Merge remote-tracking branch 'origin/master' into mvn

674 files changed, 29242 insertions, 10676 deletions

diff --git a/nixos/default.nix b/nixos/default.nix
index c11872f1441a..6beb4cd3a7df 100644
--- a/nixos/default.nix
+++ b/nixos/default.nix
@@ -9,27 +9,6 @@ let
     modules = [ configuration ];
   };
 
-  # This is for `nixos-rebuild build-vm'.
-  vmConfig = (import ./lib/eval-config.nix {
-    inherit system;
-    modules = [ configuration ./modules/virtualisation/qemu-vm.nix ];
-  }).config;
-
-  # This is for `nixos-rebuild build-vm-with-bootloader'.
-  vmWithBootLoaderConfig = (import ./lib/eval-config.nix {
-    inherit system;
-    modules =
-      [ configuration
-        ./modules/virtualisation/qemu-vm.nix
-        { virtualisation.useBootLoader = true; }
-        ({ config, ... }: {
-          virtualisation.useEFIBoot =
-            config.boot.loader.systemd-boot.enable ||
-            config.boot.loader.efi.canTouchEfiVariables;
-        })
-      ];
-  }).config;
-
 in
 
 {
@@ -37,7 +16,5 @@ in
 
   system = eval.config.system.build.toplevel;
 
-  vm = vmConfig.system.build.vm;
-
-  vmWithBootLoader = vmWithBootLoaderConfig.system.build.vm;
+  inherit (eval.config.system.build) vm vmWithBootLoader;
 }
diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix
index 31b6da01c6bd..e96bc47b4a53 100644
--- a/nixos/doc/manual/default.nix
+++ b/nixos/doc/manual/default.nix
@@ -1,4 +1,13 @@
-{ pkgs, options, config, version, revision, extraSources ? [] }:
+{ pkgs
+, options
+, config
+, version
+, revision
+, extraSources ? []
+, baseOptionsJSON ? null
+, warningsAreErrors ? true
+, prefix ? ../../..
+}:
 
 with pkgs;
 
@@ -11,11 +20,11 @@ let
   #
   # E.g. if some `options` came from modules in ${pkgs.customModules}/nix,
   # you'd need to include `extraSources = [ pkgs.customModules ]`
-  prefixesToStrip = map (p: "${toString p}/") ([ ../../.. ] ++ extraSources);
+  prefixesToStrip = map (p: "${toString p}/") ([ prefix ] ++ extraSources);
   stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip;
 
   optionsDoc = buildPackages.nixosOptionsDoc {
-    inherit options revision;
+    inherit options revision baseOptionsJSON warningsAreErrors;
     transformOptions = opt: opt // {
       # Clean up declaration sites to not refer to the NixOS source tree.
       declarations = map stripAnyPrefixes opt.declarations;
@@ -161,7 +170,7 @@ let
 in rec {
   inherit generatedSources;
 
-  inherit (optionsDoc) optionsJSON optionsDocBook;
+  inherit (optionsDoc) optionsJSON optionsNix optionsDocBook;
 
   # Generate the NixOS manual.
   manualHTML = runCommand "nixos-manual-html"
@@ -205,7 +214,7 @@ in rec {
 
   manualEpub = runCommand "nixos-manual-epub"
     { inherit sources;
-      buildInputs = [ libxml2.bin libxslt.bin zip ];
+      nativeBuildInputs = [ buildPackages.libxml2.bin buildPackages.libxslt.bin buildPackages.zip ];
     }
     ''
       # Generate the epub manual.
diff --git a/nixos/doc/manual/development/activation-script.section.md b/nixos/doc/manual/development/activation-script.section.md
new file mode 100644
index 000000000000..df6836624040
--- /dev/null
+++ b/nixos/doc/manual/development/activation-script.section.md
@@ -0,0 +1,72 @@
+# Activation script {#sec-activation-script}
+
+The activation script is a bash script called to activate the new
+configuration which resides in a NixOS system in `$out/activate`. Since its
+contents depend on your system configuration, the contents may differ.
+This chapter explains how the script works in general and some common NixOS
+snippets. Please be aware that the script is executed on every boot and system
+switch, so tasks that can be performed in other places should be performed
+there (for example letting a directory of a service be created by systemd using
+mechanisms like `StateDirectory`, `CacheDirectory`, ... or if that's not
+possible using `preStart` of the service).
+
+Activation scripts are defined as snippets using
+[](#opt-system.activationScripts). They can either be a simple multiline string
+or an attribute set that can depend on other snippets. The builder for the
+activation script will take these dependencies into account and order the
+snippets accordingly. As a simple example:
+
+```nix
+system.activationScripts.my-activation-script = {
+  deps = [ "etc" ];
+  # supportsDryActivation = true;
+  text = ''
+    echo "Hallo i bims"
+  '';
+};
+```
+
+This example creates an activation script snippet that is run after the `etc`
+snippet. The special variable `supportsDryActivation` can be set so the snippet
+is also run when `nixos-rebuild dry-activate` is run. To differentiate between
+real and dry activation, the `$NIXOS_ACTION` environment variable can be
+read which is set to `dry-activate` when a dry activation is done.
+
+An activation script can write to special files instructing
+`switch-to-configuration` to restart/reload units. The script will take these
+requests into account and will incorperate the unit configuration as described
+above. This means that the activation script will "fake" a modified unit file
+and `switch-to-configuration` will act accordingly. By doing so, configuration
+like [systemd.services.\<name\>.restartIfChanged](#opt-systemd.services) is
+respected. Since the activation script is run **after** services are already
+stopped, [systemd.services.\<name\>.stopIfChanged](#opt-systemd.services)
+cannot be taken into account anymore and the unit is always restarted instead
+of being stopped and started afterwards.
+
+The files that can be written to are `/run/nixos/activation-restart-list` and
+`/run/nixos/activation-reload-list` with their respective counterparts for
+dry activation being `/run/nixos/dry-activation-restart-list` and
+`/run/nixos/dry-activation-reload-list`. Those files can contain
+newline-separated lists of unit names where duplicates are being ignored. These
+files are not create automatically and activation scripts must take the
+possiblility into account that they have to create them first.
+
+## NixOS snippets {#sec-activation-script-nixos-snippets}
+
+There are some snippets NixOS enables by default because disabling them would
+most likely break you system. This section lists a few of them and what they
+do:
+
+- `binsh` creates `/bin/sh` which points to the runtime shell
+- `etc` sets up the contents of `/etc`, this includes systemd units and
+  excludes `/etc/passwd`, `/etc/group`, and `/etc/shadow` (which are managed by
+  the `users` snippet)
+- `hostname` sets the system's hostname in the kernel (not in `/etc`)
+- `modprobe` sets the path to the `modprobe` binary for module auto-loading
+- `nix` prepares the nix store and adds a default initial channel
+- `specialfs` is responsible for mounting filesystems like `/proc` and `sys`
+- `users` creates and removes users and groups by managing `/etc/passwd`,
+  `/etc/group` and `/etc/shadow`. This also creates home directories
+- `usrbinenv` creates `/usr/bin/env`
+- `var` creates some directories in `/var` that are not service-specific
+- `wrappers` creates setuid wrappers like `ping` and `sudo`
diff --git a/nixos/doc/manual/development/development.xml b/nixos/doc/manual/development/development.xml
index 0b2ad60a878b..21286cdbd2b4 100644
--- a/nixos/doc/manual/development/development.xml
+++ b/nixos/doc/manual/development/development.xml
@@ -12,6 +12,7 @@
  <xi:include href="../from_md/development/sources.chapter.xml" />
  <xi:include href="../from_md/development/writing-modules.chapter.xml" />
  <xi:include href="../from_md/development/building-parts.chapter.xml" />
+ <xi:include href="../from_md/development/what-happens-during-a-system-switch.chapter.xml" />
  <xi:include href="../from_md/development/writing-documentation.chapter.xml" />
  <xi:include href="../from_md/development/building-nixos.chapter.xml" />
  <xi:include href="../from_md/development/nixos-tests.chapter.xml" />
diff --git a/nixos/doc/manual/development/meta-attributes.section.md b/nixos/doc/manual/development/meta-attributes.section.md
index ca4ba007f7dc..946c08efd0a3 100644
--- a/nixos/doc/manual/development/meta-attributes.section.md
+++ b/nixos/doc/manual/development/meta-attributes.section.md
@@ -5,7 +5,7 @@ extra information. Module meta attributes are defined in the `meta.nix`
 special module.
 
 `meta` is a top level attribute like `options` and `config`. Available
-meta-attributes are `maintainers` and `doc`.
+meta-attributes are `maintainers`, `doc`, and `buildDocsInSandbox`.
 
 Each of the meta-attributes must be defined at most once per module
 file.
@@ -24,6 +24,7 @@ file.
   meta = {
     maintainers = with lib.maintainers; [ ericsagnes ];
     doc = ./default.xml;
+    buildDocsInSandbox = true;
   };
 }
 ```
@@ -38,3 +39,28 @@ file.
     ```ShellSession
     $ nix-build nixos/release.nix -A manual.x86_64-linux
     ```
+
+-  `buildDocsInSandbox` indicates whether the option documentation for the
+   module can be built in a derivation sandbox. This option is currently only
+   honored for modules shipped by nixpkgs. User modules and modules taken from
+   `NIXOS_EXTRA_MODULE_PATH` are always built outside of the sandbox, as has
+   been the case in previous releases.
+
+   Building NixOS option documentation in a sandbox allows caching of the built
+   documentation, which greatly decreases the amount of time needed to evaluate
+   a system configuration that has NixOS documentation enabled. The sandbox also
+   restricts which attributes may be referenced by documentation attributes
+   (such as option descriptions) to the `options` and `lib` module arguments and
+   the `pkgs.formats` attribute of the `pkgs` argument, `config` and the rest of
+   `pkgs` are disallowed and will cause doc build failures when used. This
+   restriction is necessary because we cannot reproduce the full nixpkgs
+   instantiation with configuration and overlays from a system configuration
+   inside the sandbox. The `options` argument only includes options of modules
+   that are also built inside the sandbox, referencing an option of a module
+   that isn't built in the sandbox is also forbidden.
+
+   The default is `true` and should usually not be changed; set it to `false`
+   only if the module requires access to `pkgs` in its documentation (e.g.
+   because it loads information from a linked package to build an option type)
+   or if its documentation depends on other modules that also aren't sandboxed
+   (e.g. by using types defined in the other module).
diff --git a/nixos/doc/manual/development/option-declarations.section.md b/nixos/doc/manual/development/option-declarations.section.md
index be56529992ab..53ecb9b3a624 100644
--- a/nixos/doc/manual/development/option-declarations.section.md
+++ b/nixos/doc/manual/development/option-declarations.section.md
@@ -27,9 +27,10 @@ The function `mkOption` accepts the following arguments.
 
 `type`
 
-:   The type of the option (see [](#sec-option-types)). It may be
-    omitted, but that's not advisable since it may lead to errors that
-    are hard to diagnose.
+:   The type of the option (see [](#sec-option-types)). This
+    argument is mandatory for nixpkgs modules. Setting this is highly
+    recommended for the sake of documentation and type checking. In case it is
+    not set, a fallback type with unspecified behavior is used.
 
 `default`
 
@@ -57,6 +58,80 @@ The function `mkOption` accepts the following arguments.
 :   A textual description of the option, in DocBook format, that will be
     included in the NixOS manual.
 
+## Utility functions for common option patterns {#sec-option-declarations-util}
+
+### `mkEnableOption` {#sec-option-declarations-util-mkEnableOption}
+
+Creates an Option attribute set for a boolean value option i.e an
+option to be toggled on or off.
+
+This function takes a single string argument, the name of the thing to be toggled.
+
+The option's description is "Whether to enable \<name\>.".
+
+For example:
+
+::: {#ex-options-declarations-util-mkEnableOption-magic .example}
+```nix
+lib.mkEnableOption "magic"
+# is like
+lib.mkOption {
+  type = lib.types.bool;
+  default = false;
+  example = true;
+  description = "Whether to enable magic.";
+}
+```
+
+### `mkPackageOption` {#sec-option-declarations-util-mkPackageOption}
+
+Usage:
+
+```nix
+mkPackageOption pkgs "name" { default = [ "path" "in" "pkgs" ]; example = "literal example"; }
+```
+
+Creates an Option attribute set for an option that specifies the package a module should use for some purpose.
+
+**Note**: You shouldn’t necessarily make package options for all of your modules. You can always overwrite a specific package throughout nixpkgs by using [nixpkgs overlays](https://nixos.org/manual/nixpkgs/stable/#chap-overlays).
+
+The default package is specified as a list of strings representing its attribute path in nixpkgs. Because of this, you need to pass nixpkgs itself as the first argument.
+
+The second argument is the name of the option, used in the description "The \<name\> package to use.". You can also pass an example value, either a literal string or a package's attribute path.
+
+You can omit the default path if the name of the option is also attribute path in nixpkgs.
+
+::: {#ex-options-declarations-util-mkPackageOption .title}
+Examples:
+
+::: {#ex-options-declarations-util-mkPackageOption-hello .example}
+```nix
+lib.mkPackageOption pkgs "hello" { }
+# is like
+lib.mkOption {
+  type = lib.types.package;
+  default = pkgs.hello;
+  defaultText = lib.literalExpression "pkgs.hello";
+  description = "The hello package to use.";
+}
+```
+
+::: {#ex-options-declarations-util-mkPackageOption-ghc .example}
+```nix
+lib.mkPackageOption pkgs "GHC" {
+  default = [ "ghc" ];
+  example = "pkgs.haskell.package.ghc921.ghc.withPackages (hkgs: [ hkgs.primes ])";
+}
+# is like
+lib.mkOption {
+  type = lib.types.package;
+  default = pkgs.ghc;
+  defaultText = lib.literalExpression "pkgs.ghc";
+  example = lib.literalExpression "pkgs.haskell.package.ghc921.ghc.withPackages (hkgs: [ hkgs.primes ])";
+  description = "The GHC package to use.";
+}
+```
+
 ## Extensible Option Types {#sec-option-declarations-eot}
 
 Extensible option types is a feature that allow to extend certain types
@@ -71,26 +146,26 @@ As an example, we will take the case of display managers. There is a
 central display manager module for generic display manager options and a
 module file per display manager backend (sddm, gdm \...).
 
-There are two approach to this module structure:
+There are two approaches we could take with this module structure:
 
--   Managing the display managers independently by adding an enable
+-   Configuring the display managers independently by adding an enable
     option to every display manager module backend. (NixOS)
 
--   Managing the display managers in the central module by adding an
-    option to select which display manager backend to use.
+-   Configuring the display managers in the central module by adding
+    an option to select which display manager backend to use.
 
 Both approaches have problems.
 
 Making backends independent can quickly become hard to manage. For
-display managers, there can be only one enabled at a time, but the type
-system can not enforce this restriction as there is no relation between
-each backend `enable` option. As a result, this restriction has to be
-done explicitely by adding assertions in each display manager backend
-module.
+display managers, there can only be one enabled at a time, but the
+type system cannot enforce this restriction as there is no relation
+between each backend's `enable` option. As a result, this restriction
+has to be done explicitly by adding assertions in each display manager
+backend module.
 
-On the other hand, managing the display managers backends in the central
-module will require to change the central module option every time a new
-backend is added or removed.
+On the other hand, managing the display manager backends in the
+central module will require changing the central module option every
+time a new backend is added or removed.
 
 By using extensible option types, it is possible to create a placeholder
 option in the central module
@@ -101,7 +176,7 @@ and to extend it in each backend module
 
 As a result, `displayManager.enable` option values can be added without
 changing the main service module file and the type system automatically
-enforce that there can only be a single display manager enabled.
+enforces that there can only be a single display manager enabled.
 
 ::: {#ex-option-declaration-eot-service .example}
 ::: {.title}
diff --git a/nixos/doc/manual/development/option-types.section.md b/nixos/doc/manual/development/option-types.section.md
index ed557206659f..00f1d85bdb61 100644
--- a/nixos/doc/manual/development/option-types.section.md
+++ b/nixos/doc/manual/development/option-types.section.md
@@ -16,13 +16,14 @@ merging is handled.
 
 `types.path`
 
-:   A filesystem path, defined as anything that when coerced to a string
-    starts with a slash. Even if derivations can be considered as path,
-    the more specific `types.package` should be preferred.
+:   A filesystem path is anything that starts with a slash when
+    coerced to a string. Even if derivations can be considered as
+    paths, the more specific `types.package` should be preferred.
 
 `types.package`
 
-:   A derivation or a store path.
+:   A top-level store path. This can be an attribute set pointing
+    to a store path, like a derivation or a flake input.
 
 `types.anything`
 
@@ -63,6 +64,24 @@ merging is handled.
     ```
     :::
 
+`types.raw`
+
+:   A type which doesn't do any checking, merging or nested evaluation. It
+    accepts a single arbitrary value that is not recursed into, making it
+    useful for values coming from outside the module system, such as package
+    sets or arbitrary data. Options of this type are still evaluated according
+    to priorities and conditionals, so `mkForce`, `mkIf` and co. still work on
+    the option value itself, but not for any value nested within it. This type
+    should only be used when checking, merging and nested evaluation are not
+    desirable.
+
+`types.optionType`
+
+:   The type of an option's type. Its merging operation ensures that nested
+    options have the correct file location annotated, and that if possible,
+    multiple option definitions are correctly merged together. The main use
+    case is as the type of the `_module.freeformType` option.
+
 `types.attrs`
 
 :   A free-form attribute set.
@@ -250,6 +269,12 @@ Composed types are types that take a type as parameter. `listOf
 :   Ensures that type *`t`* cannot be merged. It is used to ensure option
     definitions are declared only once.
 
+`types.unique` `{ message = m }` *`t`*
+
+:   Ensures that type *`t`* cannot be merged. Prints the message *`m`*, after
+    the line `The option <option path> is defined multiple times.` and before
+    a list of definition locations.
+
 `types.either` *`t1 t2`*
 
 :   Type *`t1`* or type *`t2`*, e.g. `with types; either int str`.
diff --git a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md
index f87298201791..a1431859ff59 100644
--- a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md
+++ b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md
@@ -4,19 +4,19 @@ The test itself can be run interactively. This is particularly useful
 when developing or debugging a test:
 
 ```ShellSession
-$ nix-build nixos/tests/login.nix -A driverInteractive
-$ ./result/bin/nixos-test-driver --interactive
-starting VDE switch for network 1
->
+$ nix-build . -A nixosTests.login.driverInteractive
+$ ./result/bin/nixos-test-driver
+[...]
+>>>
 ```
 
 You can then take any Python statement, e.g.
 
 ```py
-> start_all()
-> test_script()
-> machine.succeed("touch /tmp/foo")
-> print(machine.succeed("pwd")) # Show stdout of command
+>>> start_all()
+>>> test_script()
+>>> machine.succeed("touch /tmp/foo")
+>>> print(machine.succeed("pwd")) # Show stdout of command
 ```
 
 The function `test_script` executes the entire test script and drops you
@@ -28,7 +28,7 @@ You can re-use the VM states coming from a previous run by setting the
 `--keep-vm-state` flag.
 
 ```ShellSession
-$ ./result/bin/nixos-test-driver --interactive --keep-vm-state
+$ ./result/bin/nixos-test-driver --keep-vm-state
 ```
 
 The machine state is stored in the `$TMPDIR/vm-state-machinename`
diff --git a/nixos/doc/manual/development/running-nixos-tests.section.md b/nixos/doc/manual/development/running-nixos-tests.section.md
index d6a456f01883..1bec023b613a 100644
--- a/nixos/doc/manual/development/running-nixos-tests.section.md
+++ b/nixos/doc/manual/development/running-nixos-tests.section.md
@@ -24,8 +24,8 @@ After building/downloading all required dependencies, this will perform
 a build that starts a QEMU/KVM virtual machine containing a NixOS
 system. The virtual machine mounts the Nix store of the host; this makes
 VM creation very fast, as no disk image needs to be created. Afterwards,
-you can view a pretty-printed log of the test:
+you can view a log of the test:
 
 ```ShellSession
-$ firefox result/log.html
+$ nix-store --read-log result
 ```
diff --git a/nixos/doc/manual/development/settings-options.section.md b/nixos/doc/manual/development/settings-options.section.md
index 58a3d8448af5..f9bb6ff9cc41 100644
--- a/nixos/doc/manual/development/settings-options.section.md
+++ b/nixos/doc/manual/development/settings-options.section.md
@@ -66,6 +66,45 @@ have a predefined type and string generator already declared under
     and returning a set with TOML-specific attributes `type` and
     `generate` as specified [below](#pkgs-formats-result).
 
+`pkgs.formats.elixirConf { elixir ? pkgs.elixir }`
+
+:   A function taking an attribute set with values
+
+    `elixir`
+
+    :   The Elixir package which will be used to format the generated output
+
+    It returns a set with Elixir-Config-specific attributes `type`, `lib`, and
+    `generate` as specified [below](#pkgs-formats-result).
+
+    The `lib` attribute contains functions to be used in settings, for
+    generating special Elixir values:
+
+    `mkRaw elixirCode`
+
+    :   Outputs the given string as raw Elixir code
+
+    `mkGetEnv { envVariable, fallback ? null }`
+
+    :   Makes the configuration fetch an environment variable at runtime
+
+    `mkAtom atom`
+
+    :   Outputs the given string as an Elixir atom, instead of the default
+        Elixir binary string. Note: lowercase atoms still needs to be prefixed
+        with `:`
+
+    `mkTuple array`
+
+    :   Outputs the given array as an Elixir tuple, instead of the default
+        Elixir list
+
+    `mkMap attrset`
+
+    :   Outputs the given attribute set as an Elixir map, instead of the
+        default Elixir keyword list
+
+
 ::: {#pkgs-formats-result}
 These functions all return an attribute set with these values:
 :::
@@ -74,6 +113,12 @@ These functions all return an attribute set with these values:
 
 :   A module system type representing a value of the format
 
+`lib`
+
+:   Utility functions for convenience, or special interactions with the format.
+    This attribute is optional. It may contain inside a `types` attribute
+    containing types specific to this format.
+
 `generate` *`filename jsonValue`*
 
 :   A function that can render a value of the format to a file. Returns
diff --git a/nixos/doc/manual/development/unit-handling.section.md b/nixos/doc/manual/development/unit-handling.section.md
new file mode 100644
index 000000000000..a7ccb3dbd042
--- /dev/null
+++ b/nixos/doc/manual/development/unit-handling.section.md
@@ -0,0 +1,62 @@
+# Unit handling {#sec-unit-handling}
+
+To figure out what units need to be started/stopped/restarted/reloaded, the
+script first checks the current state of the system, similar to what `systemctl
+list-units` shows. For each of the units, the script goes through the following
+checks:
+
+- Is the unit file still in the new system? If not, **stop** the service unless
+  it sets `X-StopOnRemoval` in the `[Unit]` section to `false`.
+
+- Is it a `.target` unit? If so, **start** it unless it sets
+  `RefuseManualStart` in the `[Unit]` section to `true` or `X-OnlyManualStart`
+  in the `[Unit]` section to `true`. Also **stop** the unit again unless it
+  sets `X-StopOnReconfiguration` to `false`.
+
+- Are the contents of the unit files different? They are compared by parsing
+  them and comparing their contents. If they are different but only
+  `X-Reload-Triggers` in the `[Unit]` section is changed, **reload** the unit.
+  The NixOS module system allows setting these triggers with the option
+  [systemd.services.\<name\>.reloadTriggers](#opt-systemd.services). There are
+  some additional keys in the `[Unit]` section that are ignored as well. If the
+  unit files differ in any way, the following actions are performed:
+
+  - `.path` and `.slice` units are ignored. There is no need to restart them
+    since changes in their values are applied by systemd when systemd is
+    reloaded.
+
+  - `.mount` units are **reload**ed. These mostly come from the `/etc/fstab`
+    parser.
+
+  - `.socket` units are currently ignored. This is to be fixed at a later
+    point.
+
+  - The rest of the units (mostly `.service` units) are then **reload**ed if
+    `X-ReloadIfChanged` in the `[Service]` section is set to `true` (exposed
+    via [systemd.services.\<name\>.reloadIfChanged](#opt-systemd.services)).
+    A little exception is done for units that were deactivated in the meantime,
+    for example because they require a unit that got stopped before. These
+    are **start**ed instead of reloaded.
+
+  - If the reload flag is not set, some more flags decide if the unit is
+    skipped. These flags are `X-RestartIfChanged` in the `[Service]` section
+    (exposed via
+    [systemd.services.\<name\>.restartIfChanged](#opt-systemd.services)),
+    `RefuseManualStop` in the `[Unit]` section, and `X-OnlyManualStart` in the
+    `[Unit]` section.
+
+  - Further behavior depends on the unit having `X-StopIfChanged` in the
+    `[Service]` section set to `true` (exposed via
+    [systemd.services.\<name\>.stopIfChanged](#opt-systemd.services)). This is
+    set to `true` by default and must be explicitly turned off if not wanted.
+    If the flag is enabled, the unit is **stop**ped and then **start**ed. If
+    not, the unit is **restart**ed. The goal of the flag is to make sure that
+    the new unit never runs in the old environment which is still in place
+    before the activation script is run. This behavior is different when the
+    service is socket-activated, as outlined in the following steps.
+
+  - The last thing that is taken into account is whether the unit is a service
+    and socket-activated. If `X-StopIfChanged` is **not** set, the service
+    is **restart**ed with the others. If it is set, both the service and the
+    socket are **stop**ped and the socket is **start**ed, leaving socket
+    activation to start the service when it's needed.
diff --git a/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md
new file mode 100644
index 000000000000..aad82831a3c2
--- /dev/null
+++ b/nixos/doc/manual/development/what-happens-during-a-system-switch.chapter.md
@@ -0,0 +1,53 @@
+# What happens during a system switch? {#sec-switching-systems}
+
+Running `nixos-rebuild switch` is one of the more common tasks under NixOS.
+This chapter explains some of the internals of this command to make it simpler
+for new module developers to configure their units correctly and to make it
+easier to understand what is happening and why for curious administrators.
+
+`nixos-rebuild`, like many deployment solutions, calls `switch-to-configuration`
+which resides in a NixOS system at `$out/bin/switch-to-configuration`. The
+script is called with the action that is to be performed like `switch`, `test`,
+`boot`. There is also the `dry-activate` action which does not really perform
+the actions but rather prints what it would do if you called it with `test`.
+This feature can be used to check what service states would be changed if the
+configuration was switched to.
+
+If the action is `switch` or `boot`, the bootloader is updated first so the
+configuration will be the next one to boot. Unless `NIXOS_NO_SYNC` is set to
+`1`, `/nix/store` is synced to disk.
+
+If the action is `switch` or `test`, the currently running system is inspected
+and the actions to switch to the new system are calculated. This process takes
+two data sources into account: `/etc/fstab` and the current systemd status.
+Mounts and swaps are read from `/etc/fstab` and the corresponding actions are
+generated. If a new mount is added, for example, the proper `.mount` unit is
+marked to be started. The current systemd state is inspected, the difference
+between the current system and the desired configuration is calculated and
+actions are generated to get to this state. There are a lot of nuances that can
+be controlled by the units which are explained here.
+
+After calculating what should be done, the actions are carried out. The order
+of actions is always the same:
+- Stop units (`systemctl stop`)
+- Run activation script (`$out/activate`)
+- See if the activation script requested more units to restart
+- Restart systemd if needed (`systemd daemon-reexec`)
+- Forget about the failed state of units (`systemctl reset-failed`)
+- Reload systemd (`systemctl daemon-reload`)
+- Reload systemd user instances (`systemctl --user daemon-reload`)
+- Set up tmpfiles (`systemd-tmpfiles --create`)
+- Reload units (`systemctl reload`)
+- Restart units (`systemctl restart`)
+- Start units (`systemctl start`)
+- Inspect what changed during these actions and print units that failed and
+  that were newly started
+
+Most of these actions are either self-explaining but some of them have to do
+with our units or the activation script. For this reason, these topics are
+explained in the next sections.
+
+```{=docbook}
+<xi:include href="unit-handling.section.xml" />
+<xi:include href="activation-script.section.xml" />
+```
diff --git a/nixos/doc/manual/development/writing-modules.chapter.md b/nixos/doc/manual/development/writing-modules.chapter.md
index 2e3c6b34f1f5..0c41cbd3cb75 100644
--- a/nixos/doc/manual/development/writing-modules.chapter.md
+++ b/nixos/doc/manual/development/writing-modules.chapter.md
@@ -90,6 +90,17 @@ modules: `systemd.services` (the set of all systemd services) and
 `systemd.timers` (the list of commands to be executed periodically by
 `systemd`).
 
+Care must be taken when writing systemd services using `Exec*` directives. By
+default systemd performs substitution on `%<char>` specifiers in these
+directives, expands environment variables from `$FOO` and `${FOO}`, splits
+arguments on whitespace, and splits commands on `;`. All of these must be escaped
+to avoid unexpected substitution or splitting when interpolating into an `Exec*`
+directive, e.g. when using an `extraArgs` option to pass additional arguments to
+the service. The functions `utils.escapeSystemdExecArg` and
+`utils.escapeSystemdExecArgs` are provided for this, see [Example: Escaping in
+Exec directives](#exec-escaping-example) for an example. When using these
+functions system environment substitution should *not* be disabled explicitly.
+
 ::: {#locate-example .example}
 ::: {.title}
 **Example: NixOS Module for the "locate" Service**
@@ -153,6 +164,37 @@ in {
 ```
 :::
 
+::: {#exec-escaping-example .example}
+::: {.title}
+**Example: Escaping in Exec directives**
+:::
+```nix
+{ config, lib, pkgs, utils, ... }:
+
+with lib;
+
+let
+  cfg = config.services.echo;
+  echoAll = pkgs.writeScript "echo-all" ''
+    #! ${pkgs.runtimeShell}
+    for s in "$@"; do
+      printf '%s\n' "$s"
+    done
+  '';
+  args = [ "a%Nything" "lang=\${LANG}" ";" "/bin/sh -c date" ];
+in {
+  systemd.services.echo =
+    { description = "Echo to the journal";
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig.Type = "oneshot";
+      serviceConfig.ExecStart = ''
+        ${echoAll} ${utils.escapeSystemdExecArgs args}
+      '';
+    };
+}
+```
+:::
+
 ```{=docbook}
 <xi:include href="option-declarations.section.xml" />
 <xi:include href="option-types.section.xml" />
diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md
index d9749d37da79..7de57d0d2a37 100644
--- a/nixos/doc/manual/development/writing-nixos-tests.section.md
+++ b/nixos/doc/manual/development/writing-nixos-tests.section.md
@@ -88,6 +88,8 @@ starting them in parallel:
 start_all()
 ```
 
+## Machine objects {#ssec-machine-objects}
+
 The following methods are available on machine objects:
 
 `start`
@@ -313,3 +315,52 @@ repository):
       # fmt: on
     '';
 ```
+
+## Failing tests early {#ssec-failing-tests-early}
+
+To fail tests early when certain invariables are no longer met (instead of waiting for the build to time out), the decorator `polling_condition` is provided. For example, if we are testing a program `foo` that should not quit after being started, we might write the following:
+
+```py
+@polling_condition
+def foo_running():
+    machine.succeed("pgrep -x foo")
+
+
+machine.succeed("foo --start")
+machine.wait_until_succeeds("pgrep -x foo")
+
+with foo_running:
+    ...  # Put `foo` through its paces
+```
+
+
+`polling_condition` takes the following (optional) arguments:
+
+`seconds_interval`
+
+:
+    specifies how often the condition should be polled:
+
+    ```py
+    @polling_condition(seconds_interval=10)
+    def foo_running():
+        machine.succeed("pgrep -x foo")
+    ```
+
+`description`
+
+:
+    is used in the log when the condition is checked. If this is not provided, the description is pulled from the docstring of the function. These two are therefore equivalent:
+
+    ```py
+    @polling_condition
+    def foo_running():
+        "check that foo is running"
+        machine.succeed("pgrep -x foo")
+    ```
+
+    ```py
+    @polling_condition(description="check that foo is running")
+    def foo_running():
+        machine.succeed("pgrep -x foo")
+    ```
diff --git a/nixos/doc/manual/from_md/development/activation-script.section.xml b/nixos/doc/manual/from_md/development/activation-script.section.xml
new file mode 100644
index 000000000000..0d9e911216ef
--- /dev/null
+++ b/nixos/doc/manual/from_md/development/activation-script.section.xml
@@ -0,0 +1,150 @@
+<section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-activation-script">
+  <title>Activation script</title>
+  <para>
+    The activation script is a bash script called to activate the new
+    configuration which resides in a NixOS system in
+    <literal>$out/activate</literal>. Since its contents depend on your
+    system configuration, the contents may differ. This chapter explains
+    how the script works in general and some common NixOS snippets.
+    Please be aware that the script is executed on every boot and system
+    switch, so tasks that can be performed in other places should be
+    performed there (for example letting a directory of a service be
+    created by systemd using mechanisms like
+    <literal>StateDirectory</literal>,
+    <literal>CacheDirectory</literal>, … or if that’s not possible using
+    <literal>preStart</literal> of the service).
+  </para>
+  <para>
+    Activation scripts are defined as snippets using
+    <xref linkend="opt-system.activationScripts" />. They can either be
+    a simple multiline string or an attribute set that can depend on
+    other snippets. The builder for the activation script will take
+    these dependencies into account and order the snippets accordingly.
+    As a simple example:
+  </para>
+  <programlisting language="bash">
+system.activationScripts.my-activation-script = {
+  deps = [ &quot;etc&quot; ];
+  # supportsDryActivation = true;
+  text = ''
+    echo &quot;Hallo i bims&quot;
+  '';
+};
+</programlisting>
+  <para>
+    This example creates an activation script snippet that is run after
+    the <literal>etc</literal> snippet. The special variable
+    <literal>supportsDryActivation</literal> can be set so the snippet
+    is also run when <literal>nixos-rebuild dry-activate</literal> is
+    run. To differentiate between real and dry activation, the
+    <literal>$NIXOS_ACTION</literal> environment variable can be read
+    which is set to <literal>dry-activate</literal> when a dry
+    activation is done.
+  </para>
+  <para>
+    An activation script can write to special files instructing
+    <literal>switch-to-configuration</literal> to restart/reload units.
+    The script will take these requests into account and will
+    incorperate the unit configuration as described above. This means
+    that the activation script will <quote>fake</quote> a modified unit
+    file and <literal>switch-to-configuration</literal> will act
+    accordingly. By doing so, configuration like
+    <link linkend="opt-systemd.services">systemd.services.&lt;name&gt;.restartIfChanged</link>
+    is respected. Since the activation script is run
+    <emphasis role="strong">after</emphasis> services are already
+    stopped,
+    <link linkend="opt-systemd.services">systemd.services.&lt;name&gt;.stopIfChanged</link>
+    cannot be taken into account anymore and the unit is always
+    restarted instead of being stopped and started afterwards.
+  </para>
+  <para>
+    The files that can be written to are
+    <literal>/run/nixos/activation-restart-list</literal> and
+    <literal>/run/nixos/activation-reload-list</literal> with their
+    respective counterparts for dry activation being
+    <literal>/run/nixos/dry-activation-restart-list</literal> and
+    <literal>/run/nixos/dry-activation-reload-list</literal>. Those
+    files can contain newline-separated lists of unit names where
+    duplicates are being ignored. These files are not create
+    automatically and activation scripts must take the possiblility into
+    account that they have to create them first.
+  </para>
+  <section xml:id="sec-activation-script-nixos-snippets">
+    <title>NixOS snippets</title>
+    <para>
+      There are some snippets NixOS enables by default because disabling
+      them would most likely break you system. This section lists a few
+      of them and what they do:
+    </para>
+    <itemizedlist spacing="compact">
+      <listitem>
+        <para>
+          <literal>binsh</literal> creates <literal>/bin/sh</literal>
+          which points to the runtime shell
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>etc</literal> sets up the contents of
+          <literal>/etc</literal>, this includes systemd units and
+          excludes <literal>/etc/passwd</literal>,
+          <literal>/etc/group</literal>, and
+          <literal>/etc/shadow</literal> (which are managed by the
+          <literal>users</literal> snippet)
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>hostname</literal> sets the system’s hostname in the
+          kernel (not in <literal>/etc</literal>)
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>modprobe</literal> sets the path to the
+          <literal>modprobe</literal> binary for module auto-loading
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>nix</literal> prepares the nix store and adds a
+          default initial channel
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>specialfs</literal> is responsible for mounting
+          filesystems like <literal>/proc</literal> and
+          <literal>sys</literal>
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>users</literal> creates and removes users and groups
+          by managing <literal>/etc/passwd</literal>,
+          <literal>/etc/group</literal> and
+          <literal>/etc/shadow</literal>. This also creates home
+          directories
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>usrbinenv</literal> creates
+          <literal>/usr/bin/env</literal>
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>var</literal> creates some directories in
+          <literal>/var</literal> that are not service-specific
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>wrappers</literal> creates setuid wrappers like
+          <literal>ping</literal> and <literal>sudo</literal>
+        </para>
+      </listitem>
+    </itemizedlist>
+  </section>
+</section>
diff --git a/nixos/doc/manual/from_md/development/meta-attributes.section.xml b/nixos/doc/manual/from_md/development/meta-attributes.section.xml
index f535d94602bd..1eb6e0f30368 100644
--- a/nixos/doc/manual/from_md/development/meta-attributes.section.xml
+++ b/nixos/doc/manual/from_md/development/meta-attributes.section.xml
@@ -8,8 +8,8 @@
   <para>
     <literal>meta</literal> is a top level attribute like
     <literal>options</literal> and <literal>config</literal>. Available
-    meta-attributes are <literal>maintainers</literal> and
-    <literal>doc</literal>.
+    meta-attributes are <literal>maintainers</literal>,
+    <literal>doc</literal>, and <literal>buildDocsInSandbox</literal>.
   </para>
   <para>
     Each of the meta-attributes must be defined at most once per module
@@ -29,6 +29,7 @@
   meta = {
     maintainers = with lib.maintainers; [ ericsagnes ];
     doc = ./default.xml;
+    buildDocsInSandbox = true;
   };
 }
 </programlisting>
@@ -51,5 +52,44 @@
 $ nix-build nixos/release.nix -A manual.x86_64-linux
 </programlisting>
     </listitem>
+    <listitem>
+      <para>
+        <literal>buildDocsInSandbox</literal> indicates whether the
+        option documentation for the module can be built in a derivation
+        sandbox. This option is currently only honored for modules
+        shipped by nixpkgs. User modules and modules taken from
+        <literal>NIXOS_EXTRA_MODULE_PATH</literal> are always built
+        outside of the sandbox, as has been the case in previous
+        releases.
+      </para>
+      <para>
+        Building NixOS option documentation in a sandbox allows caching
+        of the built documentation, which greatly decreases the amount
+        of time needed to evaluate a system configuration that has NixOS
+        documentation enabled. The sandbox also restricts which
+        attributes may be referenced by documentation attributes (such
+        as option descriptions) to the <literal>options</literal> and
+        <literal>lib</literal> module arguments and the
+        <literal>pkgs.formats</literal> attribute of the
+        <literal>pkgs</literal> argument, <literal>config</literal> and
+        the rest of <literal>pkgs</literal> are disallowed and will
+        cause doc build failures when used. This restriction is
+        necessary because we cannot reproduce the full nixpkgs
+        instantiation with configuration and overlays from a system
+        configuration inside the sandbox. The <literal>options</literal>
+        argument only includes options of modules that are also built
+        inside the sandbox, referencing an option of a module that isn’t
+        built in the sandbox is also forbidden.
+      </para>
+      <para>
+        The default is <literal>true</literal> and should usually not be
+        changed; set it to <literal>false</literal> only if the module
+        requires access to <literal>pkgs</literal> in its documentation
+        (e.g. because it loads information from a linked package to
+        build an option type) or if its documentation depends on other
+        modules that also aren’t sandboxed (e.g. by using types defined
+        in the other module).
+      </para>
+    </listitem>
   </itemizedlist>
 </section>
diff --git a/nixos/doc/manual/from_md/development/option-declarations.section.xml b/nixos/doc/manual/from_md/development/option-declarations.section.xml
index 2845e37659b1..0ac5e0eeca2d 100644
--- a/nixos/doc/manual/from_md/development/option-declarations.section.xml
+++ b/nixos/doc/manual/from_md/development/option-declarations.section.xml
@@ -38,9 +38,11 @@ options = {
       <listitem>
         <para>
           The type of the option (see
-          <xref linkend="sec-option-types" />). It may be omitted, but
-          that’s not advisable since it may lead to errors that are hard
-          to diagnose.
+          <xref linkend="sec-option-types" />). This argument is
+          mandatory for nixpkgs modules. Setting this is highly
+          recommended for the sake of documentation and type checking.
+          In case it is not set, a fallback type with unspecified
+          behavior is used.
         </para>
       </listitem>
     </varlistentry>
@@ -97,125 +99,229 @@ options = {
       </listitem>
     </varlistentry>
   </variablelist>
-  <section xml:id="sec-option-declarations-eot">
-    <title>Extensible Option Types</title>
-    <para>
-      Extensible option types is a feature that allow to extend certain
-      types declaration through multiple module files. This feature only
-      work with a restricted set of types, namely
-      <literal>enum</literal> and <literal>submodules</literal> and any
-      composed forms of them.
-    </para>
-    <para>
-      Extensible option types can be used for <literal>enum</literal>
-      options that affects multiple modules, or as an alternative to
-      related <literal>enable</literal> options.
-    </para>
-    <para>
-      As an example, we will take the case of display managers. There is
-      a central display manager module for generic display manager
-      options and a module file per display manager backend (sddm, gdm
-      ...).
-    </para>
-    <para>
-      There are two approach to this module structure:
-    </para>
-    <itemizedlist>
-      <listitem>
+  <section xml:id="sec-option-declarations-util">
+    <title>Utility functions for common option patterns</title>
+    <section xml:id="sec-option-declarations-util-mkEnableOption">
+      <title><literal>mkEnableOption</literal></title>
+      <para>
+        Creates an Option attribute set for a boolean value option i.e
+        an option to be toggled on or off.
+      </para>
+      <para>
+        This function takes a single string argument, the name of the
+        thing to be toggled.
+      </para>
+      <para>
+        The option’s description is <quote>Whether to enable
+        &lt;name&gt;.</quote>.
+      </para>
+      <para>
+        For example:
+      </para>
+      <anchor xml:id="ex-options-declarations-util-mkEnableOption-magic" />
+      <programlisting language="bash">
+lib.mkEnableOption &quot;magic&quot;
+# is like
+lib.mkOption {
+  type = lib.types.bool;
+  default = false;
+  example = true;
+  description = &quot;Whether to enable magic.&quot;;
+}
+</programlisting>
+      <section xml:id="sec-option-declarations-util-mkPackageOption">
+        <title><literal>mkPackageOption</literal></title>
         <para>
-          Managing the display managers independently by adding an
-          enable option to every display manager module backend. (NixOS)
+          Usage:
         </para>
-      </listitem>
-      <listitem>
+        <programlisting language="bash">
+mkPackageOption pkgs &quot;name&quot; { default = [ &quot;path&quot; &quot;in&quot; &quot;pkgs&quot; ]; example = &quot;literal example&quot;; }
+</programlisting>
         <para>
-          Managing the display managers in the central module by adding
-          an option to select which display manager backend to use.
+          Creates an Option attribute set for an option that specifies
+          the package a module should use for some purpose.
         </para>
-      </listitem>
-    </itemizedlist>
-    <para>
-      Both approaches have problems.
-    </para>
-    <para>
-      Making backends independent can quickly become hard to manage. For
-      display managers, there can be only one enabled at a time, but the
-      type system can not enforce this restriction as there is no
-      relation between each backend <literal>enable</literal> option. As
-      a result, this restriction has to be done explicitely by adding
-      assertions in each display manager backend module.
-    </para>
-    <para>
-      On the other hand, managing the display managers backends in the
-      central module will require to change the central module option
-      every time a new backend is added or removed.
-    </para>
-    <para>
-      By using extensible option types, it is possible to create a
-      placeholder option in the central module
-      (<link linkend="ex-option-declaration-eot-service">Example:
-      Extensible type placeholder in the service module</link>), and to
-      extend it in each backend module
-      (<link linkend="ex-option-declaration-eot-backend-gdm">Example:
-      Extending
-      <literal>services.xserver.displayManager.enable</literal> in the
-      <literal>gdm</literal> module</link>,
-      <link linkend="ex-option-declaration-eot-backend-sddm">Example:
-      Extending
-      <literal>services.xserver.displayManager.enable</literal> in the
-      <literal>sddm</literal> module</link>).
-    </para>
-    <para>
-      As a result, <literal>displayManager.enable</literal> option
-      values can be added without changing the main service module file
-      and the type system automatically enforce that there can only be a
-      single display manager enabled.
-    </para>
-    <anchor xml:id="ex-option-declaration-eot-service" />
-    <para>
-      <emphasis role="strong">Example: Extensible type placeholder in
-      the service module</emphasis>
-    </para>
-    <programlisting language="bash">
+        <para>
+          <emphasis role="strong">Note</emphasis>: You shouldn’t
+          necessarily make package options for all of your modules. You
+          can always overwrite a specific package throughout nixpkgs by
+          using
+          <link xlink:href="https://nixos.org/manual/nixpkgs/stable/#chap-overlays">nixpkgs
+          overlays</link>.
+        </para>
+        <para>
+          The default package is specified as a list of strings
+          representing its attribute path in nixpkgs. Because of this,
+          you need to pass nixpkgs itself as the first argument.
+        </para>
+        <para>
+          The second argument is the name of the option, used in the
+          description <quote>The &lt;name&gt; package to use.</quote>.
+          You can also pass an example value, either a literal string or
+          a package’s attribute path.
+        </para>
+        <para>
+          You can omit the default path if the name of the option is
+          also attribute path in nixpkgs.
+        </para>
+        <anchor xml:id="ex-options-declarations-util-mkPackageOption" />
+        <para>
+          Examples:
+        </para>
+        <anchor xml:id="ex-options-declarations-util-mkPackageOption-hello" />
+        <programlisting language="bash">
+lib.mkPackageOption pkgs &quot;hello&quot; { }
+# is like
+lib.mkOption {
+  type = lib.types.package;
+  default = pkgs.hello;
+  defaultText = lib.literalExpression &quot;pkgs.hello&quot;;
+  description = &quot;The hello package to use.&quot;;
+}
+</programlisting>
+        <anchor xml:id="ex-options-declarations-util-mkPackageOption-ghc" />
+        <programlisting language="bash">
+lib.mkPackageOption pkgs &quot;GHC&quot; {
+  default = [ &quot;ghc&quot; ];
+  example = &quot;pkgs.haskell.package.ghc921.ghc.withPackages (hkgs: [ hkgs.primes ])&quot;;
+}
+# is like
+lib.mkOption {
+  type = lib.types.package;
+  default = pkgs.ghc;
+  defaultText = lib.literalExpression &quot;pkgs.ghc&quot;;
+  example = lib.literalExpression &quot;pkgs.haskell.package.ghc921.ghc.withPackages (hkgs: [ hkgs.primes ])&quot;;
+  description = &quot;The GHC package to use.&quot;;
+}
+</programlisting>
+        <section xml:id="sec-option-declarations-eot">
+          <title>Extensible Option Types</title>
+          <para>
+            Extensible option types is a feature that allow to extend
+            certain types declaration through multiple module files.
+            This feature only work with a restricted set of types,
+            namely <literal>enum</literal> and
+            <literal>submodules</literal> and any composed forms of
+            them.
+          </para>
+          <para>
+            Extensible option types can be used for
+            <literal>enum</literal> options that affects multiple
+            modules, or as an alternative to related
+            <literal>enable</literal> options.
+          </para>
+          <para>
+            As an example, we will take the case of display managers.
+            There is a central display manager module for generic
+            display manager options and a module file per display
+            manager backend (sddm, gdm ...).
+          </para>
+          <para>
+            There are two approaches we could take with this module
+            structure:
+          </para>
+          <itemizedlist>
+            <listitem>
+              <para>
+                Configuring the display managers independently by adding
+                an enable option to every display manager module
+                backend. (NixOS)
+              </para>
+            </listitem>
+            <listitem>
+              <para>
+                Configuring the display managers in the central module
+                by adding an option to select which display manager
+                backend to use.
+              </para>
+            </listitem>
+          </itemizedlist>
+          <para>
+            Both approaches have problems.
+          </para>
+          <para>
+            Making backends independent can quickly become hard to
+            manage. For display managers, there can only be one enabled
+            at a time, but the type system cannot enforce this
+            restriction as there is no relation between each backend’s
+            <literal>enable</literal> option. As a result, this
+            restriction has to be done explicitly by adding assertions
+            in each display manager backend module.
+          </para>
+          <para>
+            On the other hand, managing the display manager backends in
+            the central module will require changing the central module
+            option every time a new backend is added or removed.
+          </para>
+          <para>
+            By using extensible option types, it is possible to create a
+            placeholder option in the central module
+            (<link linkend="ex-option-declaration-eot-service">Example:
+            Extensible type placeholder in the service module</link>),
+            and to extend it in each backend module
+            (<link linkend="ex-option-declaration-eot-backend-gdm">Example:
+            Extending
+            <literal>services.xserver.displayManager.enable</literal> in
+            the <literal>gdm</literal> module</link>,
+            <link linkend="ex-option-declaration-eot-backend-sddm">Example:
+            Extending
+            <literal>services.xserver.displayManager.enable</literal> in
+            the <literal>sddm</literal> module</link>).
+          </para>
+          <para>
+            As a result, <literal>displayManager.enable</literal> option
+            values can be added without changing the main service module
+            file and the type system automatically enforces that there
+            can only be a single display manager enabled.
+          </para>
+          <anchor xml:id="ex-option-declaration-eot-service" />
+          <para>
+            <emphasis role="strong">Example: Extensible type placeholder
+            in the service module</emphasis>
+          </para>
+          <programlisting language="bash">
 services.xserver.displayManager.enable = mkOption {
   description = &quot;Display manager to use&quot;;
   type = with types; nullOr (enum [ ]);
 };
 </programlisting>
-    <anchor xml:id="ex-option-declaration-eot-backend-gdm" />
-    <para>
-      <emphasis role="strong">Example: Extending
-      <literal>services.xserver.displayManager.enable</literal> in the
-      <literal>gdm</literal> module</emphasis>
-    </para>
-    <programlisting language="bash">
+          <anchor xml:id="ex-option-declaration-eot-backend-gdm" />
+          <para>
+            <emphasis role="strong">Example: Extending
+            <literal>services.xserver.displayManager.enable</literal> in
+            the <literal>gdm</literal> module</emphasis>
+          </para>
+          <programlisting language="bash">
 services.xserver.displayManager.enable = mkOption {
   type = with types; nullOr (enum [ &quot;gdm&quot; ]);
 };
 </programlisting>
-    <anchor xml:id="ex-option-declaration-eot-backend-sddm" />
-    <para>
-      <emphasis role="strong">Example: Extending
-      <literal>services.xserver.displayManager.enable</literal> in the
-      <literal>sddm</literal> module</emphasis>
-    </para>
-    <programlisting language="bash">
+          <anchor xml:id="ex-option-declaration-eot-backend-sddm" />
+          <para>
+            <emphasis role="strong">Example: Extending
+            <literal>services.xserver.displayManager.enable</literal> in
+            the <literal>sddm</literal> module</emphasis>
+          </para>
+          <programlisting language="bash">
 services.xserver.displayManager.enable = mkOption {
   type = with types; nullOr (enum [ &quot;sddm&quot; ]);
 };
 </programlisting>
-    <para>
-      The placeholder declaration is a standard
-      <literal>mkOption</literal> declaration, but it is important that
-      extensible option declarations only use the
-      <literal>type</literal> argument.
-    </para>
-    <para>
-      Extensible option types work with any of the composed variants of
-      <literal>enum</literal> such as
-      <literal>with types; nullOr (enum [ &quot;foo&quot; &quot;bar&quot; ])</literal>
-      or
-      <literal>with types; listOf (enum [ &quot;foo&quot; &quot;bar&quot; ])</literal>.
-    </para>
+          <para>
+            The placeholder declaration is a standard
+            <literal>mkOption</literal> declaration, but it is important
+            that extensible option declarations only use the
+            <literal>type</literal> argument.
+          </para>
+          <para>
+            Extensible option types work with any of the composed
+            variants of <literal>enum</literal> such as
+            <literal>with types; nullOr (enum [ &quot;foo&quot; &quot;bar&quot; ])</literal>
+            or
+            <literal>with types; listOf (enum [ &quot;foo&quot; &quot;bar&quot; ])</literal>.
+          </para>
+        </section>
+      </section>
+    </section>
   </section>
 </section>
diff --git a/nixos/doc/manual/from_md/development/option-types.section.xml b/nixos/doc/manual/from_md/development/option-types.section.xml
index c83ffa2add53..444729292702 100644
--- a/nixos/doc/manual/from_md/development/option-types.section.xml
+++ b/nixos/doc/manual/from_md/development/option-types.section.xml
@@ -30,10 +30,10 @@
         </term>
         <listitem>
           <para>
-            A filesystem path, defined as anything that when coerced to
-            a string starts with a slash. Even if derivations can be
-            considered as path, the more specific
-            <literal>types.package</literal> should be preferred.
+            A filesystem path is anything that starts with a slash when
+            coerced to a string. Even if derivations can be considered
+            as paths, the more specific <literal>types.package</literal>
+            should be preferred.
           </para>
         </listitem>
       </varlistentry>
@@ -43,7 +43,9 @@
         </term>
         <listitem>
           <para>
-            A derivation or a store path.
+            A top-level store path. This can be an attribute set
+            pointing to a store path, like a derivation or a flake
+            input.
           </para>
         </listitem>
       </varlistentry>
@@ -94,6 +96,39 @@
       </varlistentry>
       <varlistentry>
         <term>
+          <literal>types.raw</literal>
+        </term>
+        <listitem>
+          <para>
+            A type which doesn’t do any checking, merging or nested
+            evaluation. It accepts a single arbitrary value that is not
+            recursed into, making it useful for values coming from
+            outside the module system, such as package sets or arbitrary
+            data. Options of this type are still evaluated according to
+            priorities and conditionals, so <literal>mkForce</literal>,
+            <literal>mkIf</literal> and co. still work on the option
+            value itself, but not for any value nested within it. This
+            type should only be used when checking, merging and nested
+            evaluation are not desirable.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>types.optionType</literal>
+        </term>
+        <listitem>
+          <para>
+            The type of an option’s type. Its merging operation ensures
+            that nested options have the correct file location
+            annotated, and that if possible, multiple option definitions
+            are correctly merged together. The main use case is as the
+            type of the <literal>_module.freeformType</literal> option.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
           <literal>types.attrs</literal>
         </term>
         <listitem>
@@ -498,6 +533,22 @@
       </varlistentry>
       <varlistentry>
         <term>
+          <literal>types.unique</literal>
+          <literal>{ message = m }</literal>
+          <emphasis><literal>t</literal></emphasis>
+        </term>
+        <listitem>
+          <para>
+            Ensures that type <emphasis><literal>t</literal></emphasis>
+            cannot be merged. Prints the message
+            <emphasis><literal>m</literal></emphasis>, after the line
+            <literal>The option &lt;option path&gt; is defined multiple times.</literal>
+            and before a list of definition locations.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
           <literal>types.either</literal>
           <emphasis><literal>t1 t2</literal></emphasis>
         </term>
diff --git a/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml b/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml
index 17003cbcbfdc..0e47350a0d24 100644
--- a/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml
+++ b/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml
@@ -5,19 +5,19 @@
     useful when developing or debugging a test:
   </para>
   <programlisting>
-$ nix-build nixos/tests/login.nix -A driverInteractive
-$ ./result/bin/nixos-test-driver --interactive
-starting VDE switch for network 1
-&gt;
+$ nix-build . -A nixosTests.login.driverInteractive
+$ ./result/bin/nixos-test-driver
+[...]
+&gt;&gt;&gt;
 </programlisting>
   <para>
     You can then take any Python statement, e.g.
   </para>
   <programlisting language="python">
-&gt; start_all()
-&gt; test_script()
-&gt; machine.succeed(&quot;touch /tmp/foo&quot;)
-&gt; print(machine.succeed(&quot;pwd&quot;)) # Show stdout of command
+&gt;&gt;&gt; start_all()
+&gt;&gt;&gt; test_script()
+&gt;&gt;&gt; machine.succeed(&quot;touch /tmp/foo&quot;)
+&gt;&gt;&gt; print(machine.succeed(&quot;pwd&quot;)) # Show stdout of command
 </programlisting>
   <para>
     The function <literal>test_script</literal> executes the entire test
@@ -30,7 +30,7 @@ starting VDE switch for network 1
     the <literal>--keep-vm-state</literal> flag.
   </para>
   <programlisting>
-$ ./result/bin/nixos-test-driver --interactive --keep-vm-state
+$ ./result/bin/nixos-test-driver --keep-vm-state
 </programlisting>
   <para>
     The machine state is stored in the
diff --git a/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml b/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml
index 7159b95b22b0..da2e5076c956 100644
--- a/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml
+++ b/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml
@@ -26,9 +26,9 @@ machine: QEMU running (pid 8841)
     perform a build that starts a QEMU/KVM virtual machine containing a
     NixOS system. The virtual machine mounts the Nix store of the host;
     this makes VM creation very fast, as no disk image needs to be
-    created. Afterwards, you can view a pretty-printed log of the test:
+    created. Afterwards, you can view a log of the test:
   </para>
   <programlisting>
-$ firefox result/log.html
+$ nix-store --read-log result
 </programlisting>
 </section>
diff --git a/nixos/doc/manual/from_md/development/settings-options.section.xml b/nixos/doc/manual/from_md/development/settings-options.section.xml
index c9430b77579c..746011a2d075 100644
--- a/nixos/doc/manual/from_md/development/settings-options.section.xml
+++ b/nixos/doc/manual/from_md/development/settings-options.section.xml
@@ -137,6 +137,97 @@
           </para>
         </listitem>
       </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>pkgs.formats.elixirConf { elixir ? pkgs.elixir }</literal>
+        </term>
+        <listitem>
+          <para>
+            A function taking an attribute set with values
+          </para>
+          <variablelist>
+            <varlistentry>
+              <term>
+                <literal>elixir</literal>
+              </term>
+              <listitem>
+                <para>
+                  The Elixir package which will be used to format the
+                  generated output
+                </para>
+              </listitem>
+            </varlistentry>
+          </variablelist>
+          <para>
+            It returns a set with Elixir-Config-specific attributes
+            <literal>type</literal>, <literal>lib</literal>, and
+            <literal>generate</literal> as specified
+            <link linkend="pkgs-formats-result">below</link>.
+          </para>
+          <para>
+            The <literal>lib</literal> attribute contains functions to
+            be used in settings, for generating special Elixir values:
+          </para>
+          <variablelist>
+            <varlistentry>
+              <term>
+                <literal>mkRaw elixirCode</literal>
+              </term>
+              <listitem>
+                <para>
+                  Outputs the given string as raw Elixir code
+                </para>
+              </listitem>
+            </varlistentry>
+            <varlistentry>
+              <term>
+                <literal>mkGetEnv { envVariable, fallback ? null }</literal>
+              </term>
+              <listitem>
+                <para>
+                  Makes the configuration fetch an environment variable
+                  at runtime
+                </para>
+              </listitem>
+            </varlistentry>
+            <varlistentry>
+              <term>
+                <literal>mkAtom atom</literal>
+              </term>
+              <listitem>
+                <para>
+                  Outputs the given string as an Elixir atom, instead of
+                  the default Elixir binary string. Note: lowercase
+                  atoms still needs to be prefixed with
+                  <literal>:</literal>
+                </para>
+              </listitem>
+            </varlistentry>
+            <varlistentry>
+              <term>
+                <literal>mkTuple array</literal>
+              </term>
+              <listitem>
+                <para>
+                  Outputs the given array as an Elixir tuple, instead of
+                  the default Elixir list
+                </para>
+              </listitem>
+            </varlistentry>
+            <varlistentry>
+              <term>
+                <literal>mkMap attrset</literal>
+              </term>
+              <listitem>
+                <para>
+                  Outputs the given attribute set as an Elixir map,
+                  instead of the default Elixir keyword list
+                </para>
+              </listitem>
+            </varlistentry>
+          </variablelist>
+        </listitem>
+      </varlistentry>
     </variablelist>
     <para xml:id="pkgs-formats-result">
       These functions all return an attribute set with these values:
@@ -154,6 +245,19 @@
       </varlistentry>
       <varlistentry>
         <term>
+          <literal>lib</literal>
+        </term>
+        <listitem>
+          <para>
+            Utility functions for convenience, or special interactions
+            with the format. This attribute is optional. It may contain
+            inside a <literal>types</literal> attribute containing types
+            specific to this format.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
           <literal>generate</literal>
           <emphasis><literal>filename jsonValue</literal></emphasis>
         </term>
diff --git a/nixos/doc/manual/from_md/development/unit-handling.section.xml b/nixos/doc/manual/from_md/development/unit-handling.section.xml
new file mode 100644
index 000000000000..4c980e1213a8
--- /dev/null
+++ b/nixos/doc/manual/from_md/development/unit-handling.section.xml
@@ -0,0 +1,131 @@
+<section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-unit-handling">
+  <title>Unit handling</title>
+  <para>
+    To figure out what units need to be
+    started/stopped/restarted/reloaded, the script first checks the
+    current state of the system, similar to what
+    <literal>systemctl list-units</literal> shows. For each of the
+    units, the script goes through the following checks:
+  </para>
+  <itemizedlist>
+    <listitem>
+      <para>
+        Is the unit file still in the new system? If not,
+        <emphasis role="strong">stop</emphasis> the service unless it
+        sets <literal>X-StopOnRemoval</literal> in the
+        <literal>[Unit]</literal> section to <literal>false</literal>.
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Is it a <literal>.target</literal> unit? If so,
+        <emphasis role="strong">start</emphasis> it unless it sets
+        <literal>RefuseManualStart</literal> in the
+        <literal>[Unit]</literal> section to <literal>true</literal> or
+        <literal>X-OnlyManualStart</literal> in the
+        <literal>[Unit]</literal> section to <literal>true</literal>.
+        Also <emphasis role="strong">stop</emphasis> the unit again
+        unless it sets <literal>X-StopOnReconfiguration</literal> to
+        <literal>false</literal>.
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Are the contents of the unit files different? They are compared
+        by parsing them and comparing their contents. If they are
+        different but only <literal>X-Reload-Triggers</literal> in the
+        <literal>[Unit]</literal> section is changed,
+        <emphasis role="strong">reload</emphasis> the unit. The NixOS
+        module system allows setting these triggers with the option
+        <link linkend="opt-systemd.services">systemd.services.&lt;name&gt;.reloadTriggers</link>.
+        There are some additional keys in the <literal>[Unit]</literal>
+        section that are ignored as well. If the unit files differ in
+        any way, the following actions are performed:
+      </para>
+      <itemizedlist>
+        <listitem>
+          <para>
+            <literal>.path</literal> and <literal>.slice</literal> units
+            are ignored. There is no need to restart them since changes
+            in their values are applied by systemd when systemd is
+            reloaded.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            <literal>.mount</literal> units are
+            <emphasis role="strong">reload</emphasis>ed. These mostly
+            come from the <literal>/etc/fstab</literal> parser.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            <literal>.socket</literal> units are currently ignored. This
+            is to be fixed at a later point.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            The rest of the units (mostly <literal>.service</literal>
+            units) are then <emphasis role="strong">reload</emphasis>ed
+            if <literal>X-ReloadIfChanged</literal> in the
+            <literal>[Service]</literal> section is set to
+            <literal>true</literal> (exposed via
+            <link linkend="opt-systemd.services">systemd.services.&lt;name&gt;.reloadIfChanged</link>).
+            A little exception is done for units that were deactivated
+            in the meantime, for example because they require a unit
+            that got stopped before. These are
+            <emphasis role="strong">start</emphasis>ed instead of
+            reloaded.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            If the reload flag is not set, some more flags decide if the
+            unit is skipped. These flags are
+            <literal>X-RestartIfChanged</literal> in the
+            <literal>[Service]</literal> section (exposed via
+            <link linkend="opt-systemd.services">systemd.services.&lt;name&gt;.restartIfChanged</link>),
+            <literal>RefuseManualStop</literal> in the
+            <literal>[Unit]</literal> section, and
+            <literal>X-OnlyManualStart</literal> in the
+            <literal>[Unit]</literal> section.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            Further behavior depends on the unit having
+            <literal>X-StopIfChanged</literal> in the
+            <literal>[Service]</literal> section set to
+            <literal>true</literal> (exposed via
+            <link linkend="opt-systemd.services">systemd.services.&lt;name&gt;.stopIfChanged</link>).
+            This is set to <literal>true</literal> by default and must
+            be explicitly turned off if not wanted. If the flag is
+            enabled, the unit is
+            <emphasis role="strong">stop</emphasis>ped and then
+            <emphasis role="strong">start</emphasis>ed. If not, the unit
+            is <emphasis role="strong">restart</emphasis>ed. The goal of
+            the flag is to make sure that the new unit never runs in the
+            old environment which is still in place before the
+            activation script is run. This behavior is different when
+            the service is socket-activated, as outlined in the
+            following steps.
+          </para>
+        </listitem>
+        <listitem>
+          <para>
+            The last thing that is taken into account is whether the
+            unit is a service and socket-activated. If
+            <literal>X-StopIfChanged</literal> is
+            <emphasis role="strong">not</emphasis> set, the service is
+            <emphasis role="strong">restart</emphasis>ed with the
+            others. If it is set, both the service and the socket are
+            <emphasis role="strong">stop</emphasis>ped and the socket is
+            <emphasis role="strong">start</emphasis>ed, leaving socket
+            activation to start the service when it’s needed.
+          </para>
+        </listitem>
+      </itemizedlist>
+    </listitem>
+  </itemizedlist>
+</section>
diff --git a/nixos/doc/manual/from_md/development/what-happens-during-a-system-switch.chapter.xml b/nixos/doc/manual/from_md/development/what-happens-during-a-system-switch.chapter.xml
new file mode 100644
index 000000000000..66ba792ddacb
--- /dev/null
+++ b/nixos/doc/manual/from_md/development/what-happens-during-a-system-switch.chapter.xml
@@ -0,0 +1,122 @@
+<chapter xmlns="http://docbook.org/ns/docbook"  xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude" xml:id="sec-switching-systems">
+  <title>What happens during a system switch?</title>
+  <para>
+    Running <literal>nixos-rebuild switch</literal> is one of the more
+    common tasks under NixOS. This chapter explains some of the
+    internals of this command to make it simpler for new module
+    developers to configure their units correctly and to make it easier
+    to understand what is happening and why for curious administrators.
+  </para>
+  <para>
+    <literal>nixos-rebuild</literal>, like many deployment solutions,
+    calls <literal>switch-to-configuration</literal> which resides in a
+    NixOS system at <literal>$out/bin/switch-to-configuration</literal>.
+    The script is called with the action that is to be performed like
+    <literal>switch</literal>, <literal>test</literal>,
+    <literal>boot</literal>. There is also the
+    <literal>dry-activate</literal> action which does not really perform
+    the actions but rather prints what it would do if you called it with
+    <literal>test</literal>. This feature can be used to check what
+    service states would be changed if the configuration was switched
+    to.
+  </para>
+  <para>
+    If the action is <literal>switch</literal> or
+    <literal>boot</literal>, the bootloader is updated first so the
+    configuration will be the next one to boot. Unless
+    <literal>NIXOS_NO_SYNC</literal> is set to <literal>1</literal>,
+    <literal>/nix/store</literal> is synced to disk.
+  </para>
+  <para>
+    If the action is <literal>switch</literal> or
+    <literal>test</literal>, the currently running system is inspected
+    and the actions to switch to the new system are calculated. This
+    process takes two data sources into account:
+    <literal>/etc/fstab</literal> and the current systemd status. Mounts
+    and swaps are read from <literal>/etc/fstab</literal> and the
+    corresponding actions are generated. If a new mount is added, for
+    example, the proper <literal>.mount</literal> unit is marked to be
+    started. The current systemd state is inspected, the difference
+    between the current system and the desired configuration is
+    calculated and actions are generated to get to this state. There are
+    a lot of nuances that can be controlled by the units which are
+    explained here.
+  </para>
+  <para>
+    After calculating what should be done, the actions are carried out.
+    The order of actions is always the same:
+  </para>
+  <itemizedlist spacing="compact">
+    <listitem>
+      <para>
+        Stop units (<literal>systemctl stop</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Run activation script (<literal>$out/activate</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        See if the activation script requested more units to restart
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Restart systemd if needed
+        (<literal>systemd daemon-reexec</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Forget about the failed state of units
+        (<literal>systemctl reset-failed</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Reload systemd (<literal>systemctl daemon-reload</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Reload systemd user instances
+        (<literal>systemctl --user daemon-reload</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Set up tmpfiles (<literal>systemd-tmpfiles --create</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Reload units (<literal>systemctl reload</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Restart units (<literal>systemctl restart</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Start units (<literal>systemctl start</literal>)
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Inspect what changed during these actions and print units that
+        failed and that were newly started
+      </para>
+    </listitem>
+  </itemizedlist>
+  <para>
+    Most of these actions are either self-explaining but some of them
+    have to do with our units or the activation script. For this reason,
+    these topics are explained in the next sections.
+  </para>
+  <xi:include href="unit-handling.section.xml" />
+  <xi:include href="activation-script.section.xml" />
+</chapter>
diff --git a/nixos/doc/manual/from_md/development/writing-modules.chapter.xml b/nixos/doc/manual/from_md/development/writing-modules.chapter.xml
index e33c24f4f12c..367731eda090 100644
--- a/nixos/doc/manual/from_md/development/writing-modules.chapter.xml
+++ b/nixos/doc/manual/from_md/development/writing-modules.chapter.xml
@@ -122,6 +122,25 @@
     services) and <literal>systemd.timers</literal> (the list of
     commands to be executed periodically by <literal>systemd</literal>).
   </para>
+  <para>
+    Care must be taken when writing systemd services using
+    <literal>Exec*</literal> directives. By default systemd performs
+    substitution on <literal>%&lt;char&gt;</literal> specifiers in these
+    directives, expands environment variables from
+    <literal>$FOO</literal> and <literal>${FOO}</literal>, splits
+    arguments on whitespace, and splits commands on
+    <literal>;</literal>. All of these must be escaped to avoid
+    unexpected substitution or splitting when interpolating into an
+    <literal>Exec*</literal> directive, e.g. when using an
+    <literal>extraArgs</literal> option to pass additional arguments to
+    the service. The functions
+    <literal>utils.escapeSystemdExecArg</literal> and
+    <literal>utils.escapeSystemdExecArgs</literal> are provided for
+    this, see <link linkend="exec-escaping-example">Example: Escaping in
+    Exec directives</link> for an example. When using these functions
+    system environment substitution should <emphasis>not</emphasis> be
+    disabled explicitly.
+  </para>
   <anchor xml:id="locate-example" />
   <para>
     <emphasis role="strong">Example: NixOS Module for the
@@ -184,6 +203,36 @@ in {
   };
 }
 </programlisting>
+  <anchor xml:id="exec-escaping-example" />
+  <para>
+    <emphasis role="strong">Example: Escaping in Exec
+    directives</emphasis>
+  </para>
+  <programlisting language="bash">
+{ config, lib, pkgs, utils, ... }:
+
+with lib;
+
+let
+  cfg = config.services.echo;
+  echoAll = pkgs.writeScript &quot;echo-all&quot; ''
+    #! ${pkgs.runtimeShell}
+    for s in &quot;$@&quot;; do
+      printf '%s\n' &quot;$s&quot;
+    done
+  '';
+  args = [ &quot;a%Nything&quot; &quot;lang=\${LANG}&quot; &quot;;&quot; &quot;/bin/sh -c date&quot; ];
+in {
+  systemd.services.echo =
+    { description = &quot;Echo to the journal&quot;;
+      wantedBy = [ &quot;multi-user.target&quot; ];
+      serviceConfig.Type = &quot;oneshot&quot;;
+      serviceConfig.ExecStart = ''
+        ${echoAll} ${utils.escapeSystemdExecArgs args}
+      '';
+    };
+}
+</programlisting>
   <xi:include href="option-declarations.section.xml" />
   <xi:include href="option-types.section.xml" />
   <xi:include href="option-def.section.xml" />
diff --git a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
index 0d523681b639..45c9c40c6095 100644
--- a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
+++ b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
@@ -117,407 +117,413 @@ if not &quot;Linux&quot; in machine.succeed(&quot;uname&quot;):
   <programlisting language="python">
 start_all()
 </programlisting>
-  <para>
-    The following methods are available on machine objects:
-  </para>
-  <variablelist>
-    <varlistentry>
-      <term>
-        <literal>start</literal>
-      </term>
-      <listitem>
-        <para>
-          Start the virtual machine. This method is asynchronous — it
-          does not wait for the machine to finish booting.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>shutdown</literal>
-      </term>
-      <listitem>
-        <para>
-          Shut down the machine, waiting for the VM to exit.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>crash</literal>
-      </term>
-      <listitem>
-        <para>
-          Simulate a sudden power failure, by telling the VM to exit
-          immediately.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>block</literal>
-      </term>
-      <listitem>
-        <para>
-          Simulate unplugging the Ethernet cable that connects the
-          machine to the other machines.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>unblock</literal>
-      </term>
-      <listitem>
-        <para>
-          Undo the effect of <literal>block</literal>.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>screenshot</literal>
-      </term>
-      <listitem>
-        <para>
-          Take a picture of the display of the virtual machine, in PNG
-          format. The screenshot is linked from the HTML log.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>get_screen_text_variants</literal>
-      </term>
-      <listitem>
-        <para>
-          Return a list of different interpretations of what is
-          currently visible on the machine's screen using optical
-          character recognition. The number and order of the
-          interpretations is not specified and is subject to change, but
-          if no exception is raised at least one will be returned.
-        </para>
-        <note>
+  <section xml:id="ssec-machine-objects">
+    <title>Machine objects</title>
+    <para>
+      The following methods are available on machine objects:
+    </para>
+    <variablelist>
+      <varlistentry>
+        <term>
+          <literal>start</literal>
+        </term>
+        <listitem>
           <para>
-            This requires passing <literal>enableOCR</literal> to the
-            test attribute set.
+            Start the virtual machine. This method is asynchronous — it
+            does not wait for the machine to finish booting.
           </para>
-        </note>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>get_screen_text</literal>
-      </term>
-      <listitem>
-        <para>
-          Return a textual representation of what is currently visible
-          on the machine's screen using optical character recognition.
-        </para>
-        <note>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>shutdown</literal>
+        </term>
+        <listitem>
           <para>
-            This requires passing <literal>enableOCR</literal> to the
-            test attribute set.
+            Shut down the machine, waiting for the VM to exit.
           </para>
-        </note>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>send_monitor_command</literal>
-      </term>
-      <listitem>
-        <para>
-          Send a command to the QEMU monitor. This is rarely used, but
-          allows doing stuff such as attaching virtual USB disks to a
-          running machine.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>send_key</literal>
-      </term>
-      <listitem>
-        <para>
-          Simulate pressing keys on the virtual keyboard, e.g.,
-          <literal>send_key(&quot;ctrl-alt-delete&quot;)</literal>.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>send_chars</literal>
-      </term>
-      <listitem>
-        <para>
-          Simulate typing a sequence of characters on the virtual
-          keyboard, e.g.,
-          <literal>send_chars(&quot;foobar\n&quot;)</literal> will type
-          the string <literal>foobar</literal> followed by the Enter
-          key.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>execute</literal>
-      </term>
-      <listitem>
-        <para>
-          Execute a shell command, returning a list
-          <literal>(status, stdout)</literal>. If the command detaches,
-          it must close stdout, as <literal>execute</literal> will wait
-          for this to consume all output reliably. This can be achieved
-          by redirecting stdout to stderr <literal>&gt;&amp;2</literal>,
-          to <literal>/dev/console</literal>,
-          <literal>/dev/null</literal> or a file. Examples of detaching
-          commands are <literal>sleep 365d &amp;</literal>, where the
-          shell forks a new process that can write to stdout and
-          <literal>xclip -i</literal>, where the
-          <literal>xclip</literal> command itself forks without closing
-          stdout. Takes an optional parameter
-          <literal>check_return</literal> that defaults to
-          <literal>True</literal>. Setting this parameter to
-          <literal>False</literal> will not check for the return code
-          and return -1 instead. This can be used for commands that shut
-          down the VM and would therefore break the pipe that would be
-          used for retrieving the return code.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>succeed</literal>
-      </term>
-      <listitem>
-        <para>
-          Execute a shell command, raising an exception if the exit
-          status is not zero, otherwise returning the standard output.
-          Commands are run with <literal>set -euo pipefail</literal>
-          set:
-        </para>
-        <itemizedlist>
-          <listitem>
-            <para>
-              If several commands are separated by <literal>;</literal>
-              and one fails, the command as a whole will fail.
-            </para>
-          </listitem>
-          <listitem>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>crash</literal>
+        </term>
+        <listitem>
+          <para>
+            Simulate a sudden power failure, by telling the VM to exit
+            immediately.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>block</literal>
+        </term>
+        <listitem>
+          <para>
+            Simulate unplugging the Ethernet cable that connects the
+            machine to the other machines.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>unblock</literal>
+        </term>
+        <listitem>
+          <para>
+            Undo the effect of <literal>block</literal>.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>screenshot</literal>
+        </term>
+        <listitem>
+          <para>
+            Take a picture of the display of the virtual machine, in PNG
+            format. The screenshot is linked from the HTML log.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>get_screen_text_variants</literal>
+        </term>
+        <listitem>
+          <para>
+            Return a list of different interpretations of what is
+            currently visible on the machine's screen using optical
+            character recognition. The number and order of the
+            interpretations is not specified and is subject to change,
+            but if no exception is raised at least one will be returned.
+          </para>
+          <note>
             <para>
-              For pipelines, the last non-zero exit status will be
-              returned (if there is one, zero will be returned
-              otherwise).
+              This requires passing <literal>enableOCR</literal> to the
+              test attribute set.
             </para>
-          </listitem>
-          <listitem>
+          </note>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>get_screen_text</literal>
+        </term>
+        <listitem>
+          <para>
+            Return a textual representation of what is currently visible
+            on the machine's screen using optical character recognition.
+          </para>
+          <note>
             <para>
-              Dereferencing unset variables fail the command.
+              This requires passing <literal>enableOCR</literal> to the
+              test attribute set.
             </para>
-          </listitem>
-          <listitem>
+          </note>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>send_monitor_command</literal>
+        </term>
+        <listitem>
+          <para>
+            Send a command to the QEMU monitor. This is rarely used, but
+            allows doing stuff such as attaching virtual USB disks to a
+            running machine.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>send_key</literal>
+        </term>
+        <listitem>
+          <para>
+            Simulate pressing keys on the virtual keyboard, e.g.,
+            <literal>send_key(&quot;ctrl-alt-delete&quot;)</literal>.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>send_chars</literal>
+        </term>
+        <listitem>
+          <para>
+            Simulate typing a sequence of characters on the virtual
+            keyboard, e.g.,
+            <literal>send_chars(&quot;foobar\n&quot;)</literal> will
+            type the string <literal>foobar</literal> followed by the
+            Enter key.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>execute</literal>
+        </term>
+        <listitem>
+          <para>
+            Execute a shell command, returning a list
+            <literal>(status, stdout)</literal>. If the command
+            detaches, it must close stdout, as
+            <literal>execute</literal> will wait for this to consume all
+            output reliably. This can be achieved by redirecting stdout
+            to stderr <literal>&gt;&amp;2</literal>, to
+            <literal>/dev/console</literal>,
+            <literal>/dev/null</literal> or a file. Examples of
+            detaching commands are <literal>sleep 365d &amp;</literal>,
+            where the shell forks a new process that can write to stdout
+            and <literal>xclip -i</literal>, where the
+            <literal>xclip</literal> command itself forks without
+            closing stdout. Takes an optional parameter
+            <literal>check_return</literal> that defaults to
+            <literal>True</literal>. Setting this parameter to
+            <literal>False</literal> will not check for the return code
+            and return -1 instead. This can be used for commands that
+            shut down the VM and would therefore break the pipe that
+            would be used for retrieving the return code.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>succeed</literal>
+        </term>
+        <listitem>
+          <para>
+            Execute a shell command, raising an exception if the exit
+            status is not zero, otherwise returning the standard output.
+            Commands are run with <literal>set -euo pipefail</literal>
+            set:
+          </para>
+          <itemizedlist>
+            <listitem>
+              <para>
+                If several commands are separated by
+                <literal>;</literal> and one fails, the command as a
+                whole will fail.
+              </para>
+            </listitem>
+            <listitem>
+              <para>
+                For pipelines, the last non-zero exit status will be
+                returned (if there is one, zero will be returned
+                otherwise).
+              </para>
+            </listitem>
+            <listitem>
+              <para>
+                Dereferencing unset variables fail the command.
+              </para>
+            </listitem>
+            <listitem>
+              <para>
+                It will wait for stdout to be closed. See
+                <literal>execute</literal> for the implications.
+              </para>
+            </listitem>
+          </itemizedlist>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>fail</literal>
+        </term>
+        <listitem>
+          <para>
+            Like <literal>succeed</literal>, but raising an exception if
+            the command returns a zero status.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_until_succeeds</literal>
+        </term>
+        <listitem>
+          <para>
+            Repeat a shell command with 1-second intervals until it
+            succeeds.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_until_fails</literal>
+        </term>
+        <listitem>
+          <para>
+            Repeat a shell command with 1-second intervals until it
+            fails.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_unit</literal>
+        </term>
+        <listitem>
+          <para>
+            Wait until the specified systemd unit has reached the
+            <quote>active</quote> state.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_file</literal>
+        </term>
+        <listitem>
+          <para>
+            Wait until the specified file exists.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_open_port</literal>
+        </term>
+        <listitem>
+          <para>
+            Wait until a process is listening on the given TCP port (on
+            <literal>localhost</literal>, at least).
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_closed_port</literal>
+        </term>
+        <listitem>
+          <para>
+            Wait until nobody is listening on the given TCP port.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_x</literal>
+        </term>
+        <listitem>
+          <para>
+            Wait until the X11 server is accepting connections.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_text</literal>
+        </term>
+        <listitem>
+          <para>
+            Wait until the supplied regular expressions matches the
+            textual contents of the screen by using optical character
+            recognition (see <literal>get_screen_text</literal> and
+            <literal>get_screen_text_variants</literal>).
+          </para>
+          <note>
             <para>
-              It will wait for stdout to be closed. See
-              <literal>execute</literal> for the implications.
+              This requires passing <literal>enableOCR</literal> to the
+              test attribute set.
             </para>
-          </listitem>
-        </itemizedlist>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>fail</literal>
-      </term>
-      <listitem>
-        <para>
-          Like <literal>succeed</literal>, but raising an exception if
-          the command returns a zero status.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_until_succeeds</literal>
-      </term>
-      <listitem>
-        <para>
-          Repeat a shell command with 1-second intervals until it
-          succeeds.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_until_fails</literal>
-      </term>
-      <listitem>
-        <para>
-          Repeat a shell command with 1-second intervals until it fails.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_unit</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until the specified systemd unit has reached the
-          <quote>active</quote> state.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_file</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until the specified file exists.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_open_port</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until a process is listening on the given TCP port (on
-          <literal>localhost</literal>, at least).
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_closed_port</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until nobody is listening on the given TCP port.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_x</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until the X11 server is accepting connections.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_text</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until the supplied regular expressions matches the
-          textual contents of the screen by using optical character
-          recognition (see <literal>get_screen_text</literal> and
-          <literal>get_screen_text_variants</literal>).
-        </para>
-        <note>
+          </note>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_console_text</literal>
+        </term>
+        <listitem>
           <para>
-            This requires passing <literal>enableOCR</literal> to the
-            test attribute set.
+            Wait until the supplied regular expressions match a line of
+            the serial console output. This method is useful when OCR is
+            not possibile or accurate enough.
           </para>
-        </note>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_console_text</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until the supplied regular expressions match a line of
-          the serial console output. This method is useful when OCR is
-          not possibile or accurate enough.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>wait_for_window</literal>
-      </term>
-      <listitem>
-        <para>
-          Wait until an X11 window has appeared whose name matches the
-          given regular expression, e.g.,
-          <literal>wait_for_window(&quot;Terminal&quot;)</literal>.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>copy_from_host</literal>
-      </term>
-      <listitem>
-        <para>
-          Copies a file from host to machine, e.g.,
-          <literal>copy_from_host(&quot;myfile&quot;, &quot;/etc/my/important/file&quot;)</literal>.
-        </para>
-        <para>
-          The first argument is the file on the host. The file needs to
-          be accessible while building the nix derivation. The second
-          argument is the location of the file on the machine.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>systemctl</literal>
-      </term>
-      <listitem>
-        <para>
-          Runs <literal>systemctl</literal> commands with optional
-          support for <literal>systemctl --user</literal>
-        </para>
-        <programlisting language="python">
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>wait_for_window</literal>
+        </term>
+        <listitem>
+          <para>
+            Wait until an X11 window has appeared whose name matches the
+            given regular expression, e.g.,
+            <literal>wait_for_window(&quot;Terminal&quot;)</literal>.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>copy_from_host</literal>
+        </term>
+        <listitem>
+          <para>
+            Copies a file from host to machine, e.g.,
+            <literal>copy_from_host(&quot;myfile&quot;, &quot;/etc/my/important/file&quot;)</literal>.
+          </para>
+          <para>
+            The first argument is the file on the host. The file needs
+            to be accessible while building the nix derivation. The
+            second argument is the location of the file on the machine.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>systemctl</literal>
+        </term>
+        <listitem>
+          <para>
+            Runs <literal>systemctl</literal> commands with optional
+            support for <literal>systemctl --user</literal>
+          </para>
+          <programlisting language="python">
 machine.systemctl(&quot;list-jobs --no-pager&quot;) # runs `systemctl list-jobs --no-pager`
 machine.systemctl(&quot;list-jobs --no-pager&quot;, &quot;any-user&quot;) # spawns a shell for `any-user` and runs `systemctl --user list-jobs --no-pager`
 </programlisting>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>shell_interact</literal>
-      </term>
-      <listitem>
-        <para>
-          Allows you to directly interact with the guest shell. This
-          should only be used during test development, not in production
-          tests. Killing the interactive session with
-          <literal>Ctrl-d</literal> or <literal>Ctrl-c</literal> also
-          ends the guest session.
-        </para>
-      </listitem>
-    </varlistentry>
-  </variablelist>
-  <para>
-    To test user units declared by
-    <literal>systemd.user.services</literal> the optional
-    <literal>user</literal> argument can be used:
-  </para>
-  <programlisting language="python">
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <literal>shell_interact</literal>
+        </term>
+        <listitem>
+          <para>
+            Allows you to directly interact with the guest shell. This
+            should only be used during test development, not in
+            production tests. Killing the interactive session with
+            <literal>Ctrl-d</literal> or <literal>Ctrl-c</literal> also
+            ends the guest session.
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
+    <para>
+      To test user units declared by
+      <literal>systemd.user.services</literal> the optional
+      <literal>user</literal> argument can be used:
+    </para>
+    <programlisting language="python">
 machine.start()
 machine.wait_for_x()
 machine.wait_for_unit(&quot;xautolock.service&quot;, &quot;x-session-user&quot;)
 </programlisting>
-  <para>
-    This applies to <literal>systemctl</literal>,
-    <literal>get_unit_info</literal>, <literal>wait_for_unit</literal>,
-    <literal>start_job</literal> and <literal>stop_job</literal>.
-  </para>
-  <para>
-    For faster dev cycles it's also possible to disable the code-linters
-    (this shouldn't be commited though):
-  </para>
-  <programlisting language="bash">
+    <para>
+      This applies to <literal>systemctl</literal>,
+      <literal>get_unit_info</literal>,
+      <literal>wait_for_unit</literal>, <literal>start_job</literal> and
+      <literal>stop_job</literal>.
+    </para>
+    <para>
+      For faster dev cycles it's also possible to disable the
+      code-linters (this shouldn't be commited though):
+    </para>
+    <programlisting language="bash">
 import ./make-test-python.nix {
   skipLint = true;
   machine =
@@ -531,13 +537,13 @@ import ./make-test-python.nix {
     '';
 }
 </programlisting>
-  <para>
-    This will produce a Nix warning at evaluation time. To fully disable
-    the linter, wrap the test script in comment directives to disable
-    the Black linter directly (again, don't commit this within the
-    Nixpkgs repository):
-  </para>
-  <programlisting language="bash">
+    <para>
+      This will produce a Nix warning at evaluation time. To fully
+      disable the linter, wrap the test script in comment directives to
+      disable the Black linter directly (again, don't commit this within
+      the Nixpkgs repository):
+    </para>
+    <programlisting language="bash">
   testScript =
     ''
       # fmt: off
@@ -545,4 +551,66 @@ import ./make-test-python.nix {
       # fmt: on
     '';
 </programlisting>
+  </section>
+  <section xml:id="ssec-failing-tests-early">
+    <title>Failing tests early</title>
+    <para>
+      To fail tests early when certain invariables are no longer met
+      (instead of waiting for the build to time out), the decorator
+      <literal>polling_condition</literal> is provided. For example, if
+      we are testing a program <literal>foo</literal> that should not
+      quit after being started, we might write the following:
+    </para>
+    <programlisting language="python">
+@polling_condition
+def foo_running():
+    machine.succeed(&quot;pgrep -x foo&quot;)
+
+
+machine.succeed(&quot;foo --start&quot;)
+machine.wait_until_succeeds(&quot;pgrep -x foo&quot;)
+
+with foo_running:
+    ...  # Put `foo` through its paces
+</programlisting>
+    <para>
+      <literal>polling_condition</literal> takes the following
+      (optional) arguments:
+    </para>
+    <para>
+      <literal>seconds_interval</literal>
+    </para>
+    <para>
+      : specifies how often the condition should be polled:
+    </para>
+    <programlisting>
+```py
+@polling_condition(seconds_interval=10)
+def foo_running():
+    machine.succeed(&quot;pgrep -x foo&quot;)
+```
+</programlisting>
+    <para>
+      <literal>description</literal>
+    </para>
+    <para>
+      : is used in the log when the condition is checked. If this is not
+      provided, the description is pulled from the docstring of the
+      function. These two are therefore equivalent:
+    </para>
+    <programlisting>
+```py
+@polling_condition
+def foo_running():
+    &quot;check that foo is running&quot;
+    machine.succeed(&quot;pgrep -x foo&quot;)
+```
+
+```py
+@polling_condition(description=&quot;check that foo is running&quot;)
+def foo_running():
+    machine.succeed(&quot;pgrep -x foo&quot;)
+```
+</programlisting>
+  </section>
 </section>
diff --git a/nixos/doc/manual/from_md/installation/installing-pxe.section.xml b/nixos/doc/manual/from_md/installation/installing-pxe.section.xml
index 1dd15ddacba8..94172de65ea0 100644
--- a/nixos/doc/manual/from_md/installation/installing-pxe.section.xml
+++ b/nixos/doc/manual/from_md/installation/installing-pxe.section.xml
@@ -7,11 +7,11 @@
   <para>
     These instructions assume that you have an existing PXE or iPXE
     infrastructure and simply want to add the NixOS installer as another
-    option. To build the necessary files from a recent version of
+    option. To build the necessary files from your current version of
     nixpkgs, you can run:
   </para>
   <programlisting>
-nix-build -A netboot.x86_64-linux nixos/release.nix
+nix-build -A netboot.x86_64-linux '&lt;nixpkgs/nixos/release.nix&gt;'
 </programlisting>
   <para>
     This will create a <literal>result</literal> directory containing: *
diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
index e2bda7604e48..b61a0268dee2 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
@@ -26,8 +26,36 @@
       </listitem>
       <listitem>
         <para>
-          <literal>iptables</literal> now uses
-          <literal>nf_tables</literal> backend.
+          <literal>iptables</literal> is now using
+          <literal>nf_tables</literal> under the hood, by using
+          <literal>iptables-nft</literal>, similar to
+          <link xlink:href="https://wiki.debian.org/nftables#Current_status">Debian</link>
+          and
+          <link xlink:href="https://fedoraproject.org/wiki/Changes/iptables-nft-default">Fedora</link>.
+          This means, <literal>ip[6]tables</literal>,
+          <literal>arptables</literal> and <literal>ebtables</literal>
+          commands will actually show rules from some specific tables in
+          the <literal>nf_tables</literal> kernel subsystem. In case
+          you’re migrating from an older release without rebooting,
+          there might be cases where you end up with iptable rules
+          configured both in the legacy <literal>iptables</literal>
+          kernel backend, as well as in the <literal>nf_tables</literal>
+          backend. This can lead to confusing firewall behaviour. An
+          <literal>iptables-save</literal> after switching will complain
+          about <quote>iptables-legacy tables present</quote>. It’s
+          probably best to reboot after the upgrade, or manually
+          removing all legacy iptables rules (via the
+          <literal>iptables-legacy</literal> package).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          systemd got an <literal>nftables</literal> backend, and
+          configures (networkd) rules in their own
+          <literal>io.systemd.*</literal> tables. Check
+          <literal>nft list ruleset</literal> to see these rules, not
+          <literal>iptables-save</literal> (which only shows
+          <literal>iptables</literal>-created rules.
         </para>
       </listitem>
       <listitem>
@@ -275,13 +303,6 @@
       </listitem>
       <listitem>
         <para>
-          <link xlink:href="https://maddy.email">maddy</link>, a
-          composable all-in-one mail server. Available as
-          <link xlink:href="options.html#opt-services.maddy.enable">services.maddy</link>.
-        </para>
-      </listitem>
-      <listitem>
-        <para>
           <link xlink:href="https://sr.ht">sourcehut</link>, a
           collection of tools useful for software development. Available
           as
@@ -1427,6 +1448,26 @@ Superuser created successfully.
           for those who want to have all RetroArch cores available.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The Linux kernel for security reasons now restricts access to
+          BPF syscalls via <literal>BPF_UNPRIV_DEFAULT_OFF=y</literal>.
+          Unprivileged access can be reenabled via the
+          <literal>kernel.unprivileged_bpf_disabled</literal> sysctl
+          knob.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>/usr</literal> will always be included in the initial
+          ramdisk. See the
+          <literal>fileSystems.&lt;name&gt;.neededForBoot</literal>
+          option. If any files exist under <literal>/usr</literal>
+          (which is not typical for NixOS), they will be included in the
+          initial ramdisk, increasing its size to a possibly problematic
+          extent.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-21.11-notable-changes">
diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
index c84a3e3b0193..fc7cb28ffdfd 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
@@ -14,13 +14,320 @@
   </itemizedlist>
   <section xml:id="sec-release-22.05-highlights">
     <title>Highlights</title>
-    <para>
-    </para>
+    <itemizedlist>
+      <listitem>
+        <para>
+          <literal>security.acme.defaults</literal> has been added to
+          simplify configuring settings for many certificates at once.
+          This also opens up the the option to use DNS-01 validation
+          when using <literal>enableACME</literal> on web server virtual
+          hosts (e.g.
+          <literal>services.nginx.virtualHosts.*.enableACME</literal>).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          PHP 8.1 is now available
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Mattermost has been updated to extended support release 6.3,
+          as the previously packaged extended support release 5.37 is
+          <link xlink:href="https://docs.mattermost.com/upgrade/extended-support-release.html">reaching
+          its end of life</link>. Migrations may take a while, see the
+          <link xlink:href="https://docs.mattermost.com/install/self-managed-changelog.html#release-v6-3-extended-support-release">changelog</link>
+          and
+          <link xlink:href="https://docs.mattermost.com/upgrade/important-upgrade-notes.html">important
+          upgrade notes</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          systemd services can now set
+          <link linkend="opt-systemd.services">systemd.services.&lt;name&gt;.reloadTriggers</link>
+          instead of <literal>reloadIfChanged</literal> for a more
+          granular distinction between reloads and restarts.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://kops.sigs.k8s.io"><literal>kops</literal></link>
+          defaults to 1.22.4, which will enable
+          <link xlink:href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html">Instance
+          Metadata Service Version 2</link> and require tokens on new
+          clusters with Kubernetes 1.22. This will increase security by
+          default, but may break some types of workloads. See the
+          <link xlink:href="https://kops.sigs.k8s.io/releases/1.22-notes/">release
+          notes</link> for details.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Module authors can use
+          <literal>mkRenamedOptionModuleWith</literal> to automate the
+          deprecation cycle without annoying out-of-tree module authors
+          and their users.
+        </para>
+      </listitem>
+    </itemizedlist>
   </section>
   <section xml:id="sec-release-22.05-new-services">
     <title>New Services</title>
-    <para>
-    </para>
+    <itemizedlist>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw">aesmd</link>,
+          the Intel SGX Architectural Enclave Service Manager. Available
+          as
+          <link linkend="opt-services.aesmd.enable">services.aesmd</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://docs.docker.com/engine/security/rootless/">rootless
+          Docker</link>, a <literal>systemd --user</literal> Docker
+          service which runs without root permissions. Available as
+          <link xlink:href="options.html#opt-virtualisation.docker.rootless.enable">virtualisation.docker.rootless.enable</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://conduit.rs/">matrix-conduit</link>,
+          a simple, fast and reliable chat server powered by matrix.
+          Available as
+          <link xlink:href="option.html#opt-services.matrix-conduit.enable">services.matrix-conduit</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html">filebeat</link>,
+          a lightweight shipper for forwarding and centralizing log
+          data. Available as
+          <link linkend="opt-services.filebeat.enable">services.filebeat</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/linux-apfs/linux-apfs-rw">apfs</link>,
+          a kernel module for mounting the Apple File System (APFS).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://frrouting.org/">FRRouting</link>, a
+          popular suite of Internet routing protocol daemons (BGP, BFD,
+          OSPF, IS-IS, VVRP and others). Available as
+          <link linkend="opt-services.frr.babel.enable">services.frr</link>
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/hifi/heisenbridge">heisenbridge</link>,
+          a bouncer-style Matrix IRC bridge. Available as
+          <link xlink:href="options.html#opt-services.heisenbridge.enable">services.heisenbridge</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://snowflake.torproject.org/">snowflake-proxy</link>,
+          a system to defeat internet censorship. Available as
+          <link xlink:href="options.html#opt-services.snowflake-proxy.enable">services.snowflake-proxy</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://ergo.chat">ergochat</link>, a modern
+          IRC with IRCv3 features. Available as
+          <link xlink:href="options.html#opt-services.ergochat.enable">services.ergochat</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/ngoduykhanh/PowerDNS-Admin">PowerDNS-Admin</link>,
+          a web interface for the PowerDNS server. Available at
+          <link xlink:href="options.html#opt-services.powerdns-admin.enable">services.powerdns-admin</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/postgres/pgadmin4">pgadmin4</link>,
+          an admin interface for the PostgreSQL database. Available at
+          <link xlink:href="options.html#opt-services.pgadmin.enable">services.pgadmin</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/sezanzeb/input-remapper">input-remapper</link>,
+          an easy to use tool to change the mapping of your input device
+          buttons. Available at
+          <link xlink:href="options.html#opt-services.input-remapper.enable">services.input-remapper</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://invoiceplane.com">InvoicePlane</link>,
+          web application for managing and creating invoices. Available
+          at
+          <link xlink:href="options.html#opt-services.invoiceplane.enable">services.invoiceplane</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://maddy.email">maddy</link>, a
+          composable all-in-one mail server. Available as
+          <link xlink:href="options.html#opt-services.maddy.enable">services.maddy</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://www.scorchworks.com/K40whisperer/k40whisperer.html">K40-Whisperer</link>,
+          a program to control cheap Chinese laser cutters. Available as
+          <link xlink:href="options.html#opt-programs.k4-whisperer.enable">programs.k40-whisperer.enable</link>.
+          Users must add themselves to the <literal>k40</literal> group
+          to be able to access the device.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/mgumz/mtr-exporter">mtr-exporter</link>,
+          a Prometheus exporter for mtr metrics. Available as
+          <link xlink:href="options.html#opt-services.mtr-exporter.enable">services.mtr-exporter</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/prometheus-pve/prometheus-pve-exporter">prometheus-pve-exporter</link>,
+          a tool that exposes information from the Proxmox VE API for
+          use by Prometheus. Available as
+          <link xlink:href="options.html#opt-services.prometheus.exporters.pve">services.prometheus.exporters.pve</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://tetrd.app">tetrd</link>, share your
+          internet connection from your device to your PC and vice versa
+          through a USB cable. Available at
+          <link linkend="opt-services.tetrd.enable">services.tetrd</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/mbrubeck/agate">agate</link>,
+          a very simple server for the Gemini hypertext protocol.
+          Available as
+          <link xlink:href="options.html#opt-services.agate.enable">services.agate</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/JustArchiNET/ArchiSteamFarm">ArchiSteamFarm</link>,
+          a C# application with primary purpose of idling Steam cards
+          from multiple accounts simultaneously. Available as
+          <link xlink:href="options.html#opt-services.archisteamfarm.enable">services.archisteamfarm</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://goteleport.com">teleport</link>,
+          allows engineers and security professionals to unify access
+          for SSH servers, Kubernetes clusters, web applications, and
+          databases across all environments. Available at
+          <link linkend="opt-services.teleport.enable">services.teleport</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://loic-sharma.github.io/BaGet/">BaGet</link>,
+          a lightweight NuGet and symbol server. Available at
+          <link linkend="opt-services.baget.enable">services.baget</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://moosefs.com">moosefs</link>, fault
+          tolerant petabyte distributed file system. Available as
+          <link linkend="opt-services.moosefs.client.enable">moosefs</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/ThomasLeister/prosody-filer">prosody-filer</link>,
+          a server for handling XMPP HTTP Upload requests. Available at
+          <link linkend="opt-services.prosody-filer.enable">services.prosody-filer</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/rfjakob/systembus-notify">systembus-notify</link>,
+          allow system level notifications to reach the users. Available
+          as
+          <link xlink:href="opt-services.systembus-notify.enable">services.systembus-notify</link>.
+          Please keep in mind that this service should only be enabled
+          on machines with fully trusted users, as any local user is
+          able to DoS user sessions by spamming notifications.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/audreyt/ethercalc">ethercalc</link>,
+          an online collaborative spreadsheet. Available as
+          <link xlink:href="options.html#opt-services.ethercalc.enable">services.ethercalc</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://nbd.sourceforge.io/">nbd</link>, a
+          Network Block Device server. Available as
+          <link xlink:href="options.html#opt-services.nbd.server.enable">services.nbd</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/Mic92/nix-ld">nix-ld</link>,
+          Run unpatched dynamic binaries on NixOS. Available as
+          <link xlink:href="options.html#opt-programs.nix-ld.enable">programs.nix-ld</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://timetagger.app">timetagger</link>,
+          an open source time-tracker with an intuitive user experience
+          and powerful reporting.
+          <link xlink:href="options.html#opt-services.timetagger.enable">services.timetagger</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://www.rstudio.com/products/rstudio/#rstudio-server">rstudio-server</link>,
+          a browser-based version of the RStudio IDE for the R
+          programming language. Available as
+          <link xlink:href="options.html#opt-services.rstudio-server.enable">services.rstudio-server</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/juanfont/headscale">headscale</link>,
+          an Open Source implementation of the
+          <link xlink:href="https://tailscale.io">Tailscale</link>
+          Control Server. Available as
+          <link xlink:href="options.html#opt-services.headscale.enable">services.headscale</link>
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://0xerr0r.github.io/blocky/">blocky</link>,
+          fast and lightweight DNS proxy as ad-blocker for local network
+          with many features.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://clusterlabs.org/pacemaker/">pacemaker</link>
+          cluster resource manager
+        </para>
+      </listitem>
+    </itemizedlist>
   </section>
   <section xml:id="sec-release-22.05-incompatibilities">
     <title>Backward Incompatibilities</title>
@@ -48,6 +355,36 @@
       </listitem>
       <listitem>
         <para>
+          <literal>pkgs.ghc.withPackages</literal> as well as
+          <literal>haskellPackages.ghcWithPackages</literal> etc. now
+          needs be overridden directly, as opposed to overriding the
+          result of calling it. Additionally, the
+          <literal>withLLVM</literal> parameter has been renamed to
+          <literal>useLLVM</literal>. So instead of
+          <literal>(ghc.withPackages (p: [])).override { withLLVM = true; }</literal>,
+          one needs to use
+          <literal>(ghc.withPackages.override { useLLVM = true; }) (p: [])</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>home-assistant</literal> module now requires
+          users that don’t want their configuration to be managed
+          declaratively to set
+          <literal>services.home-assistant.config = null;</literal>.
+          This is required due to the way default settings are handled
+          with the new settings style.
+        </para>
+        <para>
+          Additionally the default list of
+          <literal>extraComponents</literal> now includes the minimal
+          dependencies to successfully complete the
+          <link xlink:href="https://www.home-assistant.io/getting-started/onboarding/">onboarding</link>
+          procedure.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
           <literal>pkgs.emacsPackages.orgPackages</literal> is removed
           because org elpa is deprecated. The packages in the top level
           of <literal>pkgs.emacsPackages</literal>, such as org and
@@ -59,6 +396,228 @@
       </listitem>
       <listitem>
         <para>
+          <literal>services.kubernetes.addons.dashboard</literal> was
+          removed due to it being an outdated version.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.kubernetes.scheduler.{port,address}</literal>
+          now set <literal>--secure-port</literal> and
+          <literal>--bind-address</literal> instead of
+          <literal>--port</literal> and <literal>--address</literal>,
+          since the former have been deprecated and are no longer
+          functional in kubernetes&gt;=1.23. Ensure that you are not
+          relying on the insecure behaviour before upgrading.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.k3s.enable</literal> no longer implies
+          <literal>systemd.enableUnifiedCgroupHierarchy = false</literal>,
+          and will default to the <quote>systemd</quote> cgroup driver
+          when using <literal>services.k3s.docker = true</literal>. This
+          change may require a reboot to take effect, and k3s may not be
+          able to run if the boot cgroup hierarchy does not match its
+          configuration. The previous behavior may be retained by
+          explicitly setting
+          <literal>systemd.enableUnifiedCgroupHierarchy = false</literal>
+          in your configuration.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>fonts.fonts</literal> no longer includes ancient
+          bitmap fonts when both
+          <literal>config.services.xserver.enable</literal> and
+          <literal>config.nixpkgs.config.allowUnfree</literal> are
+          enabled. If you still want these fonts, use:
+        </para>
+        <programlisting language="bash">
+{
+  fonts.fonts = [
+    pkgs.xorg.fontbhlucidatypewriter100dpi
+    pkgs.xorg.fontbhlucidatypewriter75dpi
+    pkgs.xorg.fontbh100dpi
+  ];
+}
+</programlisting>
+      </listitem>
+      <listitem>
+        <para>
+          The DHCP server (<literal>services.dhcpd4</literal>,
+          <literal>services.dhcpd6</literal>) has been hardened. The
+          service is now using the systemd’s
+          <literal>DynamicUser</literal> mechanism to run as an
+          unprivileged dynamically-allocated user with limited
+          capabilities. The dhcpd state files are now always stored in
+          <literal>/var/lib/dhcpd{4,6}</literal> and the
+          <literal>services.dhcpd4.stateDir</literal> and
+          <literal>service.dhcpd6.stateDir</literal> options have been
+          removed. If you were depending on root privileges or
+          set{uid,gid,cap} binaries in dhcpd shell hooks, you may give
+          dhcpd more capabilities with e.g.
+          <literal>systemd.services.dhcpd6.serviceConfig.AmbientCapabilities</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>mailpile</literal> email webclient
+          (<literal>services.mailpile</literal>) has been removed due to
+          its reliance on python2.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>matrix-synapse</literal> service
+          (<literal>services.matrix-synapse</literal>) has been
+          converted to use the <literal>settings</literal> option
+          defined in RFC42. This means that options that are part of
+          your <literal>homeserver.yaml</literal> configuration, and
+          that were specified at the top-level of the module
+          (<literal>services.matrix-synapse</literal>) now need to be
+          moved into
+          <literal>services.matrix-synapse.settings</literal>. And while
+          not all options you may use are defined in there, they are
+          still supported, because you can set arbitrary values in this
+          freeform type.
+        </para>
+        <para>
+          The <literal>listeners.*.bind_address</literal> option was
+          renamed to <literal>bind_addresses</literal> in order to match
+          the upstream <literal>homeserver.yaml</literal> option name.
+          It is now also a list of strings instead of a string.
+        </para>
+        <para>
+          An example to make the required migration clearer:
+        </para>
+        <para>
+          Before:
+        </para>
+        <programlisting language="bash">
+{
+  services.matrix-synapse = {
+    enable = true;
+
+    server_name = &quot;example.com&quot;;
+    public_baseurl = &quot;https://example.com:8448&quot;;
+
+    enable_registration = false;
+    registration_shared_secret = &quot;xohshaeyui8jic7uutuDogahkee3aehuaf6ei3Xouz4iicie5thie6nohNahceut&quot;;
+    macaroon_secret_key = &quot;xoo8eder9seivukaiPh1cheikohquuw8Yooreid0The4aifahth3Ou0aiShaiz4l&quot;;
+
+    tls_certificate_path = &quot;/var/lib/acme/example.com/fullchain.pem&quot;;
+    tls_certificate_path = &quot;/var/lib/acme/example.com/fullchain.pem&quot;;
+
+    listeners = [ {
+      port = 8448;
+      bind_address = &quot;&quot;;
+      type = &quot;http&quot;;
+      tls = true;
+      resources = [ {
+        names = [ &quot;client&quot; ];
+        compress = true;
+      } {
+        names = [ &quot;federation&quot; ];
+        compress = false;
+      } ];
+    } ];
+
+  };
+}
+</programlisting>
+        <para>
+          After:
+        </para>
+        <programlisting language="bash">
+{
+  services.matrix-synapse = {
+    enable = true;
+
+    # this attribute set holds all values that go into your homeserver.yaml configuration
+    # See https://github.com/matrix-org/synapse/blob/develop/docs/sample_config.yaml for
+    # possible values.
+    settings = {
+      server_name = &quot;example.com&quot;;
+      public_baseurl = &quot;https://example.com:8448&quot;;
+
+      enable_registration = false;
+      # pass `registration_shared_secret` and `macaroon_secret_key` via `extraConfigFiles` instead
+
+      tls_certificate_path = &quot;/var/lib/acme/example.com/fullchain.pem&quot;;
+      tls_certificate_path = &quot;/var/lib/acme/example.com/fullchain.pem&quot;;
+
+      listeners = [ {
+        port = 8448;
+        bind_addresses = [
+          &quot;::&quot;
+          &quot;0.0.0.0&quot;
+        ];
+        type = &quot;http&quot;;
+        tls = true;
+        resources = [ {
+          names = [ &quot;client&quot; ];
+          compress = true;
+        } {
+          names = [ &quot;federation&quot; ];
+          compress = false;
+        } ];
+      } ];
+    };
+
+    extraConfigFiles = [
+      /run/keys/matrix-synapse/secrets.yaml
+    ];
+  };
+}
+</programlisting>
+        <para>
+          The secrets in your original config should be migrated into a
+          YAML file that is included via
+          <literal>extraConfigFiles</literal>.
+        </para>
+        <para>
+          Additionally a few option defaults have been synced up with
+          upstream default values, for example the
+          <literal>max_upload_size</literal> grew from
+          <literal>10M</literal> to <literal>50M</literal>. For the same
+          reason, the default <literal>media_store_path</literal> was
+          changed from <literal>${dataDir}/media</literal> to
+          <literal>${dataDir}/media_store</literal> if
+          <literal>system.stateVersion</literal> is at least
+          <literal>22.05</literal>. Files will need to be manually moved
+          to the new location if the <literal>stateVersion</literal> is
+          updated.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The MoinMoin wiki engine
+          (<literal>services.moinmoin</literal>) has been removed,
+          because Python 2 is being retired from nixpkgs.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Services in the <literal>hadoop</literal> module previously
+          set <literal>openFirewall</literal> to true by default. This
+          has now been changed to false. Node definitions for multi-node
+          clusters would need <literal>openFirewall = true;</literal> to
+          be added to to hadoop services when upgrading from NixOS
+          21.11.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.hadoop.yarn.nodemanager</literal> now uses
+          cgroup-based CPU limit enforcement by default. Additionally,
+          the option <literal>useCGroups</literal> was added to
+          nodemanagers as an easy way to switch back to the old
+          behavior.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
           The <literal>wafHook</literal> hook now honors
           <literal>NIX_BUILD_CORES</literal> when
           <literal>enableParallelBuilding</literal> is not set
@@ -75,11 +634,1013 @@
           release based on GTK+3 and Python 3.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The <literal>writers.writePython2</literal> and corresponding
+          <literal>writers.writePython2Bin</literal> convenience
+          functions to create executable Python 2 scripts in the store
+          were removed in preparation of removal of the Python 2
+          interpreter. Scripts have to be converted to Python 3 for use
+          with <literal>writers.writePython3</literal> or
+          <literal>writers.writePyPy2</literal> needs to be used.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>buildGoModule</literal> was updated to use
+          <literal>go_1_17</literal>, third party derivations that
+          specify &gt;= go 1.17 in the main <literal>go.mod</literal>
+          will need to regenerate their <literal>vendorSha256</literal>
+          hash.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>gnome-passwordsafe</literal> package updated to
+          <link xlink:href="https://gitlab.gnome.org/World/secrets/-/tags/6.0">version
+          6.x</link> and renamed to <literal>gnome-secrets</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          If you previously used
+          <literal>/etc/docker/daemon.json</literal>, you need to
+          incorporate the changes into the new option
+          <literal>virtualisation.docker.daemon.settings</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Ntopng (<literal>services.ntopng</literal>) is updated to
+          5.2.1 and uses a separate Redis instance if
+          <literal>system.stateVersion</literal> is at least
+          <literal>22.05</literal>. Existing setups shouldn’t be
+          affected.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The backward compatibility in
+          <literal>services.wordpress</literal> to configure sites with
+          the old interface has been removed. Please use
+          <literal>services.wordpress.sites</literal> instead.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The backward compatibility in
+          <literal>services.dokuwiki</literal> to configure sites with
+          the old interface has been removed. Please use
+          <literal>services.dokuwiki.sites</literal> instead.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          opensmtpd-extras is no longer build with python2 scripting
+          support due to python2 deprecation in nixpkgs
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.miniflux.adminCredentialFiles</literal> is
+          now required, instead of defaulting to
+          <literal>admin</literal> and <literal>password</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>autorestic</literal> package has been upgraded
+          from 1.3.0 to 1.5.0 which introduces breaking changes in
+          config file, check
+          <link xlink:href="https://autorestic.vercel.app/migration/1.4_1.5">their
+          migration guide</link> for more details.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          For <literal>pkgs.python3.pkgs.ipython</literal>, its direct
+          dependency
+          <literal>pkgs.python3.pkgs.matplotlib-inline</literal> (which
+          is really an adapter to integrate matplotlib in ipython if it
+          is installed) does not depend on
+          <literal>pkgs.python3.pkgs.matplotlib</literal> anymore. This
+          is closer to a non-Nix install of ipython. This has the added
+          benefit to reduce the closure size of
+          <literal>ipython</literal> from ~400MB to ~160MB (including
+          ~100MB for python itself).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>documentation.man</literal> has been refactored to
+          support choosing a man implementation other than GNU’s
+          <literal>man-db</literal>. For this,
+          <literal>documentation.man.manualPages</literal> has been
+          renamed to
+          <literal>documentation.man.man-db.manualPages</literal>. If
+          you want to use the new alternative man implementation
+          <literal>mandoc</literal>, add
+          <literal>documentation.man = { enable = true; man-db.enable = false; mandoc.enable = true; }</literal>
+          to your configuration.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Normal users (with <literal>isNormalUser = true</literal>)
+          which have non-empty <literal>subUidRanges</literal> or
+          <literal>subGidRanges</literal> set no longer have additional
+          implicit ranges allocated. To enable automatic allocation back
+          set <literal>autoSubUidGidRange = true</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>idris2</literal> now requires
+          <literal>--package</literal> when using packages
+          <literal>contrib</literal> and <literal>network</literal>,
+          while previously these idris2 packages were automatically
+          loaded.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The iputils package, which is installed by default, no longer
+          provides the legacy tools <literal>tftpd</literal> and
+          <literal>traceroute6</literal>. More tools
+          (<literal>ninfod</literal>, <literal>rarpd</literal>, and
+          <literal>rdisc</literal>) are going to be removed in the next
+          release. See
+          <link xlink:href="https://github.com/iputils/iputils/releases/tag/20211215">upstream’s
+          release notes</link> for more details and available
+          replacements.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.thelounge.private</literal> was removed in
+          favor of <literal>services.thelounge.public</literal>, to
+          follow with upstream changes.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>pkgs.docbookrx</literal> was removed since it’s
+          unmaintained
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>pkgs._7zz</literal> is now correctly licensed as
+          LGPL3+ and BSD3 with optional unfree unRAR licensed code
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>tilp2</literal> was removed together with its module
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The F-PROT antivirus (<literal>fprot</literal> package) and
+          its service module were removed because it reached
+          <link xlink:href="https://kb.cyren.com/av-support/index.php?/Knowledgebase/Article/View/434/0/end-of-sale--end-of-life-for-f-prot-and-csam">end-of-life</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>bird1</literal> and its modules
+          <literal>services.bird</literal> as well as
+          <literal>services.bird6</literal> have been removed. Upgrade
+          to <literal>services.bird2</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The options
+          <literal>networking.interfaces.&lt;name&gt;.ipv4.routes</literal>
+          and
+          <literal>networking.interfaces.&lt;name&gt;.ipv6.routes</literal>
+          are no longer ignored when using networkd instead of the
+          default scripted network backend by setting
+          <literal>networking.useNetworkd</literal> to
+          <literal>true</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          MultiMC has been replaced with the fork PolyMC due to upstream
+          developers being hostile to 3rd party package maintainers.
+          PolyMC removes all MultiMC branding and is aimed at providing
+          proper 3rd party packages like the one contained in Nixpkgs.
+          This change affects the data folder where game instances and
+          other save and configuration files are stored. Users with
+          existing installations should rename
+          <literal>~/.local/share/multimc</literal> to
+          <literal>~/.local/share/polymc</literal>. The main config
+          file’s path has also moved from
+          <literal>~/.local/share/multimc/multimc.cfg</literal> to
+          <literal>~/.local/share/polymc/polymc.cfg</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>systemd-nspawn@.service</literal> settings have been
+          reverted to the default systemd behaviour. User namespaces are
+          now activated by default. If you want to keep running nspawn
+          containers without user namespaces you need to set
+          <literal>systemd.nspawn.&lt;name&gt;.execConfig.PrivateUsers = false</literal>
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The Tor SOCKS proxy is now actually disabled if
+          <literal>services.tor.client.enable</literal> is set to
+          <literal>false</literal> (the default). If you are using this
+          functionality but didn’t change the setting or set it to
+          <literal>false</literal>, you now need to set it to
+          <literal>true</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The terraform 0.12 compatibility has been removed and the
+          <literal>terraform.withPlugins</literal> and
+          <literal>terraform-providers.mkProvider</literal>
+          implementations simplified. Providers now need to be stored
+          under
+          <literal>$out/libexec/terraform-providers/&lt;registry&gt;/&lt;owner&gt;/&lt;name&gt;/&lt;version&gt;/&lt;os&gt;_&lt;arch&gt;/terraform-provider-&lt;name&gt;_v&lt;version&gt;</literal>
+          (which mkProvider does).
+        </para>
+        <para>
+          This breaks back-compat so it’s not possible to mix-and-match
+          with previous versions of nixpkgs. In exchange, it now becomes
+          possible to use the providers from
+          <link xlink:href="https://github.com/numtide/nixpkgs-terraform-providers-bin">nixpkgs-terraform-providers-bin</link>
+          directly.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>dendrite</literal> package has been upgraded from
+          0.5.1 to
+          <link xlink:href="https://github.com/matrix-org/dendrite/releases/tag/v0.6.5">0.6.5</link>.
+          Instances configured with split sqlite databases, which has
+          been the default in NixOS, require merging of the federation
+          sender and signing key databases. See upstream
+          <link xlink:href="https://github.com/matrix-org/dendrite/releases/tag/v0.6.0">release
+          notes</link> on version 0.6.0 for details on database changes.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The existing <literal>pkgs.opentelemetry-collector</literal>
+          has been moved to
+          <literal>pkgs.opentelemetry-collector-contrib</literal> to
+          match the actual source being the <quote>contrib</quote>
+          edition. <literal>pkgs.opentelemetry-collector</literal> is
+          now the actual core release of opentelemetry-collector. If you
+          use the community contributions you should change the package
+          you refer to. If you don’t need them update your commands from
+          <literal>otelcontribcol</literal> to
+          <literal>otelcorecol</literal> and enjoy a 7x smaller binary.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>pkgs.pgadmin</literal> now refers to
+          <literal>pkgs.pgadmin4</literal>. If you still need pgadmin3,
+          use <literal>pkgs.pgadmin3</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>pkgs.noto-fonts-cjk</literal> is now deprecated in
+          favor of <literal>pkgs.noto-fonts-cjk-sans</literal> and
+          <literal>pkgs.noto-fonts-cjk-serif</literal> because they each
+          have different release schedules. To maintain compatibility
+          with prior releases of Nixpkgs,
+          <literal>pkgs.noto-fonts-cjk</literal> is currently an alias
+          of <literal>pkgs.noto-fonts-cjk-sans</literal> and doesn’t
+          include serif fonts.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>pkgs.epgstation</literal> has been upgraded from v1
+          to v2, resulting in incompatible changes in the database
+          scheme and configuration format.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Some top-level settings under
+          <link linkend="opt-services.epgstation.enable">services.epgstation</link>
+          is now deprecated because it was redudant due to the same
+          options being present in
+          <link linkend="opt-services.epgstation.settings">services.epgstation.settings</link>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option <literal>services.epgstation.basicAuth</literal>
+          was removed because basic authentication support was dropped
+          by upstream.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option
+          <link linkend="opt-services.epgstation.database.passwordFile">services.epgstation.database.passwordFile</link>
+          no longer has a default value. Make sure to set this option
+          explicitly before upgrading. Change the database password if
+          necessary.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The
+          <link linkend="opt-services.epgstation.settings">services.epgstation.settings</link>
+          option now expects options for <literal>config.yml</literal>
+          in EPGStation v2.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Existing data for the
+          <link linkend="opt-services.epgstation.enable">services.epgstation</link>
+          module would have to be backed up prior to the upgrade. To
+          back up exising data to
+          <literal>/tmp/epgstation.bak</literal>, run
+          <literal>sudo -u epgstation epgstation run backup /tmp/epgstation.bak</literal>.
+          To import that data after to the upgrade, run
+          <literal>sudo -u epgstation epgstation run v1migrate /tmp/epgstation.bak</literal>
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>switch-to-configuration</literal> (the script that is
+          run when running <literal>nixos-rebuild switch</literal> for
+          example) has been reworked
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              The interface that allows activation scripts to restart
+              units has been streamlined. Restarting and reloading is
+              now done by a single file
+              <literal>/run/nixos/activation-restart-list</literal> that
+              honors <literal>restartIfChanged</literal> and
+              <literal>reloadIfChanged</literal> of the units.
+            </para>
+            <itemizedlist spacing="compact">
+              <listitem>
+                <para>
+                  Preferring to reload instead of restarting can still
+                  be achieved using
+                  <literal>/run/nixos/activation-reload-list</literal>.
+                </para>
+              </listitem>
+            </itemizedlist>
+          </listitem>
+          <listitem>
+            <para>
+              The script now uses a proper ini-file parser to parse
+              systemd units. Some values are now only searched in one
+              section instead of in the entire unit. This is only
+              relevant for units that don’t use the NixOS systemd moule.
+            </para>
+            <itemizedlist spacing="compact">
+              <listitem>
+                <para>
+                  <literal>RefuseManualStop</literal>,
+                  <literal>X-OnlyManualStart</literal>,
+                  <literal>X-StopOnRemoval</literal>,
+                  <literal>X-StopOnReconfiguration</literal> are only
+                  searched in the <literal>[Unit]</literal> section
+                </para>
+              </listitem>
+              <listitem>
+                <para>
+                  <literal>X-ReloadIfChanged</literal>,
+                  <literal>X-RestartIfChanged</literal>,
+                  <literal>X-StopIfChanged</literal> are only searched
+                  in the <literal>[Service]</literal> section
+                </para>
+              </listitem>
+            </itemizedlist>
+          </listitem>
+        </itemizedlist>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>services.bookstack.cacheDir</literal> option has
+          been removed, since the cache directory is now handled by
+          systemd.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>services.bookstack.extraConfig</literal> option
+          has been replaced by
+          <literal>services.bookstack.config</literal> which implements
+          a
+          <link xlink:href="https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md">settings-style</link>
+          configuration.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>lib.assertMsg</literal> and
+          <literal>lib.assertOneOf</literal> no longer return
+          <literal>false</literal> if the passed condition is
+          <literal>false</literal>, <literal>throw</literal>ing the
+          given error message instead (which makes the resulting error
+          message less cluttered). This will not impact the behaviour of
+          code using these functions as intended, namely as top-level
+          wrapper for <literal>assert</literal> conditions.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>vpnc</literal> package has been changed to use
+          GnuTLS instead of OpenSSL by default for licensing reasons.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>pkgs.vimPlugins.onedark-nvim</literal> now refers to
+          <link xlink:href="https://github.com/navarasu/onedark.nvim">navarasu/onedark.nvim</link>
+          (formerly refers to
+          <link xlink:href="https://github.com/olimorris/onedarkpro.nvim">olimorris/onedarkpro.nvim</link>).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.pipewire.enable</literal> will default to
+          enabling the WirePlumber session manager instead of
+          pipewire-media-session. pipewire-media-session is deprecated
+          by upstream and not recommended, but can still be manually
+          enabled by setting
+          <literal>services.pipewire.media-session.enable</literal> to
+          <literal>true</literal> and
+          <literal>services.pipewire.wireplumber.enable</literal> to
+          <literal>false</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>pkgs.makeDesktopItem</literal> has been refactored to
+          provide a more idiomatic API. Specifically:
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              All valid options as of FDO Desktop Entry specification
+              version 1.4 can now be passed in as explicit arguments
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>exec</literal> can now be null, for entries that
+              are not of type Application
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>mimeType</literal> argument is renamed to
+              <literal>mimeTypes</literal> for consistency
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>mimeTypes</literal>,
+              <literal>categories</literal>,
+              <literal>implements</literal>,
+              <literal>keywords</literal>, <literal>onlyShowIn</literal>
+              and <literal>notShowIn</literal> take lists of strings
+              instead of one string with semicolon separators
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>extraDesktopEntries</literal> renamed to
+              <literal>extraConfig</literal> for consistency
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              Actions should now be provided as an attrset
+              <literal>actions</literal>, the <literal>Actions</literal>
+              line will be autogenerated.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>extraEntries</literal> is removed.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              Additional validation is added both at eval time and at
+              build time.
+            </para>
+          </listitem>
+        </itemizedlist>
+        <para>
+          See the <literal>vscode</literal> package for a more detailed
+          example.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-22.05-notable-changes">
     <title>Other Notable Changes</title>
-    <para>
-    </para>
+    <itemizedlist>
+      <listitem>
+        <para>
+          The option
+          <link linkend="opt-services.redis.servers">services.redis.servers</link>
+          was added to support per-application
+          <literal>redis-server</literal> which is more secure since
+          Redis databases are only mere key prefixes without any
+          configuration or ACL of their own. Backward-compatibility is
+          preserved by mapping old
+          <literal>services.redis.settings</literal> to
+          <literal>services.redis.servers.&quot;&quot;.settings</literal>,
+          but you are strongly encouraged to name each
+          <literal>redis-server</literal> instance after the application
+          using it, instead of keeping that nameless one. Except for the
+          nameless
+          <literal>services.redis.servers.&quot;&quot;</literal> still
+          accessible at <literal>127.0.0.1:6379</literal>, and to the
+          members of the Unix group <literal>redis</literal> through the
+          Unix socket <literal>/run/redis/redis.sock</literal>, all
+          other <literal>services.redis.servers.${serverName}</literal>
+          are only accessible by default to the members of the Unix
+          group <literal>redis-${serverName}</literal> through the Unix
+          socket <literal>/run/redis-${serverName}/redis.sock</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option
+          <link linkend="opt-virtualisation.vmVariant">virtualisation.vmVariant</link>
+          was added to allow users to make changes to the
+          <literal>nixos-rebuild build-vm</literal> configuration that
+          do not apply to their normal system.
+        </para>
+        <para>
+          The <literal>config.system.build.vm</literal> attribute now
+          always exists and defaults to the value from
+          <literal>vmVariant</literal>. Configurations that import the
+          <literal>virtualisation/qemu-vm.nix</literal> module
+          themselves will override this value, such that
+          <literal>vmVariant</literal> is not used.
+        </para>
+        <para>
+          Similarly
+          <link linkend="opt-virtualisation.vmVariantWithBootLoader">virtualisation.vmVariantWithBootloader</link>
+          was added.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The configuration portion of the <literal>nix-daemon</literal>
+          module has been reworked and exposed as
+          <link xlink:href="options.html#opt-nix-settings">nix.settings</link>:
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              Legacy options have been mapped to the corresponding
+              options under under
+              <link xlink:href="options.html#opt-nix.settings">nix.settings</link>
+              and will be deprecated when NixOS 21.11 reaches end of
+              life.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <link xlink:href="options.html#opt-nix.buildMachines.publicHostKey">nix.buildMachines.publicHostKey</link>
+              has been added.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </listitem>
+      <listitem>
+        <para>
+          The
+          <literal>writers.writePyPy2</literal>/<literal>writers.writePyPy3</literal>
+          and corresponding
+          <literal>writers.writePyPy2Bin</literal>/<literal>writers.writePyPy3Bin</literal>
+          convenience functions to create executable Python 2/3 scripts
+          using the PyPy interpreter were added.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Some improvements have been made to the
+          <literal>hadoop</literal> module:
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              A <literal>gatewayRole</literal> option has been added,
+              for deploying hadoop cluster configuration files to a node
+              that does not have any active services
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              Support for older versions of hadoop have been added to
+              the module
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              Overriding and extending site XML files has been made
+              easier
+            </para>
+          </listitem>
+        </itemizedlist>
+      </listitem>
+      <listitem>
+        <para>
+          If you are using Wayland you can choose to use the Ozone
+          Wayland support in Chrome and several Electron apps by setting
+          the environment variable <literal>NIXOS_OZONE_WL=1</literal>
+          (for example via
+          <literal>environment.sessionVariables.NIXOS_OZONE_WL = &quot;1&quot;</literal>).
+          This is not enabled by default because Ozone Wayland is still
+          under heavy development and behavior is not always flawless.
+          Furthermore, not all Electron apps use the latest Electron
+          versions.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>influxdb2</literal> package was split into
+          <literal>influxdb2-server</literal> and
+          <literal>influxdb2-cli</literal>, matching the split that took
+          place upstream. A combined <literal>influxdb2</literal>
+          package is still provided in this release for backwards
+          compatibilty, but will be removed at a later date.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>unifi</literal> package was switched from
+          <literal>unifi6</literal> to <literal>unifi7</literal>. Direct
+          downgrades from Unifi 7 to Unifi 6 are not possible and
+          require restoring from a backup made by Unifi 6.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>programs.zsh.autosuggestions.strategy</literal> now
+          takes a list of strings instead of a string.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>services.unifi.openPorts</literal> option default
+          value of <literal>true</literal> is now deprecated and will be
+          changed to <literal>false</literal> in 22.11. Configurations
+          using this default will print a warning when rebuilt.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>security.acme</literal> certificates will now
+          correctly check for CA revokation before reaching their
+          minimum age.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          Removing domains from
+          <literal>security.acme.certs._name_.extraDomainNames</literal>
+          will now correctly remove those domains during rebuild/renew.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          MariaDB is now offered in several versions, not just the
+          newest one. So if you have a need for running MariaDB 10.4 for
+          example, you can now just set
+          <literal>services.mysql.package = pkgs.mariadb_104;</literal>.
+          In general, it is recommended to run the newest version, to
+          get the newest features, while sticking with an LTS version
+          will most likely provide a more stable experience. Sometimes
+          software is also incompatible with the newest version of
+          MariaDB.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option
+          <link linkend="opt-programs.ssh.enableAskPassword">programs.ssh.enableAskPassword</link>
+          was added, decoupling the setting of
+          <literal>SSH_ASKPASS</literal> from
+          <literal>services.xserver.enable</literal>. This allows easy
+          usage in non-X11 environments, e.g. Wayland.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <link linkend="opt-programs.ssh.knownHosts">programs.ssh.knownHosts</link>
+          has gained an <literal>extraHostNames</literal> option to
+          replace <literal>hostNames</literal>.
+          <literal>hostNames</literal> is deprecated, but still
+          available for now.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>services.stubby</literal> module was converted to
+          a
+          <link xlink:href="https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md">settings-style</link>
+          configuration.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option <literal>services.duplicati.dataDir</literal> has
+          been added to allow changing the location of duplicati’s
+          files.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The options <literal>boot.extraModprobeConfig</literal> and
+          <literal>boot.blacklistedKernelModules</literal> now also take
+          effect in the initrd by copying the file
+          <literal>/etc/modprobe.d/nixos.conf</literal> into the initrd.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>nixos-generate-config</literal> now puts the dhcp
+          configuration in <literal>hardware-configuration.nix</literal>
+          instead of <literal>configuration.nix</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          ORY Kratos was updated to version 0.8.3-alpha.1.pre.0, which
+          introduces some breaking changes:
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              If you are relying on the SQLite images, update your
+              Docker Pull commands as follows:
+            </para>
+            <itemizedlist spacing="compact">
+              <listitem>
+                <para>
+                  <literal>docker pull oryd/kratos:{version}</literal>
+                </para>
+              </listitem>
+            </itemizedlist>
+          </listitem>
+          <listitem>
+            <para>
+              Additionally, all passwords now have to be at least 8
+              characters long.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              For more details, see:
+            </para>
+            <itemizedlist spacing="compact">
+              <listitem>
+                <para>
+                  <link xlink:href="https://github.com/ory/kratos/releases/tag/v0.8.1-alpha.1">Release
+                  Notes for v0.8.1-alpha-1</link>
+                </para>
+              </listitem>
+              <listitem>
+                <para>
+                  <link xlink:href="https://github.com/ory/kratos/releases/tag/v0.8.2-alpha.1">Release
+                  Notes for v0.8.2-alpha-1</link>
+                </para>
+              </listitem>
+            </itemizedlist>
+          </listitem>
+        </itemizedlist>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>fetchFromSourcehut</literal> now allows fetching
+          repositories recursively using <literal>fetchgit</literal> or
+          <literal>fetchhg</literal> if the argument
+          <literal>fetchSubmodules</literal> is set to
+          <literal>true</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>element-desktop</literal> package now has an
+          <literal>useKeytar</literal> option (defaults to
+          <literal>true</literal>), which allows disabling
+          <literal>keytar</literal> and in turn
+          <literal>libsecret</literal> usage (which binds to native
+          credential managers / keychain libraries).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option <literal>services.thelounge.plugins</literal> has
+          been added to allow installing plugins for The Lounge. Plugins
+          can be found in
+          <literal>pkgs.theLoungePlugins.plugins</literal> and
+          <literal>pkgs.theLoungePlugins.themes</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option
+          <literal>services.xserver.videoDriver = [ &quot;nvidia&quot; ];</literal>
+          will now also install
+          <link xlink:href="https://github.com/elFarto/nvidia-vaapi-driver">nvidia
+          VA-API drivers</link> by default.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>firmwareLinuxNonfree</literal> package has been
+          renamed to <literal>linux-firmware</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          It is now possible to specify wordlists to include as handy to
+          access environment variables using the
+          <literal>config.environment.wordlist</literal> configuration
+          options.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>services.mbpfan</literal> module was converted to
+          a
+          <link xlink:href="https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md">RFC
+          0042</link> configuration.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The default value for
+          <literal>programs.spacefm.settings.graphical_su</literal> got
+          unset. It previously pointed to <literal>gksu</literal> which
+          has been removed.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          A new module was added for the
+          <link xlink:href="https://starship.rs/">Starship</link> shell
+          prompt, providing the options
+          <literal>programs.starship.enable</literal> and
+          <literal>programs.starship.settings</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <link xlink:href="https://dino.im">Dino</link> XMPP client
+          was updated to 0.3, adding support for audio and video calls.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.mattermost.plugins</literal> has been added
+          to allow the declarative installation of Mattermost plugins.
+          Plugins are automatically repackaged using autoPatchelf.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>services.logrotate.enable</literal> now defaults to
+          true if any rotate path has been defined, and some paths have
+          been added by default.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>zrepl</literal> package has been updated from
+          0.4.0 to 0.5:
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              The RPC protocol version was bumped; all zrepl daemons in
+              a setup must be updated and restarted before replication
+              can resume.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              A bug involving encrypt-on-receive has been fixed. Read
+              the
+              <link xlink:href="https://zrepl.github.io/configuration/sendrecvoptions.html#job-recv-options-placeholder">zrepl
+              documentation</link> and check the output of
+              <literal>zfs get -r encryption,zrepl:placeholder PATH_TO_ROOTFS</literal>
+              on the receiver.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </listitem>
+      <listitem>
+        <para>
+          Renamed option
+          <literal>services.openssh.challengeResponseAuthentication</literal>
+          to
+          <literal>services.openssh.kbdInteractiveAuthentication</literal>.
+          Reason is that the old name has been deprecated upstream.
+          Using the old option name will still work, but produce a
+          warning.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>pomerium-cli</literal> command has been moved out
+          of the <literal>pomerium</literal> package into the
+          <literal>pomerium-cli</literal> package, following upstream’s
+          repository split. If you are using the
+          <literal>pomerium-cli</literal> command, you should now
+          install the <literal>pomerium-cli</literal> package.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The option
+          <link linkend="opt-networking.networkmanager.enableFccUnlock">services.networking.networkmanager.enableFccUnlock</link>
+          was added to support FCC unlock procedures. Since release
+          1.18.4, the ModemManager daemon no longer automatically
+          performs the FCC unlock procedure by default. See
+          <link xlink:href="https://modemmanager.org/docs/modemmanager/fcc-unlock/">the
+          docs</link> for more details.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          <literal>programs.tmux</literal> has a new option
+          <literal>plugins</literal> that accepts a list of packages
+          from the <literal>tmuxPlugins</literal> group. The specified
+          packages are added to the system and loaded by
+          <literal>tmux</literal>.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The polkit service, available at
+          <literal>security.polkit.enable</literal>, is now disabled by
+          default. It will automatically be enabled through services and
+          desktop environments as needed.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>hadoop</literal> package has added support for
+          <literal>aarch64-linux</literal> and
+          <literal>aarch64-darwin</literal> as of 3.3.1
+          (<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158613">#158613</link>).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>R</literal> package now builds again on
+          <literal>aarch64-darwin</literal>
+          (<link xlink:href="https://github.com/NixOS/nixpkgs/pull/158992">#158992</link>).
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The <literal>spark3</literal> package has been updated from
+          3.1.2 to 3.2.1
+          (<link xlink:href="https://github.com/NixOS/nixpkgs/pull/160075">#160075</link>):
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              Testing has been enabled for
+              <literal>aarch64-linux</literal> in addition to
+              <literal>x86_64-linux</literal>.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              The <literal>spark3</literal> package is now usable on
+              <literal>aarch64-darwin</literal> as a result of
+              <link xlink:href="https://github.com/NixOS/nixpkgs/pull/158613">#158613</link>
+              and
+              <link xlink:href="https://github.com/NixOS/nixpkgs/pull/158992">#158992</link>.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </listitem>
+    </itemizedlist>
   </section>
 </section>
diff --git a/nixos/doc/manual/installation/installing-pxe.section.md b/nixos/doc/manual/installation/installing-pxe.section.md
index 2016a258251f..4fbd6525f8c3 100644
--- a/nixos/doc/manual/installation/installing-pxe.section.md
+++ b/nixos/doc/manual/installation/installing-pxe.section.md
@@ -5,11 +5,11 @@ setup.
 
 These instructions assume that you have an existing PXE or iPXE
 infrastructure and simply want to add the NixOS installer as another
-option. To build the necessary files from a recent version of nixpkgs,
+option. To build the necessary files from your current version of nixpkgs,
 you can run:
 
 ```ShellSession
-nix-build -A netboot.x86_64-linux nixos/release.nix
+nix-build -A netboot.x86_64-linux '<nixpkgs/nixos/release.nix>'
 ```
 
 This will create a `result` directory containing: \* `bzImage` -- the
diff --git a/nixos/doc/manual/man-nixos-rebuild.xml b/nixos/doc/manual/man-nixos-rebuild.xml
index 0e0ea5d74b0b..ab2a5d83a089 100644
--- a/nixos/doc/manual/man-nixos-rebuild.xml
+++ b/nixos/doc/manual/man-nixos-rebuild.xml
@@ -453,7 +453,7 @@
       Allow ad-hoc remote builders for building the new system. This requires
       the user executing <command>nixos-rebuild</command> (usually root) to be
       configured as a trusted user in the Nix daemon. This can be achieved by
-      using the <literal>nix.trustedUsers</literal> NixOS option. Examples
+      using the <literal>nix.settings.trusted-users</literal> NixOS option. Examples
       values for that option are described in the <literal>Remote builds
       chapter</literal> in the Nix manual, (i.e. <command>--builders
       "ssh://bigbrother x86_64-linux"</command>). By specifying an empty string
@@ -535,12 +535,8 @@
      </para>
 
      <para>
-      If <option>--build-host</option> is not explicitly specified,
-      <option>--build-host</option> will implicitly be set to the same value as
-      <option>--target-host</option>. So, if you only specify
-      <option>--target-host</option> both building and activation will take
-      place remotely (and no build artifacts will be copied to the local
-      machine).
+      If <option>--build-host</option> is not explicitly specified, building
+      will take place locally.
      </para>
 
      <para>
diff --git a/nixos/doc/manual/release-notes/release-notes.xml b/nixos/doc/manual/release-notes/release-notes.xml
index 74ca57850ea5..216fea677757 100644
--- a/nixos/doc/manual/release-notes/release-notes.xml
+++ b/nixos/doc/manual/release-notes/release-notes.xml
@@ -8,6 +8,7 @@
   This section lists the release notes for each stable version of NixOS and
   current unstable revision.
  </para>
+ <xi:include href="../from_md/release-notes/rl-2205.section.xml" />
  <xi:include href="../from_md/release-notes/rl-2111.section.xml" />
  <xi:include href="../from_md/release-notes/rl-2105.section.xml" />
  <xi:include href="../from_md/release-notes/rl-2009.section.xml" />
diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md
index 2520d176096a..310d32cfdd72 100644
--- a/nixos/doc/manual/release-notes/rl-2111.section.md
+++ b/nixos/doc/manual/release-notes/rl-2111.section.md
@@ -8,7 +8,22 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - Nix has been updated to version 2.4, reference its [release notes](https://discourse.nixos.org/t/nix-2-4-released/15822) for more information on what has changed. The previous version of Nix, 2.3.16, remains available for the time being in the `nix_2_3` package.
 
-- `iptables` now uses `nf_tables` backend.
+- `iptables` is now using `nf_tables` under the hood, by using `iptables-nft`,
+  similar to [Debian](https://wiki.debian.org/nftables#Current_status) and
+  [Fedora](https://fedoraproject.org/wiki/Changes/iptables-nft-default).
+  This means, `ip[6]tables`, `arptables` and `ebtables` commands  will actually
+  show rules from some specific tables in the `nf_tables` kernel subsystem.
+  In case you're migrating from an older release without rebooting, there might
+  be cases where you end up with iptable rules configured both in the legacy
+  `iptables` kernel backend, as well as in the `nf_tables` backend.
+  This can lead to confusing firewall behaviour. An `iptables-save` after
+  switching will complain about "iptables-legacy tables present".
+  It's probably best to reboot after the upgrade, or manually removing all
+  legacy iptables rules (via the `iptables-legacy` package).
+
+- systemd got an `nftables` backend, and configures (networkd) rules in their
+  own `io.systemd.*` tables. Check `nft list ruleset` to see these rules, not
+  `iptables-save` (which only shows `iptables`-created rules.
 
 - PHP now defaults to PHP 8.0, updated from 7.4.
 
@@ -74,8 +89,6 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [PeerTube](https://joinpeertube.org/), developed by Framasoft, is the free and decentralized alternative to video platforms. Available at [services.peertube](options.html#opt-services.peertube.enable).
 
-- [maddy](https://maddy.email), a composable all-in-one mail server. Available as [services.maddy](options.html#opt-services.maddy.enable).
-
 - [sourcehut](https://sr.ht), a collection of tools useful for software development. Available as [services.sourcehut](options.html#opt-services.sourcehut.enable).
 
 - [ucarp](https://download.pureftpd.org/pub/ucarp/README), an userspace implementation of the Common Address Redundancy Protocol (CARP). Available as [networking.ucarp](options.html#opt-networking.ucarp.enable).
@@ -419,6 +432,11 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - `retroArchCores` has been removed. This means that using `nixpkgs.config.retroarch` to customize RetroArch cores is not supported anymore. Instead, use package overrides, for example: `retroarch.override { cores = with libretro; [ citra snes9x ]; };`. Also, `retroarchFull` derivation is available for those who want to have all RetroArch cores available.
 
+- The Linux kernel for security reasons now restricts access to BPF syscalls via `BPF_UNPRIV_DEFAULT_OFF=y`. Unprivileged access can be reenabled via the `kernel.unprivileged_bpf_disabled` sysctl knob.
+
+- `/usr` will always be included in the initial ramdisk. See the `fileSystems.<name>.neededForBoot` option.
+  If any files exist under `/usr` (which is not typical for NixOS), they will be included in the initial ramdisk, increasing its size to a possibly problematic extent.
+
 ## Other Notable Changes {#sec-release-21.11-notable-changes}
 
 
diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md
index 45ed69cf1b03..2ff5865b7355 100644
--- a/nixos/doc/manual/release-notes/rl-2205.section.md
+++ b/nixos/doc/manual/release-notes/rl-2205.section.md
@@ -6,12 +6,99 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 ## Highlights {#sec-release-22.05-highlights}
 
+- `security.acme.defaults` has been added to simplify configuring
+  settings for many certificates at once. This also opens up the
+  the option to use DNS-01 validation when using `enableACME` on
+  web server virtual hosts (e.g. `services.nginx.virtualHosts.*.enableACME`).
+
+- PHP 8.1 is now available
+
+- Mattermost has been updated to extended support release 6.3, as the previously packaged extended support release 5.37 is [reaching its end of life](https://docs.mattermost.com/upgrade/extended-support-release.html).
+  Migrations may take a while, see the [changelog](https://docs.mattermost.com/install/self-managed-changelog.html#release-v6-3-extended-support-release)
+  and [important upgrade notes](https://docs.mattermost.com/upgrade/important-upgrade-notes.html).
+
+- systemd services can now set [systemd.services.\<name\>.reloadTriggers](#opt-systemd.services) instead of `reloadIfChanged` for a more granular distinction between reloads and restarts.
+
+- [`kops`](https://kops.sigs.k8s.io) defaults to 1.22.4, which will enable [Instance Metadata Service Version 2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) and require tokens on new clusters with Kubernetes 1.22. This will increase security by default, but may break some types of workloads. See the [release notes](https://kops.sigs.k8s.io/releases/1.22-notes/) for details.
+
+- Module authors can use `mkRenamedOptionModuleWith` to automate the deprecation cycle without annoying out-of-tree module authors and their users.
+
 ## New Services {#sec-release-22.05-new-services}
 
+- [aesmd](https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw), the Intel SGX Architectural Enclave Service Manager. Available as [services.aesmd](#opt-services.aesmd.enable).
+
+- [rootless Docker](https://docs.docker.com/engine/security/rootless/), a `systemd --user` Docker service which runs without root permissions. Available as [virtualisation.docker.rootless.enable](options.html#opt-virtualisation.docker.rootless.enable).
+
+- [matrix-conduit](https://conduit.rs/), a simple, fast and reliable chat server powered by matrix. Available as [services.matrix-conduit](option.html#opt-services.matrix-conduit.enable).
+
+- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable).
+
+- [apfs](https://github.com/linux-apfs/linux-apfs-rw), a kernel module for mounting the Apple File System (APFS).
+
+- [FRRouting](https://frrouting.org/), a popular suite of Internet routing protocol daemons (BGP, BFD, OSPF, IS-IS, VVRP and others). Available as [services.frr](#opt-services.frr.babel.enable)
+
+- [heisenbridge](https://github.com/hifi/heisenbridge), a bouncer-style Matrix IRC bridge. Available as [services.heisenbridge](options.html#opt-services.heisenbridge.enable).
+
+- [snowflake-proxy](https://snowflake.torproject.org/), a system to defeat internet censorship. Available as [services.snowflake-proxy](options.html#opt-services.snowflake-proxy.enable).
+
+- [ergochat](https://ergo.chat), a modern IRC with IRCv3 features. Available as [services.ergochat](options.html#opt-services.ergochat.enable).
+
+- [PowerDNS-Admin](https://github.com/ngoduykhanh/PowerDNS-Admin), a web interface for the PowerDNS server. Available at [services.powerdns-admin](options.html#opt-services.powerdns-admin.enable).
+
+- [pgadmin4](https://github.com/postgres/pgadmin4), an admin interface for the PostgreSQL database. Available at [services.pgadmin](options.html#opt-services.pgadmin.enable).
+
+- [input-remapper](https://github.com/sezanzeb/input-remapper), an easy to use tool to change the mapping of your input device buttons. Available at [services.input-remapper](options.html#opt-services.input-remapper.enable).
+
+- [InvoicePlane](https://invoiceplane.com), web application for managing and creating invoices. Available at [services.invoiceplane](options.html#opt-services.invoiceplane.enable).
+
+- [maddy](https://maddy.email), a composable all-in-one mail server. Available as [services.maddy](options.html#opt-services.maddy.enable).
+
+- [K40-Whisperer](https://www.scorchworks.com/K40whisperer/k40whisperer.html), a program to control cheap Chinese laser cutters. Available as [programs.k40-whisperer.enable](options.html#opt-programs.k4-whisperer.enable). Users must add themselves to the `k40` group to be able to access the device.
+
+- [mtr-exporter](https://github.com/mgumz/mtr-exporter), a Prometheus exporter for mtr metrics. Available as [services.mtr-exporter](options.html#opt-services.mtr-exporter.enable).
+
+- [prometheus-pve-exporter](https://github.com/prometheus-pve/prometheus-pve-exporter), a tool that exposes information from the Proxmox VE API for use by Prometheus. Available as [services.prometheus.exporters.pve](options.html#opt-services.prometheus.exporters.pve).
+
+- [tetrd](https://tetrd.app), share your internet connection from your device to your PC and vice versa through a USB cable. Available at [services.tetrd](#opt-services.tetrd.enable).
+
+- [agate](https://github.com/mbrubeck/agate), a very simple server for the Gemini hypertext protocol. Available as [services.agate](options.html#opt-services.agate.enable).
+
+- [ArchiSteamFarm](https://github.com/JustArchiNET/ArchiSteamFarm), a C# application with primary purpose of idling Steam cards from multiple accounts simultaneously. Available as [services.archisteamfarm](options.html#opt-services.archisteamfarm.enable).
+
+- [teleport](https://goteleport.com), allows engineers and security professionals to unify access for SSH servers, Kubernetes clusters, web applications, and databases across all environments. Available at [services.teleport](#opt-services.teleport.enable).
+
+- [BaGet](https://loic-sharma.github.io/BaGet/), a lightweight NuGet and symbol server. Available at [services.baget](#opt-services.baget.enable).
+
+- [moosefs](https://moosefs.com), fault tolerant petabyte distributed file system.
+  Available as [moosefs](#opt-services.moosefs.client.enable).
+
+- [prosody-filer](https://github.com/ThomasLeister/prosody-filer), a server for handling XMPP HTTP Upload requests. Available at [services.prosody-filer](#opt-services.prosody-filer.enable).
+
+- [systembus-notify](https://github.com/rfjakob/systembus-notify), allow system level notifications to reach the users. Available as [services.systembus-notify](opt-services.systembus-notify.enable). Please keep in mind that this service should only be enabled on machines with fully trusted users, as any local user is able to DoS user sessions by spamming notifications.
+
+- [ethercalc](https://github.com/audreyt/ethercalc), an online collaborative
+  spreadsheet. Available as [services.ethercalc](options.html#opt-services.ethercalc.enable).
+
+- [nbd](https://nbd.sourceforge.io/), a Network Block Device server. Available as [services.nbd](options.html#opt-services.nbd.server.enable).
+
+- [nix-ld](https://github.com/Mic92/nix-ld), Run unpatched dynamic binaries on NixOS. Available as [programs.nix-ld](options.html#opt-programs.nix-ld.enable).
+
+- [timetagger](https://timetagger.app), an open source time-tracker with an intuitive user experience and powerful reporting. [services.timetagger](options.html#opt-services.timetagger.enable).
+
+- [rstudio-server](https://www.rstudio.com/products/rstudio/#rstudio-server), a browser-based version of the RStudio IDE for the R programming language. Available as [services.rstudio-server](options.html#opt-services.rstudio-server.enable).
+
+- [headscale](https://github.com/juanfont/headscale), an Open Source implementation of the [Tailscale](https://tailscale.io) Control Server. Available as [services.headscale](options.html#opt-services.headscale.enable)
+
+- [blocky](https://0xerr0r.github.io/blocky/), fast and lightweight DNS proxy as ad-blocker for local network with many features.
+
+- [pacemaker](https://clusterlabs.org/pacemaker/) cluster resource manager
+
+<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
+
 ## Backward Incompatibilities {#sec-release-22.05-incompatibilities}
 
 - `pkgs.ghc` now refers to `pkgs.targetPackages.haskellPackages.ghc`.
-  This *only* makes a difference if you are cross-compiling and will
+  This _only_ makes a difference if you are cross-compiling and will
   ensure that `pkgs.ghc` always runs on the host platform and compiles
   for the target platform (similar to `pkgs.gcc` for example).
   `haskellPackages.ghc` still behaves as before, running on the build
@@ -22,14 +109,478 @@ In addition to numerous new and upgraded packages, this release has the followin
   instead to ensure cross compilation keeps working (or switch to
   `haskellPackages.callPackage`).
 
+- `pkgs.ghc.withPackages` as well as `haskellPackages.ghcWithPackages` etc.
+  now needs be overridden directly, as opposed to overriding the result of
+  calling it. Additionally, the `withLLVM` parameter has been renamed to
+  `useLLVM`. So instead of `(ghc.withPackages (p: [])).override { withLLVM = true; }`,
+  one needs to use `(ghc.withPackages.override { useLLVM = true; }) (p: [])`.
+
+- The `home-assistant` module now requires users that don't want their
+  configuration to be managed declaratively to set
+  `services.home-assistant.config = null;`. This is required
+  due to the way default settings are handled with the new settings style.
+
+  Additionally the default list of `extraComponents` now includes the minimal
+  dependencies to successfully complete the [onboarding](https://www.home-assistant.io/getting-started/onboarding/)
+  procedure.
+
 - `pkgs.emacsPackages.orgPackages` is removed because org elpa is deprecated.
   The packages in the top level of `pkgs.emacsPackages`, such as org and
   org-contrib, refer to the ones in `pkgs.emacsPackages.elpaPackages` and
   `pkgs.emacsPackages.nongnuPackages` where the new versions will release.
 
+- `services.kubernetes.addons.dashboard` was removed due to it being an outdated version.
+
+- `services.kubernetes.scheduler.{port,address}` now set `--secure-port` and `--bind-address` instead of `--port` and `--address`, since the former have been deprecated and are no longer functional in kubernetes>=1.23. Ensure that you are not relying on the insecure behaviour before upgrading.
+
+- `services.k3s.enable` no longer implies `systemd.enableUnifiedCgroupHierarchy = false`, and will default to the 'systemd' cgroup driver when using `services.k3s.docker = true`.
+  This change may require a reboot to take effect, and k3s may not be able to run if the boot cgroup hierarchy does not match its configuration.
+  The previous behavior may be retained by explicitly setting `systemd.enableUnifiedCgroupHierarchy = false` in your configuration.
+
+- `fonts.fonts` no longer includes ancient bitmap fonts when both `config.services.xserver.enable` and `config.nixpkgs.config.allowUnfree` are enabled.
+  If you still want these fonts, use:
+
+  ```nix
+  {
+    fonts.fonts = [
+      pkgs.xorg.fontbhlucidatypewriter100dpi
+      pkgs.xorg.fontbhlucidatypewriter75dpi
+      pkgs.xorg.fontbh100dpi
+    ];
+  }
+  ```
+
+- The DHCP server (`services.dhcpd4`, `services.dhcpd6`) has been hardened.
+  The service is now using the systemd's `DynamicUser` mechanism to run as an unprivileged dynamically-allocated user with limited capabilities.
+  The dhcpd state files are now always stored in `/var/lib/dhcpd{4,6}` and the `services.dhcpd4.stateDir` and `service.dhcpd6.stateDir` options have been removed.
+  If you were depending on root privileges or set{uid,gid,cap} binaries in dhcpd shell hooks, you may give dhcpd more capabilities with e.g. `systemd.services.dhcpd6.serviceConfig.AmbientCapabilities`.
+
+- The `mailpile` email webclient (`services.mailpile`) has been removed due to its reliance on python2.
+
+- The `matrix-synapse` service (`services.matrix-synapse`) has been converted to use the `settings` option defined in RFC42.
+  This means that options that are part of your `homeserver.yaml` configuration, and that were specified at the top-level of the
+  module (`services.matrix-synapse`) now need to be moved into `services.matrix-synapse.settings`. And while not all options you
+  may use are defined in there, they are still supported, because you can set arbitrary values in this freeform type.
+
+  The `listeners.*.bind_address` option was renamed to `bind_addresses` in order to match the upstream `homeserver.yaml` option
+  name. It is now also a list of strings instead of a string.
+
+  An example to make the required migration clearer:
+
+  Before:
+  ```nix
+  {
+    services.matrix-synapse = {
+      enable = true;
+
+      server_name = "example.com";
+      public_baseurl = "https://example.com:8448";
+
+      enable_registration = false;
+      registration_shared_secret = "xohshaeyui8jic7uutuDogahkee3aehuaf6ei3Xouz4iicie5thie6nohNahceut";
+      macaroon_secret_key = "xoo8eder9seivukaiPh1cheikohquuw8Yooreid0The4aifahth3Ou0aiShaiz4l";
+
+      tls_certificate_path = "/var/lib/acme/example.com/fullchain.pem";
+      tls_certificate_path = "/var/lib/acme/example.com/fullchain.pem";
+
+      listeners = [ {
+        port = 8448;
+        bind_address = "";
+        type = "http";
+        tls = true;
+        resources = [ {
+          names = [ "client" ];
+          compress = true;
+        } {
+          names = [ "federation" ];
+          compress = false;
+        } ];
+      } ];
+
+    };
+  }
+  ```
+
+  After:
+  ```nix
+  {
+    services.matrix-synapse = {
+      enable = true;
+
+      # this attribute set holds all values that go into your homeserver.yaml configuration
+      # See https://github.com/matrix-org/synapse/blob/develop/docs/sample_config.yaml for
+      # possible values.
+      settings = {
+        server_name = "example.com";
+        public_baseurl = "https://example.com:8448";
+
+        enable_registration = false;
+        # pass `registration_shared_secret` and `macaroon_secret_key` via `extraConfigFiles` instead
+
+        tls_certificate_path = "/var/lib/acme/example.com/fullchain.pem";
+        tls_certificate_path = "/var/lib/acme/example.com/fullchain.pem";
+
+        listeners = [ {
+          port = 8448;
+          bind_addresses = [
+            "::"
+            "0.0.0.0"
+          ];
+          type = "http";
+          tls = true;
+          resources = [ {
+            names = [ "client" ];
+            compress = true;
+          } {
+            names = [ "federation" ];
+            compress = false;
+          } ];
+        } ];
+      };
+
+      extraConfigFiles = [
+        /run/keys/matrix-synapse/secrets.yaml
+      ];
+    };
+  }
+  ```
+
+  The secrets in your original config should be migrated into a YAML file that is included via `extraConfigFiles`.
+
+  Additionally a few option defaults have been synced up with upstream default values, for example the `max_upload_size` grew from `10M` to `50M`. For the same reason, the default
+  `media_store_path` was changed from `${dataDir}/media` to `${dataDir}/media_store` if `system.stateVersion` is at least `22.05`. Files will need to be manually moved to the new
+  location if the `stateVersion` is updated.
+
+- The MoinMoin wiki engine (`services.moinmoin`) has been removed, because Python 2 is being retired from nixpkgs.
+
+- Services in the `hadoop` module previously set `openFirewall` to true by default.
+  This has now been changed to false. Node definitions for multi-node clusters would need
+  `openFirewall = true;` to be added to to hadoop services when upgrading from NixOS 21.11.
+
+- `services.hadoop.yarn.nodemanager` now uses cgroup-based CPU limit enforcement by default.
+  Additionally, the option `useCGroups` was added to nodemanagers as an easy way to switch
+  back to the old behavior.
+
 - The `wafHook` hook now honors `NIX_BUILD_CORES` when `enableParallelBuilding` is not set explicitly. Packages can restore the old behaviour by setting `enableParallelBuilding=false`.
 
 - `pkgs.claws-mail-gtk2`, representing Claws Mail's older release version three, was removed in order to get rid of Python 2.
   Please switch to `claws-mail`, which is Claws Mail's latest release based on GTK+3 and Python 3.
 
+- The `writers.writePython2` and corresponding `writers.writePython2Bin` convenience functions to create executable Python 2 scripts in the store were removed in preparation of removal of the Python 2 interpreter.
+  Scripts have to be converted to Python 3 for use with `writers.writePython3` or `writers.writePyPy2` needs to be used.
+
+- `buildGoModule` was updated to use `go_1_17`, third party derivations that specify >= go 1.17 in the main `go.mod` will need to regenerate their `vendorSha256` hash.
+
+- The `gnome-passwordsafe` package updated to [version 6.x](https://gitlab.gnome.org/World/secrets/-/tags/6.0) and renamed to `gnome-secrets`.
+
+- If you previously used `/etc/docker/daemon.json`, you need to incorporate the changes into the new option `virtualisation.docker.daemon.settings`.
+
+- Ntopng (`services.ntopng`) is updated to 5.2.1 and uses a separate Redis instance if `system.stateVersion` is at least `22.05`. Existing setups shouldn't be affected.
+
+- The backward compatibility in `services.wordpress` to configure sites with
+  the old interface has been removed. Please use `services.wordpress.sites`
+  instead.
+
+- The backward compatibility in `services.dokuwiki` to configure sites with the
+  old interface has been removed. Please use `services.dokuwiki.sites` instead.
+
+- opensmtpd-extras is no longer build with python2 scripting support due to python2 deprecation in nixpkgs
+
+- `services.miniflux.adminCredentialFiles` is now required, instead of defaulting to `admin` and `password`.
+
+- The `autorestic` package has been upgraded from 1.3.0 to 1.5.0 which introduces breaking changes in config file, check [their migration guide](https://autorestic.vercel.app/migration/1.4_1.5) for more details.
+
+- For `pkgs.python3.pkgs.ipython`, its direct dependency `pkgs.python3.pkgs.matplotlib-inline`
+  (which is really an adapter to integrate matplotlib in ipython if it is installed) does
+  not depend on `pkgs.python3.pkgs.matplotlib` anymore.
+  This is closer to a non-Nix install of ipython.
+  This has the added benefit to reduce the closure size of `ipython` from ~400MB to ~160MB
+  (including ~100MB for python itself).
+
+- `documentation.man` has been refactored to support choosing a man implementation other than GNU's `man-db`. For this, `documentation.man.manualPages` has been renamed to `documentation.man.man-db.manualPages`. If you want to use the new alternative man implementation `mandoc`, add `documentation.man = { enable = true; man-db.enable = false; mandoc.enable = true; }` to your configuration.
+
+- Normal users (with `isNormalUser = true`) which have non-empty `subUidRanges` or `subGidRanges` set no longer have additional implicit ranges allocated. To enable automatic allocation back set `autoSubUidGidRange = true`.
+
+- `idris2` now requires `--package` when using packages `contrib` and `network`, while previously these idris2 packages were automatically loaded.
+
+- The iputils package, which is installed by default, no longer provides the
+  legacy tools `tftpd` and `traceroute6`. More tools (`ninfod`, `rarpd`, and
+  `rdisc`) are going to be removed in the next release. See
+  [upstream's release notes](https://github.com/iputils/iputils/releases/tag/20211215)
+  for more details and available replacements.
+
+- `services.thelounge.private` was removed in favor of `services.thelounge.public`, to follow with upstream changes.
+
+- `pkgs.docbookrx` was removed since it's unmaintained
+
+- `pkgs._7zz` is now correctly licensed as LGPL3+ and BSD3 with optional unfree unRAR licensed code
+
+- `tilp2` was removed together with its module
+
+- The F-PROT antivirus (`fprot` package) and its service module were removed because it
+  reached [end-of-life](https://kb.cyren.com/av-support/index.php?/Knowledgebase/Article/View/434/0/end-of-sale--end-of-life-for-f-prot-and-csam).
+
+- `bird1` and its modules `services.bird` as well as `services.bird6` have been removed. Upgrade to `services.bird2`.
+
+- The options `networking.interfaces.<name>.ipv4.routes` and `networking.interfaces.<name>.ipv6.routes` are no longer ignored when using networkd instead of the default scripted network backend by setting `networking.useNetworkd` to `true`.
+
+- MultiMC has been replaced with the fork PolyMC due to upstream developers being hostile to 3rd party package maintainers. PolyMC removes all MultiMC branding and is aimed at providing proper 3rd party packages like the one contained in Nixpkgs. This change affects the data folder where game instances and other save and configuration files are stored. Users with existing installations should rename `~/.local/share/multimc` to `~/.local/share/polymc`. The main config file's path has also moved from `~/.local/share/multimc/multimc.cfg` to `~/.local/share/polymc/polymc.cfg`.
+
+- `systemd-nspawn@.service` settings have been reverted to the default systemd behaviour. User namespaces are now activated by default. If you want to keep running nspawn containers without user namespaces you need to set `systemd.nspawn.<name>.execConfig.PrivateUsers = false`
+
+- The Tor SOCKS proxy is now actually disabled if `services.tor.client.enable` is set to `false` (the default). If you are using this functionality but didn't change the setting or set it to `false`, you now need to set it to `true`.
+
+- The terraform 0.12 compatibility has been removed and the `terraform.withPlugins` and `terraform-providers.mkProvider` implementations simplified. Providers now need to be stored under
+`$out/libexec/terraform-providers/<registry>/<owner>/<name>/<version>/<os>_<arch>/terraform-provider-<name>_v<version>` (which mkProvider does).
+
+  This breaks back-compat so it's not possible to mix-and-match with previous versions of nixpkgs. In exchange, it now becomes possible to use the providers from [nixpkgs-terraform-providers-bin](https://github.com/numtide/nixpkgs-terraform-providers-bin) directly.
+
+- The `dendrite` package has been upgraded from 0.5.1 to
+  [0.6.5](https://github.com/matrix-org/dendrite/releases/tag/v0.6.5). Instances
+  configured with split sqlite databases, which has been the default
+  in NixOS, require merging of the federation sender and signing key
+  databases. See upstream [release
+  notes](https://github.com/matrix-org/dendrite/releases/tag/v0.6.0)
+  on version 0.6.0 for details on database changes.
+
+- The existing `pkgs.opentelemetry-collector` has been moved to
+  `pkgs.opentelemetry-collector-contrib` to match the actual source being the
+  "contrib" edition. `pkgs.opentelemetry-collector` is now the actual core
+  release of opentelemetry-collector. If you use the community contributions
+  you should change the package you refer to. If you don't need them update your
+  commands from `otelcontribcol` to `otelcorecol` and enjoy a 7x smaller binary.
+
+- `pkgs.pgadmin` now refers to `pkgs.pgadmin4`.
+  If you still need pgadmin3, use `pkgs.pgadmin3`.
+
+- `pkgs.noto-fonts-cjk` is now deprecated in favor of `pkgs.noto-fonts-cjk-sans`
+  and `pkgs.noto-fonts-cjk-serif` because they each have different release
+  schedules. To maintain compatibility with prior releases of Nixpkgs,
+  `pkgs.noto-fonts-cjk` is currently an alias of `pkgs.noto-fonts-cjk-sans` and
+  doesn't include serif fonts.
+
+- `pkgs.epgstation` has been upgraded from v1 to v2, resulting in incompatible
+  changes in the database scheme and configuration format.
+
+- Some top-level settings under [services.epgstation](#opt-services.epgstation.enable)
+  is now deprecated because it was redudant due to the same options being
+  present in [services.epgstation.settings](#opt-services.epgstation.settings).
+
+- The option `services.epgstation.basicAuth` was removed because basic
+  authentication support was dropped by upstream.
+
+- The option [services.epgstation.database.passwordFile](#opt-services.epgstation.database.passwordFile)
+  no longer has a default value. Make sure to set this option explicitly before
+  upgrading. Change the database password if necessary.
+
+- The [services.epgstation.settings](#opt-services.epgstation.settings)
+  option now expects options for `config.yml` in EPGStation v2.
+
+- Existing data for the [services.epgstation](#opt-services.epgstation.enable)
+  module would have to be backed up prior to the upgrade. To back up exising
+  data to `/tmp/epgstation.bak`, run
+  `sudo -u epgstation epgstation run backup /tmp/epgstation.bak`.
+  To import that data after to the upgrade, run
+  `sudo -u epgstation epgstation run v1migrate /tmp/epgstation.bak`
+
+- `switch-to-configuration` (the script that is run when running `nixos-rebuild switch` for example) has been reworked
+    * The interface that allows activation scripts to restart units has been streamlined. Restarting and reloading is now done by a single file `/run/nixos/activation-restart-list` that honors `restartIfChanged` and `reloadIfChanged` of the units.
+        * Preferring to reload instead of restarting can still be achieved using `/run/nixos/activation-reload-list`.
+    * The script now uses a proper ini-file parser to parse systemd units. Some values are now only searched in one section instead of in the entire unit. This is only relevant for units that don't use the NixOS systemd moule.
+        * `RefuseManualStop`, `X-OnlyManualStart`, `X-StopOnRemoval`, `X-StopOnReconfiguration` are only searched in the `[Unit]` section
+        * `X-ReloadIfChanged`, `X-RestartIfChanged`, `X-StopIfChanged` are only searched in the `[Service]` section
+
+- The `services.bookstack.cacheDir` option has been removed, since the
+  cache directory is now handled by systemd.
+
+- The `services.bookstack.extraConfig` option has been replaced by
+  `services.bookstack.config` which implements a
+  [settings-style](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md)
+  configuration.
+
+- `lib.assertMsg` and `lib.assertOneOf` no longer return `false` if the passed condition is `false`, `throw`ing the given error message instead (which makes the resulting error message less cluttered). This will not impact the behaviour of code using these functions as intended, namely as top-level wrapper for `assert` conditions.
+
+- The `vpnc` package has been changed to use GnuTLS instead of OpenSSL by default for licensing reasons.
+
+- `pkgs.vimPlugins.onedark-nvim` now refers to [navarasu/onedark.nvim](https://github.com/navarasu/onedark.nvim)
+  (formerly refers to [olimorris/onedarkpro.nvim](https://github.com/olimorris/onedarkpro.nvim)).
+
+- `services.pipewire.enable` will default to enabling the WirePlumber session manager instead of pipewire-media-session.
+  pipewire-media-session is deprecated by upstream and not recommended, but can still be manually enabled by setting
+  `services.pipewire.media-session.enable` to `true` and `services.pipewire.wireplumber.enable` to `false`.
+
+- `pkgs.makeDesktopItem` has been refactored to provide a more idiomatic API. Specifically:
+  - All valid options as of FDO Desktop Entry specification version 1.4 can now be passed in as explicit arguments
+  - `exec` can now be null, for entries that are not of type Application
+  - `mimeType` argument is renamed to `mimeTypes` for consistency
+  - `mimeTypes`, `categories`, `implements`, `keywords`, `onlyShowIn` and `notShowIn` take lists of strings instead of one string with semicolon separators
+  - `extraDesktopEntries` renamed to `extraConfig` for consistency
+  - Actions should now be provided as an attrset `actions`, the `Actions` line will be autogenerated.
+  - `extraEntries` is removed.
+  - Additional validation is added both at eval time and at build time.
+
+  See the `vscode` package for a more detailed example.
+
+<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
+
 ## Other Notable Changes {#sec-release-22.05-notable-changes}
+
+- The option [services.redis.servers](#opt-services.redis.servers) was added
+  to support per-application `redis-server` which is more secure since Redis databases
+  are only mere key prefixes without any configuration or ACL of their own.
+  Backward-compatibility is preserved by mapping old `services.redis.settings`
+  to `services.redis.servers."".settings`, but you are strongly encouraged
+  to name each `redis-server` instance after the application using it,
+  instead of keeping that nameless one.
+  Except for the nameless `services.redis.servers.""`
+  still accessible at `127.0.0.1:6379`,
+  and to the members of the Unix group `redis`
+  through the Unix socket `/run/redis/redis.sock`,
+  all other `services.redis.servers.${serverName}`
+  are only accessible by default
+  to the members of the Unix group `redis-${serverName}`
+  through the Unix socket `/run/redis-${serverName}/redis.sock`.
+
+- The option [virtualisation.vmVariant](#opt-virtualisation.vmVariant) was added
+  to allow users to make changes to the `nixos-rebuild build-vm` configuration
+  that do not apply to their normal system.
+
+  The `config.system.build.vm` attribute now always exists and defaults to the
+  value from `vmVariant`. Configurations that import the `virtualisation/qemu-vm.nix`
+  module themselves will override this value, such that `vmVariant` is not used.
+
+  Similarly [virtualisation.vmVariantWithBootloader](#opt-virtualisation.vmVariantWithBootLoader) was added.
+
+- The configuration portion of the `nix-daemon` module has been reworked and exposed as [nix.settings](options.html#opt-nix-settings):
+  * Legacy options have been mapped to the corresponding options under under [nix.settings](options.html#opt-nix.settings) and will be deprecated when NixOS 21.11 reaches end of life.
+  * [nix.buildMachines.publicHostKey](options.html#opt-nix.buildMachines.publicHostKey) has been added.
+
+- The `writers.writePyPy2`/`writers.writePyPy3` and corresponding `writers.writePyPy2Bin`/`writers.writePyPy3Bin` convenience functions to create executable Python 2/3 scripts using the PyPy interpreter were added.
+
+- Some improvements have been made to the `hadoop` module:
+  - A `gatewayRole` option has been added, for deploying hadoop cluster configuration files to a node that does not have any active services
+  - Support for older versions of hadoop have been added to the module
+  - Overriding and extending site XML files has been made easier
+
+- If you are using Wayland you can choose to use the Ozone Wayland support
+  in Chrome and several Electron apps by setting the environment variable
+  `NIXOS_OZONE_WL=1` (for example via
+  `environment.sessionVariables.NIXOS_OZONE_WL = "1"`).
+  This is not enabled by default because Ozone Wayland is
+  still under heavy development and behavior is not always flawless.
+  Furthermore, not all Electron apps use the latest Electron versions.
+
+- The `influxdb2` package was split into `influxdb2-server` and
+  `influxdb2-cli`, matching the split that took place upstream. A
+  combined `influxdb2` package is still provided in this release for
+  backwards compatibilty, but will be removed at a later date.
+
+- The `unifi` package was switched from `unifi6` to `unifi7`.
+  Direct downgrades from Unifi 7 to Unifi 6 are not possible and require restoring from a backup made by Unifi 6.
+
+- `programs.zsh.autosuggestions.strategy` now takes a list of strings instead of a string.
+
+- The `services.unifi.openPorts` option default value of `true` is now deprecated and will be changed to `false` in 22.11.
+  Configurations using this default will print a warning when rebuilt.
+
+- `security.acme` certificates will now correctly check for CA
+  revokation before reaching their minimum age.
+
+- Removing domains from `security.acme.certs._name_.extraDomainNames`
+  will now correctly remove those domains during rebuild/renew.
+
+- MariaDB is now offered in several versions, not just the newest one.
+  So if you have a need for running MariaDB 10.4 for example, you can now just set `services.mysql.package = pkgs.mariadb_104;`.
+  In general, it is recommended to run the newest version, to get the newest features, while sticking with an LTS version will most likely provide a more stable experience.
+  Sometimes software is also incompatible with the newest version of MariaDB.
+
+- The option
+  [programs.ssh.enableAskPassword](#opt-programs.ssh.enableAskPassword) was
+  added, decoupling the setting of `SSH_ASKPASS` from
+  `services.xserver.enable`. This allows easy usage in non-X11 environments,
+  e.g. Wayland.
+
+- [programs.ssh.knownHosts](#opt-programs.ssh.knownHosts) has gained an `extraHostNames`
+  option to replace `hostNames`. `hostNames` is deprecated, but still available for now.
+
+- The `services.stubby` module was converted to a [settings-style](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration.
+
+- The option `services.duplicati.dataDir` has been added to allow changing the location of duplicati's files.
+
+- The options `boot.extraModprobeConfig` and `boot.blacklistedKernelModules` now also take effect in the initrd by copying the file `/etc/modprobe.d/nixos.conf` into the initrd.
+
+- `nixos-generate-config` now puts the dhcp configuration in `hardware-configuration.nix` instead of `configuration.nix`.
+
+- ORY Kratos was updated to version 0.8.3-alpha.1.pre.0, which introduces some breaking changes:
+  - If you are relying on the SQLite images, update your Docker Pull commands as follows:
+    - `docker pull oryd/kratos:{version}`
+  - Additionally, all passwords now have to be at least 8 characters long.
+  - For more details, see:
+    - [Release Notes for v0.8.1-alpha-1](https://github.com/ory/kratos/releases/tag/v0.8.1-alpha.1)
+    - [Release Notes for v0.8.2-alpha-1](https://github.com/ory/kratos/releases/tag/v0.8.2-alpha.1)
+
+- `fetchFromSourcehut` now allows fetching repositories recursively
+  using `fetchgit` or `fetchhg` if the argument `fetchSubmodules`
+  is set to `true`.
+
+- The `element-desktop` package now has an `useKeytar` option (defaults to `true`),
+  which allows disabling `keytar` and in turn `libsecret` usage
+  (which binds to native credential managers / keychain libraries).
+
+- The option `services.thelounge.plugins` has been added to allow installing plugins for The Lounge. Plugins can be found in `pkgs.theLoungePlugins.plugins` and `pkgs.theLoungePlugins.themes`.
+
+- The option `services.xserver.videoDriver = [ "nvidia" ];` will now also install [nvidia VA-API drivers](https://github.com/elFarto/nvidia-vaapi-driver) by default.
+
+- The `firmwareLinuxNonfree` package has been renamed to `linux-firmware`.
+
+- It is now possible to specify wordlists to include as handy to access environment variables using the `config.environment.wordlist` configuration options.
+
+- The `services.mbpfan` module was converted to a [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration.
+
+- The default value for `programs.spacefm.settings.graphical_su` got unset. It previously pointed to `gksu` which has been removed.
+
+- A new module was added for the [Starship](https://starship.rs/) shell prompt,
+  providing the options `programs.starship.enable` and `programs.starship.settings`.
+
+- The [Dino](https://dino.im) XMPP client was updated to 0.3, adding support for audio and video calls.
+
+- `services.mattermost.plugins` has been added to allow the declarative installation of Mattermost plugins.
+  Plugins are automatically repackaged using autoPatchelf.
+
+- `services.logrotate.enable` now defaults to true if any rotate path has
+  been defined, and some paths have been added by default.
+
+- The `zrepl` package has been updated from 0.4.0 to 0.5:
+
+  - The RPC protocol version was bumped; all zrepl daemons in a setup must be updated and restarted before replication can resume.
+  - A bug involving encrypt-on-receive has been fixed. Read the [zrepl documentation](https://zrepl.github.io/configuration/sendrecvoptions.html#job-recv-options-placeholder) and check the output of `zfs get -r encryption,zrepl:placeholder PATH_TO_ROOTFS` on the receiver.
+
+- Renamed option `services.openssh.challengeResponseAuthentication` to `services.openssh.kbdInteractiveAuthentication`.
+  Reason is that the old name has been deprecated upstream.
+  Using the old option name will still work, but produce a warning.
+
+- The `pomerium-cli` command has been moved out of the `pomerium` package into
+  the `pomerium-cli` package, following upstream's repository split. If you are
+  using the `pomerium-cli` command, you should now install the `pomerium-cli`
+  package.
+
+- The option
+  [services.networking.networkmanager.enableFccUnlock](#opt-networking.networkmanager.enableFccUnlock)
+  was added to support FCC unlock procedures. Since release 1.18.4, the ModemManager
+  daemon no longer automatically performs the FCC unlock procedure by default. See
+  [the docs](https://modemmanager.org/docs/modemmanager/fcc-unlock/) for more details.
+
+- `programs.tmux` has a new option `plugins` that accepts a list of packages from the `tmuxPlugins` group. The specified packages are added to the system and loaded by `tmux`.
+
+- The polkit service, available at `security.polkit.enable`, is now disabled by default. It will automatically be enabled through services and desktop environments as needed.
+
+- The `hadoop` package has added support for `aarch64-linux` and `aarch64-darwin` as of 3.3.1 ([#158613](https://github.com/NixOS/nixpkgs/pull/158613)).
+
+- The `R` package now builds again on `aarch64-darwin` ([#158992](https://github.com/NixOS/nixpkgs/pull/158992)).
+
+- The `spark3` package has been updated from 3.1.2 to 3.2.1 ([#160075](https://github.com/NixOS/nixpkgs/pull/160075)):
+
+  - Testing has been enabled for `aarch64-linux` in addition to `x86_64-linux`.
+  - The `spark3` package is now usable on `aarch64-darwin` as a result of [#158613](https://github.com/NixOS/nixpkgs/pull/158613) and [#158992](https://github.com/NixOS/nixpkgs/pull/158992).
+
+<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
diff --git a/nixos/lib/default.nix b/nixos/lib/default.nix
new file mode 100644
index 000000000000..2b3056e01457
--- /dev/null
+++ b/nixos/lib/default.nix
@@ -0,0 +1,33 @@
+let
+  # The warning is in a top-level let binding so it is only printed once.
+  minimalModulesWarning = warn "lib.nixos.evalModules is experimental and subject to change. See nixos/lib/default.nix" null;
+  inherit (nonExtendedLib) warn;
+  nonExtendedLib = import ../../lib;
+in
+{ # Optional. Allows an extended `lib` to be used instead of the regular Nixpkgs lib.
+  lib ? nonExtendedLib,
+
+  # Feature flags allow you to opt in to unfinished code. These may change some
+  # behavior or disable warnings.
+  featureFlags ? {},
+
+  # This file itself is rather new, so we accept unknown parameters to be forward
+  # compatible. This is generally not recommended, because typos go undetected.
+  ...
+}:
+let
+  seqIf = cond: if cond then builtins.seq else a: b: b;
+  # If cond, force `a` before returning any attr
+  seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v);
+
+  eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; };
+in
+/*
+  This attribute set appears as lib.nixos in the flake, or can be imported
+  using a binding like `nixosLib = import (nixpkgs + "/nixos/lib") { }`.
+*/
+{
+  inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal)
+    evalModules
+    ;
+}
diff --git a/nixos/lib/eval-cacheable-options.nix b/nixos/lib/eval-cacheable-options.nix
new file mode 100644
index 000000000000..c3ba2ce66375
--- /dev/null
+++ b/nixos/lib/eval-cacheable-options.nix
@@ -0,0 +1,53 @@
+{ libPath
+, pkgsLibPath
+, nixosPath
+, modules
+, stateVersion
+, release
+}:
+
+let
+  lib = import libPath;
+  modulesPath = "${nixosPath}/modules";
+  # dummy pkgs set that contains no packages, only `pkgs.lib` from the full set.
+  # not having `pkgs.lib` causes all users of `pkgs.formats` to fail.
+  pkgs = import pkgsLibPath {
+    inherit lib;
+    pkgs = null;
+  };
+  utils = import "${nixosPath}/lib/utils.nix" {
+    inherit config lib;
+    pkgs = null;
+  };
+  # this is used both as a module and as specialArgs.
+  # as a module it sets the _module special values, as specialArgs it makes `config`
+  # unusable. this causes documentation attributes depending on `config` to fail.
+  config = {
+    _module.check = false;
+    _module.args = {};
+    system.stateVersion = stateVersion;
+  };
+  eval = lib.evalModules {
+    modules = (map (m: "${modulesPath}/${m}") modules) ++ [
+      config
+    ];
+    specialArgs = {
+      inherit config pkgs utils;
+    };
+  };
+  docs = import "${nixosPath}/doc/manual" {
+    pkgs = pkgs // {
+      inherit lib;
+      # duplicate of the declaration in all-packages.nix
+      buildPackages.nixosOptionsDoc = attrs:
+        (import "${nixosPath}/lib/make-options-doc")
+          ({ inherit pkgs lib; } // attrs);
+    };
+    config = config.config;
+    options = eval.options;
+    version = release;
+    revision = "release-${release}";
+    prefix = modulesPath;
+  };
+in
+  docs.optionsNix
diff --git a/nixos/lib/eval-config-minimal.nix b/nixos/lib/eval-config-minimal.nix
new file mode 100644
index 000000000000..d45b9ffd4261
--- /dev/null
+++ b/nixos/lib/eval-config-minimal.nix
@@ -0,0 +1,49 @@
+
+# DO NOT IMPORT. Use nixpkgsFlake.lib.nixos, or import (nixpkgs + "/nixos/lib")
+{ lib }: # read -^
+
+let
+
+  /*
+    Invoke NixOS. Unlike traditional NixOS, this does not include all modules.
+    Any such modules have to be explicitly added via the `modules` parameter,
+    or imported using `imports` in a module.
+
+    A minimal module list improves NixOS evaluation performance and allows
+    modules to be independently usable, supporting new use cases.
+
+    Parameters:
+
+      modules:        A list of modules that constitute the configuration.
+
+      specialArgs:    An attribute set of module arguments. Unlike
+                      `config._module.args`, these are available for use in
+                      `imports`.
+                      `config._module.args` should be preferred when possible.
+
+    Return:
+
+      An attribute set containing `config.system.build.toplevel` among other
+      attributes. See `lib.evalModules` in the Nixpkgs library.
+
+   */
+  evalModules = {
+    prefix ? [],
+    modules ? [],
+    specialArgs ? {},
+  }:
+  # NOTE: Regular NixOS currently does use this function! Don't break it!
+  #       Ideally we don't diverge, unless we learn that we should.
+  #       In other words, only the public interface of nixos.evalModules
+  #       is experimental.
+  lib.evalModules {
+    inherit prefix modules;
+    specialArgs = {
+      modulesPath = builtins.toString ../modules;
+    } // specialArgs;
+  };
+
+in
+{
+  inherit evalModules;
+}
diff --git a/nixos/lib/eval-config.nix b/nixos/lib/eval-config.nix
index 62d09b8173bd..2daaa8a11863 100644
--- a/nixos/lib/eval-config.nix
+++ b/nixos/lib/eval-config.nix
@@ -21,6 +21,7 @@ evalConfigArgs@
 , # !!! See comment about args in lib/modules.nix
   specialArgs ? {}
 , modules
+, modulesLocation ? (builtins.unsafeGetAttrPos "modules" evalConfigArgs).file or null
 , # !!! See comment about check in lib/modules.nix
   check ? true
 , prefix ? []
@@ -33,6 +34,12 @@ let pkgs_ = pkgs;
 in
 
 let
+  evalModulesMinimal = (import ./default.nix {
+    inherit lib;
+    # Implicit use of feature is noted in implementation.
+    featureFlags.minimalModules = { };
+  }).evalModules;
+
   pkgsModule = rec {
     _file = ./eval-config.nix;
     key = _file;
@@ -68,13 +75,22 @@ let
         _module.check = lib.mkDefault check;
       };
     };
-  allUserModules = modules ++ legacyModules;
 
-  noUserModules = lib.evalModules ({
-    inherit prefix;
+  allUserModules =
+    let
+      # Add the invoking file (or specified modulesLocation) as error message location
+      # for modules that don't have their own locations; presumably inline modules.
+      locatedModules =
+        if modulesLocation == null then
+          modules
+        else
+          map (lib.setDefaultModuleLocation modulesLocation) modules;
+    in
+      locatedModules ++ legacyModules;
+
+  noUserModules = evalModulesMinimal ({
+    inherit prefix specialArgs;
     modules = baseModules ++ extraModules ++ [ pkgsModule modulesModule ];
-    specialArgs =
-      { modulesPath = builtins.toString ../modules; } // specialArgs;
   });
 
   # Extra arguments that are useful for constructing a similar configuration.
@@ -88,13 +104,8 @@ let
 
   nixosWithUserModules = noUserModules.extendModules { modules = allUserModules; };
 
-in withWarnings {
-
-  # Merge the option definitions in all modules, forming the full
-  # system configuration.
-  inherit (nixosWithUserModules) config options _module type;
-
+in
+withWarnings nixosWithUserModules // {
   inherit extraArgs;
-
   inherit (nixosWithUserModules._module.args) pkgs;
 }
diff --git a/nixos/lib/make-iso9660-image.sh b/nixos/lib/make-iso9660-image.sh
index 4740b05f9557..9273b8d3db8d 100644
--- a/nixos/lib/make-iso9660-image.sh
+++ b/nixos/lib/make-iso9660-image.sh
@@ -105,6 +105,7 @@ mkdir -p $out/iso
 # version-5 UUID's work)
 xorriso="xorriso
  -boot_image any gpt_disk_guid=$(uuid -v 5 daed2280-b91e-42c0-aed6-82c825ca41f3 $out | tr -d -)
+ -volume_date all_file_dates =$SOURCE_DATE_EPOCH
  -as mkisofs
  -iso-level 3
  -volid ${volumeID}
diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix
index 493006c92e7f..57652dd5db1e 100644
--- a/nixos/lib/make-options-doc/default.nix
+++ b/nixos/lib/make-options-doc/default.nix
@@ -21,6 +21,13 @@
 , options
 , transformOptions ? lib.id  # function for additional tranformations of the options
 , revision ? "" # Specify revision for the options
+# a set of options the docs we are generating will be merged into, as if by recursiveUpdate.
+# used to split the options doc build into a static part (nixos/modules) and a dynamic part
+# (non-nixos modules imported via configuration.nix, other module sources).
+, baseOptionsJSON ? null
+# instead of printing warnings for eg options with missing descriptions (which may be lost
+# by nix build unless -L is given), emit errors instead and fail the build
+, warningsAreErrors ? true
 }:
 
 let
@@ -51,10 +58,15 @@ let
   # ../../../lib/options.nix influences.
   #
   # Each element of `relatedPackages` can be either
-  # - a string:  that will be interpreted as an attribute name from `pkgs`,
-  # - a list:    that will be interpreted as an attribute path from `pkgs`,
-  # - an attrset: that can specify `name`, `path`, `package`, `comment`
+  # - a string:  that will be interpreted as an attribute name from `pkgs` and turned into a link
+  #              to search.nixos.org,
+  # - a list:    that will be interpreted as an attribute path from `pkgs` and turned into a link
+  #              to search.nixos.org,
+  # - an attrset: that can specify `name`, `path`, `comment`
   #   (either of `name`, `path` is required, the rest are optional).
+  #
+  # NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists.
+  # Such checks are not compatible with option docs caching.
   genRelatedPackages = packages: optName:
     let
       unpack = p: if lib.isString p then { name = p; }
@@ -64,16 +76,16 @@ let
         let
           title = args.title or null;
           name = args.name or (lib.concatStringsSep "." args.path);
-          path = args.path or [ args.name ];
-          package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}' found while evaluating `relatedPackages' of option `${optName}'") pkgs);
-        in "<listitem>"
-        + "<para><literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name} (${package.meta.name})</literal>"
-        + lib.optionalString (!package.meta.available) " <emphasis>[UNAVAILABLE]</emphasis>"
-        + ": ${package.meta.description or "???"}.</para>"
-        + lib.optionalString (args ? comment) "\n<para>${args.comment}</para>"
-        # Lots of `longDescription's break DocBook, so we just wrap them into <programlisting>
-        + lib.optionalString (package.meta ? longDescription) "\n<programlisting>${package.meta.longDescription}</programlisting>"
-        + "</listitem>";
+        in ''
+          <listitem>
+            <para>
+              <link xlink:href="https://search.nixos.org/packages?show=${name}&amp;sort=relevance&amp;query=${name}">
+                <literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name}</literal>
+              </link>
+            </para>
+            ${lib.optionalString (args ? comment) "<para>${args.comment}</para>"}
+          </listitem>
+        '';
     in "<itemizedlist>${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}</itemizedlist>";
 
   # Remove invisible and internal options.
@@ -99,13 +111,24 @@ in rec {
   optionsJSON = pkgs.runCommand "options.json"
     { meta.description = "List of NixOS options in JSON format";
       buildInputs = [ pkgs.brotli ];
+      options = builtins.toFile "options.json"
+        (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix));
     }
     ''
       # Export list of options in different format.
       dst=$out/share/doc/nixos
       mkdir -p $dst
 
-      cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix))} $dst/options.json
+      ${
+        if baseOptionsJSON == null
+          then "cp $options $dst/options.json"
+          else ''
+            ${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \
+              ${lib.optionalString warningsAreErrors "--warnings-are-errors"} \
+              ${baseOptionsJSON} $options \
+              > $dst/options.json
+          ''
+      }
 
       brotli -9 < $dst/options.json > $dst/options.json.br
 
@@ -118,8 +141,9 @@ in rec {
   # The actual generation of the xml file is done in nix purely for the convenience
   # of not having to generate the xml some other way
   optionsXML = pkgs.runCommand "options.xml" {} ''
+    export NIX_STORE_DIR=$TMPDIR/store
+    export NIX_STATE_DIR=$TMPDIR/state
     ${pkgs.nix}/bin/nix-instantiate \
-      --store dummy:// \
       --eval --xml --strict ${./optionsJSONtoXML.nix} \
       --argstr file ${optionsJSON}/share/doc/nixos/options.json \
       > "$out"
diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py
new file mode 100644
index 000000000000..8e2ea322dc89
--- /dev/null
+++ b/nixos/lib/make-options-doc/mergeJSON.py
@@ -0,0 +1,93 @@
+import collections
+import json
+import sys
+from typing import Any, Dict, List
+
+JSON = Dict[str, Any]
+
+class Key:
+    def __init__(self, path: List[str]):
+        self.path = path
+    def __hash__(self):
+        result = 0
+        for id in self.path:
+            result ^= hash(id)
+        return result
+    def __eq__(self, other):
+        return type(self) is type(other) and self.path == other.path
+
+Option = collections.namedtuple('Option', ['name', 'value'])
+
+# pivot a dict of options keyed by their display name to a dict keyed by their path
+def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]:
+    result: Dict[Key, Option] = dict()
+    for (name, opt) in options.items():
+        result[Key(opt['loc'])] = Option(name, opt)
+    return result
+
+# pivot back to indexed-by-full-name
+# like the docbook build we'll just fail if multiple options with differing locs
+# render to the same option name.
+def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]:
+    result: Dict[str, Dict] = dict()
+    for (key, opt) in options.items():
+        if opt.name in result:
+            raise RuntimeError(
+                'multiple options with colliding ids found',
+                opt.name,
+                result[opt.name]['loc'],
+                opt.value['loc'],
+            )
+        result[opt.name] = opt.value
+    return result
+
+warningsAreErrors = sys.argv[1] == "--warnings-are-errors"
+optOffset = 1 if warningsAreErrors else 0
+options = pivot(json.load(open(sys.argv[1 + optOffset], 'r')))
+overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r')))
+
+# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir
+for (k, v) in options.items():
+    v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}', v.value['declarations']))
+
+# merge both descriptions
+for (k, v) in overrides.items():
+    cur = options.setdefault(k, v).value
+    for (ok, ov) in v.value.items():
+        if ok == 'declarations':
+            decls = cur[ok]
+            for d in ov:
+                if d not in decls:
+                    decls += [d]
+        elif ok == "type":
+            # ignore types of placeholder options
+            if ov != "_unspecified" or cur[ok] == "_unspecified":
+                cur[ok] = ov
+        elif ov is not None or cur.get(ok, None) is None:
+            cur[ok] = ov
+
+severity = "error" if warningsAreErrors else "warning"
+
+# check that every option has a description
+hasWarnings = False
+for (k, v) in options.items():
+    if v.value.get('description', None) is None:
+        hasWarnings = True
+        print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr)
+        v.value['description'] = "This option has no description."
+    if v.value.get('type', "unspecified") == "unspecified":
+        hasWarnings = True
+        print(
+            f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " +
+            "https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr)
+
+if hasWarnings and warningsAreErrors:
+    print(
+        "\x1b[1;31m" +
+        "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " +
+        "to false to ignore these warnings." +
+        "\x1b[0m",
+        file=sys.stderr)
+    sys.exit(1)
+
+json.dump(unpivot(options), fp=sys.stdout)
diff --git a/nixos/lib/make-options-doc/options-to-docbook.xsl b/nixos/lib/make-options-doc/options-to-docbook.xsl
index b9ac26450514..b286f7b5e2c0 100644
--- a/nixos/lib/make-options-doc/options-to-docbook.xsl
+++ b/nixos/lib/make-options-doc/options-to-docbook.xsl
@@ -20,7 +20,7 @@
       <title>Configuration Options</title>
       <variablelist xml:id="configuration-variable-list">
         <xsl:for-each select="attrs">
-          <xsl:variable name="id" select="concat('opt-', str:replace(str:replace(str:replace(attr[@name = 'name']/string/@value, '*', '_'), '&lt;', '_'), '>', '_'))" />
+          <xsl:variable name="id" select="concat('opt-', str:replace(str:replace(str:replace(str:replace(attr[@name = 'name']/string/@value, '*', '_'), '&lt;', '_'), '>', '_'), ':', '_'))" />
           <varlistentry>
             <term xlink:href="#{$id}">
               <xsl:attribute name="xml:id"><xsl:value-of select="$id"/></xsl:attribute>
diff --git a/nixos/lib/qemu-common.nix b/nixos/lib/qemu-common.nix
index f3af85040bd6..20bbe9ff5d99 100644
--- a/nixos/lib/qemu-common.nix
+++ b/nixos/lib/qemu-common.nix
@@ -17,12 +17,12 @@ rec {
       ''-netdev vde,id=vlan${toString nic},sock="$QEMU_VDE_SOCKET_${toString net}"''
     ];
 
-  qemuSerialDevice = if pkgs.stdenv.hostPlatform.isx86 then "ttyS0"
+  qemuSerialDevice = if pkgs.stdenv.hostPlatform.isx86 || pkgs.stdenv.hostPlatform.isRiscV then "ttyS0"
         else if (with pkgs.stdenv.hostPlatform; isAarch32 || isAarch64 || isPower) then "ttyAMA0"
         else throw "Unknown QEMU serial device for system '${pkgs.stdenv.hostPlatform.system}'";
 
   qemuBinary = qemuPkg: {
-    x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu qemu64";
+    x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu max";
     armv7l-linux = "${qemuPkg}/bin/qemu-system-arm -enable-kvm -machine virt -cpu host";
     aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -enable-kvm -machine virt,gic-version=host -cpu host";
     powerpc64le-linux = "${qemuPkg}/bin/qemu-system-ppc64 -machine powernv";
diff --git a/nixos/lib/systemd-lib.nix b/nixos/lib/systemd-lib.nix
index 6c4d27018eed..a472d97f5cc7 100644
--- a/nixos/lib/systemd-lib.nix
+++ b/nixos/lib/systemd-lib.nix
@@ -5,12 +5,16 @@ with lib;
 let
   cfg = config.systemd;
   lndir = "${pkgs.buildPackages.xorg.lndir}/bin/lndir";
+  systemd = cfg.package;
 in rec {
 
   shellEscape = s: (replaceChars [ "\\" ] [ "\\\\" ] s);
 
   mkPathSafeName = lib.replaceChars ["@" ":" "\\" "[" "]"] ["-" "-" "-" "" ""];
 
+  # a type for options that take a unit name
+  unitNameType = types.strMatching "[a-zA-Z0-9@%:_.\\-]+[.](service|socket|device|mount|automount|swap|target|path|timer|scope|slice)";
+
   makeUnit = name: unit:
     if unit.enable then
       pkgs.runCommand "unit-${mkPathSafeName name}"
@@ -228,10 +232,209 @@ in rec {
         mkdir -p $out/getty.target.wants/
         ln -s ../autovt@tty1.service $out/getty.target.wants/
 
-        ln -s ../local-fs.target ../remote-fs.target \
-        ../nss-lookup.target ../nss-user-lookup.target ../swap.target \
-        $out/multi-user.target.wants/
+        ln -s ../remote-fs.target $out/multi-user.target.wants/
       ''}
     ''; # */
 
+  makeJobScript = name: text:
+    let
+      scriptName = replaceChars [ "\\" "@" ] [ "-" "_" ] (shellEscape name);
+      out = (pkgs.writeShellScriptBin scriptName ''
+        set -e
+        ${text}
+      '').overrideAttrs (_: {
+        # The derivation name is different from the script file name
+        # to keep the script file name short to avoid cluttering logs.
+        name = "unit-script-${scriptName}";
+      });
+    in "${out}/bin/${scriptName}";
+
+  unitConfig = { config, options, ... }: {
+    config = {
+      unitConfig =
+        optionalAttrs (config.requires != [])
+          { Requires = toString config.requires; }
+        // optionalAttrs (config.wants != [])
+          { Wants = toString config.wants; }
+        // optionalAttrs (config.after != [])
+          { After = toString config.after; }
+        // optionalAttrs (config.before != [])
+          { Before = toString config.before; }
+        // optionalAttrs (config.bindsTo != [])
+          { BindsTo = toString config.bindsTo; }
+        // optionalAttrs (config.partOf != [])
+          { PartOf = toString config.partOf; }
+        // optionalAttrs (config.conflicts != [])
+          { Conflicts = toString config.conflicts; }
+        // optionalAttrs (config.requisite != [])
+          { Requisite = toString config.requisite; }
+        // optionalAttrs (config.restartTriggers != [])
+          { X-Restart-Triggers = toString config.restartTriggers; }
+        // optionalAttrs (config.reloadTriggers != [])
+          { X-Reload-Triggers = toString config.reloadTriggers; }
+        // optionalAttrs (config.description != "") {
+          Description = config.description; }
+        // optionalAttrs (config.documentation != []) {
+          Documentation = toString config.documentation; }
+        // optionalAttrs (config.onFailure != []) {
+          OnFailure = toString config.onFailure; }
+        // optionalAttrs (options.startLimitIntervalSec.isDefined) {
+          StartLimitIntervalSec = toString config.startLimitIntervalSec;
+        } // optionalAttrs (options.startLimitBurst.isDefined) {
+          StartLimitBurst = toString config.startLimitBurst;
+        };
+    };
+  };
+
+  serviceConfig = { name, config, ... }: {
+    config = mkMerge
+      [ { # Default path for systemd services.  Should be quite minimal.
+          path = mkAfter
+            [ pkgs.coreutils
+              pkgs.findutils
+              pkgs.gnugrep
+              pkgs.gnused
+              systemd
+            ];
+          environment.PATH = "${makeBinPath config.path}:${makeSearchPathOutput "bin" "sbin" config.path}";
+        }
+        (mkIf (config.preStart != "")
+          { serviceConfig.ExecStartPre =
+              [ (makeJobScript "${name}-pre-start" config.preStart) ];
+          })
+        (mkIf (config.script != "")
+          { serviceConfig.ExecStart =
+              makeJobScript "${name}-start" config.script + " " + config.scriptArgs;
+          })
+        (mkIf (config.postStart != "")
+          { serviceConfig.ExecStartPost =
+              [ (makeJobScript "${name}-post-start" config.postStart) ];
+          })
+        (mkIf (config.reload != "")
+          { serviceConfig.ExecReload =
+              makeJobScript "${name}-reload" config.reload;
+          })
+        (mkIf (config.preStop != "")
+          { serviceConfig.ExecStop =
+              makeJobScript "${name}-pre-stop" config.preStop;
+          })
+        (mkIf (config.postStop != "")
+          { serviceConfig.ExecStopPost =
+              makeJobScript "${name}-post-stop" config.postStop;
+          })
+      ];
+  };
+
+  mountConfig = { config, ... }: {
+    config = {
+      mountConfig =
+        { What = config.what;
+          Where = config.where;
+        } // optionalAttrs (config.type != "") {
+          Type = config.type;
+        } // optionalAttrs (config.options != "") {
+          Options = config.options;
+        };
+    };
+  };
+
+  automountConfig = { config, ... }: {
+    config = {
+      automountConfig =
+        { Where = config.where;
+        };
+    };
+  };
+
+  commonUnitText = def: ''
+      [Unit]
+      ${attrsToSection def.unitConfig}
+    '';
+
+  targetToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text =
+        ''
+          [Unit]
+          ${attrsToSection def.unitConfig}
+        '';
+    };
+
+  serviceToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text = commonUnitText def +
+        ''
+          [Service]
+          ${let env = cfg.globalEnvironment // def.environment;
+            in concatMapStrings (n:
+              let s = optionalString (env.${n} != null)
+                "Environment=${builtins.toJSON "${n}=${env.${n}}"}\n";
+              # systemd max line length is now 1MiB
+              # https://github.com/systemd/systemd/commit/e6dde451a51dc5aaa7f4d98d39b8fe735f73d2af
+              in if stringLength s >= 1048576 then throw "The value of the environment variable ‘${n}’ in systemd service ‘${name}.service’ is too long." else s) (attrNames env)}
+          ${if def.reloadIfChanged then ''
+            X-ReloadIfChanged=true
+          '' else if !def.restartIfChanged then ''
+            X-RestartIfChanged=false
+          '' else ""}
+          ${optionalString (!def.stopIfChanged) "X-StopIfChanged=false"}
+          ${attrsToSection def.serviceConfig}
+        '';
+    };
+
+  socketToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text = commonUnitText def +
+        ''
+          [Socket]
+          ${attrsToSection def.socketConfig}
+          ${concatStringsSep "\n" (map (s: "ListenStream=${s}") def.listenStreams)}
+          ${concatStringsSep "\n" (map (s: "ListenDatagram=${s}") def.listenDatagrams)}
+        '';
+    };
+
+  timerToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text = commonUnitText def +
+        ''
+          [Timer]
+          ${attrsToSection def.timerConfig}
+        '';
+    };
+
+  pathToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text = commonUnitText def +
+        ''
+          [Path]
+          ${attrsToSection def.pathConfig}
+        '';
+    };
+
+  mountToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text = commonUnitText def +
+        ''
+          [Mount]
+          ${attrsToSection def.mountConfig}
+        '';
+    };
+
+  automountToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text = commonUnitText def +
+        ''
+          [Automount]
+          ${attrsToSection def.automountConfig}
+        '';
+    };
+
+  sliceToUnit = name: def:
+    { inherit (def) aliases wantedBy requiredBy enable;
+      text = commonUnitText def +
+        ''
+          [Slice]
+          ${attrsToSection def.sliceConfig}
+        '';
+    };
 }
diff --git a/nixos/lib/systemd-unit-options.nix b/nixos/lib/systemd-unit-options.nix
index 01f954a4d3e0..8029ba0e3f6c 100644
--- a/nixos/lib/systemd-unit-options.nix
+++ b/nixos/lib/systemd-unit-options.nix
@@ -45,7 +45,7 @@ in rec {
 
     requiredBy = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         Units that require (i.e. depend on and need to go down with)
         this unit. The discussion under <literal>wantedBy</literal>
@@ -56,7 +56,7 @@ in rec {
 
     wantedBy = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         Units that want (i.e. depend on) this unit. The standard way
         to make a unit start by default at boot is to set this option
@@ -73,7 +73,7 @@ in rec {
 
     aliases = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = "Aliases of that unit.";
     };
 
@@ -98,7 +98,7 @@ in rec {
 
     description = mkOption {
       default = "";
-      type = types.str;
+      type = types.singleLineStr;
       description = "Description of this unit used in systemd messages and progress indicators.";
     };
 
@@ -110,7 +110,7 @@ in rec {
 
     requires = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         Start the specified units when this unit is started, and stop
         this unit when the specified units are stopped or fail.
@@ -119,7 +119,7 @@ in rec {
 
     wants = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         Start the specified units when this unit is started.
       '';
@@ -127,7 +127,7 @@ in rec {
 
     after = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         If the specified units are started at the same time as
         this unit, delay this unit until they have started.
@@ -136,7 +136,7 @@ in rec {
 
     before = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         If the specified units are started at the same time as
         this unit, delay them until this unit has started.
@@ -145,7 +145,7 @@ in rec {
 
     bindsTo = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         Like ‘requires’, but in addition, if the specified units
         unexpectedly disappear, this unit will be stopped as well.
@@ -154,7 +154,7 @@ in rec {
 
     partOf = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         If the specified units are stopped or restarted, then this
         unit is stopped or restarted as well.
@@ -163,7 +163,7 @@ in rec {
 
     conflicts = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         If the specified units are started, then this unit is stopped
         and vice versa.
@@ -172,7 +172,7 @@ in rec {
 
     requisite = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         Similar to requires. However if the units listed are not started,
         they will not be started and the transaction will fail.
@@ -201,9 +201,20 @@ in rec {
       '';
     };
 
+    reloadTriggers = mkOption {
+      default = [];
+      type = types.listOf unitOption;
+      description = ''
+        An arbitrary list of items such as derivations.  If any item
+        in the list changes between reconfigurations, the service will
+        be reloaded.  If anything but a reload trigger changes in the
+        unit file, the unit will be restarted instead.
+      '';
+    };
+
     onFailure = mkOption {
       default = [];
-      type = types.listOf types.str;
+      type = types.listOf unitNameType;
       description = ''
         A list of one or more units that are activated when
         this unit enters the "failed" state.
@@ -338,6 +349,11 @@ in rec {
         configuration switch if its definition has changed.  If
         enabled, the value of <option>restartIfChanged</option> is
         ignored.
+
+        This option should not be used anymore in favor of
+        <option>reloadTriggers</option> which allows more granular
+        control of when a service is reloaded and when a service
+        is restarted.
       '';
     };
 
diff --git a/nixos/lib/test-driver/default.nix b/nixos/lib/test-driver/default.nix
index 3f63bc705b90..3aee91343189 100644
--- a/nixos/lib/test-driver/default.nix
+++ b/nixos/lib/test-driver/default.nix
@@ -14,7 +14,7 @@
 
 python3Packages.buildPythonApplication rec {
   pname = "nixos-test-driver";
-  version = "1.0";
+  version = "1.1";
   src = ./.;
 
   propagatedBuildInputs = [ coreutils netpbm python3Packages.colorama python3Packages.ptpython qemu_pkg socat vde2 ]
@@ -26,7 +26,7 @@ python3Packages.buildPythonApplication rec {
     mypy --disallow-untyped-defs \
           --no-implicit-optional \
           --ignore-missing-imports ${src}/test_driver
-    pylint --errors-only ${src}/test_driver
+    pylint --errors-only --enable=unused-import ${src}/test_driver
     black --check --diff ${src}/test_driver
   '';
 }
diff --git a/nixos/lib/test-driver/setup.py b/nixos/lib/test-driver/setup.py
index 156995472169..476c7b2dab2a 100644
--- a/nixos/lib/test-driver/setup.py
+++ b/nixos/lib/test-driver/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
   name="nixos-test-driver",
-  version='1.0',
+  version='1.1',
   packages=find_packages(),
   entry_points={
     "console_scripts": [
diff --git a/nixos/lib/test-driver/test_driver/__init__.py b/nixos/lib/test-driver/test_driver/__init__.py
index 5477ab5cd038..61d91c9ed654 100755
--- a/nixos/lib/test-driver/test_driver/__init__.py
+++ b/nixos/lib/test-driver/test_driver/__init__.py
@@ -33,6 +33,22 @@ class EnvDefault(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
+def writeable_dir(arg: str) -> Path:
+    """Raises an ArgumentTypeError if the given argument isn't a writeable directory
+    Note: We want to fail as early as possible if a directory isn't writeable,
+    since an executed nixos-test could fail (very late) because of the test-driver
+    writing in a directory without proper permissions.
+    """
+    path = Path(arg)
+    if not path.is_dir():
+        raise argparse.ArgumentTypeError("{0} is not a directory".format(path))
+    if not os.access(path, os.W_OK):
+        raise argparse.ArgumentTypeError(
+            "{0} is not a writeable directory".format(path)
+        )
+    return path
+
+
 def main() -> None:
     arg_parser = argparse.ArgumentParser(prog="nixos-test-driver")
     arg_parser.add_argument(
@@ -45,7 +61,7 @@ def main() -> None:
         "-I",
         "--interactive",
         help="drop into a python repl and run the tests interactively",
-        action="store_true",
+        action=argparse.BooleanOptionalAction,
     )
     arg_parser.add_argument(
         "--start-scripts",
@@ -64,6 +80,14 @@ def main() -> None:
         help="vlans to span by the driver",
     )
     arg_parser.add_argument(
+        "-o",
+        "--output_directory",
+        help="""The path to the directory where outputs copied from the VM will be placed.
+                By e.g. Machine.copy_from_vm or Machine.screenshot""",
+        default=Path.cwd(),
+        type=writeable_dir,
+    )
+    arg_parser.add_argument(
         "testscript",
         action=EnvDefault,
         envvar="testScript",
@@ -77,7 +101,11 @@ def main() -> None:
         rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state")
 
     with Driver(
-        args.start_scripts, args.vlans, args.testscript.read_text(), args.keep_vm_state
+        args.start_scripts,
+        args.vlans,
+        args.testscript.read_text(),
+        args.output_directory.resolve(),
+        args.keep_vm_state,
     ) as driver:
         if args.interactive:
             ptpython.repl.embed(driver.test_symbols(), {})
@@ -94,7 +122,7 @@ def generate_driver_symbols() -> None:
     in user's test scripts. That list is then used by pyflakes to lint those
     scripts.
     """
-    d = Driver([], [], "")
+    d = Driver([], [], "", Path())
     test_symbols = d.test_symbols()
     with open("driver-symbols", "w") as fp:
         fp.write(",".join(test_symbols.keys()))
diff --git a/nixos/lib/test-driver/test_driver/driver.py b/nixos/lib/test-driver/test_driver/driver.py
index f3af98537ad6..880b1c5fdec0 100644
--- a/nixos/lib/test-driver/test_driver/driver.py
+++ b/nixos/lib/test-driver/test_driver/driver.py
@@ -1,12 +1,35 @@
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, Iterator, List
+from typing import Any, Dict, Iterator, List, Union, Optional, Callable, ContextManager
 import os
 import tempfile
 
 from test_driver.logger import rootlog
 from test_driver.machine import Machine, NixStartScript, retry
 from test_driver.vlan import VLan
+from test_driver.polling_condition import PollingCondition
+
+
+def get_tmp_dir() -> Path:
+    """Returns a temporary directory that is defined by TMPDIR, TEMP, TMP or CWD
+    Raises an exception in case the retrieved temporary directory is not writeable
+    See https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir
+    """
+    tmp_dir = Path(tempfile.gettempdir())
+    tmp_dir.mkdir(mode=0o700, exist_ok=True)
+    if not tmp_dir.is_dir():
+        raise NotADirectoryError(
+            "The directory defined by TMPDIR, TEMP, TMP or CWD: {0} is not a directory".format(
+                tmp_dir
+            )
+        )
+    if not os.access(tmp_dir, os.W_OK):
+        raise PermissionError(
+            "The directory defined by TMPDIR, TEMP, TMP, or CWD: {0} is not writeable".format(
+                tmp_dir
+            )
+        )
+    return tmp_dir
 
 
 class Driver:
@@ -16,18 +39,20 @@ class Driver:
     tests: str
     vlans: List[VLan]
     machines: List[Machine]
+    polling_conditions: List[PollingCondition]
 
     def __init__(
         self,
         start_scripts: List[str],
         vlans: List[int],
         tests: str,
+        out_dir: Path,
         keep_vm_state: bool = False,
     ):
         self.tests = tests
+        self.out_dir = out_dir
 
-        tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
-        tmp_dir.mkdir(mode=0o700, exist_ok=True)
+        tmp_dir = get_tmp_dir()
 
         with rootlog.nested("start all VLans"):
             self.vlans = [VLan(nr, tmp_dir) for nr in vlans]
@@ -36,12 +61,16 @@ class Driver:
             for s in scripts:
                 yield NixStartScript(s)
 
+        self.polling_conditions = []
+
         self.machines = [
             Machine(
                 start_command=cmd,
                 keep_vm_state=keep_vm_state,
                 name=cmd.machine_name,
                 tmp_dir=tmp_dir,
+                callbacks=[self.check_polling_conditions],
+                out_dir=self.out_dir,
             )
             for cmd in cmd(start_scripts)
         ]
@@ -84,6 +113,7 @@ class Driver:
             retry=retry,
             serial_stdout_off=self.serial_stdout_off,
             serial_stdout_on=self.serial_stdout_on,
+            polling_condition=self.polling_condition,
             Machine=Machine,  # for typing
         )
         machine_symbols = {m.name: m for m in self.machines}
@@ -135,8 +165,8 @@ class Driver:
             "Using legacy create_machine(), please instantiate the"
             "Machine class directly, instead"
         )
-        tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
-        tmp_dir.mkdir(mode=0o700, exist_ok=True)
+
+        tmp_dir = get_tmp_dir()
 
         if args.get("startCommand"):
             start_command: str = args.get("startCommand", "")
@@ -148,6 +178,7 @@ class Driver:
 
         return Machine(
             tmp_dir=tmp_dir,
+            out_dir=self.out_dir,
             start_command=cmd,
             name=name,
             keep_vm_state=args.get("keep_vm_state", False),
@@ -159,3 +190,36 @@ class Driver:
 
     def serial_stdout_off(self) -> None:
         rootlog._print_serial_logs = False
+
+    def check_polling_conditions(self) -> None:
+        for condition in self.polling_conditions:
+            condition.maybe_raise()
+
+    def polling_condition(
+        self,
+        fun_: Optional[Callable] = None,
+        *,
+        seconds_interval: float = 2.0,
+        description: Optional[str] = None,
+    ) -> Union[Callable[[Callable], ContextManager], ContextManager]:
+        driver = self
+
+        class Poll:
+            def __init__(self, fun: Callable):
+                self.condition = PollingCondition(
+                    fun,
+                    seconds_interval,
+                    description,
+                )
+
+            def __enter__(self) -> None:
+                driver.polling_conditions.append(self.condition)
+
+            def __exit__(self, a, b, c) -> None:  # type: ignore
+                res = driver.polling_conditions.pop()
+                assert res is self.condition
+
+        if fun_ is None:
+            return Poll
+        else:
+            return Poll(fun_)
diff --git a/nixos/lib/test-driver/test_driver/machine.py b/nixos/lib/test-driver/test_driver/machine.py
index b3dbe5126fcc..569a0f3c61e4 100644
--- a/nixos/lib/test-driver/test_driver/machine.py
+++ b/nixos/lib/test-driver/test_driver/machine.py
@@ -241,9 +241,15 @@ class LegacyStartCommand(StartCommand):
         cdrom: Optional[str] = None,
         usb: Optional[str] = None,
         bios: Optional[str] = None,
+        qemuBinary: Optional[str] = None,
         qemuFlags: Optional[str] = None,
     ):
-        self._cmd = "qemu-kvm -m 384"
+        if qemuBinary is not None:
+            self._cmd = qemuBinary
+        else:
+            self._cmd = "qemu-kvm"
+
+        self._cmd += " -m 384"
 
         # networking
         net_backend = "-netdev user,id=net0"
@@ -297,6 +303,7 @@ class Machine:
     the machine lifecycle with the help of a start script / command."""
 
     name: str
+    out_dir: Path
     tmp_dir: Path
     shared_dir: Path
     state_dir: Path
@@ -318,23 +325,28 @@ class Machine:
     # Store last serial console lines for use
     # of wait_for_console_text
     last_lines: Queue = Queue()
+    callbacks: List[Callable]
 
     def __repr__(self) -> str:
         return f"<Machine '{self.name}'>"
 
     def __init__(
         self,
+        out_dir: Path,
         tmp_dir: Path,
         start_command: StartCommand,
         name: str = "machine",
         keep_vm_state: bool = False,
         allow_reboot: bool = False,
+        callbacks: Optional[List[Callable]] = None,
     ) -> None:
+        self.out_dir = out_dir
         self.tmp_dir = tmp_dir
         self.keep_vm_state = keep_vm_state
         self.allow_reboot = allow_reboot
         self.name = name
         self.start_command = start_command
+        self.callbacks = callbacks if callbacks is not None else []
 
         # set up directories
         self.shared_dir = self.tmp_dir / "shared-xchg"
@@ -375,6 +387,7 @@ class Machine:
             cdrom=args.get("cdrom"),
             usb=args.get("usb"),
             bios=args.get("bios"),
+            qemuBinary=args.get("qemuBinary"),
             qemuFlags=args.get("qemuFlags"),
         )
 
@@ -406,6 +419,7 @@ class Machine:
             return answer
 
     def send_monitor_command(self, command: str) -> str:
+        self.run_callbacks()
         with self.nested("sending monitor command: {}".format(command)):
             message = ("{}\n".format(command)).encode()
             assert self.monitor is not None
@@ -509,6 +523,7 @@ class Machine:
     def execute(
         self, command: str, check_return: bool = True, timeout: Optional[int] = 900
     ) -> Tuple[int, str]:
+        self.run_callbacks()
         self.connect()
 
         if timeout is not None:
@@ -535,11 +550,11 @@ class Machine:
 
         Should only be used during test development, not in the production test."""
         self.connect()
-        self.log("Terminal is ready (there is no prompt):")
+        self.log("Terminal is ready (there is no initial prompt):")
 
         assert self.shell
         subprocess.run(
-            ["socat", "READLINE", f"FD:{self.shell.fileno()}"],
+            ["socat", "READLINE,prompt=$ ", f"FD:{self.shell.fileno()}"],
             pass_fds=[self.shell.fileno()],
         )
 
@@ -697,10 +712,9 @@ class Machine:
             self.connected = True
 
     def screenshot(self, filename: str) -> None:
-        out_dir = os.environ.get("out", os.getcwd())
         word_pattern = re.compile(r"^\w+$")
         if word_pattern.match(filename):
-            filename = os.path.join(out_dir, "{}.png".format(filename))
+            filename = os.path.join(self.out_dir, "{}.png".format(filename))
         tmp = "{}.ppm".format(filename)
 
         with self.nested(
@@ -751,7 +765,6 @@ class Machine:
         all the VMs (using a temporary directory).
         """
         # Compute the source, target, and intermediate shared file names
-        out_dir = Path(os.environ.get("out", os.getcwd()))
         vm_src = Path(source)
         with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td:
             shared_temp = Path(shared_td)
@@ -761,7 +774,7 @@ class Machine:
             # Copy the file to the shared directory inside VM
             self.succeed(make_command(["mkdir", "-p", vm_shared_temp]))
             self.succeed(make_command(["cp", "-r", vm_src, vm_intermediate]))
-            abs_target = out_dir / target_dir / vm_src.name
+            abs_target = self.out_dir / target_dir / vm_src.name
             abs_target.parent.mkdir(exist_ok=True, parents=True)
             # Copy the file from the shared directory outside VM
             if intermediate.is_dir():
@@ -969,3 +982,7 @@ class Machine:
         self.shell.close()
         self.monitor.close()
         self.serial_thread.join()
+
+    def run_callbacks(self) -> None:
+        for callback in self.callbacks:
+            callback()
diff --git a/nixos/lib/test-driver/test_driver/polling_condition.py b/nixos/lib/test-driver/test_driver/polling_condition.py
new file mode 100644
index 000000000000..459845452fa1
--- /dev/null
+++ b/nixos/lib/test-driver/test_driver/polling_condition.py
@@ -0,0 +1,77 @@
+from typing import Callable, Optional
+import time
+
+from .logger import rootlog
+
+
+class PollingConditionFailed(Exception):
+    pass
+
+
+class PollingCondition:
+    condition: Callable[[], bool]
+    seconds_interval: float
+    description: Optional[str]
+
+    last_called: float
+    entered: bool
+
+    def __init__(
+        self,
+        condition: Callable[[], Optional[bool]],
+        seconds_interval: float = 2.0,
+        description: Optional[str] = None,
+    ):
+        self.condition = condition  # type: ignore
+        self.seconds_interval = seconds_interval
+
+        if description is None:
+            if condition.__doc__:
+                self.description = condition.__doc__
+            else:
+                self.description = condition.__name__
+        else:
+            self.description = str(description)
+
+        self.last_called = float("-inf")
+        self.entered = False
+
+    def check(self) -> bool:
+        if self.entered or not self.overdue:
+            return True
+
+        with self, rootlog.nested(self.nested_message):
+            rootlog.info(f"Time since last: {time.monotonic() - self.last_called:.2f}s")
+            try:
+                res = self.condition()  # type: ignore
+            except Exception:
+                res = False
+            res = res is None or res
+            rootlog.info(self.status_message(res))
+            return res
+
+    def maybe_raise(self) -> None:
+        if not self.check():
+            raise PollingConditionFailed(self.status_message(False))
+
+    def status_message(self, status: bool) -> str:
+        return f"Polling condition {'succeeded' if status else 'failed'}: {self.description}"
+
+    @property
+    def nested_message(self) -> str:
+        nested_message = ["Checking polling condition"]
+        if self.description is not None:
+            nested_message.append(repr(self.description))
+
+        return " ".join(nested_message)
+
+    @property
+    def overdue(self) -> bool:
+        return self.last_called + self.seconds_interval < time.monotonic()
+
+    def __enter__(self) -> None:
+        self.entered = True
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:  # type: ignore
+        self.entered = False
+        self.last_called = time.monotonic()
diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix
index 365e22714573..0d3c3a89e783 100644
--- a/nixos/lib/testing-python.nix
+++ b/nixos/lib/testing-python.nix
@@ -17,7 +17,7 @@ rec {
   inherit pkgs;
 
   # Run an automated test suite in the given virtual network.
-  runTests = { driver, pos }:
+  runTests = { driver, driverInteractive, pos }:
     stdenv.mkDerivation {
       name = "vm-test-run-${driver.testName}";
 
@@ -30,11 +30,11 @@ rec {
           # effectively mute the XMLLogger
           export LOGFILE=/dev/null
 
-          ${driver}/bin/nixos-test-driver
+          ${driver}/bin/nixos-test-driver -o $out
         '';
 
       passthru = driver.passthru // {
-        inherit driver;
+        inherit driver driverInteractive;
       };
 
       inherit pos; # for better debugging
@@ -51,6 +51,7 @@ rec {
     , enableOCR ? false
     , skipLint ? false
     , passthru ? {}
+    , interactive ? false
   }:
     let
       # Reifies and correctly wraps the python test driver for
@@ -139,7 +140,8 @@ rec {
         wrapProgram $out/bin/nixos-test-driver \
           --set startScripts "''${vmStartScripts[*]}" \
           --set testScript "$out/test-script" \
-          --set vlans '${toString vlans}'
+          --set vlans '${toString vlans}' \
+          ${lib.optionalString (interactive) "--add-flags --interactive"}
       '');
 
   # Make a full-blown test
@@ -217,6 +219,7 @@ rec {
         testName = name;
         qemu_pkg = pkgs.qemu;
         nodes = nodes pkgs.qemu;
+        interactive = true;
       };
 
       test =
@@ -224,7 +227,7 @@ rec {
           passMeta = drv: drv // lib.optionalAttrs (t ? meta) {
             meta = (drv.meta or { }) // t.meta;
           };
-        in passMeta (runTests { inherit driver pos; });
+        in passMeta (runTests { inherit driver pos driverInteractive; });
 
     in
       test // {
diff --git a/nixos/lib/utils.nix b/nixos/lib/utils.nix
index bbebf8ba35a0..ae68c3920c5b 100644
--- a/nixos/lib/utils.nix
+++ b/nixos/lib/utils.nix
@@ -45,6 +45,26 @@ rec {
    replaceChars ["/" "-" " "] ["-" "\\x2d" "\\x20"]
    (removePrefix "/" s);
 
+  # Quotes an argument for use in Exec* service lines.
+  # systemd accepts "-quoted strings with escape sequences, toJSON produces
+  # a subset of these.
+  # Additionally we escape % to disallow expansion of % specifiers. Any lone ;
+  # in the input will be turned it ";" and thus lose its special meaning.
+  # Every $ is escaped to $$, this makes it unnecessary to disable environment
+  # substitution for the directive.
+  escapeSystemdExecArg = arg:
+    let
+      s = if builtins.isPath arg then "${arg}"
+        else if builtins.isString arg then arg
+        else if builtins.isInt arg || builtins.isFloat arg then toString arg
+        else throw "escapeSystemdExecArg only allows strings, paths and numbers";
+    in
+      replaceChars [ "%" "$" ] [ "%%" "$$" ] (builtins.toJSON s);
+
+  # Quotes a list of arguments into a single string for use in a Exec*
+  # line.
+  escapeSystemdExecArgs = concatMapStringsSep " " escapeSystemdExecArg;
+
   # Returns a system path for a given shell package
   toShellPath = shell:
     if types.shellPackage.check shell then
@@ -149,10 +169,17 @@ rec {
       if [[ -h '${output}' ]]; then
         rm '${output}'
       fi
+
+      inherit_errexit_enabled=0
+      shopt -pq inherit_errexit && inherit_errexit_enabled=1
+      shopt -s inherit_errexit
     ''
     + concatStringsSep
         "\n"
-        (imap1 (index: name: "export secret${toString index}=$(<'${secrets.${name}}')")
+        (imap1 (index: name: ''
+                  secret${toString index}=$(<'${secrets.${name}}')
+                  export secret${toString index}
+                '')
                (attrNames secrets))
     + "\n"
     + "${pkgs.jq}/bin/jq >'${output}' '"
@@ -164,6 +191,7 @@ rec {
       ' <<'EOF'
       ${builtins.toJSON set}
       EOF
+      (( ! $inherit_errexit_enabled )) && shopt -u inherit_errexit
     '';
 
   systemdUtils = {
diff --git a/nixos/maintainers/scripts/azure-new/examples/basic/system.nix b/nixos/maintainers/scripts/azure-new/examples/basic/system.nix
index 855bd3bab719..d283742701d1 100644
--- a/nixos/maintainers/scripts/azure-new/examples/basic/system.nix
+++ b/nixos/maintainers/scripts/azure-new/examples/basic/system.nix
@@ -17,7 +17,7 @@ in
     description = "Azure NixOS Test User";
     openssh.authorizedKeys.keys = [ (builtins.readFile ~/.ssh/id_ed25519.pub) ];
   };
-  nix.trustedUsers = [ username ];
+  nix.settings.trusted-users = [ username ];
 
   virtualisation.azureImage.diskSize = 2500;
 
diff --git a/nixos/modules/config/fonts/fonts.nix b/nixos/modules/config/fonts/fonts.nix
index 04952898cb76..adc6654afc79 100644
--- a/nixos/modules/config/fonts/fonts.nix
+++ b/nixos/modules/config/fonts/fonts.nix
@@ -39,11 +39,6 @@ let
   defaultXFonts =
     [ (if hasHidpi then fontcursormisc_hidpi else pkgs.xorg.fontcursormisc)
       pkgs.xorg.fontmiscmisc
-    ] ++ optionals (config.nixpkgs.config.allowUnfree or false)
-    [ # these are unfree, and will make usage with xserver fail
-      pkgs.xorg.fontbhlucidatypewriter100dpi
-      pkgs.xorg.fontbhlucidatypewriter75dpi
-      pkgs.xorg.fontbh100dpi
     ];
 
 in
diff --git a/nixos/modules/config/malloc.nix b/nixos/modules/config/malloc.nix
index 84da5643004f..a3fed33afa18 100644
--- a/nixos/modules/config/malloc.nix
+++ b/nixos/modules/config/malloc.nix
@@ -22,8 +22,15 @@ let
       '';
     };
 
-    scudo = {
-      libPath = "${pkgs.llvmPackages_latest.compiler-rt}/lib/linux/libclang_rt.scudo-x86_64.so";
+    scudo = let
+      platformMap = {
+        aarch64-linux = "aarch64";
+        x86_64-linux  = "x86_64";
+      };
+
+      systemPlatform = platformMap.${pkgs.stdenv.hostPlatform.system} or (throw "scudo not supported on ${pkgs.stdenv.hostPlatform.system}");
+    in {
+      libPath = "${pkgs.llvmPackages_latest.compiler-rt}/lib/linux/libclang_rt.scudo-${systemPlatform}.so";
       description = ''
         A user-mode allocator based on LLVM Sanitizer’s CombinedAllocator,
         which aims at providing additional mitigations against heap based
diff --git a/nixos/modules/config/networking.nix b/nixos/modules/config/networking.nix
index 11307e331200..bebfeb352c01 100644
--- a/nixos/modules/config/networking.nix
+++ b/nixos/modules/config/networking.nix
@@ -1,12 +1,13 @@
 # /etc files related to networking, such as /etc/services.
 
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
 
   cfg = config.networking;
+  opt = options.networking;
 
   localhostMultiple = any (elem "localhost") (attrValues (removeAttrs cfg.hosts [ "127.0.0.1" "::1" ]));
 
@@ -78,6 +79,7 @@ in
       httpProxy = lib.mkOption {
         type = types.nullOr types.str;
         default = cfg.proxy.default;
+        defaultText = literalExpression "config.${opt.proxy.default}";
         description = ''
           This option specifies the http_proxy environment variable.
         '';
@@ -87,6 +89,7 @@ in
       httpsProxy = lib.mkOption {
         type = types.nullOr types.str;
         default = cfg.proxy.default;
+        defaultText = literalExpression "config.${opt.proxy.default}";
         description = ''
           This option specifies the https_proxy environment variable.
         '';
@@ -96,6 +99,7 @@ in
       ftpProxy = lib.mkOption {
         type = types.nullOr types.str;
         default = cfg.proxy.default;
+        defaultText = literalExpression "config.${opt.proxy.default}";
         description = ''
           This option specifies the ftp_proxy environment variable.
         '';
@@ -105,6 +109,7 @@ in
       rsyncProxy = lib.mkOption {
         type = types.nullOr types.str;
         default = cfg.proxy.default;
+        defaultText = literalExpression "config.${opt.proxy.default}";
         description = ''
           This option specifies the rsync_proxy environment variable.
         '';
@@ -114,6 +119,7 @@ in
       allProxy = lib.mkOption {
         type = types.nullOr types.str;
         default = cfg.proxy.default;
+        defaultText = literalExpression "config.${opt.proxy.default}";
         description = ''
           This option specifies the all_proxy environment variable.
         '';
@@ -190,9 +196,7 @@ in
         protocols.source  = pkgs.iana-etc + "/etc/protocols";
 
         # /etc/hosts: Hostname-to-IP mappings.
-        hosts.source = pkgs.runCommand "hosts" {} ''
-          cat ${escapeShellArgs cfg.hostFiles} > $out
-        '';
+        hosts.source = pkgs.concatText "hosts" cfg.hostFiles;
 
         # /etc/netgroup: Network-wide groups.
         netgroup.text = mkDefault "";
diff --git a/nixos/modules/config/system-path.nix b/nixos/modules/config/system-path.nix
index 6ff4ec2921cf..875c4c9c4415 100644
--- a/nixos/modules/config/system-path.nix
+++ b/nixos/modules/config/system-path.nix
@@ -41,12 +41,17 @@ let
       pkgs.zstd
     ];
 
-    defaultPackages = map (pkg: setPrio ((pkg.meta.priority or 5) + 3) pkg)
-      [ pkgs.nano
-        pkgs.perl
-        pkgs.rsync
-        pkgs.strace
-      ];
+  defaultPackageNames =
+    [ "nano"
+      "perl"
+      "rsync"
+      "strace"
+    ];
+  defaultPackages =
+    map
+      (n: let pkg = pkgs.${n}; in setPrio ((pkg.meta.priority or 5) + 3) pkg)
+      defaultPackageNames;
+  defaultPackagesText = "[ ${concatMapStringsSep " " (n: "pkgs.${n}") defaultPackageNames } ]";
 
 in
 
@@ -73,6 +78,11 @@ in
       defaultPackages = mkOption {
         type = types.listOf types.package;
         default = defaultPackages;
+        defaultText = literalDocBook ''
+          these packages, with their <literal>meta.priority</literal> numerically increased
+          (thus lowering their installation priority):
+          <programlisting>${defaultPackagesText}</programlisting>
+        '';
         example = [];
         description = ''
           Set of default packages that aren't strictly necessary
diff --git a/nixos/modules/config/update-users-groups.pl b/nixos/modules/config/update-users-groups.pl
index 232f886789d3..26ce561013b6 100644
--- a/nixos/modules/config/update-users-groups.pl
+++ b/nixos/modules/config/update-users-groups.pl
@@ -351,7 +351,7 @@ foreach my $u (values %usersOut) {
         push @subGids, $value;
     }
 
-    if($u->{isNormalUser}) {
+    if($u->{autoSubUidGidRange}) {
         my $subordinate = allocSubUid($name);
         $subUidMap->{$name} = $subordinate;
         my $value = join(":", ($name, $subordinate, 65536));
diff --git a/nixos/modules/config/users-groups.nix b/nixos/modules/config/users-groups.nix
index a34d28143418..b0f96c754fa5 100644
--- a/nixos/modules/config/users-groups.nix
+++ b/nixos/modules/config/users-groups.nix
@@ -204,6 +204,16 @@ let
         '';
       };
 
+      autoSubUidGidRange = mkOption {
+        type = types.bool;
+        default = false;
+        example = true;
+        description = ''
+          Automatically allocate subordinate user and group ids for this user.
+          Allocated range is currently always of size 65536.
+        '';
+      };
+
       createHome = mkOption {
         type = types.bool;
         default = false;
@@ -320,6 +330,9 @@ let
         (mkIf (!cfg.mutableUsers && config.initialHashedPassword != null) {
           hashedPassword = mkDefault config.initialHashedPassword;
         })
+        (mkIf (config.isNormalUser && config.subUidRanges == [] && config.subGidRanges == []) {
+          autoSubUidGidRange = mkDefault true;
+        })
       ];
 
   };
@@ -419,7 +432,7 @@ let
       { inherit (u)
           name uid group description home createHome isSystemUser
           password passwordFile hashedPassword
-          isNormalUser subUidRanges subGidRanges
+          autoSubUidGidRange subUidRanges subGidRanges
           initialPassword initialHashedPassword;
         shell = utils.toShellPath u.shell;
       }) cfg.users;
@@ -436,16 +449,10 @@ in {
   imports = [
     (mkAliasOptionModule [ "users" "extraUsers" ] [ "users" "users" ])
     (mkAliasOptionModule [ "users" "extraGroups" ] [ "users" "groups" ])
-    (mkChangedOptionModule
-      [ "security" "initialRootPassword" ]
-      [ "users" "users" "root" "initialHashedPassword" ]
-      (cfg: if cfg.security.initialRootPassword == "!"
-            then null
-            else cfg.security.initialRootPassword))
+    (mkRenamedOptionModule ["security" "initialRootPassword"] ["users" "users" "root" "initialHashedPassword"])
   ];
 
   ###### interface
-
   options = {
 
     users.mutableUsers = mkOption {
@@ -513,6 +520,17 @@ in {
       '';
     };
 
+
+    users.allowNoPasswordLogin = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Disable checking that at least the <literal>root</literal> user or a user in the <literal>wheel</literal> group can log in using
+        a password or an SSH key.
+
+        WARNING: enabling this can lock you out of your system. Enable this only if you know what are you doing.
+      '';
+    };
   };
 
 
@@ -527,6 +545,7 @@ in {
         home = "/root";
         shell = mkDefault cfg.defaultUserShell;
         group = "root";
+        initialHashedPassword = mkDefault "!";
       };
       nobody = {
         uid = ids.uids.nobody;
@@ -603,9 +622,11 @@ in {
         # there is at least one "privileged" account that has a
         # password or an SSH authorized key. Privileged accounts are
         # root and users in the wheel group.
-        assertion = !cfg.mutableUsers ->
-          any id ((mapAttrsToList (_: cfg:
-            (cfg.name == "root"
+        # The check does not apply when users.disableLoginPossibilityAssertion
+        # The check does not apply when users.mutableUsers
+        assertion = !cfg.mutableUsers -> !cfg.allowNoPasswordLogin ->
+          any id (mapAttrsToList (name: cfg:
+            (name == "root"
              || cfg.group == "wheel"
              || elem "wheel" cfg.extraGroups)
             &&
@@ -614,12 +635,16 @@ in {
              || cfg.passwordFile != null
              || cfg.openssh.authorizedKeys.keys != []
              || cfg.openssh.authorizedKeys.keyFiles != [])
-          ) cfg.users) ++ [
+          ) cfg.users ++ [
             config.security.googleOsLogin.enable
           ]);
         message = ''
           Neither the root account nor any wheel user has a password or SSH authorized key.
-          You must set one to prevent being locked out of your system.'';
+          You must set one to prevent being locked out of your system.
+          If you really want to be locked out of your system, set users.allowNoPasswordLogin = true;
+          However you are most probably better off by setting users.mutableUsers = true; and
+          manually running passwd root to set the root password.
+          '';
       }
     ] ++ flatten (flip mapAttrsToList cfg.users (name: user:
       [
diff --git a/nixos/modules/config/xdg/portal.nix b/nixos/modules/config/xdg/portal.nix
index 80ec3126ca54..088f2af59e22 100644
--- a/nixos/modules/config/xdg/portal.nix
+++ b/nixos/modules/config/xdg/portal.nix
@@ -1,4 +1,4 @@
-{ config, pkgs ,lib ,... }:
+{ config, pkgs, lib, ... }:
 
 with lib;
 
@@ -13,13 +13,13 @@ with lib;
 
   options.xdg.portal = {
     enable =
-      mkEnableOption "<link xlink:href='https://github.com/flatpak/xdg-desktop-portal'>xdg desktop integration</link>"//{
+      mkEnableOption "<link xlink:href='https://github.com/flatpak/xdg-desktop-portal'>xdg desktop integration</link>" // {
         default = false;
       };
 
     extraPortals = mkOption {
       type = types.listOf types.package;
-      default = [];
+      default = [ ];
       description = ''
         List of additional portals to add to path. Portals allow interaction
         with system, like choosing files or taking screenshots. At minimum,
@@ -46,25 +46,36 @@ with lib;
     let
       cfg = config.xdg.portal;
       packages = [ pkgs.xdg-desktop-portal ] ++ cfg.extraPortals;
-      joinedPortals = pkgs.symlinkJoin {
+      joinedPortals = pkgs.buildEnv {
         name = "xdg-portals";
-        paths = cfg.extraPortals;
+        paths = packages;
+        pathsToLink = [ "/share/xdg-desktop-portal/portals" "/share/applications" ];
       };
 
-    in mkIf cfg.enable {
+    in
+    mkIf cfg.enable {
 
       assertions = [
-        { assertion = (cfg.gtkUsePortal -> cfg.extraPortals != []);
-          message = "Setting xdg.portal.gtkUsePortal to true requires a portal implementation in xdg.portal.extraPortals such as xdg-desktop-portal-gtk or xdg-desktop-portal-kde.";
+        {
+          assertion = cfg.extraPortals != [ ];
+          message = "Setting xdg.portal.enable to true requires a portal implementation in xdg.portal.extraPortals such as xdg-desktop-portal-gtk or xdg-desktop-portal-kde.";
         }
       ];
 
-      services.dbus.packages  = packages;
+      services.dbus.packages = packages;
       systemd.packages = packages;
 
-      environment.sessionVariables = {
-        GTK_USE_PORTAL = mkIf cfg.gtkUsePortal "1";
-        XDG_DESKTOP_PORTAL_DIR = "${joinedPortals}/share/xdg-desktop-portal/portals";
+      environment = {
+        # fixes screen sharing on plasmawayland on non-chromium apps by linking
+        # share/applications/*.desktop files
+        # see https://github.com/NixOS/nixpkgs/issues/145174
+        systemPackages = [ joinedPortals ];
+        pathsToLink = [ "/share/applications" ];
+
+        sessionVariables = {
+          GTK_USE_PORTAL = mkIf cfg.gtkUsePortal "1";
+          XDG_DESKTOP_PORTAL_DIR = "${joinedPortals}/share/xdg-desktop-portal/portals";
+        };
       };
     };
 }
diff --git a/nixos/modules/hardware/all-firmware.nix b/nixos/modules/hardware/all-firmware.nix
index ce87f9e8be8a..5b60b17312f9 100644
--- a/nixos/modules/hardware/all-firmware.nix
+++ b/nixos/modules/hardware/all-firmware.nix
@@ -31,7 +31,6 @@ in {
       type = types.bool;
       description = ''
         Turn on this option if you want to enable all the firmware with a license allowing redistribution.
-        (i.e. free firmware and <literal>firmware-linux-nonfree</literal>)
       '';
     };
 
@@ -51,14 +50,13 @@ in {
   config = mkMerge [
     (mkIf (cfg.enableAllFirmware || cfg.enableRedistributableFirmware) {
       hardware.firmware = with pkgs; [
-        firmwareLinuxNonfree
+        linux-firmware
         intel2200BGFirmware
         rtl8192su-firmware
         rt5677-firmware
         rtl8723bs-firmware
         rtl8761b-firmware
         rtw88-firmware
-        rtw89-firmware
         zd1211fw
         alsa-firmware
         sof-firmware
@@ -66,6 +64,8 @@ in {
       ] ++ optional (pkgs.stdenv.hostPlatform.isAarch32 || pkgs.stdenv.hostPlatform.isAarch64) raspberrypiWirelessFirmware
         ++ optionals (versionOlder config.boot.kernelPackages.kernel.version "4.13") [
         rtl8723bs-firmware
+      ] ++ optionals (versionOlder config.boot.kernelPackages.kernel.version "5.16") [
+        rtw89-firmware
       ];
       hardware.wirelessRegulatoryDatabase = true;
     })
@@ -83,6 +83,7 @@ in {
         b43Firmware_5_1_138
         b43Firmware_6_30_163_46
         b43FirmwareCutter
+        xow_dongle-firmware
       ] ++ optional pkgs.stdenv.hostPlatform.isx86 facetimehd-firmware;
     })
     (mkIf cfg.wirelessRegulatoryDatabase {
diff --git a/nixos/modules/hardware/cpu/intel-sgx.nix b/nixos/modules/hardware/cpu/intel-sgx.nix
new file mode 100644
index 000000000000..1355ee753f0d
--- /dev/null
+++ b/nixos/modules/hardware/cpu/intel-sgx.nix
@@ -0,0 +1,69 @@
+{ config, lib, ... }:
+with lib;
+let
+  cfg = config.hardware.cpu.intel.sgx;
+  defaultPrvGroup = "sgx_prv";
+in
+{
+  options.hardware.cpu.intel.sgx.enableDcapCompat = mkOption {
+    description = ''
+      Whether to enable backward compatibility for SGX software build for the
+      out-of-tree Intel SGX DCAP driver.
+
+      Creates symbolic links for the SGX devices <literal>/dev/sgx_enclave</literal>
+      and <literal>/dev/sgx_provision</literal> to make them available as
+      <literal>/dev/sgx/enclave</literal>  and <literal>/dev/sgx/provision</literal>,
+      respectively.
+    '';
+    type = types.bool;
+    default = true;
+  };
+
+  options.hardware.cpu.intel.sgx.provision = {
+    enable = mkEnableOption "access to the Intel SGX provisioning device";
+    user = mkOption {
+      description = "Owner to assign to the SGX provisioning device.";
+      type = types.str;
+      default = "root";
+    };
+    group = mkOption {
+      description = "Group to assign to the SGX provisioning device.";
+      type = types.str;
+      default = defaultPrvGroup;
+    };
+    mode = mkOption {
+      description = "Mode to set for the SGX provisioning device.";
+      type = types.str;
+      default = "0660";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.provision.enable {
+      assertions = [
+        {
+          assertion = hasAttr cfg.provision.user config.users.users;
+          message = "Given user does not exist";
+        }
+        {
+          assertion = (cfg.provision.group == defaultPrvGroup) || (hasAttr cfg.provision.group config.users.groups);
+          message = "Given group does not exist";
+        }
+      ];
+
+      users.groups = optionalAttrs (cfg.provision.group == defaultPrvGroup) {
+        "${cfg.provision.group}" = { };
+      };
+
+      services.udev.extraRules = with cfg.provision; ''
+        SUBSYSTEM=="misc", KERNEL=="sgx_provision", OWNER="${user}", GROUP="${group}", MODE="${mode}"
+      '';
+    })
+    (mkIf cfg.enableDcapCompat {
+      services.udev.extraRules = ''
+        SUBSYSTEM=="misc", KERNEL=="sgx_enclave",   SYMLINK+="sgx/enclave"
+        SUBSYSTEM=="misc", KERNEL=="sgx_provision", SYMLINK+="sgx/provision"
+      '';
+    })
+  ];
+}
diff --git a/nixos/modules/hardware/hackrf.nix b/nixos/modules/hardware/hackrf.nix
new file mode 100644
index 000000000000..7f03b765bbda
--- /dev/null
+++ b/nixos/modules/hardware/hackrf.nix
@@ -0,0 +1,23 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.hardware.hackrf;
+
+in
+{
+  options.hardware.hackrf = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Enables hackrf udev rules and ensures 'plugdev' group exists.
+        This is a prerequisite to using HackRF devices without being root, since HackRF USB descriptors will be owned by plugdev through udev.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.udev.packages = [ pkgs.hackrf ];
+    users.groups.plugdev = { };
+  };
+}
diff --git a/nixos/modules/hardware/network/b43.nix b/nixos/modules/hardware/network/b43.nix
index e63f2d04d1a6..eb03bf223ccf 100644
--- a/nixos/modules/hardware/network/b43.nix
+++ b/nixos/modules/hardware/network/b43.nix
@@ -24,10 +24,6 @@ let kernelVersion = config.boot.kernelPackages.kernel.version; in
   ###### implementation
 
   config = mkIf config.networking.enableB43Firmware {
-    assertions = singleton
-      { assertion = lessThan 0 (builtins.compareVersions kernelVersion "3.2");
-        message = "b43 firmware for kernels older than 3.2 not packaged yet!";
-      };
     hardware.firmware = [ pkgs.b43Firmware_5_1_138 ];
   };
 
diff --git a/nixos/modules/hardware/onlykey/onlykey.udev b/nixos/modules/hardware/onlykey/onlykey.udev
index 61e3ee4e8828..9c8873aafc9e 100644
--- a/nixos/modules/hardware/onlykey/onlykey.udev
+++ b/nixos/modules/hardware/onlykey/onlykey.udev
@@ -14,5 +14,5 @@ KERNEL=="ttyACM*", ATTRS{idVendor}=="1d50", ATTRS{idProduct}=="60fc", MODE:="066
 #
 ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1"
 ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1"
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", GROUP+="plugdev"
-KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", GROUP+="plugdev"
+SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", GROUP="plugdev"
+KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", GROUP="plugdev"
diff --git a/nixos/modules/hardware/rtl-sdr.nix b/nixos/modules/hardware/rtl-sdr.nix
index 9605c7967f61..e85fc04e29bb 100644
--- a/nixos/modules/hardware/rtl-sdr.nix
+++ b/nixos/modules/hardware/rtl-sdr.nix
@@ -5,10 +5,14 @@ let
 
 in {
   options.hardware.rtl-sdr = {
-    enable = lib.mkEnableOption ''
-      Enables rtl-sdr udev rules, ensures 'plugdev' group exists, and blacklists DVB kernel modules.
-      This is a prerequisite to using devices supported by rtl-sdr without being root, since rtl-sdr USB descriptors will be owned by plugdev through udev.
-    '';
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Enables rtl-sdr udev rules, ensures 'plugdev' group exists, and blacklists DVB kernel modules.
+        This is a prerequisite to using devices supported by rtl-sdr without being root, since rtl-sdr USB descriptors will be owned by plugdev through udev.
+       '';
+    };
   };
 
   config = lib.mkIf cfg.enable {
diff --git a/nixos/modules/hardware/system-76.nix b/nixos/modules/hardware/system-76.nix
index d4896541dbae..ca40ee0ebb37 100644
--- a/nixos/modules/hardware/system-76.nix
+++ b/nixos/modules/hardware/system-76.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
-  inherit (lib) mkOption mkEnableOption types mkIf mkMerge optional versionOlder;
+  inherit (lib) literalExpression mkOption mkEnableOption types mkIf mkMerge optional versionOlder;
   cfg = config.hardware.system76;
+  opt = options.hardware.system76;
 
   kpkgs = config.boot.kernelPackages;
   modules = [ "system76" "system76-io" ] ++ (optional (versionOlder kpkgs.kernel.version "5.5") "system76-acpi");
@@ -60,6 +61,7 @@ in {
 
       firmware-daemon.enable = mkOption {
         default = cfg.enableAll;
+        defaultText = literalExpression "config.${opt.enableAll}";
         example = true;
         description = "Whether to enable the system76 firmware daemon";
         type = types.bool;
@@ -67,6 +69,7 @@ in {
 
       kernel-modules.enable = mkOption {
         default = cfg.enableAll;
+        defaultText = literalExpression "config.${opt.enableAll}";
         example = true;
         description = "Whether to make the system76 out-of-tree kernel modules available";
         type = types.bool;
@@ -74,6 +77,7 @@ in {
 
       power-daemon.enable = mkOption {
         default = cfg.enableAll;
+        defaultText = literalExpression "config.${opt.enableAll}";
         example = true;
         description = "Whether to enable the system76 power daemon";
         type = types.bool;
diff --git a/nixos/modules/hardware/video/amdgpu-pro.nix b/nixos/modules/hardware/video/amdgpu-pro.nix
index ec1c8c2d57a1..d784befc9b88 100644
--- a/nixos/modules/hardware/video/amdgpu-pro.nix
+++ b/nixos/modules/hardware/video/amdgpu-pro.nix
@@ -11,23 +11,17 @@ let
   enabled = elem "amdgpu-pro" drivers;
 
   package = config.boot.kernelPackages.amdgpu-pro;
-  package32 = pkgs.pkgsi686Linux.linuxPackages.amdgpu-pro.override { libsOnly = true; kernel = null; };
+  package32 = pkgs.pkgsi686Linux.linuxPackages.amdgpu-pro.override { kernel = null; };
 
   opengl = config.hardware.opengl;
 
-  kernel = pkgs.linux_4_9.override {
-    extraConfig = ''
-      KALLSYMS_ALL y
-    '';
-  };
-
 in
 
 {
 
   config = mkIf enabled {
 
-    nixpkgs.config.xorg.abiCompat = "1.19";
+    nixpkgs.config.xorg.abiCompat = "1.20";
 
     services.xserver.drivers = singleton
       { name = "amdgpu"; modules = [ package ]; display = true; };
@@ -36,31 +30,39 @@ in
     hardware.opengl.package32 = package32;
     hardware.opengl.setLdLibraryPath = true;
 
-    boot.extraModulePackages = [ package ];
-
-    boot.kernelPackages =
-      pkgs.recurseIntoAttrs (pkgs.linuxPackagesFor kernel);
+    boot.extraModulePackages = [ package.kmod ];
 
-    boot.blacklistedKernelModules = [ "radeon" ];
+    boot.kernelPackages = pkgs.linuxKernel.packagesFor
+      (pkgs.linuxKernel.kernels.linux_5_10.override {
+        structuredExtraConfig = {
+          DEVICE_PRIVATE = kernel.yes;
+          KALLSYMS_ALL = kernel.yes;
+        };
+      });
 
-    hardware.firmware = [ package ];
+    hardware.firmware = [ package.fw ];
 
     system.activationScripts.setup-amdgpu-pro = ''
-      mkdir -p /run/lib
-      ln -sfn ${package}/lib ${package.libCompatDir}
-      ln -sfn ${package} /run/amdgpu-pro
-    '' + optionalString opengl.driSupport32Bit ''
-      ln -sfn ${package32}/lib ${package32.libCompatDir}
+      ln -sfn ${package}/opt/amdgpu{,-pro} /run
     '';
 
     system.requiredKernelConfig = with config.lib.kernelConfig; [
+      (isYes "DEVICE_PRIVATE")
       (isYes "KALLSYMS_ALL")
     ];
 
+    boot.initrd.extraUdevRulesCommands = ''
+      cp -v ${package}/etc/udev/rules.d/*.rules $out/
+    '';
+
+    environment.systemPackages =
+      [ package.vulkan ] ++
+      # this isn't really DRI, but we'll reuse this option for now
+      optional config.hardware.opengl.driSupport32Bit package32.vulkan;
+
     environment.etc = {
-      "amd/amdrc".source = package + "/etc/amd/amdrc";
-      "amd/amdapfxx.blb".source = package + "/etc/amd/amdapfxx.blb";
-      "gbm/gbm.conf".source = package + "/etc/gbm/gbm.conf";
+      "modprobe.d/blacklist-radeon.conf".source = package + "/etc/modprobe.d/blacklist-radeon.conf";
+      amd.source = package + "/etc/amd";
     };
 
   };
diff --git a/nixos/modules/hardware/video/capture/mwprocapture.nix b/nixos/modules/hardware/video/capture/mwprocapture.nix
index 61bab533edaf..76cb4c6ee9bf 100644
--- a/nixos/modules/hardware/video/capture/mwprocapture.nix
+++ b/nixos/modules/hardware/video/capture/mwprocapture.nix
@@ -16,11 +16,6 @@ in
 
   config = mkIf cfg.enable {
 
-    assertions = singleton {
-      assertion = versionAtLeast kernelPackages.kernel.version "3.2";
-      message = "Magewell Pro Capture family module is not supported for kernels older than 3.2";
-    };
-
     boot.kernelModules = [ "ProCapture" ];
 
     environment.systemPackages = [ kernelPackages.mwprocapture ];
diff --git a/nixos/modules/hardware/video/nvidia.nix b/nixos/modules/hardware/video/nvidia.nix
index ff4225dc29ad..6de5b99a1ee6 100644
--- a/nixos/modules/hardware/video/nvidia.nix
+++ b/nixos/modules/hardware/video/nvidia.nix
@@ -179,11 +179,6 @@ in
   in mkIf enabled {
     assertions = [
       {
-        assertion = with config.services.xserver.displayManager; (gdm.enable && gdm.nvidiaWayland) -> cfg.modesetting.enable;
-        message = "You cannot use wayland with GDM without modesetting enabled for NVIDIA drivers, set `hardware.nvidia.modesetting.enable = true`";
-      }
-
-      {
         assertion = primeEnabled -> pCfg.intelBusId == "" || pCfg.amdgpuBusId == "";
         message = ''
           You cannot configure both an Intel iGPU and an AMD APU. Pick the one corresponding to your processor.
@@ -249,7 +244,7 @@ in
       modules = optional (igpuDriver == "amdgpu") [ pkgs.xorg.xf86videoamdgpu ];
       deviceSection = ''
         BusID "${igpuBusId}"
-        ${optionalString syncCfg.enable ''Option "AccelMethod" "none"''}
+        ${optionalString (syncCfg.enable && igpuDriver != "amdgpu") ''Option "AccelMethod" "none"''}
       '';
     } ++ singleton {
       name = "nvidia";
@@ -274,9 +269,15 @@ in
       Option "AllowNVIDIAGPUScreens"
     '';
 
-    services.xserver.displayManager.setupCommands = optionalString syncCfg.enable ''
+    services.xserver.displayManager.setupCommands = let
+      sinkGpuProviderName = if igpuDriver == "amdgpu" then
+        # find the name of the provider if amdgpu
+        "`${pkgs.xorg.xrandr}/bin/xrandr --listproviders | ${pkgs.gnugrep}/bin/grep -i AMD | ${pkgs.gnused}/bin/sed -n 's/^.*name://p'`"
+      else
+        igpuDriver;
+    in optionalString syncCfg.enable ''
       # Added by nvidia configuration module for Optimus/PRIME.
-      ${pkgs.xorg.xrandr}/bin/xrandr --setprovideroutputsource ${igpuDriver} NVIDIA-0
+      ${pkgs.xorg.xrandr}/bin/xrandr --setprovideroutputsource "${sinkGpuProviderName}" NVIDIA-0
       ${pkgs.xorg.xrandr}/bin/xrandr --auto
     '';
 
@@ -288,10 +289,14 @@ in
     environment.etc."egl/egl_external_platform.d".source =
       "/run/opengl-driver/share/egl/egl_external_platform.d/";
 
-    hardware.opengl.package = mkIf (!offloadCfg.enable) nvidia_x11.out;
-    hardware.opengl.package32 = mkIf (!offloadCfg.enable) nvidia_x11.lib32;
-    hardware.opengl.extraPackages = optional offloadCfg.enable nvidia_x11.out;
-    hardware.opengl.extraPackages32 = optional offloadCfg.enable nvidia_x11.lib32;
+    hardware.opengl.extraPackages = [
+      nvidia_x11.out
+      pkgs.nvidia-vaapi-driver
+    ];
+    hardware.opengl.extraPackages32 = [
+      nvidia_x11.lib32
+      pkgs.pkgsi686Linux.nvidia-vaapi-driver
+    ];
 
     environment.systemPackages = [ nvidia_x11.bin ]
       ++ optionals cfg.nvidiaSettings [ nvidia_x11.settings ]
diff --git a/nixos/modules/hardware/xone.nix b/nixos/modules/hardware/xone.nix
new file mode 100644
index 000000000000..89690d8c6fb1
--- /dev/null
+++ b/nixos/modules/hardware/xone.nix
@@ -0,0 +1,23 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.hardware.xone;
+in
+{
+  options.hardware.xone = {
+    enable = mkEnableOption "the xone driver for Xbox One and Xbobx Series X|S accessories";
+  };
+
+  config = mkIf cfg.enable {
+    boot = {
+      blacklistedKernelModules = [ "xpad" "mt76x2u" ];
+      extraModulePackages = with config.boot.kernelPackages; [ xone ];
+    };
+    hardware.firmware = [ pkgs.xow_dongle-firmware ];
+  };
+
+  meta = {
+    maintainers = with maintainers; [ rhysmdnz ];
+  };
+}
diff --git a/nixos/modules/i18n/input-method/fcitx.nix b/nixos/modules/i18n/input-method/fcitx.nix
index 57960cc365b6..7738581b893a 100644
--- a/nixos/modules/i18n/input-method/fcitx.nix
+++ b/nixos/modules/i18n/input-method/fcitx.nix
@@ -40,4 +40,7 @@ in
     };
     services.xserver.displayManager.sessionCommands = "${fcitxPackage}/bin/fcitx";
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/i18n/input-method/ibus.nix b/nixos/modules/i18n/input-method/ibus.nix
index 92f8c64338a4..c5b0cbc21502 100644
--- a/nixos/modules/i18n/input-method/ibus.nix
+++ b/nixos/modules/i18n/input-method/ibus.nix
@@ -80,4 +80,7 @@ in
       ibusPackage
     ];
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/i18n/input-method/kime.nix b/nixos/modules/i18n/input-method/kime.nix
index e462cae2437b..729a665614ae 100644
--- a/nixos/modules/i18n/input-method/kime.nix
+++ b/nixos/modules/i18n/input-method/kime.nix
@@ -45,5 +45,7 @@ in
 
     environment.etc."xdg/kime/config.yaml".text = replaceStrings [ "\\\\" ] [ "\\" ] (builtins.toJSON cfg.config);
   };
-}
 
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
+}
diff --git a/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix b/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix
index 12ad8a4ae004..303493741f3d 100644
--- a/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix
+++ b/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix
@@ -10,10 +10,10 @@ with lib;
   isoImage.edition = "gnome";
 
   services.xserver.desktopManager.gnome = {
-    # Add firefox to favorite-apps
+    # Add Firefox and other tools useful for installation to the launcher
     favoriteAppsOverride = ''
       [org.gnome.shell]
-      favorite-apps=[ 'firefox.desktop', 'org.gnome.Geary.desktop', 'org.gnome.Calendar.desktop', 'org.gnome.Music.desktop', 'org.gnome.Photos.desktop', 'org.gnome.Nautilus.desktop' ]
+      favorite-apps=[ 'firefox.desktop', 'nixos-manual.desktop', 'org.gnome.Terminal.desktop', 'org.gnome.Nautilus.desktop', 'gparted.desktop' ]
     '';
     enable = true;
   };
diff --git a/nixos/modules/installer/cd-dvd/iso-image.nix b/nixos/modules/installer/cd-dvd/iso-image.nix
index 30610b4f4260..3ff1b3d670e9 100644
--- a/nixos/modules/installer/cd-dvd/iso-image.nix
+++ b/nixos/modules/installer/cd-dvd/iso-image.nix
@@ -734,13 +734,13 @@ in
         { source = config.system.build.squashfsStore;
           target = "/nix-store.squashfs";
         }
-        { source = config.isoImage.splashImage;
-          target = "/isolinux/background.png";
-        }
         { source = pkgs.writeText "version" config.system.nixos.label;
           target = "/version.txt";
         }
       ] ++ optionals canx86BiosBoot [
+        { source = config.isoImage.splashImage;
+          target = "/isolinux/background.png";
+        }
         { source = pkgs.substituteAll  {
             name = "isolinux.cfg";
             src = pkgs.writeText "isolinux.cfg-in" isolinuxCfg;
@@ -761,6 +761,9 @@ in
         { source = (pkgs.writeTextDir "grub/loopback.cfg" "source /EFI/boot/grub.cfg") + "/grub";
           target = "/boot/grub";
         }
+        { source = config.isoImage.efiSplashImage;
+          target = "/EFI/boot/efi-background.png";
+        }
       ] ++ optionals (config.boot.loader.grub.memtest86.enable && canx86BiosBoot) [
         { source = "${pkgs.memtest86plus}/memtest.bin";
           target = "/boot/memtest.bin";
@@ -769,10 +772,6 @@ in
         { source = config.isoImage.grubTheme;
           target = "/EFI/boot/grub-theme";
         }
-      ] ++ [
-        { source = config.isoImage.efiSplashImage;
-          target = "/EFI/boot/efi-background.png";
-        }
       ];
 
     boot.loader.timeout = 10;
diff --git a/nixos/modules/installer/netboot/netboot.nix b/nixos/modules/installer/netboot/netboot.nix
index 145f71b5d0c7..a459e7304cd4 100644
--- a/nixos/modules/installer/netboot/netboot.nix
+++ b/nixos/modules/installer/netboot/netboot.nix
@@ -94,7 +94,9 @@ with lib;
 
     system.build.netbootIpxeScript = pkgs.writeTextDir "netboot.ipxe" ''
       #!ipxe
-      kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ${toString config.boot.kernelParams}
+      # Use the cmdline variable to allow the user to specify custom kernel params
+      # when chainloading this script from other iPXE scripts like netboot.xyz
+      kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ${toString config.boot.kernelParams} ''${cmdline}
       initrd initrd
       boot
     '';
diff --git a/nixos/modules/installer/sd-card/sd-image-aarch64.nix b/nixos/modules/installer/sd-card/sd-image-aarch64.nix
index 165e2aac27b4..321793882f4c 100644
--- a/nixos/modules/installer/sd-card/sd-image-aarch64.nix
+++ b/nixos/modules/installer/sd-card/sd-image-aarch64.nix
@@ -24,6 +24,9 @@
         [pi3]
         kernel=u-boot-rpi3.bin
 
+        [pi02]
+        kernel=u-boot-rpi3.bin
+
         [pi4]
         kernel=u-boot-rpi4.bin
         enable_gic=1
@@ -33,6 +36,9 @@
         # what the pi3 firmware does by default.
         disable_overscan=1
 
+        # Supported in newer board revisions
+        arm_boost=1
+
         [all]
         # Boot in 64-bit mode.
         arm_64bit=1
diff --git a/nixos/modules/installer/sd-card/sd-image-riscv64-qemu-installer.nix b/nixos/modules/installer/sd-card/sd-image-riscv64-qemu-installer.nix
new file mode 100644
index 000000000000..90c1b8413adc
--- /dev/null
+++ b/nixos/modules/installer/sd-card/sd-image-riscv64-qemu-installer.nix
@@ -0,0 +1,10 @@
+{
+  imports = [
+    ../../profiles/installation-device.nix
+    ./sd-image-riscv64-qemu.nix
+  ];
+
+  # the installation media is also the installation target,
+  # so we don't want to provide the installation configuration.nix.
+  installer.cloneConfig = false;
+}
diff --git a/nixos/modules/installer/sd-card/sd-image-riscv64-qemu.nix b/nixos/modules/installer/sd-card/sd-image-riscv64-qemu.nix
new file mode 100644
index 000000000000..a3e30768da45
--- /dev/null
+++ b/nixos/modules/installer/sd-card/sd-image-riscv64-qemu.nix
@@ -0,0 +1,32 @@
+# To build, use:
+# nix-build nixos -I nixos-config=nixos/modules/installer/sd-card/sd-image-riscv64-qemu.nix -A config.system.build.sdImage
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    ../../profiles/base.nix
+    ./sd-image.nix
+  ];
+
+  boot.loader = {
+    grub.enable = false;
+    generic-extlinux-compatible = {
+      enable = true;
+
+      # Don't even specify FDTDIR - We do not have the correct DT
+      # The DTB is generated by QEMU at runtime
+      useGenerationDeviceTree = false;
+    };
+  };
+
+  boot.consoleLogLevel = lib.mkDefault 7;
+  boot.kernelParams = [ "console=tty0" "console=ttyS0,115200n8" ];
+
+  sdImage = {
+    populateFirmwareCommands = "";
+    populateRootCommands = ''
+      mkdir -p ./files/boot
+      ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d ./files/boot
+    '';
+  };
+}
diff --git a/nixos/modules/installer/sd-card/sd-image-x86_64.nix b/nixos/modules/installer/sd-card/sd-image-x86_64.nix
new file mode 100644
index 000000000000..b44c0a4eeca5
--- /dev/null
+++ b/nixos/modules/installer/sd-card/sd-image-x86_64.nix
@@ -0,0 +1,27 @@
+# To build, use:
+# nix-build nixos -I nixos-config=nixos/modules/installer/sd-card/sd-image-x86_64.nix -A config.system.build.sdImage
+
+# This image is primarily used in NixOS tests (boot.nix) to test `boot.loader.generic-extlinux-compatible`.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    ../../profiles/base.nix
+    ./sd-image.nix
+  ];
+
+  boot.loader = {
+    grub.enable = false;
+    generic-extlinux-compatible.enable = true;
+  };
+
+  boot.consoleLogLevel = lib.mkDefault 7;
+
+  sdImage = {
+    populateFirmwareCommands = "";
+    populateRootCommands = ''
+      mkdir -p ./files/boot
+      ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d ./files/boot
+    '';
+  };
+}
diff --git a/nixos/modules/installer/sd-card/sd-image.nix b/nixos/modules/installer/sd-card/sd-image.nix
index a964cf2d6f85..7560c682517a 100644
--- a/nixos/modules/installer/sd-card/sd-image.nix
+++ b/nixos/modules/installer/sd-card/sd-image.nix
@@ -176,7 +176,7 @@ in
 
       nativeBuildInputs = [ dosfstools e2fsprogs mtools libfaketime util-linux zstd ];
 
-      inherit (config.sdImage) compressImage;
+      inherit (config.sdImage) imageName compressImage;
 
       buildCommand = ''
         mkdir -p $out/nix-support $out/sd-image
diff --git a/nixos/modules/installer/tools/nix-fallback-paths.nix b/nixos/modules/installer/tools/nix-fallback-paths.nix
index 065cea470fbb..dfafda77cb56 100644
--- a/nixos/modules/installer/tools/nix-fallback-paths.nix
+++ b/nixos/modules/installer/tools/nix-fallback-paths.nix
@@ -1,7 +1,7 @@
 {
-  x86_64-linux = "/nix/store/hapw7q1fkjxvprnkcgw9ppczavg4daj2-nix-2.4";
-  i686-linux = "/nix/store/8qlvh8pp5j8wgrzj3is2jlbhgrwgsiy9-nix-2.4";
-  aarch64-linux = "/nix/store/h48lkygcqj4hdibbdnpl67q7ks6vkrd6-nix-2.4";
-  x86_64-darwin = "/nix/store/c3mvzszvyzakvcp9spnjvsb8m2bpjk7m-nix-2.4";
-  aarch64-darwin = "/nix/store/hbfqs62r0hga2yr4zi5kc7fzhf71bq9n-nix-2.4";
+  x86_64-linux = "/nix/store/0n2wfvi1i3fg97cjc54wslvk0804y0sn-nix-2.7.0";
+  i686-linux = "/nix/store/4p27c1k9z99pli6x8cxfph20yfyzn9nh-nix-2.7.0";
+  aarch64-linux = "/nix/store/r9yr8ijsb0gi9r7y92y3yzyld59yp0kj-nix-2.7.0";
+  x86_64-darwin = "/nix/store/hyfj5imsd0c4amlcjpf8l6w4q2draaj3-nix-2.7.0";
+  aarch64-darwin = "/nix/store/9l96qllhbb6xrsjaai76dn74ap7rq92n-nix-2.7.0";
 }
diff --git a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
index 8aedce2fb49c..b4a94f62ad93 100644
--- a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
+++ b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
@@ -25,4 +25,7 @@ pkgs.runCommand "nixos-build-vms" { nativeBuildInputs = [ pkgs.makeWrapper ]; }
   ln -s ${interactiveDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
   wrapProgram $out/bin/nixos-test-driver \
     --add-flags "--interactive"
+  wrapProgram $out/bin/nixos-run-vms \
+     --set testScript "${pkgs.writeText "start-all" "start_all(); join_all();"}" \
+     --add-flags "--no-interactive"
 ''
diff --git a/nixos/modules/installer/tools/nixos-enter.sh b/nixos/modules/installer/tools/nixos-enter.sh
index 6469d9faa038..89beeee7cf9e 100644
--- a/nixos/modules/installer/tools/nixos-enter.sh
+++ b/nixos/modules/installer/tools/nixos-enter.sh
@@ -63,32 +63,32 @@ mount --rbind /sys "$mountPoint/sys"
 
 # modified from https://github.com/archlinux/arch-install-scripts/blob/bb04ab435a5a89cd5e5ee821783477bc80db797f/arch-chroot.in#L26-L52
 chroot_add_resolv_conf() {
-    local chrootdir=$1 resolv_conf=$1/etc/resolv.conf
+    local chrootDir="$1" resolvConf="$1/etc/resolv.conf"
 
     [[ -e /etc/resolv.conf ]] || return 0
 
     # Handle resolv.conf as a symlink to somewhere else.
-    if [[ -L $chrootdir/etc/resolv.conf ]]; then
+    if [[ -L "$resolvConf" ]]; then
       # readlink(1) should always give us *something* since we know at this point
       # it's a symlink. For simplicity, ignore the case of nested symlinks.
-      # We also ignore the possibility if `../`s escaping the root.
-      resolv_conf=$(readlink "$chrootdir/etc/resolv.conf")
-      if [[ $resolv_conf = /* ]]; then
-        resolv_conf=$chrootdir$resolv_conf
+      # We also ignore the possibility of `../`s escaping the root.
+      resolvConf="$(readlink "$resolvConf")"
+      if [[ "$resolvConf" = /* ]]; then
+        resolvConf="$chrootDir$resolvConf"
       else
-        resolv_conf=$chrootdir/etc/$resolv_conf
+        resolvConf="$chrootDir/etc/$resolvConf"
       fi
     fi
 
     # ensure file exists to bind mount over
-    if [[ ! -f $resolv_conf ]]; then
-      install -Dm644 /dev/null "$resolv_conf" || return 1
+    if [[ ! -f "$resolvConf" ]]; then
+      install -Dm644 /dev/null "$resolvConf" || return 1
     fi
 
-    mount --bind /etc/resolv.conf "$resolv_conf"
+    mount --bind /etc/resolv.conf "$resolvConf"
 }
 
-chroot_add_resolv_conf "$mountPoint" || print "ERROR: failed to set up resolv.conf"
+chroot_add_resolv_conf "$mountPoint" || echo "$0: failed to set up resolv.conf" >&2
 
 (
     # If silent, write both stdout and stderr of activation script to /dev/null
@@ -104,4 +104,6 @@ chroot_add_resolv_conf "$mountPoint" || print "ERROR: failed to set up resolv.co
     chroot "$mountPoint" systemd-tmpfiles --create --remove --exclude-prefix=/dev 1>&2 || true
 )
 
+unset TMPDIR
+
 exec chroot "$mountPoint" "${command[@]}"
diff --git a/nixos/modules/installer/tools/nixos-generate-config.pl b/nixos/modules/installer/tools/nixos-generate-config.pl
index fe8c4fb1a6b5..57aef50a0f6b 100644
--- a/nixos/modules/installer/tools/nixos-generate-config.pl
+++ b/nixos/modules/installer/tools/nixos-generate-config.pl
@@ -279,7 +279,7 @@ if (`lsblk -o TYPE` =~ "lvm") {
     push @initrdKernelModules, "dm-snapshot";
 }
 
-my $virt = `systemd-detect-virt`;
+my $virt = `@detectvirt@`;
 chomp $virt;
 
 
@@ -398,7 +398,7 @@ foreach my $fs (read_file("/proc/self/mountinfo")) {
     # Maybe this is a bind-mount of a filesystem we saw earlier?
     if (defined $fsByDev{$fields[2]}) {
         # Make sure this isn't a btrfs subvolume.
-        my $msg = `btrfs subvol show $rootDir$mountPoint`;
+        my $msg = `@btrfs@ subvol show $rootDir$mountPoint`;
         if ($? != 0 || $msg =~ /ERROR:/s) {
             my $path = $fields[3]; $path = "" if $path eq "/";
             my $base = $fsByDev{$fields[2]};
@@ -436,7 +436,7 @@ EOF
 
     # Is this a btrfs filesystem?
     if ($fsType eq "btrfs") {
-        my ($status, @info) = runCommand("btrfs subvol show $rootDir$mountPoint");
+        my ($status, @info) = runCommand("@btrfs@ subvol show $rootDir$mountPoint");
         if ($status != 0 || join("", @info) =~ /ERROR:/) {
             die "Failed to retrieve subvolume info for $mountPoint\n";
         }
@@ -558,6 +558,8 @@ if (!$noFilesystems) {
     $fsAndSwap .= "swapDevices =" . multiLineList("    ", @swapDevices) . ";\n";
 }
 
+my $networkingDhcpConfig = generateNetworkingDhcpConfig();
+
 my $hwConfig = <<EOF;
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
@@ -572,6 +574,7 @@ my $hwConfig = <<EOF;
   boot.kernelModules = [$kernelModules ];
   boot.extraModulePackages = [$modulePackages ];
 $fsAndSwap
+$networkingDhcpConfig
 ${\join "", (map { "  $_\n" } (uniq @attrs))}}
 EOF
 
@@ -580,13 +583,13 @@ sub generateNetworkingDhcpConfig {
   # The global useDHCP flag is deprecated, therefore explicitly set to false here.
   # Per-interface useDHCP will be mandatory in the future, so this generated config
   # replicates the default behaviour.
-  networking.useDHCP = false;
+  networking.useDHCP = lib.mkDefault false;
 EOF
 
     foreach my $path (glob "/sys/class/net/*") {
         my $dev = basename($path);
         if ($dev ne "lo") {
-            $config .= "  networking.interfaces.$dev.useDHCP = true;\n";
+            $config .= "  networking.interfaces.$dev.useDHCP = lib.mkDefault true;\n";
         }
     }
 
diff --git a/nixos/modules/installer/tools/nixos-install.sh b/nixos/modules/installer/tools/nixos-install.sh
index fc4a69aa17d3..e7cf52f5e32b 100644
--- a/nixos/modules/installer/tools/nixos-install.sh
+++ b/nixos/modules/installer/tools/nixos-install.sh
@@ -143,6 +143,23 @@ export TMPDIR=${TMPDIR:-$tmpdir}
 
 sub="auto?trusted=1"
 
+# Copy the NixOS/Nixpkgs sources to the target as the initial contents
+# of the NixOS channel.
+if [[ -z $noChannelCopy ]]; then
+    if [[ -z $channelPath ]]; then
+        channelPath="$(nix-env -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")"
+    fi
+    if [[ -n $channelPath ]]; then
+        echo "copying channel..."
+        mkdir -p "$mountPoint"/nix/var/nix/profiles/per-user/root
+        nix-env --store "$mountPoint" "${extraBuildFlags[@]}" --extra-substituters "$sub" \
+                -p "$mountPoint"/nix/var/nix/profiles/per-user/root/channels --set "$channelPath" --quiet \
+                "${verbosity[@]}"
+        install -m 0700 -d "$mountPoint"/root/.nix-defexpr
+        ln -sfn /nix/var/nix/profiles/per-user/root/channels "$mountPoint"/root/.nix-defexpr/channels
+    fi
+fi
+
 # Build the system configuration in the target filesystem.
 if [[ -z $system ]]; then
     outLink="$tmpdir/system"
@@ -167,23 +184,6 @@ nix-env --store "$mountPoint" "${extraBuildFlags[@]}" \
         --extra-substituters "$sub" \
         -p "$mountPoint"/nix/var/nix/profiles/system --set "$system" "${verbosity[@]}"
 
-# Copy the NixOS/Nixpkgs sources to the target as the initial contents
-# of the NixOS channel.
-if [[ -z $noChannelCopy ]]; then
-    if [[ -z $channelPath ]]; then
-        channelPath="$(nix-env -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")"
-    fi
-    if [[ -n $channelPath ]]; then
-        echo "copying channel..."
-        mkdir -p "$mountPoint"/nix/var/nix/profiles/per-user/root
-        nix-env --store "$mountPoint" "${extraBuildFlags[@]}" --extra-substituters "$sub" \
-                -p "$mountPoint"/nix/var/nix/profiles/per-user/root/channels --set "$channelPath" --quiet \
-                "${verbosity[@]}"
-        install -m 0700 -d "$mountPoint"/root/.nix-defexpr
-        ln -sfn /nix/var/nix/profiles/per-user/root/channels "$mountPoint"/root/.nix-defexpr/channels
-    fi
-fi
-
 # Mark the target as a NixOS installation, otherwise switch-to-configuration will chicken out.
 mkdir -m 0755 -p "$mountPoint/etc"
 touch "$mountPoint/etc/NIXOS"
diff --git a/nixos/modules/installer/tools/tools.nix b/nixos/modules/installer/tools/tools.nix
index 2f3b0cdd48f2..71aaf7f253d9 100644
--- a/nixos/modules/installer/tools/tools.nix
+++ b/nixos/modules/installer/tools/tools.nix
@@ -33,8 +33,9 @@ let
   nixos-generate-config = makeProg {
     name = "nixos-generate-config";
     src = ./nixos-generate-config.pl;
-    path = lib.optionals (lib.elem "btrfs" config.boot.supportedFilesystems) [ pkgs.btrfs-progs ];
     perl = "${pkgs.perl.withPackages (p: [ p.FileSlurp ])}/bin/perl";
+    detectvirt = "${pkgs.systemd}/bin/systemd-detect-virt";
+    btrfs = "${pkgs.btrfs-progs}/bin/btrfs";
     inherit (config.system.nixos-generate-config) configuration desktopConfiguration;
     xserverEnabled = config.services.xserver.enable;
   };
@@ -133,12 +134,13 @@ in
 
       $bootLoaderConfig
         # networking.hostName = "nixos"; # Define your hostname.
+        # Pick only one of the below networking options.
         # networking.wireless.enable = true;  # Enables wireless support via wpa_supplicant.
+        # networking.networkmanager.enable = true;  # Easiest to use and most distros use this by default.
 
         # Set your time zone.
         # time.timeZone = "Europe/Amsterdam";
 
-      $networkingDhcpConfig
         # Configure network proxy if necessary
         # networking.proxy.default = "http://user:password\@proxy:port/";
         # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain";
@@ -148,6 +150,7 @@ in
         # console = {
         #   font = "Lat2-Terminus16";
         #   keyMap = "us";
+        #   useXkbConfig = true; # use xkbOptions in tty.
         # };
 
       $xserverConfig
@@ -155,7 +158,10 @@ in
       $desktopConfiguration
         # Configure keymap in X11
         # services.xserver.layout = "us";
-        # services.xserver.xkbOptions = "eurosign:e";
+        # services.xserver.xkbOptions = {
+        #   "eurosign:e";
+        #   "caps:escape" # map caps to escape.
+        # };
 
         # Enable CUPS to print documents.
         # services.printing.enable = true;
diff --git a/nixos/modules/installer/virtualbox-demo.nix b/nixos/modules/installer/virtualbox-demo.nix
index 2768e17590b3..27a7651382b2 100644
--- a/nixos/modules/installer/virtualbox-demo.nix
+++ b/nixos/modules/installer/virtualbox-demo.nix
@@ -25,7 +25,7 @@ with lib;
 
   installer.cloneConfigExtra = ''
   # Let demo build as a trusted user.
-  # nix.trustedUsers = [ "demo" ];
+  # nix.settings.trusted-users = [ "demo" ];
 
   # Mount a VirtualBox shared folder.
   # This is configurable in the VirtualBox menu at
diff --git a/nixos/modules/misc/documentation.nix b/nixos/modules/misc/documentation.nix
index 64b1c15086fc..9304c307af2f 100644
--- a/nixos/modules/misc/documentation.nix
+++ b/nixos/modules/misc/documentation.nix
@@ -1,19 +1,35 @@
-{ config, lib, pkgs, extendModules, noUserModules, ... }:
+{ config, options, lib, pkgs, utils, modules, baseModules, extraModules, modulesPath, ... }:
 
 with lib;
 
 let
 
   cfg = config.documentation;
+  allOpts = options;
 
   /* Modules for which to show options even when not imported. */
   extraDocModules = [ ../virtualisation/qemu-vm.nix ];
 
-  /* For the purpose of generating docs, evaluate options with each derivation
-    in `pkgs` (recursively) replaced by a fake with path "\${pkgs.attribute.path}".
-    It isn't perfect, but it seems to cover a vast majority of use cases.
-    Caveat: even if the package is reached by a different means,
-    the path above will be shown and not e.g. `${config.services.foo.package}`. */
+  canCacheDocs = m:
+    let
+      f = import m;
+      instance = f (mapAttrs (n: _: abort "evaluating ${n} for `meta` failed") (functionArgs f));
+    in
+      cfg.nixos.options.splitBuild
+        && builtins.isPath m
+        && isFunction f
+        && instance ? options
+        && instance.meta.buildDocsInSandbox or true;
+
+  docModules =
+    let
+      p = partition canCacheDocs (baseModules ++ extraDocModules);
+    in
+      {
+        lazy = p.right;
+        eager = p.wrong ++ optionals cfg.nixos.includeAllModules (extraModules ++ modules);
+      };
+
   manual = import ../../doc/manual rec {
     inherit pkgs config;
     version = config.system.nixos.release;
@@ -21,10 +37,17 @@ let
     extraSources = cfg.nixos.extraModuleSources;
     options =
       let
-        extendNixOS = if cfg.nixos.includeAllModules then extendModules else noUserModules.extendModules;
-        scrubbedEval = extendNixOS {
-          modules = extraDocModules;
-          specialArgs.pkgs = scrubDerivations "pkgs" pkgs;
+        scrubbedEval = evalModules {
+          modules = [ {
+            _module.check = false;
+          } ] ++ docModules.eager;
+          specialArgs = {
+            pkgs = scrubDerivations "pkgs" pkgs;
+            # allow access to arbitrary options for eager modules, eg for getting
+            # option types from lazy modules
+            options = allOpts;
+            inherit modulesPath utils;
+          };
         };
         scrubDerivations = namePrefix: pkgSet: mapAttrs
           (name: value:
@@ -36,6 +59,49 @@ let
           )
           pkgSet;
       in scrubbedEval.options;
+    baseOptionsJSON =
+      let
+        filter =
+          builtins.filterSource
+            (n: t:
+              (t == "directory" -> baseNameOf n != "tests")
+              && (t == "file" -> hasSuffix ".nix" n)
+            );
+      in
+        pkgs.runCommand "lazy-options.json" {
+          libPath = filter "${toString pkgs.path}/lib";
+          pkgsLibPath = filter "${toString pkgs.path}/pkgs/pkgs-lib";
+          nixosPath = filter "${toString pkgs.path}/nixos";
+          modules = map (p: ''"${removePrefix "${modulesPath}/" (toString p)}"'') docModules.lazy;
+        } ''
+          export NIX_STORE_DIR=$TMPDIR/store
+          export NIX_STATE_DIR=$TMPDIR/state
+          ${pkgs.buildPackages.nix}/bin/nix-instantiate \
+            --show-trace \
+            --eval --json --strict \
+            --argstr libPath "$libPath" \
+            --argstr pkgsLibPath "$pkgsLibPath" \
+            --argstr nixosPath "$nixosPath" \
+            --arg modules "[ $modules ]" \
+            --argstr stateVersion "${options.system.stateVersion.default}" \
+            --argstr release "${config.system.nixos.release}" \
+            $nixosPath/lib/eval-cacheable-options.nix > $out \
+            || {
+              echo -en "\e[1;31m"
+              echo 'Cacheable portion of option doc build failed.'
+              echo 'Usually this means that an option attribute that ends up in documentation (eg' \
+                '`default` or `description`) depends on the restricted module arguments' \
+                '`config` or `pkgs`.'
+              echo
+              echo 'Rebuild your configuration with `--show-trace` to find the offending' \
+                'location. Remove the references to restricted arguments (eg by escaping' \
+                'their antiquotations or adding a `defaultText`) or disable the sandboxed' \
+                'build for the failing module by setting `meta.buildDocsInSandbox = false`.'
+              echo -en "\e[0m"
+              exit 1
+            } >&2
+        '';
+    inherit (cfg.nixos.options) warningsAreErrors;
   };
 
 
@@ -63,7 +129,7 @@ let
       genericName = "View NixOS documentation in a web browser";
       icon = "nix-snowflake";
       exec = "nixos-help";
-      categories = "System";
+      categories = ["System"];
     };
 
     in pkgs.symlinkJoin {
@@ -74,10 +140,6 @@ let
       ];
     };
 
-  # list of man outputs currently active intended for use as default values
-  # for man-related options, thus "man" is included unconditionally.
-  activeManOutputs = [ "man" ] ++ lib.optionals cfg.dev.enable [ "devman" ];
-
 in
 
 {
@@ -107,8 +169,8 @@ in
         type = types.bool;
         default = true;
         description = ''
-          Whether to install manual pages and the <command>man</command> command.
-          This also includes "man" outputs.
+          Whether to install manual pages.
+          This also includes <literal>man</literal> outputs.
         '';
       };
 
@@ -116,27 +178,18 @@ in
         type = types.bool;
         default = false;
         description = ''
-          Whether to generate the manual page index caches using
-          <literal>mandb(8)</literal>. This allows searching for a page or
-          keyword using utilities like <literal>apropos(1)</literal>.
-        '';
-      };
-
-      man.manualPages = mkOption {
-        type = types.path;
-        default = pkgs.buildEnv {
-          name = "man-paths";
-          paths = config.environment.systemPackages;
-          pathsToLink = [ "/share/man" ];
-          extraOutputsToInstall = activeManOutputs;
-          ignoreCollisions = true;
-        };
-        defaultText = literalDocBook "all man pages in <option>config.environment.systemPackages</option>";
-        description = ''
-          The manual pages to generate caches for if <option>generateCaches</option>
-          is enabled. Must be a path to a directory with man pages under
-          <literal>/share/man</literal>; see the source for an example.
-          Advanced users can make this a content-addressed derivation to save a few rebuilds.
+          Whether to generate the manual page index caches.
+          This allows searching for a page or
+          keyword using utilities like
+          <citerefentry>
+            <refentrytitle>apropos</refentrytitle>
+            <manvolnum>1</manvolnum>
+          </citerefentry>
+          and the <literal>-k</literal> option of
+          <citerefentry>
+            <refentrytitle>man</refentrytitle>
+            <manvolnum>1</manvolnum>
+          </citerefentry>.
         '';
       };
 
@@ -191,6 +244,25 @@ in
         '';
       };
 
+      nixos.options.splitBuild = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Whether to split the option docs build into a cacheable and an uncacheable part.
+          Splitting the build can substantially decrease the amount of time needed to build
+          the manual, but some user modules may be incompatible with this splitting.
+        '';
+      };
+
+      nixos.options.warningsAreErrors = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Treat warning emitted during the option documentation build (eg for missing option
+          descriptions) as errors.
+        '';
+      };
+
       nixos.includeAllModules = mkOption {
         type = types.bool;
         default = false;
@@ -220,30 +292,22 @@ in
   };
 
   config = mkIf cfg.enable (mkMerge [
+    {
+      assertions = [
+        {
+          assertion = !(cfg.man.man-db.enable && cfg.man.mandoc.enable);
+          message = ''
+            man-db and mandoc can't be used as the default man page viewer at the same time!
+          '';
+        }
+      ];
+    }
 
+    # The actual implementation for this lives in man-db.nix or mandoc.nix,
+    # depending on which backend is active.
     (mkIf cfg.man.enable {
-      environment.systemPackages = [ pkgs.man-db ];
       environment.pathsToLink = [ "/share/man" ];
-      environment.extraOutputsToInstall = activeManOutputs;
-      environment.etc."man_db.conf".text =
-        let
-          manualCache = pkgs.runCommandLocal "man-cache" { } ''
-            echo "MANDB_MAP ${cfg.man.manualPages}/share/man $out" > man.conf
-            ${pkgs.man-db}/bin/mandb -C man.conf -psc >/dev/null 2>&1
-          '';
-        in
-        ''
-          # Manual pages paths for NixOS
-          MANPATH_MAP /run/current-system/sw/bin /run/current-system/sw/share/man
-          MANPATH_MAP /run/wrappers/bin          /run/current-system/sw/share/man
-
-          ${optionalString cfg.man.generateCaches ''
-          # Generated manual pages cache for NixOS (immutable)
-          MANDB_MAP /run/current-system/sw/share/man ${manualCache}
-          ''}
-          # Manual pages caches for NixOS
-          MANDB_MAP /run/current-system/sw/share/man /var/cache/man/nixos
-        '';
+      environment.extraOutputsToInstall = [ "man" ] ++ optional cfg.dev.enable "devman";
     })
 
     (mkIf cfg.info.enable {
diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix
index a9f2031d1e1a..7d1faa50f4bf 100644
--- a/nixos/modules/misc/ids.nix
+++ b/nixos/modules/misc/ids.nix
@@ -82,14 +82,14 @@ in
       git = 41;
       #fourstore = 42; # dropped in 20.03
       #fourstorehttp = 43; # dropped in 20.03
-      virtuoso = 44;
+      #virtuoso = 44;  dropped module
       #rtkit = 45; # dynamically allocated 2021-09-03
       dovecot2 = 46;
       dovenull2 = 47;
       prayer = 49;
       mpd = 50;
       clamav = 51;
-      fprot = 52;
+      #fprot = 52; # unused
       # bind = 53; #dynamically allocated as of 2021-09-03
       wwwrun = 54;
       #adm = 55; # unused
@@ -182,7 +182,7 @@ in
       yandexdisk = 143;
       mxisd = 144; # was once collectd
       #consul = 145;# dynamically allocated as of 2021-09-03
-      mailpile = 146;
+      #mailpile = 146; # removed 2022-01-12
       redmine = 147;
       #seeks = 148; # removed 2020-06-21
       prosody = 149;
@@ -296,7 +296,7 @@ in
       infinoted = 264;
       sickbeard = 265;
       headphones = 266;
-      couchpotato = 267;
+      # couchpotato = 267; # unused, removed 2022-01-01
       gogs = 268;
       #pdns-recursor = 269; # dynamically allocated as of 2020-20-18
       #kresd = 270; # switched to "knot-resolver" with dynamic ID
@@ -352,6 +352,8 @@ in
       moonraker = 320;
       distcc = 321;
       webdav = 322;
+      pipewire = 323;
+      rstudio-server = 324;
 
       # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
 
@@ -410,7 +412,7 @@ in
       prayer = 49;
       mpd = 50;
       clamav = 51;
-      fprot = 52;
+      #fprot = 52; # unused
       #bind = 53; # unused
       wwwrun = 54;
       adm = 55;
@@ -501,7 +503,7 @@ in
       #yandexdisk = 143; # unused
       mxisd = 144; # was once collectd
       #consul = 145; # unused
-      mailpile = 146;
+      #mailpile = 146; # removed 2022-01-12
       redmine = 147;
       #seeks = 148; # removed 2020-06-21
       prosody = 149;
@@ -603,7 +605,7 @@ in
       infinoted = 264;
       sickbeard = 265;
       headphones = 266;
-      couchpotato = 267;
+      # couchpotato = 267; # unused, removed 2022-01-01
       gogs = 268;
       #kresd = 270; # switched to "knot-resolver" with dynamic ID
       #rpc = 271; # unused
@@ -658,6 +660,8 @@ in
       moonraker = 320;
       distcc = 321;
       webdav = 322;
+      pipewire = 323;
+      rstudio-server = 324;
 
       # When adding a gid, make sure it doesn't match an existing
       # uid. Users and groups with the same name should have equal
diff --git a/nixos/modules/misc/locate.nix b/nixos/modules/misc/locate.nix
index 5fd82aa963bf..204a89143008 100644
--- a/nixos/modules/misc/locate.nix
+++ b/nixos/modules/misc/locate.nix
@@ -5,11 +5,14 @@ with lib;
 let
   cfg = config.services.locate;
   isMLocate = hasPrefix "mlocate" cfg.locate.name;
+  isPLocate = hasPrefix "plocate" cfg.locate.name;
+  isMorPLocate = (isMLocate || isPLocate);
   isFindutils = hasPrefix "findutils" cfg.locate.name;
-in {
+in
+{
   imports = [
     (mkRenamedOptionModule [ "services" "locate" "period" ] [ "services" "locate" "interval" ])
-    (mkRemovedOptionModule [ "services" "locate" "includeStore" ] "Use services.locate.prunePaths" )
+    (mkRemovedOptionModule [ "services" "locate" "includeStore" ] "Use services.locate.prunePaths")
   ];
 
   options.services.locate = with types; {
@@ -163,7 +166,16 @@ in {
 
     prunePaths = mkOption {
       type = listOf path;
-      default = [ "/tmp" "/var/tmp" "/var/cache" "/var/lock" "/var/run" "/var/spool" "/nix/store" "/nix/var/log/nix" ];
+      default = [
+        "/tmp"
+        "/var/tmp"
+        "/var/cache"
+        "/var/lock"
+        "/var/run"
+        "/var/spool"
+        "/nix/store"
+        "/nix/var/log/nix"
+      ];
       description = ''
         Which paths to exclude from indexing
       '';
@@ -171,7 +183,11 @@ in {
 
     pruneNames = mkOption {
       type = listOf str;
-      default = [ ".bzr" ".cache" ".git" ".hg" ".svn" ];
+      default = lib.optionals (!isFindutils) [ ".bzr" ".cache" ".git" ".hg" ".svn" ];
+      defaultText = literalDocBook ''
+        <literal>[ ".bzr" ".cache" ".git" ".hg" ".svn" ]</literal>, if
+        supported by the locate implementation (i.e. mlocate or plocate).
+      '';
       description = ''
         Directory components which should exclude paths containing them from indexing
       '';
@@ -188,26 +204,38 @@ in {
   };
 
   config = mkIf cfg.enable {
-    users.groups = mkIf isMLocate { mlocate = {}; };
+    users.groups = mkMerge [
+      (mkIf isMLocate { mlocate = { }; })
+      (mkIf isPLocate { plocate = { }; })
+    ];
 
-    security.wrappers = mkIf isMLocate {
-      locate = {
-        group = "mlocate";
-        owner = "root";
-        permissions = "u+rx,g+x,o+x";
-        setgid = true;
-        setuid = false;
-        source = "${cfg.locate}/bin/locate";
+    security.wrappers =
+      let
+        common = {
+          owner = "root";
+          permissions = "u+rx,g+x,o+x";
+          setgid = true;
+          setuid = false;
+        };
+        mlocate = (mkIf isMLocate {
+          group = "mlocate";
+          source = "${cfg.locate}/bin/locate";
+        });
+        plocate = (mkIf isPLocate {
+          group = "plocate";
+          source = "${cfg.locate}/bin/plocate";
+        });
+      in
+      mkIf isMorPLocate {
+        locate = mkMerge [ common mlocate plocate ];
+        plocate = (mkIf isPLocate (mkMerge [ common plocate ]));
       };
-    };
 
     nixpkgs.config = { locate.dbfile = cfg.output; };
 
     environment.systemPackages = [ cfg.locate ];
 
-    environment.variables = mkIf (!isMLocate)
-      { LOCATE_PATH = cfg.output;
-      };
+    environment.variables = mkIf (!isMorPLocate) { LOCATE_PATH = cfg.output; };
 
     environment.etc = {
       # write /etc/updatedb.conf for manual calls to `updatedb`
@@ -221,57 +249,65 @@ in {
       };
     };
 
-    warnings = optional (isMLocate && cfg.localuser != null) "mlocate does not support the services.locate.localuser option; updatedb will run as root. (Silence with services.locate.localuser = null.)"
-            ++ optional (isFindutils && cfg.pruneNames != []) "findutils locate does not support pruning by directory component"
-            ++ optional (isFindutils && cfg.pruneBindMounts) "findutils locate does not support skipping bind mounts";
+    warnings = optional (isMorPLocate && cfg.localuser != null)
+      "mlocate does not support the services.locate.localuser option; updatedb will run as root. (Silence with services.locate.localuser = null.)"
+    ++ optional (isFindutils && cfg.pruneNames != [ ])
+      "findutils locate does not support pruning by directory component"
+    ++ optional (isFindutils && cfg.pruneBindMounts)
+      "findutils locate does not support skipping bind mounts";
 
-    systemd.services.update-locatedb =
-      { description = "Update Locate Database";
-        path = mkIf (!isMLocate) [ pkgs.su ];
+    systemd.services.update-locatedb = {
+      description = "Update Locate Database";
+      path = mkIf (!isMorPLocate) [ pkgs.su ];
 
-        # mlocate's updatedb takes flags via a configuration file or
-        # on the command line, but not by environment variable.
-        script =
-          if isMLocate
-          then let toFlags = x: optional (cfg.${x} != [])
-                                         "--${lib.toLower x} '${concatStringsSep " " cfg.${x}}'";
-                   args = concatLists (map toFlags ["pruneFS" "pruneNames" "prunePaths"]);
-               in ''
+      # mlocate's updatedb takes flags via a configuration file or
+      # on the command line, but not by environment variable.
+      script =
+        if isMorPLocate then
+          let
+            toFlags = x:
+              optional (cfg.${x} != [ ])
+                "--${lib.toLower x} '${concatStringsSep " " cfg.${x}}'";
+            args = concatLists (map toFlags [ "pruneFS" "pruneNames" "prunePaths" ]);
+          in
+          ''
             exec ${cfg.locate}/bin/updatedb \
               --output ${toString cfg.output} ${concatStringsSep " " args} \
               --prune-bind-mounts ${if cfg.pruneBindMounts then "yes" else "no"} \
               ${concatStringsSep " " cfg.extraFlags}
           ''
-          else ''
-            exec ${cfg.locate}/bin/updatedb \
-              ${optionalString (cfg.localuser != null && ! isMLocate) "--localuser=${cfg.localuser}"} \
-              --output=${toString cfg.output} ${concatStringsSep " " cfg.extraFlags}
-          '';
-        environment = optionalAttrs (!isMLocate) {
-          PRUNEFS = concatStringsSep " " cfg.pruneFS;
-          PRUNEPATHS = concatStringsSep " " cfg.prunePaths;
-          PRUNENAMES = concatStringsSep " " cfg.pruneNames;
-          PRUNE_BIND_MOUNTS = if cfg.pruneBindMounts then "yes" else "no";
-        };
-        serviceConfig.Nice = 19;
-        serviceConfig.IOSchedulingClass = "idle";
-        serviceConfig.PrivateTmp = "yes";
-        serviceConfig.PrivateNetwork = "yes";
-        serviceConfig.NoNewPrivileges = "yes";
-        serviceConfig.ReadOnlyPaths = "/";
-        # Use dirOf cfg.output because mlocate creates temporary files next to
-        # the actual database. We could specify and create them as well,
-        # but that would make this quite brittle when they change something.
-        # NOTE: If /var/cache does not exist, this leads to the misleading error message:
-        # update-locatedb.service: Failed at step NAMESPACE spawning …/update-locatedb-start: No such file or directory
-        serviceConfig.ReadWritePaths = dirOf cfg.output;
+        else ''
+          exec ${cfg.locate}/bin/updatedb \
+            ${optionalString (cfg.localuser != null && !isMorPLocate) "--localuser=${cfg.localuser}"} \
+            --output=${toString cfg.output} ${concatStringsSep " " cfg.extraFlags}
+        '';
+      environment = optionalAttrs (!isMorPLocate) {
+        PRUNEFS = concatStringsSep " " cfg.pruneFS;
+        PRUNEPATHS = concatStringsSep " " cfg.prunePaths;
+        PRUNENAMES = concatStringsSep " " cfg.pruneNames;
+        PRUNE_BIND_MOUNTS = if cfg.pruneBindMounts then "yes" else "no";
       };
+      serviceConfig.Nice = 19;
+      serviceConfig.IOSchedulingClass = "idle";
+      serviceConfig.PrivateTmp = "yes";
+      serviceConfig.PrivateNetwork = "yes";
+      serviceConfig.NoNewPrivileges = "yes";
+      serviceConfig.ReadOnlyPaths = "/";
+      # Use dirOf cfg.output because mlocate creates temporary files next to
+      # the actual database. We could specify and create them as well,
+      # but that would make this quite brittle when they change something.
+      # NOTE: If /var/cache does not exist, this leads to the misleading error message:
+      # update-locatedb.service: Failed at step NAMESPACE spawning …/update-locatedb-start: No such file or directory
+      serviceConfig.ReadWritePaths = dirOf cfg.output;
+    };
 
-    systemd.timers.update-locatedb = mkIf (cfg.interval != "never")
-      { description = "Update timer for locate database";
-        partOf      = [ "update-locatedb.service" ];
-        wantedBy    = [ "timers.target" ];
-        timerConfig.OnCalendar = cfg.interval;
-      };
+    systemd.timers.update-locatedb = mkIf (cfg.interval != "never") {
+      description = "Update timer for locate database";
+      partOf = [ "update-locatedb.service" ];
+      wantedBy = [ "timers.target" ];
+      timerConfig.OnCalendar = cfg.interval;
+    };
   };
+
+  meta.maintainers = with lib.maintainers; [ SuperSandro2000 ];
 }
diff --git a/nixos/modules/misc/man-db.nix b/nixos/modules/misc/man-db.nix
new file mode 100644
index 000000000000..8bd329bc4e0c
--- /dev/null
+++ b/nixos/modules/misc/man-db.nix
@@ -0,0 +1,73 @@
+{ config, pkgs, lib, ... }:
+
+let
+  cfg = config.documentation.man.man-db;
+in
+
+{
+  options = {
+    documentation.man.man-db = {
+      enable = lib.mkEnableOption "man-db as the default man page viewer" // {
+        default = config.documentation.man.enable;
+        defaultText = lib.literalExpression "config.documentation.man.enable";
+        example = false;
+      };
+
+      manualPages = lib.mkOption {
+        type = lib.types.path;
+        default = pkgs.buildEnv {
+          name = "man-paths";
+          paths = config.environment.systemPackages;
+          pathsToLink = [ "/share/man" ];
+          extraOutputsToInstall = [ "man" ]
+            ++ lib.optionals config.documentation.dev.enable [ "devman" ];
+          ignoreCollisions = true;
+        };
+        defaultText = lib.literalDocBook "all man pages in <option>config.environment.systemPackages</option>";
+        description = ''
+          The manual pages to generate caches for if <option>documentation.man.generateCaches</option>
+          is enabled. Must be a path to a directory with man pages under
+          <literal>/share/man</literal>; see the source for an example.
+          Advanced users can make this a content-addressed derivation to save a few rebuilds.
+        '';
+      };
+
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.man-db;
+        defaultText = lib.literalExpression "pkgs.man-db";
+        description = ''
+          The <literal>man-db</literal> derivation to use. Useful to override
+          configuration options used for the package.
+        '';
+      };
+    };
+  };
+
+  imports = [
+    (lib.mkRenamedOptionModule [ "documentation" "man" "manualPages" ] [ "documentation" "man" "man-db" "manualPages" ])
+  ];
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+    environment.etc."man_db.conf".text =
+      let
+        manualCache = pkgs.runCommandLocal "man-cache" { } ''
+          echo "MANDB_MAP ${cfg.manualPages}/share/man $out" > man.conf
+          ${cfg.package}/bin/mandb -C man.conf -psc >/dev/null 2>&1
+        '';
+      in
+      ''
+        # Manual pages paths for NixOS
+        MANPATH_MAP /run/current-system/sw/bin /run/current-system/sw/share/man
+        MANPATH_MAP /run/wrappers/bin          /run/current-system/sw/share/man
+
+        ${lib.optionalString config.documentation.man.generateCaches ''
+        # Generated manual pages cache for NixOS (immutable)
+        MANDB_MAP /run/current-system/sw/share/man ${manualCache}
+        ''}
+        # Manual pages caches for NixOS
+        MANDB_MAP /run/current-system/sw/share/man /var/cache/man/nixos
+      '';
+  };
+}
diff --git a/nixos/modules/misc/mandoc.nix b/nixos/modules/misc/mandoc.nix
new file mode 100644
index 000000000000..3da60f2f8e65
--- /dev/null
+++ b/nixos/modules/misc/mandoc.nix
@@ -0,0 +1,61 @@
+{ config, lib, pkgs, ... }:
+
+let
+  makewhatis = "${lib.getBin cfg.package}/bin/makewhatis";
+
+  cfg = config.documentation.man.mandoc;
+
+in {
+  meta.maintainers = [ lib.maintainers.sternenseemann ];
+
+  options = {
+    documentation.man.mandoc = {
+      enable = lib.mkEnableOption "mandoc as the default man page viewer";
+
+      manPath = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ "share/man" ];
+        example = lib.literalExpression "[ \"share/man\" \"share/man/fr\" ]";
+        description = ''
+          Change the manpath, i. e. the directories where
+          <citerefentry><refentrytitle>man</refentrytitle><manvolnum>1</manvolnum></citerefentry>
+          looks for section-specific directories of man pages.
+          You only need to change this setting if you want extra man pages
+          (e. g. in non-english languages). All values must be strings that
+          are a valid path from the target prefix (without including it).
+          The first value given takes priority.
+        '';
+      };
+
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.mandoc;
+        defaultText = lib.literalExpression "pkgs.mandoc";
+        description = ''
+          The <literal>mandoc</literal> derivation to use. Useful to override
+          configuration options used for the package.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment = {
+      systemPackages = [ cfg.package ];
+
+      # tell mandoc about man pages
+      etc."man.conf".text = lib.concatMapStrings (path: ''
+        manpath /run/current-system/sw/${path}
+      '') cfg.manPath;
+
+      # create mandoc.db for whatis(1), apropos(1) and man(1) -k
+      # TODO(@sternenseemman): fix symlinked directories not getting indexed,
+      # see: https://inbox.vuxu.org/mandoc-tech/20210906171231.GF83680@athene.usta.de/T/#e85f773c1781e3fef85562b2794f9cad7b2909a3c
+      extraSetup = lib.mkIf config.documentation.man.generateCaches ''
+        ${makewhatis} -T utf8 ${
+          lib.concatMapStringsSep " " (path: "\"$out/${path}\"") cfg.manPath
+        }
+      '';
+    };
+  };
+}
diff --git a/nixos/modules/misc/meta.nix b/nixos/modules/misc/meta.nix
index 3dd97cbec235..8e689a63f6bf 100644
--- a/nixos/modules/misc/meta.nix
+++ b/nixos/modules/misc/meta.nix
@@ -54,6 +54,21 @@ in
         '';
       };
 
+      buildDocsInSandbox = mkOption {
+        type = types.bool // {
+          merge = loc: defs: defs;
+        };
+        internal = true;
+        default = true;
+        description = ''
+          Whether to include this module in the split options doc build.
+          Disable if the module references `config`, `pkgs` or other module
+          arguments that cannot be evaluated as constants.
+
+          This option should be defined at most once per module.
+        '';
+      };
+
     };
   };
 
diff --git a/nixos/modules/misc/nixpkgs.nix b/nixos/modules/misc/nixpkgs.nix
index 08bc4398555b..69967c8a7601 100644
--- a/nixos/modules/misc/nixpkgs.nix
+++ b/nixos/modules/misc/nixpkgs.nix
@@ -64,6 +64,11 @@ let
 in
 
 {
+  imports = [
+    ./assertions.nix
+    ./meta.nix
+  ];
+
   options.nixpkgs = {
 
     pkgs = mkOption {
@@ -248,4 +253,7 @@ in
       )
     ];
   };
+
+  # needs a full nixpkgs path to import nixpkgs
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/misc/nixpkgs/test.nix b/nixos/modules/misc/nixpkgs/test.nix
new file mode 100644
index 000000000000..ec5fab9fb4a5
--- /dev/null
+++ b/nixos/modules/misc/nixpkgs/test.nix
@@ -0,0 +1,8 @@
+{ evalMinimalConfig, pkgs, lib, stdenv }:
+lib.recurseIntoAttrs {
+  invokeNixpkgsSimple =
+    (evalMinimalConfig ({ config, modulesPath, ... }: {
+      imports = [ (modulesPath + "/misc/nixpkgs.nix") ];
+      nixpkgs.system = stdenv.hostPlatform.system;
+    }))._module.args.pkgs.hello;
+}
diff --git a/nixos/modules/misc/version.nix b/nixos/modules/misc/version.nix
index 8f246a9278b7..d825f4beb301 100644
--- a/nixos/modules/misc/version.nix
+++ b/nixos/modules/misc/version.nix
@@ -1,11 +1,21 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
+{ config, lib, options, pkgs, ... }:
 
 let
   cfg = config.system.nixos;
-in
+  opt = options.system.nixos;
+
+  inherit (lib)
+    concatStringsSep mapAttrsToList toLower
+    literalExpression mkRenamedOptionModule mkDefault mkOption trivial types;
+
+  needsEscaping = s: null != builtins.match "[a-zA-Z0-9]+" s;
+  escapeIfNeccessary = s: if needsEscaping s then s else ''"${lib.escape [ "\$" "\"" "\\" "\`" ] s}"'';
+  attrsToText = attrs:
+    concatStringsSep "\n" (
+      mapAttrsToList (n: v: ''${n}=${escapeIfNeccessary (toString v)}'') attrs
+    );
 
+in
 {
   imports = [
     (mkRenamedOptionModule [ "system" "nixosVersion" ] [ "system" "nixos" "version" ])
@@ -53,6 +63,7 @@ in
     stateVersion = mkOption {
       type = types.str;
       default = cfg.release;
+      defaultText = literalExpression "config.${opt.release}";
       description = ''
         Every once in a while, a new NixOS release may change
         configuration defaults in a way incompatible with stateful
@@ -99,22 +110,32 @@ in
     # Generate /etc/os-release.  See
     # https://www.freedesktop.org/software/systemd/man/os-release.html for the
     # format.
-    environment.etc.os-release.text =
-      ''
-        NAME=NixOS
-        ID=nixos
-        VERSION="${cfg.release} (${cfg.codeName})"
-        VERSION_CODENAME=${toLower cfg.codeName}
-        VERSION_ID="${cfg.release}"
-        BUILD_ID="${cfg.version}"
-        PRETTY_NAME="NixOS ${cfg.release} (${cfg.codeName})"
-        LOGO="nix-snowflake"
-        HOME_URL="https://nixos.org/"
-        DOCUMENTATION_URL="https://nixos.org/learn.html"
-        SUPPORT_URL="https://nixos.org/community.html"
-        BUG_REPORT_URL="https://github.com/NixOS/nixpkgs/issues"
-      '';
-
+    environment.etc = {
+      "lsb-release".text = attrsToText {
+        LSB_VERSION = "${cfg.release} (${cfg.codeName})";
+        DISTRIB_ID = "nixos";
+        DISTRIB_RELEASE = cfg.release;
+        DISTRIB_CODENAME = toLower cfg.codeName;
+        DISTRIB_DESCRIPTION = "NixOS ${cfg.release} (${cfg.codeName})";
+      };
+
+      "os-release".text = attrsToText {
+        NAME = "NixOS";
+        ID = "nixos";
+        VERSION = "${cfg.release} (${cfg.codeName})";
+        VERSION_CODENAME = toLower cfg.codeName;
+        VERSION_ID = cfg.release;
+        BUILD_ID = cfg.version;
+        PRETTY_NAME = "NixOS ${cfg.release} (${cfg.codeName})";
+        LOGO = "nix-snowflake";
+        HOME_URL = "https://nixos.org/";
+        DOCUMENTATION_URL = "https://nixos.org/learn.html";
+        SUPPORT_URL = "https://nixos.org/community.html";
+        BUG_REPORT_URL = "https://github.com/NixOS/nixpkgs/issues";
+      };
+    };
   };
 
+  # uses version info nixpkgs, which requires a full nixpkgs path
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/misc/wordlist.nix b/nixos/modules/misc/wordlist.nix
new file mode 100644
index 000000000000..988b522d7431
--- /dev/null
+++ b/nixos/modules/misc/wordlist.nix
@@ -0,0 +1,59 @@
+{ config, lib, pkgs, ... }:
+with lib;
+let
+  concatAndSort = name: files: pkgs.runCommand name {} ''
+    awk 1 ${lib.escapeShellArgs files} | sed '{ /^\s*$/d; s/^\s\+//; s/\s\+$// }' | sort | uniq > $out
+  '';
+in
+{
+  options = {
+    environment.wordlist = {
+      enable = mkEnableOption "environment variables for lists of words";
+
+      lists = mkOption {
+        type = types.attrsOf (types.nonEmptyListOf types.path);
+
+        default = {
+          WORDLIST = [ "${pkgs.scowl}/share/dict/words.txt" ];
+        };
+
+        defaultText = literalExpression ''
+          {
+            WORDLIST = [ "''${pkgs.scowl}/share/dict/words.txt" ];
+          }
+        '';
+
+        description = ''
+          A set with the key names being the environment variable you'd like to
+          set and the values being a list of paths to text documents containing
+          lists of words. The various files will be merged, sorted, duplicates
+          removed, and extraneous spacing removed.
+
+          If you have a handful of words that you want to add to an already
+          existing wordlist, you may find `builtins.toFile` useful for this
+          task.
+        '';
+
+        example = literalExpression ''
+          {
+            WORDLIST = [ "''${pkgs.scowl}/share/dict/words.txt" ];
+            AUGMENTED_WORDLIST = [
+              "''${pkgs.scowl}/share/dict/words.txt"
+              "''${pkgs.scowl}/share/dict/words.variants.txt"
+              (builtins.toFile "extra-words" '''
+                desynchonization
+                oobleck''')
+            ];
+          }
+        '';
+      };
+    };
+  };
+
+  config = mkIf config.environment.wordlist.enable {
+    environment.variables =
+      lib.mapAttrs
+        (name: value: "${concatAndSort "wordlist-${name}" value}")
+        config.environment.wordlist.lists;
+  };
+}
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 3cc9ea88e17b..e80c6cf90f54 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -45,6 +45,7 @@
   ./hardware/ckb-next.nix
   ./hardware/cpu/amd-microcode.nix
   ./hardware/cpu/intel-microcode.nix
+  ./hardware/cpu/intel-sgx.nix
   ./hardware/corectrl.nix
   ./hardware/digitalbitbox.nix
   ./hardware/device-tree.nix
@@ -52,6 +53,7 @@
   ./hardware/flirc.nix
   ./hardware/gpgsmartcards.nix
   ./hardware/i2c.nix
+  ./hardware/hackrf.nix
   ./hardware/sensor/hddtemp.nix
   ./hardware/sensor/iio.nix
   ./hardware/keyboard/teck.nix
@@ -89,6 +91,7 @@
   ./hardware/video/switcheroo-control.nix
   ./hardware/video/uvcvideo/default.nix
   ./hardware/video/webcam/facetimehd.nix
+  ./hardware/xone.nix
   ./hardware/xpadneo.nix
   ./i18n/input-method/default.nix
   ./i18n/input-method/fcitx.nix
@@ -107,10 +110,13 @@
   ./misc/lib.nix
   ./misc/label.nix
   ./misc/locate.nix
+  ./misc/man-db.nix
+  ./misc/mandoc.nix
   ./misc/meta.nix
   ./misc/nixpkgs.nix
   ./misc/passthru.nix
   ./misc/version.nix
+  ./misc/wordlist.nix
   ./misc/nixops-autoluks.nix
   ./programs/adb.nix
   ./programs/appgate-sdp.nix
@@ -162,6 +168,8 @@
   ./programs/iftop.nix
   ./programs/iotop.nix
   ./programs/java.nix
+  ./programs/k40-whisperer.nix
+  ./programs/kclock.nix
   ./programs/kdeconnect.nix
   ./programs/kbdlight.nix
   ./programs/less.nix
@@ -172,6 +180,8 @@
   ./programs/msmtp.nix
   ./programs/mtr.nix
   ./programs/nano.nix
+  ./programs/nbd.nix
+  ./programs/nix-ld.nix
   ./programs/neovim.nix
   ./programs/nm-applet.nix
   ./programs/npm.nix
@@ -194,11 +204,11 @@
   ./programs/ssmtp.nix
   ./programs/sysdig.nix
   ./programs/systemtap.nix
+  ./programs/starship.nix
   ./programs/steam.nix
   ./programs/sway.nix
   ./programs/system-config-printer.nix
   ./programs/thefuck.nix
-  ./programs/tilp2.nix
   ./programs/tmux.nix
   ./programs/traceroute.nix
   ./programs/tsm-client.nix
@@ -223,7 +233,7 @@
   ./programs/zsh/zsh-autosuggestions.nix
   ./programs/zsh/zsh-syntax-highlighting.nix
   ./rename.nix
-  ./security/acme.nix
+  ./security/acme
   ./security/apparmor.nix
   ./security/audit.nix
   ./security/auditd.nix
@@ -248,6 +258,7 @@
   ./security/tpm2.nix
   ./services/admin/meshcentral.nix
   ./services/admin/oxidized.nix
+  ./services/admin/pgadmin.nix
   ./services/admin/salt/master.nix
   ./services/admin/salt/minion.nix
   ./services/amqp/activemq/default.nix
@@ -292,10 +303,10 @@
   ./services/backup/znapzend.nix
   ./services/blockchain/ethereum/geth.nix
   ./services/backup/zrepl.nix
+  ./services/cluster/corosync/default.nix
   ./services/cluster/hadoop/default.nix
   ./services/cluster/k3s/default.nix
   ./services/cluster/kubernetes/addons/dns.nix
-  ./services/cluster/kubernetes/addons/dashboard.nix
   ./services/cluster/kubernetes/addon-manager.nix
   ./services/cluster/kubernetes/apiserver.nix
   ./services/cluster/kubernetes/controller-manager.nix
@@ -305,6 +316,7 @@
   ./services/cluster/kubernetes/pki.nix
   ./services/cluster/kubernetes/proxy.nix
   ./services/cluster/kubernetes/scheduler.nix
+  ./services/cluster/pacemaker/default.nix
   ./services/cluster/spark/default.nix
   ./services/computing/boinc/client.nix
   ./services/computing/foldingathome/client.nix
@@ -346,14 +358,12 @@
   ./services/databases/redis.nix
   ./services/databases/riak.nix
   ./services/databases/victoriametrics.nix
-  ./services/databases/virtuoso.nix
   ./services/desktops/accountsservice.nix
   ./services/desktops/bamf.nix
   ./services/desktops/blueman.nix
   ./services/desktops/cpupower-gui.nix
   ./services/desktops/dleyna-renderer.nix
   ./services/desktops/dleyna-server.nix
-  ./services/desktops/pantheon/files.nix
   ./services/desktops/espanso.nix
   ./services/desktops/flatpak.nix
   ./services/desktops/geoclue2.nix
@@ -362,6 +372,7 @@
   ./services/desktops/malcontent.nix
   ./services/desktops/pipewire/pipewire.nix
   ./services/desktops/pipewire/pipewire-media-session.nix
+  ./services/desktops/pipewire/wireplumber.nix
   ./services/desktops/gnome/at-spi2-core.nix
   ./services/desktops/gnome/chrome-gnome-shell.nix
   ./services/desktops/gnome/evolution-data-server.nix
@@ -389,11 +400,14 @@
   ./services/development/hoogle.nix
   ./services/development/jupyter/default.nix
   ./services/development/jupyterhub/default.nix
+  ./services/development/rstudio-server/default.nix
   ./services/development/lorri.nix
+  ./services/development/zammad.nix
   ./services/display-managers/greetd.nix
   ./services/editors/emacs.nix
   ./services/editors/infinoted.nix
   ./services/finance/odoo.nix
+  ./services/games/asf.nix
   ./services/games/crossfire-server.nix
   ./services/games/deliantra-server.nix
   ./services/games/factorio.nix
@@ -445,8 +459,11 @@
   ./services/hardware/undervolt.nix
   ./services/hardware/vdr.nix
   ./services/hardware/xow.nix
+  ./services/home-automation/home-assistant.nix
+  ./services/home-automation/zigbee2mqtt.nix
   ./services/logging/SystemdJournal2Gelf.nix
   ./services/logging/awstats.nix
+  ./services/logging/filebeat.nix
   ./services/logging/fluentd.nix
   ./services/logging/graylog.nix
   ./services/logging/heartbeat.nix
@@ -488,6 +505,7 @@
   ./services/mail/roundcube.nix
   ./services/mail/sympa.nix
   ./services/mail/nullmailer.nix
+  ./services/matrix/matrix-synapse.nix
   ./services/matrix/mjolnir.nix
   ./services/matrix/pantalaimon.nix
   ./services/misc/ananicy.nix
@@ -508,7 +526,6 @@
   ./services/misc/cpuminer-cryptonight.nix
   ./services/misc/cgminer.nix
   ./services/misc/confd.nix
-  ./services/misc/couchpotato.nix
   ./services/misc/dendrite.nix
   ./services/misc/devmon.nix
   ./services/misc/dictd.nix
@@ -538,9 +555,10 @@
   ./services/misc/gollum.nix
   ./services/misc/gpsd.nix
   ./services/misc/headphones.nix
+  ./services/misc/heisenbridge.nix
   ./services/misc/greenclip.nix
-  ./services/misc/home-assistant.nix
   ./services/misc/ihaskell.nix
+  ./services/misc/input-remapper.nix
   ./services/misc/irkerd.nix
   ./services/misc/jackett.nix
   ./services/misc/jellyfin.nix
@@ -553,14 +571,13 @@
   ./services/misc/mame.nix
   ./services/misc/matrix-appservice-discord.nix
   ./services/misc/matrix-appservice-irc.nix
-  ./services/misc/matrix-synapse.nix
+  ./services/misc/matrix-conduit.nix
   ./services/misc/mautrix-facebook.nix
   ./services/misc/mautrix-telegram.nix
   ./services/misc/mbpfan.nix
   ./services/misc/mediatomb.nix
   ./services/misc/metabase.nix
   ./services/misc/moonraker.nix
-  ./services/misc/mwlib.nix
   ./services/misc/mx-puppet-discord.nix
   ./services/misc/n8n.nix
   ./services/misc/nitter.nix
@@ -589,6 +606,7 @@
   ./services/misc/redmine.nix
   ./services/misc/rippled.nix
   ./services/misc/ripple-data-api.nix
+  ./services/misc/rmfakecloud.nix
   ./services/misc/serviio.nix
   ./services/misc/safeeyes.nix
   ./services/misc/sdrplay.nix
@@ -614,7 +632,6 @@
   ./services/misc/weechat.nix
   ./services/misc/xmr-stak.nix
   ./services/misc/xmrig.nix
-  ./services/misc/zigbee2mqtt.nix
   ./services/misc/zoneminder.nix
   ./services/misc/zookeeper.nix
   ./services/monitoring/alerta.nix
@@ -676,6 +693,7 @@
   ./services/network-filesystems/litestream/default.nix
   ./services/network-filesystems/netatalk.nix
   ./services/network-filesystems/nfsd.nix
+  ./services/network-filesystems/moosefs.nix
   ./services/network-filesystems/openafs/client.nix
   ./services/network-filesystems/openafs/server.nix
   ./services/network-filesystems/orangefs/server.nix
@@ -709,6 +727,7 @@
   ./services/networking/bird.nix
   ./services/networking/bitlbee.nix
   ./services/networking/blockbook-frontend.nix
+  ./services/networking/blocky.nix
   ./services/networking/charybdis.nix
   ./services/networking/cjdns.nix
   ./services/networking/cntlm.nix
@@ -733,15 +752,16 @@
   ./services/networking/ejabberd.nix
   ./services/networking/epmd.nix
   ./services/networking/ergo.nix
+  ./services/networking/ergochat.nix
   ./services/networking/eternal-terminal.nix
   ./services/networking/fakeroute.nix
   ./services/networking/ferm.nix
-  ./services/networking/firefox/sync-server.nix
   ./services/networking/fireqos.nix
   ./services/networking/firewall.nix
   ./services/networking/flannel.nix
   ./services/networking/freenet.nix
   ./services/networking/freeradius.nix
+  ./services/networking/frr.nix
   ./services/networking/gateone.nix
   ./services/networking/gdomap.nix
   ./services/networking/ghostunnel.nix
@@ -751,10 +771,10 @@
   ./services/networking/go-neb.nix
   ./services/networking/go-shadowsocks2.nix
   ./services/networking/gobgpd.nix
-  ./services/networking/gogoclient.nix
   ./services/networking/gvpe.nix
   ./services/networking/hans.nix
   ./services/networking/haproxy.nix
+  ./services/networking/headscale.nix
   ./services/networking/hostapd.nix
   ./services/networking/htpdate.nix
   ./services/networking/hylafax/default.nix
@@ -783,7 +803,6 @@
   ./services/networking/lldpd.nix
   ./services/networking/logmein-hamachi.nix
   ./services/networking/lxd-image-server.nix
-  ./services/networking/mailpile.nix
   ./services/networking/magic-wormhole-mailbox-server.nix
   ./services/networking/matterbridge.nix
   ./services/networking/mjpg-streamer.nix
@@ -795,6 +814,7 @@
   ./services/networking/miredo.nix
   ./services/networking/mstpd.nix
   ./services/networking/mtprotoproxy.nix
+  ./services/networking/mtr-exporter.nix
   ./services/networking/mullvad-vpn.nix
   ./services/networking/multipath.nix
   ./services/networking/murmur.nix
@@ -803,6 +823,7 @@
   ./services/networking/nar-serve.nix
   ./services/networking/nat.nix
   ./services/networking/nats.nix
+  ./services/networking/nbd.nix
   ./services/networking/ndppd.nix
   ./services/networking/nebula.nix
   ./services/networking/networkmanager.nix
@@ -843,7 +864,6 @@
   ./services/networking/quassel.nix
   ./services/networking/quorum.nix
   ./services/networking/quicktun.nix
-  ./services/networking/racoon.nix
   ./services/networking/radicale.nix
   ./services/networking/radvd.nix
   ./services/networking/rdnssd.nix
@@ -863,6 +883,7 @@
   ./services/networking/shorewall6.nix
   ./services/networking/shout.nix
   ./services/networking/sniproxy.nix
+  ./services/networking/snowflake-proxy.nix
   ./services/networking/smartdns.nix
   ./services/networking/smokeping.nix
   ./services/networking/softether.nix
@@ -887,6 +908,8 @@
   ./services/networking/tcpcrypt.nix
   ./services/networking/teamspeak3.nix
   ./services/networking/tedicross.nix
+  ./services/networking/tetrd.nix
+  ./services/networking/teleport.nix
   ./services/networking/thelounge.nix
   ./services/networking/tinc.nix
   ./services/networking/tinydns.nix
@@ -905,6 +928,7 @@
   ./services/networking/vsftpd.nix
   ./services/networking/wasabibackend.nix
   ./services/networking/websockify.nix
+  ./services/networking/wg-netmanager.nix
   ./services/networking/wg-quick.nix
   ./services/networking/wireguard.nix
   ./services/networking/wpa_supplicant.nix
@@ -928,12 +952,12 @@
   ./services/search/kibana.nix
   ./services/search/meilisearch.nix
   ./services/search/solr.nix
+  ./services/security/aesmd.nix
   ./services/security/certmgr.nix
   ./services/security/cfssl.nix
   ./services/security/clamav.nix
   ./services/security/fail2ban.nix
   ./services/security/fprintd.nix
-  ./services/security/fprot.nix
   ./services/security/haka.nix
   ./services/security/haveged.nix
   ./services/security/hockeypuck.nix
@@ -957,6 +981,7 @@
   ./services/security/vault.nix
   ./services/security/vaultwarden/default.nix
   ./services/security/yubikey-agent.nix
+  ./services/system/cachix-agent/default.nix
   ./services/system/cloud-init.nix
   ./services/system/dbus.nix
   ./services/system/earlyoom.nix
@@ -965,6 +990,7 @@
   ./services/system/nscd.nix
   ./services/system/saslauthd.nix
   ./services/system/self-deploy.nix
+  ./services/system/systembus-notify.nix
   ./services/system/uptimed.nix
   ./services/torrent/deluge.nix
   ./services/torrent/flexget.nix
@@ -986,6 +1012,7 @@
   ./services/web-apps/bookstack.nix
   ./services/web-apps/calibre-web.nix
   ./services/web-apps/code-server.nix
+  ./services/web-apps/baget.nix
   ./services/web-apps/convos.nix
   ./services/web-apps/cryptpad.nix
   ./services/web-apps/dex.nix
@@ -993,6 +1020,7 @@
   ./services/web-apps/documize.nix
   ./services/web-apps/dokuwiki.nix
   ./services/web-apps/engelsystem.nix
+  ./services/web-apps/ethercalc.nix
   ./services/web-apps/fluidd.nix
   ./services/web-apps/galene.nix
   ./services/web-apps/gerrit.nix
@@ -1009,6 +1037,7 @@
   ./services/web-apps/keycloak.nix
   ./services/web-apps/lemmy.nix
   ./services/web-apps/invidious.nix
+  ./services/web-apps/invoiceplane.nix
   ./services/web-apps/limesurvey.nix
   ./services/web-apps/mastodon.nix
   ./services/web-apps/mattermost.nix
@@ -1023,14 +1052,14 @@
   ./services/web-apps/plantuml-server.nix
   ./services/web-apps/plausible.nix
   ./services/web-apps/pgpkeyserver-lite.nix
+  ./services/web-apps/powerdns-admin.nix
+  ./services/web-apps/prosody-filer.nix
   ./services/web-apps/matomo.nix
-  ./services/web-apps/moinmoin.nix
   ./services/web-apps/openwebrx.nix
   ./services/web-apps/restya-board.nix
   ./services/web-apps/sogo.nix
   ./services/web-apps/rss-bridge.nix
   ./services/web-apps/tt-rss.nix
-  ./services/web-apps/trac.nix
   ./services/web-apps/trilium.nix
   ./services/web-apps/selfoss.nix
   ./services/web-apps/shiori.nix
@@ -1041,6 +1070,7 @@
   ./services/web-apps/wordpress.nix
   ./services/web-apps/youtrack.nix
   ./services/web-apps/zabbix.nix
+  ./services/web-servers/agate.nix
   ./services/web-servers/apache-httpd/default.nix
   ./services/web-servers/caddy/default.nix
   ./services/web-servers/darkhttpd.nix
@@ -1060,7 +1090,6 @@
   ./services/web-servers/phpfpm/default.nix
   ./services/web-servers/pomerium.nix
   ./services/web-servers/unit/default.nix
-  ./services/web-servers/shellinabox.nix
   ./services/web-servers/tomcat.nix
   ./services/web-servers/traefik.nix
   ./services/web-servers/trafficserver/default.nix
@@ -1140,15 +1169,21 @@
   ./system/boot/stage-1.nix
   ./system/boot/stage-2.nix
   ./system/boot/systemd.nix
-  ./system/boot/systemd-nspawn.nix
+  ./system/boot/systemd/coredump.nix
+  ./system/boot/systemd/journald.nix
+  ./system/boot/systemd/logind.nix
+  ./system/boot/systemd/nspawn.nix
+  ./system/boot/systemd/tmpfiles.nix
+  ./system/boot/systemd/user.nix
   ./system/boot/timesyncd.nix
   ./system/boot/tmp.nix
-  ./system/etc/etc.nix
+  ./system/etc/etc-activation.nix
   ./tasks/auto-upgrade.nix
   ./tasks/bcache.nix
   ./tasks/cpu-freq.nix
   ./tasks/encrypted-devices.nix
   ./tasks/filesystems.nix
+  ./tasks/filesystems/apfs.nix
   ./tasks/filesystems/bcachefs.nix
   ./tasks/filesystems/btrfs.nix
   ./tasks/filesystems/cifs.nix
@@ -1176,6 +1211,7 @@
   ./tasks/powertop.nix
   ./testing/service-runner.nix
   ./virtualisation/anbox.nix
+  ./virtualisation/build-vm.nix
   ./virtualisation/container-config.nix
   ./virtualisation/containerd.nix
   ./virtualisation/containers.nix
@@ -1183,6 +1219,7 @@
   ./virtualisation/oci-containers.nix
   ./virtualisation/cri-o.nix
   ./virtualisation/docker.nix
+  ./virtualisation/docker-rootless.nix
   ./virtualisation/ecs-agent.nix
   ./virtualisation/libvirtd.nix
   ./virtualisation/lxc.nix
diff --git a/nixos/modules/profiles/all-hardware.nix b/nixos/modules/profiles/all-hardware.nix
index 797fcddb8c90..25f68123a1da 100644
--- a/nixos/modules/profiles/all-hardware.nix
+++ b/nixos/modules/profiles/all-hardware.nix
@@ -44,12 +44,12 @@ in
       "ohci1394" "sbp2"
 
       # Virtio (QEMU, KVM etc.) support.
-      "virtio_net" "virtio_pci" "virtio_blk" "virtio_scsi" "virtio_balloon" "virtio_console"
+      "virtio_net" "virtio_pci" "virtio_mmio" "virtio_blk" "virtio_scsi" "virtio_balloon" "virtio_console"
 
       # VMware support.
       "mptspi" "vmxnet3" "vsock"
     ] ++ lib.optional platform.isx86 "vmw_balloon"
-    ++ lib.optionals (!platform.isAarch64 && !platform.isAarch32) [ # not sure where else they're missing
+    ++ lib.optionals (pkgs.stdenv.isi686 || pkgs.stdenv.isx86_64) [
       "vmw_vmci" "vmwgfx" "vmw_vsock_vmci_transport"
 
       # Hyper-V support.
diff --git a/nixos/modules/profiles/hardened.nix b/nixos/modules/profiles/hardened.nix
index 3f8f78f012a7..856ee480fc0b 100644
--- a/nixos/modules/profiles/hardened.nix
+++ b/nixos/modules/profiles/hardened.nix
@@ -17,7 +17,7 @@ with lib;
 
   boot.kernelPackages = mkDefault pkgs.linuxPackages_hardened;
 
-  nix.allowedUsers = mkDefault [ "@users" ];
+  nix.settings.allowed-users = mkDefault [ "@users" ];
 
   environment.memoryAllocator.provider = mkDefault "scudo";
   environment.variables.SCUDO_OPTIONS = mkDefault "ZeroContents=1";
diff --git a/nixos/modules/programs/calls.nix b/nixos/modules/programs/calls.nix
index 59961625e5d9..08a223b408d4 100644
--- a/nixos/modules/programs/calls.nix
+++ b/nixos/modules/programs/calls.nix
@@ -14,6 +14,8 @@ in {
   };
 
   config = mkIf cfg.enable {
+    programs.dconf.enable = true;
+
     environment.systemPackages = [
       pkgs.calls
     ];
diff --git a/nixos/modules/programs/captive-browser.nix b/nixos/modules/programs/captive-browser.nix
index 0f5d087e8d87..aad554c2bd66 100644
--- a/nixos/modules/programs/captive-browser.nix
+++ b/nixos/modules/programs/captive-browser.nix
@@ -1,8 +1,33 @@
 { config, lib, pkgs, ... }:
 
-with lib;
 let
   cfg = config.programs.captive-browser;
+
+  inherit (lib)
+    concatStringsSep escapeShellArgs optionalString
+    literalExpression mkEnableOption mkIf mkOption mkOptionDefault types;
+
+  browserDefault = chromium: concatStringsSep " " [
+    ''env XDG_CONFIG_HOME="$PREV_CONFIG_HOME"''
+    ''${chromium}/bin/chromium''
+    ''--user-data-dir=''${XDG_DATA_HOME:-$HOME/.local/share}/chromium-captive''
+    ''--proxy-server="socks5://$PROXY"''
+    ''--host-resolver-rules="MAP * ~NOTFOUND , EXCLUDE localhost"''
+    ''--no-first-run''
+    ''--new-window''
+    ''--incognito''
+    ''-no-default-browser-check''
+    ''http://cache.nixos.org/''
+  ];
+
+  desktopItem = pkgs.makeDesktopItem {
+    name = "captive-browser";
+    desktopName = "Captive Portal Browser";
+    exec = "/run/wrappers/bin/captive-browser";
+    icon = "nix-snowflake";
+    categories = [ "Network" ];
+  };
+
 in
 {
   ###### interface
@@ -26,18 +51,8 @@ in
       # the options below are the same as in "captive-browser.toml"
       browser = mkOption {
         type = types.str;
-        default = concatStringsSep " " [
-          ''env XDG_CONFIG_HOME="$PREV_CONFIG_HOME"''
-          ''${pkgs.chromium}/bin/chromium''
-          ''--user-data-dir=''${XDG_DATA_HOME:-$HOME/.local/share}/chromium-captive''
-          ''--proxy-server="socks5://$PROXY"''
-          ''--host-resolver-rules="MAP * ~NOTFOUND , EXCLUDE localhost"''
-          ''--no-first-run''
-          ''--new-window''
-          ''--incognito''
-          ''-no-default-browser-check''
-          ''http://cache.nixos.org/''
-        ];
+        default = browserDefault pkgs.chromium;
+        defaultText = literalExpression (browserDefault "\${pkgs.chromium}");
         description = ''
           The shell (/bin/sh) command executed once the proxy starts.
           When browser exits, the proxy exits. An extra env var PROXY is available.
@@ -82,6 +97,11 @@ in
   ###### implementation
 
   config = mkIf cfg.enable {
+    environment.systemPackages = [
+      (pkgs.runCommandNoCC "captive-browser-desktop-item" { } ''
+        install -Dm444 -t $out/share/applications ${desktopItem}/share/applications/*.desktop
+      '')
+    ];
 
     programs.captive-browser.dhcp-dns =
       let
diff --git a/nixos/modules/programs/chromium.nix b/nixos/modules/programs/chromium.nix
index 602253a321d7..8a1653318ab5 100644
--- a/nixos/modules/programs/chromium.nix
+++ b/nixos/modules/programs/chromium.nix
@@ -7,6 +7,7 @@ let
 
   defaultProfile = filterAttrs (k: v: v != null) {
     HomepageLocation = cfg.homepageLocation;
+    DefaultSearchProviderEnabled = cfg.defaultSearchProviderEnabled;
     DefaultSearchProviderSearchURL = cfg.defaultSearchProviderSearchURL;
     DefaultSearchProviderSuggestURL = cfg.defaultSearchProviderSuggestURL;
     ExtensionInstallForcelist = cfg.extensions;
@@ -50,6 +51,13 @@ in
         example = "https://nixos.org";
       };
 
+      defaultSearchProviderEnabled = mkOption {
+        type = types.nullOr types.bool;
+        description = "Enable the default search provider.";
+        default = null;
+        example = true;
+      };
+
       defaultSearchProviderSearchURL = mkOption {
         type = types.nullOr types.str;
         description = "Chromium default search provider url.";
diff --git a/nixos/modules/programs/command-not-found/command-not-found.pl b/nixos/modules/programs/command-not-found/command-not-found.pl
index 220d057b7f4f..72e246c81ae9 100644
--- a/nixos/modules/programs/command-not-found/command-not-found.pl
+++ b/nixos/modules/programs/command-not-found/command-not-found.pl
@@ -21,11 +21,24 @@ my $res = $dbh->selectall_arrayref(
     "select package from Programs where system = ? and name = ?",
     { Slice => {} }, $system, $program);
 
-if (!defined $res || scalar @$res == 0) {
+my $len = !defined $res ? 0 : scalar @$res;
+
+if ($len == 0) {
     print STDERR "$program: command not found\n";
-} elsif (scalar @$res == 1) {
+} elsif ($len == 1) {
     my $package = @$res[0]->{package};
     if ($ENV{"NIX_AUTO_RUN"} // "") {
+        if ($ENV{"NIX_AUTO_RUN_INTERACTIVE"} // "") {
+            while (1) {
+                print STDERR "'$program' from package '$package' will be run, confirm? [yn]: ";
+                chomp(my $comfirm = <STDIN>);
+                if (lc $comfirm eq "n") {
+                    exit 0;
+                } elsif (lc $comfirm eq "y") {
+                    last;
+                }
+            }
+        }
         exec("nix-shell", "-p", $package, "--run", shell_quote("exec", @ARGV));
     } else {
         print STDERR <<EOF;
@@ -35,11 +48,30 @@ ephemeral shell by typing:
 EOF
     }
 } else {
-    print STDERR <<EOF;
+    if ($ENV{"NIX_AUTO_RUN"} // "") {
+        print STDERR "Select a package that provides '$program':\n";
+        for my $i (0 .. $len - 1) {
+            print STDERR "  [", $i + 1, "]: @$res[$i]->{package}\n";
+        }
+        my $choice = 0;
+        while (1) { # exec will break this loop
+            no warnings "numeric";
+            print STDERR "Your choice [1-${len}]: ";
+            # 0 can be invalid user input like non-number string
+            # so we start from 1
+            $choice = <STDIN> + 0;
+            if (1 <= $choice && $choice <= $len) {
+                exec("nix-shell", "-p", @$res[$choice - 1]->{package},
+                    "--run", shell_quote("exec", @ARGV));
+            }
+        }
+    } else {
+        print STDERR <<EOF;
 The program '$program' is not in your PATH. It is provided by several packages.
 You can make it available in an ephemeral shell by typing one of the following:
 EOF
-    print STDERR "  nix-shell -p $_->{package}\n" foreach @$res;
+        print STDERR "  nix-shell -p $_->{package}\n" foreach @$res;
+    }
 }
 
 exit 127;
diff --git a/nixos/modules/programs/dconf.nix b/nixos/modules/programs/dconf.nix
index 298abac8afa9..265c41cbbbc9 100644
--- a/nixos/modules/programs/dconf.nix
+++ b/nixos/modules/programs/dconf.nix
@@ -60,7 +60,7 @@ in
     environment.systemPackages = [ pkgs.dconf ];
 
     # Needed for unwrapped applications
-    environment.variables.GIO_EXTRA_MODULES = mkIf cfg.enable [ "${pkgs.dconf.lib}/lib/gio/modules" ];
+    environment.sessionVariables.GIO_EXTRA_MODULES = mkIf cfg.enable [ "${pkgs.dconf.lib}/lib/gio/modules" ];
   };
 
 }
diff --git a/nixos/modules/programs/environment.nix b/nixos/modules/programs/environment.nix
index d552c751afd7..a448727be778 100644
--- a/nixos/modules/programs/environment.nix
+++ b/nixos/modules/programs/environment.nix
@@ -40,13 +40,15 @@ in
         KDEDIRS = [ "" ];
         QT_PLUGIN_PATH = [ "/lib/qt4/plugins" "/lib/kde4/plugins" ];
         QTWEBKIT_PLUGIN_PATH = [ "/lib/mozilla/plugins/" ];
-        GTK_PATH = [ "/lib/gtk-2.0" "/lib/gtk-3.0" ];
+        GTK_PATH = [ "/lib/gtk-2.0" "/lib/gtk-3.0" "/lib/gtk-4.0" ];
         XDG_CONFIG_DIRS = [ "/etc/xdg" ];
         XDG_DATA_DIRS = [ "/share" ];
         MOZ_PLUGIN_PATH = [ "/lib/mozilla/plugins" ];
         LIBEXEC_PATH = [ "/lib/libexec" ];
       };
 
+    environment.pathsToLink = [ "/lib/gtk-2.0" "/lib/gtk-3.0" "/lib/gtk-4.0" ];
+
     environment.extraInit =
       ''
          unset ASPELL_CONF
diff --git a/nixos/modules/programs/firejail.nix b/nixos/modules/programs/firejail.nix
index 41db4f0136ef..76b42168c198 100644
--- a/nixos/modules/programs/firejail.nix
+++ b/nixos/modules/programs/firejail.nix
@@ -17,8 +17,8 @@ let
         then value
         else { executable = value; profile = null; extraArgs = []; };
         args = lib.escapeShellArgs (
-          (optional (opts.profile != null) "--profile=${toString opts.profile}")
-          ++ opts.extraArgs
+          opts.extraArgs
+          ++ (optional (opts.profile != null) "--profile=${toString opts.profile}")
           );
       in
       ''
@@ -74,8 +74,10 @@ in {
         </para>
         <para>
         You will get file collisions if you put the actual application binary in
-        the global environment and applications started via .desktop files are
-        not wrapped if they specify the absolute path to the binary.
+        the global environment (such as by adding the application package to
+        <code>environment.systemPackages</code>), and applications started via
+        .desktop files are not wrapped if they specify the absolute path to the
+        binary.
       '';
     };
   };
diff --git a/nixos/modules/programs/gnupg.nix b/nixos/modules/programs/gnupg.nix
index 06f49182e4df..b41f30287ea5 100644
--- a/nixos/modules/programs/gnupg.nix
+++ b/nixos/modules/programs/gnupg.nix
@@ -71,6 +71,7 @@ in
       type = types.nullOr (types.enum pkgs.pinentry.flavors);
       example = "gnome3";
       default = defaultPinentryFlavor;
+      defaultText = literalDocBook ''matching the configured desktop environment'';
       description = ''
         Which pinentry interface to use. If not null, the path to the
         pinentry binary will be passed to gpg-agent via commandline and
@@ -148,4 +149,6 @@ in
     ];
   };
 
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/programs/k40-whisperer.nix b/nixos/modules/programs/k40-whisperer.nix
new file mode 100644
index 000000000000..3163e45f57e4
--- /dev/null
+++ b/nixos/modules/programs/k40-whisperer.nix
@@ -0,0 +1,40 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.programs.k40-whisperer;
+  pkg = cfg.package.override {
+    udevGroup = cfg.group;
+  };
+in
+{
+  options.programs.k40-whisperer = {
+    enable = mkEnableOption "K40-Whisperer";
+
+    group = mkOption {
+      type = types.str;
+      description = ''
+        Group assigned to the device when connected.
+      '';
+      default = "k40";
+    };
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.k40-whisperer;
+      defaultText = literalExpression "pkgs.k40-whisperer";
+      example = literalExpression "pkgs.k40-whisperer";
+      description = ''
+        K40 Whisperer package to use.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.groups.${cfg.group} = {};
+
+    environment.systemPackages = [ pkg ];
+    services.udev.packages = [ pkg ];
+  };
+}
diff --git a/nixos/modules/programs/kclock.nix b/nixos/modules/programs/kclock.nix
new file mode 100644
index 000000000000..42d81d2798ba
--- /dev/null
+++ b/nixos/modules/programs/kclock.nix
@@ -0,0 +1,13 @@
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.programs.kclock;
+  kclockPkg = pkgs.libsForQt5.kclock;
+in {
+  options.programs.kclock = { enable = mkEnableOption "Enable KClock"; };
+
+  config = mkIf cfg.enable {
+    services.dbus.packages = [ kclockPkg ];
+    environment.systemPackages = [ kclockPkg ];
+  };
+}
diff --git a/nixos/modules/programs/nbd.nix b/nixos/modules/programs/nbd.nix
new file mode 100644
index 000000000000..fea9bc1ff71a
--- /dev/null
+++ b/nixos/modules/programs/nbd.nix
@@ -0,0 +1,19 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.programs.nbd;
+in
+{
+  options = {
+    programs.nbd = {
+      enable = mkEnableOption "Network Block Device (nbd) support";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [ nbd ];
+    boot.kernelModules = [ "nbd" ];
+  };
+}
diff --git a/nixos/modules/programs/nix-ld.nix b/nixos/modules/programs/nix-ld.nix
new file mode 100644
index 000000000000..810a74ab50b7
--- /dev/null
+++ b/nixos/modules/programs/nix-ld.nix
@@ -0,0 +1,12 @@
+{ pkgs, lib, config, ... }:
+{
+  meta.maintainers = [ lib.maintainers.mic92 ];
+  options = {
+    programs.nix-ld.enable = lib.mkEnableOption ''nix-ld, Documentation: <link xlink:href="https://github.com/Mic92/nix-ld"/>'';
+  };
+  config = lib.mkIf config.programs.nix-ld.enable {
+    systemd.tmpfiles.rules = [
+      "L+ ${pkgs.nix-ld.ldPath} - - - - ${pkgs.nix-ld}/libexec/nix-ld"
+    ];
+  };
+}
diff --git a/nixos/modules/programs/phosh.nix b/nixos/modules/programs/phosh.nix
index cba3f73768ec..ad875616ac9e 100644
--- a/nixos/modules/programs/phosh.nix
+++ b/nixos/modules/programs/phosh.nix
@@ -8,18 +8,17 @@ let
   # Based on https://source.puri.sm/Librem5/librem5-base/-/blob/4596c1056dd75ac7f043aede07887990fd46f572/default/sm.puri.OSK0.desktop
   oskItem = pkgs.makeDesktopItem {
     name = "sm.puri.OSK0";
-    type = "Application";
     desktopName = "On-screen keyboard";
     exec = "${pkgs.squeekboard}/bin/squeekboard";
-    categories = "GNOME;Core;";
-    extraEntries = ''
-      OnlyShowIn=GNOME;
-      NoDisplay=true
-      X-GNOME-Autostart-Phase=Panel
-      X-GNOME-Provides=inputmethod
-      X-GNOME-Autostart-Notify=true
-      X-GNOME-AutoRestart=true
-    '';
+    categories = [ "GNOME" "Core" ];
+    onlyShowIn = [ "GNOME" ];
+    noDisplay = true;
+    extraConfig = {
+      X-GNOME-Autostart-Phase = "Panel";
+      X-GNOME-Provides = "inputmethod";
+      X-GNOME-Autostart-Notify = "true";
+      X-GNOME-AutoRestart = "true";
+    };
   };
 
   phocConfigType = types.submodule {
diff --git a/nixos/modules/programs/qt5ct.nix b/nixos/modules/programs/qt5ct.nix
index 3f2bcf622836..88e861bf4031 100644
--- a/nixos/modules/programs/qt5ct.nix
+++ b/nixos/modules/programs/qt5ct.nix
@@ -26,6 +26,6 @@ with lib;
   ###### implementation
   config = mkIf config.programs.qt5ct.enable {
     environment.variables.QT_QPA_PLATFORMTHEME = "qt5ct";
-    environment.systemPackages = with pkgs; [ qt5ct ];
+    environment.systemPackages = with pkgs; [ libsForQt5.qt5ct ];
   };
 }
diff --git a/nixos/modules/programs/spacefm.nix b/nixos/modules/programs/spacefm.nix
index 822fca3ecec7..f71abcaa3325 100644
--- a/nixos/modules/programs/spacefm.nix
+++ b/nixos/modules/programs/spacefm.nix
@@ -27,13 +27,11 @@ in
         default = {
           tmp_dir = "/tmp";
           terminal_su = "${pkgs.sudo}/bin/sudo";
-          graphical_su = "${pkgs.gksu}/bin/gksu";
         };
         defaultText = literalExpression ''
           {
             tmp_dir = "/tmp";
             terminal_su = "''${pkgs.sudo}/bin/sudo";
-            graphical_su = "''${pkgs.gksu}/bin/gksu";
           }
         '';
         description = ''
diff --git a/nixos/modules/programs/ssh.nix b/nixos/modules/programs/ssh.nix
index 5da15b68cf7d..b31fce915240 100644
--- a/nixos/modules/programs/ssh.nix
+++ b/nixos/modules/programs/ssh.nix
@@ -17,7 +17,7 @@ let
       exec ${askPassword} "$@"
     '';
 
-  knownHosts = map (h: getAttr h cfg.knownHosts) (attrNames cfg.knownHosts);
+  knownHosts = attrValues cfg.knownHosts;
 
   knownHostsText = (flip (concatMapStringsSep "\n") knownHosts
     (h: assert h.hostNames != [];
@@ -25,6 +25,9 @@ let
       + (if h.publicKey != null then h.publicKey else readFile h.publicKeyFile)
     )) + "\n";
 
+  knownHostsFiles = [ "/etc/ssh/ssh_known_hosts" "/etc/ssh/ssh_known_hosts2" ]
+    ++ map pkgs.copyPathToStore cfg.knownHostsFiles;
+
 in
 {
   ###### interface
@@ -33,6 +36,13 @@ in
 
     programs.ssh = {
 
+      enableAskPassword = mkOption {
+        type = types.bool;
+        default = config.services.xserver.enable;
+        defaultText = literalExpression "config.services.xserver.enable";
+        description = "Whether to configure SSH_ASKPASS in the environment.";
+      };
+
       askPassword = mkOption {
         type = types.str;
         default = "${pkgs.x11_ssh_askpass}/libexec/x11-ssh-askpass";
@@ -132,7 +142,7 @@ in
 
       knownHosts = mkOption {
         default = {};
-        type = types.attrsOf (types.submodule ({ name, ... }: {
+        type = types.attrsOf (types.submodule ({ name, config, options, ... }: {
           options = {
             certAuthority = mkOption {
               type = types.bool;
@@ -144,12 +154,22 @@ in
             };
             hostNames = mkOption {
               type = types.listOf types.str;
-              default = [];
+              default = [ name ] ++ config.extraHostNames;
+              defaultText = literalExpression "[ ${name} ] ++ config.${options.extraHostNames}";
               description = ''
+                DEPRECATED, please use <literal>extraHostNames</literal>.
                 A list of host names and/or IP numbers used for accessing
                 the host's ssh service.
               '';
             };
+            extraHostNames = mkOption {
+              type = types.listOf types.str;
+              default = [];
+              description = ''
+                A list of additional host names and/or IP numbers used for
+                accessing the host's ssh service.
+              '';
+            };
             publicKey = mkOption {
               default = null;
               type = types.nullOr types.str;
@@ -170,13 +190,12 @@ in
                 You can fetch a public key file from a running SSH server
                 with the <command>ssh-keyscan</command> command. The content
                 of the file should follow the same format as described for
-                the <literal>publicKey</literal> option.
+                the <literal>publicKey</literal> option. Only a single key
+                is supported. If a host has multiple keys, use
+                <option>programs.ssh.knownHostsFiles</option> instead.
               '';
             };
           };
-          config = {
-            hostNames = mkDefault [ name ];
-          };
         }));
         description = ''
           The set of system-wide known SSH hosts.
@@ -184,17 +203,36 @@ in
         example = literalExpression ''
           {
             myhost = {
-              hostNames = [ "myhost" "myhost.mydomain.com" "10.10.1.4" ];
+              extraHostNames = [ "myhost.mydomain.com" "10.10.1.4" ];
               publicKeyFile = ./pubkeys/myhost_ssh_host_dsa_key.pub;
             };
-            myhost2 = {
-              hostNames = [ "myhost2" ];
-              publicKeyFile = ./pubkeys/myhost2_ssh_host_dsa_key.pub;
-            };
+            "myhost2.net".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIRuJ8p1Fi+m6WkHV0KWnRfpM1WxoW8XAS+XvsSKsTK";
           }
         '';
       };
 
+      knownHostsFiles = mkOption {
+        default = [];
+        type = with types; listOf path;
+        description = ''
+          Files containing SSH host keys to set as global known hosts.
+          <literal>/etc/ssh/ssh_known_hosts</literal> (which is
+          generated by <option>programs.ssh.knownHosts</option>) and
+          <literal>/etc/ssh/ssh_known_hosts2</literal> are always
+          included.
+        '';
+        example = literalExpression ''
+          [
+            ./known_hosts
+            (writeText "github.keys" '''
+              github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+              github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+              github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+            ''')
+          ]
+        '';
+      };
+
       kexAlgorithms = mkOption {
         type = types.nullOr (types.listOf types.str);
         default = null;
@@ -241,6 +279,9 @@ in
         message = "knownHost ${name} must contain either a publicKey or publicKeyFile";
       });
 
+    warnings = mapAttrsToList (name: _: ''programs.ssh.knownHosts.${name}.hostNames is deprecated, use programs.ssh.knownHosts.${name}.extraHostNames'')
+      (filterAttrs (name: {hostNames, extraHostNames, ...}: hostNames != [ name ] ++ extraHostNames) cfg.knownHosts);
+
     # SSH configuration. Slight duplication of the sshd_config
     # generation in the sshd service.
     environment.etc."ssh/ssh_config".text =
@@ -251,6 +292,7 @@ in
         # Generated options from other settings
         Host *
         AddressFamily ${if config.networking.enableIPv6 then "any" else "inet"}
+        GlobalKnownHostsFile ${concatStringsSep " " knownHostsFiles}
 
         ${optionalString cfg.setXAuthLocation ''
           XAuthLocation ${pkgs.xorg.xauth}/bin/xauth
@@ -287,7 +329,7 @@ in
         # Allow ssh-agent to ask for confirmation. This requires the
         # unit to know about the user's $DISPLAY (via ‘systemctl
         # import-environment’).
-        environment.SSH_ASKPASS = optionalString config.services.xserver.enable askPasswordWrapper;
+        environment.SSH_ASKPASS = optionalString cfg.enableAskPassword askPasswordWrapper;
         environment.DISPLAY = "fake"; # required to make ssh-agent start $SSH_ASKPASS
       };
 
@@ -298,7 +340,7 @@ in
         fi
       '';
 
-    environment.variables.SSH_ASKPASS = optionalString config.services.xserver.enable askPassword;
+    environment.variables.SSH_ASKPASS = optionalString cfg.enableAskPassword askPassword;
 
   };
 }
diff --git a/nixos/modules/programs/starship.nix b/nixos/modules/programs/starship.nix
new file mode 100644
index 000000000000..83d2272003c6
--- /dev/null
+++ b/nixos/modules/programs/starship.nix
@@ -0,0 +1,51 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.programs.starship;
+
+  settingsFormat = pkgs.formats.toml { };
+
+  settingsFile = settingsFormat.generate "starship.toml" cfg.settings;
+
+in {
+  options.programs.starship = {
+    enable = mkEnableOption "the Starship shell prompt";
+
+    settings = mkOption {
+      inherit (settingsFormat) type;
+      default = { };
+      description = ''
+        Configuration included in <literal>starship.toml</literal>.
+
+        See https://starship.rs/config/#prompt for documentation.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    programs.bash.promptInit = ''
+      if [[ $TERM != "dumb" && (-z $INSIDE_EMACS || $INSIDE_EMACS == "vterm") ]]; then
+        export STARSHIP_CONFIG=${settingsFile}
+        eval "$(${pkgs.starship}/bin/starship init bash)"
+      fi
+    '';
+
+    programs.fish.promptInit = ''
+      if test "$TERM" != "dumb" -a \( -z "$INSIDE_EMACS" -o "$INSIDE_EMACS" = "vterm" \)
+        set -x STARSHIP_CONFIG ${settingsFile}
+        eval (${pkgs.starship}/bin/starship init fish)
+      end
+    '';
+
+    programs.zsh.promptInit = ''
+      if [[ $TERM != "dumb" && (-z $INSIDE_EMACS || $INSIDE_EMACS == "vterm") ]]; then
+        export STARSHIP_CONFIG=${settingsFile}
+        eval "$(${pkgs.starship}/bin/starship init zsh)"
+      fi
+    '';
+  };
+
+  meta.maintainers = pkgs.starship.meta.maintainers;
+}
diff --git a/nixos/modules/programs/sway.nix b/nixos/modules/programs/sway.nix
index c64e01a20cb3..01b047281344 100644
--- a/nixos/modules/programs/sway.nix
+++ b/nixos/modules/programs/sway.nix
@@ -90,10 +90,10 @@ in {
     extraPackages = mkOption {
       type = with types; listOf package;
       default = with pkgs; [
-        swaylock swayidle alacritty dmenu
+        swaylock swayidle foot dmenu
       ];
       defaultText = literalExpression ''
-        with pkgs; [ swaylock swayidle alacritty dmenu ];
+        with pkgs; [ swaylock swayidle foot dmenu ];
       '';
       example = literalExpression ''
         with pkgs; [
@@ -134,6 +134,7 @@ in {
         '';
       };
     };
+    security.polkit.enable = true;
     security.pam.services.swaylock = {};
     hardware.opengl.enable = mkDefault true;
     fonts.enableDefaultFonts = mkDefault true;
diff --git a/nixos/modules/programs/tilp2.nix b/nixos/modules/programs/tilp2.nix
deleted file mode 100644
index da9e32e3e6c6..000000000000
--- a/nixos/modules/programs/tilp2.nix
+++ /dev/null
@@ -1,28 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-
-let
-  cfg = config.programs.tilp2;
-
-in {
-  options.programs.tilp2 = {
-    enable = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Enable tilp2 and udev rules for supported calculators.
-      '';
-    };
-  };
-
-  config = mkIf cfg.enable {
-    services.udev.packages = [
-      pkgs.libticables2
-    ];
-
-    environment.systemPackages = [
-      pkgs.tilp2
-    ];
-  };
-}
diff --git a/nixos/modules/programs/tmux.nix b/nixos/modules/programs/tmux.nix
index c39908751d29..74b3fbd9ac06 100644
--- a/nixos/modules/programs/tmux.nix
+++ b/nixos/modules/programs/tmux.nix
@@ -52,6 +52,12 @@ let
     set  -s escape-time       ${toString cfg.escapeTime}
     set  -g history-limit     ${toString cfg.historyLimit}
 
+    ${lib.optionalString (cfg.plugins != []) ''
+    # Run plugins
+    ${lib.concatMapStringsSep "\n" (x: "run-shell ${x.rtp}") cfg.plugins}
+
+    ''}
+
     ${cfg.extraConfig}
   '';
 
@@ -165,6 +171,13 @@ in {
           downside it doesn't survive user logout.
         '';
       };
+
+      plugins = mkOption {
+        default = [];
+        type = types.listOf types.package;
+        description = "List of plugins to install.";
+        example = lib.literalExpression "[ pkgs.tmuxPlugins.nord ]";
+      };
     };
   };
 
@@ -174,7 +187,7 @@ in {
     environment = {
       etc."tmux.conf".text = tmuxConf;
 
-      systemPackages = [ pkgs.tmux ];
+      systemPackages = [ pkgs.tmux ] ++ cfg.plugins;
 
       variables = {
         TMUX_TMPDIR = lib.optional cfg.secureSocket ''''${XDG_RUNTIME_DIR:-"/run/user/$(id -u)"}'';
diff --git a/nixos/modules/programs/tsm-client.nix b/nixos/modules/programs/tsm-client.nix
index 65d4db7834ff..28db96253875 100644
--- a/nixos/modules/programs/tsm-client.nix
+++ b/nixos/modules/programs/tsm-client.nix
@@ -7,7 +7,7 @@ let
   inherit (lib.modules) mkDefault mkIf;
   inherit (lib.options) literalExpression mkEnableOption mkOption;
   inherit (lib.strings) concatStringsSep optionalString toLower;
-  inherit (lib.types) addCheck attrsOf lines nullOr package path port str strMatching submodule;
+  inherit (lib.types) addCheck attrsOf lines nonEmptyStr nullOr package path port str strMatching submodule;
 
   # Checks if given list of strings contains unique
   # elements when compared without considering case.
@@ -35,7 +35,7 @@ let
       '';
     };
     options.server = mkOption {
-      type = strMatching ".+";
+      type = nonEmptyStr;
       example = "tsmserver.company.com";
       description = ''
         Host/domain name or IP address of the IBM TSM server.
@@ -56,7 +56,7 @@ let
       '';
     };
     options.node = mkOption {
-      type = strMatching ".+";
+      type = nonEmptyStr;
       example = "MY-TSM-NODE";
       description = ''
         Target node name on the IBM TSM server.
@@ -144,7 +144,7 @@ let
     };
     config.name = mkDefault name;
     # Client system-options file directives are explained here:
-    # https://www.ibm.com/support/knowledgecenter/SSEQVQ_8.1.8/client/c_opt_usingopts.html
+    # https://www.ibm.com/docs/en/spectrum-protect/8.1.13?topic=commands-processing-options
     config.extraConfig =
       mapAttrs (lib.trivial.const mkDefault) (
         {
diff --git a/nixos/modules/programs/zsh/zsh-autosuggestions.nix b/nixos/modules/programs/zsh/zsh-autosuggestions.nix
index fee324cc7326..2e53e907d547 100644
--- a/nixos/modules/programs/zsh/zsh-autosuggestions.nix
+++ b/nixos/modules/programs/zsh/zsh-autosuggestions.nix
@@ -22,17 +22,18 @@ in
     };
 
     strategy = mkOption {
-      type = types.enum [ "history" "match_prev_cmd" ];
-      default = "history";
+      type = types.listOf (types.enum [ "history" "completion" "match_prev_cmd" ]);
+      default = [ "history" ];
       description = ''
-        Set ZSH_AUTOSUGGEST_STRATEGY to choose the strategy for generating suggestions.
-        There are currently two to choose from:
+        `ZSH_AUTOSUGGEST_STRATEGY` is an array that specifies how suggestions should be generated.
+        The strategies in the array are tried successively until a suggestion is found.
+        There are currently three built-in strategies to choose from:
 
-          * history: Chooses the most recent match.
-          * match_prev_cmd: Chooses the most recent match whose preceding history item matches
-            the most recently executed command (more info). Note that this strategy won't work as
-            expected with ZSH options that don't preserve the history order such as
-            HIST_IGNORE_ALL_DUPS or HIST_EXPIRE_DUPS_FIRST.
+        - `history`: Chooses the most recent match from history.
+        - `completion`: Chooses a suggestion based on what tab-completion would suggest. (requires `zpty` module)
+        - `match_prev_cmd`: Like `history`, but chooses the most recent match whose preceding history item matches
+            the most recently executed command. Note that this strategy won't work as expected with ZSH options that
+            don't preserve the history order such as `HIST_IGNORE_ALL_DUPS` or `HIST_EXPIRE_DUPS_FIRST`.
       '';
     };
 
@@ -62,7 +63,7 @@ in
       source ${pkgs.zsh-autosuggestions}/share/zsh-autosuggestions/zsh-autosuggestions.zsh
 
       export ZSH_AUTOSUGGEST_HIGHLIGHT_STYLE="${cfg.highlightStyle}"
-      export ZSH_AUTOSUGGEST_STRATEGY=("${cfg.strategy}")
+      export ZSH_AUTOSUGGEST_STRATEGY=(${concatStringsSep " " cfg.strategy})
       ${optionalString (!cfg.async) "unset ZSH_AUTOSUGGEST_USE_ASYNC"}
 
       ${concatStringsSep "\n" (mapAttrsToList (key: value: ''export ${key}="${value}"'') cfg.extraConfig)}
diff --git a/nixos/modules/programs/zsh/zsh.nix b/nixos/modules/programs/zsh/zsh.nix
index e5c5b08f8d4d..5fe98b6801bb 100644
--- a/nixos/modules/programs/zsh/zsh.nix
+++ b/nixos/modules/programs/zsh/zsh.nix
@@ -1,6 +1,6 @@
 # This module defines global configuration for the zshell.
 
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
@@ -9,6 +9,7 @@ let
   cfge = config.environment;
 
   cfg = config.programs.zsh;
+  opt = options.programs.zsh;
 
   zshAliases = concatStringsSep "\n" (
     mapAttrsFlatten (k: v: "alias ${k}=${escapeShellArg v}")
@@ -147,6 +148,7 @@ in
 
       enableGlobalCompInit = mkOption {
         default = cfg.enableCompletion;
+        defaultText = literalExpression "config.${opt.enableCompletion}";
         description = ''
           Enable execution of compinit call for all interactive zsh shells.
 
diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix
index 8e1d6f7bc4a5..195cf87e6a85 100644
--- a/nixos/modules/rename.nix
+++ b/nixos/modules/rename.nix
@@ -17,33 +17,61 @@ with lib;
     (mkAliasOptionModule [ "environment" "checkConfigurationOptions" ] [ "_module" "check" ])
 
     # Completely removed modules
+    (mkRemovedOptionModule [ "environment" "blcr" "enable" ] "The BLCR module has been removed")
     (mkRemovedOptionModule [ "fonts" "fontconfig" "penultimate" ] "The corresponding package has removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "quagga" ] "the corresponding package has been removed from nixpkgs")
-    (mkRemovedOptionModule [ "services" "chronos" ] "The corresponding package was removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "deepin" ] "The corresponding packages were removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "firefox" "syncserver" "user" ] "")
-    (mkRemovedOptionModule [ "services" "firefox" "syncserver" "group" ] "")
-    (mkRemovedOptionModule [ "services" "marathon" ] "The corresponding package was removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "mesos" ] "The corresponding package was removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "winstone" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "hardware" "brightnessctl" ] ''
+      The brightnessctl module was removed because newer versions of
+      brightnessctl don't require the udev rules anymore (they can use the
+      systemd-logind API). Instead of using the module you can now
+      simply add the brightnessctl package to environment.systemPackages.
+    '')
+    (mkRemovedOptionModule [ "hardware" "u2f" ] ''
+      The U2F modules module was removed, as all it did was adding the
+      udev rules from libu2f-host to the system. Udev gained native support
+      to handle FIDO security tokens, so this isn't necessary anymore.
+    '')
     (mkRemovedOptionModule [ "networking" "vpnc" ] "Use environment.etc.\"vpnc/service.conf\" instead.")
     (mkRemovedOptionModule [ "networking" "wicd" ] "The corresponding package was removed from nixpkgs.")
-    (mkRemovedOptionModule [ "environment" "blcr" "enable" ] "The BLCR module has been removed")
-    (mkRemovedOptionModule [ "services" "beegfsEnable" ] "The BeeGFS module has been removed")
+    (mkRemovedOptionModule [ "programs" "tilp2" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "programs" "way-cooler" ] ("way-cooler is abandoned by its author: " +
+      "https://way-cooler.org/blog/2020/01/09/way-cooler-post-mortem.html"))
+    (mkRemovedOptionModule [ "security" "hideProcessInformation" ] ''
+        The hidepid module was removed, since the underlying machinery
+        is broken when using cgroups-v2.
+    '')
     (mkRemovedOptionModule [ "services" "beegfs" ] "The BeeGFS module has been removed")
-    (mkRemovedOptionModule ["services" "cgmanager" "enable"] "cgmanager was deprecated by lxc and therefore removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "osquery" ] "The osquery module has been removed")
+    (mkRemovedOptionModule [ "services" "beegfsEnable" ] "The BeeGFS module has been removed")
+    (mkRemovedOptionModule [ "services" "cgmanager" "enable"] "cgmanager was deprecated by lxc and therefore removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "chronos" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "couchpotato" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "deepin" ] "The corresponding packages were removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "dnscrypt-proxy" ] "Use services.dnscrypt-proxy2 instead")
+    (mkRemovedOptionModule [ "services" "firefox" "syncserver" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "flashpolicyd" ] "The flashpolicyd module has been removed. Adobe Flash Player is deprecated.")
     (mkRemovedOptionModule [ "services" "fourStore" ] "The fourStore module has been removed")
-    (mkRemovedOptionModule [ "services" "frab" ] "The frab module has been removed")
     (mkRemovedOptionModule [ "services" "fourStoreEndpoint" ] "The fourStoreEndpoint module has been removed")
+    (mkRemovedOptionModule [ "services" "fprot" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "frab" ] "The frab module has been removed")
+    (mkRemovedOptionModule [ "services" "kippo" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "mailpile" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "marathon" ] "The corresponding package was removed from nixpkgs.")
     (mkRemovedOptionModule [ "services" "mathics" ] "The Mathics module has been removed")
-    (mkRemovedOptionModule [ "programs" "way-cooler" ] ("way-cooler is abandoned by its author: " +
-      "https://way-cooler.org/blog/2020/01/09/way-cooler-post-mortem.html"))
-    (mkRemovedOptionModule [ "services" "xserver" "multitouch" ] ''
-      services.xserver.multitouch (which uses xf86_input_mtrack) has been removed
-      as the underlying package isn't being maintained. Working alternatives are
-      libinput and synaptics.
+    (mkRemovedOptionModule [ "services" "meguca" ] "Use meguca has been removed from nixpkgs")
+    (mkRemovedOptionModule [ "services" "mesos" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "moinmoin" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "mwlib" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "osquery" ] "The osquery module has been removed")
+    (mkRemovedOptionModule [ "services" "pantheon" "files" ] ''
+      This module was removed, please add pkgs.pantheon.elementary-files to environment.systemPackages directly.
+    '')
+    (mkRemovedOptionModule [ "services" "prey" ] ''
+      prey-bash-client is deprecated upstream
     '')
+    (mkRemovedOptionModule [ "services" "quagga" ] "the corresponding package has been removed from nixpkgs")
+    (mkRemovedOptionModule [ "services" "seeks" ] "")
+    (mkRemovedOptionModule [ "services" "venus" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "wakeonlan"] "This module was removed in favor of enabling it with networking.interfaces.<name>.wakeOnLan")
+    (mkRemovedOptionModule [ "services" "winstone" ] "The corresponding package was removed from nixpkgs.")
     (mkRemovedOptionModule [ "services" "xserver" "displayManager" "auto" ] ''
       The services.xserver.displayManager.auto module has been removed
       because it was only intended for use in internal NixOS tests, and gave the
@@ -51,37 +79,18 @@ with lib;
       LightDM. Please use the services.xserver.displayManager.autoLogin options
       instead, or any other display manager in NixOS as they all support auto-login.
     '')
-    (mkRemovedOptionModule [ "services" "dnscrypt-proxy" ] "Use services.dnscrypt-proxy2 instead")
-    (mkRemovedOptionModule [ "services" "meguca" ] "Use meguca has been removed from nixpkgs")
-    (mkRemovedOptionModule ["hardware" "brightnessctl" ] ''
-      The brightnessctl module was removed because newer versions of
-      brightnessctl don't require the udev rules anymore (they can use the
-      systemd-logind API). Instead of using the module you can now
-      simply add the brightnessctl package to environment.systemPackages.
+    (mkRemovedOptionModule [ "services" "xserver" "multitouch" ] ''
+      services.xserver.multitouch (which uses xf86_input_mtrack) has been removed
+      as the underlying package isn't being maintained. Working alternatives are
+      libinput and synaptics.
     '')
     (mkRemovedOptionModule [ "virtualisation" "rkt" ] "The rkt module has been removed, it was archived by upstream")
-
-    (mkRemovedOptionModule ["services" "prey" ] ''
-      prey-bash-client is deprecated upstream
-    '')
-
-    (mkRemovedOptionModule ["hardware" "u2f" ] ''
-      The U2F modules module was removed, as all it did was adding the
-      udev rules from libu2f-host to the system. Udev gained native support
-      to handle FIDO security tokens, so this isn't necessary anymore.
+    (mkRemovedOptionModule [ "services" "racoon" ] ''
+      The racoon module has been removed, because the software project was abandoned upstream.
     '')
-
-    (mkRemovedOptionModule [ "services" "seeks" ] "")
-    (mkRemovedOptionModule [ "services" "venus" ] "The corresponding package was removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "flashpolicyd" ] "The flashpolicyd module has been removed. Adobe Flash Player is deprecated.")
-
-    (mkRemovedOptionModule [ "security" "hideProcessInformation" ] ''
-        The hidepid module was removed, since the underlying machinery
-        is broken when using cgroups-v2.
-    '')
-    (mkRemovedOptionModule ["services" "wakeonlan"] "This module was removed in favor of enabling it with networking.interfaces.<name>.wakeOnLan")
-
-    (mkRemovedOptionModule [ "services" "kippo" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "shellinabox" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "gogoclient" ] "The corresponding package was removed from nixpkgs.")
+    (mkRemovedOptionModule [ "services" "virtuoso" ] "The corresponding package was removed from nixpkgs.")
 
     # Do NOT add any option renames here, see top of the file
   ];
diff --git a/nixos/modules/security/acme.nix b/nixos/modules/security/acme/default.nix
index b50eeddfa40a..d827c448055b 100644
--- a/nixos/modules/security/acme.nix
+++ b/nixos/modules/security/acme/default.nix
@@ -2,6 +2,8 @@
 with lib;
 let
   cfg = config.security.acme;
+  opt = options.security.acme;
+  user = if cfg.useRoot then "root" else "acme";
 
   # Used to calculate timer accuracy for coalescing
   numCerts = length (builtins.attrNames cfg.certs);
@@ -22,7 +24,7 @@ let
   # security.acme.certs.<cert>.group on some of the services.
   commonServiceConfig = {
     Type = "oneshot";
-    User = "acme";
+    User = user;
     Group = mkDefault "acme";
     UMask = 0022;
     StateDirectoryMode = 750;
@@ -100,12 +102,12 @@ let
   # is configurable on a per-cert basis.
   userMigrationService = let
     script = with builtins; ''
-      chown -R acme .lego/accounts
+      chown -R ${user} .lego/accounts
     '' + (concatStringsSep "\n" (mapAttrsToList (cert: data: ''
       for fixpath in ${escapeShellArg cert} .lego/${escapeShellArg cert}; do
         if [ -d "$fixpath" ]; then
           chmod -R u=rwX,g=rX,o= "$fixpath"
-          chown -R acme:${data.group} "$fixpath"
+          chown -R ${user}:${data.group} "$fixpath"
         fi
       done
     '') certConfigs));
@@ -127,7 +129,7 @@ let
   };
 
   certToConfig = cert: data: let
-    acmeServer = if data.server != null then data.server else cfg.server;
+    acmeServer = data.server;
     useDns = data.dnsProvider != null;
     destPath = "/var/lib/acme/${cert}";
     selfsignedDeps = optionals (cfg.preliminarySelfsigned) [ "acme-selfsigned-${cert}.service" ];
@@ -155,6 +157,7 @@ let
       ${toString data.ocspMustStaple} ${data.keyType}
     '';
     certDir = mkHash hashData;
+    # TODO remove domainHash usage entirely. Waiting on go-acme/lego#1532
     domainHash = mkHash "${concatStringsSep " " extraDomains} ${data.domain}";
     accountHash = (mkAccountHash acmeServer data);
     accountDir = accountDirRoot + accountHash;
@@ -209,7 +212,7 @@ let
       description = "Renew ACME Certificate for ${cert}";
       wantedBy = [ "timers.target" ];
       timerConfig = {
-        OnCalendar = cfg.renewInterval;
+        OnCalendar = data.renewInterval;
         Unit = "acme-${cert}.service";
         Persistent = "yes";
 
@@ -266,7 +269,7 @@ let
         cat key.pem fullchain.pem > full.pem
 
         # Group might change between runs, re-apply it
-        chown 'acme:${data.group}' *
+        chown '${user}:${data.group}' *
 
         # Default permissions make the files unreadable by group + anon
         # Need to be readable by group
@@ -321,7 +324,7 @@ let
           fi
         '');
       } // optionalAttrs (data.listenHTTP != null && toInt (elemAt (splitString ":" data.listenHTTP) 1) < 1024) {
-        AmbientCapabilities = "CAP_NET_BIND_SERVICE";
+        CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
       };
 
       # Working directory will be /tmp
@@ -354,7 +357,7 @@ let
           expiration_s=$[expiration_date - now]
           expiration_days=$[expiration_s / (3600 * 24)]   # rounds down
 
-          [[ $expiration_days -gt ${toString cfg.validMinDays} ]]
+          [[ $expiration_days -gt ${toString data.validMinDays} ]]
         }
 
         ${optionalString (data.webroot != null) ''
@@ -371,37 +374,40 @@ let
 
         echo '${domainHash}' > domainhash.txt
 
-        # Check if we can renew
-        if [ -e 'certificates/${keyName}.key' -a -e 'certificates/${keyName}.crt' -a -n "$(ls -1 accounts)" ]; then
+        # Check if we can renew.
+        # We can only renew if the list of domains has not changed.
+        if cmp -s domainhash.txt certificates/domainhash.txt && [ -e 'certificates/${keyName}.key' -a -e 'certificates/${keyName}.crt' -a -n "$(ls -1 accounts)" ]; then
 
-          # When domains are updated, there's no need to do a full
-          # Lego run, but it's likely renew won't work if days is too low.
-          if [ -e certificates/domainhash.txt ] && cmp -s domainhash.txt certificates/domainhash.txt; then
+          # Even if a cert is not expired, it may be revoked by the CA.
+          # Try to renew, and silently fail if the cert is not expired.
+          # Avoids #85794 and resolves #129838
+          if ! lego ${renewOpts} --days ${toString data.validMinDays}; then
             if is_expiration_skippable out/full.pem; then
-              echo 1>&2 "nixos-acme: skipping renewal because expiration isn't within the coming ${toString cfg.validMinDays} days"
+              echo 1>&2 "nixos-acme: Ignoring failed renewal because expiration isn't within the coming ${toString data.validMinDays} days"
             else
-              echo 1>&2 "nixos-acme: renewing now, because certificate expires within the configured ${toString cfg.validMinDays} days"
-              lego ${renewOpts} --days ${toString cfg.validMinDays}
+              # High number to avoid Systemd reserved codes.
+              exit 11
             fi
-          else
-            echo 1>&2 "certificate domain(s) have changed; will renew now"
-            # Any number > 90 works, but this one is over 9000 ;-)
-            lego ${renewOpts} --days 9001
           fi
 
         # Otherwise do a full run
-        else
-          lego ${runOpts}
+        elif ! lego ${runOpts}; then
+          # Produce a nice error for those doing their first nixos-rebuild with these certs
+          echo Failed to fetch certificates. \
+            This may mean your DNS records are set up incorrectly. \
+            ${optionalString (cfg.preliminarySelfsigned) "Selfsigned certs are in place and dependant services will still start."}
+          # Exit 10 so that users can potentially amend SuccessExitStatus to ignore this error.
+          # High number to avoid Systemd reserved codes.
+          exit 10
         fi
 
         mv domainhash.txt certificates/
 
         # Group might change between runs, re-apply it
-        chown 'acme:${data.group}' certificates/*
+        chown '${user}:${data.group}' certificates/*
 
         # Copy all certs to the "real" certs directory
-        CERT='certificates/${keyName}.crt'
-        if [ -e "$CERT" ] && ! cmp -s "$CERT" out/fullchain.pem; then
+        if ! cmp -s 'certificates/${keyName}.crt' out/fullchain.pem; then
           touch out/renewed
           echo Installing new certificate
           cp -vp 'certificates/${keyName}.crt' out/fullchain.pem
@@ -420,31 +426,45 @@ let
 
   certConfigs = mapAttrs certToConfig cfg.certs;
 
-  certOpts = { name, ... }: {
+  # These options can be specified within
+  # security.acme.defaults or security.acme.certs.<name>
+  inheritableModule = isDefaults: { config, ... }: let
+    defaultAndText = name: default: {
+      # When ! isDefaults then this is the option declaration for the
+      # security.acme.certs.<name> path, which has the extra inheritDefaults
+      # option, which if disabled means that we can't inherit it
+      default = if isDefaults || ! config.inheritDefaults then default else cfg.defaults.${name};
+      # The docs however don't need to depend on inheritDefaults, they should
+      # stay constant. Though notably it wouldn't matter much, because to get
+      # the option information, a submodule with name `<name>` is evaluated
+      # without any definitions.
+      defaultText = if isDefaults then default else literalExpression "config.security.acme.defaults.${name}";
+    };
+  in {
     options = {
-      # user option has been removed
-      user = mkOption {
-        visible = false;
-        default = "_mkRemovedOptionModule";
+      validMinDays = mkOption {
+        type = types.int;
+        inherit (defaultAndText "validMinDays" 30) default defaultText;
+        description = "Minimum remaining validity before renewal in days.";
       };
 
-      # allowKeysForGroup option has been removed
-      allowKeysForGroup = mkOption {
-        visible = false;
-        default = "_mkRemovedOptionModule";
+      renewInterval = mkOption {
+        type = types.str;
+        inherit (defaultAndText "renewInterval" "daily") default defaultText;
+        description = ''
+          Systemd calendar expression when to check for renewal. See
+          <citerefentry><refentrytitle>systemd.time</refentrytitle>
+          <manvolnum>7</manvolnum></citerefentry>.
+        '';
       };
 
-      # extraDomains was replaced with extraDomainNames
-      extraDomains = mkOption {
-        visible = false;
-        default = "_mkMergedOptionModule";
+      enableDebugLogs = mkEnableOption "debug logging for this certificate" // {
+        inherit (defaultAndText "enableDebugLogs" true) default defaultText;
       };
 
-      enableDebugLogs = mkEnableOption "debug logging for this certificate" // { default = cfg.enableDebugLogs; };
-
       webroot = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        inherit (defaultAndText "webroot" null) default defaultText;
         example = "/var/lib/acme/acme-challenge";
         description = ''
           Where the webroot of the HTTP vhost is located.
@@ -455,20 +475,9 @@ let
         '';
       };
 
-      listenHTTP = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = ":1360";
-        description = ''
-          Interface and port to listen on to solve HTTP challenges
-          in the form [INTERFACE]:PORT.
-          If you use a port other than 80, you must proxy port 80 to this port.
-        '';
-      };
-
       server = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        inherit (defaultAndText "server" null) default defaultText;
         description = ''
           ACME Directory Resource URI. Defaults to Let's Encrypt's
           production endpoint,
@@ -476,27 +485,25 @@ let
         '';
       };
 
-      domain = mkOption {
-        type = types.str;
-        default = name;
-        description = "Domain to fetch certificate for (defaults to the entry name).";
-      };
-
       email = mkOption {
-        type = types.nullOr types.str;
-        default = cfg.email;
-        description = "Contact email address for the CA to be able to reach you.";
+        type = types.str;
+        inherit (defaultAndText "email" null) default defaultText;
+        description = ''
+          Email address for account creation and correspondence from the CA.
+          It is recommended to use the same email for all certs to avoid account
+          creation limits.
+        '';
       };
 
       group = mkOption {
         type = types.str;
-        default = "acme";
+        inherit (defaultAndText "group" "acme") default defaultText;
         description = "Group running the ACME client.";
       };
 
       reloadServices = mkOption {
         type = types.listOf types.str;
-        default = [];
+        inherit (defaultAndText "reloadServices" []) default defaultText;
         description = ''
           The list of systemd services to call <code>systemctl try-reload-or-restart</code>
           on.
@@ -505,7 +512,7 @@ let
 
       postRun = mkOption {
         type = types.lines;
-        default = "";
+        inherit (defaultAndText "postRun" "") default defaultText;
         example = "cp full.pem backup.pem";
         description = ''
           Commands to run after new certificates go live. Note that
@@ -515,30 +522,9 @@ let
         '';
       };
 
-      directory = mkOption {
-        type = types.str;
-        readOnly = true;
-        default = "/var/lib/acme/${name}";
-        description = "Directory where certificate and other state is stored.";
-      };
-
-      extraDomainNames = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        example = literalExpression ''
-          [
-            "example.org"
-            "mydomain.org"
-          ]
-        '';
-        description = ''
-          A list of extra domain names, which are included in the one certificate to be issued.
-        '';
-      };
-
       keyType = mkOption {
         type = types.str;
-        default = "ec256";
+        inherit (defaultAndText "keyType" "ec256") default defaultText;
         description = ''
           Key type to use for private keys.
           For an up to date list of supported values check the --key-type option
@@ -548,7 +534,7 @@ let
 
       dnsProvider = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        inherit (defaultAndText "dnsProvider" null) default defaultText;
         example = "route53";
         description = ''
           DNS Challenge provider. For a list of supported providers, see the "code"
@@ -558,7 +544,7 @@ let
 
       dnsResolver = mkOption {
         type = types.nullOr types.str;
-        default = null;
+        inherit (defaultAndText "dnsResolver" null) default defaultText;
         example = "1.1.1.1:53";
         description = ''
           Set the resolver to use for performing recursive DNS queries. Supported:
@@ -569,6 +555,7 @@ let
 
       credentialsFile = mkOption {
         type = types.path;
+        inherit (defaultAndText "credentialsFile" null) default defaultText;
         description = ''
           Path to an EnvironmentFile for the cert's service containing any required and
           optional environment variables for your selected dnsProvider.
@@ -580,7 +567,7 @@ let
 
       dnsPropagationCheck = mkOption {
         type = types.bool;
-        default = true;
+        inherit (defaultAndText "dnsPropagationCheck" true) default defaultText;
         description = ''
           Toggles lego DNS propagation check, which is used alongside DNS-01
           challenge to ensure the DNS entries required are available.
@@ -589,7 +576,7 @@ let
 
       ocspMustStaple = mkOption {
         type = types.bool;
-        default = false;
+        inherit (defaultAndText "ocspMustStaple" false) default defaultText;
         description = ''
           Turns on the OCSP Must-Staple TLS extension.
           Make sure you know what you're doing! See:
@@ -602,7 +589,7 @@ let
 
       extraLegoFlags = mkOption {
         type = types.listOf types.str;
-        default = [];
+        inherit (defaultAndText "extraLegoFlags" []) default defaultText;
         description = ''
           Additional global flags to pass to all lego commands.
         '';
@@ -610,7 +597,7 @@ let
 
       extraLegoRenewFlags = mkOption {
         type = types.listOf types.str;
-        default = [];
+        inherit (defaultAndText "extraLegoRenewFlags" []) default defaultText;
         description = ''
           Additional flags to pass to lego renew.
         '';
@@ -618,7 +605,7 @@ let
 
       extraLegoRunFlags = mkOption {
         type = types.listOf types.str;
-        default = [];
+        inherit (defaultAndText "extraLegoRunFlags" []) default defaultText;
         description = ''
           Additional flags to pass to lego run.
         '';
@@ -626,45 +613,80 @@ let
     };
   };
 
-in {
-
-  options = {
-    security.acme = {
+  certOpts = { name, config, ... }: {
+    options = {
+      # user option has been removed
+      user = mkOption {
+        visible = false;
+        default = "_mkRemovedOptionModule";
+      };
 
-      enableDebugLogs = mkEnableOption "debug logging for all certificates by default" // { default = true; };
+      # allowKeysForGroup option has been removed
+      allowKeysForGroup = mkOption {
+        visible = false;
+        default = "_mkRemovedOptionModule";
+      };
 
-      validMinDays = mkOption {
-        type = types.int;
-        default = 30;
-        description = "Minimum remaining validity before renewal in days.";
+      # extraDomains was replaced with extraDomainNames
+      extraDomains = mkOption {
+        visible = false;
+        default = "_mkMergedOptionModule";
       };
 
-      email = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = "Contact email address for the CA to be able to reach you.";
+      directory = mkOption {
+        type = types.str;
+        readOnly = true;
+        default = "/var/lib/acme/${name}";
+        description = "Directory where certificate and other state is stored.";
       };
 
-      renewInterval = mkOption {
+      domain = mkOption {
         type = types.str;
-        default = "daily";
+        default = name;
+        description = "Domain to fetch certificate for (defaults to the entry name).";
+      };
+
+      extraDomainNames = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        example = literalExpression ''
+          [
+            "example.org"
+            "mydomain.org"
+          ]
+        '';
         description = ''
-          Systemd calendar expression when to check for renewal. See
-          <citerefentry><refentrytitle>systemd.time</refentrytitle>
-          <manvolnum>7</manvolnum></citerefentry>.
+          A list of extra domain names, which are included in the one certificate to be issued.
         '';
       };
 
-      server = mkOption {
+      # This setting must be different for each configured certificate, otherwise
+      # two or more renewals may fail to bind to the address. Hence, it is not in
+      # the inheritableOpts.
+      listenHTTP = mkOption {
         type = types.nullOr types.str;
         default = null;
+        example = ":1360";
         description = ''
-          ACME Directory Resource URI. Defaults to Let's Encrypt's
-          production endpoint,
-          <link xlink:href="https://acme-v02.api.letsencrypt.org/directory"/>, if unset.
+          Interface and port to listen on to solve HTTP challenges
+          in the form [INTERFACE]:PORT.
+          If you use a port other than 80, you must proxy port 80 to this port.
         '';
       };
 
+      inheritDefaults = mkOption {
+        default = true;
+        example = true;
+        description = "Whether to inherit values set in `security.acme.defaults` or not.";
+        type = lib.types.bool;
+      };
+    };
+  };
+
+in {
+
+  options = {
+    security.acme = {
       preliminarySelfsigned = mkOption {
         type = types.bool;
         default = true;
@@ -687,9 +709,31 @@ in {
         '';
       };
 
+      useRoot = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to use the root user when generating certs. This is not recommended
+          for security + compatiblity reasons. If a service requires root owned certificates
+          consider following the guide on "Using ACME with services demanding root
+          owned certificates" in the NixOS manual, and only using this as a fallback
+          or for testing.
+        '';
+      };
+
+      defaults = mkOption {
+        type = types.submodule (inheritableModule true);
+        description = ''
+          Default values inheritable by all configured certs. You can
+          use this to define options shared by all your certs. These defaults
+          can also be ignored on a per-cert basis using the
+          `security.acme.certs.''${cert}.inheritDefaults' option.
+        '';
+      };
+
       certs = mkOption {
         default = { };
-        type = with types; attrsOf (submodule certOpts);
+        type = with types; attrsOf (submodule [ (inheritableModule false) certOpts ]);
         description = ''
           Attribute set of certificates to get signed and renewed. Creates
           <literal>acme-''${cert}.{service,timer}</literal> systemd units for
@@ -720,12 +764,16 @@ in {
 
       To use the let's encrypt staging server, use security.acme.server =
       "https://acme-staging-v02.api.letsencrypt.org/directory".
-    ''
-    )
+    '')
     (mkRemovedOptionModule [ "security" "acme" "directory" ] "ACME Directory is now hardcoded to /var/lib/acme and its permisisons are managed by systemd. See https://github.com/NixOS/nixpkgs/issues/53852 for more info.")
     (mkRemovedOptionModule [ "security" "acme" "preDelay" ] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
     (mkRemovedOptionModule [ "security" "acme" "activationDelay" ] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
-    (mkChangedOptionModule [ "security" "acme" "validMin" ] [ "security" "acme" "validMinDays" ] (config: config.security.acme.validMin / (24 * 3600)))
+    (mkChangedOptionModule [ "security" "acme" "validMin" ] [ "security" "acme" "defaults" "validMinDays" ] (config: config.security.acme.validMin / (24 * 3600)))
+    (mkChangedOptionModule [ "security" "acme" "validMinDays" ] [ "security" "acme" "defaults" "validMinDays" ] (config: config.security.acme.validMinDays))
+    (mkChangedOptionModule [ "security" "acme" "renewInterval" ] [ "security" "acme" "defaults" "renewInterval" ] (config: config.security.acme.renewInterval))
+    (mkChangedOptionModule [ "security" "acme" "email" ] [ "security" "acme" "defaults" "email" ] (config: config.security.acme.email))
+    (mkChangedOptionModule [ "security" "acme" "server" ] [ "security" "acme" "defaults" "server" ] (config: config.security.acme.server))
+    (mkChangedOptionModule [ "security" "acme" "enableDebugLogs" ] [ "security" "acme" "defaults" "enableDebugLogs" ] (config: config.security.acme.enableDebugLogs))
   ];
 
   config = mkMerge [
@@ -840,8 +888,8 @@ in {
         # Create some targets which can be depended on to be "active" after cert renewals
         finishedTargets = mapAttrs' (cert: conf: nameValuePair "acme-finished-${cert}" {
           wantedBy = [ "default.target" ];
-          requires = [ "acme-${cert}.service" ] ++ conf.selfsignedDeps;
-          after = [ "acme-${cert}.service" ] ++ conf.selfsignedDeps;
+          requires = [ "acme-${cert}.service" ];
+          after = [ "acme-${cert}.service" ];
         }) certConfigs;
 
         # Create targets to limit the number of simultaneous account creations
@@ -868,6 +916,6 @@ in {
 
   meta = {
     maintainers = lib.teams.acme.members;
-    doc = ./acme.xml;
+    doc = ./doc.xml;
   };
 }
diff --git a/nixos/modules/security/acme.xml b/nixos/modules/security/acme/doc.xml
index bf93800a0af4..f623cc509be6 100644
--- a/nixos/modules/security/acme.xml
+++ b/nixos/modules/security/acme/doc.xml
@@ -7,8 +7,9 @@
  <para>
   NixOS supports automatic domain validation &amp; certificate retrieval and
   renewal using the ACME protocol. Any provider can be used, but by default
-  NixOS uses Let's Encrypt. The alternative ACME client <literal>lego</literal>
-  is used under the hood.
+  NixOS uses Let's Encrypt. The alternative ACME client
+  <link xlink:href="https://go-acme.github.io/lego/">lego</link> is used under
+  the hood.
  </para>
  <para>
   Automatic cert validation and configuration for Apache and Nginx virtual
@@ -29,7 +30,7 @@
   <para>
    You must also set an email address to be used when creating accounts with
    Let's Encrypt. You can set this for all certs with
-   <literal><xref linkend="opt-security.acme.email" /></literal>
+   <literal><xref linkend="opt-security.acme.defaults.email" /></literal>
    and/or on a per-cert basis with
    <literal><xref linkend="opt-security.acme.certs._name_.email" /></literal>.
    This address is only used for registration and renewal reminders,
@@ -38,7 +39,7 @@
 
   <para>
    Alternatively, you can use a different ACME server by changing the
-   <literal><xref linkend="opt-security.acme.server" /></literal> option
+   <literal><xref linkend="opt-security.acme.defaults.server" /></literal> option
    to a provider of your choosing, or just change the server for one cert with
    <literal><xref linkend="opt-security.acme.certs._name_.server" /></literal>.
   </para>
@@ -60,12 +61,12 @@
    = true;</literal> in a virtualHost config. We first create self-signed
    placeholder certificates in place of the real ACME certs. The placeholder
    certs are overwritten when the ACME certs arrive. For
-   <literal>foo.example.com</literal> the config would look like.
+   <literal>foo.example.com</literal> the config would look like this:
   </para>
 
 <programlisting>
 <xref linkend="opt-security.acme.acceptTerms" /> = true;
-<xref linkend="opt-security.acme.email" /> = "admin+acme@example.com";
+<xref linkend="opt-security.acme.defaults.email" /> = "admin+acme@example.com";
 services.nginx = {
   <link linkend="opt-services.nginx.enable">enable</link> = true;
   <link linkend="opt-services.nginx.virtualHosts">virtualHosts</link> = {
@@ -114,7 +115,7 @@ services.nginx = {
 
 <programlisting>
 <xref linkend="opt-security.acme.acceptTerms" /> = true;
-<xref linkend="opt-security.acme.email" /> = "admin+acme@example.com";
+<xref linkend="opt-security.acme.defaults.email" /> = "admin+acme@example.com";
 
 # /var/lib/acme/.challenges must be writable by the ACME user
 # and readable by the Nginx user. The easiest way to achieve
@@ -218,7 +219,7 @@ services.bind = {
 
 # Now we can configure ACME
 <xref linkend="opt-security.acme.acceptTerms" /> = true;
-<xref linkend="opt-security.acme.email" /> = "admin+acme@example.com";
+<xref linkend="opt-security.acme.defaults.email" /> = "admin+acme@example.com";
 <xref linkend="opt-security.acme.certs" />."example.com" = {
   <link linkend="opt-security.acme.certs._name_.domain">domain</link> = "*.example.com";
   <link linkend="opt-security.acme.certs._name_.dnsProvider">dnsProvider</link> = "rfc2136";
@@ -231,25 +232,39 @@ services.bind = {
   <para>
    The <filename>dnskeys.conf</filename> and <filename>certs.secret</filename>
    must be kept secure and thus you should not keep their contents in your
-   Nix config. Instead, generate them one time with these commands:
+   Nix config. Instead, generate them one time with a systemd service:
   </para>
 
 <programlisting>
-mkdir -p /var/lib/secrets
-tsig-keygen rfc2136key.example.com &gt; /var/lib/secrets/dnskeys.conf
-chown named:root /var/lib/secrets/dnskeys.conf
-chmod 400 /var/lib/secrets/dnskeys.conf
-
-# Copy the secret value from the dnskeys.conf, and put it in
-# RFC2136_TSIG_SECRET below
-
-cat &gt; /var/lib/secrets/certs.secret &lt;&lt; EOF
-RFC2136_NAMESERVER='127.0.0.1:53'
-RFC2136_TSIG_ALGORITHM='hmac-sha256.'
-RFC2136_TSIG_KEY='rfc2136key.example.com'
-RFC2136_TSIG_SECRET='your secret key'
-EOF
-chmod 400 /var/lib/secrets/certs.secret
+systemd.services.dns-rfc2136-conf = {
+  requiredBy = ["acme-example.com.service", "bind.service"];
+  before = ["acme-example.com.service", "bind.service"];
+  unitConfig = {
+    ConditionPathExists = "!/var/lib/secrets/dnskeys.conf";
+  };
+  serviceConfig = {
+    Type = "oneshot";
+    UMask = 0077;
+  };
+  path = [ pkgs.bind ];
+  script = ''
+    mkdir -p /var/lib/secrets
+    tsig-keygen rfc2136key.example.com &gt; /var/lib/secrets/dnskeys.conf
+    chown named:root /var/lib/secrets/dnskeys.conf
+    chmod 400 /var/lib/secrets/dnskeys.conf
+
+    # Copy the secret value from the dnskeys.conf, and put it in
+    # RFC2136_TSIG_SECRET below
+
+    cat &gt; /var/lib/secrets/certs.secret &lt;&lt; EOF
+    RFC2136_NAMESERVER='127.0.0.1:53'
+    RFC2136_TSIG_ALGORITHM='hmac-sha256.'
+    RFC2136_TSIG_KEY='rfc2136key.example.com'
+    RFC2136_TSIG_SECRET='your secret key'
+    EOF
+    chmod 400 /var/lib/secrets/certs.secret
+  '';
+};
 </programlisting>
 
   <para>
@@ -258,6 +273,106 @@ chmod 400 /var/lib/secrets/certs.secret
    journalctl -fu acme-example.com.service</literal> and watching its log output.
   </para>
  </section>
+
+ <section xml:id="module-security-acme-config-dns-with-vhosts">
+  <title>Using DNS validation with web server virtual hosts</title>
+
+  <para>
+   It is possible to use DNS-01 validation with all certificates,
+   including those automatically configured via the Nginx/Apache
+   <literal><link linkend="opt-services.nginx.virtualHosts._name_.enableACME">enableACME</link></literal>
+   option. This configuration pattern is fully
+   supported and part of the module's test suite for Nginx + Apache.
+  </para>
+
+  <para>
+   You must follow the guide above on configuring DNS-01 validation
+   first, however instead of setting the options for one certificate
+   (e.g. <xref linkend="opt-security.acme.certs._name_.dnsProvider" />)
+   you will set them as defaults
+   (e.g. <xref linkend="opt-security.acme.defaults.dnsProvider" />).
+  </para>
+
+<programlisting>
+# Configure ACME appropriately
+<xref linkend="opt-security.acme.acceptTerms" /> = true;
+<xref linkend="opt-security.acme.defaults.email" /> = "admin+acme@example.com";
+<xref linkend="opt-security.acme.defaults" /> = {
+  <link linkend="opt-security.acme.defaults.dnsProvider">dnsProvider</link> = "rfc2136";
+  <link linkend="opt-security.acme.defaults.credentialsFile">credentialsFile</link> = "/var/lib/secrets/certs.secret";
+  # We don't need to wait for propagation since this is a local DNS server
+  <link linkend="opt-security.acme.defaults.dnsPropagationCheck">dnsPropagationCheck</link> = false;
+};
+
+# For each virtual host you would like to use DNS-01 validation with,
+# set acmeRoot = null
+services.nginx = {
+  <link linkend="opt-services.nginx.enable">enable</link> = true;
+  <link linkend="opt-services.nginx.virtualHosts">virtualHosts</link> = {
+    "foo.example.com" = {
+      <link linkend="opt-services.nginx.virtualHosts._name_.enableACME">enableACME</link> = true;
+      <link linkend="opt-services.nginx.virtualHosts._name_.acmeRoot">acmeRoot</link> = null;
+    };
+  };
+}
+</programlisting>
+
+  <para>
+   And that's it! Next time your configuration is rebuilt, or when
+   you add a new virtualHost, it will be DNS-01 validated.
+  </para>
+ </section>
+
+ <section xml:id="module-security-acme-root-owned">
+  <title>Using ACME with services demanding root owned certificates</title>
+
+  <para>
+   Some services refuse to start if the configured certificate files
+   are not owned by root. PostgreSQL and OpenSMTPD are examples of these.
+   There is no way to change the user the ACME module uses (it will always be
+   <literal>acme</literal>), however you can use systemd's
+   <literal>LoadCredential</literal> feature to resolve this elegantly.
+   Below is an example configuration for OpenSMTPD, but this pattern
+   can be applied to any service.
+  </para>
+
+<programlisting>
+# Configure ACME however you like (DNS or HTTP validation), adding
+# the following configuration for the relevant certificate.
+# Note: You cannot use `systemctl reload` here as that would mean
+# the LoadCredential configuration below would be skipped and
+# the service would continue to use old certificates.
+security.acme.certs."mail.example.com".postRun = ''
+  systemctl restart opensmtpd
+'';
+
+# Now you must augment OpenSMTPD's systemd service to load
+# the certificate files.
+<link linkend="opt-systemd.services._name_.requires">systemd.services.opensmtpd.requires</link> = ["acme-finished-mail.example.com.target"];
+<link linkend="opt-systemd.services._name_.serviceConfig">systemd.services.opensmtpd.serviceConfig.LoadCredential</link> = let
+  certDir = config.security.acme.certs."mail.example.com".directory;
+in [
+  "cert.pem:${certDir}/cert.pem"
+  "key.pem:${certDir}/key.pem"
+];
+
+# Finally, configure OpenSMTPD to use these certs.
+services.opensmtpd = let
+  credsDir = "/run/credentials/opensmtpd.service";
+in {
+  enable = true;
+  setSendmail = false;
+  serverConfiguration = ''
+    pki mail.example.com cert "${credsDir}/cert.pem"
+    pki mail.example.com key "${credsDir}/key.pem"
+    listen on localhost tls pki mail.example.com
+    action act1 relay host smtp://127.0.0.1:10027
+    match for local action act1
+  '';
+};
+</programlisting>
+ </section>
+
  <section xml:id="module-security-acme-regenerate">
   <title>Regenerating certificates</title>
 
diff --git a/nixos/modules/security/acme/mk-cert-ownership-assertion.nix b/nixos/modules/security/acme/mk-cert-ownership-assertion.nix
new file mode 100644
index 000000000000..b80d89aeb9fc
--- /dev/null
+++ b/nixos/modules/security/acme/mk-cert-ownership-assertion.nix
@@ -0,0 +1,4 @@
+{ cert, group, groups, user }: {
+  assertion = cert.group == group || builtins.any (u: u == user) groups.${cert.group}.members;
+  message = "Group for certificate ${cert.domain} must be ${group}, or user ${user} must be a member of group ${cert.group}";
+}
diff --git a/nixos/modules/security/dhparams.nix b/nixos/modules/security/dhparams.nix
index 012be2887d89..cfa9003f12fb 100644
--- a/nixos/modules/security/dhparams.nix
+++ b/nixos/modules/security/dhparams.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
-  inherit (lib) mkOption types;
+  inherit (lib) literalExpression mkOption types;
   cfg = config.security.dhparams;
+  opt = options.security.dhparams;
 
   bitType = types.addCheck types.int (b: b >= 16) // {
     name = "bits";
@@ -13,6 +14,7 @@ let
     options.bits = mkOption {
       type = bitType;
       default = cfg.defaultBitSize;
+      defaultText = literalExpression "config.${opt.defaultBitSize}";
       description = ''
         The bit size for the prime that is used during a Diffie-Hellman
         key exchange.
diff --git a/nixos/modules/security/google_oslogin.nix b/nixos/modules/security/google_oslogin.nix
index c2889a0f0d1d..cf416035ef60 100644
--- a/nixos/modules/security/google_oslogin.nix
+++ b/nixos/modules/security/google_oslogin.nix
@@ -5,7 +5,7 @@ with lib;
 let
 
   cfg = config.security.googleOsLogin;
-  package = pkgs.google-compute-engine-oslogin;
+  package = pkgs.google-guest-oslogin;
 
 in
 
@@ -17,7 +17,7 @@ in
       type = types.bool;
       default = false;
       description = ''
-        Whether to enable Google OS Login
+        Whether to enable Google OS Login.
 
         The OS Login package enables the following components:
         AuthorizedKeysCommand to query valid SSH keys from the user's OS Login
@@ -36,7 +36,7 @@ in
     security.pam.services.sshd = {
       makeHomeDir = true;
       googleOsLoginAccountVerification = true;
-      # disabled for now: googleOsLoginAuthentication = true;
+      googleOsLoginAuthentication = true;
     };
 
     security.sudo.extraConfig = ''
@@ -47,6 +47,9 @@ in
       "d /var/google-users.d 750 root root -"
     ];
 
+    systemd.packages = [ package ];
+    systemd.timers.google-oslogin-cache.wantedBy = [ "timers.target" ];
+
     # enable the nss module, so user lookups etc. work
     system.nssModules = [ package ];
     system.nssDatabases.passwd = [ "cache_oslogin" "oslogin" ];
diff --git a/nixos/modules/security/misc.nix b/nixos/modules/security/misc.nix
index e7abc1e0d597..c20e067b8cc7 100644
--- a/nixos/modules/security/misc.nix
+++ b/nixos/modules/security/misc.nix
@@ -123,8 +123,8 @@ with lib;
       boot.kernel.sysctl."user.max_user_namespaces" = 0;
 
       assertions = [
-        { assertion = config.nix.useSandbox -> config.security.allowUserNamespaces;
-          message = "`nix.useSandbox = true` conflicts with `!security.allowUserNamespaces`.";
+        { assertion = config.nix.settings.sandbox -> config.security.allowUserNamespaces;
+          message = "`nix.settings.sandbox = true` conflicts with `!security.allowUserNamespaces`.";
         }
       ];
     })
diff --git a/nixos/modules/security/pam.nix b/nixos/modules/security/pam.nix
index 0944b36c6d19..c0ef8b5f30bd 100644
--- a/nixos/modules/security/pam.nix
+++ b/nixos/modules/security/pam.nix
@@ -444,15 +444,15 @@ let
             account sufficient ${pam_krb5}/lib/security/pam_krb5.so
           '' +
           optionalString cfg.googleOsLoginAccountVerification ''
-            account [success=ok ignore=ignore default=die] ${pkgs.google-compute-engine-oslogin}/lib/pam_oslogin_login.so
-            account [success=ok default=ignore] ${pkgs.google-compute-engine-oslogin}/lib/pam_oslogin_admin.so
+            account [success=ok ignore=ignore default=die] ${pkgs.google-guest-oslogin}/lib/security/pam_oslogin_login.so
+            account [success=ok default=ignore] ${pkgs.google-guest-oslogin}/lib/security/pam_oslogin_admin.so
           '' +
           ''
 
             # Authentication management.
           '' +
           optionalString cfg.googleOsLoginAuthentication ''
-            auth [success=done perm_denied=bad default=ignore] ${pkgs.google-compute-engine-oslogin}/lib/pam_oslogin_login.so
+            auth [success=done perm_denied=die default=ignore] ${pkgs.google-guest-oslogin}/lib/security/pam_oslogin_login.so
           '' +
           optionalString cfg.rootOK ''
             auth sufficient pam_rootok.so
@@ -518,7 +518,7 @@ let
                 auth optional ${pkgs.pam_gnupg}/lib/security/pam_gnupg.so ${optionalString cfg.gnupg.storeOnly " store-only"}
               '' +
               optionalString cfg.googleAuthenticator.enable ''
-                auth required ${pkgs.googleAuthenticator}/lib/security/pam_google_authenticator.so no_increment_hotp
+                auth required ${pkgs.google-authenticator}/lib/security/pam_google_authenticator.so no_increment_hotp
               '' +
               optionalString cfg.duoSecurity.enable ''
                 auth required ${pkgs.duo-unix}/lib/security/pam_duo.so
@@ -1035,7 +1035,7 @@ in
         setuid = true;
         owner = "root";
         group = "root";
-        source = "${pkgs.pam}/sbin/unix_chkpwd.orig";
+        source = "${pkgs.pam}/bin/unix_chkpwd";
       };
     };
 
@@ -1072,8 +1072,8 @@ in
     security.apparmor.includes."abstractions/pam" = let
       isEnabled = test: fold or false (map test (attrValues config.security.pam.services));
       in
-      lib.concatMapStringsSep "\n"
-        (name: "r ${config.environment.etc."pam.d/${name}".source},")
+      lib.concatMapStrings
+        (name: "r ${config.environment.etc."pam.d/${name}".source},\n")
         (attrNames config.security.pam.services) +
       ''
       mr ${getLib pkgs.pam}/lib/security/pam_filter/*,
@@ -1091,11 +1091,11 @@ in
         mr ${pam_ccreds}/lib/security/pam_ccreds.so,
       '' +
       optionalString (isEnabled (cfg: cfg.googleOsLoginAccountVerification)) ''
-        mr ${pkgs.google-compute-engine-oslogin}/lib/pam_oslogin_login.so,
-        mr ${pkgs.google-compute-engine-oslogin}/lib/pam_oslogin_admin.so,
+        mr ${pkgs.google-guest-oslogin}/lib/security/pam_oslogin_login.so,
+        mr ${pkgs.google-guest-oslogin}/lib/security/pam_oslogin_admin.so,
       '' +
       optionalString (isEnabled (cfg: cfg.googleOsLoginAuthentication)) ''
-        mr ${pkgs.google-compute-engine-oslogin}/lib/pam_oslogin_login.so,
+        mr ${pkgs.google-guest-oslogin}/lib/security/pam_oslogin_login.so,
       '' +
       optionalString (config.security.pam.enableSSHAgentAuth
                      && isEnabled (cfg: cfg.sshAgentAuth)) ''
diff --git a/nixos/modules/security/polkit.nix b/nixos/modules/security/polkit.nix
index d9c58152f1fa..1ba149745c65 100644
--- a/nixos/modules/security/polkit.nix
+++ b/nixos/modules/security/polkit.nix
@@ -12,11 +12,7 @@ in
 
   options = {
 
-    security.polkit.enable = mkOption {
-      type = types.bool;
-      default = true;
-      description = "Whether to enable PolKit.";
-    };
+    security.polkit.enable = mkEnableOption "polkit";
 
     security.polkit.extraConfig = mkOption {
       type = types.lines;
diff --git a/nixos/modules/security/systemd-confinement.nix b/nixos/modules/security/systemd-confinement.nix
index 0e3ec5af323e..f3a2de3bf87a 100644
--- a/nixos/modules/security/systemd-confinement.nix
+++ b/nixos/modules/security/systemd-confinement.nix
@@ -175,8 +175,8 @@ in {
       serviceName = "${name}.service";
       excludedPath = rootPaths;
     } ''
-      mkdir -p "$out/lib/systemd/system"
-      serviceFile="$out/lib/systemd/system/$serviceName"
+      mkdir -p "$out/lib/systemd/system/$serviceName.d"
+      serviceFile="$out/lib/systemd/system/$serviceName.d/confinement.conf"
 
       echo '[Service]' > "$serviceFile"
 
diff --git a/nixos/modules/security/wrappers/default.nix b/nixos/modules/security/wrappers/default.nix
index a47de7e04f7a..e63f19010de8 100644
--- a/nixos/modules/security/wrappers/default.nix
+++ b/nixos/modules/security/wrappers/default.nix
@@ -92,7 +92,6 @@ let
     , permissions
     , ...
     }:
-    assert (lib.versionAtLeast (lib.getVersion config.boot.kernelPackages.kernel) "4.3");
     ''
       cp ${securityWrapper}/bin/security-wrapper "$wrapperDir/${program}"
       echo -n "${source}" > "$wrapperDir/${program}.real"
@@ -244,8 +243,6 @@ in
     security.apparmor.includes."nixos/security.wrappers" = ''
       include "${pkgs.apparmorRulesFromClosure { name="security.wrappers"; } [
         securityWrapper
-        pkgs.stdenv.cc.cc
-        pkgs.stdenv.cc.libc
       ]}"
     '';
 
diff --git a/nixos/modules/services/admin/pgadmin.nix b/nixos/modules/services/admin/pgadmin.nix
new file mode 100644
index 000000000000..80b681454104
--- /dev/null
+++ b/nixos/modules/services/admin/pgadmin.nix
@@ -0,0 +1,127 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  pkg = pkgs.pgadmin4;
+  cfg = config.services.pgadmin;
+
+  _base = with types; [ int bool str ];
+  base = with types; oneOf ([ (listOf (oneOf _base)) (attrsOf (oneOf _base)) ] ++ _base);
+
+  formatAttrset = attr:
+    "{${concatStringsSep "\n" (mapAttrsToList (key: value: "${builtins.toJSON key}: ${formatPyValue value},") attr)}}";
+
+  formatPyValue = value:
+    if builtins.isString value then builtins.toJSON value
+    else if value ? _expr then value._expr
+    else if builtins.isInt value then toString value
+    else if builtins.isBool value then (if value then "True" else "False")
+    else if builtins.isAttrs value then (formatAttrset value)
+    else if builtins.isList value then "[${concatStringsSep "\n" (map (v: "${formatPyValue v},") value)}]"
+    else throw "Unrecognized type";
+
+  formatPy = attrs:
+    concatStringsSep "\n" (mapAttrsToList (key: value: "${key} = ${formatPyValue value}") attrs);
+
+  pyType = with types; attrsOf (oneOf [ (attrsOf base) (listOf base) base ]);
+in
+{
+  options.services.pgadmin = {
+    enable = mkEnableOption "PostgreSQL Admin 4";
+
+    port = mkOption {
+      description = "Port for pgadmin4 to run on";
+      type = types.port;
+      default = 5050;
+    };
+
+    initialEmail = mkOption {
+      description = "Initial email for the pgAdmin account.";
+      type = types.str;
+    };
+
+    initialPasswordFile = mkOption {
+      description = ''
+        Initial password file for the pgAdmin account.
+        NOTE: Should be string not a store path, to prevent the password from being world readable.
+      '';
+      type = types.path;
+    };
+
+    openFirewall = mkEnableOption "firewall passthrough for pgadmin4";
+
+    settings = mkOption {
+      description = ''
+        Settings for pgadmin4.
+        <link xlink:href="https://www.pgadmin.org/docs/pgadmin4/development/config_py.html">Documentation</link>.
+      '';
+      type = pyType;
+      default= {};
+    };
+  };
+
+  config = mkIf (cfg.enable) {
+    networking.firewall.allowedTCPPorts = mkIf (cfg.openFirewall) [ cfg.port ];
+
+    services.pgadmin.settings = {
+      DEFAULT_SERVER_PORT = cfg.port;
+      SERVER_MODE = true;
+    } // (optionalAttrs cfg.openFirewall {
+      DEFAULT_SERVER = mkDefault "::";
+    });
+
+    systemd.services.pgadmin = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      requires = [ "network.target" ];
+      # we're adding this optionally so just in case there's any race it'll be caught
+      # in case postgres doesn't start, pgadmin will just start normally
+      wants = [ "postgresql.service" ];
+
+      path = [ config.services.postgresql.package pkgs.coreutils pkgs.bash ];
+
+      preStart = ''
+        # NOTE: this is idempotent (aka running it twice has no effect)
+        (
+          # Email address:
+          echo ${escapeShellArg cfg.initialEmail}
+
+          # file might not contain newline. echo hack fixes that.
+          PW=$(cat ${escapeShellArg cfg.initialPasswordFile})
+
+          # Password:
+          echo "$PW"
+          # Retype password:
+          echo "$PW"
+        ) | ${pkg}/bin/pgadmin4-setup
+      '';
+
+      restartTriggers = [
+        "/etc/pgadmin/config_system.py"
+      ];
+
+      serviceConfig = {
+        User = "pgadmin";
+        DynamicUser = true;
+        LogsDirectory = "pgadmin";
+        StateDirectory = "pgadmin";
+        ExecStart = "${pkg}/bin/pgadmin4";
+      };
+    };
+
+    users.users.pgadmin = {
+      isSystemUser = true;
+      group = "pgadmin";
+    };
+
+    users.groups.pgadmin = {};
+
+    environment.etc."pgadmin/config_system.py" = {
+      text = formatPy cfg.settings;
+      mode = "0600";
+      user = "pgadmin";
+      group = "pgadmin";
+    };
+  };
+}
diff --git a/nixos/modules/services/audio/jmusicbot.nix b/nixos/modules/services/audio/jmusicbot.nix
index f573bd2ab8dd..e0f8d461af07 100644
--- a/nixos/modules/services/audio/jmusicbot.nix
+++ b/nixos/modules/services/audio/jmusicbot.nix
@@ -9,6 +9,13 @@ in
     services.jmusicbot = {
       enable = mkEnableOption "jmusicbot, a Discord music bot that's easy to set up and run yourself";
 
+      package = mkOption {
+        type = types.package;
+        default = pkgs.jmusicbot;
+        defaultText = literalExpression "pkgs.jmusicbot";
+        description = "JMusicBot package to use";
+      };
+
       stateDir = mkOption {
         type = types.path;
         description = ''
@@ -27,7 +34,7 @@ in
       after = [ "network-online.target" ];
       description = "Discord music bot that's easy to set up and run yourself!";
       serviceConfig = mkMerge [{
-        ExecStart = "${pkgs.jmusicbot}/bin/JMusicBot";
+        ExecStart = "${cfg.package}/bin/JMusicBot";
         WorkingDirectory = cfg.stateDir;
         Restart = "always";
         RestartSec = 20;
diff --git a/nixos/modules/services/audio/mpd.nix b/nixos/modules/services/audio/mpd.nix
index 560264e249d0..586b9ffa6888 100644
--- a/nixos/modules/services/audio/mpd.nix
+++ b/nixos/modules/services/audio/mpd.nix
@@ -209,62 +209,42 @@ in {
 
   config = mkIf cfg.enable {
 
+    # install mpd units
+    systemd.packages = [ pkgs.mpd ];
+
     systemd.sockets.mpd = mkIf cfg.startWhenNeeded {
-      description = "Music Player Daemon Socket";
       wantedBy = [ "sockets.target" ];
       listenStreams = [
         (if pkgs.lib.hasPrefix "/" cfg.network.listenAddress
           then cfg.network.listenAddress
           else "${optionalString (cfg.network.listenAddress != "any") "${cfg.network.listenAddress}:"}${toString cfg.network.port}")
       ];
-      socketConfig = {
-        Backlog = 5;
-        KeepAlive = true;
-        PassCredentials = true;
-      };
     };
 
     systemd.services.mpd = {
-      after = [ "network.target" "sound.target" ];
-      description = "Music Player Daemon";
       wantedBy = optional (!cfg.startWhenNeeded) "multi-user.target";
 
-      serviceConfig = mkMerge [
+      preStart =
+        ''
+          set -euo pipefail
+          install -m 600 ${mpdConf} /run/mpd/mpd.conf
+        '' + optionalString (cfg.credentials != [])
+        (concatStringsSep "\n"
+          (imap0
+            (i: c: ''${pkgs.replace-secret}/bin/replace-secret '{{password-${toString i}}}' '${c.passwordFile}' /run/mpd/mpd.conf'')
+            cfg.credentials));
+
+      serviceConfig =
         {
           User = "${cfg.user}";
-          ExecStart = "${pkgs.mpd}/bin/mpd --no-daemon /run/mpd/mpd.conf";
-          ExecStartPre = pkgs.writeShellScript "mpd-start-pre" (''
-            set -euo pipefail
-            install -m 600 ${mpdConf} /run/mpd/mpd.conf
-          '' + optionalString (cfg.credentials != [])
-            (concatStringsSep "\n"
-              (imap0
-                (i: c: ''${pkgs.replace-secret}/bin/replace-secret '{{password-${toString i}}}' '${c.passwordFile}' /run/mpd/mpd.conf'')
-                cfg.credentials))
-          );
+          # Note: the first "" overrides the ExecStart from the upstream unit
+          ExecStart = [ "" "${pkgs.mpd}/bin/mpd --systemd /run/mpd/mpd.conf" ];
           RuntimeDirectory = "mpd";
-          Type = "notify";
-          LimitRTPRIO = 50;
-          LimitRTTIME = "infinity";
-          ProtectSystem = true;
-          NoNewPrivileges = true;
-          ProtectKernelTunables = true;
-          ProtectControlGroups = true;
-          ProtectKernelModules = true;
-          RestrictAddressFamilies = "AF_INET AF_INET6 AF_UNIX AF_NETLINK";
-          RestrictNamespaces = true;
-          Restart = "always";
-        }
-        (mkIf (cfg.dataDir == "/var/lib/${name}") {
-          StateDirectory = [ name ];
-        })
-        (mkIf (cfg.playlistDirectory == "/var/lib/${name}/playlists") {
-          StateDirectory = [ name "${name}/playlists" ];
-        })
-        (mkIf (cfg.musicDirectory == "/var/lib/${name}/music") {
-          StateDirectory = [ name "${name}/music" ];
-        })
-      ];
+          StateDirectory = []
+            ++ optionals (cfg.dataDir == "/var/lib/${name}") [ name ]
+            ++ optionals (cfg.playlistDirectory == "/var/lib/${name}/playlists") [ name "${name}/playlists" ]
+            ++ optionals (cfg.musicDirectory == "/var/lib/${name}/music")        [ name "${name}/music" ];
+        };
     };
 
     users.users = optionalAttrs (cfg.user == name) {
diff --git a/nixos/modules/services/audio/mpdscribble.nix b/nixos/modules/services/audio/mpdscribble.nix
index 1368543ae1a4..333ffb709410 100644
--- a/nixos/modules/services/audio/mpdscribble.nix
+++ b/nixos/modules/services/audio/mpdscribble.nix
@@ -1,10 +1,11 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.mpdscribble;
   mpdCfg = config.services.mpd;
+  mpdOpt = options.services.mpd;
 
   endpointUrls = {
     "last.fm" = "http://post.audioscrobbler.com";
@@ -108,6 +109,11 @@ in {
         mpdCfg.network.listenAddress
       else
         "localhost");
+      defaultText = literalExpression ''
+        if config.${mpdOpt.network.listenAddress} != "any"
+        then config.${mpdOpt.network.listenAddress}
+        else "localhost"
+      '';
       type = types.str;
       description = ''
         Host for the mpdscribble daemon to search for a mpd daemon on.
@@ -122,6 +128,10 @@ in {
           mpdCfg.credentials).passwordFile
       else
         null;
+      defaultText = literalDocBook ''
+        The first password file with read access configured for MPD when using a local instance,
+        otherwise <literal>null</literal>.
+      '';
       type = types.nullOr types.str;
       description = ''
         File containing the password for the mpd daemon.
@@ -132,6 +142,7 @@ in {
 
     port = mkOption {
       default = mpdCfg.network.port;
+      defaultText = literalExpression "config.${mpdOpt.network.port}";
       type = types.port;
       description = ''
         Port for the mpdscribble daemon to search for a mpd daemon on.
diff --git a/nixos/modules/services/audio/roon-server.nix b/nixos/modules/services/audio/roon-server.nix
index 566c7cae42ce..de1f61c8e73b 100644
--- a/nixos/modules/services/audio/roon-server.nix
+++ b/nixos/modules/services/audio/roon-server.nix
@@ -51,7 +51,10 @@ in {
     };
 
     networking.firewall = mkIf cfg.openFirewall {
-      allowedTCPPortRanges = [{ from = 9100; to = 9200; }];
+      allowedTCPPortRanges = [
+        { from = 9100; to = 9200; }
+        { from = 9330; to = 9332; }
+      ];
       allowedUDPPorts = [ 9003 ];
       extraCommands = ''
         iptables -A INPUT -s 224.0.0.0/4 -j ACCEPT
diff --git a/nixos/modules/services/audio/snapserver.nix b/nixos/modules/services/audio/snapserver.nix
index d3e97719f357..6d5ce98df895 100644
--- a/nixos/modules/services/audio/snapserver.nix
+++ b/nixos/modules/services/audio/snapserver.nix
@@ -44,24 +44,24 @@ let
 
   optionString = concatStringsSep " " (mapAttrsToList streamToOption cfg.streams
     # global options
-    ++ [ "--stream.bind_to_address ${cfg.listenAddress}" ]
-    ++ [ "--stream.port ${toString cfg.port}" ]
-    ++ optionalNull cfg.sampleFormat "--stream.sampleformat ${cfg.sampleFormat}"
-    ++ optionalNull cfg.codec "--stream.codec ${cfg.codec}"
-    ++ optionalNull cfg.streamBuffer "--stream.stream_buffer ${toString cfg.streamBuffer}"
-    ++ optionalNull cfg.buffer "--stream.buffer ${toString cfg.buffer}"
+    ++ [ "--stream.bind_to_address=${cfg.listenAddress}" ]
+    ++ [ "--stream.port=${toString cfg.port}" ]
+    ++ optionalNull cfg.sampleFormat "--stream.sampleformat=${cfg.sampleFormat}"
+    ++ optionalNull cfg.codec "--stream.codec=${cfg.codec}"
+    ++ optionalNull cfg.streamBuffer "--stream.stream_buffer=${toString cfg.streamBuffer}"
+    ++ optionalNull cfg.buffer "--stream.buffer=${toString cfg.buffer}"
     ++ optional cfg.sendToMuted "--stream.send_to_muted"
     # tcp json rpc
-    ++ [ "--tcp.enabled ${toString cfg.tcp.enable}" ]
+    ++ [ "--tcp.enabled=${toString cfg.tcp.enable}" ]
     ++ optionals cfg.tcp.enable [
-      "--tcp.address ${cfg.tcp.listenAddress}"
-      "--tcp.port ${toString cfg.tcp.port}" ]
+      "--tcp.bind_to_address=${cfg.tcp.listenAddress}"
+      "--tcp.port=${toString cfg.tcp.port}" ]
      # http json rpc
-    ++ [ "--http.enabled ${toString cfg.http.enable}" ]
+    ++ [ "--http.enabled=${toString cfg.http.enable}" ]
     ++ optionals cfg.http.enable [
-      "--http.address ${cfg.http.listenAddress}"
-      "--http.port ${toString cfg.http.port}"
-    ] ++ optional (cfg.http.docRoot != null) "--http.doc_root \"${toString cfg.http.docRoot}\"");
+      "--http.bind_to_address=${cfg.http.listenAddress}"
+      "--http.port=${toString cfg.http.port}"
+    ] ++ optional (cfg.http.docRoot != null) "--http.doc_root=\"${toString cfg.http.docRoot}\"");
 
 in {
   imports = [
diff --git a/nixos/modules/services/audio/squeezelite.nix b/nixos/modules/services/audio/squeezelite.nix
index 05506f5bcc7a..36295e21c60f 100644
--- a/nixos/modules/services/audio/squeezelite.nix
+++ b/nixos/modules/services/audio/squeezelite.nix
@@ -1,50 +1,46 @@
 { config, lib, pkgs, ... }:
 
-with lib;
-
 let
+  inherit (lib) mkEnableOption mkIf mkOption optionalString types;
+
   dataDir = "/var/lib/squeezelite";
   cfg = config.services.squeezelite;
+  pkg = if cfg.pulseAudio then pkgs.squeezelite-pulse else pkgs.squeezelite;
+  bin = "${pkg}/bin/${pkg.pname}";
 
-in {
+in
+{
 
   ###### interface
 
-  options = {
-
-    services.squeezelite= {
+  options.services.squeezelite = {
+    enable = mkEnableOption "Squeezelite, a software Squeezebox emulator";
 
-      enable = mkEnableOption "Squeezelite, a software Squeezebox emulator";
-
-      extraArguments = mkOption {
-        default = "";
-        type = types.str;
-        description = ''
-          Additional command line arguments to pass to Squeezelite.
-        '';
-      };
+    pulseAudio = mkEnableOption "pulseaudio support";
 
+    extraArguments = mkOption {
+      default = "";
+      type = types.str;
+      description = ''
+        Additional command line arguments to pass to Squeezelite.
+      '';
     };
-
   };
 
 
   ###### implementation
 
   config = mkIf cfg.enable {
-
-    systemd.services.squeezelite= {
+    systemd.services.squeezelite = {
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" "sound.target" ];
       description = "Software Squeezebox emulator";
       serviceConfig = {
         DynamicUser = true;
-        ExecStart = "${pkgs.squeezelite}/bin/squeezelite -N ${dataDir}/player-name ${cfg.extraArguments}";
+        ExecStart = "${bin} -N ${dataDir}/player-name ${cfg.extraArguments}";
         StateDirectory = builtins.baseNameOf dataDir;
         SupplementaryGroups = "audio";
       };
     };
-
   };
-
 }
diff --git a/nixos/modules/services/backup/borgbackup.nix b/nixos/modules/services/backup/borgbackup.nix
index 220c571b927e..4c9ddfe4674b 100644
--- a/nixos/modules/services/backup/borgbackup.nix
+++ b/nixos/modules/services/backup/borgbackup.nix
@@ -30,7 +30,7 @@ let
     }
     trap 'on_exit' INT TERM QUIT EXIT
 
-    archiveName="${cfg.archiveBaseName}-$(date ${cfg.dateFormat})"
+    archiveName="${if cfg.archiveBaseName == null then "" else cfg.archiveBaseName + "-"}$(date ${cfg.dateFormat})"
     archiveSuffix="${optionalString cfg.appendFailedSuffix ".failed"}"
     ${cfg.preHook}
   '' + optionalString cfg.doInit ''
@@ -60,7 +60,7 @@ let
   '' + optionalString (cfg.prune.keep != { }) ''
     borg prune $extraArgs \
       ${mkKeepArgs cfg} \
-      --prefix ${escapeShellArg cfg.prune.prefix} \
+      ${optionalString (cfg.prune.prefix != null) "--prefix ${escapeShellArg cfg.prune.prefix} \\"}
       $extraPruneArgs
     ${cfg.postPrune}
   '';
@@ -99,7 +99,18 @@ let
         BORG_REPO = cfg.repo;
         inherit (cfg) extraArgs extraInitArgs extraCreateArgs extraPruneArgs;
       } // (mkPassEnv cfg) // cfg.environment;
-      inherit (cfg) startAt;
+    };
+
+  mkBackupTimers = name: cfg:
+    nameValuePair "borgbackup-job-${name}" {
+      description = "BorgBackup job ${name} timer";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        Persistent = cfg.persistentTimer;
+        OnCalendar = cfg.startAt;
+      };
+      # if remote-backup wait for network
+      after = optional (cfg.persistentTimer && !isLocalPath cfg.repo) "network-online.target";
     };
 
   # utility function around makeWrapper
@@ -284,7 +295,7 @@ in {
           };
 
           archiveBaseName = mkOption {
-            type = types.strMatching "[^/{}]+";
+            type = types.nullOr (types.strMatching "[^/{}]+");
             default = "${globalConfig.networking.hostName}-${name}";
             defaultText = literalExpression ''"''${config.networking.hostName}-<name>"'';
             description = ''
@@ -292,6 +303,7 @@ in {
               determined by <option>dateFormat</option>, will be appended. The full
               name can be modified at runtime (<literal>$archiveName</literal>).
               Placeholders like <literal>{hostname}</literal> must not be used.
+              Use <literal>null</literal> for no base name.
             '';
           };
 
@@ -320,6 +332,19 @@ in {
             '';
           };
 
+          persistentTimer = mkOption {
+            default = false;
+            type = types.bool;
+            example = true;
+            description = ''
+              Set the <literal>persistentTimer</literal> option for the
+              <citerefentry><refentrytitle>systemd.timer</refentrytitle>
+              <manvolnum>5</manvolnum></citerefentry>
+              which triggers the backup immediately if the last trigger
+              was missed (e.g. if the system was powered down).
+            '';
+          };
+
           user = mkOption {
             type = types.str;
             description = ''
@@ -471,11 +496,11 @@ in {
           };
 
           prune.prefix = mkOption {
-            type = types.str;
+            type = types.nullOr (types.str);
             description = ''
               Only consider archive names starting with this prefix for pruning.
               By default, only archives created by this job are considered.
-              Use <literal>""</literal> to consider all archives.
+              Use <literal>""</literal> or <literal>null</literal> to consider all archives.
             '';
             default = config.archiveBaseName;
             defaultText = literalExpression "archiveBaseName";
@@ -694,6 +719,10 @@ in {
         # A repo named "foo" is mapped to systemd.services.borgbackup-repo-foo
         // mapAttrs' mkRepoService repos;
 
+      # A job named "foo" is mapped to systemd.timers.borgbackup-job-foo
+      # only generate the timer if interval (startAt) is set
+      systemd.timers = mapAttrs' mkBackupTimers (filterAttrs (_: cfg: cfg.startAt != []) jobs);
+
       users = mkMerge (mapAttrsToList mkUsersConfig repos);
 
       environment.systemPackages = with pkgs; [ borgbackup ] ++ (mapAttrsToList mkBorgWrapper jobs);
diff --git a/nixos/modules/services/backup/duplicati.nix b/nixos/modules/services/backup/duplicati.nix
index cf5aebdecd28..97864c44691b 100644
--- a/nixos/modules/services/backup/duplicati.nix
+++ b/nixos/modules/services/backup/duplicati.nix
@@ -18,6 +18,20 @@ in
         '';
       };
 
+      dataDir = mkOption {
+        type = types.str;
+        default = "/var/lib/duplicati";
+        description = ''
+          The directory where Duplicati stores its data files.
+
+          <note><para>
+            If left as the default value this directory will automatically be created
+            before the Duplicati server starts, otherwise you are responsible for ensuring
+            the directory exists with appropriate ownership and permissions.
+          </para></note>
+        '';
+      };
+
       interface = mkOption {
         default = "127.0.0.1";
         type = types.str;
@@ -45,20 +59,23 @@ in
       description = "Duplicati backup";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        User = cfg.user;
-        Group = "duplicati";
-        StateDirectory = "duplicati";
-        ExecStart = "${pkgs.duplicati}/bin/duplicati-server --webservice-interface=${cfg.interface} --webservice-port=${toString cfg.port} --server-datafolder=/var/lib/duplicati";
-        Restart = "on-failure";
-      };
+      serviceConfig = mkMerge [
+        {
+          User = cfg.user;
+          Group = "duplicati";
+          ExecStart = "${pkgs.duplicati}/bin/duplicati-server --webservice-interface=${cfg.interface} --webservice-port=${toString cfg.port} --server-datafolder=${cfg.dataDir}";
+          Restart = "on-failure";
+        }
+        (mkIf (cfg.dataDir == "/var/lib/duplicati") {
+          StateDirectory = "duplicati";
+        })
+      ];
     };
 
     users.users = lib.optionalAttrs (cfg.user == "duplicati") {
       duplicati = {
         uid = config.ids.uids.duplicati;
-        home = "/var/lib/duplicati";
-        createHome = true;
+        home = cfg.dataDir;
         group = "duplicati";
       };
     };
diff --git a/nixos/modules/services/backup/mysql-backup.nix b/nixos/modules/services/backup/mysql-backup.nix
index 9fca21002733..c40a0b5abc40 100644
--- a/nixos/modules/services/backup/mysql-backup.nix
+++ b/nixos/modules/services/backup/mysql-backup.nix
@@ -113,9 +113,10 @@ in
         };
       };
       services.mysql-backup = {
-        description = "Mysql backup service";
+        description = "MySQL backup service";
         enable = true;
         serviceConfig = {
+          Type = "oneshot";
           User = cfg.user;
         };
         script = backupScript;
diff --git a/nixos/modules/services/backup/restic-rest-server.nix b/nixos/modules/services/backup/restic-rest-server.nix
index 86744637f85d..4717119f178a 100644
--- a/nixos/modules/services/backup/restic-rest-server.nix
+++ b/nixos/modules/services/backup/restic-rest-server.nix
@@ -95,6 +95,10 @@ in
       };
     };
 
+    systemd.tmpfiles.rules = mkIf cfg.privateRepos [
+        "f ${cfg.dataDir}/.htpasswd 0700 restic restic -"
+    ];
+
     users.users.restic = {
       group = "restic";
       home = cfg.dataDir;
diff --git a/nixos/modules/services/backup/sanoid.nix b/nixos/modules/services/backup/sanoid.nix
index e70063415ec0..5eb031b2e9f0 100644
--- a/nixos/modules/services/backup/sanoid.nix
+++ b/nixos/modules/services/backup/sanoid.nix
@@ -51,7 +51,10 @@ let
   datasetOptions = rec {
     use_template = mkOption {
       description = "Names of the templates to use for this dataset.";
-      type = types.listOf (types.enum (attrNames cfg.templates));
+      type = types.listOf (types.str // {
+        check = (types.enum (attrNames cfg.templates)).check;
+        description = "configured template name";
+      });
       default = [ ];
     };
     useTemplate = use_template;
diff --git a/nixos/modules/services/backup/tarsnap.nix b/nixos/modules/services/backup/tarsnap.nix
index 9cce86836612..9b5fd90012e0 100644
--- a/nixos/modules/services/backup/tarsnap.nix
+++ b/nixos/modules/services/backup/tarsnap.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, utils, ... }:
+{ config, lib, options, pkgs, utils, ... }:
 
 with lib;
 
 let
   gcfg = config.services.tarsnap;
+  opt = options.services.tarsnap;
 
   configFile = name: cfg: ''
     keyfile ${cfg.keyfile}
@@ -59,12 +60,13 @@ in
       };
 
       archives = mkOption {
-        type = types.attrsOf (types.submodule ({ config, ... }:
+        type = types.attrsOf (types.submodule ({ config, options, ... }:
           {
             options = {
               keyfile = mkOption {
                 type = types.str;
                 default = gcfg.keyfile;
+                defaultText = literalExpression "config.${opt.keyfile}";
                 description = ''
                   Set a specific keyfile for this archive. This defaults to
                   <literal>"/root/tarsnap.key"</literal> if left unspecified.
@@ -87,6 +89,9 @@ in
               cachedir = mkOption {
                 type = types.nullOr types.path;
                 default = "/var/cache/tarsnap/${utils.escapeSystemdPath config.keyfile}";
+                defaultText = literalExpression ''
+                  "/var/cache/tarsnap/''${utils.escapeSystemdPath config.${options.keyfile}}"
+                '';
                 description = ''
                   The cache allows tarsnap to identify previously stored data
                   blocks, reducing archival time and bandwidth usage.
@@ -320,21 +325,22 @@ in
                         ${optionalString cfg.explicitSymlinks "-H"} \
                         ${optionalString cfg.followSymlinks "-L"} \
                         ${concatStringsSep " " cfg.directories}'';
+          cachedir = escapeShellArg cfg.cachedir;
           in if (cfg.cachedir != null) then ''
-            mkdir -p ${cfg.cachedir}
-            chmod 0700 ${cfg.cachedir}
+            mkdir -p ${cachedir}
+            chmod 0700 ${cachedir}
 
             ( flock 9
-              if [ ! -e ${cfg.cachedir}/firstrun ]; then
+              if [ ! -e ${cachedir}/firstrun ]; then
                 ( flock 10
                   flock -u 9
                   ${tarsnap} --fsck
                   flock 9
-                ) 10>${cfg.cachedir}/firstrun
+                ) 10>${cachedir}/firstrun
               fi
-            ) 9>${cfg.cachedir}/lockf
+            ) 9>${cachedir}/lockf
 
-             exec flock ${cfg.cachedir}/firstrun ${run}
+             exec flock ${cachedir}/firstrun ${run}
           '' else "exec ${run}";
 
         serviceConfig = {
@@ -356,22 +362,23 @@ in
           tarsnap = ''tarsnap --configfile "/etc/tarsnap/${name}.conf"'';
           lastArchive = "$(${tarsnap} --list-archives | sort | tail -1)";
           run = ''${tarsnap} -x -f "${lastArchive}" ${optionalString cfg.verbose "-v"}'';
+          cachedir = escapeShellArg cfg.cachedir;
 
         in if (cfg.cachedir != null) then ''
-          mkdir -p ${cfg.cachedir}
-          chmod 0700 ${cfg.cachedir}
+          mkdir -p ${cachedir}
+          chmod 0700 ${cachedir}
 
           ( flock 9
-            if [ ! -e ${cfg.cachedir}/firstrun ]; then
+            if [ ! -e ${cachedir}/firstrun ]; then
               ( flock 10
                 flock -u 9
                 ${tarsnap} --fsck
                 flock 9
-              ) 10>${cfg.cachedir}/firstrun
+              ) 10>${cachedir}/firstrun
             fi
-          ) 9>${cfg.cachedir}/lockf
+          ) 9>${cachedir}/lockf
 
-           exec flock ${cfg.cachedir}/firstrun ${run}
+           exec flock ${cachedir}/firstrun ${run}
         '' else "exec ${run}";
 
         serviceConfig = {
diff --git a/nixos/modules/services/backup/tsm.nix b/nixos/modules/services/backup/tsm.nix
index 6c238745797e..4e690ac6ecda 100644
--- a/nixos/modules/services/backup/tsm.nix
+++ b/nixos/modules/services/backup/tsm.nix
@@ -5,7 +5,7 @@ let
   inherit (lib.attrsets) hasAttr;
   inherit (lib.modules) mkDefault mkIf;
   inherit (lib.options) mkEnableOption mkOption;
-  inherit (lib.types) nullOr strMatching;
+  inherit (lib.types) nonEmptyStr nullOr;
 
   options.services.tsmBackup = {
     enable = mkEnableOption ''
@@ -15,7 +15,7 @@ let
       <option>programs.tsmClient.enable</option>
     '';
     command = mkOption {
-      type = strMatching ".+";
+      type = nonEmptyStr;
       default = "backup";
       example = "incr";
       description = ''
@@ -24,7 +24,7 @@ let
       '';
     };
     servername = mkOption {
-      type = strMatching ".+";
+      type = nonEmptyStr;
       example = "mainTsmServer";
       description = ''
         Create a systemd system service
@@ -41,7 +41,7 @@ let
       '';
     };
     autoTime = mkOption {
-      type = nullOr (strMatching ".+");
+      type = nullOr nonEmptyStr;
       default = null;
       example = "12:00";
       description = ''
@@ -87,16 +87,35 @@ in
       environment.DSM_LOG = "/var/log/tsm-backup/";
       # TSM needs a HOME dir to store certificates.
       environment.HOME = "/var/lib/tsm-backup";
-      # for exit status description see
-      # https://www.ibm.com/support/knowledgecenter/en/SSEQVQ_8.1.8/client/c_sched_rtncode.html
-      serviceConfig.SuccessExitStatus = "4 8";
-      # The `-se` option must come after the command.
-      # The `-optfile` option suppresses a `dsm.opt`-not-found warning.
-      serviceConfig.ExecStart =
-        "${cfgPrg.wrappedPackage}/bin/dsmc ${cfg.command} -se='${cfg.servername}' -optfile=/dev/null";
-      serviceConfig.LogsDirectory = "tsm-backup";
-      serviceConfig.StateDirectory = "tsm-backup";
-      serviceConfig.StateDirectoryMode = "0750";
+      serviceConfig = {
+        # for exit status description see
+        # https://www.ibm.com/docs/en/spectrum-protect/8.1.13?topic=clients-client-return-codes
+        SuccessExitStatus = "4 8";
+        # The `-se` option must come after the command.
+        # The `-optfile` option suppresses a `dsm.opt`-not-found warning.
+        ExecStart =
+          "${cfgPrg.wrappedPackage}/bin/dsmc ${cfg.command} -se='${cfg.servername}' -optfile=/dev/null";
+        LogsDirectory = "tsm-backup";
+        StateDirectory = "tsm-backup";
+        StateDirectoryMode = "0750";
+        # systemd sandboxing
+        LockPersonality = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        #PrivateTmp = true;  # would break backup of {/var,}/tmp
+        #PrivateUsers = true;  # would block backup of /home/*
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = "read-only";
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "noaccess";
+        ProtectSystem = "strict";
+        RestrictNamespaces = true;
+        RestrictSUIDSGID = true;
+      };
       startAt = mkIf (cfg.autoTime!=null) cfg.autoTime;
     };
   };
diff --git a/nixos/modules/services/cluster/corosync/default.nix b/nixos/modules/services/cluster/corosync/default.nix
new file mode 100644
index 000000000000..b4144917feea
--- /dev/null
+++ b/nixos/modules/services/cluster/corosync/default.nix
@@ -0,0 +1,112 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.corosync;
+in
+{
+  # interface
+  options.services.corosync = {
+    enable = mkEnableOption "corosync";
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.corosync;
+      defaultText = literalExpression "pkgs.corosync";
+      description = "Package that should be used for corosync.";
+    };
+
+    clusterName = mkOption {
+      type = types.str;
+      default = "nixcluster";
+      description = "Name of the corosync cluster.";
+    };
+
+    extraOptions = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = "Additional options with which to start corosync.";
+    };
+
+    nodelist = mkOption {
+      description = "Corosync nodelist: all cluster members.";
+      default = [];
+      type = with types; listOf (submodule {
+        options = {
+          nodeid = mkOption {
+            type = int;
+            description = "Node ID number";
+          };
+          name = mkOption {
+            type = str;
+            description = "Node name";
+          };
+          ring_addrs = mkOption {
+            type = listOf str;
+            description = "List of addresses, one for each ring.";
+          };
+        };
+      });
+    };
+  };
+
+  # implementation
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    environment.etc."corosync/corosync.conf".text = ''
+      totem {
+        version: 2
+        secauth: on
+        cluster_name: ${cfg.clusterName}
+        transport: knet
+      }
+
+      nodelist {
+        ${concatMapStrings ({ nodeid, name, ring_addrs }: ''
+          node {
+            nodeid: ${toString nodeid}
+            name: ${name}
+            ${concatStrings (imap0 (i: addr: ''
+              ring${toString i}_addr: ${addr}
+            '') ring_addrs)}
+          }
+        '') cfg.nodelist}
+      }
+
+      quorum {
+        # only corosync_votequorum is supported
+        provider: corosync_votequorum
+        wait_for_all: 0
+        ${optionalString (builtins.length cfg.nodelist < 3) ''
+          two_node: 1
+        ''}
+      }
+
+      logging {
+        to_syslog: yes
+      }
+    '';
+
+    environment.etc."corosync/uidgid.d/root".text = ''
+      # allow pacemaker connection by root
+      uidgid {
+        uid: 0
+        gid: 0
+      }
+    '';
+
+    systemd.packages = [ cfg.package ];
+    systemd.services.corosync = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        StateDirectory = "corosync";
+        StateDirectoryMode = "0700";
+      };
+    };
+
+    environment.etc."sysconfig/corosync".text = lib.optionalString (cfg.extraOptions != []) ''
+      COROSYNC_OPTIONS="${lib.escapeShellArgs cfg.extraOptions}"
+    '';
+  };
+}
diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix
index 0caec5cfc203..e3c26a0d5505 100644
--- a/nixos/modules/services/cluster/hadoop/conf.nix
+++ b/nixos/modules/services/cluster/hadoop/conf.nix
@@ -1,6 +1,6 @@
 { cfg, pkgs, lib }:
 let
-  propertyXml = name: value: ''
+  propertyXml = name: value: lib.optionalString (value != null) ''
     <property>
       <name>${name}</name>
       <value>${builtins.toString value}</value>
@@ -29,16 +29,16 @@ let
     export HADOOP_LOG_DIR=/tmp/hadoop/$USER
   '';
 in
-pkgs.runCommand "hadoop-conf" {} ''
+pkgs.runCommand "hadoop-conf" {} (with cfg; ''
   mkdir -p $out/
-  cp ${siteXml "core-site.xml" cfg.coreSite}/* $out/
-  cp ${siteXml "hdfs-site.xml" cfg.hdfsSite}/* $out/
-  cp ${siteXml "mapred-site.xml" cfg.mapredSite}/* $out/
-  cp ${siteXml "yarn-site.xml" cfg.yarnSite}/* $out/
-  cp ${siteXml "httpfs-site.xml" cfg.httpfsSite}/* $out/
-  cp ${cfgFile "container-executor.cfg" cfg.containerExecutorCfg}/* $out/
+  cp ${siteXml "core-site.xml" (coreSite // coreSiteInternal)}/* $out/
+  cp ${siteXml "hdfs-site.xml" (hdfsSiteDefault // hdfsSite // hdfsSiteInternal)}/* $out/
+  cp ${siteXml "mapred-site.xml" (mapredSiteDefault // mapredSite)}/* $out/
+  cp ${siteXml "yarn-site.xml" (yarnSiteDefault // yarnSite // yarnSiteInternal)}/* $out/
+  cp ${siteXml "httpfs-site.xml" httpfsSite}/* $out/
+  cp ${cfgFile "container-executor.cfg" containerExecutorCfg}/* $out/
   cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/
   cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/
-  cp ${cfg.log4jProperties} $out/log4j.properties
-  ${lib.concatMapStringsSep "\n" (dir: "cp -r ${dir}/* $out/") cfg.extraConfDirs}
-''
+  cp ${log4jProperties} $out/log4j.properties
+  ${lib.concatMapStringsSep "\n" (dir: "cp -r ${dir}/* $out/") extraConfDirs}
+'')
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix
index 90f22c48e055..a4fdea81037c 100644
--- a/nixos/modules/services/cluster/hadoop/default.nix
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@@ -1,6 +1,7 @@
-{ config, lib, pkgs, ...}:
+{ config, lib, options, pkgs, ...}:
 let
   cfg = config.services.hadoop;
+  opt = options.services.hadoop;
 in
 with lib;
 {
@@ -20,43 +21,84 @@ with lib;
         <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml"/>
       '';
     };
+    coreSiteInternal = mkOption {
+      default = {};
+      type = types.attrsOf types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to core-site.xml based on module options
+      '';
+    };
 
-    hdfsSite = mkOption {
+    hdfsSiteDefault = mkOption {
       default = {
         "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+        "dfs.namenode.http-address" = "0.0.0.0:9870";
+        "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
+        "dfs.namenode.http-bind-host" = "0.0.0.0";
       };
       type = types.attrsOf types.anything;
+      description = ''
+        Default options for hdfs-site.xml
+      '';
+    };
+    hdfsSite = mkOption {
+      default = {};
+      type = types.attrsOf types.anything;
       example = literalExpression ''
         {
           "dfs.nameservices" = "namenode1";
         }
       '';
       description = ''
-        Hadoop hdfs-site.xml definition
+        Additional options and overrides for hdfs-site.xml
         <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
       '';
     };
+    hdfsSiteInternal = mkOption {
+      default = {};
+      type = types.attrsOf types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to hdfs-site.xml based on module options
+      '';
+    };
 
-    mapredSite = mkOption {
+    mapredSiteDefault = mkOption {
       default = {
         "mapreduce.framework.name" = "yarn";
         "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
         "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
         "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
       };
+      defaultText = literalExpression ''
+        {
+          "mapreduce.framework.name" = "yarn";
+          "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
+          "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
+          "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
+        }
+      '';
+      type = types.attrsOf types.anything;
+      description = ''
+        Default options for mapred-site.xml
+      '';
+    };
+    mapredSite = mkOption {
+      default = {};
       type = types.attrsOf types.anything;
       example = literalExpression ''
-        options.services.hadoop.mapredSite.default // {
+        {
           "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
         }
       '';
       description = ''
-        Hadoop mapred-site.xml definition
+        Additional options and overrides for mapred-site.xml
         <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
       '';
     };
 
-    yarnSite = mkOption {
+    yarnSiteDefault = mkOption {
       default = {
         "yarn.nodemanager.admin-env" = "PATH=$PATH";
         "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
@@ -68,19 +110,34 @@ with lib;
         "yarn.nodemanager.linux-container-executor.path" = "/run/wrappers/yarn-nodemanager/bin/container-executor";
         "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
         "yarn.resourcemanager.bind-host" = "0.0.0.0";
-        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
+        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
       };
       type = types.attrsOf types.anything;
+      description = ''
+        Default options for yarn-site.xml
+      '';
+    };
+    yarnSite = mkOption {
+      default = {};
+      type = types.attrsOf types.anything;
       example = literalExpression ''
-        options.services.hadoop.yarnSite.default // {
+        {
           "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
         }
       '';
       description = ''
-        Hadoop yarn-site.xml definition
+        Additional options and overrides for yarn-site.xml
         <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
       '';
     };
+    yarnSiteInternal = mkOption {
+      default = {};
+      type = types.attrsOf types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to yarn-site.xml based on module options
+      '';
+    };
 
     httpfsSite = mkOption {
       default = { };
@@ -98,6 +155,9 @@ with lib;
 
     log4jProperties = mkOption {
       default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties";
+      defaultText = literalExpression ''
+        "''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}/etc/hadoop/log4j.properties"
+      '';
       type = types.path;
       example = literalExpression ''
         "''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties";
@@ -111,6 +171,7 @@ with lib;
         "yarn.nodemanager.linux-container-executor.group"="hadoop";
         "min.user.id"=1000;
         "feature.terminal.enabled"=1;
+        "feature.mount-cgroup.enabled" = 1;
       };
       type = types.attrsOf types.anything;
       example = literalExpression ''
@@ -136,6 +197,8 @@ with lib;
       description = "Directories containing additional config files to be added to HADOOP_CONF_DIR";
     };
 
+    gatewayRole.enable = mkEnableOption "gateway role for deploying hadoop configs";
+
     package = mkOption {
       type = types.package;
       default = pkgs.hadoop;
@@ -145,20 +208,16 @@ with lib;
   };
 
 
-  config = mkMerge [
-    (mkIf (builtins.hasAttr "yarn" config.users.users ||
-           builtins.hasAttr "hdfs" config.users.users ||
-           builtins.hasAttr "httpfs" config.users.users) {
-      users.groups.hadoop = {
-        gid = config.ids.gids.hadoop;
-      };
-      environment = {
-        systemPackages = [ cfg.package ];
-        etc."hadoop-conf".source = let
-          hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
-        in "${hadoopConf}";
-      };
-    })
-
-  ];
+  config = mkIf cfg.gatewayRole.enable {
+    users.groups.hadoop = {
+      gid = config.ids.gids.hadoop;
+    };
+    environment = {
+      systemPackages = [ cfg.package ];
+      etc."hadoop-conf".source = let
+        hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+      in "${hadoopConf}";
+      variables.HADOOP_CONF_DIR = "/etc/hadoop-conf/";
+    };
+  };
 }
diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index be667aa82d8a..325a002ad32f 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -1,191 +1,191 @@
-{ config, lib, pkgs, ...}:
+{ config, lib, pkgs, ... }:
 with lib;
 let
   cfg = config.services.hadoop;
+
+  # Config files for hadoop services
   hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
-  restartIfChanged  = mkOption {
-    type = types.bool;
-    description = ''
-      Automatically restart the service on config change.
-      This can be set to false to defer restarts on clusters running critical applications.
-      Please consider the security implications of inadvertently running an older version,
-      and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
-    '';
-    default = false;
-  };
+
+  # Generator for HDFS service options
+  hadoopServiceOption = { serviceName, firewallOption ? true, extraOpts ? null }: {
+    enable = mkEnableOption serviceName;
+    restartIfChanged = mkOption {
+      type = types.bool;
+      description = ''
+        Automatically restart the service on config change.
+        This can be set to false to defer restarts on clusters running critical applications.
+        Please consider the security implications of inadvertently running an older version,
+        and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+      '';
+      default = false;
+    };
+    extraFlags = mkOption{
+      type = with types; listOf str;
+      default = [];
+      description = "Extra command line flags to pass to ${serviceName}";
+      example = [
+        "-Dcom.sun.management.jmxremote"
+        "-Dcom.sun.management.jmxremote.port=8010"
+      ];
+    };
+    extraEnv = mkOption{
+      type = with types; attrsOf str;
+      default = {};
+      description = "Extra environment variables for ${serviceName}";
+    };
+  } // (optionalAttrs firewallOption {
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Open firewall ports for ${serviceName}.";
+    };
+  }) // (optionalAttrs (extraOpts != null) extraOpts);
+
+  # Generator for HDFS service configs
+  hadoopServiceConfig =
+    { name
+    , serviceOptions ? cfg.hdfs."${toLower name}"
+    , description ? "Hadoop HDFS ${name}"
+    , User ? "hdfs"
+    , allowedTCPPorts ? [ ]
+    , preStart ? ""
+    , environment ? { }
+    , extraConfig ? { }
+    }: (
+
+      mkIf serviceOptions.enable ( mkMerge [{
+        systemd.services."hdfs-${toLower name}" = {
+          inherit description preStart;
+          environment = environment // serviceOptions.extraEnv;
+          wantedBy = [ "multi-user.target" ];
+          inherit (serviceOptions) restartIfChanged;
+          serviceConfig = {
+            inherit User;
+            SyslogIdentifier = "hdfs-${toLower name}";
+            ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} ${toLower name} ${escapeShellArgs serviceOptions.extraFlags}";
+            Restart = "always";
+          };
+        };
+
+        services.hadoop.gatewayRole.enable = true;
+
+        networking.firewall.allowedTCPPorts = mkIf
+          ((builtins.hasAttr "openFirewall" serviceOptions) && serviceOptions.openFirewall)
+          allowedTCPPorts;
+      } extraConfig])
+    );
+
 in
 {
   options.services.hadoop.hdfs = {
-    namenode = {
-      enable = mkEnableOption "Whether to run the HDFS NameNode";
+
+    namenode = hadoopServiceOption { serviceName = "HDFS NameNode"; } // {
       formatOnInit = mkOption {
         type = types.bool;
         default = false;
         description = ''
-          Format HDFS namenode on first start. This is useful for quickly spinning up ephemeral HDFS clusters with a single namenode.
-          For HA clusters, initialization involves multiple steps across multiple nodes. Follow [this guide](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html)
-          to initialize an HA cluster manually.
-        '';
-      };
-      inherit restartIfChanged;
-      openFirewall = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Open firewall ports for namenode
-        '';
-      };
-    };
-    datanode = {
-      enable = mkEnableOption "Whether to run the HDFS DataNode";
-      inherit restartIfChanged;
-      openFirewall = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Open firewall ports for datanode
+          Format HDFS namenode on first start. This is useful for quickly spinning up
+          ephemeral HDFS clusters with a single namenode.
+          For HA clusters, initialization involves multiple steps across multiple nodes.
+          Follow this guide to initialize an HA cluster manually:
+          <link xlink:href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html"/>
         '';
       };
     };
-    journalnode = {
-      enable = mkEnableOption "Whether to run the HDFS JournalNode";
-      inherit restartIfChanged;
-      openFirewall = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Open firewall ports for journalnode
-        '';
+
+    datanode = hadoopServiceOption { serviceName = "HDFS DataNode"; } // {
+      dataDirs = mkOption {
+        default = null;
+        description = "Tier and path definitions for datanode storage.";
+        type = with types; nullOr (listOf (submodule {
+          options = {
+            type = mkOption {
+              type = enum [ "SSD" "DISK" "ARCHIVE" "RAM_DISK" ];
+              description = ''
+                Storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS storage policies.
+              '';
+            };
+            path = mkOption {
+              type = path;
+              example = [ "/var/lib/hadoop/hdfs/dn" ];
+              description = "Determines where on the local filesystem a data node should store its blocks.";
+            };
+          };
+        }));
       };
     };
-    zkfc = {
-      enable = mkEnableOption "Whether to run the HDFS ZooKeeper failover controller";
-      inherit restartIfChanged;
+
+    journalnode = hadoopServiceOption { serviceName = "HDFS JournalNode"; };
+
+    zkfc = hadoopServiceOption {
+      serviceName = "HDFS ZooKeeper failover controller";
+      firewallOption = false;
     };
-    httpfs = {
-      enable = mkEnableOption "Whether to run the HDFS HTTPfs server";
+
+    httpfs = hadoopServiceOption { serviceName = "HDFS JournalNode"; } // {
       tempPath = mkOption {
         type = types.path;
         default = "/tmp/hadoop/httpfs";
-        description = ''
-          HTTPFS_TEMP path used by HTTPFS
-        '';
-      };
-      inherit restartIfChanged;
-      openFirewall = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Open firewall ports for HTTPFS
-        '';
+        description = "HTTPFS_TEMP path used by HTTPFS";
       };
     };
+
   };
 
   config = mkMerge [
-    (mkIf cfg.hdfs.namenode.enable {
-      systemd.services.hdfs-namenode = {
-        description = "Hadoop HDFS NameNode";
-        wantedBy = [ "multi-user.target" ];
-        inherit (cfg.hdfs.namenode) restartIfChanged;
-
-        preStart = (mkIf cfg.hdfs.namenode.formatOnInit ''
-          ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true
-        '');
-
-        serviceConfig = {
-          User = "hdfs";
-          SyslogIdentifier = "hdfs-namenode";
-          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode";
-          Restart = "always";
-        };
-      };
-
-      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.namenode.openFirewall [
+    (hadoopServiceConfig {
+      name = "NameNode";
+      allowedTCPPorts = [
         9870 # namenode.http-address
         8020 # namenode.rpc-address
-        8022 # namenode. servicerpc-address
-      ]);
+        8022 # namenode.servicerpc-address
+        8019 # dfs.ha.zkfc.port
+      ];
+      preStart = (mkIf cfg.hdfs.namenode.formatOnInit
+        "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true"
+      );
     })
-    (mkIf cfg.hdfs.datanode.enable {
-      systemd.services.hdfs-datanode = {
-        description = "Hadoop HDFS DataNode";
-        wantedBy = [ "multi-user.target" ];
-        inherit (cfg.hdfs.datanode) restartIfChanged;
-
-        serviceConfig = {
-          User = "hdfs";
-          SyslogIdentifier = "hdfs-datanode";
-          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode";
-          Restart = "always";
-        };
-      };
 
-      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.datanode.openFirewall [
+    (hadoopServiceConfig {
+      name = "DataNode";
+      # port numbers for datanode changed between hadoop 2 and 3
+      allowedTCPPorts = if versionAtLeast cfg.package.version "3" then [
         9864 # datanode.http.address
         9866 # datanode.address
         9867 # datanode.ipc.address
-      ]);
+      ] else [
+        50075 # datanode.http.address
+        50010 # datanode.address
+        50020 # datanode.ipc.address
+      ];
+      extraConfig.services.hadoop.hdfsSiteInternal."dfs.datanode.data.dir" = let d = cfg.hdfs.datanode.dataDirs; in
+        if (d!= null) then (concatMapStringsSep "," (x: "["+x.type+"]file://"+x.path) cfg.hdfs.datanode.dataDirs) else d;
     })
-    (mkIf cfg.hdfs.journalnode.enable {
-      systemd.services.hdfs-journalnode = {
-        description = "Hadoop HDFS JournalNode";
-        wantedBy = [ "multi-user.target" ];
-        inherit (cfg.hdfs.journalnode) restartIfChanged;
-
-        serviceConfig = {
-          User = "hdfs";
-          SyslogIdentifier = "hdfs-journalnode";
-          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} journalnode";
-          Restart = "always";
-        };
-      };
 
-      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.journalnode.openFirewall [
+    (hadoopServiceConfig {
+      name = "JournalNode";
+      allowedTCPPorts = [
         8480 # dfs.journalnode.http-address
         8485 # dfs.journalnode.rpc-address
-      ]);
+      ];
     })
-    (mkIf cfg.hdfs.zkfc.enable {
-      systemd.services.hdfs-zkfc = {
-        description = "Hadoop HDFS ZooKeeper failover controller";
-        wantedBy = [ "multi-user.target" ];
-        inherit (cfg.hdfs.zkfc) restartIfChanged;
-
-        serviceConfig = {
-          User = "hdfs";
-          SyslogIdentifier = "hdfs-zkfc";
-          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} zkfc";
-          Restart = "always";
-        };
-      };
-    })
-    (mkIf cfg.hdfs.httpfs.enable {
-      systemd.services.hdfs-httpfs = {
-        description = "Hadoop httpfs";
-        wantedBy = [ "multi-user.target" ];
-        inherit (cfg.hdfs.httpfs) restartIfChanged;
-
-        environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;
 
-        preStart = ''
-          mkdir -p $HTTPFS_TEMP
-        '';
+    (hadoopServiceConfig {
+      name = "zkfc";
+      description = "Hadoop HDFS ZooKeeper failover controller";
+    })
 
-        serviceConfig = {
-          User = "httpfs";
-          SyslogIdentifier = "hdfs-httpfs";
-          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} httpfs";
-          Restart = "always";
-        };
-      };
-      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.httpfs.openFirewall [
+    (hadoopServiceConfig {
+      name = "HTTPFS";
+      environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;
+      preStart = "mkdir -p $HTTPFS_TEMP";
+      User = "httpfs";
+      allowedTCPPorts = [
         14000 # httpfs.http.port
-      ]);
+      ];
     })
-    (mkIf (
-        cfg.hdfs.namenode.enable || cfg.hdfs.datanode.enable || cfg.hdfs.journalnode.enable || cfg.hdfs.zkfc.enable
-    ) {
+
+    (mkIf cfg.gatewayRole.enable {
       users.users.hdfs = {
         description = "Hadoop HDFS user";
         group = "hadoop";
@@ -199,5 +199,6 @@ in
         isSystemUser = true;
       };
     })
+
   ];
 }
diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix
index 37c26ea10f76..74e16bdec687 100644
--- a/nixos/modules/services/cluster/hadoop/yarn.nix
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@@ -13,23 +13,77 @@ let
     '';
     default = false;
   };
+  extraFlags = mkOption{
+    type = with types; listOf str;
+    default = [];
+    description = "Extra command line flags to pass to the service";
+    example = [
+      "-Dcom.sun.management.jmxremote"
+      "-Dcom.sun.management.jmxremote.port=8010"
+    ];
+  };
+  extraEnv = mkOption{
+    type = with types; attrsOf str;
+    default = {};
+    description = "Extra environment variables";
+  };
 in
 {
   options.services.hadoop.yarn = {
     resourcemanager = {
-      enable = mkEnableOption "Whether to run the Hadoop YARN ResourceManager";
-      inherit restartIfChanged;
+      enable = mkEnableOption "Hadoop YARN ResourceManager";
+      inherit restartIfChanged extraFlags extraEnv;
+
       openFirewall = mkOption {
         type = types.bool;
-        default = true;
+        default = false;
         description = ''
           Open firewall ports for resourcemanager
         '';
       };
     };
     nodemanager = {
-      enable = mkEnableOption "Whether to run the Hadoop YARN NodeManager";
-      inherit restartIfChanged;
+      enable = mkEnableOption "Hadoop YARN NodeManager";
+      inherit restartIfChanged extraFlags extraEnv;
+
+      resource = {
+        cpuVCores = mkOption {
+          description = "Number of vcores that can be allocated for containers.";
+          type = with types; nullOr ints.positive;
+          default = null;
+        };
+        maximumAllocationVCores = mkOption {
+          description = "The maximum virtual CPU cores any container can be allocated.";
+          type = with types; nullOr ints.positive;
+          default = null;
+        };
+        memoryMB = mkOption {
+          description = "Amount of physical memory, in MB, that can be allocated for containers.";
+          type = with types; nullOr ints.positive;
+          default = null;
+        };
+        maximumAllocationMB = mkOption {
+          description = "The maximum physical memory any container can be allocated.";
+          type = with types; nullOr ints.positive;
+          default = null;
+        };
+      };
+
+      useCGroups = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Use cgroups to enforce resource limits on containers
+        '';
+      };
+
+      localDir = mkOption {
+        description = "List of directories to store localized files in.";
+        type = with types; nullOr (listOf path);
+        example = [ "/var/lib/hadoop/yarn/nm" ];
+        default = null;
+      };
+
       addBinBash = mkOption {
         type = types.bool;
         default = true;
@@ -39,7 +93,7 @@ in
       };
       openFirewall = mkOption {
         type = types.bool;
-        default = true;
+        default = false;
         description = ''
           Open firewall ports for nodemanager.
           Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened.
@@ -49,10 +103,7 @@ in
   };
 
   config = mkMerge [
-    (mkIf (
-        cfg.yarn.resourcemanager.enable || cfg.yarn.nodemanager.enable
-    ) {
-
+    (mkIf cfg.gatewayRole.enable {
       users.users.yarn = {
         description = "Hadoop YARN user";
         group = "hadoop";
@@ -65,15 +116,19 @@ in
         description = "Hadoop YARN ResourceManager";
         wantedBy = [ "multi-user.target" ];
         inherit (cfg.yarn.resourcemanager) restartIfChanged;
+        environment = cfg.yarn.resourcemanager.extraEnv;
 
         serviceConfig = {
           User = "yarn";
           SyslogIdentifier = "yarn-resourcemanager";
           ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
-                      " resourcemanager";
+                      " resourcemanager ${escapeShellArgs cfg.yarn.resourcemanager.extraFlags}";
           Restart = "always";
         };
       };
+
+      services.hadoop.gatewayRole.enable = true;
+
       networking.firewall.allowedTCPPorts = (mkIf cfg.yarn.resourcemanager.openFirewall [
         8088 # resourcemanager.webapp.address
         8030 # resourcemanager.scheduler.address
@@ -94,6 +149,7 @@ in
         description = "Hadoop YARN NodeManager";
         wantedBy = [ "multi-user.target" ];
         inherit (cfg.yarn.nodemanager) restartIfChanged;
+        environment = cfg.yarn.nodemanager.extraEnv;
 
         preStart = ''
           # create log dir
@@ -101,8 +157,9 @@ in
           chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
 
           # set up setuid container executor binary
+          umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
           rm -rf /run/wrappers/yarn-nodemanager/ || true
-          mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop}
+          mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
           cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
           chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
           chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
@@ -114,11 +171,26 @@ in
           SyslogIdentifier = "yarn-nodemanager";
           PermissionsStartOnly = true;
           ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
-                      " nodemanager";
+                      " nodemanager ${escapeShellArgs cfg.yarn.nodemanager.extraFlags}";
           Restart = "always";
         };
       };
 
+      services.hadoop.gatewayRole.enable = true;
+
+      services.hadoop.yarnSiteInternal = with cfg.yarn.nodemanager; {
+        "yarn.nodemanager.local-dirs" = localDir;
+        "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores;
+        "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB;
+        "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores;
+        "yarn.nodemanager.resource.memory-mb" = resource.memoryMB;
+      } // mkIf useCGroups {
+        "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
+        "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
+        "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
+        "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup";
+      };
+
       networking.firewall.allowedTCPPortRanges = [
         (mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;})
       ];
diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix
index 50b6780bbe66..3a36cfa3f37b 100644
--- a/nixos/modules/services/cluster/k3s/default.nix
+++ b/nixos/modules/services/cluster/k3s/default.nix
@@ -91,11 +91,6 @@ in
     virtualisation.docker = mkIf cfg.docker {
       enable = mkDefault true;
     };
-
-    # TODO: disable this once k3s supports cgroupsv2, either by docker
-    # supporting it, or their bundled containerd
-    systemd.enableUnifiedCgroupHierarchy = false;
-
     environment.systemPackages = [ config.services.k3s.package ];
 
     systemd.services.k3s = {
@@ -119,6 +114,7 @@ in
           [
             "${cfg.package}/bin/k3s ${cfg.role}"
           ] ++ (optional cfg.docker "--docker")
+          ++ (optional (cfg.docker && config.systemd.enableUnifiedCgroupHierarchy) "--kubelet-arg=cgroup-driver=systemd")
           ++ (optional cfg.disableAgent "--disable-agent")
           ++ (optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
           ++ (optional (cfg.token != "") "--token ${cfg.token}")
diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 3d988dc2479a..b677d900ff50 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -58,7 +58,7 @@ in
             "spec" = { ... };
           };
         }
-        // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; };
+        // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dns.nix> { cfg = config.services.kubernetes; };
       '';
     };
 
@@ -167,4 +167,5 @@ in
     };
   };
 
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
deleted file mode 100644
index 2ed7742eda09..000000000000
--- a/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
+++ /dev/null
@@ -1,332 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-
-let
-  cfg = config.services.kubernetes.addons.dashboard;
-in {
-  imports = [
-    (mkRenamedOptionModule [ "services" "kubernetes" "addons" "dashboard" "enableRBAC" ] [ "services" "kubernetes" "addons" "dashboard" "rbac" "enable" ])
-  ];
-
-  options.services.kubernetes.addons.dashboard = {
-    enable = mkEnableOption "kubernetes dashboard addon";
-
-    extraArgs = mkOption {
-      description = "Extra arguments to append to the dashboard cmdline";
-      type = types.listOf types.str;
-      default = [];
-      example = ["--enable-skip-login"];
-    };
-
-    rbac = mkOption {
-      description = "Role-based access control (RBAC) options";
-      default = {};
-      type = types.submodule {
-        options = {
-          enable = mkOption {
-            description = "Whether to enable role based access control is enabled for kubernetes dashboard";
-            type = types.bool;
-            default = elem "RBAC" config.services.kubernetes.apiserver.authorizationMode;
-          };
-
-          clusterAdmin = mkOption {
-            description = "Whether to assign cluster admin rights to the kubernetes dashboard";
-            type = types.bool;
-            default = false;
-          };
-        };
-      };
-    };
-
-    version = mkOption {
-      description = "Which version of the kubernetes dashboard to deploy";
-      type = types.str;
-      default = "v1.10.1";
-    };
-
-    image = mkOption {
-      description = "Docker image to seed for the kubernetes dashboard container.";
-      type = types.attrs;
-      default = {
-        imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
-        imageDigest = "sha256:0ae6b69432e78069c5ce2bcde0fe409c5c4d6f0f4d9cd50a17974fea38898747";
-        finalImageTag = cfg.version;
-        sha256 = "01xrr4pwgr2hcjrjsi3d14ifpzdfbxzqpzxbk2fkbjb9zkv38zxy";
-      };
-    };
-  };
-
-  config = mkIf cfg.enable {
-    services.kubernetes.kubelet.seedDockerImages = [(pkgs.dockerTools.pullImage cfg.image)];
-
-    services.kubernetes.addonManager.addons = {
-      kubernetes-dashboard-deployment = {
-        kind = "Deployment";
-        apiVersion = "apps/v1";
-        metadata = {
-          labels = {
-            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-            k8s-app = "kubernetes-dashboard";
-            version = cfg.version;
-            "kubernetes.io/cluster-service" = "true";
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-          };
-          name = "kubernetes-dashboard";
-          namespace = "kube-system";
-        };
-        spec = {
-          replicas = 1;
-          revisionHistoryLimit = 10;
-          selector.matchLabels.k8s-app = "kubernetes-dashboard";
-          template = {
-            metadata = {
-              labels = {
-                k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-                k8s-app = "kubernetes-dashboard";
-                version = cfg.version;
-                "kubernetes.io/cluster-service" = "true";
-              };
-              annotations = {
-                "scheduler.alpha.kubernetes.io/critical-pod" = "";
-              };
-            };
-            spec = {
-              priorityClassName = "system-cluster-critical";
-              containers = [{
-                name = "kubernetes-dashboard";
-                image = with cfg.image; "${imageName}:${finalImageTag}";
-                ports = [{
-                  containerPort = 8443;
-                  protocol = "TCP";
-                }];
-                resources = {
-                  limits = {
-                    cpu = "100m";
-                    memory = "300Mi";
-                  };
-                  requests = {
-                    cpu = "100m";
-                    memory = "100Mi";
-                  };
-                };
-                args = ["--auto-generate-certificates"] ++ cfg.extraArgs;
-                volumeMounts = [{
-                  name = "tmp-volume";
-                  mountPath = "/tmp";
-                } {
-                  name = "kubernetes-dashboard-certs";
-                  mountPath = "/certs";
-                }];
-                livenessProbe = {
-                  httpGet = {
-                    scheme = "HTTPS";
-                    path = "/";
-                    port = 8443;
-                  };
-                  initialDelaySeconds = 30;
-                  timeoutSeconds = 30;
-                };
-              }];
-              volumes = [{
-                name = "kubernetes-dashboard-certs";
-                secret = {
-                  secretName = "kubernetes-dashboard-certs";
-                };
-              } {
-                name = "tmp-volume";
-                emptyDir = {};
-              }];
-              serviceAccountName = "kubernetes-dashboard";
-              tolerations = [{
-                key = "node-role.kubernetes.io/master";
-                effect = "NoSchedule";
-              } {
-                key = "CriticalAddonsOnly";
-                operator = "Exists";
-              }];
-            };
-          };
-        };
-      };
-
-      kubernetes-dashboard-svc = {
-        apiVersion = "v1";
-        kind = "Service";
-        metadata = {
-          labels = {
-            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-            k8s-app = "kubernetes-dashboard";
-            "kubernetes.io/cluster-service" = "true";
-            "kubernetes.io/name" = "KubeDashboard";
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-          };
-          name = "kubernetes-dashboard";
-          namespace  = "kube-system";
-        };
-        spec = {
-          ports = [{
-            port = 443;
-            targetPort = 8443;
-          }];
-          selector.k8s-app = "kubernetes-dashboard";
-        };
-      };
-
-      kubernetes-dashboard-sa = {
-        apiVersion = "v1";
-        kind = "ServiceAccount";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-          };
-          name = "kubernetes-dashboard";
-          namespace = "kube-system";
-        };
-      };
-      kubernetes-dashboard-sec-certs = {
-        apiVersion = "v1";
-        kind = "Secret";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-certs";
-          namespace = "kube-system";
-        };
-        type = "Opaque";
-      };
-      kubernetes-dashboard-sec-kholder = {
-        apiVersion = "v1";
-        kind = "Secret";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-key-holder";
-          namespace = "kube-system";
-        };
-        type = "Opaque";
-      };
-      kubernetes-dashboard-cm = {
-        apiVersion = "v1";
-        kind = "ConfigMap";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-settings";
-          namespace = "kube-system";
-        };
-      };
-    } // (optionalAttrs cfg.rbac.enable
-      (let
-        subjects = [{
-          kind = "ServiceAccount";
-          name = "kubernetes-dashboard";
-          namespace = "kube-system";
-        }];
-        labels = {
-          k8s-app = "kubernetes-dashboard";
-          k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-          "addonmanager.kubernetes.io/mode" = "Reconcile";
-        };
-      in
-        (if cfg.rbac.clusterAdmin then {
-          kubernetes-dashboard-crb = {
-            apiVersion = "rbac.authorization.k8s.io/v1";
-            kind = "ClusterRoleBinding";
-            metadata = {
-              name = "kubernetes-dashboard";
-              inherit labels;
-            };
-            roleRef = {
-              apiGroup = "rbac.authorization.k8s.io";
-              kind = "ClusterRole";
-              name = "cluster-admin";
-            };
-            inherit subjects;
-          };
-        }
-        else
-        {
-          # Upstream role- and rolebinding as per:
-          # https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/alternative/kubernetes-dashboard.yaml
-          kubernetes-dashboard-role = {
-            apiVersion = "rbac.authorization.k8s.io/v1";
-            kind = "Role";
-            metadata = {
-              name = "kubernetes-dashboard-minimal";
-              namespace = "kube-system";
-              inherit labels;
-            };
-            rules = [
-              # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
-              {
-                apiGroups = [""];
-                resources = ["secrets"];
-                verbs = ["create"];
-              }
-              # Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
-              {
-                apiGroups = [""];
-                resources = ["configmaps"];
-                verbs = ["create"];
-              }
-              # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
-              {
-                apiGroups = [""];
-                resources = ["secrets"];
-                resourceNames = ["kubernetes-dashboard-key-holder"];
-                verbs = ["get" "update" "delete"];
-              }
-              # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
-              {
-                apiGroups = [""];
-                resources = ["configmaps"];
-                resourceNames = ["kubernetes-dashboard-settings"];
-                verbs = ["get" "update"];
-              }
-              # Allow Dashboard to get metrics from heapster.
-              {
-                apiGroups = [""];
-                resources = ["services"];
-                resourceNames = ["heapster"];
-                verbs = ["proxy"];
-              }
-              {
-                apiGroups = [""];
-                resources = ["services/proxy"];
-                resourceNames = ["heapster" "http:heapster:" "https:heapster:"];
-                verbs = ["get"];
-              }
-            ];
-          };
-
-          kubernetes-dashboard-rb = {
-            apiVersion = "rbac.authorization.k8s.io/v1";
-            kind = "RoleBinding";
-            metadata = {
-              name = "kubernetes-dashboard-minimal";
-              namespace = "kube-system";
-              inherit labels;
-            };
-            roleRef = {
-              apiGroup = "rbac.authorization.k8s.io";
-              kind = "Role";
-              name = "kubernetes-dashboard-minimal";
-            };
-            inherit subjects;
-          };
-        })
-    ));
-  };
-}
diff --git a/nixos/modules/services/cluster/kubernetes/addons/dns.nix b/nixos/modules/services/cluster/kubernetes/addons/dns.nix
index 34943fddd3d1..7bd4991f43f7 100644
--- a/nixos/modules/services/cluster/kubernetes/addons/dns.nix
+++ b/nixos/modules/services/cluster/kubernetes/addons/dns.nix
@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 
 with lib;
 
@@ -23,6 +23,10 @@ in {
           take 3 (splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
         ))
       ) + ".254";
+      defaultText = literalDocBook ''
+        The <literal>x.y.z.254</literal> IP of
+        <literal>config.${options.services.kubernetes.apiserver.serviceClusterIpRange}</literal>.
+      '';
       type = types.str;
     };
 
@@ -359,4 +363,6 @@ in {
 
     services.kubernetes.kubelet.clusterDns = mkDefault cfg.clusterIp;
   };
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 2c89310beb5a..a192e93badc2 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -1,9 +1,10 @@
-  { config, lib, pkgs, ... }:
+  { config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   top = config.services.kubernetes;
+  otop = options.services.kubernetes;
   cfg = top.apiserver;
 
   isRBACEnabled = elem "RBAC" cfg.authorizationMode;
@@ -84,6 +85,7 @@ in
     clientCaFile = mkOption {
       description = "Kubernetes apiserver CA file for client auth.";
       default = top.caFile;
+      defaultText = literalExpression "config.${otop.caFile}";
       type = nullOr path;
     };
 
@@ -138,6 +140,7 @@ in
       caFile = mkOption {
         description = "Etcd ca file.";
         default = top.caFile;
+        defaultText = literalExpression "config.${otop.caFile}";
         type = types.nullOr types.path;
       };
     };
@@ -157,6 +160,7 @@ in
     featureGates = mkOption {
       description = "List set of feature gates";
       default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
       type = listOf str;
     };
 
@@ -175,6 +179,7 @@ in
     kubeletClientCaFile = mkOption {
       description = "Path to a cert file for connecting to kubelet.";
       default = top.caFile;
+      defaultText = literalExpression "config.${otop.caFile}";
       type = nullOr path;
     };
 
@@ -491,4 +496,5 @@ in
 
   ];
 
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index 7128b5f70b1a..7c317e94deeb 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   top = config.services.kubernetes;
+  otop = options.services.kubernetes;
   cfg = top.controllerManager;
 in
 {
@@ -30,6 +31,7 @@ in
     clusterCidr = mkOption {
       description = "Kubernetes CIDR Range for Pods in cluster.";
       default = top.clusterCidr;
+      defaultText = literalExpression "config.${otop.clusterCidr}";
       type = str;
     };
 
@@ -44,6 +46,7 @@ in
     featureGates = mkOption {
       description = "List set of feature gates";
       default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
       type = listOf str;
     };
 
@@ -67,6 +70,7 @@ in
         service account's token secret.
       '';
       default = top.caFile;
+      defaultText = literalExpression "config.${otop.caFile}";
       type = nullOr path;
     };
 
@@ -167,4 +171,6 @@ in
 
     services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress;
   };
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index 433adf4d488c..35ec99d83c84 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.kubernetes;
+  opt = options.services.kubernetes;
 
   defaultContainerdSettings = {
     version = 2;
@@ -25,10 +26,7 @@ let
 
       containerd.runtimes.runc = {
         runtime_type = "io.containerd.runc.v2";
-      };
-
-      containerd.runtimes."io.containerd.runc.v2".options = {
-        SystemdCgroup = true;
+        options.SystemdCgroup = true;
       };
     };
   };
@@ -87,6 +85,7 @@ let
       description = "${prefix} certificate authority file used to connect to kube-apiserver.";
       type = types.nullOr types.path;
       default = cfg.caFile;
+      defaultText = literalExpression "config.${opt.caFile}";
     };
 
     certFile = mkOption {
@@ -104,6 +103,7 @@ let
 in {
 
   imports = [
+    (mkRemovedOptionModule [ "services" "kubernetes" "addons" "dashboard" ] "Removed due to it being an outdated version")
     (mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "")
   ];
 
@@ -196,6 +196,9 @@ in {
       description = "Default location for kubernetes secrets. Not a store location.";
       type = types.path;
       default = cfg.dataDir + "/secrets";
+      defaultText = literalExpression ''
+        config.${opt.dataDir} + "/secrets"
+      '';
     };
   };
 
@@ -307,4 +310,6 @@ in {
                           else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
     })
   ];
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index fecea7a15f3d..cb81eaaf0160 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -95,4 +95,6 @@ in
 
     };
   };
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 2806f73375bc..af3a5062febc 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   top = config.services.kubernetes;
+  otop = options.services.kubernetes;
   cfg = top.kubelet;
 
   cniConfig =
@@ -35,6 +36,7 @@ let
       key = mkOption {
         description = "Key of taint.";
         default = name;
+        defaultText = literalDocBook "Name of this submodule.";
         type = str;
       };
       value = mkOption {
@@ -76,12 +78,14 @@ in
     clusterDomain = mkOption {
       description = "Use alternative domain.";
       default = config.services.kubernetes.addons.dns.clusterDomain;
+      defaultText = literalExpression "config.${options.services.kubernetes.addons.dns.clusterDomain}";
       type = str;
     };
 
     clientCaFile = mkOption {
       description = "Kubernetes apiserver CA file for client authentication.";
       default = top.caFile;
+      defaultText = literalExpression "config.${otop.caFile}";
       type = nullOr path;
     };
 
@@ -148,6 +152,7 @@ in
     featureGates = mkOption {
       description = "List set of feature gates";
       default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
       type = listOf str;
     };
 
@@ -259,8 +264,6 @@ in
         "net.bridge.bridge-nf-call-ip6tables" = 1;
       };
 
-      systemd.enableUnifiedCgroupHierarchy = false; # true breaks node memory metrics
-
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "kubernetes.target" ];
@@ -390,4 +393,6 @@ in
     })
 
   ];
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 76ab03cd520b..7d9198d20e8c 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -266,7 +266,7 @@ in
           in
           ''
             export KUBECONFIG=${clusterAdminKubeconfig}
-            ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
+            ${kubernetes}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
           '';
         })]);
 
@@ -401,4 +401,6 @@ in
         };
       };
     });
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index a09efcef94ea..0fd98d1c1576 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   top = config.services.kubernetes;
+  otop = options.services.kubernetes;
   cfg = top.proxy;
 in
 {
@@ -31,6 +32,7 @@ in
     featureGates = mkOption {
       description = "List set of feature gates";
       default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
       type = listOf str;
     };
 
@@ -95,4 +97,6 @@ in
 
     services.kubernetes.proxy.kubeconfig.server = mkDefault top.apiserverAddress;
   };
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index 1b0c22a11426..2d95528a6ead 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   top = config.services.kubernetes;
+  otop = options.services.kubernetes;
   cfg = top.scheduler;
 in
 {
@@ -27,6 +28,7 @@ in
     featureGates = mkOption {
       description = "List set of feature gates";
       default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
       type = listOf str;
     };
 
@@ -64,12 +66,12 @@ in
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = ''${top.package}/bin/kube-scheduler \
-          --address=${cfg.address} \
+          --bind-address=${cfg.address} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
           --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \
           --leader-elect=${boolToString cfg.leaderElect} \
-          --port=${toString cfg.port} \
+          --secure-port=${toString cfg.port} \
           ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
           ${cfg.extraOpts}
         '';
@@ -94,4 +96,6 @@ in
 
     services.kubernetes.scheduler.kubeconfig.server = mkDefault top.apiserverAddress;
   };
+
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/cluster/pacemaker/default.nix b/nixos/modules/services/cluster/pacemaker/default.nix
new file mode 100644
index 000000000000..7eeadffcc586
--- /dev/null
+++ b/nixos/modules/services/cluster/pacemaker/default.nix
@@ -0,0 +1,52 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.pacemaker;
+in
+{
+  # interface
+  options.services.pacemaker = {
+    enable = mkEnableOption "pacemaker";
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.pacemaker;
+      defaultText = literalExpression "pkgs.pacemaker";
+      description = "Package that should be used for pacemaker.";
+    };
+  };
+
+  # implementation
+  config = mkIf cfg.enable {
+    assertions = [ {
+      assertion = config.services.corosync.enable;
+      message = ''
+        Enabling services.pacemaker requires a services.corosync configuration.
+      '';
+    } ];
+
+    environment.systemPackages = [ cfg.package ];
+
+    # required by pacemaker
+    users.users.hacluster = {
+      isSystemUser = true;
+      group = "pacemaker";
+      home = "/var/lib/pacemaker";
+    };
+    users.groups.pacemaker = {};
+
+    systemd.tmpfiles.rules = [
+      "d /var/log/pacemaker 0700 hacluster pacemaker -"
+    ];
+
+    systemd.packages = [ cfg.package ];
+    systemd.services.pacemaker = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        StateDirectory = "pacemaker";
+        StateDirectoryMode = "0700";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/computing/slurm/slurm.nix b/nixos/modules/services/computing/slurm/slurm.nix
index d2f3feffc970..8cbe54c60604 100644
--- a/nixos/modules/services/computing/slurm/slurm.nix
+++ b/nixos/modules/services/computing/slurm/slurm.nix
@@ -1,10 +1,11 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
 
   cfg = config.services.slurm;
+  opt = options.services.slurm;
   # configuration file can be generated by http://slurm.schedmd.com/configurator.html
 
   defaultUser = "slurm";
@@ -90,6 +91,7 @@ in
         storageUser = mkOption {
           type = types.str;
           default = cfg.user;
+          defaultText = literalExpression "config.${opt.user}";
           description = ''
             Database user name.
           '';
@@ -154,6 +156,7 @@ in
       controlAddr = mkOption {
         type = types.nullOr types.str;
         default = cfg.controlMachine;
+        defaultText = literalExpression "config.${opt.controlMachine}";
         example = null;
         description = ''
           Name that ControlMachine should be referred to in establishing a
@@ -279,6 +282,10 @@ in
         type = types.path;
         internal = true;
         default = etcSlurm;
+        defaultText = literalDocBook ''
+          Directory created from generated config files and
+          <literal>config.${opt.extraConfigPaths}</literal>.
+        '';
         description = ''
           Path to directory with slurm config files. This option is set by default from the
           Slurm module and is meant to make the Slurm config file available to other modules.
@@ -355,6 +362,7 @@ in
 
       wantedBy = [ "multi-user.target" ];
       after = [ "systemd-tmpfiles-clean.service" ];
+      requires = [ "network.target" ];
 
       serviceConfig = {
         Type = "forking";
@@ -364,12 +372,12 @@ in
         ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
         LimitMEMLOCK = "infinity";
       };
-
-      preStart = ''
-        mkdir -p /var/spool
-      '';
     };
 
+    systemd.tmpfiles.rules = mkIf cfg.client.enable [
+      "d /var/spool/slurmd 755 root root -"
+    ];
+
     services.openssh.forwardX11 = mkIf cfg.client.enable (mkDefault true);
 
     systemd.services.slurmctld = mkIf (cfg.server.enable) {
diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix
index 2dc61c21ac71..aaa159d3cb18 100644
--- a/nixos/modules/services/continuous-integration/buildbot/master.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/master.nix
@@ -1,11 +1,12 @@
 # NixOS module for Buildbot continous integration server.
 
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.buildbot-master;
+  opt = options.services.buildbot-master;
 
   python = cfg.package.pythonModule;
 
@@ -152,6 +153,7 @@ in {
 
       buildbotDir = mkOption {
         default = "${cfg.home}/master";
+        defaultText = literalExpression ''"''${config.${opt.home}}/master"'';
         type = types.path;
         description = "Specifies the Buildbot directory.";
       };
diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix
index dd4f4a4a74a9..1d7f53bb6559 100644
--- a/nixos/modules/services/continuous-integration/buildbot/worker.nix
+++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix
@@ -1,11 +1,12 @@
 # NixOS module for Buildbot Worker.
 
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.buildbot-worker;
+  opt = options.services.buildbot-worker;
 
   python = cfg.package.pythonModule;
 
@@ -77,6 +78,7 @@ in {
 
       buildbotDir = mkOption {
         default = "${cfg.home}/worker";
+        defaultText = literalExpression ''"''${config.${opt.home}}/worker"'';
         type = types.path;
         description = "Specifies the Buildbot directory.";
       };
diff --git a/nixos/modules/services/continuous-integration/github-runner.nix b/nixos/modules/services/continuous-integration/github-runner.nix
index 59370f43fe75..a7645e1f56e9 100644
--- a/nixos/modules/services/continuous-integration/github-runner.nix
+++ b/nixos/modules/services/continuous-integration/github-runner.nix
@@ -10,6 +10,8 @@ let
   stateDir = "%S/${systemdDir}";
   # %L: Log directory root (usually /var/log); see systemd.unit(5)
   logsDir = "%L/${systemdDir}";
+  # Name of file stored in service state directory
+  currentConfigTokenFilename = ".current-token";
 in
 {
   options.services.github-runner = {
@@ -32,6 +34,14 @@ in
         Repository to add the runner to.
 
         Changing this option triggers a new runner registration.
+
+        IMPORTANT: If your token is org-wide (not per repository), you need to
+        provide a github org link, not a single repository, so do it like this
+        <literal>https://github.com/nixos</literal>, not like this
+        <literal>https://github.com/nixos/nixpkgs</literal>.
+        Otherwise, you are going to get a <literal>404 NotFound</literal>
+        from <literal>POST https://api.github.com/actions/runner-registration</literal>
+        in the configure script.
       '';
       example = "https://github.com/nixos/nixpkgs";
     };
@@ -144,13 +154,11 @@ in
         ExecStart = "${cfg.package}/bin/runsvc.sh";
 
         # Does the following, sequentially:
-        # - Copy the current and the previous `tokenFile` to the $RUNTIME_DIRECTORY
-        #   and make it accessible to the service user to allow for a content
-        #   comparison.
-        # - If the module configuration or the token has changed, clear the state directory.
-        # - Configure the runner.
-        # - Copy the configured `tokenFile` to the $STATE_DIRECTORY and make it
-        #   inaccessible to the service user.
+        # - If the module configuration or the token has changed, purge the state directory,
+        #   and create the current and the new token file with the contents of the configured
+        #   token. While both files have the same content, only the later is accessible by
+        #   the service user.
+        # - Configure the runner using the new token file. When finished, delete it.
         # - Set up the directory structure by creating the necessary symlinks.
         ExecStartPre =
           let
@@ -173,37 +181,20 @@ in
             currentConfigPath = "$STATE_DIRECTORY/.nixos-current-config.json";
             runnerRegistrationConfig = getAttrs [ "name" "tokenFile" "url" "runnerGroup" "extraLabels" ] cfg;
             newConfigPath = builtins.toFile "${svcName}-config.json" (builtins.toJSON runnerRegistrationConfig);
-            currentConfigTokenFilename = ".current-token";
             newConfigTokenFilename = ".new-token";
             runnerCredFiles = [
               ".credentials"
               ".credentials_rsaparams"
               ".runner"
             ];
-            ownConfigTokens = writeScript "own-config-tokens" ''
-              # Copy current and new token file to runtime dir and make it accessible to the service user
-              cp ${escapeShellArg cfg.tokenFile} "$RUNTIME_DIRECTORY/${newConfigTokenFilename}"
-              chmod 600 "$RUNTIME_DIRECTORY/${newConfigTokenFilename}"
-              chown "$USER" "$RUNTIME_DIRECTORY/${newConfigTokenFilename}"
-
-              if [[ -e "$STATE_DIRECTORY/${currentConfigTokenFilename}" ]]; then
-                cp "$STATE_DIRECTORY/${currentConfigTokenFilename}" "$RUNTIME_DIRECTORY/${currentConfigTokenFilename}"
-                chmod 600 "$RUNTIME_DIRECTORY/${currentConfigTokenFilename}"
-                chown "$USER" "$RUNTIME_DIRECTORY/${currentConfigTokenFilename}"
-              fi
-            '';
-            disownConfigTokens = writeScript "disown-config-tokens" ''
-              # Make the token inaccessible to the runner service user
-              chmod 600 "$STATE_DIRECTORY/${currentConfigTokenFilename}"
-              chown root:root "$STATE_DIRECTORY/${currentConfigTokenFilename}"
-            '';
             unconfigureRunner = writeScript "unconfigure" ''
               differs=
               # Set `differs = 1` if current and new runner config differ or if `currentConfigPath` does not exist
               ${pkgs.diffutils}/bin/diff -q '${newConfigPath}' "${currentConfigPath}" >/dev/null 2>&1 || differs=1
               # Also trigger a registration if the token content changed
               ${pkgs.diffutils}/bin/diff -q \
-                "$RUNTIME_DIRECTORY"/{${currentConfigTokenFilename},${newConfigTokenFilename}} \
+                "$STATE_DIRECTORY"/${currentConfigTokenFilename} \
+                ${escapeShellArg cfg.tokenFile} \
                 >/dev/null 2>&1 || differs=1
 
               if [[ -n "$differs" ]]; then
@@ -211,15 +202,21 @@ in
                 echo "The old runner will still appear in the GitHub Actions UI." \
                   "You have to remove it manually."
                 find "$STATE_DIRECTORY/" -mindepth 1 -delete
+
+                # Copy the configured token file to the state dir and allow the service user to read the file
+                install --mode=666 ${escapeShellArg cfg.tokenFile} "$STATE_DIRECTORY/${newConfigTokenFilename}"
+                # Also copy current file to allow for a diff on the next start
+                install --mode=600 ${escapeShellArg cfg.tokenFile} "$STATE_DIRECTORY/${currentConfigTokenFilename}"
               fi
             '';
             configureRunner = writeScript "configure" ''
-              empty=$(ls -A "$STATE_DIRECTORY")
-              if [[ -z "$empty" ]]; then
+              if [[ -e "$STATE_DIRECTORY/${newConfigTokenFilename}" ]]; then
                 echo "Configuring GitHub Actions Runner"
-                token=$(< "$RUNTIME_DIRECTORY"/${newConfigTokenFilename})
+
+                token=$(< "$STATE_DIRECTORY"/${newConfigTokenFilename})
                 RUNNER_ROOT="$STATE_DIRECTORY" ${cfg.package}/bin/config.sh \
                   --unattended \
+                  --disableupdate \
                   --work "$RUNTIME_DIRECTORY" \
                   --url ${escapeShellArg cfg.url} \
                   --token "$token" \
@@ -234,8 +231,7 @@ in
                 rm    -rf "$STATE_DIRECTORY/_diag/"
 
                 # Cleanup token from config
-                rm -f "$RUNTIME_DIRECTORY"/${currentConfigTokenFilename}
-                mv    "$RUNTIME_DIRECTORY"/${newConfigTokenFilename} "$STATE_DIRECTORY/${currentConfigTokenFilename}"
+                rm "$STATE_DIRECTORY/${newConfigTokenFilename}"
 
                 # Symlink to new config
                 ln -s '${newConfigPath}' "${currentConfigPath}"
@@ -250,10 +246,8 @@ in
             '';
           in
           map (x: "${x} ${escapeShellArgs [ stateDir runtimeDir logsDir ]}") [
-            "+${ownConfigTokens}" # runs as root
-            unconfigureRunner
+            "+${unconfigureRunner}" # runs as root
             configureRunner
-            "+${disownConfigTokens}" # runs as root
             setupRuntimeDir
           ];
 
@@ -266,6 +260,13 @@ in
         StateDirectoryMode = "0700";
         WorkingDirectory = runtimeDir;
 
+        InaccessiblePaths = [
+          # Token file path given in the configuration
+          cfg.tokenFile
+          # Token file in the state directory
+          "${stateDir}/${currentConfigTokenFilename}"
+        ];
+
         # By default, use a dynamically allocated user
         DynamicUser = true;
 
diff --git a/nixos/modules/services/continuous-integration/gitlab-runner.nix b/nixos/modules/services/continuous-integration/gitlab-runner.nix
index d4b8541c6a1b..dc58c6345239 100644
--- a/nixos/modules/services/continuous-integration/gitlab-runner.nix
+++ b/nixos/modules/services/continuous-integration/gitlab-runner.nix
@@ -147,7 +147,7 @@ in
     concurrent = mkOption {
       type = types.int;
       default = 1;
-      example = literalExpression "config.nix.maxJobs";
+      example = literalExpression "config.nix.settings.max-jobs";
       description = ''
         Limits how many jobs globally can be run concurrently.
         The most upper limit of jobs using all defined runners.
diff --git a/nixos/modules/services/continuous-integration/gocd-agent/default.nix b/nixos/modules/services/continuous-integration/gocd-agent/default.nix
index acc3fb12484a..c63998c6736a 100644
--- a/nixos/modules/services/continuous-integration/gocd-agent/default.nix
+++ b/nixos/modules/services/continuous-integration/gocd-agent/default.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.gocd-agent;
+  opt = options.services.gocd-agent;
 in {
   options = {
     services.gocd-agent = {
@@ -98,6 +99,15 @@ in {
           "-Dcruise.console.publish.interval=10"
           "-Djava.security.egd=file:/dev/./urandom"
         ];
+        defaultText = literalExpression ''
+          [
+            "-Xms''${config.${opt.initialJavaHeapSize}}"
+            "-Xmx''${config.${opt.maxJavaHeapMemory}}"
+            "-Djava.io.tmpdir=/tmp"
+            "-Dcruise.console.publish.interval=10"
+            "-Djava.security.egd=file:/dev/./urandom"
+          ]
+        '';
         description = ''
           Specifies startup command line arguments to pass to Go.CD agent
           java process.
diff --git a/nixos/modules/services/continuous-integration/gocd-server/default.nix b/nixos/modules/services/continuous-integration/gocd-server/default.nix
index 646bf13ac67a..3540656f9344 100644
--- a/nixos/modules/services/continuous-integration/gocd-server/default.nix
+++ b/nixos/modules/services/continuous-integration/gocd-server/default.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.gocd-server;
+  opt = options.services.gocd-server;
 in {
   options = {
     services.gocd-server = {
@@ -106,6 +107,20 @@ in {
           "-Dcruise.server.port=${toString cfg.port}"
           "-Dcruise.server.ssl.port=${toString cfg.sslPort}"
         ];
+        defaultText = literalExpression ''
+          [
+            "-Xms''${config.${opt.initialJavaHeapSize}}"
+            "-Xmx''${config.${opt.maxJavaHeapMemory}}"
+            "-Dcruise.listen.host=''${config.${opt.listenAddress}}"
+            "-Duser.language=en"
+            "-Djruby.rack.request.size.threshold.bytes=30000000"
+            "-Duser.country=US"
+            "-Dcruise.config.dir=''${config.${opt.workDir}}/conf"
+            "-Dcruise.config.file=''${config.${opt.workDir}}/conf/cruise-config.xml"
+            "-Dcruise.server.port=''${toString config.${opt.port}}"
+            "-Dcruise.server.ssl.port=''${toString config.${opt.sslPort}}"
+          ]
+        '';
 
         description = ''
           Specifies startup command line arguments to pass to Go.CD server
diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix
index 968bc8f1e54e..ef1933e12284 100644
--- a/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix
+++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix
@@ -67,7 +67,7 @@ in
 
     # Trusted user allows simplified configuration and better performance
     # when operating in a cluster.
-    nix.trustedUsers = [ config.systemd.services.hercules-ci-agent.serviceConfig.User ];
+    nix.settings.trusted-users = [ config.systemd.services.hercules-ci-agent.serviceConfig.User ];
     services.hercules-ci-agent = {
       settings = {
         nixUserIsTrusted = true;
diff --git a/nixos/modules/services/continuous-integration/hydra/default.nix b/nixos/modules/services/continuous-integration/hydra/default.nix
index ccb7cc21734e..cc5de97d6d10 100644
--- a/nixos/modules/services/continuous-integration/hydra/default.nix
+++ b/nixos/modules/services/continuous-integration/hydra/default.nix
@@ -258,8 +258,6 @@ in
         uid = config.ids.uids.hydra-www;
       };
 
-    nix.trustedUsers = [ "hydra-queue-runner" ];
-
     services.hydra.extraConfig =
       ''
         using_frontend_proxy = 1
@@ -277,16 +275,21 @@ in
 
     environment.variables = hydraEnv;
 
-    nix.extraOptions = ''
-      keep-outputs = true
-      keep-derivations = true
-
-
-    '' + optionalString (versionOlder (getVersion config.nix.package.out) "2.4pre") ''
-      # The default (`true') slows Nix down a lot since the build farm
-      # has so many GC roots.
-      gc-check-reachability = false
-    '';
+    nix.settings = mkMerge [
+      {
+        keep-outputs = true;
+        keep-derivations = true;
+        trusted-users = [ "hydra-queue-runner" ];
+      }
+
+      (mkIf (versionOlder (getVersion config.nix.package.out) "2.4pre")
+        {
+          # The default (`true') slows Nix down a lot since the build farm
+          # has so many GC roots.
+          gc-check-reachability = false;
+        }
+      )
+    ];
 
     systemd.services.hydra-init =
       { wantedBy = [ "multi-user.target" ];
diff --git a/nixos/modules/services/databases/couchdb.nix b/nixos/modules/services/databases/couchdb.nix
index 16dd64f2373e..742e605d224d 100644
--- a/nixos/modules/services/databases/couchdb.nix
+++ b/nixos/modules/services/databases/couchdb.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, options, lib, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.couchdb;
+  opt = options.services.couchdb;
   configFile = pkgs.writeText "couchdb.ini" (
     ''
       [couchdb]
@@ -43,8 +44,8 @@ in {
 
       package = mkOption {
         type = types.package;
-        default = pkgs.couchdb;
-        defaultText = literalExpression "pkgs.couchdb";
+        default = pkgs.couchdb3;
+        defaultText = literalExpression "pkgs.couchdb3";
         description = ''
           CouchDB package to use.
         '';
@@ -150,6 +151,15 @@ in {
         '';
       };
 
+      argsFile = mkOption {
+        type = types.path;
+        default = "${cfg.package}/etc/vm.args";
+        defaultText = literalExpression ''"config.${opt.package}/etc/vm.args"'';
+        description = ''
+          vm.args configuration. Overrides Couchdb's Erlang VM parameters file.
+        '';
+      };
+
       configFile = mkOption {
         type = types.path;
         description = ''
@@ -186,12 +196,14 @@ in {
       '';
 
       environment = {
-        # we are actually specifying 4 configuration files:
+        # we are actually specifying 5 configuration files:
         # 1. the preinstalled default.ini
         # 2. the module configuration
         # 3. the extraConfig from the module options
         # 4. the locally writable config file, which couchdb itself writes to
         ERL_FLAGS= ''-couch_ini ${cfg.package}/etc/default.ini ${configFile} ${pkgs.writeText "couchdb-extra.ini" cfg.extraConfig} ${cfg.configFile}'';
+        # 5. the vm.args file
+        COUCHDB_ARGS_FILE=''${cfg.argsFile}'';
       };
 
       serviceConfig = {
diff --git a/nixos/modules/services/databases/hbase.nix b/nixos/modules/services/databases/hbase.nix
index 181be2d6b0b8..fe4f05eec643 100644
--- a/nixos/modules/services/databases/hbase.nix
+++ b/nixos/modules/services/databases/hbase.nix
@@ -1,14 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, options, lib, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.hbase;
-
-  defaultConfig = {
-    "hbase.rootdir" = "file://${cfg.dataDir}/hbase";
-    "hbase.zookeeper.property.dataDir" = "${cfg.dataDir}/zookeeper";
-  };
+  opt = options.services.hbase;
 
   buildProperty = configAttr:
     (builtins.concatStringsSep "\n"
@@ -23,7 +19,7 @@ let
 
   configFile = pkgs.writeText "hbase-site.xml"
     ''<configuration>
-        ${buildProperty (defaultConfig // cfg.settings)}
+        ${buildProperty (opt.settings.default // cfg.settings)}
       </configuration>
     '';
 
@@ -96,7 +92,16 @@ in {
 
       settings = mkOption {
         type = with lib.types; attrsOf (oneOf [ str int bool ]);
-        default = defaultConfig;
+        default = {
+          "hbase.rootdir" = "file://${cfg.dataDir}/hbase";
+          "hbase.zookeeper.property.dataDir" = "${cfg.dataDir}/zookeeper";
+        };
+        defaultText = literalExpression ''
+          {
+            "hbase.rootdir" = "file://''${config.${opt.dataDir}}/hbase";
+            "hbase.zookeeper.property.dataDir" = "''${config.${opt.dataDir}}/zookeeper";
+          }
+        '';
         description = ''
           configurations in hbase-site.xml, see <link xlink:href="https://github.com/apache/hbase/blob/master/hbase-server/src/test/resources/hbase-site.xml"/> for details.
         '';
diff --git a/nixos/modules/services/databases/influxdb2.nix b/nixos/modules/services/databases/influxdb2.nix
index 15f008cbc6d6..340c515bbb43 100644
--- a/nixos/modules/services/databases/influxdb2.nix
+++ b/nixos/modules/services/databases/influxdb2.nix
@@ -1,5 +1,7 @@
 { config, lib, pkgs, ... }:
+
 with lib;
+
 let
   format = pkgs.formats.json { };
   cfg = config.services.influxdb2;
@@ -9,12 +11,14 @@ in
   options = {
     services.influxdb2 = {
       enable = mkEnableOption "the influxdb2 server";
+
       package = mkOption {
-        default = pkgs.influxdb2;
+        default = pkgs.influxdb2-server;
         defaultText = literalExpression "pkgs.influxdb2";
         description = "influxdb2 derivation to use.";
         type = types.package;
       };
+
       settings = mkOption {
         default = { };
         description = ''configuration options for influxdb2, see <link xlink:href="https://docs.influxdata.com/influxdb/v2.0/reference/config-options"/> for details.'';
@@ -28,18 +32,20 @@ in
       assertion = !(builtins.hasAttr "bolt-path" cfg.settings) && !(builtins.hasAttr "engine-path" cfg.settings);
       message = "services.influxdb2.config: bolt-path and engine-path should not be set as they are managed by systemd";
     }];
+
     systemd.services.influxdb2 = {
       description = "InfluxDB is an open-source, distributed, time series database";
       documentation = [ "https://docs.influxdata.com/influxdb/" ];
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
       environment = {
-        INFLUXD_CONFIG_PATH = "${configFile}";
+        INFLUXD_CONFIG_PATH = configFile;
       };
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/influxd --bolt-path \${STATE_DIRECTORY}/influxd.bolt --engine-path \${STATE_DIRECTORY}/engine";
         StateDirectory = "influxdb2";
-        DynamicUser = true;
+        User = "influxdb2";
+        Group = "influxdb2";
         CapabilityBoundingSet = "";
         SystemCallFilter = "@system-service";
         LimitNOFILE = 65536;
@@ -47,6 +53,13 @@ in
         Restart = "on-failure";
       };
     };
+
+    users.extraUsers.influxdb2 = {
+      isSystemUser = true;
+      group = "influxdb2";
+    };
+
+    users.extraGroups.influxdb2 = {};
   };
 
   meta.maintainers = with lib.maintainers; [ nickcao ];
diff --git a/nixos/modules/services/databases/mysql.nix b/nixos/modules/services/databases/mysql.nix
index a9d9a6d80588..625b31d081c9 100644
--- a/nixos/modules/services/databases/mysql.nix
+++ b/nixos/modules/services/databases/mysql.nix
@@ -11,10 +11,8 @@ let
   mysqldOptions =
     "--user=${cfg.user} --datadir=${cfg.dataDir} --basedir=${cfg.package}";
 
-  settingsFile = pkgs.writeText "my.cnf" (
-    generators.toINI { listsAsDuplicateKeys = true; } cfg.settings +
-    optionalString (cfg.extraOptions != null) "[mysqld]\n${cfg.extraOptions}"
-  );
+  format = pkgs.formats.ini { listsAsDuplicateKeys = true; };
+  configFile = format.generate "my.cnf" cfg.settings;
 
 in
 
@@ -22,6 +20,9 @@ in
   imports = [
     (mkRemovedOptionModule [ "services" "mysql" "pidDir" ] "Don't wait for pidfiles, describe dependencies through systemd.")
     (mkRemovedOptionModule [ "services" "mysql" "rootPassword" ] "Use socket authentication or set the password outside of the nix store.")
+    (mkRemovedOptionModule [ "services" "mysql" "extraOptions" ] "Use services.mysql.settings.mysqld instead.")
+    (mkRemovedOptionModule [ "services" "mysql" "bind" ] "Use services.mysql.settings.mysqld.bind-address instead.")
+    (mkRemovedOptionModule [ "services" "mysql" "port" ] "Use services.mysql.settings.mysqld.port instead.")
   ];
 
   ###### interface
@@ -40,41 +41,53 @@ in
         ";
       };
 
-      bind = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = "0.0.0.0";
-        description = "Address to bind to. The default is to bind to all addresses.";
-      };
-
-      port = mkOption {
-        type = types.port;
-        default = 3306;
-        description = "Port of MySQL.";
-      };
-
       user = mkOption {
         type = types.str;
         default = "mysql";
-        description = "User account under which MySQL runs.";
+        description = ''
+          User account under which MySQL runs.
+
+          <note><para>
+          If left as the default value this user will automatically be created
+          on system activation, otherwise you are responsible for
+          ensuring the user exists before the MySQL service starts.
+          </para></note>
+        '';
       };
 
       group = mkOption {
         type = types.str;
         default = "mysql";
-        description = "Group under which MySQL runs.";
+        description = ''
+          Group account under which MySQL runs.
+
+          <note><para>
+          If left as the default value this group will automatically be created
+          on system activation, otherwise you are responsible for
+          ensuring the user exists before the MySQL service starts.
+          </para></note>
+        '';
       };
 
       dataDir = mkOption {
         type = types.path;
         example = "/var/lib/mysql";
-        description = "Location where MySQL stores its table files.";
+        description = ''
+          The data directory for MySQL.
+
+          <note><para>
+          If left as the default value of <literal>/var/lib/mysql</literal> this directory will automatically be created before the MySQL
+          server starts, otherwise you are responsible for ensuring the directory exists with appropriate ownership and permissions.
+          </para></note>
+        '';
       };
 
       configFile = mkOption {
         type = types.path;
-        default = settingsFile;
-        defaultText = literalExpression "settingsFile";
+        default = configFile;
+        defaultText = ''
+          A configuration file automatically generated by NixOS.
+        '';
         description = ''
           Override the configuration file used by MySQL. By default,
           NixOS generates one automatically from <option>services.mysql.settings</option>.
@@ -92,7 +105,7 @@ in
       };
 
       settings = mkOption {
-        type = with types; attrsOf (attrsOf (oneOf [ bool int str (listOf str) ]));
+        type = format.type;
         default = {};
         description = ''
           MySQL configuration. Refer to
@@ -125,23 +138,6 @@ in
         '';
       };
 
-      extraOptions = mkOption {
-        type = with types; nullOr lines;
-        default = null;
-        example = ''
-          key_buffer_size = 6G
-          table_cache = 1600
-          log-error = /var/log/mysql_err.log
-        '';
-        description = ''
-          Provide extra options to the MySQL configuration file.
-
-          Please note, that these options are added to the
-          <literal>[mysqld]</literal> section so you don't need to explicitly
-          state it again.
-        '';
-      };
-
       initialDatabases = mkOption {
         type = types.listOf (types.submodule {
           options = {
@@ -287,7 +283,7 @@ in
         };
 
         masterPort = mkOption {
-          type = types.int;
+          type = types.port;
           default = 3306;
           description = "Port number on which the MySQL master server runs.";
         };
@@ -299,9 +295,7 @@ in
 
   ###### implementation
 
-  config = mkIf config.services.mysql.enable {
-
-    warnings = optional (cfg.extraOptions != null) "services.mysql.`extraOptions` is deprecated, please use services.mysql.`settings`.";
+  config = mkIf cfg.enable {
 
     services.mysql.dataDir =
       mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/mysql"
@@ -310,8 +304,7 @@ in
     services.mysql.settings.mysqld = mkMerge [
       {
         datadir = cfg.dataDir;
-        bind-address = mkIf (cfg.bind != null) cfg.bind;
-        port = cfg.port;
+        port = mkDefault 3306;
       }
       (mkIf (cfg.replication.role == "master" || cfg.replication.role == "slave") {
         log-bin = "mysql-bin-${toString cfg.replication.serverId}";
@@ -341,156 +334,150 @@ in
 
     environment.etc."my.cnf".source = cfg.configFile;
 
-    systemd.tmpfiles.rules = [
-      "d '${cfg.dataDir}' 0700 '${cfg.user}' '${cfg.group}' - -"
-      "z '${cfg.dataDir}' 0700 '${cfg.user}' '${cfg.group}' - -"
-    ];
-
-    systemd.services.mysql = let
-      hasNotify = isMariaDB;
-    in {
-        description = "MySQL Server";
-
-        after = [ "network.target" ];
-        wantedBy = [ "multi-user.target" ];
-        restartTriggers = [ cfg.configFile ];
-
-        unitConfig.RequiresMountsFor = "${cfg.dataDir}";
-
-        path = [
-          # Needed for the mysql_install_db command in the preStart script
-          # which calls the hostname command.
-          pkgs.nettools
-        ];
-
-        preStart = if isMariaDB then ''
-          if ! test -e ${cfg.dataDir}/mysql; then
-            ${cfg.package}/bin/mysql_install_db --defaults-file=/etc/my.cnf ${mysqldOptions}
-            touch ${cfg.dataDir}/mysql_init
-          fi
-        '' else ''
-          if ! test -e ${cfg.dataDir}/mysql; then
-            ${cfg.package}/bin/mysqld --defaults-file=/etc/my.cnf ${mysqldOptions} --initialize-insecure
-            touch ${cfg.dataDir}/mysql_init
-          fi
-        '';
-
-        script = ''
-          # https://mariadb.com/kb/en/getting-started-with-mariadb-galera-cluster/#systemd-and-galera-recovery
-          if test -n "''${_WSREP_START_POSITION}"; then
-            if test -e "${cfg.package}/bin/galera_recovery"; then
-              VAR=$(cd ${cfg.package}/bin/..; ${cfg.package}/bin/galera_recovery); [[ $? -eq 0 ]] && export _WSREP_START_POSITION=$VAR || exit 1
-            fi
-          fi
-
-          # The last two environment variables are used for starting Galera clusters
-          exec ${cfg.package}/bin/mysqld --defaults-file=/etc/my.cnf ${mysqldOptions} $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION
-        '';
-
-        postStart = let
-          # The super user account to use on *first* run of MySQL server
-          superUser = if isMariaDB then cfg.user else "root";
-        in ''
-          ${optionalString (!hasNotify) ''
-            # Wait until the MySQL server is available for use
-            count=0
-            while [ ! -e /run/mysqld/mysqld.sock ]
-            do
-                if [ $count -eq 30 ]
-                then
-                    echo "Tried 30 times, giving up..."
-                    exit 1
-                fi
-
-                echo "MySQL daemon not yet started. Waiting for 1 second..."
-                count=$((count++))
-                sleep 1
-            done
-          ''}
-
-          if [ -f ${cfg.dataDir}/mysql_init ]
-          then
-              # While MariaDB comes with a 'mysql' super user account since 10.4.x, MySQL does not
-              # Since we don't want to run this service as 'root' we need to ensure the account exists on first run
-              ( echo "CREATE USER IF NOT EXISTS '${cfg.user}'@'localhost' IDENTIFIED WITH ${if isMariaDB then "unix_socket" else "auth_socket"};"
-                echo "GRANT ALL PRIVILEGES ON *.* TO '${cfg.user}'@'localhost' WITH GRANT OPTION;"
-              ) | ${cfg.package}/bin/mysql -u ${superUser} -N
-
-              ${concatMapStrings (database: ''
-                # Create initial databases
-                if ! test -e "${cfg.dataDir}/${database.name}"; then
-                    echo "Creating initial database: ${database.name}"
-                    ( echo 'create database `${database.name}`;'
-
-                      ${optionalString (database.schema != null) ''
-                      echo 'use `${database.name}`;'
-
-                      # TODO: this silently falls through if database.schema does not exist,
-                      # we should catch this somehow and exit, but can't do it here because we're in a subshell.
-                      if [ -f "${database.schema}" ]
-                      then
-                          cat ${database.schema}
-                      elif [ -d "${database.schema}" ]
-                      then
-                          cat ${database.schema}/mysql-databases/*.sql
-                      fi
-                      ''}
-                    ) | ${cfg.package}/bin/mysql -u ${superUser} -N
-                fi
-              '') cfg.initialDatabases}
-
-              ${optionalString (cfg.replication.role == "master")
-                ''
-                  # Set up the replication master
-
-                  ( echo "use mysql;"
-                    echo "CREATE USER '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}' IDENTIFIED WITH mysql_native_password;"
-                    echo "SET PASSWORD FOR '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}' = PASSWORD('${cfg.replication.masterPassword}');"
-                    echo "GRANT REPLICATION SLAVE ON *.* TO '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}';"
-                  ) | ${cfg.package}/bin/mysql -u ${superUser} -N
-                ''}
-
-              ${optionalString (cfg.replication.role == "slave")
-                ''
-                  # Set up the replication slave
-
-                  ( echo "stop slave;"
-                    echo "change master to master_host='${cfg.replication.masterHost}', master_user='${cfg.replication.masterUser}', master_password='${cfg.replication.masterPassword}';"
-                    echo "start slave;"
-                  ) | ${cfg.package}/bin/mysql -u ${superUser} -N
-                ''}
-
-              ${optionalString (cfg.initialScript != null)
-                ''
-                  # Execute initial script
-                  # using toString to avoid copying the file to nix store if given as path instead of string,
-                  # as it might contain credentials
-                  cat ${toString cfg.initialScript} | ${cfg.package}/bin/mysql -u ${superUser} -N
-                ''}
-
-              rm ${cfg.dataDir}/mysql_init
+    systemd.services.mysql = {
+      description = "MySQL Server";
+
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      restartTriggers = [ cfg.configFile ];
+
+      unitConfig.RequiresMountsFor = cfg.dataDir;
+
+      path = [
+        # Needed for the mysql_install_db command in the preStart script
+        # which calls the hostname command.
+        pkgs.nettools
+      ];
+
+      preStart = if isMariaDB then ''
+        if ! test -e ${cfg.dataDir}/mysql; then
+          ${cfg.package}/bin/mysql_install_db --defaults-file=/etc/my.cnf ${mysqldOptions}
+          touch ${cfg.dataDir}/mysql_init
+        fi
+      '' else ''
+        if ! test -e ${cfg.dataDir}/mysql; then
+          ${cfg.package}/bin/mysqld --defaults-file=/etc/my.cnf ${mysqldOptions} --initialize-insecure
+          touch ${cfg.dataDir}/mysql_init
+        fi
+      '';
+
+      script = ''
+        # https://mariadb.com/kb/en/getting-started-with-mariadb-galera-cluster/#systemd-and-galera-recovery
+        if test -n "''${_WSREP_START_POSITION}"; then
+          if test -e "${cfg.package}/bin/galera_recovery"; then
+            VAR=$(cd ${cfg.package}/bin/..; ${cfg.package}/bin/galera_recovery); [[ $? -eq 0 ]] && export _WSREP_START_POSITION=$VAR || exit 1
           fi
+        fi
+
+        # The last two environment variables are used for starting Galera clusters
+        exec ${cfg.package}/bin/mysqld --defaults-file=/etc/my.cnf ${mysqldOptions} $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION
+      '';
+
+      postStart = let
+        # The super user account to use on *first* run of MySQL server
+        superUser = if isMariaDB then cfg.user else "root";
+      in ''
+        ${optionalString (!isMariaDB) ''
+          # Wait until the MySQL server is available for use
+          count=0
+          while [ ! -e /run/mysqld/mysqld.sock ]
+          do
+              if [ $count -eq 30 ]
+              then
+                  echo "Tried 30 times, giving up..."
+                  exit 1
+              fi
+
+              echo "MySQL daemon not yet started. Waiting for 1 second..."
+              count=$((count++))
+              sleep 1
+          done
+        ''}
+
+        if [ -f ${cfg.dataDir}/mysql_init ]
+        then
+            # While MariaDB comes with a 'mysql' super user account since 10.4.x, MySQL does not
+            # Since we don't want to run this service as 'root' we need to ensure the account exists on first run
+            ( echo "CREATE USER IF NOT EXISTS '${cfg.user}'@'localhost' IDENTIFIED WITH ${if isMariaDB then "unix_socket" else "auth_socket"};"
+              echo "GRANT ALL PRIVILEGES ON *.* TO '${cfg.user}'@'localhost' WITH GRANT OPTION;"
+            ) | ${cfg.package}/bin/mysql -u ${superUser} -N
 
-          ${optionalString (cfg.ensureDatabases != []) ''
-            (
             ${concatMapStrings (database: ''
-              echo "CREATE DATABASE IF NOT EXISTS \`${database}\`;"
-            '') cfg.ensureDatabases}
+              # Create initial databases
+              if ! test -e "${cfg.dataDir}/${database.name}"; then
+                  echo "Creating initial database: ${database.name}"
+                  ( echo 'create database `${database.name}`;'
+
+                    ${optionalString (database.schema != null) ''
+                    echo 'use `${database.name}`;'
+
+                    # TODO: this silently falls through if database.schema does not exist,
+                    # we should catch this somehow and exit, but can't do it here because we're in a subshell.
+                    if [ -f "${database.schema}" ]
+                    then
+                        cat ${database.schema}
+                    elif [ -d "${database.schema}" ]
+                    then
+                        cat ${database.schema}/mysql-databases/*.sql
+                    fi
+                    ''}
+                  ) | ${cfg.package}/bin/mysql -u ${superUser} -N
+              fi
+            '') cfg.initialDatabases}
+
+            ${optionalString (cfg.replication.role == "master")
+              ''
+                # Set up the replication master
+
+                ( echo "use mysql;"
+                  echo "CREATE USER '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}' IDENTIFIED WITH mysql_native_password;"
+                  echo "SET PASSWORD FOR '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}' = PASSWORD('${cfg.replication.masterPassword}');"
+                  echo "GRANT REPLICATION SLAVE ON *.* TO '${cfg.replication.masterUser}'@'${cfg.replication.slaveHost}';"
+                ) | ${cfg.package}/bin/mysql -u ${superUser} -N
+              ''}
+
+            ${optionalString (cfg.replication.role == "slave")
+              ''
+                # Set up the replication slave
+
+                ( echo "stop slave;"
+                  echo "change master to master_host='${cfg.replication.masterHost}', master_user='${cfg.replication.masterUser}', master_password='${cfg.replication.masterPassword}';"
+                  echo "start slave;"
+                ) | ${cfg.package}/bin/mysql -u ${superUser} -N
+              ''}
+
+            ${optionalString (cfg.initialScript != null)
+              ''
+                # Execute initial script
+                # using toString to avoid copying the file to nix store if given as path instead of string,
+                # as it might contain credentials
+                cat ${toString cfg.initialScript} | ${cfg.package}/bin/mysql -u ${superUser} -N
+              ''}
+
+            rm ${cfg.dataDir}/mysql_init
+        fi
+
+        ${optionalString (cfg.ensureDatabases != []) ''
+          (
+          ${concatMapStrings (database: ''
+            echo "CREATE DATABASE IF NOT EXISTS \`${database}\`;"
+          '') cfg.ensureDatabases}
+          ) | ${cfg.package}/bin/mysql -N
+        ''}
+
+        ${concatMapStrings (user:
+          ''
+            ( echo "CREATE USER IF NOT EXISTS '${user.name}'@'localhost' IDENTIFIED WITH ${if isMariaDB then "unix_socket" else "auth_socket"};"
+              ${concatStringsSep "\n" (mapAttrsToList (database: permission: ''
+                echo "GRANT ${permission} ON ${database} TO '${user.name}'@'localhost';"
+              '') user.ensurePermissions)}
             ) | ${cfg.package}/bin/mysql -N
-          ''}
-
-          ${concatMapStrings (user:
-            ''
-              ( echo "CREATE USER IF NOT EXISTS '${user.name}'@'localhost' IDENTIFIED WITH ${if isMariaDB then "unix_socket" else "auth_socket"};"
-                ${concatStringsSep "\n" (mapAttrsToList (database: permission: ''
-                  echo "GRANT ${permission} ON ${database} TO '${user.name}'@'localhost';"
-                '') user.ensurePermissions)}
-              ) | ${cfg.package}/bin/mysql -N
-            '') cfg.ensureUsers}
-        '';
+          '') cfg.ensureUsers}
+      '';
 
-        serviceConfig = {
-          Type = if hasNotify then "notify" else "simple";
+      serviceConfig = mkMerge [
+        {
+          Type = if isMariaDB then "notify" else "simple";
           Restart = "on-abort";
           RestartSec = "5s";
 
@@ -523,9 +510,12 @@ in
           PrivateMounts = true;
           # System Call Filtering
           SystemCallArchitectures = "native";
-        };
-      };
-
+        }
+        (mkIf (cfg.dataDir == "/var/lib/mysql") {
+          StateDirectory = "mysql";
+          StateDirectoryMode = "0700";
+        })
+      ];
+    };
   };
-
 }
diff --git a/nixos/modules/services/databases/neo4j.nix b/nixos/modules/services/databases/neo4j.nix
index f37e5ad16939..8816f3b2e4b6 100644
--- a/nixos/modules/services/databases/neo4j.nix
+++ b/nixos/modules/services/databases/neo4j.nix
@@ -4,6 +4,7 @@ with lib;
 
 let
   cfg = config.services.neo4j;
+  opt = options.services.neo4j;
   certDirOpt = options.services.neo4j.directories.certificates;
   isDefaultPathOption = opt: isOption opt && opt.type == types.path && opt.highestPrio >= 1500;
 
@@ -256,6 +257,7 @@ in {
       certificates = mkOption {
         type = types.path;
         default = "${cfg.directories.home}/certificates";
+        defaultText = literalExpression ''"''${config.${opt.directories.home}}/certificates"'';
         description = ''
           Directory for storing certificates to be used by Neo4j for
           TLS connections.
@@ -280,6 +282,7 @@ in {
       data = mkOption {
         type = types.path;
         default = "${cfg.directories.home}/data";
+        defaultText = literalExpression ''"''${config.${opt.directories.home}}/data"'';
         description = ''
           Path of the data directory. You must not configure more than one
           Neo4j installation to use the same data directory.
@@ -305,6 +308,7 @@ in {
       imports = mkOption {
         type = types.path;
         default = "${cfg.directories.home}/import";
+        defaultText = literalExpression ''"''${config.${opt.directories.home}}/import"'';
         description = ''
           The root directory for file URLs used with the Cypher
           <literal>LOAD CSV</literal> clause. Only meaningful when
@@ -321,6 +325,7 @@ in {
       plugins = mkOption {
         type = types.path;
         default = "${cfg.directories.home}/plugins";
+        defaultText = literalExpression ''"''${config.${opt.directories.home}}/plugins"'';
         description = ''
           Path of the database plugin directory. Compiled Java JAR files that
           contain database procedures will be loaded if they are placed in
@@ -432,6 +437,7 @@ in {
           baseDirectory = mkOption {
             type = types.path;
             default = "${cfg.directories.certificates}/${name}";
+            defaultText = literalExpression ''"''${config.${opt.directories.certificates}}/''${name}"'';
             description = ''
               The mandatory base directory for cryptographic objects of this
               policy. This path is only automatically generated when this
@@ -493,6 +499,7 @@ in {
           revokedDir = mkOption {
             type = types.path;
             default = "${config.baseDirectory}/revoked";
+            defaultText = literalExpression ''"''${config.${options.baseDirectory}}/revoked"'';
             description = ''
               Path to directory of CRLs (Certificate Revocation Lists) in
               PEM format. Must be an absolute path. The existence of this
@@ -528,6 +535,7 @@ in {
           trustedDir = mkOption {
             type = types.path;
             default = "${config.baseDirectory}/trusted";
+            defaultText = literalExpression ''"''${config.${options.baseDirectory}}/trusted"'';
             description = ''
               Path to directory of X.509 certificates in PEM format for
               trusted parties. Must be an absolute path. The existence of this
diff --git a/nixos/modules/services/databases/redis.nix b/nixos/modules/services/databases/redis.nix
index 578d9d9ec8d7..a1bd73c9e371 100644
--- a/nixos/modules/services/databases/redis.nix
+++ b/nixos/modules/services/databases/redis.nix
@@ -5,17 +5,18 @@ with lib;
 let
   cfg = config.services.redis;
 
-  ulimitNofile = cfg.maxclients + 32;
-
   mkValueString = value:
     if value == true then "yes"
     else if value == false then "no"
     else generators.mkValueStringDefault { } value;
 
-  redisConfig = pkgs.writeText "redis.conf" (generators.toKeyValue {
+  redisConfig = settings: pkgs.writeText "redis.conf" (generators.toKeyValue {
     listsAsDuplicateKeys = true;
     mkKeyValue = generators.mkKeyValueDefault { inherit mkValueString; } " ";
-  } cfg.settings);
+  } settings);
+
+  redisName = name: "redis" + optionalString (name != "") ("-"+name);
+  enabledServers = filterAttrs (name: conf: conf.enable) config.services.redis.servers;
 
 in {
   imports = [
@@ -24,7 +25,28 @@ in {
     (mkRemovedOptionModule [ "services" "redis" "dbFilename" ] "The redis module now uses /var/lib/redis/dump.rdb as database dump location.")
     (mkRemovedOptionModule [ "services" "redis" "appendOnlyFilename" ] "This option was never used.")
     (mkRemovedOptionModule [ "services" "redis" "pidFile" ] "This option was removed.")
-    (mkRemovedOptionModule [ "services" "redis" "extraConfig" ] "Use services.redis.settings instead.")
+    (mkRemovedOptionModule [ "services" "redis" "extraConfig" ] "Use services.redis.servers.*.settings instead.")
+    (mkRenamedOptionModule [ "services" "redis" "enable"] [ "services" "redis" "servers" "" "enable" ])
+    (mkRenamedOptionModule [ "services" "redis" "port"] [ "services" "redis" "servers" "" "port" ])
+    (mkRenamedOptionModule [ "services" "redis" "openFirewall"] [ "services" "redis" "servers" "" "openFirewall" ])
+    (mkRenamedOptionModule [ "services" "redis" "bind"] [ "services" "redis" "servers" "" "bind" ])
+    (mkRenamedOptionModule [ "services" "redis" "unixSocket"] [ "services" "redis" "servers" "" "unixSocket" ])
+    (mkRenamedOptionModule [ "services" "redis" "unixSocketPerm"] [ "services" "redis" "servers" "" "unixSocketPerm" ])
+    (mkRenamedOptionModule [ "services" "redis" "logLevel"] [ "services" "redis" "servers" "" "logLevel" ])
+    (mkRenamedOptionModule [ "services" "redis" "logfile"] [ "services" "redis" "servers" "" "logfile" ])
+    (mkRenamedOptionModule [ "services" "redis" "syslog"] [ "services" "redis" "servers" "" "syslog" ])
+    (mkRenamedOptionModule [ "services" "redis" "databases"] [ "services" "redis" "servers" "" "databases" ])
+    (mkRenamedOptionModule [ "services" "redis" "maxclients"] [ "services" "redis" "servers" "" "maxclients" ])
+    (mkRenamedOptionModule [ "services" "redis" "save"] [ "services" "redis" "servers" "" "save" ])
+    (mkRenamedOptionModule [ "services" "redis" "slaveOf"] [ "services" "redis" "servers" "" "slaveOf" ])
+    (mkRenamedOptionModule [ "services" "redis" "masterAuth"] [ "services" "redis" "servers" "" "masterAuth" ])
+    (mkRenamedOptionModule [ "services" "redis" "requirePass"] [ "services" "redis" "servers" "" "requirePass" ])
+    (mkRenamedOptionModule [ "services" "redis" "requirePassFile"] [ "services" "redis" "servers" "" "requirePassFile" ])
+    (mkRenamedOptionModule [ "services" "redis" "appendOnly"] [ "services" "redis" "servers" "" "appendOnly" ])
+    (mkRenamedOptionModule [ "services" "redis" "appendFsync"] [ "services" "redis" "servers" "" "appendFsync" ])
+    (mkRenamedOptionModule [ "services" "redis" "slowLogLogSlowerThan"] [ "services" "redis" "servers" "" "slowLogLogSlowerThan" ])
+    (mkRenamedOptionModule [ "services" "redis" "slowLogMaxLen"] [ "services" "redis" "servers" "" "slowLogMaxLen" ])
+    (mkRenamedOptionModule [ "services" "redis" "settings"] [ "services" "redis" "servers" "" "settings" ])
   ];
 
   ###### interface
@@ -32,18 +54,6 @@ in {
   options = {
 
     services.redis = {
-
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to enable the Redis server. Note that the NixOS module for
-          Redis disables kernel support for Transparent Huge Pages (THP),
-          because this features causes major performance problems for Redis,
-          e.g. (https://redis.io/topics/latency).
-        '';
-      };
-
       package = mkOption {
         type = types.package;
         default = pkgs.redis;
@@ -51,176 +61,233 @@ in {
         description = "Which Redis derivation to use.";
       };
 
-      port = mkOption {
-        type = types.port;
-        default = 6379;
-        description = "The port for Redis to listen to.";
-      };
-
-      vmOverCommit = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Set vm.overcommit_memory to 1 (Suggested for Background Saving: http://redis.io/topics/faq)
-        '';
-      };
+      vmOverCommit = mkEnableOption ''
+        setting of vm.overcommit_memory to 1
+        (Suggested for Background Saving: http://redis.io/topics/faq)
+      '';
 
-      openFirewall = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to open ports in the firewall for the server.
-        '';
-      };
+      servers = mkOption {
+        type = with types; attrsOf (submodule ({config, name, ...}@args: {
+          options = {
+            enable = mkEnableOption ''
+              Redis server.
+
+              Note that the NixOS module for Redis disables kernel support
+              for Transparent Huge Pages (THP),
+              because this features causes major performance problems for Redis,
+              e.g. (https://redis.io/topics/latency).
+            '';
+
+            user = mkOption {
+              type = types.str;
+              default = redisName name;
+              defaultText = literalExpression ''
+                if name == "" then "redis" else "redis-''${name}"
+              '';
+              description = "The username and groupname for redis-server.";
+            };
 
-      bind = mkOption {
-        type = with types; nullOr str;
-        default = "127.0.0.1";
-        description = ''
-          The IP interface to bind to.
-          <literal>null</literal> means "all interfaces".
-        '';
-        example = "192.0.2.1";
-      };
+            port = mkOption {
+              type = types.port;
+              default = if name == "" then 6379 else 0;
+              defaultText = literalExpression ''if name == "" then 6379 else 0'';
+              description = ''
+                The TCP port to accept connections.
+                If port 0 is specified Redis will not listen on a TCP socket.
+              '';
+            };
 
-      unixSocket = mkOption {
-        type = with types; nullOr path;
-        default = null;
-        description = "The path to the socket to bind to.";
-        example = "/run/redis/redis.sock";
-      };
+            openFirewall = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Whether to open ports in the firewall for the server.
+              '';
+            };
 
-      unixSocketPerm = mkOption {
-        type = types.int;
-        default = 750;
-        description = "Change permissions for the socket";
-        example = 700;
-      };
+            bind = mkOption {
+              type = with types; nullOr str;
+              default = "127.0.0.1";
+              description = ''
+                The IP interface to bind to.
+                <literal>null</literal> means "all interfaces".
+              '';
+              example = "192.0.2.1";
+            };
 
-      logLevel = mkOption {
-        type = types.str;
-        default = "notice"; # debug, verbose, notice, warning
-        example = "debug";
-        description = "Specify the server verbosity level, options: debug, verbose, notice, warning.";
-      };
+            unixSocket = mkOption {
+              type = with types; nullOr path;
+              default = "/run/${redisName name}/redis.sock";
+              defaultText = literalExpression ''
+                if name == "" then "/run/redis/redis.sock" else "/run/redis-''${name}/redis.sock"
+              '';
+              description = "The path to the socket to bind to.";
+            };
 
-      logfile = mkOption {
-        type = types.str;
-        default = "/dev/null";
-        description = "Specify the log file name. Also 'stdout' can be used to force Redis to log on the standard output.";
-        example = "/var/log/redis.log";
-      };
+            unixSocketPerm = mkOption {
+              type = types.int;
+              default = 660;
+              description = "Change permissions for the socket";
+              example = 600;
+            };
 
-      syslog = mkOption {
-        type = types.bool;
-        default = true;
-        description = "Enable logging to the system logger.";
-      };
+            logLevel = mkOption {
+              type = types.str;
+              default = "notice"; # debug, verbose, notice, warning
+              example = "debug";
+              description = "Specify the server verbosity level, options: debug, verbose, notice, warning.";
+            };
 
-      databases = mkOption {
-        type = types.int;
-        default = 16;
-        description = "Set the number of databases.";
-      };
+            logfile = mkOption {
+              type = types.str;
+              default = "/dev/null";
+              description = "Specify the log file name. Also 'stdout' can be used to force Redis to log on the standard output.";
+              example = "/var/log/redis.log";
+            };
 
-      maxclients = mkOption {
-        type = types.int;
-        default = 10000;
-        description = "Set the max number of connected clients at the same time.";
-      };
+            syslog = mkOption {
+              type = types.bool;
+              default = true;
+              description = "Enable logging to the system logger.";
+            };
 
-      save = mkOption {
-        type = with types; listOf (listOf int);
-        default = [ [900 1] [300 10] [60 10000] ];
-        description = "The schedule in which data is persisted to disk, represented as a list of lists where the first element represent the amount of seconds and the second the number of changes.";
-      };
+            databases = mkOption {
+              type = types.int;
+              default = 16;
+              description = "Set the number of databases.";
+            };
 
-      slaveOf = mkOption {
-        type = with types; nullOr (submodule ({ ... }: {
-          options = {
-            ip = mkOption {
-              type = str;
-              description = "IP of the Redis master";
-              example = "192.168.1.100";
+            maxclients = mkOption {
+              type = types.int;
+              default = 10000;
+              description = "Set the max number of connected clients at the same time.";
             };
 
-            port = mkOption {
-              type = port;
-              description = "port of the Redis master";
-              default = 6379;
+            save = mkOption {
+              type = with types; listOf (listOf int);
+              default = [ [900 1] [300 10] [60 10000] ];
+              description = "The schedule in which data is persisted to disk, represented as a list of lists where the first element represent the amount of seconds and the second the number of changes.";
             };
-          };
-        }));
 
-        default = null;
-        description = "IP and port to which this redis instance acts as a slave.";
-        example = { ip = "192.168.1.100"; port = 6379; };
-      };
+            slaveOf = mkOption {
+              type = with types; nullOr (submodule ({ ... }: {
+                options = {
+                  ip = mkOption {
+                    type = str;
+                    description = "IP of the Redis master";
+                    example = "192.168.1.100";
+                  };
+
+                  port = mkOption {
+                    type = port;
+                    description = "port of the Redis master";
+                    default = 6379;
+                  };
+                };
+              }));
+
+              default = null;
+              description = "IP and port to which this redis instance acts as a slave.";
+              example = { ip = "192.168.1.100"; port = 6379; };
+            };
 
-      masterAuth = mkOption {
-        type = with types; nullOr str;
-        default = null;
-        description = ''If the master is password protected (using the requirePass configuration)
-        it is possible to tell the slave to authenticate before starting the replication synchronization
-        process, otherwise the master will refuse the slave request.
-        (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE)'';
-      };
+            masterAuth = mkOption {
+              type = with types; nullOr str;
+              default = null;
+              description = ''If the master is password protected (using the requirePass configuration)
+              it is possible to tell the slave to authenticate before starting the replication synchronization
+              process, otherwise the master will refuse the slave request.
+              (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE)'';
+            };
 
-      requirePass = mkOption {
-        type = with types; nullOr str;
-        default = null;
-        description = ''
-          Password for database (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE).
-          Use requirePassFile to store it outside of the nix store in a dedicated file.
-        '';
-        example = "letmein!";
-      };
+            requirePass = mkOption {
+              type = with types; nullOr str;
+              default = null;
+              description = ''
+                Password for database (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE).
+                Use requirePassFile to store it outside of the nix store in a dedicated file.
+              '';
+              example = "letmein!";
+            };
 
-      requirePassFile = mkOption {
-        type = with types; nullOr path;
-        default = null;
-        description = "File with password for the database.";
-        example = "/run/keys/redis-password";
-      };
+            requirePassFile = mkOption {
+              type = with types; nullOr path;
+              default = null;
+              description = "File with password for the database.";
+              example = "/run/keys/redis-password";
+            };
 
-      appendOnly = mkOption {
-        type = types.bool;
-        default = false;
-        description = "By default data is only periodically persisted to disk, enable this option to use an append-only file for improved persistence.";
-      };
+            appendOnly = mkOption {
+              type = types.bool;
+              default = false;
+              description = "By default data is only periodically persisted to disk, enable this option to use an append-only file for improved persistence.";
+            };
 
-      appendFsync = mkOption {
-        type = types.str;
-        default = "everysec"; # no, always, everysec
-        description = "How often to fsync the append-only log, options: no, always, everysec.";
-      };
+            appendFsync = mkOption {
+              type = types.str;
+              default = "everysec"; # no, always, everysec
+              description = "How often to fsync the append-only log, options: no, always, everysec.";
+            };
 
-      slowLogLogSlowerThan = mkOption {
-        type = types.int;
-        default = 10000;
-        description = "Log queries whose execution take longer than X in milliseconds.";
-        example = 1000;
-      };
+            slowLogLogSlowerThan = mkOption {
+              type = types.int;
+              default = 10000;
+              description = "Log queries whose execution take longer than X in milliseconds.";
+              example = 1000;
+            };
 
-      slowLogMaxLen = mkOption {
-        type = types.int;
-        default = 128;
-        description = "Maximum number of items to keep in slow log.";
-      };
+            slowLogMaxLen = mkOption {
+              type = types.int;
+              default = 128;
+              description = "Maximum number of items to keep in slow log.";
+            };
 
-      settings = mkOption {
-        type = with types; attrsOf (oneOf [ bool int str (listOf str) ]);
+            settings = mkOption {
+              # TODO: this should be converted to freeformType
+              type = with types; attrsOf (oneOf [ bool int str (listOf str) ]);
+              default = {};
+              description = ''
+                Redis configuration. Refer to
+                <link xlink:href="https://redis.io/topics/config"/>
+                for details on supported values.
+              '';
+              example = literalExpression ''
+                {
+                  loadmodule = [ "/path/to/my_module.so" "/path/to/other_module.so" ];
+                }
+              '';
+            };
+          };
+          config.settings = mkMerge [
+            {
+              port = config.port;
+              daemonize = false;
+              supervised = "systemd";
+              loglevel = config.logLevel;
+              logfile = config.logfile;
+              syslog-enabled = config.syslog;
+              databases = config.databases;
+              maxclients = config.maxclients;
+              save = map (d: "${toString (builtins.elemAt d 0)} ${toString (builtins.elemAt d 1)}") config.save;
+              dbfilename = "dump.rdb";
+              dir = "/var/lib/${redisName name}";
+              appendOnly = config.appendOnly;
+              appendfsync = config.appendFsync;
+              slowlog-log-slower-than = config.slowLogLogSlowerThan;
+              slowlog-max-len = config.slowLogMaxLen;
+            }
+            (mkIf (config.bind != null) { bind = config.bind; })
+            (mkIf (config.unixSocket != null) {
+              unixsocket = config.unixSocket;
+              unixsocketperm = toString config.unixSocketPerm;
+            })
+            (mkIf (config.slaveOf != null) { slaveof = "${config.slaveOf.ip} ${toString config.slaveOf.port}"; })
+            (mkIf (config.masterAuth != null) { masterauth = config.masterAuth; })
+            (mkIf (config.requirePass != null) { requirepass = config.requirePass; })
+          ];
+        }));
+        description = "Configuration of multiple <literal>redis-server</literal> instances.";
         default = {};
-        description = ''
-          Redis configuration. Refer to
-          <link xlink:href="https://redis.io/topics/config"/>
-          for details on supported values.
-        '';
-        example = literalExpression ''
-          {
-            loadmodule = [ "/path/to/my_module.so" "/path/to/other_module.so" ];
-          }
-        '';
       };
     };
 
@@ -229,78 +296,61 @@ in {
 
   ###### implementation
 
-  config = mkIf config.services.redis.enable {
-    assertions = [{
-      assertion = cfg.requirePass != null -> cfg.requirePassFile == null;
-      message = "You can only set one services.redis.requirePass or services.redis.requirePassFile";
-    }];
-    boot.kernel.sysctl = (mkMerge [
+  config = mkIf (enabledServers != {}) {
+
+    assertions = attrValues (mapAttrs (name: conf: {
+      assertion = conf.requirePass != null -> conf.requirePassFile == null;
+      message = ''
+        You can only set one services.redis.servers.${name}.requirePass
+        or services.redis.servers.${name}.requirePassFile
+      '';
+    }) enabledServers);
+
+    boot.kernel.sysctl = mkMerge [
       { "vm.nr_hugepages" = "0"; }
       ( mkIf cfg.vmOverCommit { "vm.overcommit_memory" = "1"; } )
-    ]);
+    ];
 
-    networking.firewall = mkIf cfg.openFirewall {
-      allowedTCPPorts = [ cfg.port ];
-    };
-
-    users.users.redis = {
-      description = "Redis database user";
-      group = "redis";
-      isSystemUser = true;
-    };
-    users.groups.redis = {};
+    networking.firewall.allowedTCPPorts = concatMap (conf:
+      optional conf.openFirewall conf.port
+    ) (attrValues enabledServers);
 
     environment.systemPackages = [ cfg.package ];
 
-    services.redis.settings = mkMerge [
-      {
-        port = cfg.port;
-        daemonize = false;
-        supervised = "systemd";
-        loglevel = cfg.logLevel;
-        logfile = cfg.logfile;
-        syslog-enabled = cfg.syslog;
-        databases = cfg.databases;
-        maxclients = cfg.maxclients;
-        save = map (d: "${toString (builtins.elemAt d 0)} ${toString (builtins.elemAt d 1)}") cfg.save;
-        dbfilename = "dump.rdb";
-        dir = "/var/lib/redis";
-        appendOnly = cfg.appendOnly;
-        appendfsync = cfg.appendFsync;
-        slowlog-log-slower-than = cfg.slowLogLogSlowerThan;
-        slowlog-max-len = cfg.slowLogMaxLen;
-      }
-      (mkIf (cfg.bind != null) { bind = cfg.bind; })
-      (mkIf (cfg.unixSocket != null) { unixsocket = cfg.unixSocket; unixsocketperm = "${toString cfg.unixSocketPerm}"; })
-      (mkIf (cfg.slaveOf != null) { slaveof = "${cfg.slaveOf.ip} ${toString cfg.slaveOf.port}"; })
-      (mkIf (cfg.masterAuth != null) { masterauth = cfg.masterAuth; })
-      (mkIf (cfg.requirePass != null) { requirepass = cfg.requirePass; })
-    ];
+    users.users = mapAttrs' (name: conf: nameValuePair (redisName name) {
+      description = "System user for the redis-server instance ${name}";
+      isSystemUser = true;
+      group = redisName name;
+    }) enabledServers;
+    users.groups = mapAttrs' (name: conf: nameValuePair (redisName name) {
+    }) enabledServers;
 
-    systemd.services.redis = {
-      description = "Redis Server";
+    systemd.services = mapAttrs' (name: conf: nameValuePair (redisName name) {
+      description = "Redis Server - ${redisName name}";
 
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
 
-      preStart = ''
-        install -m 600 ${redisConfig} /run/redis/redis.conf
-      '' + optionalString (cfg.requirePassFile != null) ''
-        password=$(cat ${escapeShellArg cfg.requirePassFile})
-        echo "requirePass $password" >> /run/redis/redis.conf
-      '';
-
       serviceConfig = {
-        ExecStart = "${cfg.package}/bin/redis-server /run/redis/redis.conf";
+        ExecStart = "${cfg.package}/bin/redis-server /run/${redisName name}/redis.conf";
+        ExecStartPre = [("+"+pkgs.writeShellScript "${redisName name}-credentials" (''
+            install -o '${conf.user}' -m 600 ${redisConfig conf.settings} /run/${redisName name}/redis.conf
+          '' + optionalString (conf.requirePassFile != null) ''
+            {
+              printf requirePass' '
+              cat ${escapeShellArg conf.requirePassFile}
+            } >>/run/${redisName name}/redis.conf
+          '')
+        )];
         Type = "notify";
         # User and group
-        User = "redis";
-        Group = "redis";
+        User = conf.user;
+        Group = conf.user;
         # Runtime directory and mode
-        RuntimeDirectory = "redis";
+        RuntimeDirectory = redisName name;
         RuntimeDirectoryMode = "0750";
         # State directory and mode
-        StateDirectory = "redis";
+        StateDirectory = redisName name;
         StateDirectoryMode = "0700";
         # Access write directories
         UMask = "0077";
@@ -309,7 +359,7 @@ in {
         # Security
         NoNewPrivileges = true;
         # Process Properties
-        LimitNOFILE = "${toString ulimitNofile}";
+        LimitNOFILE = mkDefault "${toString (conf.maxclients + 32)}";
         # Sandboxing
         ProtectSystem = "strict";
         ProtectHome = true;
@@ -322,7 +372,9 @@ in {
         ProtectKernelModules = true;
         ProtectKernelTunables = true;
         ProtectControlGroups = true;
-        RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
+        RestrictAddressFamilies =
+          optionals (conf.port != 0) ["AF_INET" "AF_INET6"] ++
+          optional (conf.unixSocket != null) "AF_UNIX";
         RestrictNamespaces = true;
         LockPersonality = true;
         MemoryDenyWriteExecute = true;
@@ -333,6 +385,7 @@ in {
         SystemCallArchitectures = "native";
         SystemCallFilter = "~@cpu-emulation @debug @keyring @memlock @mount @obsolete @privileged @resources @setuid";
       };
-    };
+    }) enabledServers;
+
   };
 }
diff --git a/nixos/modules/services/databases/virtuoso.nix b/nixos/modules/services/databases/virtuoso.nix
deleted file mode 100644
index 8b01622ecb03..000000000000
--- a/nixos/modules/services/databases/virtuoso.nix
+++ /dev/null
@@ -1,99 +0,0 @@
-{ config, lib, pkgs, ... }:
-let
-  cfg = config.services.virtuoso;
-  virtuosoUser = "virtuoso";
-  stateDir = "/var/lib/virtuoso";
-in
-with lib;
-{
-
-  ###### interface
-
-  options = {
-
-    services.virtuoso = {
-
-      enable = mkEnableOption "Virtuoso Opensource database server";
-
-      config = mkOption {
-        type = types.lines;
-        default = "";
-        description = "Extra options to put into Virtuoso configuration file.";
-      };
-
-      parameters = mkOption {
-        type = types.lines;
-        default = "";
-        description = "Extra options to put into [Parameters] section of Virtuoso configuration file.";
-      };
-
-      listenAddress = mkOption {
-        type = types.str;
-        default = "1111";
-        example = "myserver:1323";
-        description = "ip:port or port to listen on.";
-      };
-
-      httpListenAddress = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = "myserver:8080";
-        description = "ip:port or port for Virtuoso HTTP server to listen on.";
-      };
-
-      dirsAllowed = mkOption {
-        type = types.nullOr types.str; # XXX Maybe use a list in the future?
-        default = null;
-        example = "/www, /home/";
-        description = "A list of directories Virtuoso is allowed to access";
-      };
-    };
-
-  };
-
-
-  ###### implementation
-
-  config = mkIf cfg.enable {
-
-    users.users.${virtuosoUser} =
-      { uid = config.ids.uids.virtuoso;
-        description = "virtuoso user";
-        home = stateDir;
-      };
-
-    systemd.services.virtuoso = {
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-
-      preStart = ''
-        mkdir -p ${stateDir}
-        chown ${virtuosoUser} ${stateDir}
-      '';
-
-      script = ''
-        cd ${stateDir}
-        ${pkgs.virtuoso}/bin/virtuoso-t +foreground +configfile ${pkgs.writeText "virtuoso.ini" cfg.config}
-      '';
-    };
-
-    services.virtuoso.config = ''
-      [Database]
-      DatabaseFile=${stateDir}/x-virtuoso.db
-      TransactionFile=${stateDir}/x-virtuoso.trx
-      ErrorLogFile=${stateDir}/x-virtuoso.log
-      xa_persistent_file=${stateDir}/x-virtuoso.pxa
-
-      [Parameters]
-      ServerPort=${cfg.listenAddress}
-      RunAs=${virtuosoUser}
-      ${optionalString (cfg.dirsAllowed != null) "DirsAllowed=${cfg.dirsAllowed}"}
-      ${cfg.parameters}
-
-      [HTTPServer]
-      ${optionalString (cfg.httpListenAddress != null) "ServerPort=${cfg.httpListenAddress}"}
-    '';
-
-  };
-
-}
diff --git a/nixos/modules/services/desktops/flatpak.nix b/nixos/modules/services/desktops/flatpak.nix
index 7da92cc9f264..5fecc64b4f70 100644
--- a/nixos/modules/services/desktops/flatpak.nix
+++ b/nixos/modules/services/desktops/flatpak.nix
@@ -30,6 +30,8 @@ in {
 
     environment.systemPackages = [ pkgs.flatpak ];
 
+    security.polkit.enable = true;
+
     services.dbus.packages = [ pkgs.flatpak ];
 
     systemd.packages = [ pkgs.flatpak ];
diff --git a/nixos/modules/services/desktops/gnome/glib-networking.nix b/nixos/modules/services/desktops/gnome/glib-networking.nix
index 4288b6b5de61..1039605391ab 100644
--- a/nixos/modules/services/desktops/gnome/glib-networking.nix
+++ b/nixos/modules/services/desktops/gnome/glib-networking.nix
@@ -38,7 +38,7 @@ with lib;
 
     systemd.packages = [ pkgs.glib-networking ];
 
-    environment.variables.GIO_EXTRA_MODULES = [ "${pkgs.glib-networking.out}/lib/gio/modules" ];
+    environment.sessionVariables.GIO_EXTRA_MODULES = [ "${pkgs.glib-networking.out}/lib/gio/modules" ];
 
   };
 
diff --git a/nixos/modules/services/desktops/gnome/gnome-settings-daemon.nix b/nixos/modules/services/desktops/gnome/gnome-settings-daemon.nix
index 05b5c86ddcb3..9c68c9b76e9e 100644
--- a/nixos/modules/services/desktops/gnome/gnome-settings-daemon.nix
+++ b/nixos/modules/services/desktops/gnome/gnome-settings-daemon.nix
@@ -57,26 +57,12 @@ in
       pkgs.gnome.gnome-settings-daemon
     ];
 
-    systemd.user.targets."gnome-session-initialized".wants = [
-      "gsd-color.target"
-      "gsd-datetime.target"
-      "gsd-keyboard.target"
-      "gsd-media-keys.target"
-      "gsd-print-notifications.target"
-      "gsd-rfkill.target"
-      "gsd-screensaver-proxy.target"
-      "gsd-sharing.target"
-      "gsd-smartcard.target"
-      "gsd-sound.target"
-      "gsd-wacom.target"
-      "gsd-wwan.target"
-      "gsd-a11y-settings.target"
-      "gsd-housekeeping.target"
-      "gsd-power.target"
+    systemd.user.targets."gnome-session-x11-services".wants = [
+      "org.gnome.SettingsDaemon.XSettings.service"
     ];
 
-    systemd.user.targets."gnome-session-x11-services".wants = [
-      "gsd-xsettings.target"
+    systemd.user.targets."gnome-session-x11-services-ready".wants = [
+      "org.gnome.SettingsDaemon.XSettings.service"
     ];
 
   };
diff --git a/nixos/modules/services/desktops/gnome/tracker-miners.nix b/nixos/modules/services/desktops/gnome/tracker-miners.nix
index c9101f0caa63..9351007d30b5 100644
--- a/nixos/modules/services/desktops/gnome/tracker-miners.nix
+++ b/nixos/modules/services/desktops/gnome/tracker-miners.nix
@@ -47,6 +47,8 @@ with lib;
 
     systemd.packages = [ pkgs.tracker-miners ];
 
+    services.gnome.tracker.subcommandPackages = [ pkgs.tracker-miners ];
+
   };
 
 }
diff --git a/nixos/modules/services/desktops/gnome/tracker.nix b/nixos/modules/services/desktops/gnome/tracker.nix
index 29d9662b0b8f..fef399d0112e 100644
--- a/nixos/modules/services/desktops/gnome/tracker.nix
+++ b/nixos/modules/services/desktops/gnome/tracker.nix
@@ -4,6 +4,9 @@
 
 with lib;
 
+let
+  cfg = config.services.gnome.tracker;
+in
 {
 
   meta = {
@@ -33,6 +36,15 @@ with lib;
         '';
       };
 
+      subcommandPackages = mkOption {
+        type = types.listOf types.package;
+        default = [ ];
+        internal = true;
+        description = ''
+          List of packages containing tracker3 subcommands.
+        '';
+      };
+
     };
 
   };
@@ -40,7 +52,7 @@ with lib;
 
   ###### implementation
 
-  config = mkIf config.services.gnome.tracker.enable {
+  config = mkIf cfg.enable {
 
     environment.systemPackages = [ pkgs.tracker ];
 
@@ -48,6 +60,17 @@ with lib;
 
     systemd.packages = [ pkgs.tracker ];
 
+    environment.variables = {
+      TRACKER_CLI_SUBCOMMANDS_DIR =
+        let
+          subcommandPackagesTree = pkgs.symlinkJoin {
+            name = "tracker-with-subcommands-${pkgs.tracker.version}";
+            paths = [ pkgs.tracker ] ++ cfg.subcommandPackages;
+          };
+        in
+        "${subcommandPackagesTree}/libexec/tracker3";
+    };
+
   };
 
 }
diff --git a/nixos/modules/services/desktops/gvfs.nix b/nixos/modules/services/desktops/gvfs.nix
index b6a27279bdf8..1aa64ea37db5 100644
--- a/nixos/modules/services/desktops/gvfs.nix
+++ b/nixos/modules/services/desktops/gvfs.nix
@@ -54,10 +54,10 @@ in
 
     systemd.packages = [ cfg.package ];
 
-    services.udev.packages = [ pkgs.libmtp.bin ];
+    services.udev.packages = [ pkgs.libmtp.out ];
 
     # Needed for unwrapped applications
-    environment.variables.GIO_EXTRA_MODULES = [ "${cfg.package}/lib/gio/modules" ];
+    environment.sessionVariables.GIO_EXTRA_MODULES = [ "${cfg.package}/lib/gio/modules" ];
 
   };
 
diff --git a/nixos/modules/services/desktops/pantheon/files.nix b/nixos/modules/services/desktops/pantheon/files.nix
deleted file mode 100644
index 8cee9f42b62f..000000000000
--- a/nixos/modules/services/desktops/pantheon/files.nix
+++ /dev/null
@@ -1,13 +0,0 @@
-# pantheon files daemon.
-
-{ config, pkgs, lib, ... }:
-
-with lib;
-
-{
-
-  imports = [
-    (mkRemovedOptionModule [ "services" "pantheon" "files" "enable" ] "Use `environment.systemPackages [ pkgs.pantheon.elementary-files ];`")
-  ];
-
-}
diff --git a/nixos/modules/services/desktops/pipewire/daemon/client-rt.conf.json b/nixos/modules/services/desktops/pipewire/daemon/client-rt.conf.json
index 284d8c394a61..9aa51b61431d 100644
--- a/nixos/modules/services/desktops/pipewire/daemon/client-rt.conf.json
+++ b/nixos/modules/services/desktops/pipewire/daemon/client-rt.conf.json
@@ -8,7 +8,7 @@
   },
   "context.modules": [
     {
-      "name": "libpipewire-module-rtkit",
+      "name": "libpipewire-module-rt",
       "args": {},
       "flags": [
         "ifexists",
diff --git a/nixos/modules/services/desktops/pipewire/daemon/minimal.conf.json b/nixos/modules/services/desktops/pipewire/daemon/minimal.conf.json
new file mode 100644
index 000000000000..c7f58fd5799a
--- /dev/null
+++ b/nixos/modules/services/desktops/pipewire/daemon/minimal.conf.json
@@ -0,0 +1,118 @@
+{
+  "context.properties": {
+    "link.max-buffers": 16,
+    "core.daemon": true,
+    "core.name": "pipewire-0",
+    "settings.check-quantum": true,
+    "settings.check-rate": true,
+    "vm.overrides": {
+      "default.clock.min-quantum": 1024
+    }
+  },
+  "context.spa-libs": {
+    "audio.convert.*": "audioconvert/libspa-audioconvert",
+    "api.alsa.*": "alsa/libspa-alsa",
+    "support.*": "support/libspa-support"
+  },
+  "context.modules": [
+    {
+      "name": "libpipewire-module-rt",
+      "args": {
+        "nice.level": -11
+      },
+      "flags": [
+        "ifexists",
+        "nofail"
+      ]
+    },
+    {
+      "name": "libpipewire-module-protocol-native"
+    },
+    {
+      "name": "libpipewire-module-profiler"
+    },
+    {
+      "name": "libpipewire-module-metadata"
+    },
+    {
+      "name": "libpipewire-module-spa-node-factory"
+    },
+    {
+      "name": "libpipewire-module-client-node"
+    },
+    {
+      "name": "libpipewire-module-access",
+      "args": {}
+    },
+    {
+      "name": "libpipewire-module-adapter"
+    },
+    {
+      "name": "libpipewire-module-link-factory"
+    }
+  ],
+  "context.objects": [
+    {
+      "factory": "metadata",
+      "args": {
+        "metadata.name": "default"
+      }
+    },
+    {
+      "factory": "spa-node-factory",
+      "args": {
+        "factory.name": "support.node.driver",
+        "node.name": "Dummy-Driver",
+        "node.group": "pipewire.dummy",
+        "priority.driver": 20000
+      }
+    },
+    {
+      "factory": "spa-node-factory",
+      "args": {
+        "factory.name": "support.node.driver",
+        "node.name": "Freewheel-Driver",
+        "priority.driver": 19000,
+        "node.group": "pipewire.freewheel",
+        "node.freewheel": true
+      }
+    },
+    {
+      "factory": "adapter",
+      "args": {
+        "factory.name": "api.alsa.pcm.source",
+        "node.name": "system",
+        "node.description": "system",
+        "media.class": "Audio/Source",
+        "api.alsa.path": "hw:0",
+        "node.suspend-on-idle": true,
+        "resample.disable": true,
+        "channelmix.disable": true,
+        "adapter.auto-port-config": {
+          "mode": "dsp",
+          "monitor": false,
+          "position": "unknown"
+        }
+      }
+    },
+    {
+      "factory": "adapter",
+      "args": {
+        "factory.name": "api.alsa.pcm.sink",
+        "node.name": "system",
+        "node.description": "system",
+        "media.class": "Audio/Sink",
+        "api.alsa.path": "hw:0",
+        "node.suspend-on-idle": true,
+        "resample.disable": true,
+        "channelmix.disable": true,
+        "adapter.auto-port-config": {
+          "mode": "dsp",
+          "monitor": false,
+          "position": "unknown"
+        }
+      }
+    }
+  ],
+  "context.exec": []
+}
diff --git a/nixos/modules/services/desktops/pipewire/daemon/pipewire-pulse.conf.json b/nixos/modules/services/desktops/pipewire/daemon/pipewire-pulse.conf.json
index 3ed994f11145..df0f62556dff 100644
--- a/nixos/modules/services/desktops/pipewire/daemon/pipewire-pulse.conf.json
+++ b/nixos/modules/services/desktops/pipewire/daemon/pipewire-pulse.conf.json
@@ -6,8 +6,10 @@
   },
   "context.modules": [
     {
-      "name": "libpipewire-module-rtkit",
-      "args": {},
+      "name": "libpipewire-module-rt",
+      "args": {
+        "nice.level": -11
+      },
       "flags": [
         "ifexists",
         "nofail"
@@ -37,6 +39,61 @@
       }
     }
   ],
-  "context.exec": [],
-  "stream.properties": {}
+  "context.exec": [
+    {
+      "path": "pactl",
+      "args": "load-module module-always-sink"
+    }
+  ],
+  "stream.properties": {},
+  "pulse.rules": [
+    {
+      "matches": [
+        {}
+      ],
+      "actions": {
+        "update-props": {}
+      }
+    },
+    {
+      "matches": [
+        {
+          "application.process.binary": "teams"
+        },
+        {
+          "application.process.binary": "skypeforlinux"
+        }
+      ],
+      "actions": {
+        "quirks": [
+          "force-s16-info"
+        ]
+      }
+    },
+    {
+      "matches": [
+        {
+          "application.process.binary": "firefox"
+        }
+      ],
+      "actions": {
+        "quirks": [
+          "remove-capture-dont-move"
+        ]
+      }
+    },
+    {
+      "matches": [
+        {
+          "application.name": "~speech-dispatcher*"
+        }
+      ],
+      "actions": {
+        "update-props": {
+          "pulse.min.req": "1024/48000",
+          "pulse.min.quantum": "1024/48000"
+        }
+      }
+    }
+  ]
 }
diff --git a/nixos/modules/services/desktops/pipewire/daemon/pipewire.conf.json b/nixos/modules/services/desktops/pipewire/daemon/pipewire.conf.json
index a923ab4db235..7c79f0168c02 100644
--- a/nixos/modules/services/desktops/pipewire/daemon/pipewire.conf.json
+++ b/nixos/modules/services/desktops/pipewire/daemon/pipewire.conf.json
@@ -3,6 +3,7 @@
     "link.max-buffers": 16,
     "core.daemon": true,
     "core.name": "pipewire-0",
+    "default.clock.min-quantum": 16,
     "vm.overrides": {
       "default.clock.min-quantum": 1024
     }
@@ -19,8 +20,10 @@
   },
   "context.modules": [
     {
-      "name": "libpipewire-module-rtkit",
-      "args": {},
+      "name": "libpipewire-module-rt",
+      "args": {
+        "nice.level": -11
+      },
       "flags": [
         "ifexists",
         "nofail"
diff --git a/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix b/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix
index 4be3e881a9dc..109c91134b99 100644
--- a/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix
+++ b/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix
@@ -29,6 +29,8 @@ in {
 
   meta = {
     maintainers = teams.freedesktop.members;
+    # uses attributes of the linked package
+    buildDocsInSandbox = false;
   };
 
   ###### interface
@@ -36,9 +38,8 @@ in {
     services.pipewire.media-session = {
       enable = mkOption {
         type = types.bool;
-        default = config.services.pipewire.enable;
-        defaultText = literalExpression "config.services.pipewire.enable";
-        description = "Example pipewire session manager";
+        default = false;
+        description = "Whether to enable the deprecated example Pipewire session manager";
       };
 
       package = mkOption {
@@ -94,6 +95,12 @@ in {
   config = mkIf cfg.enable {
     environment.systemPackages = [ cfg.package ];
     systemd.packages = [ cfg.package ];
+
+    # Enable either system or user units.
+    systemd.services.pipewire-media-session.enable = config.services.pipewire.systemWide;
+    systemd.user.services.pipewire-media-session.enable = !config.services.pipewire.systemWide;
+
+    systemd.services.pipewire-media-session.wantedBy = [ "pipewire.service" ];
     systemd.user.services.pipewire-media-session.wantedBy = [ "pipewire.service" ];
 
     environment.etc."pipewire/media-session.d/media-session.conf" = {
diff --git a/nixos/modules/services/desktops/pipewire/pipewire.nix b/nixos/modules/services/desktops/pipewire/pipewire.nix
index 55755ecd6457..59e9342a6ea1 100644
--- a/nixos/modules/services/desktops/pipewire/pipewire.nix
+++ b/nixos/modules/services/desktops/pipewire/pipewire.nix
@@ -25,21 +25,26 @@ let
     client = lib.importJSON ./daemon/client.conf.json;
     client-rt = lib.importJSON ./daemon/client-rt.conf.json;
     jack = lib.importJSON ./daemon/jack.conf.json;
+    minimal = lib.importJSON ./daemon/minimal.conf.json;
     pipewire = lib.importJSON ./daemon/pipewire.conf.json;
     pipewire-pulse = lib.importJSON ./daemon/pipewire-pulse.conf.json;
   };
 
+  useSessionManager = cfg.wireplumber.enable || cfg.media-session.enable;
+
   configs = {
     client = recursiveUpdate defaults.client cfg.config.client;
     client-rt = recursiveUpdate defaults.client-rt cfg.config.client-rt;
     jack = recursiveUpdate defaults.jack cfg.config.jack;
-    pipewire = recursiveUpdate defaults.pipewire cfg.config.pipewire;
+    pipewire = recursiveUpdate (if useSessionManager then defaults.pipewire else defaults.minimal) cfg.config.pipewire;
     pipewire-pulse = recursiveUpdate defaults.pipewire-pulse cfg.config.pipewire-pulse;
   };
 in {
 
   meta = {
     maintainers = teams.freedesktop.members;
+    # uses attributes of the linked package
+    buildDocsInSandbox = false;
   };
 
   ###### interface
@@ -123,6 +128,22 @@ in {
       pulse = {
         enable = mkEnableOption "PulseAudio server emulation";
       };
+
+      systemWide = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          If true, a system-wide PipeWire service and socket is enabled
+          allowing all users in the "pipewire" group to use it simultaneously.
+          If false, then user units are used instead, restricting access to
+          only one user.
+
+          Enabling system-wide PipeWire is however not recommended and disabled
+          by default according to
+          https://github.com/PipeWire/pipewire/blob/master/NEWS
+        '';
+      };
+
     };
   };
 
@@ -148,9 +169,20 @@ in {
 
     # PipeWire depends on DBUS but doesn't list it. Without this booting
     # into a terminal results in the service crashing with an error.
+    systemd.services.pipewire.bindsTo = [ "dbus.service" ];
+    systemd.user.services.pipewire.bindsTo = [ "dbus.service" ];
+
+    # Enable either system or user units.  Note that for pipewire-pulse there
+    # are only user units, which work in both cases.
+    systemd.sockets.pipewire.enable = cfg.systemWide;
+    systemd.services.pipewire.enable = cfg.systemWide;
+    systemd.user.sockets.pipewire.enable = !cfg.systemWide;
+    systemd.user.services.pipewire.enable = !cfg.systemWide;
+
+    systemd.sockets.pipewire.wantedBy = lib.mkIf cfg.socketActivation [ "sockets.target" ];
     systemd.user.sockets.pipewire.wantedBy = lib.mkIf cfg.socketActivation [ "sockets.target" ];
     systemd.user.sockets.pipewire-pulse.wantedBy = lib.mkIf (cfg.socketActivation && cfg.pulse.enable) ["sockets.target"];
-    systemd.user.services.pipewire.bindsTo = [ "dbus.service" ];
+
     services.udev.packages = [ cfg.package ];
 
     # If any paths are updated here they must also be updated in the package test.
@@ -194,7 +226,22 @@ in {
     environment.sessionVariables.LD_LIBRARY_PATH =
       lib.optional cfg.jack.enable "${cfg.package.jack}/lib";
 
+    users = lib.mkIf cfg.systemWide {
+      users.pipewire = {
+        uid = config.ids.uids.pipewire;
+        group = "pipewire";
+        extraGroups = [
+          "audio"
+          "video"
+        ] ++ lib.optional config.security.rtkit.enable "rtkit";
+        description = "Pipewire system service user";
+        isSystemUser = true;
+      };
+      groups.pipewire.gid = config.ids.gids.pipewire;
+    };
+
     # https://gitlab.freedesktop.org/pipewire/pipewire/-/issues/464#note_723554
+    systemd.services.pipewire.environment."PIPEWIRE_LINK_PASSIVE" = "1";
     systemd.user.services.pipewire.environment."PIPEWIRE_LINK_PASSIVE" = "1";
   };
 }
diff --git a/nixos/modules/services/desktops/pipewire/wireplumber.nix b/nixos/modules/services/desktops/pipewire/wireplumber.nix
new file mode 100644
index 000000000000..52ec17b95db4
--- /dev/null
+++ b/nixos/modules/services/desktops/pipewire/wireplumber.nix
@@ -0,0 +1,44 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.pipewire.wireplumber;
+in
+{
+  meta.maintainers = [ lib.maintainers.k900 ];
+
+  options = {
+    services.pipewire.wireplumber = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = config.services.pipewire.enable;
+        defaultText = lib.literalExpression "config.services.pipewire.enable";
+        description = "Whether to enable Wireplumber, a modular session / policy manager for PipeWire";
+      };
+
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.wireplumber;
+        defaultText = lib.literalExpression "pkgs.wireplumber";
+        description = "The wireplumber derivation to use.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !config.services.pipewire.media-session.enable;
+        message = "WirePlumber and pipewire-media-session can't be enabled at the same time.";
+      }
+    ];
+
+    environment.systemPackages = [ cfg.package ];
+    systemd.packages = [ cfg.package ];
+
+    systemd.services.wireplumber.enable = config.services.pipewire.systemWide;
+    systemd.user.services.wireplumber.enable = !config.services.pipewire.systemWide;
+
+    systemd.services.wireplumber.wantedBy = [ "pipewire.service" ];
+    systemd.user.services.wireplumber.wantedBy = [ "pipewire.service" ];
+  };
+}
diff --git a/nixos/modules/services/development/rstudio-server/default.nix b/nixos/modules/services/development/rstudio-server/default.nix
new file mode 100644
index 000000000000..cd903c7e55bf
--- /dev/null
+++ b/nixos/modules/services/development/rstudio-server/default.nix
@@ -0,0 +1,107 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.services.rstudio-server;
+
+  rserver-conf = builtins.toFile "rserver.conf" ''
+    server-working-dir=${cfg.serverWorkingDir}
+    www-address=${cfg.listenAddr}
+    ${cfg.rserverExtraConfig}
+  '';
+
+  rsession-conf = builtins.toFile "rsession.conf" ''
+    ${cfg.rsessionExtraConfig}
+  '';
+
+in
+{
+  meta.maintainers = with maintainers; [ jbedo cfhammill ];
+
+  options.services.rstudio-server = {
+    enable = mkEnableOption "RStudio server";
+
+    serverWorkingDir = mkOption {
+      type = types.str;
+      default = "/var/lib/rstudio-server";
+      description = ''
+        Default working directory for server (server-working-dir in rserver.conf).
+      '';
+    };
+
+    listenAddr = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = ''
+        Address to listen on (www-address in rserver.conf).
+      '';
+    };
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.rstudio-server;
+      defaultText = literalExpression "pkgs.rstudio-server";
+      example = literalExpression "pkgs.rstudioServerWrapper.override { packages = [ pkgs.rPackages.ggplot2 ]; }";
+      description = ''
+        Rstudio server package to use. Can be set to rstudioServerWrapper to provide packages.
+      '';
+    };
+
+    rserverExtraConfig = mkOption {
+      type = types.str;
+      default = "";
+      description = ''
+        Extra contents for rserver.conf.
+      '';
+    };
+
+    rsessionExtraConfig = mkOption {
+      type = types.str;
+      default = "";
+      description = ''
+        Extra contents for resssion.conf.
+      '';
+    };
+
+  };
+
+  config = mkIf cfg.enable
+    {
+      systemd.services.rstudio-server = {
+        description = "Rstudio server";
+
+        after = [ "network.target" ];
+        wantedBy = [ "multi-user.target" ];
+        restartTriggers = [ rserver-conf rsession-conf ];
+
+        serviceConfig = {
+          Restart = "on-failure";
+          Type = "forking";
+          ExecStart = "${cfg.package}/bin/rserver";
+          StateDirectory = "rstudio-server";
+          RuntimeDirectory = "rstudio-server";
+        };
+      };
+
+      environment.etc = {
+        "rstudio/rserver.conf".source = rserver-conf;
+        "rstudio/rsession.conf".source = rsession-conf;
+        "pam.d/rstudio".source = "/etc/pam.d/login";
+      };
+      environment.systemPackages = [ cfg.package ];
+
+      users = {
+        users.rstudio-server = {
+          uid = config.ids.uids.rstudio-server;
+          description = "rstudio-server";
+          group = "rstudio-server";
+        };
+        groups.rstudio-server = {
+          gid = config.ids.gids.rstudio-server;
+        };
+      };
+
+    };
+}
diff --git a/nixos/modules/services/development/zammad.nix b/nixos/modules/services/development/zammad.nix
new file mode 100644
index 000000000000..d457a6071873
--- /dev/null
+++ b/nixos/modules/services/development/zammad.nix
@@ -0,0 +1,323 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.zammad;
+  settingsFormat = pkgs.formats.yaml { };
+  filterNull = filterAttrs (_: v: v != null);
+  serviceConfig = {
+    Type = "simple";
+    Restart = "always";
+
+    User = "zammad";
+    Group = "zammad";
+    PrivateTmp = true;
+    StateDirectory = "zammad";
+    WorkingDirectory = cfg.dataDir;
+  };
+  environment = {
+    RAILS_ENV = "production";
+    NODE_ENV = "production";
+    RAILS_SERVE_STATIC_FILES = "true";
+    RAILS_LOG_TO_STDOUT = "true";
+  };
+  databaseConfig = settingsFormat.generate "database.yml" cfg.database.settings;
+in
+{
+
+  options = {
+    services.zammad = {
+      enable = mkEnableOption "Zammad, a web-based, open source user support/ticketing solution.";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.zammad;
+        defaultText = literalExpression "pkgs.zammad";
+        description = "Zammad package to use.";
+      };
+
+      dataDir = mkOption {
+        type = types.path;
+        default = "/var/lib/zammad";
+        description = ''
+          Path to a folder that will contain Zammad working directory.
+        '';
+      };
+
+      host = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        example = "192.168.23.42";
+        description = "Host address.";
+      };
+
+      openPorts = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether to open firewall ports for Zammad";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 3000;
+        description = "Web service port.";
+      };
+
+      websocketPort = mkOption {
+        type = types.port;
+        default = 6042;
+        description = "Websocket service port.";
+      };
+
+      database = {
+        type = mkOption {
+          type = types.enum [ "PostgreSQL" "MySQL" ];
+          default = "PostgreSQL";
+          example = "MySQL";
+          description = "Database engine to use.";
+        };
+
+        host = mkOption {
+          type = types.nullOr types.str;
+          default = {
+            PostgreSQL = "/run/postgresql";
+            MySQL = "localhost";
+          }.${cfg.database.type};
+          defaultText = literalExpression ''
+            {
+              PostgreSQL = "/run/postgresql";
+              MySQL = "localhost";
+            }.''${config.services.zammad.database.type};
+          '';
+          description = ''
+            Database host address.
+          '';
+        };
+
+        port = mkOption {
+          type = types.nullOr types.port;
+          default = null;
+          description = "Database port. Use <literal>null</literal> for default port.";
+        };
+
+        name = mkOption {
+          type = types.str;
+          default = "zammad";
+          description = ''
+            Database name.
+          '';
+        };
+
+        user = mkOption {
+          type = types.nullOr types.str;
+          default = "zammad";
+          description = "Database user.";
+        };
+
+        passwordFile = mkOption {
+          type = types.nullOr types.path;
+          default = null;
+          example = "/run/keys/zammad-dbpassword";
+          description = ''
+            A file containing the password for <option>services.zammad.database.user</option>.
+          '';
+        };
+
+        createLocally = mkOption {
+          type = types.bool;
+          default = true;
+          description = "Whether to create a local database automatically.";
+        };
+
+        settings = mkOption {
+          type = settingsFormat.type;
+          default = { };
+          example = literalExpression ''
+            {
+            }
+          '';
+          description = ''
+            The <filename>database.yml</filename> configuration file as key value set.
+            See <link xlink:href='TODO' />
+            for list of configuration parameters.
+          '';
+        };
+      };
+
+      secretKeyBaseFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        example = "/run/keys/secret_key_base";
+        description = ''
+          The path to a file containing the
+          <literal>secret_key_base</literal> secret.
+
+          Zammad uses <literal>secret_key_base</literal> to encrypt
+          the cookie store, which contains session data, and to digest
+          user auth tokens.
+
+          Needs to be a 64 byte long string of hexadecimal
+          characters. You can generate one by running
+
+          <screen>
+          <prompt>$ </prompt>openssl rand -hex 64 >/path/to/secret_key_base_file
+          </screen>
+
+          This should be a string, not a nix path, since nix paths are
+          copied into the world-readable nix store.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    services.zammad.database.settings = {
+      production = mapAttrs (_: v: mkDefault v) (filterNull {
+        adapter = {
+          PostgreSQL = "postgresql";
+          MySQL = "mysql2";
+        }.${cfg.database.type};
+        database = cfg.database.name;
+        pool = 50;
+        timeout = 5000;
+        encoding = "utf8";
+        username = cfg.database.user;
+        host = cfg.database.host;
+        port = cfg.database.port;
+      });
+    };
+
+    networking.firewall.allowedTCPPorts = mkIf cfg.openPorts [
+      config.services.zammad.port
+      config.services.zammad.websocketPort
+    ];
+
+    users.users.zammad = {
+      isSystemUser = true;
+      home = cfg.dataDir;
+      group = "zammad";
+    };
+
+    users.groups.zammad = { };
+
+    assertions = [
+      {
+        assertion = cfg.database.createLocally -> cfg.database.user == "zammad";
+        message = "services.zammad.database.user must be set to \"zammad\" if services.zammad.database.createLocally is set to true";
+      }
+      {
+        assertion = cfg.database.createLocally -> cfg.database.passwordFile == null;
+        message = "a password cannot be specified if services.zammad.database.createLocally is set to true";
+      }
+    ];
+
+    services.mysql = optionalAttrs (cfg.database.createLocally && cfg.database.type == "MySQL") {
+      enable = true;
+      package = mkDefault pkgs.mariadb;
+      ensureDatabases = [ cfg.database.name ];
+      ensureUsers = [
+        {
+          name = cfg.database.user;
+          ensurePermissions = { "${cfg.database.name}.*" = "ALL PRIVILEGES"; };
+        }
+      ];
+    };
+
+    services.postgresql = optionalAttrs (cfg.database.createLocally && cfg.database.type == "PostgreSQL") {
+      enable = true;
+      ensureDatabases = [ cfg.database.name ];
+      ensureUsers = [
+        {
+          name = cfg.database.user;
+          ensurePermissions = { "DATABASE ${cfg.database.name}" = "ALL PRIVILEGES"; };
+        }
+      ];
+    };
+
+    systemd.services.zammad-web = {
+      inherit environment;
+      serviceConfig = serviceConfig // {
+        # loading all the gems takes time
+        TimeoutStartSec = 1200;
+      };
+      after = [
+        "network.target"
+        "postgresql.service"
+      ];
+      requires = [
+        "postgresql.service"
+      ];
+      description = "Zammad web";
+      wantedBy = [ "multi-user.target" ];
+      preStart = ''
+        # Blindly copy the whole project here.
+        chmod -R +w .
+        rm -rf ./public/assets/*
+        rm -rf ./tmp/*
+        rm -rf ./log/*
+        cp -r --no-preserve=owner ${cfg.package}/* .
+        chmod -R +w .
+        # config file
+        cp ${databaseConfig} ./config/database.yml
+        chmod -R +w .
+        ${optionalString (cfg.database.passwordFile != null) ''
+        {
+          echo -n "  password: "
+          cat ${cfg.database.passwordFile}
+        } >> ./config/database.yml
+        ''}
+        ${optionalString (cfg.secretKeyBaseFile != null) ''
+        {
+          echo "production: "
+          echo -n "  secret_key_base: "
+          cat ${cfg.secretKeyBaseFile}
+        } > ./config/secrets.yml
+        ''}
+
+        if [ `${config.services.postgresql.package}/bin/psql \
+                  --host ${cfg.database.host} \
+                  ${optionalString
+                    (cfg.database.port != null)
+                    "--port ${toString cfg.database.port}"} \
+                  --username ${cfg.database.user} \
+                  --dbname ${cfg.database.name} \
+                  --command "SELECT COUNT(*) FROM pg_class c \
+                            JOIN pg_namespace s ON s.oid = c.relnamespace \
+                            WHERE s.nspname NOT IN ('pg_catalog', 'pg_toast', 'information_schema') \
+                              AND s.nspname NOT LIKE 'pg_temp%';" | sed -n 3p` -eq 0 ]; then
+          echo "Initialize database"
+          ./bin/rake --no-system db:migrate
+          ./bin/rake --no-system db:seed
+        else
+          echo "Migrate database"
+          ./bin/rake --no-system db:migrate
+        fi
+        echo "Done"
+      '';
+      script = "./script/rails server -b ${cfg.host} -p ${toString cfg.port}";
+    };
+
+    systemd.services.zammad-websocket = {
+      inherit serviceConfig environment;
+      after = [ "zammad-web.service" ];
+      requires = [ "zammad-web.service" ];
+      description = "Zammad websocket";
+      wantedBy = [ "multi-user.target" ];
+      script = "./script/websocket-server.rb -b ${cfg.host} -p ${toString cfg.websocketPort} start";
+    };
+
+    systemd.services.zammad-scheduler = {
+      inherit environment;
+      serviceConfig = serviceConfig // { Type = "forking"; };
+      after = [ "zammad-web.service" ];
+      requires = [ "zammad-web.service" ];
+      description = "Zammad scheduler";
+      wantedBy = [ "multi-user.target" ];
+      script = "./script/scheduler.rb start";
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ garbas taeer ];
+}
diff --git a/nixos/modules/services/games/asf.nix b/nixos/modules/services/games/asf.nix
new file mode 100644
index 000000000000..ea2bfd40fffa
--- /dev/null
+++ b/nixos/modules/services/games/asf.nix
@@ -0,0 +1,236 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.archisteamfarm;
+
+  format = pkgs.formats.json { };
+
+  asf-config = format.generate "ASF.json" (cfg.settings // {
+    # we disable it because ASF cannot update itself anyways
+    # and nixos takes care of restarting the service
+    # is in theory not needed as this is already the default for default builds
+    UpdateChannel = 0;
+    Headless = true;
+  });
+
+  ipc-config = format.generate "IPC.config" cfg.ipcSettings;
+
+  mkBot = n: c:
+    format.generate "${n}.json" (c.settings // {
+      SteamLogin = if c.username == "" then n else c.username;
+      SteamPassword = c.passwordFile;
+      # sets the password format to file (https://github.com/JustArchiNET/ArchiSteamFarm/wiki/Security#file)
+      PasswordFormat = 4;
+      Enabled = c.enabled;
+    });
+in
+{
+  options.services.archisteamfarm = {
+    enable = mkOption {
+      type = types.bool;
+      description = ''
+        If enabled, starts the ArchisSteamFarm service.
+        For configuring the SteamGuard token you will need to use the web-ui, which is enabled by default over on 127.0.0.1:1242.
+        You cannot configure ASF in any way outside of nix, since all the config files get wiped on restart and replaced with the programatically set ones by nix.
+      '';
+      default = false;
+    };
+
+    web-ui = mkOption {
+      type = types.submodule {
+        options = {
+          enable = mkEnableOption
+            "Wheter to start the web-ui. This is the preferred way of configuring things such as the steam guard token";
+
+          package = mkOption {
+            type = types.package;
+            default = pkgs.ArchiSteamFarm.ui;
+            description =
+              "Web-UI package to use. Contents must be in lib/dist.";
+          };
+        };
+      };
+      default = {
+        enable = true;
+        package = pkgs.ArchiSteamFarm.ui;
+      };
+      example = {
+        enable = false;
+      };
+      description = "The Web-UI hosted on 127.0.0.1:1242.";
+    };
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.ArchiSteamFarm;
+      description =
+        "Package to use. Should always be the latest version, for security reasons, since this module uses very new features and to not get out of sync with the Steam API.";
+    };
+
+    dataDir = mkOption {
+      type = types.path;
+      default = "/var/lib/asf";
+      description = ''
+        The ASF home directory used to store all data.
+        If left as the default value this directory will automatically be created before the ASF server starts, otherwise the sysadmin is responsible for ensuring the directory exists with appropriate ownership and permissions.'';
+    };
+
+    settings = mkOption {
+      type = format.type;
+      description = ''
+        The ASF.json file, all the options are documented <link xlink:href="https://github.com/JustArchiNET/ArchiSteamFarm/wiki/Configuration#global-config">here</link>.
+        Do note that `AutoRestart`  and `UpdateChannel` is always to `false`
+respectively `0` because NixOS takes care of updating everything.
+        `Headless` is also always set to `true` because there is no way to provide inputs via a systemd service.
+        You should try to keep ASF up to date since upstream does not provide support for anything but the latest version and you're exposing yourself to all kinds of issues - as is outlined <link xlink:href="https://github.com/JustArchiNET/ArchiSteamFarm/wiki/Configuration#updateperiod">here</link>.
+      '';
+      example = {
+        Statistics = false;
+      };
+      default = { };
+    };
+
+    ipcSettings = mkOption {
+      type = format.type;
+      description = ''
+        Settings to write to IPC.config.
+        All options can be found <link xlink:href="https://github.com/JustArchiNET/ArchiSteamFarm/wiki/IPC#custom-configuration">here</link>.
+      '';
+      example = {
+        Kestrel = {
+          Endpoints = {
+            HTTP = {
+              Url = "http://*:1242";
+            };
+          };
+        };
+      };
+      default = { };
+    };
+
+    bots = mkOption {
+      type = types.attrsOf (types.submodule {
+        options = {
+          username = mkOption {
+            type = types.str;
+            description =
+              "Name of the user to log in. Default is attribute name.";
+            default = "";
+          };
+          passwordFile = mkOption {
+            type = types.path;
+            description =
+              "Path to a file containig the password. The file must be readable by the <literal>asf</literal> user/group.";
+          };
+          enabled = mkOption {
+            type = types.bool;
+            default = true;
+            description = "Whether to enable the bot on startup.";
+          };
+          settings = mkOption {
+            type = types.attrs;
+            description =
+              "Additional settings that are documented <link xlink:href=\"https://github.com/JustArchiNET/ArchiSteamFarm/wiki/Configuration#bot-config\">here</link>.";
+            default = { };
+          };
+        };
+      });
+      description = ''
+        Bots name and configuration.
+      '';
+      example = {
+        exampleBot = {
+          username = "alice";
+          passwordFile = "/var/lib/asf/secrets/password";
+          settings = { SteamParentalCode = "1234"; };
+        };
+      };
+      default = { };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    users = {
+      users.asf = {
+        home = cfg.dataDir;
+        isSystemUser = true;
+        group = "asf";
+        description = "Archis-Steam-Farm service user";
+      };
+      groups.asf = { };
+    };
+
+    systemd.services = {
+      asf = {
+        description = "Archis-Steam-Farm Service";
+        after = [ "network.target" ];
+        wantedBy = [ "multi-user.target" ];
+
+        serviceConfig = mkMerge [
+          (mkIf (cfg.dataDir == "/var/lib/asf") { StateDirectory = "asf"; })
+          {
+            User = "asf";
+            Group = "asf";
+            WorkingDirectory = cfg.dataDir;
+            Type = "simple";
+            ExecStart =
+              "${cfg.package}/bin/ArchiSteamFarm --path ${cfg.dataDir} --process-required --no-restart --service --no-config-migrate";
+
+            # mostly copied from the default systemd service
+            PrivateTmp = true;
+            LockPersonality = true;
+            PrivateDevices = true;
+            PrivateIPC = true;
+            PrivateMounts = true;
+            PrivateUsers = true;
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            ProtectProc = "invisible";
+            ProtectSystem = "full";
+            RemoveIPC = true;
+            RestrictAddressFamilies = "AF_INET AF_INET6";
+            RestrictNamespaces = true;
+            RestrictRealtime = true;
+            RestrictSUIDSGID = true;
+          }
+        ];
+
+        preStart = ''
+          mkdir -p config
+          rm -f www
+          rm -f config/{*.json,*.config}
+
+          ln -s ${asf-config} config/ASF.json
+
+          ${strings.optionalString (cfg.ipcSettings != {}) ''
+            ln -s ${ipc-config} config/IPC.config
+          ''}
+
+          ln -s ${pkgs.runCommandLocal "ASF-bots" {} ''
+            mkdir -p $out/lib/asf/bots
+            for i in ${strings.concatStringsSep " " (lists.map (x: "${getName x},${x}") (attrsets.mapAttrsToList mkBot cfg.bots))}; do IFS=",";
+              set -- $i
+              ln -s $2 $out/lib/asf/bots/$1
+            done
+          ''}/lib/asf/bots/* config/
+
+          ${strings.optionalString cfg.web-ui.enable ''
+            ln -s ${cfg.web-ui.package}/lib/dist www
+          ''}
+        '';
+      };
+    };
+  };
+
+  meta = {
+    buildDocsInSandbox = false;
+    maintainers = with maintainers; [ lom ];
+  };
+}
diff --git a/nixos/modules/services/games/minecraft-server.nix b/nixos/modules/services/games/minecraft-server.nix
index ddbe9508a4dc..5bb8eff57629 100644
--- a/nixos/modules/services/games/minecraft-server.nix
+++ b/nixos/modules/services/games/minecraft-server.nix
@@ -182,6 +182,27 @@ in {
         Restart = "always";
         User = "minecraft";
         WorkingDirectory = cfg.dataDir;
+        # Hardening
+        CapabilityBoundingSet = [ "" ];
+        DeviceAllow = [ "" ];
+        LockPersonality = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        UMask = "0077";
       };
 
       preStart = ''
diff --git a/nixos/modules/services/games/quake3-server.nix b/nixos/modules/services/games/quake3-server.nix
index 1dc01260e8fa..175af4a83828 100644
--- a/nixos/modules/services/games/quake3-server.nix
+++ b/nixos/modules/services/games/quake3-server.nix
@@ -71,6 +71,7 @@ in {
       baseq3 = mkOption {
         type = types.either types.package types.path;
         default = defaultBaseq3;
+        defaultText = literalDocBook "Manually downloaded Quake 3 installation directory.";
         example = "/var/lib/q3ds";
         description = ''
           Path to the baseq3 files (pak*.pk3). If this is on the nix store (type = package) all .pk3 files should be saved
diff --git a/nixos/modules/services/games/terraria.nix b/nixos/modules/services/games/terraria.nix
index 7312c7e6b635..29f976b3c2ae 100644
--- a/nixos/modules/services/games/terraria.nix
+++ b/nixos/modules/services/games/terraria.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg   = config.services.terraria;
+  opt   = options.services.terraria;
   worldSizeMap = { small = 1; medium = 2; large = 3; };
   valFlag = name: val: optionalString (val != null) "-${name} \"${escape ["\\" "\""] (toString val)}\"";
   boolFlag = name: val: optionalString val "-${name}";
@@ -36,7 +37,7 @@ in
         type        = types.bool;
         default     = false;
         description = ''
-          If enabled, starts a Terraria server. The server can be connected to via <literal>tmux -S ${cfg.dataDir}/terraria.sock attach</literal>
+          If enabled, starts a Terraria server. The server can be connected to via <literal>tmux -S ''${config.${opt.dataDir}}/terraria.sock attach</literal>
           for administration by users who are a part of the <literal>terraria</literal> group (use <literal>C-b d</literal> shortcut to detach again).
         '';
       };
diff --git a/nixos/modules/services/hardware/ddccontrol.nix b/nixos/modules/services/hardware/ddccontrol.nix
index 766bf12ee9f0..f0b5a9c81960 100644
--- a/nixos/modules/services/hardware/ddccontrol.nix
+++ b/nixos/modules/services/hardware/ddccontrol.nix
@@ -20,6 +20,9 @@ in
   ###### implementation
 
   config = lib.mkIf cfg.enable {
+    # Load the i2c-dev module
+    boot.kernelModules = [ "i2c_dev" ];
+
     # Give users access to the "gddccontrol" tool
     environment.systemPackages = [
       pkgs.ddccontrol
diff --git a/nixos/modules/services/hardware/spacenavd.nix b/nixos/modules/services/hardware/spacenavd.nix
index 74725dd23d25..69ca6f102efe 100644
--- a/nixos/modules/services/hardware/spacenavd.nix
+++ b/nixos/modules/services/hardware/spacenavd.nix
@@ -15,7 +15,6 @@ in {
   config = mkIf cfg.enable {
     systemd.user.services.spacenavd = {
       description = "Daemon for the Spacenavigator 6DOF mice by 3Dconnexion";
-      after = [ "syslog.target" ];
       wantedBy = [ "graphical.target" ];
       serviceConfig = {
         ExecStart = "${pkgs.spacenavd}/bin/spacenavd -d -l syslog";
diff --git a/nixos/modules/services/hardware/tcsd.nix b/nixos/modules/services/hardware/tcsd.nix
index c549a6775013..e414b9647c9b 100644
--- a/nixos/modules/services/hardware/tcsd.nix
+++ b/nixos/modules/services/hardware/tcsd.nix
@@ -1,11 +1,12 @@
 # tcsd daemon.
 
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 
 with lib;
 let
 
   cfg = config.services.tcsd;
+  opt = options.services.tcsd;
 
   tcsdConf = pkgs.writeText "tcsd.conf" ''
     port = 30003
@@ -83,6 +84,7 @@ in
 
       platformCred = mkOption {
         default = "${cfg.stateDir}/platform.cert";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/platform.cert"'';
         type = types.path;
         description = ''
           Path to the platform credential for your TPM. Your TPM
@@ -96,6 +98,7 @@ in
 
       conformanceCred = mkOption {
         default = "${cfg.stateDir}/conformance.cert";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/conformance.cert"'';
         type = types.path;
         description = ''
           Path to the conformance credential for your TPM.
@@ -104,6 +107,7 @@ in
 
       endorsementCred = mkOption {
         default = "${cfg.stateDir}/endorsement.cert";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/endorsement.cert"'';
         type = types.path;
         description = ''
           Path to the endorsement credential for your TPM.
diff --git a/nixos/modules/services/hardware/thermald.nix b/nixos/modules/services/hardware/thermald.nix
index 3b495d00df07..fcd02ea90c6c 100644
--- a/nixos/modules/services/hardware/thermald.nix
+++ b/nixos/modules/services/hardware/thermald.nix
@@ -4,7 +4,8 @@ with lib;
 
 let
   cfg = config.services.thermald;
-in {
+in
+{
   ###### interface
   options = {
     services.thermald = {
@@ -41,6 +42,7 @@ in {
       description = "Thermal Daemon Service";
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
+        PrivateNetwork = true;
         ExecStart = ''
           ${cfg.package}/sbin/thermald \
             --no-daemon \
diff --git a/nixos/modules/services/hardware/thinkfan.nix b/nixos/modules/services/hardware/thinkfan.nix
index 7a5a7e1c41ce..4ea829e496e8 100644
--- a/nixos/modules/services/hardware/thinkfan.nix
+++ b/nixos/modules/services/hardware/thinkfan.nix
@@ -19,7 +19,7 @@ let
         description = "tuple of" + concatMapStrings (t: " (${t.description})") ts;
       };
       level = ints.unsigned;
-      special = enum [ "level auto" "level full-speed" "level disengage" ];
+      special = enum [ "level auto" "level full-speed" "level disengaged" ];
     in
       tuple [ (either level special) level level ];
 
@@ -164,7 +164,7 @@ in {
 
           LEVEL is the fan level to use: it can be an integer (0-7 with thinkpad_acpi),
           "level auto" (to keep the default firmware behavior), "level full-speed" or
-          "level disengage" (to run the fan as fast as possible).
+          "level disengaged" (to run the fan as fast as possible).
           LOW is the temperature at which to step down to the previous level.
           HIGH is the temperature at which to step up to the next level.
           All numbers are integers.
diff --git a/nixos/modules/services/hardware/triggerhappy.nix b/nixos/modules/services/hardware/triggerhappy.nix
index 4e979c4d8fa1..c2fa87875e11 100644
--- a/nixos/modules/services/hardware/triggerhappy.nix
+++ b/nixos/modules/services/hardware/triggerhappy.nix
@@ -70,7 +70,7 @@ in
         type = types.listOf (types.submodule bindingCfg);
         default = [];
         example = lib.literalExpression ''
-          [ { keys = ["PLAYPAUSE"];  cmd = "''${pkgs.mpc_cli}/bin/mpc -q toggle"; } ]
+          [ { keys = ["PLAYPAUSE"];  cmd = "''${pkgs.mpc-cli}/bin/mpc -q toggle"; } ]
         '';
         description = ''
           Key bindings for <command>triggerhappy</command>.
diff --git a/nixos/modules/services/hardware/udev.nix b/nixos/modules/services/hardware/udev.nix
index d48b5444677c..61448af2d33b 100644
--- a/nixos/modules/services/hardware/udev.nix
+++ b/nixos/modules/services/hardware/udev.nix
@@ -317,7 +317,8 @@ in
       (isYes "NET")
     ];
 
-    boot.extraModprobeConfig = "options firmware_class path=${config.hardware.firmware}/lib/firmware";
+    # We don't place this into `extraModprobeConfig` so that stage-1 ramdisk doesn't bloat.
+    environment.etc."modprobe.d/firmware.conf".text = "options firmware_class path=${config.hardware.firmware}/lib/firmware";
 
     system.activationScripts.udevd =
       ''
diff --git a/nixos/modules/services/hardware/udisks2.nix b/nixos/modules/services/hardware/udisks2.nix
index e898f3260585..6be23f39754e 100644
--- a/nixos/modules/services/hardware/udisks2.nix
+++ b/nixos/modules/services/hardware/udisks2.nix
@@ -32,6 +32,8 @@ with lib;
 
     environment.systemPackages = [ pkgs.udisks2 ];
 
+    security.polkit.enable = true;
+
     services.dbus.packages = [ pkgs.udisks2 ];
 
     systemd.tmpfiles.rules = [ "d /var/lib/udisks2 0755 root root -" ];
diff --git a/nixos/modules/services/hardware/undervolt.nix b/nixos/modules/services/hardware/undervolt.nix
index 212c0227c0d0..a743bbf21c8c 100644
--- a/nixos/modules/services/hardware/undervolt.nix
+++ b/nixos/modules/services/hardware/undervolt.nix
@@ -164,8 +164,6 @@ in
     environment.systemPackages = [ cfg.package ];
 
     systemd.services.undervolt = {
-      path = [ pkgs.undervolt ];
-
       description = "Intel Undervolting Service";
 
       # Apply undervolt on boot, nixos generation switch and resume
@@ -175,7 +173,7 @@ in
       serviceConfig = {
         Type = "oneshot";
         Restart = "no";
-        ExecStart = "${pkgs.undervolt}/bin/undervolt ${toString cliArgs}";
+        ExecStart = "${cfg.package}/bin/undervolt ${toString cliArgs}";
       };
     };
 
diff --git a/nixos/modules/services/hardware/upower.nix b/nixos/modules/services/hardware/upower.nix
index 92c060147bfc..81bf497c993d 100644
--- a/nixos/modules/services/hardware/upower.nix
+++ b/nixos/modules/services/hardware/upower.nix
@@ -155,8 +155,8 @@ in
         default = 1200;
         description = ''
           When <literal>usePercentageForPolicy</literal> is
-          <literal>false</literal>, the time remaining at which UPower will
-          consider the battery low.
+          <literal>false</literal>, the time remaining in seconds at which
+          UPower will consider the battery low.
 
           If any value (of <literal>timeLow</literal>,
           <literal>timeCritical</literal> and <literal>timeAction</literal>) is
@@ -169,8 +169,8 @@ in
         default = 300;
         description = ''
           When <literal>usePercentageForPolicy</literal> is
-          <literal>false</literal>, the time remaining at which UPower will
-          consider the battery critical.
+          <literal>false</literal>, the time remaining in seconds at which
+          UPower will consider the battery critical.
 
           If any value (of <literal>timeLow</literal>,
           <literal>timeCritical</literal> and <literal>timeAction</literal>) is
@@ -183,8 +183,8 @@ in
         default = 120;
         description = ''
           When <literal>usePercentageForPolicy</literal> is
-          <literal>false</literal>, the time remaining at which UPower will
-          take action for the critical battery level.
+          <literal>false</literal>, the time remaining in seconds at which
+          UPower will take action for the critical battery level.
 
           If any value (of <literal>timeLow</literal>,
           <literal>timeCritical</literal> and <literal>timeAction</literal>) is
diff --git a/nixos/modules/services/misc/home-assistant.nix b/nixos/modules/services/home-automation/home-assistant.nix
index 2de25d87ed39..6022227f6ea8 100644
--- a/nixos/modules/services/misc/home-assistant.nix
+++ b/nixos/modules/services/home-automation/home-assistant.nix
@@ -4,35 +4,27 @@ with lib;
 
 let
   cfg = config.services.home-assistant;
+  format = pkgs.formats.yaml {};
 
-  # cfg.config != null can be assumed here
-  configJSON = pkgs.writeText "configuration.json"
-    (builtins.toJSON (if cfg.applyDefaultConfig then
-    (recursiveUpdate defaultConfig cfg.config) else cfg.config));
+  # Render config attribute sets to YAML
+  # Values that are null will be filtered from the output, so this is one way to have optional
+  # options shown in settings.
+  # We post-process the result to add support for YAML functions, like secrets or includes, see e.g.
+  # https://www.home-assistant.io/docs/configuration/secrets/
+  filteredConfig = lib.converge (lib.filterAttrsRecursive (_: v: ! elem v [ null ])) cfg.config or {};
   configFile = pkgs.runCommand "configuration.yaml" { preferLocalBuild = true; } ''
-    ${pkgs.remarshal}/bin/json2yaml -i ${configJSON} -o $out
-    # Hack to support custom yaml objects,
-    # i.e. secrets: https://www.home-assistant.io/docs/configuration/secrets/
+    cp ${format.generate "configuration.yaml" filteredConfig} $out
     sed -i -e "s/'\!\([a-z_]\+\) \(.*\)'/\!\1 \2/;s/^\!\!/\!/;" $out
   '';
+  lovelaceConfig = cfg.lovelaceConfig or {};
+  lovelaceConfigFile = format.generate "ui-lovelace.yaml" lovelaceConfig;
 
-  lovelaceConfigJSON = pkgs.writeText "ui-lovelace.json"
-    (builtins.toJSON cfg.lovelaceConfig);
-  lovelaceConfigFile = pkgs.runCommand "ui-lovelace.yaml" { preferLocalBuild = true; } ''
-    ${pkgs.remarshal}/bin/json2yaml -i ${lovelaceConfigJSON} -o $out
-  '';
-
+  # Components advertised by the home-assistant package
   availableComponents = cfg.package.availableComponents;
 
+  # Components that were added by overriding the package
   explicitComponents = cfg.package.extraComponents;
-
-  usedPlatforms = config:
-    if isAttrs config then
-      optional (config ? platform) config.platform
-      ++ concatMap usedPlatforms (attrValues config)
-    else if isList config then
-      concatMap usedPlatforms config
-    else [ ];
+  useExplicitComponent = component: elem component explicitComponents;
 
   # Given a component "platform", looks up whether it is used in the config
   # as `platform = "platform";`.
@@ -42,34 +34,46 @@ let
   #   platform = "mqtt";
   #   ...
   # } ];
-  useComponentPlatform = component: elem component (usedPlatforms cfg.config);
+  usedPlatforms = config:
+    if isAttrs config then
+      optional (config ? platform) config.platform
+      ++ concatMap usedPlatforms (attrValues config)
+    else if isList config then
+      concatMap usedPlatforms config
+    else [ ];
 
-  useExplicitComponent = component: elem component explicitComponents;
+  useComponentPlatform = component: elem component (usedPlatforms cfg.config);
 
-  # Returns whether component is used in config or explicitly passed into package
+  # Returns whether component is used in config, explicitly passed into package or
+  # configured in the module.
   useComponent = component:
     hasAttrByPath (splitString "." component) cfg.config
     || useComponentPlatform component
-    || useExplicitComponent component;
+    || useExplicitComponent component
+    || builtins.elem component cfg.extraComponents;
 
-  # List of components used in config
+  # Final list of components passed into the package to include required dependencies
   extraComponents = filter useComponent availableComponents;
 
-  package = if (cfg.autoExtraComponents && cfg.config != null)
-    then (cfg.package.override { inherit extraComponents; })
-    else cfg.package;
+  package = (cfg.package.override (oldArgs: {
+    # Respect overrides that already exist in the passed package and
+    # concat it with values passed via the module.
+    extraComponents = oldArgs.extraComponents or [] ++ extraComponents;
+    extraPackages = ps: (oldArgs.extraPackages or (_: []) ps) ++ (cfg.extraPackages ps);
+  }));
+in {
+  imports = [
+    # Migrations in NixOS 22.05
+    (mkRemovedOptionModule [ "services" "home-assistant" "applyDefaultConfig" ] "The default config was migrated into services.home-assistant.config")
+    (mkRemovedOptionModule [ "services" "home-assistant" "autoExtraComponents" ] "Components are now parsed from services.home-assistant.config unconditionally")
+    (mkRenamedOptionModule [ "services" "home-assistant" "port" ] [ "services" "home-assistant" "config" "http" "server_port" ])
+  ];
 
-  # If you are changing this, please update the description in applyDefaultConfig
-  defaultConfig = {
-    homeassistant.time_zone = config.time.timeZone;
-    http.server_port = cfg.port;
-  } // optionalAttrs (cfg.lovelaceConfig != null) {
-    lovelace.mode = "yaml";
+  meta = {
+    buildDocsInSandbox = false;
+    maintainers = teams.home-assistant.members;
   };
 
-in {
-  meta.maintainers = teams.home-assistant.members;
-
   options.services.home-assistant = {
     # Running home-assistant on NixOS is considered an installation method that is unsupported by the upstream project.
     # https://github.com/home-assistant/architecture/blob/master/adr/0012-define-supported-installation-method.md#decision
@@ -81,42 +85,166 @@ in {
       description = "The config directory, where your <filename>configuration.yaml</filename> is located.";
     };
 
-    port = mkOption {
-      default = 8123;
-      type = types.port;
-      description = "The port on which to listen.";
+    extraComponents = mkOption {
+      type = types.listOf (types.enum availableComponents);
+      default = [
+        # List of components required to complete the onboarding
+        "default_config"
+        "met"
+        "esphome"
+      ] ++ optionals (pkgs.stdenv.hostPlatform.isAarch32 || pkgs.stdenv.hostPlatform.isAarch64) [
+        # Use the platform as an indicator that we might be running on a RaspberryPi and include
+        # relevant components
+        "rpi_power"
+      ];
+      example = literalExpression ''
+        [
+          "analytics"
+          "default_config"
+          "esphome"
+          "my"
+          "shopping_list"
+          "wled"
+        ]
+      '';
+      description = ''
+        List of <link xlink:href="https://www.home-assistant.io/integrations/">components</link> that have their dependencies included in the package.
+
+        The component name can be found in the URL, for example <literal>https://www.home-assistant.io/integrations/ffmpeg/</literal> would map to <literal>ffmpeg</literal>.
+      '';
     };
 
-    applyDefaultConfig = mkOption {
-      default = true;
-      type = types.bool;
+    extraPackages = mkOption {
+      type = types.functionTo (types.listOf types.package);
+      default = _: [];
+      defaultText = literalExpression ''
+        python3Packages: with python3Packages; [];
+      '';
+      example = literalExpression ''
+        python3Packages: with python3Packages; [
+          # postgresql support
+          psycopg2
+        ];
+      '';
       description = ''
-        Setting this option enables a few configuration options for HA based on NixOS configuration (such as time zone) to avoid having to manually specify configuration we already have.
-        </para>
-        <para>
-        Currently one side effect of enabling this is that the <literal>http</literal> component will be enabled.
-        </para>
-        <para>
-        This only takes effect if <literal>config != null</literal> in order to ensure that a manually managed <filename>configuration.yaml</filename> is not overwritten.
+        List of packages to add to propagatedBuildInputs.
+
+        A popular example is <package>python3Packages.psycopg2</package>
+        for PostgreSQL support in the recorder component.
       '';
     };
 
     config = mkOption {
-      default = null;
-      # Migrate to new option types later: https://github.com/NixOS/nixpkgs/pull/75584
-      type =  with lib.types; let
-          valueType = nullOr (oneOf [
-            bool
-            int
-            float
-            str
-            (lazyAttrsOf valueType)
-            (listOf valueType)
-          ]) // {
-            description = "Yaml value";
-            emptyValue.value = {};
+      type = types.nullOr (types.submodule {
+        freeformType = format.type;
+        options = {
+          # This is a partial selection of the most common options, so new users can quickly
+          # pick up how to match home-assistants config structure to ours. It also lets us preset
+          # config values intelligently.
+
+          homeassistant = {
+            # https://www.home-assistant.io/docs/configuration/basic/
+            name = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              example = "Home";
+              description = ''
+                Name of the location where Home Assistant is running.
+              '';
+            };
+
+            latitude = mkOption {
+              type = types.nullOr (types.either types.float types.str);
+              default = null;
+              example = 52.3;
+              description = ''
+                Latitude of your location required to calculate the time the sun rises and sets.
+              '';
+            };
+
+            longitude = mkOption {
+              type = types.nullOr (types.either types.float types.str);
+              default = null;
+              example = 4.9;
+              description = ''
+                Longitude of your location required to calculate the time the sun rises and sets.
+              '';
+            };
+
+            unit_system = mkOption {
+              type = types.nullOr (types.enum [ "metric" "imperial" ]);
+              default = null;
+              example = "metric";
+              description = ''
+                The unit system to use. This also sets temperature_unit, Celsius for Metric and Fahrenheit for Imperial.
+              '';
+            };
+
+            temperature_unit = mkOption {
+              type = types.nullOr (types.enum [ "C" "F" ]);
+              default = null;
+              example = "C";
+              description = ''
+                Override temperature unit set by unit_system. <literal>C</literal> for Celsius, <literal>F</literal> for Fahrenheit.
+              '';
+            };
+
+            time_zone = mkOption {
+              type = types.nullOr types.str;
+              default = config.time.timeZone or null;
+              defaultText = literalExpression ''
+                config.time.timeZone or null
+              '';
+              example = "Europe/Amsterdam";
+              description = ''
+                Pick your time zone from the column TZ of Wikipedia’s <link xlink:href="https://en.wikipedia.org/wiki/List_of_tz_database_time_zones">list of tz database time zones</link>.
+              '';
+            };
           };
-        in valueType;
+
+          http = {
+            # https://www.home-assistant.io/integrations/http/
+            server_host = mkOption {
+              type = types.either types.str (types.listOf types.str);
+              default = [
+                "0.0.0.0"
+                "::"
+              ];
+              example = "::1";
+              description = ''
+                Only listen to incoming requests on specific IP/host. The default listed assumes support for IPv4 and IPv6.
+              '';
+            };
+
+            server_port = mkOption {
+              default = 8123;
+              type = types.port;
+              description = ''
+                The port on which to listen.
+              '';
+            };
+          };
+
+          lovelace = {
+            # https://www.home-assistant.io/lovelace/dashboards/
+            mode = mkOption {
+              type = types.enum [ "yaml" "storage" ];
+              default = if cfg.lovelaceConfig != null
+                then "yaml"
+                else "storage";
+              defaultText = literalExpression ''
+                if cfg.lovelaceConfig != null
+                  then "yaml"
+                else "storage";
+              '';
+              example = "yaml";
+              description = ''
+                In what mode should the main Lovelace panel be, <literal>yaml</literal> or <literal>storage</literal> (UI managed).
+              '';
+            };
+          };
+        };
+      });
       example = literalExpression ''
         {
           homeassistant = {
@@ -130,15 +258,19 @@ in {
           frontend = {
             themes = "!include_dir_merge_named themes";
           };
-          http = { };
+          http = {};
           feedreader.urls = [ "https://nixos.org/blogs.xml" ];
         }
       '';
       description = ''
         Your <filename>configuration.yaml</filename> as a Nix attribute set.
-        Beware that setting this option will delete your previous <filename>configuration.yaml</filename>.
-        <link xlink:href="https://www.home-assistant.io/docs/configuration/secrets/">Secrets</link>
-        are encoded as strings as shown in the example.
+
+        YAML functions like <link xlink:href="https://www.home-assistant.io/docs/configuration/secrets/">secrets</link>
+        can be passed as a string and will be unquoted automatically.
+
+        Unless this option is explicitly set to <literal>null</literal>
+        we assume your <filename>configuration.yaml</filename> is
+        managed through this module and thereby overwritten on startup.
       '';
     };
 
@@ -147,16 +279,18 @@ in {
       type = types.bool;
       description = ''
         Whether to make <filename>configuration.yaml</filename> writable.
-        This only has an effect if <option>config</option> is set.
+
         This will allow you to edit it from Home Assistant's web interface.
+
+        This only has an effect if <option>config</option> is set.
         However, bear in mind that it will be overwritten at every start of the service.
       '';
     };
 
     lovelaceConfig = mkOption {
       default = null;
-      type = with types; nullOr attrs;
-      # from https://www.home-assistant.io/lovelace/yaml-mode/
+      type = types.nullOr format.type;
+      # from https://www.home-assistant.io/lovelace/dashboards/
       example = literalExpression ''
         {
           title = "My Awesome Home";
@@ -172,8 +306,8 @@ in {
       '';
       description = ''
         Your <filename>ui-lovelace.yaml</filename> as a Nix attribute set.
-        Setting this option will automatically add
-        <literal>lovelace.mode = "yaml";</literal> to your <option>config</option>.
+        Setting this option will automatically set <literal>lovelace.mode</literal> to <literal>yaml</literal>.
+
         Beware that setting this option will delete your previous <filename>ui-lovelace.yaml</filename>
       '';
     };
@@ -183,8 +317,10 @@ in {
       type = types.bool;
       description = ''
         Whether to make <filename>ui-lovelace.yaml</filename> writable.
-        This only has an effect if <option>lovelaceConfig</option> is set.
+
         This will allow you to edit it from Home Assistant's web interface.
+
+        This only has an effect if <option>lovelaceConfig</option> is set.
         However, bear in mind that it will be overwritten at every start of the service.
       '';
     };
@@ -201,30 +337,18 @@ in {
       type = types.package;
       example = literalExpression ''
         pkgs.home-assistant.override {
-          extraPackages = ps: with ps; [ colorlog ];
+          extraPackages = python3Packages: with python3Packages; [
+            psycopg2
+          ];
+          extraComponents = [
+            "default_config"
+            "esphome"
+            "met"
+          ];
         }
       '';
       description = ''
-        Home Assistant package to use. By default the tests are disabled, as they take a considerable amout of time to complete.
-        Override <literal>extraPackages</literal> or <literal>extraComponents</literal> in order to add additional dependencies.
-        If you specify <option>config</option> and do not set <option>autoExtraComponents</option>
-        to <literal>false</literal>, overriding <literal>extraComponents</literal> will have no effect.
-        Avoid <literal>home-assistant.overridePythonAttrs</literal> if you use <literal>autoExtraComponents</literal>.
-      '';
-    };
-
-    autoExtraComponents = mkOption {
-      default = true;
-      type = types.bool;
-      description = ''
-        If set to <literal>true</literal>, the components used in <literal>config</literal>
-        are set as the specified package's <literal>extraComponents</literal>.
-        This in turn adds all packaged dependencies to the derivation.
-        You might still see import errors in your log.
-        In this case, you will need to package the necessary dependencies yourself
-        or ask for someone else to package them.
-        If a dependency is packaged but not automatically added to this list,
-        you might need to specify it in <literal>extraPackages</literal>.
+        The Home Assistant package to use.
       '';
     };
 
@@ -240,18 +364,30 @@ in {
 
     systemd.services.home-assistant = {
       description = "Home Assistant";
-      after = [ "network.target" ];
-      preStart = optionalString (cfg.config != null) (if cfg.configWritable then ''
-        cp --no-preserve=mode ${configFile} "${cfg.configDir}/configuration.yaml"
-      '' else ''
-        rm -f "${cfg.configDir}/configuration.yaml"
-        ln -s ${configFile} "${cfg.configDir}/configuration.yaml"
-      '') + optionalString (cfg.lovelaceConfig != null) (if cfg.lovelaceConfigWritable then ''
-        cp --no-preserve=mode ${lovelaceConfigFile} "${cfg.configDir}/ui-lovelace.yaml"
-      '' else ''
-        rm -f "${cfg.configDir}/ui-lovelace.yaml"
-        ln -s ${lovelaceConfigFile} "${cfg.configDir}/ui-lovelace.yaml"
-      '');
+      after = [
+        "network-online.target"
+
+        # prevent races with database creation
+        "mysql.service"
+        "postgresql.service"
+      ];
+      preStart = let
+        copyConfig = if cfg.configWritable then ''
+          cp --no-preserve=mode ${configFile} "${cfg.configDir}/configuration.yaml"
+        '' else ''
+          rm -f "${cfg.configDir}/configuration.yaml"
+          ln -s ${configFile} "${cfg.configDir}/configuration.yaml"
+        '';
+        copyLovelaceConfig = if cfg.lovelaceConfigWritable then ''
+          cp --no-preserve=mode ${lovelaceConfigFile} "${cfg.configDir}/ui-lovelace.yaml"
+        '' else ''
+          rm -f "${cfg.configDir}/ui-lovelace.yaml"
+          ln -s ${lovelaceConfigFile} "${cfg.configDir}/ui-lovelace.yaml"
+        '';
+      in
+        (optionalString (cfg.config != null) copyConfig) +
+        (optionalString (cfg.lovelaceConfig != null) copyLovelaceConfig)
+      ;
       serviceConfig = let
         # List of capabilities to equip home-assistant with, depending on configured components
         capabilities = [
@@ -278,6 +414,11 @@ in {
           "bluetooth_tracker"
           "bluetooth_le_tracker"
         ];
+        componentsUsingPing = [
+          # Components that require the capset syscall for the ping wrapper
+          "ping"
+          "wake_on_lan"
+        ];
         componentsUsingSerialDevices = [
           # Components that require access to serial devices (/dev/tty*)
           # List generated from home-assistant documentation:
@@ -324,7 +465,7 @@ in {
           "zwave_js"
         ];
       in {
-        ExecStart = "${package}/bin/hass --runner --config '${cfg.configDir}'";
+        ExecStart = "${package}/bin/hass --config '${cfg.configDir}'";
         ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
         User = "hass";
         Group = "hass";
@@ -382,6 +523,8 @@ in {
         SystemCallFilter = [
           "@system-service"
           "~@privileged"
+        ] ++ optionals (any useComponent componentsUsingPing) [
+          "capset"
         ];
         UMask = "0077";
       };
diff --git a/nixos/modules/services/misc/zigbee2mqtt.nix b/nixos/modules/services/home-automation/zigbee2mqtt.nix
index ff6d595e5a6e..ff6d595e5a6e 100644
--- a/nixos/modules/services/misc/zigbee2mqtt.nix
+++ b/nixos/modules/services/home-automation/zigbee2mqtt.nix
diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix
new file mode 100644
index 000000000000..223a993c505b
--- /dev/null
+++ b/nixos/modules/services/logging/filebeat.nix
@@ -0,0 +1,253 @@
+{ config, lib, utils, pkgs, ... }:
+
+let
+  inherit (lib)
+    attrValues
+    literalExpression
+    mkEnableOption
+    mkIf
+    mkOption
+    types;
+
+  cfg = config.services.filebeat;
+
+  json = pkgs.formats.json {};
+in
+{
+  options = {
+
+    services.filebeat = {
+
+      enable = mkEnableOption "filebeat";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.filebeat;
+        defaultText = literalExpression "pkgs.filebeat";
+        example = literalExpression "pkgs.filebeat7";
+        description = ''
+          The filebeat package to use.
+        '';
+      };
+
+      inputs = mkOption {
+        description = ''
+          Inputs specify how Filebeat locates and processes input data.
+
+          This is like <literal>services.filebeat.settings.filebeat.inputs</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat input.
+
+          An input type can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.inputs._name_.type"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            type = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The input type.
+
+                Look for the value after <literal>type:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            journald.id = "everything";  # Only for filebeat7
+            log = {
+              enabled = true;
+              paths = [
+                "/var/log/*.log"
+              ];
+            };
+          };
+        '';
+      };
+
+      modules = mkOption {
+        description = ''
+          Filebeat modules provide a quick way to get started
+          processing common log formats. They contain default
+          configurations, Elasticsearch ingest pipeline definitions,
+          and Kibana dashboards to help you implement and deploy a log
+          monitoring solution.
+
+          This is like <literal>services.filebeat.settings.filebeat.modules</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat module.
+
+          A module can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.modules._name_.module"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            module = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The name of the module.
+
+                Look for the value after <literal>module:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            nginx = {
+              access = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/access.log*" ];
+              };
+              error = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/error.log*" ];
+              };
+            };
+          };
+        '';
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = json.type;
+
+          options = {
+
+            output.elasticsearch.hosts = mkOption {
+              type = with types; listOf str;
+              default = [ "127.0.0.1:9200" ];
+              example = [ "myEShost:9200" ];
+              description = ''
+                The list of Elasticsearch nodes to connect to.
+
+                The events are distributed to these nodes in round
+                robin order. If one node becomes unreachable, the
+                event is automatically sent to another node. Each
+                Elasticsearch node can be defined as a URL or
+                IP:PORT. For example:
+                <literal>http://192.15.3.2</literal>,
+                <literal>https://es.found.io:9230</literal> or
+                <literal>192.24.3.2:9300</literal>. If no port is
+                specified, <literal>9200</literal> is used.
+              '';
+            };
+
+            filebeat = {
+              inputs = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Inputs specify how Filebeat locates and processes
+                  input data. Use <xref
+                  linkend="opt-services.filebeat.inputs"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+                '';
+              };
+              modules = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Filebeat modules provide a quick way to get started
+                  processing common log formats. They contain default
+                  configurations, Elasticsearch ingest pipeline
+                  definitions, and Kibana dashboards to help you
+                  implement and deploy a log monitoring solution.
+
+                  Use <xref linkend="opt-services.filebeat.modules"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+                '';
+              };
+            };
+          };
+        };
+        default = {};
+        example = literalExpression ''
+          {
+            settings = {
+              output.elasticsearch = {
+                hosts = [ "myEShost:9200" ];
+                username = "filebeat_internal";
+                password = { _secret = "/var/keys/elasticsearch_password"; };
+              };
+              logging.level = "info";
+            };
+          };
+        '';
+
+        description = ''
+          Configuration for filebeat. See
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/>
+          for supported values.
+
+          Options containing secret data should be set to an attribute
+          set containing the attribute <literal>_secret</literal> - a
+          string pointing to a file containing the value the option
+          should be set to. See the example to get a better picture of
+          this: in the resulting
+          <filename>filebeat.yml</filename> file, the
+          <literal>output.elasticsearch.password</literal>
+          key will be set to the contents of the
+          <filename>/var/keys/elasticsearch_password</filename> file.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
+    services.filebeat.settings.filebeat.modules = attrValues cfg.modules;
+
+    systemd.services.filebeat = {
+      description = "Filebeat log shipper";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
+      serviceConfig = {
+        ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" ''
+          set -euo pipefail
+
+          umask u=rwx,g=,o=
+
+          ${utils.genJqSecretsReplacementSnippet
+              cfg.settings
+              "/var/lib/filebeat/filebeat.yml"
+           }
+        '';
+        ExecStart = ''
+          ${cfg.package}/bin/filebeat -e \
+            -c "/var/lib/filebeat/filebeat.yml" \
+            --path.data "/var/lib/filebeat"
+        '';
+        Restart = "always";
+        StateDirectory = "filebeat";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix
index 2d98598c1bee..4035ab48b4b8 100644
--- a/nixos/modules/services/logging/journalbeat.nix
+++ b/nixos/modules/services/logging/journalbeat.nix
@@ -5,14 +5,10 @@ with lib;
 let
   cfg = config.services.journalbeat;
 
-  lt6 = builtins.compareVersions cfg.package.version "6" < 0;
-
   journalbeatYml = pkgs.writeText "journalbeat.yml" ''
     name: ${cfg.name}
     tags: ${builtins.toJSON cfg.tags}
 
-    ${optionalString lt6 "journalbeat.cursor_state_file: /var/lib/${cfg.stateDir}/cursor-state"}
-
     ${cfg.extraConfig}
   '';
 
@@ -28,7 +24,6 @@ in
         type = types.package;
         default = pkgs.journalbeat;
         defaultText = literalExpression "pkgs.journalbeat";
-        example = literalExpression "pkgs.journalbeat7";
         description = ''
           The journalbeat package to use
         '';
@@ -58,17 +53,7 @@ in
 
       extraConfig = mkOption {
         type = types.lines;
-        default = optionalString lt6 ''
-          journalbeat:
-            seek_position: cursor
-            cursor_seek_fallback: tail
-            write_cursor_state: true
-            cursor_flush_period: 5s
-            clean_field_names: true
-            convert_to_numbers: false
-            move_metadata_to_field: journal
-            default_type: journal
-        '';
+        default = "";
         description = "Any other configuration options you want to add";
       };
 
@@ -89,6 +74,8 @@ in
     systemd.services.journalbeat = {
       description = "Journalbeat log shipper";
       wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
       preStart = ''
         mkdir -p ${cfg.stateDir}/data
         mkdir -p ${cfg.stateDir}/logs
diff --git a/nixos/modules/services/logging/logrotate.nix b/nixos/modules/services/logging/logrotate.nix
index ba5d6e29d0bd..082cf92ff4ef 100644
--- a/nixos/modules/services/logging/logrotate.nix
+++ b/nixos/modules/services/logging/logrotate.nix
@@ -5,7 +5,7 @@ with lib;
 let
   cfg = config.services.logrotate;
 
-  pathOpts = {
+  pathOpts = { name, ... }:  {
     options = {
       enable = mkOption {
         type = types.bool;
@@ -16,10 +16,19 @@ let
         '';
       };
 
-      path = mkOption {
+      name = mkOption {
         type = types.str;
+        internal = true;
+      };
+
+      path = mkOption {
+        type = with types; either str (listOf str);
+        default = name;
+        defaultText = "attribute name";
         description = ''
           The path to log files to be rotated.
+          Spaces are allowed and normal shell quoting rules apply,
+          with ', ", and \ characters supported.
         '';
       };
 
@@ -74,15 +83,12 @@ let
       };
     };
 
-    config.extraConfig = ''
-      missingok
-      notifempty
-    '';
+    config.name = name;
   };
 
   mkConf = pathOpts: ''
     # generated by NixOS using the `services.logrotate.paths.${pathOpts.name}` attribute set
-    "${pathOpts.path}" {
+    ${concatMapStringsSep " " (path: ''"${path}"'') (toList pathOpts.path)} {
       ${optionalString (pathOpts.user != null || pathOpts.group != null) "su ${pathOpts.user} ${pathOpts.group}"}
       ${pathOpts.frequency}
       rotate ${toString pathOpts.keep}
@@ -90,8 +96,12 @@ let
     }
   '';
 
-  paths = sortProperties (mapAttrsToList (name: pathOpts: pathOpts // { name = name; }) (filterAttrs (_: pathOpts: pathOpts.enable) cfg.paths));
-  configFile = pkgs.writeText "logrotate.conf" (concatStringsSep "\n" ((map mkConf paths) ++ [ cfg.extraConfig ]));
+  paths = sortProperties (attrValues (filterAttrs (_: pathOpts: pathOpts.enable) cfg.paths));
+  configFile = pkgs.writeText "logrotate.conf" (
+    concatStringsSep "\n" (
+      [ "missingok" "notifempty" cfg.extraConfig ] ++ (map mkConf paths)
+    )
+  );
 
 in
 {
@@ -101,7 +111,10 @@ in
 
   options = {
     services.logrotate = {
-      enable = mkEnableOption "the logrotate systemd service";
+      enable = mkEnableOption "the logrotate systemd service" // {
+        default = foldr (n: a: a || n.enable) false (attrValues cfg.paths);
+        defaultText = literalExpression "cfg.paths != {}";
+      };
 
       paths = mkOption {
         type = with types; attrsOf (submodule pathOpts);
@@ -154,15 +167,12 @@ in
 
     systemd.services.logrotate = {
       description = "Logrotate Service";
-      wantedBy = [ "multi-user.target" ];
       startAt = "hourly";
-      script = ''
-        exec ${pkgs.logrotate}/sbin/logrotate ${configFile}
-      '';
 
       serviceConfig = {
         Restart = "no";
         User = "root";
+        ExecStart = "${pkgs.logrotate}/sbin/logrotate ${configFile}";
       };
     };
   };
diff --git a/nixos/modules/services/logging/promtail.nix b/nixos/modules/services/logging/promtail.nix
index 95c83796ece6..a34bc07b6ab2 100644
--- a/nixos/modules/services/logging/promtail.nix
+++ b/nixos/modules/services/logging/promtail.nix
@@ -45,7 +45,7 @@ in {
         Restart = "on-failure";
         TimeoutStopSec = 10;
 
-        ExecStart = "${pkgs.grafana-loki}/bin/promtail -config.file=${prettyJSON cfg.configuration} ${escapeShellArgs cfg.extraFlags}";
+        ExecStart = "${pkgs.promtail}/bin/promtail -config.file=${prettyJSON cfg.configuration} ${escapeShellArgs cfg.extraFlags}";
 
         ProtectSystem = "strict";
         ProtectHome = true;
diff --git a/nixos/modules/services/mail/dovecot.nix b/nixos/modules/services/mail/dovecot.nix
index c39827c5b867..a8c1f176782c 100644
--- a/nixos/modules/services/mail/dovecot.nix
+++ b/nixos/modules/services/mail/dovecot.nix
@@ -38,7 +38,7 @@ let
         ssl_cert = <${cfg.sslServerCert}
         ssl_key = <${cfg.sslServerKey}
         ${optionalString (cfg.sslCACert != null) ("ssl_ca = <" + cfg.sslCACert)}
-        ssl_dh = <${config.security.dhparams.params.dovecot2.path}
+        ${optionalString cfg.enableDHE ''ssl_dh = <${config.security.dhparams.params.dovecot2.path}''}
         disable_plaintext_auth = yes
       ''
     )
@@ -169,25 +169,13 @@ in
   ];
 
   options.services.dovecot2 = {
-    enable = mkEnableOption "Dovecot 2.x POP3/IMAP server";
+    enable = mkEnableOption "the dovecot 2.x POP3/IMAP server";
 
-    enablePop3 = mkOption {
-      type = types.bool;
-      default = false;
-      description = "Start the POP3 listener (when Dovecot is enabled).";
-    };
+    enablePop3 = mkEnableOption "starting the POP3 listener (when Dovecot is enabled).";
 
-    enableImap = mkOption {
-      type = types.bool;
-      default = true;
-      description = "Start the IMAP listener (when Dovecot is enabled).";
-    };
+    enableImap = mkEnableOption "starting the IMAP listener (when Dovecot is enabled)." // { default = true; };
 
-    enableLmtp = mkOption {
-      type = types.bool;
-      default = false;
-      description = "Start the LMTP listener (when Dovecot is enabled).";
-    };
+    enableLmtp = mkEnableOption "starting the LMTP listener (when Dovecot is enabled).";
 
     protocols = mkOption {
       type = types.listOf types.str;
@@ -279,13 +267,9 @@ in
       description = "Default group to store mail for virtual users.";
     };
 
-    createMailUser = mkOption {
-      type = types.bool;
-      default = true;
-      description = ''Whether to automatically create the user
-        given in <option>services.dovecot.user</option> and the group
-        given in <option>services.dovecot.group</option>.'';
-    };
+    createMailUser = mkEnableOption ''automatically creating the user
+      given in <option>services.dovecot.user</option> and the group
+      given in <option>services.dovecot.group</option>.'' // { default = true; };
 
     modules = mkOption {
       type = types.listOf types.package;
@@ -316,11 +300,9 @@ in
       description = "Path to the server's private key.";
     };
 
-    enablePAM = mkOption {
-      type = types.bool;
-      default = true;
-      description = "Whether to create a own Dovecot PAM service and configure PAM user logins.";
-    };
+    enablePAM = mkEnableOption "creating a own Dovecot PAM service and configure PAM user logins." // { default = true; };
+
+    enableDHE = mkEnableOption "enable ssl_dh and generation of primes for the key exchange." // { default = true; };
 
     sieveScripts = mkOption {
       type = types.attrsOf types.path;
@@ -328,11 +310,7 @@ in
       description = "Sieve scripts to be executed. Key is a sequence, e.g. 'before2', 'after' etc.";
     };
 
-    showPAMFailure = mkOption {
-      type = types.bool;
-      default = false;
-      description = "Show the PAM failure message on authentication error (useful for OTPW).";
-    };
+    showPAMFailure = mkEnableOption "showing the PAM failure message on authentication error (useful for OTPW).";
 
     mailboxes = mkOption {
       type = with types; coercedTo
@@ -348,12 +326,7 @@ in
       description = "Configure mailboxes and auto create or subscribe them.";
     };
 
-    enableQuota = mkOption {
-      type = types.bool;
-      default = false;
-      example = true;
-      description = "Whether to enable the dovecot quota service.";
-    };
+    enableQuota = mkEnableOption "the dovecot quota service.";
 
     quotaPort = mkOption {
       type = types.str;
@@ -376,7 +349,7 @@ in
   config = mkIf cfg.enable {
     security.pam.services.dovecot2 = mkIf cfg.enablePAM {};
 
-    security.dhparams = mkIf (cfg.sslServerCert != null) {
+    security.dhparams = mkIf (cfg.sslServerCert != null && cfg.enableDHE) {
       enable = true;
       params.dovecot2 = {};
     };
diff --git a/nixos/modules/services/mail/maddy.nix b/nixos/modules/services/mail/maddy.nix
index 44cfa3c2908d..0b06905ac6f1 100644
--- a/nixos/modules/services/mail/maddy.nix
+++ b/nixos/modules/services/mail/maddy.nix
@@ -3,9 +3,16 @@
 with lib;
 
 let
+
   name = "maddy";
+
   cfg = config.services.maddy;
+
   defaultConfig = ''
+    # Minimal configuration with TLS disabled, adapted from upstream example
+    # configuration here https://github.com/foxcpp/maddy/blob/master/maddy.conf
+    # Do not use this in production!
+
     tls off
 
     auth.pass_table local_authdb {
@@ -131,22 +138,34 @@ let
 in {
   options = {
     services.maddy = {
+
       enable = mkEnableOption "Maddy, a free an open source mail server";
 
       user = mkOption {
         default = "maddy";
         type = with types; uniq string;
         description = ''
-          Name of the user under which maddy will run. If not specified, a
-          default user will be created.
+          User account under which maddy runs.
+
+          <note><para>
+          If left as the default value this user will automatically be created
+          on system activation, otherwise the sysadmin is responsible for
+          ensuring the user exists before the maddy service starts.
+          </para></note>
         '';
       };
+
       group = mkOption {
         default = "maddy";
         type = with types; uniq string;
         description = ''
-          Name of the group under which maddy will run. If not specified, a
-          default group will be created.
+          Group account under which maddy runs.
+
+          <note><para>
+          If left as the default value this group will automatically be created
+          on system activation, otherwise the sysadmin is responsible for
+          ensuring the group exists before the maddy service starts.
+          </para></note>
         '';
       };
 
@@ -158,6 +177,7 @@ in {
           Hostname to use. It should be FQDN.
         '';
       };
+
       primaryDomain = mkOption {
         default = "localhost";
         type = with types; uniq string;
@@ -166,6 +186,7 @@ in {
           Primary MX domain to use. It should be FQDN.
         '';
       };
+
       localDomains = mkOption {
         type = with types; listOf str;
         default = ["$(primary_domain)"];
@@ -178,11 +199,18 @@ in {
           Define list of allowed domains.
         '';
       };
+
       config = mkOption {
         type = with types; nullOr lines;
         default = defaultConfig;
         description = ''
-          Server configuration.
+          Server configuration, see
+          <link xlink:href="https://maddy.email">https://maddy.email</link> for
+          more information. The default configuration of this module will setup
+          minimal maddy instance for mail transfer without TLS encryption.
+          <note><para>
+          This should not be used in a production environment.
+          </para></note>
         '';
       };
 
@@ -203,9 +231,11 @@ in {
       packages = [ pkgs.maddy ];
       services.maddy = {
         serviceConfig = {
-          User = "${cfg.user}";
-          Group = "${cfg.group}";
+          User = cfg.user;
+          Group = cfg.group;
+          StateDirectory = [ "maddy" ];
         };
+        restartTriggers = [ config.environment.etc."maddy/maddy.conf".source ];
         wantedBy = [ "multi-user.target" ];
       };
     };
@@ -220,20 +250,16 @@ in {
       '';
     };
 
-    users.users = optionalAttrs (cfg.user == "maddy") {
-      maddy = {
-        description = "Maddy service user";
-        group = cfg.group;
-        home = "/var/lib/maddy";
-        createHome = true;
+    users.users = optionalAttrs (cfg.user == name) {
+      ${name} = {
         isSystemUser = true;
+        group = cfg.group;
+        description = "Maddy mail transfer agent user";
       };
     };
 
-    users.groups = mkIf (cfg.group == "maddy") {
-      maddy = pkgs.lib.mkForce {
-        name = cfg.group;
-      };
+    users.groups = optionalAttrs (cfg.group == name) {
+      ${cfg.group} = { };
     };
 
     networking.firewall = mkIf cfg.openFirewall {
diff --git a/nixos/modules/services/mail/postfixadmin.nix b/nixos/modules/services/mail/postfixadmin.nix
index f5c8efb3076c..a0846ad52902 100644
--- a/nixos/modules/services/mail/postfixadmin.nix
+++ b/nixos/modules/services/mail/postfixadmin.nix
@@ -114,7 +114,7 @@ in
               location ~* \.php$ {
                 fastcgi_split_path_info ^(.+\.php)(/.+)$;
                 fastcgi_pass unix:${fpm.socket};
-                include ${pkgs.nginx}/conf/fastcgi_params;
+                include ${config.services.nginx.package}/conf/fastcgi_params;
                 include ${pkgs.nginx}/conf/fastcgi.conf;
               }
             '';
diff --git a/nixos/modules/services/mail/roundcube.nix b/nixos/modules/services/mail/roundcube.nix
index ac192c56aa60..1dd393da8822 100644
--- a/nixos/modules/services/mail/roundcube.nix
+++ b/nixos/modules/services/mail/roundcube.nix
@@ -153,7 +153,7 @@ in
               location ~* \.php$ {
                 fastcgi_split_path_info ^(.+\.php)(/.+)$;
                 fastcgi_pass unix:${fpm.socket};
-                include ${pkgs.nginx}/conf/fastcgi_params;
+                include ${config.services.nginx.package}/conf/fastcgi_params;
                 include ${pkgs.nginx}/conf/fastcgi.conf;
               }
             '';
diff --git a/nixos/modules/services/mail/rspamd.nix b/nixos/modules/services/mail/rspamd.nix
index 50208cbeb00a..a570e137a55a 100644
--- a/nixos/modules/services/mail/rspamd.nix
+++ b/nixos/modules/services/mail/rspamd.nix
@@ -5,6 +5,7 @@ with lib;
 let
 
   cfg = config.services.rspamd;
+  opt = options.services.rspamd;
   postfixCfg = config.services.postfix;
 
   bindSocketOpts = {options, config, ... }: {
@@ -285,8 +286,8 @@ in
               bindSockets = [{
                 socket = "/run/rspamd/rspamd.sock";
                 mode = "0660";
-                owner = "${cfg.user}";
-                group = "${cfg.group}";
+                owner = "''${config.${opt.user}}";
+                group = "''${config.${opt.group}}";
               }];
             };
             controller = {
diff --git a/nixos/modules/services/misc/matrix-synapse-log_config.yaml b/nixos/modules/services/matrix/matrix-synapse-log_config.yaml
index d85bdd1208f9..d85bdd1208f9 100644
--- a/nixos/modules/services/misc/matrix-synapse-log_config.yaml
+++ b/nixos/modules/services/matrix/matrix-synapse-log_config.yaml
diff --git a/nixos/modules/services/matrix/matrix-synapse.nix b/nixos/modules/services/matrix/matrix-synapse.nix
new file mode 100644
index 000000000000..c4d14dbd547e
--- /dev/null
+++ b/nixos/modules/services/matrix/matrix-synapse.nix
@@ -0,0 +1,773 @@
+{ config, lib, options, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.matrix-synapse;
+  format = pkgs.formats.yaml {};
+
+  # remove null values from the final configuration
+  finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings;
+  configFile = format.generate "homeserver.yaml" finalSettings;
+  logConfigFile = format.generate "log_config.yaml" cfg.logConfig;
+
+  pluginsEnv = cfg.package.python.buildEnv.override {
+    extraLibs = cfg.plugins;
+  };
+
+  usePostgresql = cfg.settings.database.name == "psycopg2";
+  hasLocalPostgresDB = let args = cfg.settings.database.args; in
+    usePostgresql && (!(args ? host) || (elem args.host [ "localhost" "127.0.0.1" "::1" ]));
+
+  registerNewMatrixUser =
+    let
+      isIpv6 = x: lib.length (lib.splitString ":" x) > 1;
+      listener =
+        lib.findFirst (
+          listener: lib.any (
+            resource: lib.any (
+              name: name == "client"
+            ) resource.names
+          ) listener.resources
+        ) (lib.last cfg.settings.listeners) cfg.settings.listeners;
+        # FIXME: Handle cases with missing client listener properly,
+        # don't rely on lib.last, this will not work.
+
+      # add a tail, so that without any bind_addresses we still have a useable address
+      bindAddress = head (listener.bind_addresses ++ [ "127.0.0.1" ]);
+      listenerProtocol = if listener.tls
+        then "https"
+        else "http";
+    in
+    pkgs.writeShellScriptBin "matrix-synapse-register_new_matrix_user" ''
+      exec ${cfg.package}/bin/register_new_matrix_user \
+        $@ \
+        ${lib.concatMapStringsSep " " (x: "-c ${x}") ([ configFile ] ++ cfg.extraConfigFiles)} \
+        "${listenerProtocol}://${
+          if (isIpv6 bindAddress) then
+            "[${bindAddress}]"
+          else
+            "${bindAddress}"
+        }:${builtins.toString listener.port}/"
+    '';
+in {
+
+  imports = [
+
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "trusted_third_party_id_servers" ] ''
+      The `trusted_third_party_id_servers` option as been removed in `matrix-synapse` v1.4.0
+      as the behavior is now obsolete.
+    '')
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "create_local_database" ] ''
+      Database configuration must be done manually. An exemplary setup is demonstrated in
+      <nixpkgs/nixos/tests/matrix-synapse.nix>
+    '')
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "web_client" ] "")
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "room_invite_state_types" ] ''
+      You may add additional event types via
+      `services.matrix-synapse.room_prejoin_state.additional_event_types` and
+      disable the default events via
+      `services.matrix-synapse.room_prejoin_state.disable_default_event_types`.
+    '')
+
+    # options that don't exist in synapse anymore
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "bind_host" ] "Use listener settings instead." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "bind_port" ] "Use listener settings instead." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "expire_access_tokens" ] "" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "no_tls" ] "It is no longer supported by synapse." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "tls_dh_param_path" ] "It was removed from synapse." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "unsecure_port" ] "Use settings.listeners instead." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "user_creation_max_duration" ] "It is no longer supported by synapse." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "verbose" ] "Use a log config instead." )
+
+    # options that were moved into rfc42 style settigns
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "app_service_config_files" ] "Use settings.app_service_config_Files instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "database_args" ] "Use settings.database.args instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "database_name" ] "Use settings.database.args.database instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "database_type" ] "Use settings.database.name instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "database_user" ] "Use settings.database.args.user instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "dynamic_thumbnails" ] "Use settings.dynamic_thumbnails instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "enable_metrics" ] "Use settings.enable_metrics instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "enable_registration" ] "Use settings.enable_registration instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "extraConfig" ] "Use settings instead." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "listeners" ] "Use settings.listeners instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "logConfig" ] "Use settings.log_config instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "max_image_pixels" ] "Use settings.max_image_pixels instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "max_upload_size" ] "Use settings.max_upload_size instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "presence" "enabled" ] "Use settings.presence.enabled instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "public_baseurl" ] "Use settings.public_baseurl instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "report_stats" ] "Use settings.report_stats instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "server_name" ] "Use settings.server_name instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "servers" ] "Use settings.trusted_key_servers instead." )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "tls_certificate_path" ] "Use settings.tls_certificate_path instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "tls_private_key_path" ] "Use settings.tls_private_key_path instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "turn_shared_secret" ] "Use settings.turn_shared_secret instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "turn_uris" ] "Use settings.turn_uris instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "turn_user_lifetime" ] "Use settings.turn_user_lifetime instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "url_preview_enabled" ] "Use settings.url_preview_enabled instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "url_preview_ip_range_blacklist" ] "Use settings.url_preview_ip_range_blacklist instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "url_preview_ip_range_whitelist" ] "Use settings.url_preview_ip_range_whitelist instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "url_preview_url_blacklist" ] "Use settings.url_preview_url_blacklist instead" )
+
+    # options that are too specific to mention them explicitly in settings
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "account_threepid_delegates" "email" ] "Use settings.account_threepid_delegates.email instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "account_threepid_delegates" "msisdn" ] "Use settings.account_threepid_delegates.msisdn instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "allow_guest_access" ] "Use settings.allow_guest_access instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "bcrypt_rounds" ] "Use settings.bcrypt_rounds instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "enable_registration_captcha" ] "Use settings.enable_registration_captcha instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "event_cache_size" ] "Use settings.event_cache_size instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "federation_rc_concurrent" ] "Use settings.rc_federation.concurrent instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "federation_rc_reject_limit" ] "Use settings.rc_federation.reject_limit instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "federation_rc_sleep_delay" ] "Use settings.rc_federation.sleep_delay instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "federation_rc_sleep_limit" ] "Use settings.rc_federation.sleep_limit instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "federation_rc_window_size" ] "Use settings.rc_federation.window_size instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "key_refresh_interval" ] "Use settings.key_refresh_interval instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "rc_messages_burst_count" ] "Use settings.rc_messages.burst_count instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "rc_messages_per_second" ] "Use settings.rc_messages.per_second instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "recaptcha_private_key" ] "Use settings.recaptcha_private_key instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "recaptcha_public_key" ] "Use settings.recaptcha_public_key instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "redaction_retention_period" ] "Use settings.redaction_retention_period instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "room_prejoin_state" "additional_event_types" ] "Use settings.room_prejoin_state.additional_event_types instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "room_prejoin_state" "disable_default_event_types" ] "Use settings.room_prejoin-state.disable_default_event_types instead" )
+
+    # Options that should be passed via extraConfigFiles, so they are not persisted into the nix store
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "macaroon_secret_key" ] "Pass this value via extraConfigFiles instead" )
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "registration_shared_secret" ] "Pass this value via extraConfigFiles instead" )
+
+  ];
+
+  options = {
+    services.matrix-synapse = {
+      enable = mkEnableOption "matrix.org synapse";
+
+      configFile = mkOption {
+        type = types.str;
+        readOnly = true;
+        description = ''
+          Path to the configuration file on the target system. Useful to configure e.g. workers
+          that also need this.
+        '';
+      };
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.matrix-synapse;
+        defaultText = literalExpression "pkgs.matrix-synapse";
+        description = ''
+          Overridable attribute of the matrix synapse server package to use.
+        '';
+      };
+
+      plugins = mkOption {
+        type = types.listOf types.package;
+        default = [ ];
+        example = literalExpression ''
+          with config.services.matrix-synapse.package.plugins; [
+            matrix-synapse-ldap3
+            matrix-synapse-pam
+          ];
+        '';
+        description = ''
+          List of additional Matrix plugins to make available.
+        '';
+      };
+
+      withJemalloc = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to preload jemalloc to reduce memory fragmentation and overall usage.
+        '';
+      };
+
+      dataDir = mkOption {
+        type = types.str;
+        default = "/var/lib/matrix-synapse";
+        description = ''
+          The directory where matrix-synapse stores its stateful data such as
+          certificates, media and uploads.
+        '';
+      };
+
+      settings = mkOption {
+        default = {};
+        description = ''
+          The primary synapse configuration. See the
+          <link xlink:href="https://github.com/matrix-org/synapse/blob/v${cfg.package.version}/docs/sample_config.yaml">sample configuration</link>
+          for possible values.
+
+          Secrets should be passed in by using the <literal>extraConfigFiles</literal> option.
+        '';
+        type = with types; submodule {
+          freeformType = format.type;
+          options = {
+            # This is a reduced set of popular options and defaults
+            # Do not add every available option here, they can be specified
+            # by the user at their own discretion. This is a freeform type!
+
+            server_name = mkOption {
+              type = types.str;
+              example = "example.com";
+              default = config.networking.hostName;
+              defaultText = literalExpression "config.networking.hostName";
+              description = ''
+                The domain name of the server, with optional explicit port.
+                This is used by remote servers to look up the server address.
+                This is also the last part of your UserID.
+
+                The server_name cannot be changed later so it is important to configure this correctly before you start Synapse.
+              '';
+            };
+
+            enable_registration = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Enable registration for new users.
+              '';
+            };
+
+            registration_shared_secret = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                If set, allows registration by anyone who also has the shared
+                secret, even if registration is otherwise disabled.
+
+                Secrets should be passed in via <literal>extraConfigFiles</literal>!
+              '';
+            };
+
+            macaroon_secret_key = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                Secret key for authentication tokens. If none is specified,
+                the registration_shared_secret is used, if one is given; otherwise,
+                a secret key is derived from the signing key.
+
+                Secrets should be passed in via <literal>extraConfigFiles</literal>!
+              '';
+            };
+
+            enable_metrics = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Enable collection and rendering of performance metrics
+              '';
+            };
+
+            report_stats = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Whether or not to report anonymized homeserver usage statistics.
+              '';
+            };
+
+            signing_key_path = mkOption {
+              type = types.path;
+              default = "${cfg.dataDir}/homeserver.signing.key";
+              description = ''
+                Path to the signing key to sign messages with.
+              '';
+            };
+
+            pid_file = mkOption {
+              type = types.path;
+              default = "/run/matrix-synapse.pid";
+              readOnly = true;
+              description = ''
+                The file to store the PID in.
+              '';
+            };
+
+            log_config = mkOption {
+              type = types.path;
+              default = ./matrix-synapse-log_config.yaml;
+              description = ''
+                The file that holds the logging configuration.
+              '';
+            };
+
+            media_store_path = mkOption {
+              type = types.path;
+              default = if lib.versionAtLeast config.system.stateVersion "22.05"
+                then "${cfg.dataDir}/media_store"
+                else "${cfg.dataDir}/media";
+              description = ''
+                Directory where uploaded images and attachments are stored.
+              '';
+            };
+
+            public_baseurl = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              example = "https://example.com:8448/";
+              description = ''
+                The public-facing base URL for the client API (not including _matrix/...)
+              '';
+            };
+
+            tls_certificate_path = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              example = "/var/lib/acme/example.com/fullchain.pem";
+              description = ''
+                PEM encoded X509 certificate for TLS.
+                You can replace the self-signed certificate that synapse
+                autogenerates on launch with your own SSL certificate + key pair
+                if you like.  Any required intermediary certificates can be
+                appended after the primary certificate in hierarchical order.
+              '';
+            };
+
+            tls_private_key_path = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              example = "/var/lib/acme/example.com/key.pem";
+              description = ''
+                PEM encoded private key for TLS. Specify null if synapse is not
+                speaking TLS directly.
+              '';
+            };
+
+            presence.enabled = mkOption {
+              type = types.bool;
+              default = true;
+              example = false;
+              description = ''
+                Whether to enable presence tracking.
+
+                Presence tracking allows users to see the state (e.g online/offline)
+                of other local and remote users.
+              '';
+            };
+
+            listeners = mkOption {
+              type = types.listOf (types.submodule {
+                options = {
+                  port = mkOption {
+                    type = types.port;
+                    example = 8448;
+                    description = ''
+                      The port to listen for HTTP(S) requests on.
+                    '';
+                  };
+
+                  bind_addresses = mkOption {
+                    type = types.listOf types.str;
+                    default = [
+                      "::1"
+                      "127.0.0.1"
+                    ];
+                    example = literalExpression ''
+                    [
+                      "::"
+                      "0.0.0.0"
+                    ]
+                    '';
+                    description = ''
+                     IP addresses to bind the listener to.
+                    '';
+                  };
+
+                  type = mkOption {
+                    type = types.enum [
+                      "http"
+                      "manhole"
+                      "metrics"
+                      "replication"
+                    ];
+                    default = "http";
+                    example = "metrics";
+                    description = ''
+                      The type of the listener, usually http.
+                    '';
+                  };
+
+                  tls = mkOption {
+                    type = types.bool;
+                    default = true;
+                    example = false;
+                    description = ''
+                      Whether to enable TLS on the listener socket.
+                    '';
+                  };
+
+                  x_forwarded = mkOption {
+                    type = types.bool;
+                    default = false;
+                    example = true;
+                    description = ''
+                      Use the X-Forwarded-For (XFF) header as the client IP and not the
+                      actual client IP.
+                    '';
+                  };
+
+                  resources = mkOption {
+                    type = types.listOf (types.submodule {
+                      options = {
+                        names = mkOption {
+                          type = types.listOf (types.enum [
+                            "client"
+                            "consent"
+                            "federation"
+                            "keys"
+                            "media"
+                            "metrics"
+                            "openid"
+                            "replication"
+                            "static"
+                          ]);
+                          description = ''
+                            List of resources to host on this listener.
+                          '';
+                          example = [
+                            "client"
+                          ];
+                        };
+                        compress = mkOption {
+                          type = types.bool;
+                          description = ''
+                            Should synapse compress HTTP responses to clients that support it?
+                            This should be disabled if running synapse behind a load balancer
+                            that can do automatic compression.
+                          '';
+                        };
+                      };
+                    });
+                    description = ''
+                      List of HTTP resources to serve on this listener.
+                    '';
+                  };
+                };
+              });
+              default = [ {
+                port = 8008;
+                bind_addresses = [ "127.0.0.1" ];
+                type = "http";
+                tls = false;
+                x_forwarded = true;
+                resources = [ {
+                  names = [ "client" ];
+                  compress = true;
+                } {
+                  names = [ "federation" ];
+                  compress = false;
+                } ];
+              } ];
+              description = ''
+                List of ports that Synapse should listen on, their purpose and their configuration.
+              '';
+            };
+
+            database.name = mkOption {
+              type = types.enum [
+                "sqlite3"
+                "psycopg2"
+              ];
+              default = if versionAtLeast config.system.stateVersion "18.03"
+                then "psycopg2"
+                else "sqlite3";
+               defaultText = literalExpression ''
+                if versionAtLeast config.system.stateVersion "18.03"
+                then "psycopg2"
+                else "sqlite3"
+              '';
+              description = ''
+                The database engine name. Can be sqlite3 or psycopg2.
+              '';
+            };
+
+            database.args.database = mkOption {
+              type = types.str;
+              default = {
+                sqlite3 = "${cfg.dataDir}/homeserver.db";
+                psycopg2 = "matrix-synapse";
+              }.${cfg.settings.database.name};
+              defaultText = literalExpression ''
+              {
+                sqlite3 = "''${${options.services.matrix-synapse.dataDir}}/homeserver.db";
+                psycopg2 = "matrix-synapse";
+              }.''${${options.services.matrix-synapse.settings}.database.name};
+              '';
+              description = ''
+                Name of the database when using the psycopg2 backend,
+                path to the database location when using sqlite3.
+              '';
+            };
+
+            database.args.user = mkOption {
+              type = types.nullOr types.str;
+              default = {
+                sqlite3 = null;
+                psycopg2 = "matrix-synapse";
+              }.${cfg.settings.database.name};
+              description = ''
+                Username to connect with psycopg2, set to null
+                when using sqlite3.
+              '';
+            };
+
+            url_preview_enabled = mkOption {
+              type = types.bool;
+              default = true;
+              example = false;
+              description = ''
+                Is the preview URL API enabled?  If enabled, you *must* specify an
+                explicit url_preview_ip_range_blacklist of IPs that the spider is
+                denied from accessing.
+              '';
+            };
+
+            url_preview_ip_range_blacklist = mkOption {
+              type = types.listOf types.str;
+              default = [
+                "10.0.0.0/8"
+                "100.64.0.0/10"
+                "127.0.0.0/8"
+                "169.254.0.0/16"
+                "172.16.0.0/12"
+                "192.0.0.0/24"
+                "192.0.2.0/24"
+                "192.168.0.0/16"
+                "192.88.99.0/24"
+                "198.18.0.0/15"
+                "198.51.100.0/24"
+                "2001:db8::/32"
+                "203.0.113.0/24"
+                "224.0.0.0/4"
+                "::1/128"
+                "fc00::/7"
+                "fe80::/10"
+                "fec0::/10"
+                "ff00::/8"
+              ];
+              description = ''
+                List of IP address CIDR ranges that the URL preview spider is denied
+                from accessing.
+              '';
+            };
+
+            url_preview_ip_range_whitelist = mkOption {
+              type = types.listOf types.str;
+              default = [];
+              description = ''
+                List of IP address CIDR ranges that the URL preview spider is allowed
+                to access even if they are specified in url_preview_ip_range_blacklist.
+              '';
+            };
+
+            url_preview_url_blacklist = mkOption {
+              type = types.listOf types.str;
+              default = [];
+              description = ''
+                Optional list of URL matches that the URL preview spider is
+                denied from accessing.
+              '';
+            };
+
+            max_upload_size = mkOption {
+              type = types.str;
+              default = "50M";
+              example = "100M";
+              description = ''
+                The largest allowed upload size in bytes
+              '';
+            };
+
+            max_image_pixels = mkOption {
+              type = types.str;
+              default = "32M";
+              example = "64M";
+              description = ''
+                Maximum number of pixels that will be thumbnailed
+              '';
+            };
+
+            dynamic_thumbnails = mkOption {
+              type = types.bool;
+              default = false;
+              example = true;
+              description = ''
+                Whether to generate new thumbnails on the fly to precisely match
+                the resolution requested by the client. If true then whenever
+                a new resolution is requested by the client the server will
+                generate a new thumbnail. If false the server will pick a thumbnail
+                from a precalculated list.
+              '';
+            };
+
+            turn_uris = mkOption {
+              type = types.listOf types.str;
+              default = [];
+              example = [
+                "turn:turn.example.com:3487?transport=udp"
+                "turn:turn.example.com:3487?transport=tcp"
+                "turns:turn.example.com:5349?transport=udp"
+                "turns:turn.example.com:5349?transport=tcp"
+              ];
+              description = ''
+                The public URIs of the TURN server to give to clients
+              '';
+            };
+            turn_shared_secret = mkOption {
+              type = types.str;
+              default = "";
+              example = literalExpression ''
+                config.services.coturn.static-auth-secret
+              '';
+              description = ''
+                The shared secret used to compute passwords for the TURN server.
+
+                Secrets should be passed in via <literal>extraConfigFiles</literal>!
+              '';
+            };
+
+            trusted_key_servers = mkOption {
+              type = types.listOf (types.submodule {
+                options = {
+                  server_name = mkOption {
+                    type = types.str;
+                    example = "matrix.org";
+                    description = ''
+                      Hostname of the trusted server.
+                    '';
+                  };
+
+                  verify_keys = mkOption {
+                    type = types.nullOr (types.attrsOf types.str);
+                    default = null;
+                    example = literalExpression ''
+                      {
+                        "ed25519:auto" = "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw";
+                      }
+                    '';
+                    description = ''
+                      Attribute set from key id to base64 encoded public key.
+
+                      If specified synapse will check that the response is signed
+                      by at least one of the given keys.
+                    '';
+                  };
+                };
+              });
+              default = [ {
+                server_name = "matrix.org";
+                verify_keys = {
+                  "ed25519:auto" = "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw";
+                };
+              } ];
+              description = ''
+                The trusted servers to download signing keys from.
+              '';
+            };
+
+            app_service_config_files = mkOption {
+              type = types.listOf types.path;
+              default = [ ];
+              description = ''
+                A list of application service config file to use
+              '';
+            };
+
+          };
+        };
+      };
+
+      extraConfigFiles = mkOption {
+        type = types.listOf types.path;
+        default = [];
+        description = ''
+          Extra config files to include.
+
+          The configuration files will be included based on the command line
+          argument --config-path. This allows to configure secrets without
+          having to go through the Nix store, e.g. based on deployment keys if
+          NixOps is in use.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      { assertion = hasLocalPostgresDB -> config.services.postgresql.enable;
+        message = ''
+          Cannot deploy matrix-synapse with a configuration for a local postgresql database
+            and a missing postgresql service. Since 20.03 it's mandatory to manually configure the
+            database (please read the thread in https://github.com/NixOS/nixpkgs/pull/80447 for
+            further reference).
+
+            If you
+            - try to deploy a fresh synapse, you need to configure the database yourself. An example
+              for this can be found in <nixpkgs/nixos/tests/matrix-synapse.nix>
+            - update your existing matrix-synapse instance, you simply need to add `services.postgresql.enable = true`
+              to your configuration.
+
+          For further information about this update, please read the release-notes of 20.03 carefully.
+        '';
+      }
+    ];
+
+    services.matrix-synapse.configFile = configFile;
+
+    users.users.matrix-synapse = {
+      group = "matrix-synapse";
+      home = cfg.dataDir;
+      createHome = true;
+      shell = "${pkgs.bash}/bin/bash";
+      uid = config.ids.uids.matrix-synapse;
+    };
+
+    users.groups.matrix-synapse = {
+      gid = config.ids.gids.matrix-synapse;
+    };
+
+    systemd.services.matrix-synapse = {
+      description = "Synapse Matrix homeserver";
+      after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
+      wantedBy = [ "multi-user.target" ];
+      preStart = ''
+        ${cfg.package}/bin/synapse_homeserver \
+          --config-path ${configFile} \
+          --keys-directory ${cfg.dataDir} \
+          --generate-keys
+      '';
+      environment = {
+        PYTHONPATH = makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ];
+      } // optionalAttrs (cfg.withJemalloc) {
+        LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
+      };
+      serviceConfig = {
+        Type = "notify";
+        User = "matrix-synapse";
+        Group = "matrix-synapse";
+        WorkingDirectory = cfg.dataDir;
+        ExecStartPre = [ ("+" + (pkgs.writeShellScript "matrix-synapse-fix-permissions" ''
+          chown matrix-synapse:matrix-synapse ${cfg.dataDir}/homeserver.signing.key
+          chmod 0600 ${cfg.dataDir}/homeserver.signing.key
+        '')) ];
+        ExecStart = ''
+          ${cfg.package}/bin/synapse_homeserver \
+            ${ concatMapStringsSep "\n  " (x: "--config-path ${x} \\") ([ configFile ] ++ cfg.extraConfigFiles) }
+            --keys-directory ${cfg.dataDir}
+        '';
+        ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID";
+        Restart = "on-failure";
+        UMask = "0077";
+      };
+    };
+
+    environment.systemPackages = [ registerNewMatrixUser ];
+  };
+
+  meta = {
+    buildDocsInSandbox = false;
+    doc = ./matrix-synapse.xml;
+    maintainers = teams.matrix.members;
+  };
+
+}
diff --git a/nixos/modules/services/misc/matrix-synapse.xml b/nixos/modules/services/matrix/matrix-synapse.xml
index 41a56df0f2b5..cf33957d58ec 100644
--- a/nixos/modules/services/misc/matrix-synapse.xml
+++ b/nixos/modules/services/matrix/matrix-synapse.xml
@@ -115,20 +115,21 @@ in {
   };
   services.matrix-synapse = {
     <link linkend="opt-services.matrix-synapse.enable">enable</link> = true;
-    <link linkend="opt-services.matrix-synapse.server_name">server_name</link> = config.networking.domain;
-    <link linkend="opt-services.matrix-synapse.listeners">listeners</link> = [
+    <link linkend="opt-services.matrix-synapse.settings.server_name">server_name</link> = config.networking.domain;
+    <link linkend="opt-services.matrix-synapse.settings.listeners">listeners</link> = [
       {
-        <link linkend="opt-services.matrix-synapse.listeners._.port">port</link> = 8008;
-        <link linkend="opt-services.matrix-synapse.listeners._.bind_address">bind_address</link> = "::1";
-        <link linkend="opt-services.matrix-synapse.listeners._.type">type</link> = "http";
-        <link linkend="opt-services.matrix-synapse.listeners._.tls">tls</link> = false;
-        <link linkend="opt-services.matrix-synapse.listeners._.x_forwarded">x_forwarded</link> = true;
-        <link linkend="opt-services.matrix-synapse.listeners._.resources">resources</link> = [
-          {
-            <link linkend="opt-services.matrix-synapse.listeners._.resources._.names">names</link> = [ "client" "federation" ];
-            <link linkend="opt-services.matrix-synapse.listeners._.resources._.compress">compress</link> = false;
-          }
-        ];
+        <link linkend="opt-services.matrix-synapse.settings.listeners._.port">port</link> = 8008;
+        <link linkend="opt-services.matrix-synapse.settings.listeners._.bind_addresses">bind_addresses</link> = [ "::1" ];
+        <link linkend="opt-services.matrix-synapse.settings.listeners._.type">type</link> = "http";
+        <link linkend="opt-services.matrix-synapse.settings.listeners._.tls">tls</link> = false;
+        <link linkend="opt-services.matrix-synapse.settings.listeners._.x_forwarded">x_forwarded</link> = true;
+        <link linkend="opt-services.matrix-synapse.settings.listeners._.resources">resources</link> = [ {
+          <link linkend="opt-services.matrix-synapse.settings.listeners._.resources._.names">names</link> = [ "client" ];
+          <link linkend="opt-services.matrix-synapse.settings.listeners._.resources._.compress">compress</link> = true;
+        } {
+          <link linkend="opt-services.matrix-synapse.settings.listeners._.resources._.names">names</link> = [ "federation" ];
+          <link linkend="opt-services.matrix-synapse.settings.listeners._.resources._.compress">compress</link> = false;
+        } ];
       }
     ];
   };
@@ -151,11 +152,11 @@ in {
 
   <para>
    If you want to run a server with public registration by anybody, you can
-   then enable <literal><link linkend="opt-services.matrix-synapse.enable_registration">services.matrix-synapse.enable_registration</link> =
+   then enable <literal><link linkend="opt-services.matrix-synapse.settings.enable_registration">services.matrix-synapse.settings.enable_registration</link> =
    true;</literal>. Otherwise, or you can generate a registration secret with
    <command>pwgen -s 64 1</command> and set it with
-   <option><link linkend="opt-services.matrix-synapse.registration_shared_secret">services.matrix-synapse.registration_shared_secret</link></option>. To
-   create a new user or admin, run the following after you have set the secret
+   <option><link linkend="opt-services.matrix-synapse.settings.registration_shared_secret">services.matrix-synapse.settings.registration_shared_secret</link></option>.
+   To create a new user or admin, run the following after you have set the secret
    and have rebuilt NixOS:
 <screen>
 <prompt>$ </prompt>nix run nixpkgs.matrix-synapse
@@ -170,7 +171,7 @@ Success!
    <literal>@your-username:example.org</literal>. Note that the registration
    secret ends up in the nix store and therefore is world-readable by any user
    on your machine, so it makes sense to only temporarily activate the
-   <link linkend="opt-services.matrix-synapse.registration_shared_secret">registration_shared_secret</link>
+   <link linkend="opt-services.matrix-synapse.settings.registration_shared_secret">registration_shared_secret</link>
    option until a better solution for NixOS is in place.
   </para>
  </section>
diff --git a/nixos/modules/services/matrix/mjolnir.xml b/nixos/modules/services/matrix/mjolnir.xml
index d462ddf7b01b..b07abe339791 100644
--- a/nixos/modules/services/matrix/mjolnir.xml
+++ b/nixos/modules/services/matrix/mjolnir.xml
@@ -98,7 +98,7 @@
   </para>
   <para>
    To use the Antispam Module, add <package>matrix-synapse-plugins.matrix-synapse-mjolnir-antispam</package>
-   to the Synapse plugin list and enable the <literal>mjolnir.AntiSpam</literal> module.
+   to the Synapse plugin list and enable the <literal>mjolnir.Module</literal> module.
   </para>
 <programlisting>
 {
@@ -108,7 +108,7 @@
     ];
     extraConfig = ''
       modules:
-        - module: mjolnir.AntiSpam
+        - module: mjolnir.Module
           config:
             # Prevent servers/users in the ban lists from inviting users on this
             # server to rooms. Default true.
diff --git a/nixos/modules/services/misc/airsonic.nix b/nixos/modules/services/misc/airsonic.nix
index 533a3d367a32..2b9c6d80abbd 100644
--- a/nixos/modules/services/misc/airsonic.nix
+++ b/nixos/modules/services/misc/airsonic.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.airsonic;
+  opt = options.services.airsonic;
 in {
   options = {
 
@@ -38,9 +39,11 @@ in {
         default = "127.0.0.1";
         description = ''
           The host name or IP address on which to bind Airsonic.
-          Only relevant if you have multiple network interfaces and want
-          to make Airsonic available on only one of them. The default value
-          will bind Airsonic to all available network interfaces.
+          The default value is appropriate for first launch, when the
+          default credentials are easy to guess. It is also appropriate
+          if you intend to use the virtualhost option in the service
+          module. In other cases, you may want to change this to a
+          specific IP or 0.0.0.0 to listen on all interfaces.
         '';
       };
 
@@ -78,7 +81,7 @@ in {
         description = ''
           List of paths to transcoder executables that should be accessible
           from Airsonic. Symlinks will be created to each executable inside
-          ${cfg.home}/transcoders.
+          ''${config.${opt.home}}/transcoders.
         '';
       };
 
diff --git a/nixos/modules/services/misc/ananicy.nix b/nixos/modules/services/misc/ananicy.nix
index f76f534fb450..191666bc3625 100644
--- a/nixos/modules/services/misc/ananicy.nix
+++ b/nixos/modules/services/misc/ananicy.nix
@@ -84,7 +84,7 @@ in
       } // (if ((lib.getName cfg.package) == (lib.getName pkgs.ananicy-cpp)) then {
         # https://gitlab.com/ananicy-cpp/ananicy-cpp/-/blob/master/src/config.cpp#L12
         loglevel = mkOD "warn"; # default is info but its spammy
-        cgroup_realtime_workaround = mkOD true;
+        cgroup_realtime_workaround = mkOD config.systemd.enableUnifiedCgroupHierarchy;
       } else {
         # https://github.com/Nefelim4ag/Ananicy/blob/master/ananicy.d/ananicy.conf
         check_disks_schedulers = mkOD true;
diff --git a/nixos/modules/services/misc/autorandr.nix b/nixos/modules/services/misc/autorandr.nix
index 95cee5046e81..a65c5c9d11cf 100644
--- a/nixos/modules/services/misc/autorandr.nix
+++ b/nixos/modules/services/misc/autorandr.nix
@@ -43,6 +43,7 @@ in {
         ExecStart = "${pkgs.autorandr}/bin/autorandr --batch --change --default ${cfg.defaultTarget}";
         Type = "oneshot";
         RemainAfterExit = false;
+        KillMode = "process";
       };
     };
 
diff --git a/nixos/modules/services/misc/bees.nix b/nixos/modules/services/misc/bees.nix
index cb97a86b8592..fa00d7e4f55d 100644
--- a/nixos/modules/services/misc/bees.nix
+++ b/nixos/modules/services/misc/bees.nix
@@ -21,6 +21,8 @@ let
         <para>
         This must be in a format usable by findmnt; that could be a key=value
         pair, or a bare path to a mount point.
+        Using bare paths will allow systemd to start the beesd service only
+        after mounting the associated path.
       '';
       example = "LABEL=MyBulkDataDrive";
     };
@@ -122,6 +124,7 @@ in
             StartupIOWeight = 25;
             SyslogIdentifier = "beesd"; # would otherwise be "bees-service-wrapper"
           };
+        unitConfig.RequiresMountsFor = lib.mkIf (lib.hasPrefix "/" fs.spec) fs.spec;
         wantedBy = [ "multi-user.target" ];
       })
       cfg.filesystems;
diff --git a/nixos/modules/services/misc/couchpotato.nix b/nixos/modules/services/misc/couchpotato.nix
deleted file mode 100644
index f5163cf86cf5..000000000000
--- a/nixos/modules/services/misc/couchpotato.nix
+++ /dev/null
@@ -1,42 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-
-let
-  cfg = config.services.couchpotato;
-
-in
-{
-  options = {
-    services.couchpotato = {
-      enable = mkEnableOption "CouchPotato Server";
-    };
-  };
-
-  config = mkIf cfg.enable {
-    systemd.services.couchpotato = {
-      description = "CouchPotato Server";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-
-      serviceConfig = {
-        Type = "simple";
-        User = "couchpotato";
-        Group = "couchpotato";
-        StateDirectory = "couchpotato";
-        ExecStart = "${pkgs.couchpotato}/bin/couchpotato";
-        Restart = "on-failure";
-      };
-    };
-
-    users.users.couchpotato =
-      { group = "couchpotato";
-        home = "/var/lib/couchpotato/";
-        description = "CouchPotato daemon user";
-        uid = config.ids.uids.couchpotato;
-      };
-
-    users.groups.couchpotato =
-      { gid = config.ids.gids.couchpotato; };
-  };
-}
diff --git a/nixos/modules/services/misc/dendrite.nix b/nixos/modules/services/misc/dendrite.nix
index c967fc3a362a..b2885b094153 100644
--- a/nixos/modules/services/misc/dendrite.nix
+++ b/nixos/modules/services/misc/dendrite.nix
@@ -110,6 +110,15 @@ in
             '';
           };
         };
+        options.app_service_api.database = {
+          connection_string = lib.mkOption {
+            type = lib.types.str;
+            default = "file:federationapi.db";
+            description = ''
+              Database for the Appservice API.
+            '';
+          };
+        };
         options.client_api = {
           registration_disabled = lib.mkOption {
             type = lib.types.bool;
@@ -120,6 +129,91 @@ in
             '';
           };
         };
+        options.federation_api.database = {
+          connection_string = lib.mkOption {
+            type = lib.types.str;
+            default = "file:federationapi.db";
+            description = ''
+              Database for the Federation API.
+            '';
+          };
+        };
+        options.key_server.database = {
+          connection_string = lib.mkOption {
+            type = lib.types.str;
+            default = "file:keyserver.db";
+            description = ''
+              Database for the Key Server (for end-to-end encryption).
+            '';
+          };
+        };
+        options.media_api = {
+          database = {
+            connection_string = lib.mkOption {
+              type = lib.types.str;
+              default = "file:mediaapi.db";
+              description = ''
+                Database for the Media API.
+              '';
+            };
+          };
+          base_path = lib.mkOption {
+            type = lib.types.str;
+            default = "${workingDir}/media_store";
+            description = ''
+              Storage path for uploaded media.
+            '';
+          };
+        };
+        options.room_server.database = {
+          connection_string = lib.mkOption {
+            type = lib.types.str;
+            default = "file:roomserver.db";
+            description = ''
+              Database for the Room Server.
+            '';
+          };
+        };
+        options.sync_api.database = {
+          connection_string = lib.mkOption {
+            type = lib.types.str;
+            default = "file:syncserver.db";
+            description = ''
+              Database for the Sync API.
+            '';
+          };
+        };
+        options.user_api = {
+          account_database = {
+            connection_string = lib.mkOption {
+              type = lib.types.str;
+              default = "file:userapi_accounts.db";
+              description = ''
+                Database for the User API, accounts.
+              '';
+            };
+          };
+          device_database = {
+            connection_string = lib.mkOption {
+              type = lib.types.str;
+              default = "file:userapi_devices.db";
+              description = ''
+                Database for the User API, devices.
+              '';
+            };
+          };
+        };
+        options.mscs = {
+          database = {
+            connection_string = lib.mkOption {
+              type = lib.types.str;
+              default = "file:mscs.db";
+              description = ''
+                Database for exerimental MSC's.
+              '';
+            };
+          };
+        };
       };
       default = { };
       description = ''
diff --git a/nixos/modules/services/misc/etcd.nix b/nixos/modules/services/misc/etcd.nix
index 26ad1ad5536a..3925b7dd1636 100644
--- a/nixos/modules/services/misc/etcd.nix
+++ b/nixos/modules/services/misc/etcd.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.etcd;
+  opt = options.services.etcd;
 
 in {
 
@@ -24,6 +25,7 @@ in {
     advertiseClientUrls = mkOption {
       description = "Etcd list of this member's client URLs to advertise to the rest of the cluster.";
       default = cfg.listenClientUrls;
+      defaultText = literalExpression "config.${opt.listenClientUrls}";
       type = types.listOf types.str;
     };
 
@@ -42,12 +44,14 @@ in {
     initialAdvertisePeerUrls = mkOption {
       description = "Etcd list of this member's peer URLs to advertise to rest of the cluster.";
       default = cfg.listenPeerUrls;
+      defaultText = literalExpression "config.${opt.listenPeerUrls}";
       type = types.listOf types.str;
     };
 
     initialCluster = mkOption {
       description = "Etcd initial cluster configuration for bootstrapping.";
       default = ["${cfg.name}=http://127.0.0.1:2380"];
+      defaultText = literalExpression ''["''${config.${opt.name}}=http://127.0.0.1:2380"]'';
       type = types.listOf types.str;
     };
 
@@ -96,18 +100,21 @@ in {
     peerCertFile = mkOption {
       description = "Cert file to use for peer to peer communication";
       default = cfg.certFile;
+      defaultText = literalExpression "config.${opt.certFile}";
       type = types.nullOr types.path;
     };
 
     peerKeyFile = mkOption {
       description = "Key file to use for peer to peer communication";
       default = cfg.keyFile;
+      defaultText = literalExpression "config.${opt.keyFile}";
       type = types.nullOr types.path;
     };
 
     peerTrustedCaFile = mkOption {
       description = "Certificate authority file to use for peer to peer communication";
       default = cfg.trustedCaFile;
+      defaultText = literalExpression "config.${opt.trustedCaFile}";
       type = types.nullOr types.path;
     };
 
diff --git a/nixos/modules/services/misc/exhibitor.nix b/nixos/modules/services/misc/exhibitor.nix
index 28c98edf47af..4c935efbd844 100644
--- a/nixos/modules/services/misc/exhibitor.nix
+++ b/nixos/modules/services/misc/exhibitor.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.exhibitor;
+  opt = options.services.exhibitor;
   exhibitorConfig = ''
     zookeeper-install-directory=${cfg.baseDir}/zookeeper
     zookeeper-data-directory=${cfg.zkDataDir}
@@ -165,6 +166,7 @@ in
       zkDataDir = mkOption {
         type = types.str;
         default = "${cfg.baseDir}/zkData";
+        defaultText = literalExpression ''"''${config.${opt.baseDir}}/zkData"'';
         description = ''
           The Zookeeper data directory
         '';
@@ -172,6 +174,7 @@ in
       zkLogDir = mkOption {
         type = types.path;
         default = "${cfg.baseDir}/zkLogs";
+        defaultText = literalExpression ''"''${config.${opt.baseDir}}/zkLogs"'';
         description = ''
           The Zookeeper logs directory
         '';
diff --git a/nixos/modules/services/misc/gitea.nix b/nixos/modules/services/misc/gitea.nix
index 022a73c2b596..bc7bb663ee00 100644
--- a/nixos/modules/services/misc/gitea.nix
+++ b/nixos/modules/services/misc/gitea.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.gitea;
+  opt = options.services.gitea;
   gitea = cfg.package;
   pg = config.services.postgresql;
   useMysql = cfg.database.type == "mysql";
@@ -51,6 +52,7 @@ in
       log = {
         rootPath = mkOption {
           default = "${cfg.stateDir}/log";
+          defaultText = literalExpression ''"''${config.${opt.stateDir}}/log"'';
           type = types.str;
           description = "Root path for log files.";
         };
@@ -84,6 +86,11 @@ in
         port = mkOption {
           type = types.port;
           default = (if !usePostgresql then 3306 else pg.port);
+          defaultText = literalExpression ''
+            if config.${opt.database.type} != "postgresql"
+            then 3306
+            else config.${options.services.postgresql.port}
+          '';
           description = "Database host port.";
         };
 
@@ -130,6 +137,7 @@ in
         path = mkOption {
           type = types.str;
           default = "${cfg.stateDir}/data/gitea.db";
+          defaultText = literalExpression ''"''${config.${opt.stateDir}}/data/gitea.db"'';
           description = "Path to the sqlite3 database file.";
         };
 
@@ -166,8 +174,22 @@ in
         backupDir = mkOption {
           type = types.str;
           default = "${cfg.stateDir}/dump";
+          defaultText = literalExpression ''"''${config.${opt.stateDir}}/dump"'';
           description = "Path to the dump files.";
         };
+
+        type = mkOption {
+          type = types.enum [ "zip" "rar" "tar" "sz" "tar.gz" "tar.xz" "tar.bz2" "tar.br" "tar.lz4" ];
+          default = "zip";
+          description = "Archive format used to store the dump file.";
+        };
+
+        file = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          description = "Filename to be used for the dump. If `null` a default name is choosen by gitea.";
+          example = "gitea-dump";
+        };
       };
 
       ssh = {
@@ -199,6 +221,7 @@ in
         contentDir = mkOption {
           type = types.str;
           default = "${cfg.stateDir}/data/lfs";
+          defaultText = literalExpression ''"''${config.${opt.stateDir}}/data/lfs"'';
           description = "Where to store LFS files.";
         };
       };
@@ -212,6 +235,7 @@ in
       repositoryRoot = mkOption {
         type = types.str;
         default = "${cfg.stateDir}/repositories";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/repositories"'';
         description = "Path to the git repositories.";
       };
 
@@ -623,7 +647,7 @@ in
        serviceConfig = {
          Type = "oneshot";
          User = cfg.user;
-         ExecStart = "${gitea}/bin/gitea dump";
+         ExecStart = "${gitea}/bin/gitea dump --type ${cfg.dump.type}" + optionalString (cfg.dump.file != null) " --file ${cfg.dump.file}";
          WorkingDirectory = cfg.dump.backupDir;
        };
     };
diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix
index 01a7ea42d9db..e48444f71612 100644
--- a/nixos/modules/services/misc/gitlab.nix
+++ b/nixos/modules/services/misc/gitlab.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, utils, ... }:
+{ config, lib, options, pkgs, utils, ... }:
 
 with lib;
 
 let
   cfg = config.services.gitlab;
+  opt = options.services.gitlab;
 
   ruby = cfg.packages.gitlab.ruby;
 
@@ -71,7 +72,7 @@ let
     redis = {
       bin = "${pkgs.redis}/bin/redis-cli";
       host = "127.0.0.1";
-      port = 6379;
+      port = config.services.redis.servers.gitlab.port;
       database = 0;
       namespace = "resque:gitlab";
     };
@@ -309,6 +310,7 @@ in {
       backup.path = mkOption {
         type = types.str;
         default = cfg.statePath + "/backup";
+        defaultText = literalExpression ''config.${opt.statePath} + "/backup"'';
         description = "GitLab path for backups.";
       };
 
@@ -448,7 +450,8 @@ in {
 
       redisUrl = mkOption {
         type = types.str;
-        default = "redis://localhost:6379/";
+        default = "redis://localhost:${toString config.services.redis.servers.gitlab.port}/";
+        defaultText = literalExpression ''redis://localhost:''${toString config.services.redis.servers.gitlab.port}/'';
         description = "Redis URL for all GitLab services except gitlab-shell";
       };
 
@@ -554,6 +557,7 @@ in {
         defaultForProjects = mkOption {
           type = types.bool;
           default = cfg.registry.enable;
+          defaultText = literalExpression "config.${opt.registry.enable}";
           description = "If GitLab container registry should be enabled by default for projects.";
         };
         issuer = mkOption {
@@ -958,7 +962,11 @@ in {
     };
 
     # Redis is required for the sidekiq queue runner.
-    services.redis.enable = mkDefault true;
+    services.redis.servers.gitlab = {
+      enable = mkDefault true;
+      port = mkDefault 31636;
+      bind = mkDefault "127.0.0.1";
+    };
 
     # We use postgres as the main data store.
     services.postgresql = optionalAttrs databaseActuallyCreateLocally {
@@ -1096,7 +1104,9 @@ in {
       "d ${gitlabConfig.production.shared.path} 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/artifacts 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/lfs-objects 0750 ${cfg.user} ${cfg.group} -"
+      "d ${gitlabConfig.production.shared.path}/packages 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/pages 0750 ${cfg.user} ${cfg.group} -"
+      "d ${gitlabConfig.production.shared.path}/terraform_state 0750 ${cfg.user} ${cfg.group} -"
       "L+ /run/gitlab/config - - - - ${cfg.statePath}/config"
       "L+ /run/gitlab/log - - - - ${cfg.statePath}/log"
       "L+ /run/gitlab/tmp - - - - ${cfg.statePath}/tmp"
@@ -1126,8 +1136,8 @@ in {
 
         ExecStartPre = let
           preStartFullPrivileges = ''
-            shopt -s dotglob nullglob
-            set -eu
+            set -o errexit -o pipefail -o nounset
+            shopt -s dotglob nullglob inherit_errexit
 
             chown --no-dereference '${cfg.user}':'${cfg.group}' '${cfg.statePath}'/*
             if [[ -n "$(ls -A '${cfg.statePath}'/config/)" ]]; then
@@ -1137,7 +1147,8 @@ in {
         in "+${pkgs.writeShellScript "gitlab-pre-start-full-privileges" preStartFullPrivileges}";
 
         ExecStart = pkgs.writeShellScript "gitlab-config" ''
-          set -eu
+          set -o errexit -o pipefail -o nounset
+          shopt -s inherit_errexit
 
           umask u=rwx,g=rx,o=
 
@@ -1166,7 +1177,8 @@ in {
             rm -f '${cfg.statePath}/config/database.yml'
 
             ${if cfg.databasePasswordFile != null then ''
-                export db_password="$(<'${cfg.databasePasswordFile}')"
+                db_password="$(<'${cfg.databasePasswordFile}')"
+                export db_password
 
                 if [[ -z "$db_password" ]]; then
                   >&2 echo "Database password was an empty string!"
@@ -1190,10 +1202,11 @@ in {
 
             rm -f '${cfg.statePath}/config/secrets.yml'
 
-            export secret="$(<'${cfg.secrets.secretFile}')"
-            export db="$(<'${cfg.secrets.dbFile}')"
-            export otp="$(<'${cfg.secrets.otpFile}')"
-            export jws="$(<'${cfg.secrets.jwsFile}')"
+            secret="$(<'${cfg.secrets.secretFile}')"
+            db="$(<'${cfg.secrets.dbFile}')"
+            otp="$(<'${cfg.secrets.otpFile}')"
+            jws="$(<'${cfg.secrets.jwsFile}')"
+            export secret db otp jws
             jq -n '{production: {secret_key_base: $ENV.secret,
                     otp_key_base: $ENV.otp,
                     db_key_base: $ENV.db,
@@ -1227,7 +1240,8 @@ in {
         RemainAfterExit = true;
 
         ExecStart = pkgs.writeShellScript "gitlab-db-config" ''
-          set -eu
+          set -o errexit -o pipefail -o nounset
+          shopt -s inherit_errexit
           umask u=rwx,g=rx,o=
 
           initial_root_password="$(<'${cfg.initialRootPasswordFile}')"
@@ -1240,13 +1254,13 @@ in {
     systemd.services.gitlab-sidekiq = {
       after = [
         "network.target"
-        "redis.service"
+        "redis-gitlab.service"
         "postgresql.service"
         "gitlab-config.service"
         "gitlab-db-config.service"
       ];
       bindsTo = [
-        "redis.service"
+        "redis-gitlab.service"
         "gitlab-config.service"
         "gitlab-db-config.service"
       ] ++ optional (cfg.databaseHost == "") "postgresql.service";
@@ -1361,7 +1375,7 @@ in {
 
     systemd.services.gitlab-mailroom = mkIf (gitlabConfig.production.incoming_email.enabled or false) {
       description = "GitLab incoming mail daemon";
-      after = [ "network.target" "redis.service" "gitlab-config.service" ];
+      after = [ "network.target" "redis-gitlab.service" "gitlab-config.service" ];
       bindsTo = [ "gitlab-config.service" ];
       wantedBy = [ "gitlab.target" ];
       partOf = [ "gitlab.target" ];
@@ -1382,12 +1396,12 @@ in {
       after = [
         "gitlab-workhorse.service"
         "network.target"
-        "redis.service"
+        "redis-gitlab.service"
         "gitlab-config.service"
         "gitlab-db-config.service"
       ];
       bindsTo = [
-        "redis.service"
+        "redis-gitlab.service"
         "gitlab-config.service"
         "gitlab-db-config.service"
       ] ++ optional (cfg.databaseHost == "") "postgresql.service";
diff --git a/nixos/modules/services/misc/gitweb.nix b/nixos/modules/services/misc/gitweb.nix
index 13396bf2eb02..a1180716e36b 100644
--- a/nixos/modules/services/misc/gitweb.nix
+++ b/nixos/modules/services/misc/gitweb.nix
@@ -47,6 +47,7 @@ in
         $highlight_bin = "${pkgs.highlight}/bin/highlight";
         ${cfg.extraConfig}
       '';
+      defaultText = literalDocBook "generated config file";
       type = types.path;
       readOnly = true;
       internal = true;
diff --git a/nixos/modules/services/misc/gogs.nix b/nixos/modules/services/misc/gogs.nix
index d7233f10c7cb..c7ae4f494071 100644
--- a/nixos/modules/services/misc/gogs.nix
+++ b/nixos/modules/services/misc/gogs.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.gogs;
+  opt = options.services.gogs;
   configFile = pkgs.writeText "app.ini" ''
     APP_NAME = ${cfg.appName}
     RUN_USER = ${cfg.user}
@@ -129,6 +130,7 @@ in
         path = mkOption {
           type = types.str;
           default = "${cfg.stateDir}/data/gogs.db";
+          defaultText = literalExpression ''"''${config.${opt.stateDir}}/data/gogs.db"'';
           description = "Path to the sqlite3 database file.";
         };
       };
@@ -142,6 +144,7 @@ in
       repositoryRoot = mkOption {
         type = types.str;
         default = "${cfg.stateDir}/repositories";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/repositories"'';
         description = "Path to the git repositories.";
       };
 
diff --git a/nixos/modules/services/misc/headphones.nix b/nixos/modules/services/misc/headphones.nix
index 3ee0a4458bd0..31bd61cb4c20 100644
--- a/nixos/modules/services/misc/headphones.nix
+++ b/nixos/modules/services/misc/headphones.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
@@ -7,6 +7,7 @@ let
   name = "headphones";
 
   cfg = config.services.headphones;
+  opt = options.services.headphones;
 
 in
 
@@ -29,6 +30,7 @@ in
       configFile = mkOption {
         type = types.path;
         default = "${cfg.dataDir}/config.ini";
+        defaultText = literalExpression ''"''${config.${opt.dataDir}}/config.ini"'';
         description = "Path to config file.";
       };
       host = mkOption {
diff --git a/nixos/modules/services/misc/heisenbridge.nix b/nixos/modules/services/misc/heisenbridge.nix
new file mode 100644
index 000000000000..7ce8a23d9af1
--- /dev/null
+++ b/nixos/modules/services/misc/heisenbridge.nix
@@ -0,0 +1,222 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.heisenbridge;
+
+  pkg = config.services.heisenbridge.package;
+  bin = "${pkg}/bin/heisenbridge";
+
+  jsonType = (pkgs.formats.json { }).type;
+
+  registrationFile = "/var/lib/heisenbridge/registration.yml";
+  # JSON is a proper subset of YAML
+  bridgeConfig = builtins.toFile "heisenbridge-registration.yml" (builtins.toJSON {
+    id = "heisenbridge";
+    url = cfg.registrationUrl;
+    # Don't specify as_token and hs_token
+    rate_limited = false;
+    sender_localpart = "heisenbridge";
+    namespaces = cfg.namespaces;
+  });
+in
+{
+  options.services.heisenbridge = {
+    enable = mkEnableOption "the Matrix to IRC bridge";
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.heisenbridge;
+      defaultText = "pkgs.heisenbridge";
+      example = "pkgs.heisenbridge.override { … = …; }";
+      description = ''
+        Package of the application to run, exposed for overriding purposes.
+      '';
+    };
+
+    homeserver = mkOption {
+      type = types.str;
+      description = "The URL to the home server for client-server API calls";
+      example = "http://localhost:8008";
+    };
+
+    registrationUrl = mkOption {
+      type = types.str;
+      description = ''
+        The URL where the application service is listening for HS requests, from the Matrix HS perspective.#
+        The default value assumes the bridge runs on the same host as the home server, in the same network.
+      '';
+      example = "https://matrix.example.org";
+      default = "http://${cfg.address}:${toString cfg.port}";
+      defaultText = "http://$${cfg.address}:$${toString cfg.port}";
+    };
+
+    address = mkOption {
+      type = types.str;
+      description = "Address to listen on. IPv6 does not seem to be supported.";
+      default = "127.0.0.1";
+      example = "0.0.0.0";
+    };
+
+    port = mkOption {
+      type = types.port;
+      description = "The port to listen on";
+      default = 9898;
+    };
+
+    debug = mkOption {
+      type = types.bool;
+      description = "More verbose logging. Recommended during initial setup.";
+      default = false;
+    };
+
+    owner = mkOption {
+      type = types.nullOr types.str;
+      description = ''
+        Set owner MXID otherwise first talking local user will claim the bridge
+      '';
+      default = null;
+      example = "@admin:example.org";
+    };
+
+    namespaces = mkOption {
+      description = "Configure the 'namespaces' section of the registration.yml for the bridge and the server";
+      # TODO link to Matrix documentation of the format
+      type = types.submodule {
+        freeformType = jsonType;
+      };
+
+      default = {
+        users = [
+          {
+            regex = "@irc_.*";
+            exclusive = true;
+          }
+        ];
+        aliases = [ ];
+        rooms = [ ];
+      };
+    };
+
+    identd.enable = mkEnableOption "identd service support";
+    identd.port = mkOption {
+      type = types.port;
+      description = "identd listen port";
+      default = 113;
+    };
+
+    extraArgs = mkOption {
+      type = types.listOf types.str;
+      description = "Heisenbridge is configured over the command line. Append extra arguments here";
+      default = [ ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.heisenbridge = {
+      description = "Matrix<->IRC bridge";
+      before = [ "matrix-synapse.service" ]; # So the registration file can be used by Synapse
+      wantedBy = [ "multi-user.target" ];
+
+      preStart = ''
+        umask 077
+        set -e -u -o pipefail
+
+        if ! [ -f "${registrationFile}" ]; then
+          # Generate registration file if not present (actually, we only care about the tokens in it)
+          ${bin} --generate --config ${registrationFile}
+        fi
+
+        # Overwrite the registration file with our generated one (the config may have changed since then),
+        # but keep the tokens. Two step procedure to be failure safe
+        ${pkgs.yq}/bin/yq --slurp \
+          '.[0] + (.[1] | {as_token, hs_token})' \
+          ${bridgeConfig} \
+          ${registrationFile} \
+          > ${registrationFile}.new
+        mv -f ${registrationFile}.new ${registrationFile}
+
+        # Grant Synapse access to the registration
+        if ${getBin pkgs.glibc}/bin/getent group matrix-synapse > /dev/null; then
+          chgrp -v matrix-synapse ${registrationFile}
+          chmod -v g+r ${registrationFile}
+        fi
+      '';
+
+      serviceConfig = rec {
+        Type = "simple";
+        ExecStart = lib.concatStringsSep " " (
+          [
+            bin
+            (if cfg.debug then "-vvv" else "-v")
+            "--config"
+            registrationFile
+            "--listen-address"
+            (lib.escapeShellArg cfg.address)
+            "--listen-port"
+            (toString cfg.port)
+          ]
+          ++ (lib.optionals (cfg.owner != null) [
+            "--owner"
+            (lib.escapeShellArg cfg.owner)
+          ])
+          ++ (lib.optionals cfg.identd.enable [
+            "--identd"
+            "--identd-port"
+            (toString cfg.identd.port)
+          ])
+          ++ [
+            (lib.escapeShellArg cfg.homeserver)
+          ]
+          ++ (map (lib.escapeShellArg) cfg.extraArgs)
+        );
+
+        # Hardening options
+
+        User = "heisenbridge";
+        Group = "heisenbridge";
+        RuntimeDirectory = "heisenbridge";
+        RuntimeDirectoryMode = "0700";
+        StateDirectory = "heisenbridge";
+        StateDirectoryMode = "0755";
+
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        RestrictSUIDSGID = true;
+        PrivateMounts = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectHostname = true;
+        ProtectClock = true;
+        ProtectProc = "invisible";
+        ProcSubset = "pid";
+        RestrictNamespaces = true;
+        RemoveIPC = true;
+        UMask = "0077";
+
+        CapabilityBoundingSet = [ "CAP_CHOWN" ] ++ optional (cfg.port < 1024 || (cfg.identd.enable && cfg.identd.port < 1024)) "CAP_NET_BIND_SERVICE";
+        AmbientCapabilities = CapabilityBoundingSet;
+        NoNewPrivileges = true;
+        LockPersonality = true;
+        RestrictRealtime = true;
+        SystemCallFilter = ["@system-service" "~@priviledged" "@chown"];
+        SystemCallArchitectures = "native";
+        RestrictAddressFamilies = "AF_INET AF_INET6";
+      };
+    };
+
+    users.groups.heisenbridge = {};
+    users.users.heisenbridge = {
+      description = "Service user for the Heisenbridge";
+      group = "heisenbridge";
+      isSystemUser = true;
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.piegames ];
+}
diff --git a/nixos/modules/services/misc/input-remapper.nix b/nixos/modules/services/misc/input-remapper.nix
new file mode 100644
index 000000000000..f5fb2bf53086
--- /dev/null
+++ b/nixos/modules/services/misc/input-remapper.nix
@@ -0,0 +1,30 @@
+{ pkgs, lib, config, ... }:
+
+with lib;
+
+let cfg = config.services.input-remapper; in
+{
+  options = {
+    services.input-remapper = {
+      enable = mkEnableOption "input-remapper, an easy to use tool to change the mapping of your input device buttons.";
+      package = options.mkPackageOption pkgs "input-remapper" { };
+      enableUdevRules = mkEnableOption "udev rules added by input-remapper to handle hotplugged devices. Currently disabled by default due to https://github.com/sezanzeb/input-remapper/issues/140";
+      serviceWantedBy = mkOption {
+        default = [ "graphical.target" ];
+        example = [ "multi-user.target" ];
+        type = types.listOf types.str;
+        description = "Specifies the WantedBy setting for the input-remapper service.";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.udev.packages = mkIf cfg.enableUdevRules [ cfg.package ];
+    services.dbus.packages = [ cfg.package ];
+    systemd.packages = [ cfg.package ];
+    environment.systemPackages = [ cfg.package ];
+    systemd.services.input-remapper.wantedBy = cfg.serviceWantedBy;
+  };
+
+  meta.maintainers = with lib.maintainers; [ LunNova ];
+}
diff --git a/nixos/modules/services/misc/jellyfin.nix b/nixos/modules/services/misc/jellyfin.nix
index b9d54f27edc2..04cf82f8a46b 100644
--- a/nixos/modules/services/misc/jellyfin.nix
+++ b/nixos/modules/services/misc/jellyfin.nix
@@ -70,10 +70,12 @@ in
         LockPersonality = true;
 
         PrivateTmp = true;
-        PrivateDevices = true;
+        # Disabled to allow Jellyfin to access hw accel devices endpoints
+        # PrivateDevices = true;
         PrivateUsers = true;
 
-        ProtectClock = true;
+        # Disabled as it does not allow Jellyfin to interface with CUDA devices
+        # ProtectClock = true;
         ProtectControlGroups = true;
         ProtectHostname = true;
         ProtectKernelLogs = true;
@@ -84,7 +86,7 @@ in
 
         RestrictNamespaces = true;
         # AF_NETLINK needed because Jellyfin monitors the network connection
-        RestrictAddressFamilies = [ "AF_NETLINK" "AF_INET" "AF_INET6" ];
+        RestrictAddressFamilies = [ "AF_NETLINK" "AF_INET" "AF_INET6" "AF_UNIX" ];
         RestrictRealtime = true;
         RestrictSUIDSGID = true;
 
diff --git a/nixos/modules/services/misc/matrix-appservice-discord.nix b/nixos/modules/services/misc/matrix-appservice-discord.nix
index 947471e56b46..8a8c7f41e3cb 100644
--- a/nixos/modules/services/misc/matrix-appservice-discord.nix
+++ b/nixos/modules/services/misc/matrix-appservice-discord.nix
@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 
 with lib;
 
@@ -7,6 +7,7 @@ let
   registrationFile = "${dataDir}/discord-registration.yaml";
   appDir = "${pkgs.matrix-appservice-discord}/${pkgs.matrix-appservice-discord.passthru.nodeAppDir}";
   cfg = config.services.matrix-appservice-discord;
+  opt = options.services.matrix-appservice-discord;
   # TODO: switch to configGen.json once RFC42 is implemented
   settingsFile = pkgs.writeText "matrix-appservice-discord-settings.json" (builtins.toJSON cfg.settings);
 
@@ -74,6 +75,7 @@ in {
       url = mkOption {
         type = types.str;
         default = "http://localhost:${toString cfg.port}";
+        defaultText = literalExpression ''"http://localhost:''${toString config.${opt.port}}"'';
         description = ''
           The URL where the application service is listening for HS requests.
         '';
diff --git a/nixos/modules/services/misc/matrix-appservice-irc.nix b/nixos/modules/services/misc/matrix-appservice-irc.nix
index 02627e51c932..b041c9c82c56 100644
--- a/nixos/modules/services/misc/matrix-appservice-irc.nix
+++ b/nixos/modules/services/misc/matrix-appservice-irc.nix
@@ -226,4 +226,7 @@ in {
       isSystemUser = true;
     };
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/misc/matrix-conduit.nix b/nixos/modules/services/misc/matrix-conduit.nix
new file mode 100644
index 000000000000..108f64de7aa9
--- /dev/null
+++ b/nixos/modules/services/misc/matrix-conduit.nix
@@ -0,0 +1,149 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.matrix-conduit;
+
+  format = pkgs.formats.toml {};
+  configFile = format.generate "conduit.toml" cfg.settings;
+in
+  {
+    meta.maintainers = with maintainers; [ pstn piegames ];
+    options.services.matrix-conduit = {
+      enable = mkEnableOption "matrix-conduit";
+
+      extraEnvironment = mkOption {
+        type = types.attrsOf types.str;
+        description = "Extra Environment variables to pass to the conduit server.";
+        default = {};
+        example = { RUST_BACKTRACE="yes"; };
+      };
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.matrix-conduit;
+        defaultText = "pkgs.matrix-conduit";
+        example = "pkgs.matrix-conduit";
+        description = ''
+          Package of the conduit matrix server to use.
+        '';
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = format.type;
+          options = {
+            global.server_name = mkOption {
+              type = types.str;
+              example = "example.com";
+              description = "The server_name is the name of this server. It is used as a suffix for user # and room ids.";
+            };
+            global.port = mkOption {
+              type = types.port;
+              default = 6167;
+              description = "The port Conduit will be running on. You need to set up a reverse proxy in your web server (e.g. apache or nginx), so all requests to /_matrix on port 443 and 8448 will be forwarded to the Conduit instance running on this port";
+            };
+            global.max_request_size = mkOption {
+              type = types.ints.positive;
+              default = 20000000;
+              description = "Max request size in bytes. Don't forget to also change it in the proxy.";
+            };
+            global.allow_registration = mkOption {
+              type = types.bool;
+              default = false;
+              description = "Whether new users can register on this server.";
+            };
+            global.allow_encryption = mkOption {
+              type = types.bool;
+              default = true;
+              description = "Whether new encrypted rooms can be created. Note: existing rooms will continue to work.";
+            };
+            global.allow_federation = mkOption {
+              type = types.bool;
+              default = true;
+              description = ''
+                Whether this server federates with other servers.
+              '';
+            };
+            global.trusted_servers = mkOption {
+              type = types.listOf types.str;
+              default = [ "matrix.org" ];
+              description = "Servers trusted with signing server keys.";
+            };
+            global.address = mkOption {
+              type = types.str;
+              default = "::1";
+              description = "Address to listen on for connections by the reverse proxy/tls terminator.";
+            };
+            global.database_path = mkOption {
+              type = types.str;
+              default = "/var/lib/matrix-conduit/";
+              readOnly = true;
+              description = ''
+                Path to the conduit database, the directory where conduit will save its data.
+                Note that due to using the DynamicUser feature of systemd, this value should not be changed
+                and is set to be read only.
+              '';
+            };
+            global.database_backend = mkOption {
+              type = types.enum [ "sqlite" "rocksdb" ];
+              default = "sqlite";
+              example = "rocksdb";
+              description = ''
+                The database backend for the service. Switching it on an existing
+                instance will require manual migration of data.
+              '';
+            };
+          };
+        };
+        default = {};
+        description = ''
+            Generates the conduit.toml configuration file. Refer to
+            <link xlink:href="https://gitlab.com/famedly/conduit/-/blob/master/conduit-example.toml"/>
+            for details on supported values.
+            Note that database_path can not be edited because the service's reliance on systemd StateDir.
+        '';
+      };
+    };
+
+    config = mkIf cfg.enable {
+      systemd.services.conduit = {
+        description = "Conduit Matrix Server";
+        documentation = [ "https://gitlab.com/famedly/conduit/" ];
+        wantedBy = [ "multi-user.target" ];
+        environment = lib.mkMerge ([
+          { CONDUIT_CONFIG = configFile; }
+          cfg.extraEnvironment
+        ]);
+        serviceConfig = {
+          DynamicUser = true;
+          User = "conduit";
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          PrivateDevices = true;
+          PrivateMounts = true;
+          PrivateUsers = true;
+          RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          SystemCallArchitectures = "native";
+          SystemCallFilter = [
+            "@system-service"
+            "~@privileged"
+          ];
+          StateDirectory = "matrix-conduit";
+          ExecStart = "${cfg.package}/bin/conduit";
+          Restart = "on-failure";
+          RestartSec = 10;
+          StartLimitBurst = 5;
+        };
+      };
+    };
+  }
diff --git a/nixos/modules/services/misc/matrix-synapse.nix b/nixos/modules/services/misc/matrix-synapse.nix
deleted file mode 100644
index 0f96f6b1ee22..000000000000
--- a/nixos/modules/services/misc/matrix-synapse.nix
+++ /dev/null
@@ -1,794 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  cfg = config.services.matrix-synapse;
-  pg = config.services.postgresql;
-  usePostgresql = cfg.database_type == "psycopg2";
-  logConfigFile = pkgs.writeText "log_config.yaml" cfg.logConfig;
-  mkResource = r: ''{names: ${builtins.toJSON r.names}, compress: ${boolToString r.compress}}'';
-  mkListener = l: ''{port: ${toString l.port}, bind_address: "${l.bind_address}", type: ${l.type}, tls: ${boolToString l.tls}, x_forwarded: ${boolToString l.x_forwarded}, resources: [${concatStringsSep "," (map mkResource l.resources)}]}'';
-  pluginsEnv = cfg.package.python.buildEnv.override {
-    extraLibs = cfg.plugins;
-  };
-  configFile = pkgs.writeText "homeserver.yaml" ''
-${optionalString (cfg.tls_certificate_path != null) ''
-tls_certificate_path: "${cfg.tls_certificate_path}"
-''}
-${optionalString (cfg.tls_private_key_path != null) ''
-tls_private_key_path: "${cfg.tls_private_key_path}"
-''}
-${optionalString (cfg.tls_dh_params_path != null) ''
-tls_dh_params_path: "${cfg.tls_dh_params_path}"
-''}
-no_tls: ${boolToString cfg.no_tls}
-${optionalString (cfg.bind_port != null) ''
-bind_port: ${toString cfg.bind_port}
-''}
-${optionalString (cfg.unsecure_port != null) ''
-unsecure_port: ${toString cfg.unsecure_port}
-''}
-${optionalString (cfg.bind_host != null) ''
-bind_host: "${cfg.bind_host}"
-''}
-server_name: "${cfg.server_name}"
-pid_file: "/run/matrix-synapse.pid"
-${optionalString (cfg.public_baseurl != null) ''
-public_baseurl: "${cfg.public_baseurl}"
-''}
-listeners: [${concatStringsSep "," (map mkListener cfg.listeners)}]
-database: {
-  name: "${cfg.database_type}",
-  args: {
-    ${concatStringsSep ",\n    " (
-      mapAttrsToList (n: v: "\"${n}\": ${builtins.toJSON v}") cfg.database_args
-    )}
-  }
-}
-event_cache_size: "${cfg.event_cache_size}"
-verbose: ${cfg.verbose}
-log_config: "${logConfigFile}"
-rc_messages_per_second: ${cfg.rc_messages_per_second}
-rc_message_burst_count: ${cfg.rc_message_burst_count}
-federation_rc_window_size: ${cfg.federation_rc_window_size}
-federation_rc_sleep_limit: ${cfg.federation_rc_sleep_limit}
-federation_rc_sleep_delay: ${cfg.federation_rc_sleep_delay}
-federation_rc_reject_limit: ${cfg.federation_rc_reject_limit}
-federation_rc_concurrent: ${cfg.federation_rc_concurrent}
-media_store_path: "${cfg.dataDir}/media"
-uploads_path: "${cfg.dataDir}/uploads"
-max_upload_size: "${cfg.max_upload_size}"
-max_image_pixels: "${cfg.max_image_pixels}"
-dynamic_thumbnails: ${boolToString cfg.dynamic_thumbnails}
-url_preview_enabled: ${boolToString cfg.url_preview_enabled}
-${optionalString (cfg.url_preview_enabled == true) ''
-url_preview_ip_range_blacklist: ${builtins.toJSON cfg.url_preview_ip_range_blacklist}
-url_preview_ip_range_whitelist: ${builtins.toJSON cfg.url_preview_ip_range_whitelist}
-url_preview_url_blacklist: ${builtins.toJSON cfg.url_preview_url_blacklist}
-''}
-recaptcha_private_key: "${cfg.recaptcha_private_key}"
-recaptcha_public_key: "${cfg.recaptcha_public_key}"
-enable_registration_captcha: ${boolToString cfg.enable_registration_captcha}
-turn_uris: ${builtins.toJSON cfg.turn_uris}
-turn_shared_secret: "${cfg.turn_shared_secret}"
-enable_registration: ${boolToString cfg.enable_registration}
-${optionalString (cfg.registration_shared_secret != null) ''
-registration_shared_secret: "${cfg.registration_shared_secret}"
-''}
-recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
-turn_user_lifetime: "${cfg.turn_user_lifetime}"
-user_creation_max_duration: ${cfg.user_creation_max_duration}
-bcrypt_rounds: ${cfg.bcrypt_rounds}
-allow_guest_access: ${boolToString cfg.allow_guest_access}
-
-account_threepid_delegates:
-  ${optionalString (cfg.account_threepid_delegates.email != null) "email: ${cfg.account_threepid_delegates.email}"}
-  ${optionalString (cfg.account_threepid_delegates.msisdn != null) "msisdn: ${cfg.account_threepid_delegates.msisdn}"}
-
-room_prejoin_state:
-  disable_default_event_types: ${boolToString cfg.room_prejoin_state.disable_default_event_types}
-  additional_event_types: ${builtins.toJSON cfg.room_prejoin_state.additional_event_types}
-${optionalString (cfg.macaroon_secret_key != null) ''
-  macaroon_secret_key: "${cfg.macaroon_secret_key}"
-''}
-expire_access_token: ${boolToString cfg.expire_access_token}
-enable_metrics: ${boolToString cfg.enable_metrics}
-report_stats: ${boolToString cfg.report_stats}
-signing_key_path: "${cfg.dataDir}/homeserver.signing.key"
-key_refresh_interval: "${cfg.key_refresh_interval}"
-perspectives:
-  servers: {
-    ${concatStringsSep "},\n" (mapAttrsToList (n: v: ''
-    "${n}": {
-      "verify_keys": {
-        ${concatStringsSep "},\n" (mapAttrsToList (n: v: ''
-        "${n}": {
-          "key": "${v}"
-        }'') v)}
-      }
-    '') cfg.servers)}
-    }
-  }
-redaction_retention_period: ${toString cfg.redaction_retention_period}
-app_service_config_files: ${builtins.toJSON cfg.app_service_config_files}
-
-${cfg.extraConfig}
-'';
-
-  hasLocalPostgresDB = let args = cfg.database_args; in
-    usePostgresql && (!(args ? host) || (elem args.host [ "localhost" "127.0.0.1" "::1" ]));
-in {
-  options = {
-    services.matrix-synapse = {
-      enable = mkEnableOption "matrix.org synapse";
-      configFile = mkOption {
-        type = types.str;
-        readOnly = true;
-        description = ''
-          Path to the configuration file on the target system. Useful to configure e.g. workers
-          that also need this.
-        '';
-      };
-      package = mkOption {
-        type = types.package;
-        default = pkgs.matrix-synapse;
-        defaultText = literalExpression "pkgs.matrix-synapse";
-        description = ''
-          Overridable attribute of the matrix synapse server package to use.
-        '';
-      };
-      plugins = mkOption {
-        type = types.listOf types.package;
-        default = [ ];
-        example = literalExpression ''
-          with config.services.matrix-synapse.package.plugins; [
-            matrix-synapse-ldap3
-            matrix-synapse-pam
-          ];
-        '';
-        description = ''
-          List of additional Matrix plugins to make available.
-        '';
-      };
-      withJemalloc = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to preload jemalloc to reduce memory fragmentation and overall usage.
-        '';
-      };
-      no_tls = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Don't bind to the https port
-        '';
-      };
-      bind_port = mkOption {
-        type = types.nullOr types.int;
-        default = null;
-        example = 8448;
-        description = ''
-          DEPRECATED: Use listeners instead.
-          The port to listen for HTTPS requests on.
-          For when matrix traffic is sent directly to synapse.
-        '';
-      };
-      unsecure_port = mkOption {
-        type = types.nullOr types.int;
-        default = null;
-        example = 8008;
-        description = ''
-          DEPRECATED: Use listeners instead.
-          The port to listen for HTTP requests on.
-          For when matrix traffic passes through loadbalancer that unwraps TLS.
-        '';
-      };
-      bind_host = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          DEPRECATED: Use listeners instead.
-          Local interface to listen on.
-          The empty string will cause synapse to listen on all interfaces.
-        '';
-      };
-      tls_certificate_path = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = "${cfg.dataDir}/homeserver.tls.crt";
-        description = ''
-          PEM encoded X509 certificate for TLS.
-          You can replace the self-signed certificate that synapse
-          autogenerates on launch with your own SSL certificate + key pair
-          if you like.  Any required intermediary certificates can be
-          appended after the primary certificate in hierarchical order.
-        '';
-      };
-      tls_private_key_path = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = "${cfg.dataDir}/homeserver.tls.key";
-        description = ''
-          PEM encoded private key for TLS. Specify null if synapse is not
-          speaking TLS directly.
-        '';
-      };
-      tls_dh_params_path = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = "${cfg.dataDir}/homeserver.tls.dh";
-        description = ''
-          PEM dh parameters for ephemeral keys
-        '';
-      };
-      server_name = mkOption {
-        type = types.str;
-        example = "example.com";
-        default = config.networking.hostName;
-        defaultText = literalExpression "config.networking.hostName";
-        description = ''
-          The domain name of the server, with optional explicit port.
-          This is used by remote servers to look up the server address.
-          This is also the last part of your UserID.
-
-          The server_name cannot be changed later so it is important to configure this correctly before you start Synapse.
-        '';
-      };
-      public_baseurl = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = "https://example.com:8448/";
-        description = ''
-          The public-facing base URL for the client API (not including _matrix/...)
-        '';
-      };
-      listeners = mkOption {
-        type = types.listOf (types.submodule {
-          options = {
-            port = mkOption {
-              type = types.port;
-              example = 8448;
-              description = ''
-                The port to listen for HTTP(S) requests on.
-              '';
-            };
-            bind_address = mkOption {
-              type = types.str;
-              default = "";
-              example = "203.0.113.42";
-              description = ''
-                Local interface to listen on.
-                The empty string will cause synapse to listen on all interfaces.
-              '';
-            };
-            type = mkOption {
-              type = types.str;
-              default = "http";
-              description = ''
-                Type of listener.
-              '';
-            };
-            tls = mkOption {
-              type = types.bool;
-              default = true;
-              description = ''
-                Whether to listen for HTTPS connections rather than HTTP.
-              '';
-            };
-            x_forwarded = mkOption {
-              type = types.bool;
-              default = false;
-              description = ''
-                Use the X-Forwarded-For (XFF) header as the client IP and not the
-                actual client IP.
-              '';
-            };
-            resources = mkOption {
-              type = types.listOf (types.submodule {
-                options = {
-                  names = mkOption {
-                    type = types.listOf types.str;
-                    description = ''
-                      List of resources to host on this listener.
-                    '';
-                    example = ["client" "webclient" "federation"];
-                  };
-                  compress = mkOption {
-                    type = types.bool;
-                    description = ''
-                      Should synapse compress HTTP responses to clients that support it?
-                      This should be disabled if running synapse behind a load balancer
-                      that can do automatic compression.
-                    '';
-                  };
-                };
-              });
-              description = ''
-                List of HTTP resources to serve on this listener.
-              '';
-            };
-          };
-        });
-        default = [{
-          port = 8448;
-          bind_address = "";
-          type = "http";
-          tls = true;
-          x_forwarded = false;
-          resources = [
-            { names = ["client" "webclient"]; compress = true; }
-            { names = ["federation"]; compress = false; }
-          ];
-        }];
-        description = ''
-          List of ports that Synapse should listen on, their purpose and their configuration.
-        '';
-      };
-      verbose = mkOption {
-        type = types.str;
-        default = "0";
-        description = "Logging verbosity level.";
-      };
-      rc_messages_per_second = mkOption {
-        type = types.str;
-        default = "0.2";
-        description = "Number of messages a client can send per second";
-      };
-      rc_message_burst_count = mkOption {
-        type = types.str;
-        default = "10.0";
-        description = "Number of message a client can send before being throttled";
-      };
-      federation_rc_window_size = mkOption {
-        type = types.str;
-        default = "1000";
-        description = "The federation window size in milliseconds";
-      };
-      federation_rc_sleep_limit = mkOption {
-        type = types.str;
-        default = "10";
-        description = ''
-          The number of federation requests from a single server in a window
-          before the server will delay processing the request.
-        '';
-      };
-      federation_rc_sleep_delay = mkOption {
-        type = types.str;
-        default = "500";
-        description = ''
-          The duration in milliseconds to delay processing events from
-          remote servers by if they go over the sleep limit.
-        '';
-      };
-      federation_rc_reject_limit = mkOption {
-        type = types.str;
-        default = "50";
-        description = ''
-          The maximum number of concurrent federation requests allowed
-          from a single server
-        '';
-      };
-      federation_rc_concurrent = mkOption {
-        type = types.str;
-        default = "3";
-        description = "The number of federation requests to concurrently process from a single server";
-      };
-      database_type = mkOption {
-        type = types.enum [ "sqlite3" "psycopg2" ];
-        default = if versionAtLeast config.system.stateVersion "18.03"
-          then "psycopg2"
-          else "sqlite3";
-        defaultText = literalExpression ''
-          if versionAtLeast config.system.stateVersion "18.03"
-            then "psycopg2"
-            else "sqlite3"
-        '';
-        description = ''
-          The database engine name. Can be sqlite or psycopg2.
-        '';
-      };
-      database_name = mkOption {
-        type = types.str;
-        default = "matrix-synapse";
-        description = "Database name.";
-      };
-      database_user = mkOption {
-        type = types.str;
-        default = "matrix-synapse";
-        description = "Database user name.";
-      };
-      database_args = mkOption {
-        type = types.attrs;
-        default = {
-          sqlite3 = { database = "${cfg.dataDir}/homeserver.db"; };
-          psycopg2 = {
-            user = cfg.database_user;
-            database = cfg.database_name;
-          };
-        }.${cfg.database_type};
-        description = ''
-          Arguments to pass to the engine.
-        '';
-      };
-      event_cache_size = mkOption {
-        type = types.str;
-        default = "10K";
-        description = "Number of events to cache in memory.";
-      };
-      url_preview_enabled = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Is the preview URL API enabled?  If enabled, you *must* specify an
-          explicit url_preview_ip_range_blacklist of IPs that the spider is
-          denied from accessing.
-        '';
-      };
-      url_preview_ip_range_blacklist = mkOption {
-        type = types.listOf types.str;
-        default = [
-          "127.0.0.0/8"
-          "10.0.0.0/8"
-          "172.16.0.0/12"
-          "192.168.0.0/16"
-          "100.64.0.0/10"
-          "169.254.0.0/16"
-          "::1/128"
-          "fe80::/64"
-          "fc00::/7"
-        ];
-        description = ''
-          List of IP address CIDR ranges that the URL preview spider is denied
-          from accessing.
-        '';
-      };
-      url_preview_ip_range_whitelist = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          List of IP address CIDR ranges that the URL preview spider is allowed
-          to access even if they are specified in
-          url_preview_ip_range_blacklist.
-        '';
-      };
-      url_preview_url_blacklist = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          Optional list of URL matches that the URL preview spider is
-          denied from accessing.
-        '';
-      };
-      recaptcha_private_key = mkOption {
-        type = types.str;
-        default = "";
-        description = ''
-          This Home Server's ReCAPTCHA private key.
-        '';
-      };
-      recaptcha_public_key = mkOption {
-        type = types.str;
-        default = "";
-        description = ''
-          This Home Server's ReCAPTCHA public key.
-        '';
-      };
-      enable_registration_captcha = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enables ReCaptcha checks when registering, preventing signup
-          unless a captcha is answered. Requires a valid ReCaptcha
-          public/private key.
-        '';
-      };
-      turn_uris = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          The public URIs of the TURN server to give to clients
-        '';
-      };
-      turn_shared_secret = mkOption {
-        type = types.str;
-        default = "";
-        description = ''
-          The shared secret used to compute passwords for the TURN server
-        '';
-      };
-      turn_user_lifetime = mkOption {
-        type = types.str;
-        default = "1h";
-        description = "How long generated TURN credentials last";
-      };
-      enable_registration = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enable registration for new users.
-        '';
-      };
-      registration_shared_secret = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          If set, allows registration by anyone who also has the shared
-          secret, even if registration is otherwise disabled.
-        '';
-      };
-      enable_metrics = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enable collection and rendering of performance metrics
-        '';
-      };
-      report_stats = mkOption {
-        type = types.bool;
-        default = false;
-        description = "";
-      };
-      servers = mkOption {
-        type = types.attrsOf (types.attrsOf types.str);
-        default = {
-          "matrix.org" = {
-            "ed25519:auto" = "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw";
-          };
-        };
-        description = ''
-          The trusted servers to download signing keys from.
-        '';
-      };
-      max_upload_size = mkOption {
-        type = types.str;
-        default = "10M";
-        description = "The largest allowed upload size in bytes";
-      };
-      max_image_pixels = mkOption {
-        type = types.str;
-        default = "32M";
-        description = "Maximum number of pixels that will be thumbnailed";
-      };
-      dynamic_thumbnails = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to generate new thumbnails on the fly to precisely match
-          the resolution requested by the client. If true then whenever
-          a new resolution is requested by the client the server will
-          generate a new thumbnail. If false the server will pick a thumbnail
-          from a precalculated list.
-        '';
-      };
-      user_creation_max_duration = mkOption {
-        type = types.str;
-        default = "1209600000";
-        description = ''
-          Sets the expiry for the short term user creation in
-          milliseconds. The default value is two weeks.
-        '';
-      };
-      bcrypt_rounds = mkOption {
-        type = types.str;
-        default = "12";
-        description = ''
-          Set the number of bcrypt rounds used to generate password hash.
-          Larger numbers increase the work factor needed to generate the hash.
-        '';
-      };
-      allow_guest_access = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Allows users to register as guests without a password/email/etc, and
-          participate in rooms hosted on this server which have been made
-          accessible to anonymous users.
-        '';
-      };
-      account_threepid_delegates.email = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Delegate email sending to https://example.org
-        '';
-      };
-      account_threepid_delegates.msisdn = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Delegate SMS sending to this local process (https://localhost:8090)
-        '';
-      };
-      room_prejoin_state.additional_event_types = mkOption {
-        default = [];
-        type = types.listOf types.str;
-        description = ''
-          Additional events to share with users who received an invite.
-        '';
-      };
-      room_prejoin_state.disable_default_event_types = mkOption {
-        default = false;
-        type = types.bool;
-        description = ''
-          Whether to disable the default state-event types for users invited to a room.
-          These are:
-
-          <itemizedlist>
-          <listitem><para>m.room.join_rules</para></listitem>
-          <listitem><para>m.room.canonical_alias</para></listitem>
-          <listitem><para>m.room.avatar</para></listitem>
-          <listitem><para>m.room.encryption</para></listitem>
-          <listitem><para>m.room.name</para></listitem>
-          <listitem><para>m.room.create</para></listitem>
-          </itemizedlist>
-        '';
-      };
-      macaroon_secret_key = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          Secret key for authentication tokens
-        '';
-      };
-      expire_access_token = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to enable access token expiration.
-        '';
-      };
-      key_refresh_interval = mkOption {
-        type = types.str;
-        default = "1d";
-        description = ''
-          How long key response published by this server is valid for.
-          Used to set the valid_until_ts in /key/v2 APIs.
-          Determines how quickly servers will query to check which keys
-          are still valid.
-        '';
-      };
-      app_service_config_files = mkOption {
-        type = types.listOf types.path;
-        default = [ ];
-        description = ''
-          A list of application service config file to use
-        '';
-      };
-      redaction_retention_period = mkOption {
-        type = types.int;
-        default = 7;
-        description = ''
-          How long to keep redacted events in unredacted form in the database.
-        '';
-      };
-      extraConfig = mkOption {
-        type = types.lines;
-        default = "";
-        description = ''
-          Extra config options for matrix-synapse.
-        '';
-      };
-      extraConfigFiles = mkOption {
-        type = types.listOf types.path;
-        default = [];
-        description = ''
-          Extra config files to include.
-
-          The configuration files will be included based on the command line
-          argument --config-path. This allows to configure secrets without
-          having to go through the Nix store, e.g. based on deployment keys if
-          NixOPS is in use.
-        '';
-      };
-      logConfig = mkOption {
-        type = types.lines;
-        default = readFile ./matrix-synapse-log_config.yaml;
-        description = ''
-          A yaml python logging config file
-        '';
-      };
-      dataDir = mkOption {
-        type = types.str;
-        default = "/var/lib/matrix-synapse";
-        description = ''
-          The directory where matrix-synapse stores its stateful data such as
-          certificates, media and uploads.
-        '';
-      };
-    };
-  };
-
-  config = mkIf cfg.enable {
-    assertions = [
-      { assertion = hasLocalPostgresDB -> config.services.postgresql.enable;
-        message = ''
-          Cannot deploy matrix-synapse with a configuration for a local postgresql database
-            and a missing postgresql service. Since 20.03 it's mandatory to manually configure the
-            database (please read the thread in https://github.com/NixOS/nixpkgs/pull/80447 for
-            further reference).
-
-            If you
-            - try to deploy a fresh synapse, you need to configure the database yourself. An example
-              for this can be found in <nixpkgs/nixos/tests/matrix-synapse.nix>
-            - update your existing matrix-synapse instance, you simply need to add `services.postgresql.enable = true`
-              to your configuration.
-
-          For further information about this update, please read the release-notes of 20.03 carefully.
-        '';
-      }
-    ];
-
-    services.matrix-synapse.configFile = "${configFile}";
-
-    users.users.matrix-synapse = {
-      group = "matrix-synapse";
-      home = cfg.dataDir;
-      createHome = true;
-      shell = "${pkgs.bash}/bin/bash";
-      uid = config.ids.uids.matrix-synapse;
-    };
-
-    users.groups.matrix-synapse = {
-      gid = config.ids.gids.matrix-synapse;
-    };
-
-    systemd.services.matrix-synapse = {
-      description = "Synapse Matrix homeserver";
-      after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
-      wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        ${cfg.package}/bin/homeserver \
-          --config-path ${configFile} \
-          --keys-directory ${cfg.dataDir} \
-          --generate-keys
-      '';
-      environment = {
-        PYTHONPATH = makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ];
-      } // optionalAttrs (cfg.withJemalloc) {
-        LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
-      };
-      serviceConfig = {
-        Type = "notify";
-        User = "matrix-synapse";
-        Group = "matrix-synapse";
-        WorkingDirectory = cfg.dataDir;
-        ExecStartPre = [ ("+" + (pkgs.writeShellScript "matrix-synapse-fix-permissions" ''
-          chown matrix-synapse:matrix-synapse ${cfg.dataDir}/homeserver.signing.key
-          chmod 0600 ${cfg.dataDir}/homeserver.signing.key
-        '')) ];
-        ExecStart = ''
-          ${cfg.package}/bin/homeserver \
-            ${ concatMapStringsSep "\n  " (x: "--config-path ${x} \\") ([ configFile ] ++ cfg.extraConfigFiles) }
-            --keys-directory ${cfg.dataDir}
-        '';
-        ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID";
-        Restart = "on-failure";
-        UMask = "0077";
-      };
-    };
-  };
-
-  imports = [
-    (mkRemovedOptionModule [ "services" "matrix-synapse" "trusted_third_party_id_servers" ] ''
-      The `trusted_third_party_id_servers` option as been removed in `matrix-synapse` v1.4.0
-      as the behavior is now obsolete.
-    '')
-    (mkRemovedOptionModule [ "services" "matrix-synapse" "create_local_database" ] ''
-      Database configuration must be done manually. An exemplary setup is demonstrated in
-      <nixpkgs/nixos/tests/matrix-synapse.nix>
-    '')
-    (mkRemovedOptionModule [ "services" "matrix-synapse" "web_client" ] "")
-    (mkRemovedOptionModule [ "services" "matrix-synapse" "room_invite_state_types" ] ''
-      You may add additional event types via
-      `services.matrix-synapse.room_prejoin_state.additional_event_types` and
-      disable the default events via
-      `services.matrix-synapse.room_prejoin_state.disable_default_event_types`.
-    '')
-  ];
-
-  meta.doc = ./matrix-synapse.xml;
-  meta.maintainers = teams.matrix.members;
-
-}
diff --git a/nixos/modules/services/misc/mautrix-telegram.nix b/nixos/modules/services/misc/mautrix-telegram.nix
index 3b070b873b04..794c4dd9ddcd 100644
--- a/nixos/modules/services/misc/mautrix-telegram.nix
+++ b/nixos/modules/services/misc/mautrix-telegram.nix
@@ -145,7 +145,7 @@ in {
             --config='${settingsFile}' \
             --registration='${registrationFile}'
         fi
-
+      '' + lib.optionalString (pkgs.mautrix-telegram ? alembic) ''
         # run automatic database init and migration scripts
         ${pkgs.mautrix-telegram.alembic}/bin/alembic -x config='${settingsFile}' upgrade head
       '';
diff --git a/nixos/modules/services/misc/mbpfan.nix b/nixos/modules/services/misc/mbpfan.nix
index d80b6fafc2cf..e0a4d8a13e75 100644
--- a/nixos/modules/services/misc/mbpfan.nix
+++ b/nixos/modules/services/misc/mbpfan.nix
@@ -5,6 +5,8 @@ with lib;
 let
   cfg = config.services.mbpfan;
   verbose = if cfg.verbose then "v" else "";
+  settingsFormat = pkgs.formats.ini {};
+  settingsFile = settingsFormat.generate "mbpfan.ini" cfg.settings;
 
 in {
   options.services.mbpfan = {
@@ -19,54 +21,6 @@ in {
       '';
     };
 
-    minFanSpeed = mkOption {
-      type = types.int;
-      default = 2000;
-      description = ''
-        The minimum fan speed.
-      '';
-    };
-
-    maxFanSpeed = mkOption {
-      type = types.int;
-      default = 6200;
-      description = ''
-        The maximum fan speed.
-      '';
-    };
-
-    lowTemp = mkOption {
-      type = types.int;
-      default = 63;
-      description = ''
-        The low temperature.
-      '';
-    };
-
-    highTemp = mkOption {
-      type = types.int;
-      default = 66;
-      description = ''
-        The high temperature.
-      '';
-    };
-
-    maxTemp = mkOption {
-      type = types.int;
-      default = 86;
-      description = ''
-        The maximum temperature.
-      '';
-    };
-
-    pollingInterval = mkOption {
-      type = types.int;
-      default = 7;
-      description = ''
-        The polling interval.
-      '';
-    };
-
     verbose = mkOption {
       type = types.bool;
       default = false;
@@ -74,23 +28,67 @@ in {
         If true, sets the log level to verbose.
       '';
     };
+
+    settings = mkOption {
+      default = {};
+      description = "The INI configuration for Mbpfan.";
+      type = types.submodule {
+        freeformType = settingsFormat.type;
+
+        options.general.min_fan1_speed = mkOption {
+          type = types.nullOr types.int;
+          default = 2000;
+          description = ''
+            The minimum fan speed. Setting to null enables automatic detection.
+            Check minimum fan limits with "cat /sys/devices/platform/applesmc.768/fan*_min".
+          '';
+        };
+        options.general.max_fan1_speed = mkOption {
+          type = types.nullOr types.int;
+          default = 6199;
+          description = ''
+            The maximum fan speed. Setting to null enables automatic detection.
+            Check maximum fan limits with "cat /sys/devices/platform/applesmc.768/fan*_max".
+          '';
+        };
+        options.general.low_temp = mkOption {
+          type = types.int;
+          default = 55;
+          description = "Temperature below which fan speed will be at minimum. Try ranges 55-63.";
+        };
+        options.general.high_temp = mkOption {
+          type = types.int;
+          default = 58;
+          description = "Fan will increase speed when higher than this temperature. Try ranges 58-66.";
+        };
+        options.general.max_temp = mkOption {
+          type = types.int;
+          default = 86;
+          description = "Fan will run at full speed above this temperature. Do not set it > 90.";
+        };
+        options.general.polling_interval = mkOption {
+          type = types.int;
+          default = 1;
+          description = "The polling interval.";
+        };
+      };
+    };
   };
 
+  imports = [
+    (mkRenamedOptionModule [ "services" "mbpfan" "pollingInterval" ] [ "services" "mbpfan" "settings" "general" "polling_interval" ])
+    (mkRenamedOptionModule [ "services" "mbpfan" "maxTemp" ] [ "services" "mbpfan" "settings" "general" "max_temp" ])
+    (mkRenamedOptionModule [ "services" "mbpfan" "lowTemp" ] [ "services" "mbpfan" "settings" "general" "low_temp" ])
+    (mkRenamedOptionModule [ "services" "mbpfan" "highTemp" ] [ "services" "mbpfan" "settings" "general" "high_temp" ])
+    (mkRenamedOptionModule [ "services" "mbpfan" "minFanSpeed" ] [ "services" "mbpfan" "settings" "general" "min_fan1_speed" ])
+    (mkRenamedOptionModule [ "services" "mbpfan" "maxFanSpeed" ] [ "services" "mbpfan" "settings" "general" "max_fan1_speed" ])
+  ];
+
   config = mkIf cfg.enable {
     boot.kernelModules = [ "coretemp" "applesmc" ];
 
-    environment = {
-      etc."mbpfan.conf".text = ''
-        [general]
-        min_fan_speed = ${toString cfg.minFanSpeed}
-        max_fan_speed = ${toString cfg.maxFanSpeed}
-        low_temp = ${toString cfg.lowTemp}
-        high_temp = ${toString cfg.highTemp}
-        max_temp = ${toString cfg.maxTemp}
-        polling_interval = ${toString cfg.pollingInterval}
-      '';
-      systemPackages = [ cfg.package ];
-    };
+    environment.etc."mbpfan.conf".source = settingsFile;
+    environment.systemPackages = [ cfg.package ];
 
     systemd.services.mbpfan = {
       description = "A fan manager daemon for MacBook Pro";
diff --git a/nixos/modules/services/misc/mediatomb.nix b/nixos/modules/services/misc/mediatomb.nix
index 383090575b22..ee5c0ef8d277 100644
--- a/nixos/modules/services/misc/mediatomb.nix
+++ b/nixos/modules/services/misc/mediatomb.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
@@ -6,6 +6,7 @@ let
 
   gid = config.ids.gids.mediatomb;
   cfg = config.services.mediatomb;
+  opt = options.services.mediatomb;
   name = cfg.package.pname;
   pkg = cfg.package;
   optionYesNo = option: if option then "yes" else "no";
@@ -216,7 +217,6 @@ in {
 
       package = mkOption {
         type = types.package;
-        example = literalExpression "pkgs.mediatomb";
         default = pkgs.gerbera;
         defaultText = literalExpression "pkgs.gerbera";
         description = ''
@@ -261,6 +261,7 @@ in {
       dataDir = mkOption {
         type = types.path;
         default = "/var/lib/${name}";
+        defaultText = literalExpression ''"/var/lib/''${config.${opt.package}.pname}"'';
         description = ''
           The directory where Gerbera/Mediatomb stores its state, data, etc.
         '';
@@ -277,13 +278,13 @@ in {
       user = mkOption {
         type = types.str;
         default = "mediatomb";
-        description = "User account under which ${name} runs.";
+        description = "User account under which the service runs.";
       };
 
       group = mkOption {
         type = types.str;
         default = "mediatomb";
-        description = "Group account under which ${name} runs.";
+        description = "Group account under which the service runs.";
       };
 
       port = mkOption {
@@ -340,7 +341,7 @@ in {
         type = types.bool;
         default = false;
         description = ''
-          Allow ${name} to create and use its own config file inside the <literal>dataDir</literal> as
+          Allow the service to create and use its own config file inside the <literal>dataDir</literal> as
           configured by <option>services.mediatomb.dataDir</option>.
           Deactivated by default, the service then runs with the configuration generated from this module.
           Otherwise, when enabled, no service configuration is generated. Gerbera/Mediatomb then starts using
@@ -365,6 +366,7 @@ in {
       wantedBy = [ "multi-user.target" ];
       serviceConfig.ExecStart = "${binaryCommand} --port ${toString cfg.port} ${interfaceFlag} ${configFlag} --home ${cfg.dataDir}";
       serviceConfig.User = cfg.user;
+      serviceConfig.Group = cfg.group;
     };
 
     users.groups = optionalAttrs (cfg.group == "mediatomb") {
diff --git a/nixos/modules/services/misc/moonraker.nix b/nixos/modules/services/misc/moonraker.nix
index e08d2f84212d..ae57aaa6d479 100644
--- a/nixos/modules/services/misc/moonraker.nix
+++ b/nixos/modules/services/misc/moonraker.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 with lib;
 let
   pkg = pkgs.moonraker;
   cfg = config.services.moonraker;
+  opt = options.services.moonraker;
   format = pkgs.formats.ini {
     # https://github.com/NixOS/nixpkgs/pull/121613#issuecomment-885241996
     listToValue = l:
@@ -31,6 +32,7 @@ in {
       configDir = mkOption {
         type = types.path;
         default = cfg.stateDir + "/config";
+        defaultText = literalExpression ''config.${opt.stateDir} + "/config"'';
         description = ''
           The directory containing client-writable configuration files.
 
diff --git a/nixos/modules/services/misc/mwlib.nix b/nixos/modules/services/misc/mwlib.nix
deleted file mode 100644
index 8dd17c06c0b3..000000000000
--- a/nixos/modules/services/misc/mwlib.nix
+++ /dev/null
@@ -1,258 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  cfg = config.services.mwlib;
-  pypkgs = pkgs.python27Packages;
-
-  inherit (pypkgs) python mwlib;
-
-  user = mkOption {
-    default = "nobody";
-    type = types.str;
-    description = "User to run as.";
-  };
-
-in
-{
-
-  options.services.mwlib = {
-
-    nserve = {
-      enable = mkOption {
-        default = false;
-        type = types.bool;
-        description = ''
-          Whether to enable nserve. Nserve is a HTTP
-          server.  The Collection extension is talking to
-          that program directly.  Nserve uses at least
-          one qserve instance in order to distribute
-          and manage jobs.
-        '';
-      }; # nserve.enable
-
-      port = mkOption {
-        default = 8899;
-        type = types.port;
-        description = "Specify port to listen on.";
-      }; # nserve.port
-
-      address = mkOption {
-        default = "127.0.0.1";
-        type = types.str;
-        description = "Specify network interface to listen on.";
-      }; # nserve.address
-
-      qserve = mkOption {
-        default = [ "${cfg.qserve.address}:${toString cfg.qserve.port}" ];
-        type = types.listOf types.str;
-        description = "Register qserve instance.";
-      }; # nserve.qserve
-
-      inherit user;
-    }; # nserve
-
-    qserve = {
-      enable = mkOption {
-        default = false;
-        type = types.bool;
-        description = ''
-          A job queue server used to distribute and manage
-          jobs. You should start one qserve instance
-          for each machine that is supposed to render pdf
-          files. Unless you’re operating the Wikipedia
-          installation, one machine should suffice.
-        '';
-      }; # qserve.enable
-
-      port = mkOption {
-        default = 14311;
-        type = types.port;
-        description = "Specify port to listen on.";
-      }; # qserve.port
-
-      address = mkOption {
-        default = "127.0.0.1";
-        type = types.str;
-        description = "Specify network interface to listen on.";
-      }; # qserve.address
-
-      datadir = mkOption {
-        default = "/var/lib/mwlib-qserve";
-        type = types.path;
-        description = "qserve data directory (FIXME: unused?)";
-      }; # qserve.datadir
-
-      allow = mkOption {
-        default = [ "127.0.0.1" ];
-        type = types.listOf types.str;
-        description = "List of allowed client IPs. Empty means any.";
-      }; # qserve.allow
-
-      inherit user;
-    }; # qserve
-
-    nslave = {
-      enable = mkOption {
-        default = cfg.qserve.enable;
-        type = types.bool;
-        description = ''
-          Pulls new jobs from exactly one qserve instance
-          and calls the zip and render programs
-          in order to download article collections and
-          convert them to different output formats. Nslave
-          uses a cache directory to store the generated
-          documents. Nslave also starts an internal http
-          server serving the content of the cache directory.
-        '';
-      }; # nslave.enable
-
-      cachedir = mkOption {
-        default = "/var/cache/mwlib-nslave";
-        type = types.path;
-        description = "Directory to store generated documents.";
-      }; # nslave.cachedir
-
-      numprocs = mkOption {
-        default = 10;
-        type = types.int;
-        description = "Number of parallel jobs to be executed.";
-      }; # nslave.numprocs
-
-      http = mkOption {
-        default = {};
-        description = ''
-          Internal http server serving the content of the cache directory.
-          You have to enable it, or use your own way for serving files
-          and set the http.url option accordingly.
-          '';
-        type = types.submodule ({
-          options = {
-            enable = mkOption {
-              default = true;
-              type = types.bool;
-              description = "Enable internal http server.";
-            }; # nslave.http.enable
-
-            port = mkOption {
-              default = 8898;
-              type = types.port;
-              description = "Port to listen to when serving files from cache.";
-            }; # nslave.http.port
-
-            address = mkOption {
-              default = "127.0.0.1";
-              type = types.str;
-              description = "Specify network interface to listen on.";
-            }; # nslave.http.address
-
-            url = mkOption {
-              default = "http://localhost:${toString cfg.nslave.http.port}/cache";
-              type = types.str;
-              description = ''
-                Specify URL for accessing generated files from cache.
-                The Collection extension of Mediawiki won't be able to
-                download files without it.
-                '';
-            }; # nslave.http.url
-          };
-        }); # types.submodule
-      }; # nslave.http
-
-      inherit user;
-    }; # nslave
-
-  }; # options.services
-
-  config = {
-
-    systemd.services.mwlib-nserve = mkIf cfg.nserve.enable
-    {
-      description = "mwlib network interface";
-
-      wantedBy = [ "multi-user.target" ];
-      after = [ "network.target" "mwlib-qserve.service" ];
-
-      serviceConfig = {
-        ExecStart = concatStringsSep " " (
-          [
-            "${mwlib}/bin/nserve"
-            "--port ${toString cfg.nserve.port}"
-            "--interface ${cfg.nserve.address}"
-          ] ++ cfg.nserve.qserve
-        );
-        User = cfg.nserve.user;
-      };
-    }; # systemd.services.mwlib-nserve
-
-    systemd.services.mwlib-qserve = mkIf cfg.qserve.enable
-    {
-      description = "mwlib job queue server";
-
-      wantedBy = [ "multi-user.target" ];
-
-      preStart = ''
-        mkdir -pv '${cfg.qserve.datadir}'
-        chown -Rc ${cfg.qserve.user}:`id -ng ${cfg.qserve.user}` '${cfg.qserve.datadir}'
-        chmod -Rc u=rwX,go= '${cfg.qserve.datadir}'
-      '';
-
-      serviceConfig = {
-        ExecStart = concatStringsSep " " (
-          [
-            "${mwlib}/bin/mw-qserve"
-            "-p ${toString cfg.qserve.port}"
-            "-i ${cfg.qserve.address}"
-            "-d ${cfg.qserve.datadir}"
-          ] ++ map (a: "-a ${a}") cfg.qserve.allow
-        );
-        User = cfg.qserve.user;
-        PermissionsStartOnly = true;
-      };
-    }; # systemd.services.mwlib-qserve
-
-    systemd.services.mwlib-nslave = mkIf cfg.nslave.enable
-    {
-      description = "mwlib worker";
-
-      wantedBy = [ "multi-user.target" ];
-      after = [ "network.target" ];
-
-      preStart = ''
-        mkdir -pv '${cfg.nslave.cachedir}'
-        chown -Rc ${cfg.nslave.user}:`id -ng ${cfg.nslave.user}` '${cfg.nslave.cachedir}'
-        chmod -Rc u=rwX,go= '${cfg.nslave.cachedir}'
-      '';
-
-      path = with pkgs; [ imagemagick pdftk ];
-      environment = {
-        PYTHONPATH = concatMapStringsSep ":"
-          (m: "${pypkgs.${m}}/lib/${python.libPrefix}/site-packages")
-          [ "mwlib-rl" "mwlib-ext" "pygments" "pyfribidi" ];
-      };
-
-      serviceConfig = {
-        ExecStart = concatStringsSep " " (
-          [
-            "${mwlib}/bin/nslave"
-            "--cachedir ${cfg.nslave.cachedir}"
-            "--numprocs ${toString cfg.nslave.numprocs}"
-            "--url ${cfg.nslave.http.url}"
-          ] ++ (
-            if cfg.nslave.http.enable then
-            [
-              "--serve-files-port ${toString cfg.nslave.http.port}"
-              "--serve-files-address ${cfg.nslave.http.address}"
-            ] else
-            [
-              "--no-serve-files"
-            ]
-          ));
-        User = cfg.nslave.user;
-        PermissionsStartOnly = true;
-      };
-    }; # systemd.services.mwlib-nslave
-
-  }; # config
-}
diff --git a/nixos/modules/services/misc/mx-puppet-discord.nix b/nixos/modules/services/misc/mx-puppet-discord.nix
index b6f5e04511ae..6214f7f7eb6b 100644
--- a/nixos/modules/services/misc/mx-puppet-discord.nix
+++ b/nixos/modules/services/misc/mx-puppet-discord.nix
@@ -79,10 +79,7 @@ in {
 
   config = mkIf cfg.enable {
     systemd.services.mx-puppet-discord = {
-      description = ''
-        mx-puppet-discord is a discord puppeting bridge for matrix.
-        It handles bridging private and group DMs, as well as Guilds (servers).
-      '';
+      description = "Matrix to Discord puppeting bridge";
 
       wantedBy = [ "multi-user.target" ];
       wants = [ "network-online.target" ] ++ cfg.serviceDependencies;
diff --git a/nixos/modules/services/misc/n8n.nix b/nixos/modules/services/misc/n8n.nix
index 27616e5f8226..77e717eeff9f 100644
--- a/nixos/modules/services/misc/n8n.nix
+++ b/nixos/modules/services/misc/n8n.nix
@@ -43,6 +43,7 @@ in
         # This folder must be writeable as the application is storing
         # its data in it, so the StateDirectory is a good choice
         N8N_USER_FOLDER = "/var/lib/n8n";
+        HOME = "/var/lib/n8n";
         N8N_CONFIG_FILES = "${configFile}";
       };
       serviceConfig = {
diff --git a/nixos/modules/services/misc/nitter.nix b/nixos/modules/services/misc/nitter.nix
index 6a9eeb02095c..97005c9d914f 100644
--- a/nixos/modules/services/misc/nitter.nix
+++ b/nixos/modules/services/misc/nitter.nix
@@ -49,6 +49,13 @@ in
     services.nitter = {
       enable = mkEnableOption "If enabled, start Nitter.";
 
+      package = mkOption {
+        default = pkgs.nitter;
+        type = types.package;
+        defaultText = literalExpression "pkgs.nitter";
+        description = "The nitter derivation to use.";
+      };
+
       server = {
         address = mkOption {
           type =  types.str;
@@ -78,8 +85,8 @@ in
 
         staticDir = mkOption {
           type = types.path;
-          default = "${pkgs.nitter}/share/nitter/public";
-          defaultText = literalExpression ''"''${pkgs.nitter}/share/nitter/public"'';
+          default = "${cfg.package}/share/nitter/public";
+          defaultText = literalExpression ''"''${config.services.nitter.package}/share/nitter/public"'';
           description = "Path to the static files directory.";
         };
 
@@ -306,8 +313,8 @@ in
           Environment = [ "NITTER_CONF_FILE=/var/lib/nitter/nitter.conf" ];
           # Some parts of Nitter expect `public` folder in working directory,
           # see https://github.com/zedeus/nitter/issues/414
-          WorkingDirectory = "${pkgs.nitter}/share/nitter";
-          ExecStart = "${pkgs.nitter}/bin/nitter";
+          WorkingDirectory = "${cfg.package}/share/nitter";
+          ExecStart = "${cfg.package}/bin/nitter";
           ExecStartPre = "${preStart}";
           AmbientCapabilities = lib.mkIf (cfg.server.port < 1024) [ "CAP_NET_BIND_SERVICE" ];
           Restart = "on-failure";
diff --git a/nixos/modules/services/misc/nix-daemon.nix b/nixos/modules/services/misc/nix-daemon.nix
index 869feb05eb7b..d56808c7564e 100644
--- a/nixos/modules/services/misc/nix-daemon.nix
+++ b/nixos/modules/services/misc/nix-daemon.nix
@@ -6,20 +6,20 @@ let
 
   cfg = config.nix;
 
-  nix = cfg.package.out;
+  nixPackage = cfg.package.out;
 
-  nixVersion = getVersion nix;
-
-  isNix23 = versionAtLeast nixVersion "2.3pre";
+  isNixAtLeast = versionAtLeast (getVersion nixPackage);
 
   makeNixBuildUser = nr: {
-    name  = "nixbld${toString nr}";
+    name = "nixbld${toString nr}";
     value = {
       description = "Nix build user ${toString nr}";
 
-      /* For consistency with the setgid(2), setuid(2), and setgroups(2)
-         calls in `libstore/build.cc', don't add any supplementary group
-         here except "nixbld".  */
+      /*
+        For consistency with the setgid(2), setuid(2), and setgroups(2)
+        calls in `libstore/build.cc', don't add any supplementary group
+        here except "nixbld".
+      */
       uid = builtins.add config.ids.uids.nixbld nr;
       isSystemUser = true;
       group = "nixbld";
@@ -30,53 +30,93 @@ let
   nixbldUsers = listToAttrs (map makeNixBuildUser (range 1 cfg.nrBuildUsers));
 
   nixConf =
-    assert versionAtLeast nixVersion "2.2";
-    pkgs.runCommand "nix.conf" { preferLocalBuild = true; extraOptions = cfg.extraOptions; } (
-      ''
-        cat > $out <<END
+    assert isNixAtLeast "2.2";
+    let
+
+      mkValueString = v:
+        if v == null then ""
+        else if isInt v then toString v
+        else if isBool v then boolToString v
+        else if isFloat v then floatToString v
+        else if isList v then toString v
+        else if isDerivation v then toString v
+        else if builtins.isPath v then toString v
+        else if isString v then v
+        else if isCoercibleToString v then toString v
+        else abort "The nix conf value: ${toPretty {} v} can not be encoded";
+
+      mkKeyValue = k: v: "${escape [ "=" ] k} = ${mkValueString v}";
+
+      mkKeyValuePairs = attrs: concatStringsSep "\n" (mapAttrsToList mkKeyValue attrs);
+
+    in
+    pkgs.writeTextFile {
+      name = "nix.conf";
+      text = ''
         # WARNING: this file is generated from the nix.* options in
         # your NixOS configuration, typically
         # /etc/nixos/configuration.nix.  Do not edit it!
-        build-users-group = nixbld
-        max-jobs = ${toString (cfg.maxJobs)}
-        cores = ${toString (cfg.buildCores)}
-        sandbox = ${if (builtins.isBool cfg.useSandbox) then boolToString cfg.useSandbox else cfg.useSandbox}
-        extra-sandbox-paths = ${toString cfg.sandboxPaths}
-        substituters = ${toString cfg.binaryCaches}
-        trusted-substituters = ${toString cfg.trustedBinaryCaches}
-        trusted-public-keys = ${toString cfg.binaryCachePublicKeys}
-        auto-optimise-store = ${boolToString cfg.autoOptimiseStore}
-        require-sigs = ${boolToString cfg.requireSignedBinaryCaches}
-        trusted-users = ${toString cfg.trustedUsers}
-        allowed-users = ${toString cfg.allowedUsers}
-        ${optionalString (!cfg.distributedBuilds) ''
-          builders =
-        ''}
-        system-features = ${toString cfg.systemFeatures}
-        ${optionalString isNix23 ''
-          sandbox-fallback = false
-        ''}
-        $extraOptions
-        END
-      '' + optionalString cfg.checkConfig (
-            if pkgs.stdenv.hostPlatform != pkgs.stdenv.buildPlatform then ''
-              echo "Ignore nix.checkConfig when cross-compiling"
-            '' else ''
-              echo "Checking that Nix can read nix.conf..."
-              ln -s $out ./nix.conf
-              NIX_CONF_DIR=$PWD ${cfg.package}/bin/nix show-config ${optionalString isNix23 "--no-net --option experimental-features nix-command"} >/dev/null
-            '')
-      );
+        ${mkKeyValuePairs cfg.settings}
+        ${cfg.extraOptions}
+      '';
+      checkPhase =
+        if pkgs.stdenv.hostPlatform != pkgs.stdenv.buildPlatform then ''
+          echo "Ignoring validation for cross-compilation"
+        ''
+        else ''
+          echo "Validating generated nix.conf"
+          ln -s $out ./nix.conf
+          set -e
+          set +o pipefail
+          NIX_CONF_DIR=$PWD \
+            ${cfg.package}/bin/nix show-config ${optionalString (isNixAtLeast "2.3pre") "--no-net"} \
+              ${optionalString (isNixAtLeast "2.4pre") "--option experimental-features nix-command"} \
+            |& sed -e 's/^warning:/error:/' \
+            | (! grep '${if cfg.checkConfig then "^error:" else "^error: unknown setting"}')
+          set -o pipefail
+        '';
+    };
+
+  legacyConfMappings = {
+    useSandbox = "sandbox";
+    buildCores = "cores";
+    maxJobs = "max-jobs";
+    sandboxPaths = "extra-sandbox-paths";
+    binaryCaches = "substituters";
+    trustedBinaryCaches = "trusted-substituters";
+    binaryCachePublicKeys = "trusted-public-keys";
+    autoOptimiseStore = "auto-optimise-store";
+    requireSignedBinaryCaches = "require-sigs";
+    trustedUsers = "trusted-users";
+    allowedUsers = "allowed-users";
+    systemFeatures = "system-features";
+  };
+
+  semanticConfType = with types;
+    let
+      confAtom = nullOr
+        (oneOf [
+          bool
+          int
+          float
+          str
+          path
+          package
+        ]) // {
+        description = "Nix config atom (null, bool, int, float, str, path or package)";
+      };
+    in
+    attrsOf (either confAtom (listOf confAtom));
 
 in
 
 {
   imports = [
-    (mkRenamedOptionModule [ "nix" "useChroot" ] [ "nix" "useSandbox" ])
-    (mkRenamedOptionModule [ "nix" "chrootDirs" ] [ "nix" "sandboxPaths" ])
-    (mkRenamedOptionModule [ "nix" "daemonIONiceLevel" ] [ "nix" "daemonIOSchedPriority" ])
+    (mkRenamedOptionModuleWith { sinceRelease = 2003; from = [ "nix" "useChroot" ]; to = [ "nix" "useSandbox" ]; })
+    (mkRenamedOptionModuleWith { sinceRelease = 2003; from = [ "nix" "chrootDirs" ]; to = [ "nix" "sandboxPaths" ]; })
+    (mkRenamedOptionModuleWith { sinceRelease = 2205; from = [ "nix" "daemonIONiceLevel" ]; to = [ "nix" "daemonIOSchedPriority" ]; })
     (mkRemovedOptionModule [ "nix" "daemonNiceLevel" ] "Consider nix.daemonCPUSchedPolicy instead.")
-  ];
+  ] ++ mapAttrsToList (oldConf: newConf: mkRenamedOptionModuleWith { sinceRelease = 2205; from = [ "nix" oldConf ]; to = [ "nix" "settings" newConf ]; }) legacyConfMappings;
 
   ###### interface
 
@@ -102,81 +142,6 @@ in
         '';
       };
 
-      maxJobs = mkOption {
-        type = types.either types.int (types.enum ["auto"]);
-        default = "auto";
-        example = 64;
-        description = ''
-          This option defines the maximum number of jobs that Nix will try to
-          build in parallel. The default is auto, which means it will use all
-          available logical cores. It is recommend to set it to the total
-          number of logical cores in your system (e.g., 16 for two CPUs with 4
-          cores each and hyper-threading).
-        '';
-      };
-
-      autoOptimiseStore = mkOption {
-        type = types.bool;
-        default = false;
-        example = true;
-        description = ''
-         If set to true, Nix automatically detects files in the store that have
-         identical contents, and replaces them with hard links to a single copy.
-         This saves disk space. If set to false (the default), you can still run
-         nix-store --optimise to get rid of duplicate files.
-        '';
-      };
-
-      buildCores = mkOption {
-        type = types.int;
-        default = 0;
-        example = 64;
-        description = ''
-          This option defines the maximum number of concurrent tasks during
-          one build. It affects, e.g., -j option for make.
-          The special value 0 means that the builder should use all
-          available CPU cores in the system. Some builds may become
-          non-deterministic with this option; use with care! Packages will
-          only be affected if enableParallelBuilding is set for them.
-        '';
-      };
-
-      useSandbox = mkOption {
-        type = types.either types.bool (types.enum ["relaxed"]);
-        default = true;
-        description = "
-          If set, Nix will perform builds in a sandboxed environment that it
-          will set up automatically for each build. This prevents impurities
-          in builds by disallowing access to dependencies outside of the Nix
-          store by using network and mount namespaces in a chroot environment.
-          This is enabled by default even though it has a possible performance
-          impact due to the initial setup time of a sandbox for each build. It
-          doesn't affect derivation hashes, so changing this option will not
-          trigger a rebuild of packages.
-        ";
-      };
-
-      sandboxPaths = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        example = [ "/dev" "/proc" ];
-        description =
-          ''
-            Directories from the host filesystem to be included
-            in the sandbox.
-          '';
-      };
-
-      extraOptions = mkOption {
-        type = types.lines;
-        default = "";
-        example = ''
-          keep-outputs = true
-          keep-derivations = true
-        '';
-        description = "Additional text appended to <filename>nix.conf</filename>.";
-      };
-
       distributedBuilds = mkOption {
         type = types.bool;
         default = false;
@@ -187,7 +152,7 @@ in
       };
 
       daemonCPUSchedPolicy = mkOption {
-        type = types.enum ["other" "batch" "idle"];
+        type = types.enum [ "other" "batch" "idle" ];
         default = "other";
         example = "batch";
         description = ''
@@ -218,7 +183,7 @@ in
       };
 
       daemonIOSchedClass = mkOption {
-        type = types.enum ["best-effort" "idle"];
+        type = types.enum [ "best-effort" "idle" ];
         default = "best-effort";
         example = "idle";
         description = ''
@@ -250,11 +215,11 @@ in
           scheduling policy: With idle, priorities are not used in scheduling
           decisions. best-effort supports values in the range 0 (high) to 7
           (low).
-      '';
+        '';
       };
 
       buildMachines = mkOption {
-        type = types.listOf (types.submodule ({
+        type = types.listOf (types.submodule {
           options = {
             hostName = mkOption {
               type = types.str;
@@ -276,7 +241,7 @@ in
             };
             systems = mkOption {
               type = types.listOf types.str;
-              default = [];
+              default = [ ];
               example = [ "x86_64-linux" "aarch64-linux" ];
               description = ''
                 The system types the build machine can execute derivations on.
@@ -293,7 +258,7 @@ in
                 The username to log in as on the remote host. This user must be
                 able to log in and run nix commands non-interactively. It must
                 also be privileged to build derivations, so must be included in
-                <option>nix.trustedUsers</option>.
+                <option>nix.settings.trusted-users</option>.
               '';
             };
             sshKey = mkOption {
@@ -331,7 +296,7 @@ in
             };
             mandatoryFeatures = mkOption {
               type = types.listOf types.str;
-              default = [];
+              default = [ ];
               example = [ "big-parallel" ];
               description = ''
                 A list of features mandatory for this builder. The builder will
@@ -342,7 +307,7 @@ in
             };
             supportedFeatures = mkOption {
               type = types.listOf types.str;
-              default = [];
+              default = [ ];
               example = [ "kvm" "big-parallel" ];
               description = ''
                 A list of features supported by this builder. The builder will
@@ -350,9 +315,18 @@ in
                 list.
               '';
             };
+            publicHostKey = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                The (base64-encoded) public host key of this builder. The field
+                is calculated via <command>base64 -w0 /etc/ssh/ssh_host_type_key.pub</command>.
+                If null, SSH will use its regular known-hosts file when connecting.
+              '';
+            };
           };
-        }));
-        default = [];
+        });
+        default = [ ];
         description = ''
           This option lists the machines to be used if distributed builds are
           enabled (see <option>nix.distributedBuilds</option>).
@@ -366,7 +340,7 @@ in
       envVars = mkOption {
         type = types.attrs;
         internal = true;
-        default = {};
+        default = { };
         description = "Environment variables used by Nix.";
       };
 
@@ -391,92 +365,13 @@ in
         '';
       };
 
-      binaryCaches = mkOption {
-        type = types.listOf types.str;
-        description = ''
-          List of binary cache URLs used to obtain pre-built binaries
-          of Nix packages.
-
-          By default https://cache.nixos.org/ is added,
-          to override it use <literal>lib.mkForce []</literal>.
-        '';
-      };
-
-      trustedBinaryCaches = mkOption {
-        type = types.listOf types.str;
-        default = [ ];
-        example = [ "https://hydra.nixos.org/" ];
-        description = ''
-          List of binary cache URLs that non-root users can use (in
-          addition to those specified using
-          <option>nix.binaryCaches</option>) by passing
-          <literal>--option binary-caches</literal> to Nix commands.
-        '';
-      };
-
-      requireSignedBinaryCaches = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          If enabled (the default), Nix will only download binaries from binary caches if
-          they are cryptographically signed with any of the keys listed in
-          <option>nix.binaryCachePublicKeys</option>. If disabled, signatures are neither
-          required nor checked, so it's strongly recommended that you use only
-          trustworthy caches and https to prevent man-in-the-middle attacks.
-        '';
-      };
-
-      binaryCachePublicKeys = mkOption {
-        type = types.listOf types.str;
-        example = [ "hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=" ];
-        description = ''
-          List of public keys used to sign binary caches. If
-          <option>nix.requireSignedBinaryCaches</option> is enabled,
-          then Nix will use a binary from a binary cache if and only
-          if it is signed by <emphasis>any</emphasis> of the keys
-          listed here. By default, only the key for
-          <uri>cache.nixos.org</uri> is included.
-        '';
-      };
-
-      trustedUsers = mkOption {
-        type = types.listOf types.str;
-        default = [ "root" ];
-        example = [ "root" "alice" "@wheel" ];
-        description = ''
-          A list of names of users that have additional rights when
-          connecting to the Nix daemon, such as the ability to specify
-          additional binary caches, or to import unsigned NARs. You
-          can also specify groups by prefixing them with
-          <literal>@</literal>; for instance,
-          <literal>@wheel</literal> means all users in the wheel
-          group.
-        '';
-      };
-
-      allowedUsers = mkOption {
-        type = types.listOf types.str;
-        default = [ "*" ];
-        example = [ "@wheel" "@builders" "alice" "bob" ];
-        description = ''
-          A list of names of users (separated by whitespace) that are
-          allowed to connect to the Nix daemon. As with
-          <option>nix.trustedUsers</option>, you can specify groups by
-          prefixing them with <literal>@</literal>. Also, you can
-          allow all users by specifying <literal>*</literal>. The
-          default is <literal>*</literal>. Note that trusted users are
-          always allowed to connect.
-        '';
-      };
-
       nixPath = mkOption {
         type = types.listOf types.str;
-        default =
-          [
-            "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos"
-            "nixos-config=/etc/nixos/configuration.nix"
-            "/nix/var/nix/profiles/per-user/root/channels"
-          ];
+        default = [
+          "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos"
+          "nixos-config=/etc/nixos/configuration.nix"
+          "/nix/var/nix/profiles/per-user/root/channels"
+        ];
         description = ''
           The default Nix expression search path, used by the Nix
           evaluator to look up paths enclosed in angle brackets
@@ -484,45 +379,44 @@ in
         '';
       };
 
-      systemFeatures = mkOption {
-        type = types.listOf types.str;
-        example = [ "kvm" "big-parallel" "gccarch-skylake" ];
-        description = ''
-          The supported features of a machine
-        '';
-      };
-
       checkConfig = mkOption {
         type = types.bool;
         default = true;
         description = ''
-          If enabled (the default), checks that Nix can parse the generated nix.conf.
+          If enabled (the default), checks for data type mismatches and that Nix
+          can parse the generated nix.conf.
         '';
       };
 
       registry = mkOption {
         type = types.attrsOf (types.submodule (
           let
-            inputAttrs = types.attrsOf (types.oneOf [types.str types.int types.bool types.package]);
+            referenceAttrs = with types; attrsOf (oneOf [
+              str
+              int
+              bool
+              package
+            ]);
           in
           { config, name, ... }:
-          { options = {
+          {
+            options = {
               from = mkOption {
-                type = inputAttrs;
+                type = referenceAttrs;
                 example = { type = "indirect"; id = "nixpkgs"; };
                 description = "The flake reference to be rewritten.";
               };
               to = mkOption {
-                type = inputAttrs;
+                type = referenceAttrs;
                 example = { type = "github"; owner = "my-org"; repo = "my-nixpkgs"; };
-                description = "The flake reference to which <option>from></option> is to be rewritten.";
+                description = "The flake reference <option>from></option> is rewritten to.";
               };
               flake = mkOption {
                 type = types.nullOr types.attrs;
                 default = null;
                 example = literalExpression "nixpkgs";
                 description = ''
-                  The flake input to which <option>from></option> is to be rewritten.
+                  The flake input <option>from></option> is rewritten to.
                 '';
               };
               exact = mkOption {
@@ -537,35 +431,232 @@ in
             };
             config = {
               from = mkDefault { type = "indirect"; id = name; };
-              to = mkIf (config.flake != null)
-                ({ type = "path";
-                   path = config.flake.outPath;
-                 } // lib.filterAttrs
-                   (n: v: n == "lastModified" || n == "rev" || n == "revCount" || n == "narHash")
-                   config.flake);
+              to = mkIf (config.flake != null) (mkDefault
+                {
+                  type = "path";
+                  path = config.flake.outPath;
+                } // filterAttrs
+                (n: _: n == "lastModified" || n == "rev" || n == "revCount" || n == "narHash")
+                config.flake);
             };
           }
         ));
-        default = {};
+        default = { };
         description = ''
           A system-wide flake registry.
         '';
       };
 
-    };
+      extraOptions = mkOption {
+        type = types.lines;
+        default = "";
+        example = ''
+          keep-outputs = true
+          keep-derivations = true
+        '';
+        description = "Additional text appended to <filename>nix.conf</filename>.";
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = semanticConfType;
+
+          options = {
+            max-jobs = mkOption {
+              type = types.either types.int (types.enum [ "auto" ]);
+              default = "auto";
+              example = 64;
+              description = ''
+                This option defines the maximum number of jobs that Nix will try to
+                build in parallel. The default is auto, which means it will use all
+                available logical cores. It is recommend to set it to the total
+                number of logical cores in your system (e.g., 16 for two CPUs with 4
+                cores each and hyper-threading).
+              '';
+            };
+
+            auto-optimise-store = mkOption {
+              type = types.bool;
+              default = false;
+              example = true;
+              description = ''
+                If set to true, Nix automatically detects files in the store that have
+                identical contents, and replaces them with hard links to a single copy.
+                This saves disk space. If set to false (the default), you can still run
+                nix-store --optimise to get rid of duplicate files.
+              '';
+            };
+
+            cores = mkOption {
+              type = types.int;
+              default = 0;
+              example = 64;
+              description = ''
+                This option defines the maximum number of concurrent tasks during
+                one build. It affects, e.g., -j option for make.
+                The special value 0 means that the builder should use all
+                available CPU cores in the system. Some builds may become
+                non-deterministic with this option; use with care! Packages will
+                only be affected if enableParallelBuilding is set for them.
+              '';
+            };
+
+            sandbox = mkOption {
+              type = types.either types.bool (types.enum [ "relaxed" ]);
+              default = true;
+              description = ''
+                If set, Nix will perform builds in a sandboxed environment that it
+                will set up automatically for each build. This prevents impurities
+                in builds by disallowing access to dependencies outside of the Nix
+                store by using network and mount namespaces in a chroot environment.
+                This is enabled by default even though it has a possible performance
+                impact due to the initial setup time of a sandbox for each build. It
+                doesn't affect derivation hashes, so changing this option will not
+                trigger a rebuild of packages.
+              '';
+            };
+
+            extra-sandbox-paths = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              example = [ "/dev" "/proc" ];
+              description = ''
+                Directories from the host filesystem to be included
+                in the sandbox.
+              '';
+            };
+
+            substituters = mkOption {
+              type = types.listOf types.str;
+              description = ''
+                List of binary cache URLs used to obtain pre-built binaries
+                of Nix packages.
+
+                By default https://cache.nixos.org/ is added.
+              '';
+            };
+
+            trusted-substituters = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              example = [ "https://hydra.nixos.org/" ];
+              description = ''
+                List of binary cache URLs that non-root users can use (in
+                addition to those specified using
+                <option>nix.settings.substituters</option>) by passing
+                <literal>--option binary-caches</literal> to Nix commands.
+              '';
+            };
+
+            require-sigs = mkOption {
+              type = types.bool;
+              default = true;
+              description = ''
+                If enabled (the default), Nix will only download binaries from binary caches if
+                they are cryptographically signed with any of the keys listed in
+                <option>nix.settings.trusted-public-keys</option>. If disabled, signatures are neither
+                required nor checked, so it's strongly recommended that you use only
+                trustworthy caches and https to prevent man-in-the-middle attacks.
+              '';
+            };
+
+            trusted-public-keys = mkOption {
+              type = types.listOf types.str;
+              example = [ "hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=" ];
+              description = ''
+                List of public keys used to sign binary caches. If
+                <option>nix.settings.trusted-public-keys</option> is enabled,
+                then Nix will use a binary from a binary cache if and only
+                if it is signed by <emphasis>any</emphasis> of the keys
+                listed here. By default, only the key for
+                <uri>cache.nixos.org</uri> is included.
+              '';
+            };
+
+            trusted-users = mkOption {
+              type = types.listOf types.str;
+              default = [ "root" ];
+              example = [ "root" "alice" "@wheel" ];
+              description = ''
+                A list of names of users that have additional rights when
+                connecting to the Nix daemon, such as the ability to specify
+                additional binary caches, or to import unsigned NARs. You
+                can also specify groups by prefixing them with
+                <literal>@</literal>; for instance,
+                <literal>@wheel</literal> means all users in the wheel
+                group.
+              '';
+            };
 
+            system-features = mkOption {
+              type = types.listOf types.str;
+              example = [ "kvm" "big-parallel" "gccarch-skylake" ];
+              description = ''
+                The set of features supported by the machine. Derivations
+                can express dependencies on system features through the
+                <literal>requiredSystemFeatures</literal> attribute.
+
+                By default, pseudo-features <literal>nixos-test</literal>, <literal>benchmark</literal>,
+                and <literal>big-parallel</literal> used in Nixpkgs are set, <literal>kvm</literal>
+                is also included in it is avaliable.
+              '';
+            };
+
+            allowed-users = mkOption {
+              type = types.listOf types.str;
+              default = [ "*" ];
+              example = [ "@wheel" "@builders" "alice" "bob" ];
+              description = ''
+                A list of names of users (separated by whitespace) that are
+                allowed to connect to the Nix daemon. As with
+                <option>nix.settings.trusted-users</option>, you can specify groups by
+                prefixing them with <literal>@</literal>. Also, you can
+                allow all users by specifying <literal>*</literal>. The
+                default is <literal>*</literal>. Note that trusted users are
+                always allowed to connect.
+              '';
+            };
+          };
+        };
+        default = { };
+        example = literalExpression ''
+          {
+            use-sandbox = true;
+            show-trace = true;
+
+            system-features = [ "big-parallel" "kvm" "recursive-nix" ];
+            sandbox-paths = { "/bin/sh" = "''${pkgs.busybox-sandbox-shell.out}/bin/busybox"; };
+          }
+        '';
+        description = ''
+          Configuration for Nix, see
+          <link xlink:href="https://nixos.org/manual/nix/stable/#sec-conf-file"/> or
+          <citerefentry>
+            <refentrytitle>nix.conf</refentrytitle>
+            <manvolnum>5</manvolnum>
+          </citerefentry> for avalaible options.
+          The value declared here will be translated directly to the key-value pairs Nix expects.
+          </para>
+          <para>
+          You can use <command>nix-instantiate --eval --strict '&lt;nixpkgs/nixos&gt;' -A config.nix.settings</command>
+          to view the current value. By default it is empty.
+          </para>
+          <para>
+          Nix configurations defined under <option>nix.*</option> will be translated and applied to this
+          option. In addition, configuration specified in <option>nix.extraOptions</option> which will be appended
+          verbatim to the resulting config file.
+        '';
+      };
+    };
   };
 
 
   ###### implementation
 
   config = mkIf cfg.enable {
-
-    nix.binaryCachePublicKeys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ];
-    nix.binaryCaches = [ "https://cache.nixos.org/" ];
-
     environment.systemPackages =
-      [ nix
+      [
+        nixPackage
         pkgs.nix-info
       ]
       ++ optional (config.programs.bash.enableCompletion) pkgs.nix-bash-completions;
@@ -579,44 +670,49 @@ in
 
     # List of machines for distributed Nix builds in the format
     # expected by build-remote.pl.
-    environment.etc."nix/machines" =
-      { enable = cfg.buildMachines != [];
-        text =
-          concatMapStrings (machine:
-            "${if machine.sshUser != null then "${machine.sshUser}@" else ""}${machine.hostName} "
-            + (if machine.system != null then machine.system else concatStringsSep "," machine.systems)
-            + " ${if machine.sshKey != null then machine.sshKey else "-"} ${toString machine.maxJobs} "
-            + toString (machine.speedFactor)
-            + " "
-            + concatStringsSep "," (machine.mandatoryFeatures ++ machine.supportedFeatures)
-            + " "
-            + concatStringsSep "," machine.mandatoryFeatures
+    environment.etc."nix/machines" = mkIf (cfg.buildMachines != [ ]) {
+      text =
+        concatMapStrings
+          (machine:
+            (concatStringsSep " " ([
+              "${optionalString (machine.sshUser != null) "${machine.sshUser}@"}${machine.hostName}"
+              (if machine.system != null then machine.system else if machine.systems != [ ] then concatStringsSep "," machine.systems else "-")
+              (if machine.sshKey != null then machine.sshKey else "-")
+              (toString machine.maxJobs)
+              (toString machine.speedFactor)
+              (concatStringsSep "," (machine.supportedFeatures ++ machine.mandatoryFeatures))
+              (concatStringsSep "," machine.mandatoryFeatures)
+            ]
+            ++ optional (isNixAtLeast "2.4pre") (if machine.publicHostKey != null then machine.publicHostKey else "-")))
             + "\n"
-          ) cfg.buildMachines;
-      };
+          )
+          cfg.buildMachines;
+    };
+
     assertions =
-      let badMachine = m: m.system == null && m.systems == [];
-      in [
+      let badMachine = m: m.system == null && m.systems == [ ];
+      in
+      [
         {
-          assertion = !(builtins.any badMachine cfg.buildMachines);
+          assertion = !(any badMachine cfg.buildMachines);
           message = ''
             At least one system type (via <varname>system</varname> or
               <varname>systems</varname>) must be set for every build machine.
               Invalid machine specifications:
           '' + "      " +
-          (builtins.concatStringsSep "\n      "
-            (builtins.map (m: m.hostName)
-              (builtins.filter (badMachine) cfg.buildMachines)));
+          (concatStringsSep "\n      "
+            (map (m: m.hostName)
+              (filter (badMachine) cfg.buildMachines)));
         }
       ];
 
-
-    systemd.packages = [ nix ];
+    systemd.packages = [ nixPackage ];
 
     systemd.sockets.nix-daemon.wantedBy = [ "sockets.target" ];
 
     systemd.services.nix-daemon =
-      { path = [ nix pkgs.util-linux config.programs.ssh.package ]
+      {
+        path = [ nixPackage pkgs.util-linux config.programs.ssh.package ]
           ++ optionals cfg.distributedBuilds [ pkgs.gzip ];
 
         environment = cfg.envVars
@@ -626,19 +722,52 @@ in
         unitConfig.RequiresMountsFor = "/nix/store";
 
         serviceConfig =
-          { CPUSchedulingPolicy = cfg.daemonCPUSchedPolicy;
+          {
+            CPUSchedulingPolicy = cfg.daemonCPUSchedPolicy;
             IOSchedulingClass = cfg.daemonIOSchedClass;
             IOSchedulingPriority = cfg.daemonIOSchedPriority;
             LimitNOFILE = 4096;
           };
 
         restartTriggers = [ nixConf ];
+
+        # `stopIfChanged = false` changes to switch behavior
+        # from   stop -> update units -> start
+        #   to   update units -> restart
+        #
+        # The `stopIfChanged` setting therefore controls a trade-off between a
+        # more predictable lifecycle, which runs the correct "version" of
+        # the `ExecStop` line, and on the other hand the availability of
+        # sockets during the switch, as the effectiveness of the stop operation
+        # depends on the socket being stopped as well.
+        #
+        # As `nix-daemon.service` does not make use of `ExecStop`, we prefer
+        # to keep the socket up and available. This is important for machines
+        # that run Nix-based services, such as automated build, test, and deploy
+        # services, that expect the daemon socket to be available at all times.
+        #
+        # Notably, the Nix client does not retry on failure to connect to the
+        # daemon socket, and the in-process RemoteStore instance will disable
+        # itself. This makes retries infeasible even for services that are
+        # aware of the issue. Failure to connect can affect not only new client
+        # processes, but also new RemoteStore instances in existing processes,
+        # as well as existing RemoteStore instances that have not saturated
+        # their connection pool.
+        #
+        # Also note that `stopIfChanged = true` does not kill existing
+        # connection handling daemons, as one might wish to happen before a
+        # breaking Nix upgrade (which is rare). The daemon forks that handle
+        # the individual connections split off into their own sessions, causing
+        # them not to be stopped by systemd.
+        # If a Nix upgrade does require all existing daemon processes to stop,
+        # nix-daemon must do so on its own accord, and only when the new version
+        # starts and detects that Nix's persistent state needs an upgrade.
+        stopIfChanged = false;
+
       };
 
     # Set up the environment variables for running Nix.
-    environment.sessionVariables = cfg.envVars //
-      { NIX_PATH = cfg.nixPath;
-      };
+    environment.sessionVariables = cfg.envVars // { NIX_PATH = cfg.nixPath; };
 
     environment.extraInit =
       ''
@@ -647,7 +776,7 @@ in
         fi
       '';
 
-    nix.nrBuildUsers = mkDefault (lib.max 32 (if cfg.maxJobs == "auto" then 0 else cfg.maxJobs));
+    nix.nrBuildUsers = mkDefault (max 32 (if cfg.settings.max-jobs == "auto" then 0 else cfg.settings.max-jobs));
 
     users.users = nixbldUsers;
 
@@ -663,14 +792,26 @@ in
         fi
       '';
 
-    nix.systemFeatures = mkDefault (
-      [ "nixos-test" "benchmark" "big-parallel" "kvm" ] ++
-      optionals (pkgs.hostPlatform ? gcc.arch) (
-        # a builder can run code for `gcc.arch` and inferior architectures
-        [ "gccarch-${pkgs.hostPlatform.gcc.arch}" ] ++
-        map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${pkgs.hostPlatform.gcc.arch}
-      )
-    );
+    # Legacy configuration conversion.
+    nix.settings = mkMerge [
+      {
+        trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ];
+        substituters = mkAfter [ "https://cache.nixos.org/" ];
+
+        system-features = mkDefault (
+          [ "nixos-test" "benchmark" "big-parallel" "kvm" ] ++
+          optionals (pkgs.hostPlatform ? gcc.arch) (
+            # a builder can run code for `gcc.arch` and inferior architectures
+            [ "gccarch-${pkgs.hostPlatform.gcc.arch}" ] ++
+            map (x: "gccarch-${x}") systems.architectures.inferiors.${pkgs.hostPlatform.gcc.arch}
+          )
+        );
+      }
+
+      (mkIf (!cfg.distributedBuilds) { builders = null; })
+
+      (mkIf (isNixAtLeast "2.3pre") { sandbox-fallback = false; })
+    ];
 
   };
 
diff --git a/nixos/modules/services/misc/nix-ssh-serve.nix b/nixos/modules/services/misc/nix-ssh-serve.nix
index d5c64fdb2647..355fad5db468 100644
--- a/nixos/modules/services/misc/nix-ssh-serve.nix
+++ b/nixos/modules/services/misc/nix-ssh-serve.nix
@@ -20,7 +20,7 @@ in {
       write = mkOption {
         type = types.bool;
         default = false;
-        description = "Whether to enable writing to the Nix store as a remote store via SSH. Note: the sshServe user is named nix-ssh and is not a trusted-user. nix-ssh should be added to the nix.trustedUsers option in most use cases, such as allowing remote building of derivations.";
+        description = "Whether to enable writing to the Nix store as a remote store via SSH. Note: the sshServe user is named nix-ssh and is not a trusted-user. nix-ssh should be added to the <option>nix.settings.trusted-users</option> option in most use cases, such as allowing remote building of derivations.";
       };
 
       keys = mkOption {
diff --git a/nixos/modules/services/misc/packagekit.nix b/nixos/modules/services/misc/packagekit.nix
index 93bd206bd983..9191078ef9ca 100644
--- a/nixos/modules/services/misc/packagekit.nix
+++ b/nixos/modules/services/misc/packagekit.nix
@@ -13,7 +13,7 @@ let
     (iniFmt.generate "PackageKit.conf" (recursiveUpdate
       {
         Daemon = {
-          DefaultBackend = "test_nop";
+          DefaultBackend = "nix";
           KeepCache = false;
         };
       }
@@ -35,7 +35,7 @@ let
 in
 {
   imports = [
-    (mkRemovedOptionModule [ "services" "packagekit" "backend" ] "The only backend that doesn't blow up is `test_nop`.")
+    (mkRemovedOptionModule [ "services" "packagekit" "backend" ] "Always set to Nix.")
   ];
 
   options.services.packagekit = {
@@ -62,6 +62,8 @@ in
 
     services.dbus.packages = with pkgs; [ packagekit ];
 
+    environment.systemPackages = with pkgs; [ packagekit ];
+
     systemd.packages = with pkgs; [ packagekit ];
 
     environment.etc = listToAttrs (map
diff --git a/nixos/modules/services/misc/paperless-ng.nix b/nixos/modules/services/misc/paperless-ng.nix
index db8082f072c3..11e44f5ece57 100644
--- a/nixos/modules/services/misc/paperless-ng.nix
+++ b/nixos/modules/services/misc/paperless-ng.nix
@@ -6,12 +6,18 @@ let
 
   defaultUser = "paperless";
 
+  hasCustomRedis = hasAttr "PAPERLESS_REDIS" cfg.extraConfig;
+
   env = {
     PAPERLESS_DATA_DIR = cfg.dataDir;
     PAPERLESS_MEDIA_ROOT = cfg.mediaDir;
     PAPERLESS_CONSUMPTION_DIR = cfg.consumptionDir;
     GUNICORN_CMD_ARGS = "--bind=${cfg.address}:${toString cfg.port}";
-  } // lib.mapAttrs (_: toString) cfg.extraConfig;
+  } // (
+    lib.mapAttrs (_: toString) cfg.extraConfig
+  ) // (optionalAttrs (!hasCustomRedis) {
+    PAPERLESS_REDIS = "unix://${config.services.redis.servers.paperless-ng.unixSocket}";
+  });
 
   manage = let
     setupEnv = lib.concatStringsSep "\n" (mapAttrsToList (name: val: "export ${name}=\"${val}\"") env);
@@ -30,7 +36,7 @@ let
       "-/etc/hosts"
       "-/etc/localtime"
       "-/run/postgresql"
-    ];
+    ] ++ (optional (!hasCustomRedis) config.services.redis.servers.paperless-ng.unixSocket);
     BindPaths = [
       cfg.consumptionDir
       cfg.dataDir
@@ -44,8 +50,7 @@ let
     NoNewPrivileges = true;
     PrivateDevices = true;
     PrivateMounts = true;
-    # Needs to connect to redis
-    # PrivateNetwork = true;
+    PrivateNetwork = true;
     PrivateTmp = true;
     PrivateUsers = true;
     ProcSubset = "pid";
@@ -65,6 +70,7 @@ let
     RestrictNamespaces = true;
     RestrictRealtime = true;
     RestrictSUIDSGID = true;
+    SupplementaryGroups = optional (!hasCustomRedis) config.services.redis.servers.paperless-ng.user;
     SystemCallArchitectures = "native";
     SystemCallFilter = [ "@system-service" "~@privileged @resources @setuid @keyring" ];
     # Does not work well with the temporary root
@@ -190,7 +196,7 @@ in
 
   config = mkIf cfg.enable {
     # Enable redis if no special url is set
-    services.redis.enable = mkIf (!hasAttr "PAPERLESS_REDIS" env) true;
+    services.redis.servers.paperless-ng.enable = mkIf (!hasCustomRedis) true;
 
     systemd.tmpfiles.rules = [
       "d '${cfg.dataDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
@@ -208,6 +214,8 @@ in
         User = cfg.user;
         ExecStart = "${cfg.package}/bin/paperless-ng qcluster";
         Restart = "on-failure";
+        # The `mbind` syscall is needed for running the classifier.
+        SystemCallFilter = defaultServiceConfig.SystemCallFilter ++ [ "mbind" ];
       };
       environment = env;
       wantedBy = [ "multi-user.target" ];
@@ -234,6 +242,8 @@ in
           echo "$superuserState" > "$superuserStateFile"
         fi
       '';
+    } // optionalAttrs (!hasCustomRedis) {
+      after = [ "redis-paperless-ng.service" ];
     };
 
     # Password copying can't be implemented as a privileged preStart script
@@ -248,6 +258,8 @@ in
             '${cfg.passwordFile}' '${cfg.dataDir}/superuser-password'
         '';
         Type = "oneshot";
+        # Needs to talk to mail server for automated import rules
+        PrivateNetwork = false;
       };
     };
 
@@ -279,6 +291,8 @@ in
         CapabilityBoundingSet = "CAP_NET_BIND_SERVICE";
         # gunicorn needs setuid
         SystemCallFilter = defaultServiceConfig.SystemCallFilter ++ [ "@setuid" ];
+        # Needs to serve web page
+        PrivateNetwork = false;
       };
       environment = env // {
         PATH = mkForce cfg.package.path;
diff --git a/nixos/modules/services/misc/plex.nix b/nixos/modules/services/misc/plex.nix
index 2ae4e80d5c3f..1cd8da768f48 100644
--- a/nixos/modules/services/misc/plex.nix
+++ b/nixos/modules/services/misc/plex.nix
@@ -6,6 +6,10 @@ let
   cfg = config.services.plex;
 in
 {
+  imports = [
+    (mkRemovedOptionModule [ "services" "plex" "managePlugins" ] "Please omit or define the option: `services.plex.extraPlugins' instead.")
+  ];
+
   options = {
     services.plex = {
       enable = mkEnableOption "Plex Media Server";
@@ -42,16 +46,6 @@ in
         '';
       };
 
-      managePlugins = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          If set to true, this option will cause all of the symlinks in Plex's
-          plugin directory to be removed and symlinks for paths specified in
-          <option>extraPlugins</option> to be added.
-        '';
-      };
-
       extraPlugins = mkOption {
         type = types.listOf types.path;
         default = [];
@@ -59,9 +53,20 @@ in
           A list of paths to extra plugin bundles to install in Plex's plugin
           directory. Every time the systemd unit for Plex starts up, all of the
           symlinks in Plex's plugin directory will be cleared and this module
-          will symlink all of the paths specified here to that directory. If
-          this behavior is undesired, set <option>managePlugins</option> to
-          false.
+          will symlink all of the paths specified here to that directory.
+        '';
+        example = literalExpression ''
+          [
+            (builtins.path {
+              name = "Audnexus.bundle";
+              path = pkgs.fetchFromGitHub {
+                owner = "djdembeck";
+                repo = "Audnexus.bundle";
+                rev = "v0.2.8";
+                sha256 = "sha256-IWOSz3vYL7zhdHan468xNc6C/eQ2C2BukQlaJNLXh7E=";
+              };
+            })
+          ]
         '';
       };
 
diff --git a/nixos/modules/services/misc/rippled.nix b/nixos/modules/services/misc/rippled.nix
index 9c66df2fce1c..f6ec0677774b 100644
--- a/nixos/modules/services/misc/rippled.nix
+++ b/nixos/modules/services/misc/rippled.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.rippled;
+  opt = options.services.rippled;
 
   b2i = val: if val then "1" else "0";
 
@@ -165,6 +166,7 @@ let
         description = "Location to store the database.";
         type = types.path;
         default = cfg.databasePath;
+        defaultText = literalExpression "config.${opt.databasePath}";
       };
 
       compression = mkOption {
@@ -177,6 +179,7 @@ let
         description = "Enable automatic purging of older ledger information.";
         type = types.nullOr (types.addCheck types.int (v: v > 256));
         default = cfg.ledgerHistory;
+        defaultText = literalExpression "config.${opt.ledgerHistory}";
       };
 
       advisoryDelete = mkOption {
@@ -398,6 +401,7 @@ in
       config = mkOption {
         internal = true;
         default = pkgs.writeText "rippled.conf" rippledCfg;
+        defaultText = literalDocBook "generated config file";
       };
     };
   };
diff --git a/nixos/modules/services/misc/rmfakecloud.nix b/nixos/modules/services/misc/rmfakecloud.nix
new file mode 100644
index 000000000000..fe522653c216
--- /dev/null
+++ b/nixos/modules/services/misc/rmfakecloud.nix
@@ -0,0 +1,147 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.rmfakecloud;
+  serviceDataDir = "/var/lib/rmfakecloud";
+
+in {
+  options = {
+    services.rmfakecloud = {
+      enable = mkEnableOption "rmfakecloud remarkable self-hosted cloud";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.rmfakecloud;
+        defaultText = literalExpression "pkgs.rmfakecloud";
+        description = ''
+          rmfakecloud package to use.
+
+          The default does not include the web user interface.
+        '';
+      };
+
+      storageUrl = mkOption {
+        type = types.str;
+        example = "https://local.appspot.com";
+        description = ''
+          URL used by the tablet to access the rmfakecloud service.
+        '';
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 3000;
+        description = ''
+          Listening port number.
+        '';
+      };
+
+      logLevel = mkOption {
+        type = types.enum [ "info" "debug" "warn" "error" ];
+        default = "info";
+        description = ''
+          Logging level.
+        '';
+      };
+
+      extraSettings = mkOption {
+        type = with types; attrsOf str;
+        default = { };
+        example = { DATADIR = "/custom/path/for/rmfakecloud/data"; };
+        description = ''
+          Extra settings in the form of a set of key-value pairs.
+          For tokens and secrets, use `environmentFile` instead.
+
+          Available settings are listed on
+          https://ddvk.github.io/rmfakecloud/install/configuration/.
+        '';
+      };
+
+      environmentFile = mkOption {
+        type = with types; nullOr path;
+        default = null;
+        example = "/etc/secrets/rmfakecloud.env";
+        description = ''
+          Path to an environment file loaded for the rmfakecloud service.
+
+          This can be used to securely store tokens and secrets outside of the
+          world-readable Nix store. Since this file is read by systemd, it may
+          have permission 0400 and be owned by root.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.rmfakecloud = {
+      description = "rmfakecloud remarkable self-hosted cloud";
+
+      environment = {
+        STORAGE_URL = cfg.storageUrl;
+        PORT = toString cfg.port;
+        LOGLEVEL = cfg.logLevel;
+      } // cfg.extraSettings;
+
+      preStart = ''
+        # Generate the secret key used to sign client session tokens.
+        # Replacing it invalidates the previously established sessions.
+        if [ -z "$JWT_SECRET_KEY" ] && [ ! -f jwt_secret_key ]; then
+          (umask 077; touch jwt_secret_key)
+          cat /dev/urandom | tr -cd '[:alnum:]' | head -c 48 >> jwt_secret_key
+        fi
+      '';
+
+      script = ''
+        if [ -z "$JWT_SECRET_KEY" ]; then
+          export JWT_SECRET_KEY="$(cat jwt_secret_key)"
+        fi
+
+        ${cfg.package}/bin/rmfakecloud
+      '';
+
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+
+      serviceConfig = {
+        Type = "simple";
+        Restart = "always";
+
+        EnvironmentFile =
+          mkIf (cfg.environmentFile != null) cfg.environmentFile;
+
+        AmbientCapabilities =
+          mkIf (cfg.port < 1024) [ "CAP_NET_BIND_SERVICE" ];
+
+        DynamicUser = true;
+        PrivateDevices = true;
+        ProtectHome = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectControlGroups = true;
+        CapabilityBoundingSet = [ "" ];
+        DevicePolicy = "closed";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        ProtectClock = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectProc = "invisible";
+        ProcSubset = "pid";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        WorkingDirectory = serviceDataDir;
+        StateDirectory = baseNameOf serviceDataDir;
+        UMask = 0027;
+      };
+    };
+  };
+
+  meta.maintainers = with maintainers; [ pacien ];
+}
diff --git a/nixos/modules/services/misc/sickbeard.nix b/nixos/modules/services/misc/sickbeard.nix
index 8e871309c98e..a3db99286342 100644
--- a/nixos/modules/services/misc/sickbeard.nix
+++ b/nixos/modules/services/misc/sickbeard.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
@@ -7,6 +7,7 @@ let
   name = "sickbeard";
 
   cfg = config.services.sickbeard;
+  opt = options.services.sickbeard;
   sickbeard = cfg.package;
 
 in
@@ -39,6 +40,7 @@ in
       configFile = mkOption {
         type = types.path;
         default = "${cfg.dataDir}/config.ini";
+        defaultText = literalExpression ''"''${config.${opt.dataDir}}/config.ini"'';
         description = "Path to config file.";
       };
       port = mkOption {
diff --git a/nixos/modules/services/misc/sourcehut/builds.nix b/nixos/modules/services/misc/sourcehut/builds.nix
index f806e8c51b99..685a132d3507 100644
--- a/nixos/modules/services/misc/sourcehut/builds.nix
+++ b/nixos/modules/services/misc/sourcehut/builds.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   scfg = cfg.builds;
   rcfg = config.services.redis;
   iniKey = "builds.sr.ht";
@@ -38,6 +39,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/buildsrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/buildsrht"'';
       description = ''
         State path for builds.sr.ht.
       '';
@@ -61,7 +63,7 @@ in
               rev = "ff96a0fa5635770390b184ae74debea75c3fd534";
               ref = "nixos-unstable";
           };
-          image_from_nixpkgs = pkgs_unstable: (import ("${pkgs.sourcehut.buildsrht}/lib/images/nixos/image.nix") {
+          image_from_nixpkgs = pkgs_unstable: (import ("''${pkgs.sourcehut.buildsrht}/lib/images/nixos/image.nix") {
             pkgs = (import pkgs_unstable {});
           });
         in
diff --git a/nixos/modules/services/misc/sourcehut/default.nix b/nixos/modules/services/misc/sourcehut/default.nix
index c84a75b0ca02..21551d7d5f03 100644
--- a/nixos/modules/services/misc/sourcehut/default.nix
+++ b/nixos/modules/services/misc/sourcehut/default.nix
@@ -1,14 +1,90 @@
 { config, pkgs, lib, ... }:
-
 with lib;
 let
+  inherit (config.services) nginx postfix postgresql redis;
+  inherit (config.users) users groups;
   cfg = config.services.sourcehut;
-  cfgIni = cfg.settings;
-  settingsFormat = pkgs.formats.ini { };
+  domain = cfg.settings."sr.ht".global-domain;
+  settingsFormat = pkgs.formats.ini {
+    listToValue = concatMapStringsSep "," (generators.mkValueStringDefault {});
+    mkKeyValue = k: v:
+      if v == null then ""
+      else generators.mkKeyValueDefault {
+        mkValueString = v:
+          if v == true then "yes"
+          else if v == false then "no"
+          else generators.mkValueStringDefault {} v;
+      } "=" k v;
+  };
+  configIniOfService = srv: settingsFormat.generate "sourcehut-${srv}-config.ini"
+    # Each service needs access to only a subset of sections (and secrets).
+    (filterAttrs (k: v: v != null)
+    (mapAttrs (section: v:
+      let srvMatch = builtins.match "^([a-z]*)\\.sr\\.ht(::.*)?$" section; in
+      if srvMatch == null # Include sections shared by all services
+      || head srvMatch == srv # Include sections for the service being configured
+      then v
+      # Enable Web links and integrations between services.
+      else if tail srvMatch == [ null ] && elem (head srvMatch) cfg.services
+      then {
+        inherit (v) origin;
+        # mansrht crashes without it
+        oauth-client-id = v.oauth-client-id or null;
+      }
+      # Drop sub-sections of other services
+      else null)
+    (recursiveUpdate cfg.settings {
+      # Those paths are mounted using BindPaths= or BindReadOnlyPaths=
+      # for services needing access to them.
+      "builds.sr.ht::worker".buildlogs = "/var/log/sourcehut/buildsrht-worker";
+      "git.sr.ht".post-update-script = "/usr/bin/gitsrht-update-hook";
+      "git.sr.ht".repos = "/var/lib/sourcehut/gitsrht/repos";
+      "hg.sr.ht".changegroup-script = "/usr/bin/hgsrht-hook-changegroup";
+      "hg.sr.ht".repos = "/var/lib/sourcehut/hgsrht/repos";
+      # Making this a per service option despite being in a global section,
+      # so that it uses the redis-server used by the service.
+      "sr.ht".redis-host = cfg.${srv}.redis.host;
+    })));
+  commonServiceSettings = srv: {
+    origin = mkOption {
+      description = "URL ${srv}.sr.ht is being served at (protocol://domain)";
+      type = types.str;
+      default = "https://${srv}.${domain}";
+      defaultText = "https://${srv}.example.com";
+    };
+    debug-host = mkOption {
+      description = "Address to bind the debug server to.";
+      type = with types; nullOr str;
+      default = null;
+    };
+    debug-port = mkOption {
+      description = "Port to bind the debug server to.";
+      type = with types; nullOr str;
+      default = null;
+    };
+    connection-string = mkOption {
+      description = "SQLAlchemy connection string for the database.";
+      type = types.str;
+      default = "postgresql:///localhost?user=${srv}srht&host=/run/postgresql";
+    };
+    migrate-on-upgrade = mkEnableOption "automatic migrations on package upgrade" // { default = true; };
+    oauth-client-id = mkOption {
+      description = "${srv}.sr.ht's OAuth client id for meta.sr.ht.";
+      type = types.str;
+    };
+    oauth-client-secret = mkOption {
+      description = "${srv}.sr.ht's OAuth client secret for meta.sr.ht.";
+      type = types.path;
+      apply = s: "<" + toString s;
+    };
+  };
 
   # Specialized python containing all the modules
   python = pkgs.sourcehut.python.withPackages (ps: with ps; [
     gunicorn
+    eventlet
+    # For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=5 flower
+    flower
     # Sourcehut services
     srht
     buildsrht
@@ -19,72 +95,37 @@ let
     listssrht
     mansrht
     metasrht
+    # Not a python package
+    #pagessrht
     pastesrht
     todosrht
   ]);
+  mkOptionNullOrStr = description: mkOption {
+    inherit description;
+    type = with types; nullOr str;
+    default = null;
+  };
 in
 {
-  imports =
-    [
-      ./git.nix
-      ./hg.nix
-      ./hub.nix
-      ./todo.nix
-      ./man.nix
-      ./meta.nix
-      ./paste.nix
-      ./builds.nix
-      ./lists.nix
-      ./dispatch.nix
-      (mkRemovedOptionModule [ "services" "sourcehut" "nginx" "enable" ] ''
-        The sourcehut module supports `nginx` as a local reverse-proxy by default and doesn't
-        support other reverse-proxies officially.
-
-        However it's possible to use an alternative reverse-proxy by
-
-          * disabling nginx
-          * adjusting the relevant settings for server addresses and ports directly
-
-        Further details about this can be found in the `Sourcehut`-section of the NixOS-manual.
-      '')
-    ];
-
   options.services.sourcehut = {
-    enable = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Enable sourcehut - git hosting, continuous integration, mailing list, ticket tracking,
-        task dispatching, wiki and account management services
-      '';
-    };
+    enable = mkEnableOption ''
+      sourcehut - git hosting, continuous integration, mailing list, ticket tracking,
+      task dispatching, wiki and account management services
+    '';
 
     services = mkOption {
-      type = types.nonEmptyListOf (types.enum [ "builds" "dispatch" "git" "hub" "hg" "lists" "man" "meta" "paste" "todo" ]);
-      default = [ "man" "meta" "paste" ];
-      example = [ "builds" "dispatch" "git" "hub" "hg" "lists" "man" "meta" "paste" "todo" ];
+      type = with types; listOf (enum
+        [ "builds" "dispatch" "git" "hg" "hub" "lists" "man" "meta" "pages" "paste" "todo" ]);
+      defaultText = "locally enabled services";
       description = ''
-        Services to enable on the sourcehut network.
+        Services that may be displayed as links in the title bar of the Web interface.
       '';
     };
 
-    originBase = mkOption {
+    listenAddress = mkOption {
       type = types.str;
-      default = with config.networking; hostName + lib.optionalString (domain != null) ".${domain}";
-      defaultText = literalExpression ''
-        with config.networking; hostName + optionalString (domain != null) ".''${domain}"
-      '';
-      description = ''
-        Host name used by reverse-proxy and for default settings. Will host services at git."''${originBase}". For example: git.sr.ht
-      '';
-    };
-
-    address = mkOption {
-      type = types.str;
-      default = "127.0.0.1";
-      description = ''
-        Address to bind to.
-      '';
+      default = "localhost";
+      description = "Address to bind to.";
     };
 
     python = mkOption {
@@ -97,105 +138,1249 @@ in
       '';
     };
 
-    statePath = mkOption {
-      type = types.path;
-      default = "/var/lib/sourcehut";
-      description = ''
-        Root state path for the sourcehut network. If left as the default value
-        this directory will automatically be created before the sourcehut server
-        starts, otherwise the sysadmin is responsible for ensuring the
-        directory exists with appropriate ownership and permissions.
-      '';
+    minio = {
+      enable = mkEnableOption ''local minio integration'';
+    };
+
+    nginx = {
+      enable = mkEnableOption ''local nginx integration'';
+      virtualHost = mkOption {
+        type = types.attrs;
+        default = {};
+        description = "Virtual-host configuration merged with all Sourcehut's virtual-hosts.";
+      };
+    };
+
+    postfix = {
+      enable = mkEnableOption ''local postfix integration'';
+    };
+
+    postgresql = {
+      enable = mkEnableOption ''local postgresql integration'';
+    };
+
+    redis = {
+      enable = mkEnableOption ''local redis integration in a dedicated redis-server'';
     };
 
     settings = mkOption {
       type = lib.types.submodule {
         freeformType = settingsFormat.type;
+        options."sr.ht" = {
+          global-domain = mkOption {
+            description = "Global domain name.";
+            type = types.str;
+            example = "example.com";
+          };
+          environment = mkOption {
+            description = "Values other than \"production\" adds a banner to each page.";
+            type = types.enum [ "development" "production" ];
+            default = "development";
+          };
+          network-key = mkOption {
+            description = ''
+              An absolute file path (which should be outside the Nix-store)
+              to a secret key to encrypt internal messages with. Use <code>srht-keygen network</code> to
+              generate this key. It must be consistent between all services and nodes.
+            '';
+            type = types.path;
+            apply = s: "<" + toString s;
+          };
+          owner-email = mkOption {
+            description = "Owner's email.";
+            type = types.str;
+            default = "contact@example.com";
+          };
+          owner-name = mkOption {
+            description = "Owner's name.";
+            type = types.str;
+            default = "John Doe";
+          };
+          site-blurb = mkOption {
+            description = "Blurb for your site.";
+            type = types.str;
+            default = "the hacker's forge";
+          };
+          site-info = mkOption {
+            description = "The top-level info page for your site.";
+            type = types.str;
+            default = "https://sourcehut.org";
+          };
+          service-key = mkOption {
+            description = ''
+              An absolute file path (which should be outside the Nix-store)
+              to a key used for encrypting session cookies. Use <code>srht-keygen service</code> to
+              generate the service key. This must be shared between each node of the same
+              service (e.g. git1.sr.ht and git2.sr.ht), but different services may use
+              different keys. If you configure all of your services with the same
+              config.ini, you may use the same service-key for all of them.
+            '';
+            type = types.path;
+            apply = s: "<" + toString s;
+          };
+          site-name = mkOption {
+            description = "The name of your network of sr.ht-based sites.";
+            type = types.str;
+            default = "sourcehut";
+          };
+          source-url = mkOption {
+            description = "The source code for your fork of sr.ht.";
+            type = types.str;
+            default = "https://git.sr.ht/~sircmpwn/srht";
+          };
+        };
+        options.mail = {
+          smtp-host = mkOptionNullOrStr "Outgoing SMTP host.";
+          smtp-port = mkOption {
+            description = "Outgoing SMTP port.";
+            type = with types; nullOr port;
+            default = null;
+          };
+          smtp-user = mkOptionNullOrStr "Outgoing SMTP user.";
+          smtp-password = mkOptionNullOrStr "Outgoing SMTP password.";
+          smtp-from = mkOptionNullOrStr "Outgoing SMTP FROM.";
+          error-to = mkOptionNullOrStr "Address receiving application exceptions";
+          error-from = mkOptionNullOrStr "Address sending application exceptions";
+          pgp-privkey = mkOptionNullOrStr ''
+            An absolute file path (which should be outside the Nix-store)
+            to an OpenPGP private key.
+
+            Your PGP key information (DO NOT mix up pub and priv here)
+            You must remove the password from your secret key, if present.
+            You can do this with <code>gpg --edit-key [key-id]</code>,
+            then use the <code>passwd</code> command and do not enter a new password.
+          '';
+          pgp-pubkey = mkOptionNullOrStr "OpenPGP public key.";
+          pgp-key-id = mkOptionNullOrStr "OpenPGP key identifier.";
+        };
+        options.objects = {
+          s3-upstream = mkOption {
+            description = "Configure the S3-compatible object storage service.";
+            type = with types; nullOr str;
+            default = null;
+          };
+          s3-access-key = mkOption {
+            description = "Access key to the S3-compatible object storage service";
+            type = with types; nullOr str;
+            default = null;
+          };
+          s3-secret-key = mkOption {
+            description = ''
+              An absolute file path (which should be outside the Nix-store)
+              to the secret key of the S3-compatible object storage service.
+            '';
+            type = with types; nullOr path;
+            default = null;
+            apply = mapNullable (s: "<" + toString s);
+          };
+        };
+        options.webhooks = {
+          private-key = mkOption {
+            description = ''
+              An absolute file path (which should be outside the Nix-store)
+              to a base64-encoded Ed25519 key for signing webhook payloads.
+              This should be consistent for all *.sr.ht sites,
+              as this key will be used to verify signatures
+              from other sites in your network.
+              Use the <code>srht-keygen webhook</code> command to generate a key.
+            '';
+            type = types.path;
+            apply = s: "<" + toString s;
+          };
+        };
+
+        options."dispatch.sr.ht" = commonServiceSettings "dispatch" // {
+        };
+        options."dispatch.sr.ht::github" = {
+          oauth-client-id = mkOptionNullOrStr "OAuth client id.";
+          oauth-client-secret = mkOptionNullOrStr "OAuth client secret.";
+        };
+        options."dispatch.sr.ht::gitlab" = {
+          enabled = mkEnableOption "GitLab integration";
+          canonical-upstream = mkOption {
+            type = types.str;
+            description = "Canonical upstream.";
+            default = "gitlab.com";
+          };
+          repo-cache = mkOption {
+            type = types.str;
+            description = "Repository cache directory.";
+            default = "./repo-cache";
+          };
+          "gitlab.com" = mkOption {
+            type = with types; nullOr str;
+            description = "GitLab id and secret.";
+            default = null;
+            example = "GitLab:application id:secret";
+          };
+        };
+
+        options."builds.sr.ht" = commonServiceSettings "builds" // {
+          allow-free = mkEnableOption "nonpaying users to submit builds";
+          redis = mkOption {
+            description = "The Redis connection used for the Celery worker.";
+            type = types.str;
+            default = "redis+socket:///run/redis-sourcehut-buildsrht/redis.sock?virtual_host=2";
+          };
+          shell = mkOption {
+            description = ''
+              Scripts used to launch on SSH connection.
+              <literal>/usr/bin/master-shell</literal> on master,
+              <literal>/usr/bin/runner-shell</literal> on runner.
+              If master and worker are on the same system
+              set to <literal>/usr/bin/runner-shell</literal>.
+            '';
+            type = types.enum ["/usr/bin/master-shell" "/usr/bin/runner-shell"];
+            default = "/usr/bin/master-shell";
+          };
+        };
+        options."builds.sr.ht::worker" = {
+          bind-address = mkOption {
+            description = ''
+              HTTP bind address for serving local build information/monitoring.
+            '';
+            type = types.str;
+            default = "localhost:8080";
+          };
+          buildlogs = mkOption {
+            description = "Path to write build logs.";
+            type = types.str;
+            default = "/var/log/sourcehut/buildsrht-worker";
+          };
+          name = mkOption {
+            description = ''
+              Listening address and listening port
+              of the build runner (with HTTP port if not 80).
+            '';
+            type = types.str;
+            default = "localhost:5020";
+          };
+          timeout = mkOption {
+            description = ''
+              Max build duration.
+              See <link xlink:href="https://golang.org/pkg/time/#ParseDuration"/>.
+            '';
+            type = types.str;
+            default = "3m";
+          };
+        };
+
+        options."git.sr.ht" = commonServiceSettings "git" // {
+          outgoing-domain = mkOption {
+            description = "Outgoing domain.";
+            type = types.str;
+            default = "https://git.localhost.localdomain";
+          };
+          post-update-script = mkOption {
+            description = ''
+              A post-update script which is installed in every git repo.
+              This setting is propagated to newer and existing repositories.
+            '';
+            type = types.path;
+            default = "${pkgs.sourcehut.gitsrht}/bin/gitsrht-update-hook";
+            defaultText = "\${pkgs.sourcehut.gitsrht}/bin/gitsrht-update-hook";
+          };
+          repos = mkOption {
+            description = ''
+              Path to git repositories on disk.
+              If changing the default, you must ensure that
+              the gitsrht's user as read and write access to it.
+            '';
+            type = types.str;
+            default = "/var/lib/sourcehut/gitsrht/repos";
+          };
+          webhooks = mkOption {
+            description = "The Redis connection used for the webhooks worker.";
+            type = types.str;
+            default = "redis+socket:///run/redis-sourcehut-gitsrht/redis.sock?virtual_host=1";
+          };
+        };
+        options."git.sr.ht::api" = {
+          internal-ipnet = mkOption {
+            description = ''
+              Set of IP subnets which are permitted to utilize internal API
+              authentication. This should be limited to the subnets
+              from which your *.sr.ht services are running.
+              See <xref linkend="opt-services.sourcehut.listenAddress"/>.
+            '';
+            type = with types; listOf str;
+            default = [ "127.0.0.0/8" "::1/128" ];
+          };
+        };
+
+        options."hg.sr.ht" = commonServiceSettings "hg" // {
+          changegroup-script = mkOption {
+            description = ''
+              A changegroup script which is installed in every mercurial repo.
+              This setting is propagated to newer and existing repositories.
+            '';
+            type = types.str;
+            default = "${cfg.python}/bin/hgsrht-hook-changegroup";
+            defaultText = "\${cfg.python}/bin/hgsrht-hook-changegroup";
+          };
+          repos = mkOption {
+            description = ''
+              Path to mercurial repositories on disk.
+              If changing the default, you must ensure that
+              the hgsrht's user as read and write access to it.
+            '';
+            type = types.str;
+            default = "/var/lib/sourcehut/hgsrht/repos";
+          };
+          srhtext = mkOptionNullOrStr ''
+            Path to the srht mercurial extension
+            (defaults to where the hgsrht code is)
+          '';
+          clone_bundle_threshold = mkOption {
+            description = ".hg/store size (in MB) past which the nightly job generates clone bundles.";
+            type = types.ints.unsigned;
+            default = 50;
+          };
+          hg_ssh = mkOption {
+            description = "Path to hg-ssh (if not in $PATH).";
+            type = types.str;
+            default = "${pkgs.mercurial}/bin/hg-ssh";
+            defaultText = "\${pkgs.mercurial}/bin/hg-ssh";
+          };
+          webhooks = mkOption {
+            description = "The Redis connection used for the webhooks worker.";
+            type = types.str;
+            default = "redis+socket:///run/redis-sourcehut-hgsrht/redis.sock?virtual_host=1";
+          };
+        };
+
+        options."hub.sr.ht" = commonServiceSettings "hub" // {
+        };
+
+        options."lists.sr.ht" = commonServiceSettings "lists" // {
+          allow-new-lists = mkEnableOption "Allow creation of new lists.";
+          notify-from = mkOption {
+            description = "Outgoing email for notifications generated by users.";
+            type = types.str;
+            default = "lists-notify@localhost.localdomain";
+          };
+          posting-domain = mkOption {
+            description = "Posting domain.";
+            type = types.str;
+            default = "lists.localhost.localdomain";
+          };
+          redis = mkOption {
+            description = "The Redis connection used for the Celery worker.";
+            type = types.str;
+            default = "redis+socket:///run/redis-sourcehut-listssrht/redis.sock?virtual_host=2";
+          };
+          webhooks = mkOption {
+            description = "The Redis connection used for the webhooks worker.";
+            type = types.str;
+            default = "redis+socket:///run/redis-sourcehut-listssrht/redis.sock?virtual_host=1";
+          };
+        };
+        options."lists.sr.ht::worker" = {
+          reject-mimetypes = mkOption {
+            description = ''
+              Comma-delimited list of Content-Types to reject. Messages with Content-Types
+              included in this list are rejected. Multipart messages are always supported,
+              and each part is checked against this list.
+
+              Uses fnmatch for wildcard expansion.
+            '';
+            type = with types; listOf str;
+            default = ["text/html"];
+          };
+          reject-url = mkOption {
+            description = "Reject URL.";
+            type = types.str;
+            default = "https://man.sr.ht/lists.sr.ht/etiquette.md";
+          };
+          sock = mkOption {
+            description = ''
+              Path for the lmtp daemon's unix socket. Direct incoming mail to this socket.
+              Alternatively, specify IP:PORT and an SMTP server will be run instead.
+            '';
+            type = types.str;
+            default = "/tmp/lists.sr.ht-lmtp.sock";
+          };
+          sock-group = mkOption {
+            description = ''
+              The lmtp daemon will make the unix socket group-read/write
+              for users in this group.
+            '';
+            type = types.str;
+            default = "postfix";
+          };
+        };
+
+        options."man.sr.ht" = commonServiceSettings "man" // {
+        };
+
+        options."meta.sr.ht" =
+          removeAttrs (commonServiceSettings "meta")
+            ["oauth-client-id" "oauth-client-secret"] // {
+          api-origin = mkOption {
+            description = "Origin URL for API, 100 more than web.";
+            type = types.str;
+            default = "http://${cfg.listenAddress}:${toString (cfg.meta.port + 100)}";
+            defaultText = ''http://<xref linkend="opt-services.sourcehut.listenAddress"/>:''${toString (<xref linkend="opt-services.sourcehut.meta.port"/> + 100)}'';
+          };
+          webhooks = mkOption {
+            description = "The Redis connection used for the webhooks worker.";
+            type = types.str;
+            default = "redis+socket:///run/redis-sourcehut-metasrht/redis.sock?virtual_host=1";
+          };
+          welcome-emails = mkEnableOption "sending stock sourcehut welcome emails after signup";
+        };
+        options."meta.sr.ht::api" = {
+          internal-ipnet = mkOption {
+            description = ''
+              Set of IP subnets which are permitted to utilize internal API
+              authentication. This should be limited to the subnets
+              from which your *.sr.ht services are running.
+              See <xref linkend="opt-services.sourcehut.listenAddress"/>.
+            '';
+            type = with types; listOf str;
+            default = [ "127.0.0.0/8" "::1/128" ];
+          };
+        };
+        options."meta.sr.ht::aliases" = mkOption {
+          description = "Aliases for the client IDs of commonly used OAuth clients.";
+          type = with types; attrsOf int;
+          default = {};
+          example = { "git.sr.ht" = 12345; };
+        };
+        options."meta.sr.ht::billing" = {
+          enabled = mkEnableOption "the billing system";
+          stripe-public-key = mkOptionNullOrStr "Public key for Stripe. Get your keys at https://dashboard.stripe.com/account/apikeys";
+          stripe-secret-key = mkOptionNullOrStr ''
+            An absolute file path (which should be outside the Nix-store)
+            to a secret key for Stripe. Get your keys at https://dashboard.stripe.com/account/apikeys
+          '' // {
+            apply = mapNullable (s: "<" + toString s);
+          };
+        };
+        options."meta.sr.ht::settings" = {
+          registration = mkEnableOption "public registration";
+          onboarding-redirect = mkOption {
+            description = "Where to redirect new users upon registration.";
+            type = types.str;
+            default = "https://meta.localhost.localdomain";
+          };
+          user-invites = mkOption {
+            description = ''
+              How many invites each user is issued upon registration
+              (only applicable if open registration is disabled).
+            '';
+            type = types.ints.unsigned;
+            default = 5;
+          };
+        };
+
+        options."pages.sr.ht" = commonServiceSettings "pages" // {
+          gemini-certs = mkOption {
+            description = ''
+              An absolute file path (which should be outside the Nix-store)
+              to Gemini certificates.
+            '';
+            type = with types; nullOr path;
+            default = null;
+          };
+          max-site-size = mkOption {
+            description = "Maximum size of any given site (post-gunzip), in MiB.";
+            type = types.int;
+            default = 1024;
+          };
+          user-domain = mkOption {
+            description = ''
+              Configures the user domain, if enabled.
+              All users are given &lt;username&gt;.this.domain.
+            '';
+            type = with types; nullOr str;
+            default = null;
+          };
+        };
+        options."pages.sr.ht::api" = {
+          internal-ipnet = mkOption {
+            description = ''
+              Set of IP subnets which are permitted to utilize internal API
+              authentication. This should be limited to the subnets
+              from which your *.sr.ht services are running.
+              See <xref linkend="opt-services.sourcehut.listenAddress"/>.
+            '';
+            type = with types; listOf str;
+            default = [ "127.0.0.0/8" "::1/128" ];
+          };
+        };
+
+        options."paste.sr.ht" = commonServiceSettings "paste" // {
+        };
+
+        options."todo.sr.ht" = commonServiceSettings "todo" // {
+          notify-from = mkOption {
+            description = "Outgoing email for notifications generated by users.";
+            type = types.str;
+            default = "todo-notify@localhost.localdomain";
+          };
+          webhooks = mkOption {
+            description = "The Redis connection used for the webhooks worker.";
+            type = types.str;
+            default = "redis+socket:///run/redis-sourcehut-todosrht/redis.sock?virtual_host=1";
+          };
+        };
+        options."todo.sr.ht::mail" = {
+          posting-domain = mkOption {
+            description = "Posting domain.";
+            type = types.str;
+            default = "todo.localhost.localdomain";
+          };
+          sock = mkOption {
+            description = ''
+              Path for the lmtp daemon's unix socket. Direct incoming mail to this socket.
+              Alternatively, specify IP:PORT and an SMTP server will be run instead.
+            '';
+            type = types.str;
+            default = "/tmp/todo.sr.ht-lmtp.sock";
+          };
+          sock-group = mkOption {
+            description = ''
+              The lmtp daemon will make the unix socket group-read/write
+              for users in this group.
+            '';
+            type = types.str;
+            default = "postfix";
+          };
+        };
       };
       default = { };
       description = ''
         The configuration for the sourcehut network.
       '';
     };
+
+    builds = {
+      enableWorker = mkEnableOption ''
+        worker for builds.sr.ht
+
+        <warning><para>
+        For smaller deployments, job runners can be installed alongside the master server
+        but even if you only build your own software, integration with other services
+        may cause you to run untrusted builds
+        (e.g. automatic testing of patches via listssrht).
+        See <link xlink:href="https://man.sr.ht/builds.sr.ht/configuration.md#security-model"/>.
+        </para></warning>
+      '';
+
+      images = mkOption {
+        type = with types; attrsOf (attrsOf (attrsOf package));
+        default = { };
+        example = lib.literalExpression ''(let
+            # Pinning unstable to allow usage with flakes and limit rebuilds.
+            pkgs_unstable = builtins.fetchGit {
+                url = "https://github.com/NixOS/nixpkgs";
+                rev = "ff96a0fa5635770390b184ae74debea75c3fd534";
+                ref = "nixos-unstable";
+            };
+            image_from_nixpkgs = (import ("''${pkgs.sourcehut.buildsrht}/lib/images/nixos/image.nix") {
+              pkgs = (import pkgs_unstable {});
+            });
+          in
+          {
+            nixos.unstable.x86_64 = image_from_nixpkgs;
+          }
+        )'';
+        description = ''
+          Images for builds.sr.ht. Each package should be distro.release.arch and point to a /nix/store/package/root.img.qcow2.
+        '';
+      };
+    };
+
+    git = {
+      package = mkOption {
+        type = types.package;
+        default = pkgs.git;
+        defaultText = literalExpression "pkgs.git";
+        example = literalExpression "pkgs.gitFull";
+        description = ''
+          Git package for git.sr.ht. This can help silence collisions.
+        '';
+      };
+      fcgiwrap.preforkProcess = mkOption {
+        description = "Number of fcgiwrap processes to prefork.";
+        type = types.int;
+        default = 4;
+      };
+    };
+
+    hg = {
+      package = mkOption {
+        type = types.package;
+        default = pkgs.mercurial;
+        defaultText = literalExpression "pkgs.mercurial";
+        description = ''
+          Mercurial package for hg.sr.ht. This can help silence collisions.
+        '';
+      };
+      cloneBundles = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Generate clonebundles (which require more disk space but dramatically speed up cloning large repositories).
+        '';
+      };
+    };
+
+    lists = {
+      process = {
+        extraArgs = mkOption {
+          type = with types; listOf str;
+          default = [ "--loglevel DEBUG" "--pool eventlet" "--without-heartbeat" ];
+          description = "Extra arguments passed to the Celery responsible for processing mails.";
+        };
+        celeryConfig = mkOption {
+          type = types.lines;
+          default = "";
+          description = "Content of the <literal>celeryconfig.py</literal> used by the Celery of <literal>listssrht-process</literal>.";
+        };
+      };
+    };
   };
 
-  config = mkIf cfg.enable {
-    assertions =
-      [
+  config = mkIf cfg.enable (mkMerge [
+    {
+      environment.systemPackages = [ pkgs.sourcehut.coresrht ];
+
+      services.sourcehut.settings = {
+        "git.sr.ht".outgoing-domain = mkDefault "https://git.${domain}";
+        "lists.sr.ht".notify-from = mkDefault "lists-notify@${domain}";
+        "lists.sr.ht".posting-domain = mkDefault "lists.${domain}";
+        "meta.sr.ht::settings".onboarding-redirect = mkDefault "https://meta.${domain}";
+        "todo.sr.ht".notify-from = mkDefault "todo-notify@${domain}";
+        "todo.sr.ht::mail".posting-domain = mkDefault "todo.${domain}";
+      };
+    }
+    (mkIf cfg.postgresql.enable {
+      assertions = [
+        { assertion = postgresql.enable;
+          message = "postgresql must be enabled and configured";
+        }
+      ];
+    })
+    (mkIf cfg.postfix.enable {
+      assertions = [
+        { assertion = postfix.enable;
+          message = "postfix must be enabled and configured";
+        }
+      ];
+      # Needed for sharing the LMTP sockets with JoinsNamespaceOf=
+      systemd.services.postfix.serviceConfig.PrivateTmp = true;
+    })
+    (mkIf cfg.redis.enable {
+      services.redis.vmOverCommit = mkDefault true;
+    })
+    (mkIf cfg.nginx.enable {
+      assertions = [
+        { assertion = nginx.enable;
+          message = "nginx must be enabled and configured";
+        }
+      ];
+      # For proxyPass= in virtual-hosts for Sourcehut services.
+      services.nginx.recommendedProxySettings = mkDefault true;
+    })
+    (mkIf (cfg.builds.enable || cfg.git.enable || cfg.hg.enable) {
+      services.openssh = {
+        # Note that sshd will continue to honor AuthorizedKeysFile.
+        # Note that you may want automatically rotate
+        # or link to /dev/null the following log files:
+        # - /var/log/gitsrht-dispatch
+        # - /var/log/{build,git,hg}srht-keys
+        # - /var/log/{git,hg}srht-shell
+        # - /var/log/gitsrht-update-hook
+        authorizedKeysCommand = ''/etc/ssh/sourcehut/subdir/srht-dispatch "%u" "%h" "%t" "%k"'';
+        # srht-dispatch will setuid/setgid according to [git.sr.ht::dispatch]
+        authorizedKeysCommandUser = "root";
+        extraConfig = ''
+          PermitUserEnvironment SRHT_*
+        '';
+      };
+      environment.etc."ssh/sourcehut/config.ini".source =
+        settingsFormat.generate "sourcehut-dispatch-config.ini"
+          (filterAttrs (k: v: k == "git.sr.ht::dispatch")
+          cfg.settings);
+      environment.etc."ssh/sourcehut/subdir/srht-dispatch" = {
+        # sshd_config(5): The program must be owned by root, not writable by group or others
+        mode = "0755";
+        source = pkgs.writeShellScript "srht-dispatch" ''
+          set -e
+          cd /etc/ssh/sourcehut/subdir
+          ${cfg.python}/bin/gitsrht-dispatch "$@"
+        '';
+      };
+      systemd.services.sshd = {
+        #path = optional cfg.git.enable [ cfg.git.package ];
+        serviceConfig = {
+          BindReadOnlyPaths =
+            # Note that those /usr/bin/* paths are hardcoded in multiple places in *.sr.ht,
+            # for instance to get the user from the [git.sr.ht::dispatch] settings.
+            # *srht-keys needs to:
+            # - access a redis-server in [sr.ht] redis-host,
+            # - access the PostgreSQL server in [*.sr.ht] connection-string,
+            # - query metasrht-api (through the HTTP API).
+            # Using this has the side effect of creating empty files in /usr/bin/
+            optionals cfg.builds.enable [
+              "${pkgs.writeShellScript "buildsrht-keys-wrapper" ''
+                set -e
+                cd /run/sourcehut/buildsrht/subdir
+                set -x
+                exec -a "$0" ${pkgs.sourcehut.buildsrht}/bin/buildsrht-keys "$@"
+              ''}:/usr/bin/buildsrht-keys"
+              "${pkgs.sourcehut.buildsrht}/bin/master-shell:/usr/bin/master-shell"
+              "${pkgs.sourcehut.buildsrht}/bin/runner-shell:/usr/bin/runner-shell"
+            ] ++
+            optionals cfg.git.enable [
+              # /path/to/gitsrht-keys calls /path/to/gitsrht-shell,
+              # or [git.sr.ht] shell= if set.
+              "${pkgs.writeShellScript "gitsrht-keys-wrapper" ''
+                set -e
+                cd /run/sourcehut/gitsrht/subdir
+                set -x
+                exec -a "$0" ${pkgs.sourcehut.gitsrht}/bin/gitsrht-keys "$@"
+              ''}:/usr/bin/gitsrht-keys"
+              "${pkgs.writeShellScript "gitsrht-shell-wrapper" ''
+                set -e
+                cd /run/sourcehut/gitsrht/subdir
+                set -x
+                exec -a "$0" ${pkgs.sourcehut.gitsrht}/bin/gitsrht-shell "$@"
+              ''}:/usr/bin/gitsrht-shell"
+              "${pkgs.writeShellScript "gitsrht-update-hook" ''
+                set -e
+                test -e "''${PWD%/*}"/config.ini ||
+                # Git hooks are run relative to their repository's directory,
+                # but gitsrht-update-hook looks up ../config.ini
+                ln -s /run/sourcehut/gitsrht/config.ini "''${PWD%/*}"/config.ini
+                # hooks/post-update calls /usr/bin/gitsrht-update-hook as hooks/stage-3
+                # but this wrapper being a bash script, it overrides $0 with /usr/bin/gitsrht-update-hook
+                # hence this hack to put hooks/stage-3 back into gitsrht-update-hook's $0
+                if test "''${STAGE3:+set}"
+                then
+                  set -x
+                  exec -a hooks/stage-3 ${pkgs.sourcehut.gitsrht}/bin/gitsrht-update-hook "$@"
+                else
+                  export STAGE3=set
+                  set -x
+                  exec -a "$0" ${pkgs.sourcehut.gitsrht}/bin/gitsrht-update-hook "$@"
+                fi
+              ''}:/usr/bin/gitsrht-update-hook"
+            ] ++
+            optionals cfg.hg.enable [
+              # /path/to/hgsrht-keys calls /path/to/hgsrht-shell,
+              # or [hg.sr.ht] shell= if set.
+              "${pkgs.writeShellScript "hgsrht-keys-wrapper" ''
+                set -e
+                cd /run/sourcehut/hgsrht/subdir
+                set -x
+                exec -a "$0" ${pkgs.sourcehut.hgsrht}/bin/hgsrht-keys "$@"
+              ''}:/usr/bin/hgsrht-keys"
+              "${pkgs.writeShellScript "hgsrht-shell-wrapper" ''
+                set -e
+                cd /run/sourcehut/hgsrht/subdir
+                set -x
+                exec -a "$0" ${pkgs.sourcehut.hgsrht}/bin/hgsrht-shell "$@"
+              ''}:/usr/bin/hgsrht-shell"
+              # Mercurial's changegroup hooks are run relative to their repository's directory,
+              # but hgsrht-hook-changegroup looks up ./config.ini
+              "${pkgs.writeShellScript "hgsrht-hook-changegroup" ''
+                set -e
+                test -e "''$PWD"/config.ini ||
+                ln -s /run/sourcehut/hgsrht/config.ini "''$PWD"/config.ini
+                set -x
+                exec -a "$0" ${cfg.python}/bin/hgsrht-hook-changegroup "$@"
+              ''}:/usr/bin/hgsrht-hook-changegroup"
+            ];
+        };
+      };
+    })
+  ]);
+
+  imports = [
+
+    (import ./service.nix "builds" {
+      inherit configIniOfService;
+      srvsrht = "buildsrht";
+      port = 5002;
+      # TODO: a celery worker on the master and worker are apparently needed
+      extraServices.buildsrht-worker = let
+        qemuPackage = pkgs.qemu_kvm;
+        serviceName = "buildsrht-worker";
+        statePath = "/var/lib/sourcehut/${serviceName}";
+        in mkIf cfg.builds.enableWorker {
+        path = [ pkgs.openssh pkgs.docker ];
+        preStart = ''
+          set -x
+          if test -z "$(docker images -q qemu:latest 2>/dev/null)" \
+          || test "$(cat ${statePath}/docker-image-qemu)" != "${qemuPackage.version}"
+          then
+            # Create and import qemu:latest image for docker
+            ${pkgs.dockerTools.streamLayeredImage {
+              name = "qemu";
+              tag = "latest";
+              contents = [ qemuPackage ];
+            }} | docker load
+            # Mark down current package version
+            echo '${qemuPackage.version}' >${statePath}/docker-image-qemu
+          fi
+        '';
+        serviceConfig = {
+          ExecStart = "${pkgs.sourcehut.buildsrht}/bin/builds.sr.ht-worker";
+          BindPaths = [ cfg.settings."builds.sr.ht::worker".buildlogs ];
+          LogsDirectory = [ "sourcehut/${serviceName}" ];
+          RuntimeDirectory = [ "sourcehut/${serviceName}/subdir" ];
+          StateDirectory = [ "sourcehut/${serviceName}" ];
+          TimeoutStartSec = "1800s";
+          # builds.sr.ht-worker looks up ../config.ini
+          WorkingDirectory = "-"+"/run/sourcehut/${serviceName}/subdir";
+        };
+      };
+      extraConfig = let
+        image_dirs = flatten (
+          mapAttrsToList (distro: revs:
+            mapAttrsToList (rev: archs:
+              mapAttrsToList (arch: image:
+                pkgs.runCommand "buildsrht-images" { } ''
+                  mkdir -p $out/${distro}/${rev}/${arch}
+                  ln -s ${image}/*.qcow2 $out/${distro}/${rev}/${arch}/root.img.qcow2
+                ''
+              ) archs
+            ) revs
+          ) cfg.builds.images
+        );
+        image_dir_pre = pkgs.symlinkJoin {
+          name = "builds.sr.ht-worker-images-pre";
+          paths = image_dirs;
+            # FIXME: not working, apparently because ubuntu/latest is a broken link
+            # ++ [ "${pkgs.sourcehut.buildsrht}/lib/images" ];
+        };
+        image_dir = pkgs.runCommand "builds.sr.ht-worker-images" { } ''
+          mkdir -p $out/images
+          cp -Lr ${image_dir_pre}/* $out/images
+        '';
+        in mkMerge [
         {
-          assertion = with cfgIni.webhooks; private-key != null && stringLength private-key == 44;
-          message = "The webhook's private key must be defined and of a 44 byte length.";
+          users.users.${cfg.builds.user}.shell = pkgs.bash;
+
+          virtualisation.docker.enable = true;
+
+          services.sourcehut.settings = mkMerge [
+            { # Note that git.sr.ht::dispatch is not a typo,
+              # gitsrht-dispatch always use this section
+              "git.sr.ht::dispatch"."/usr/bin/buildsrht-keys" =
+                mkDefault "${cfg.builds.user}:${cfg.builds.group}";
+            }
+            (mkIf cfg.builds.enableWorker {
+              "builds.sr.ht::worker".shell = "/usr/bin/runner-shell";
+              "builds.sr.ht::worker".images = mkDefault "${image_dir}/images";
+              "builds.sr.ht::worker".controlcmd = mkDefault "${image_dir}/images/control";
+            })
+          ];
         }
+        (mkIf cfg.builds.enableWorker {
+          users.groups = {
+            docker.members = [ cfg.builds.user ];
+          };
+        })
+        (mkIf (cfg.builds.enableWorker && cfg.nginx.enable) {
+          # Allow nginx access to buildlogs
+          users.users.${nginx.user}.extraGroups = [ cfg.builds.group ];
+          systemd.services.nginx = {
+            serviceConfig.BindReadOnlyPaths = [ cfg.settings."builds.sr.ht::worker".buildlogs ];
+          };
+          services.nginx.virtualHosts."logs.${domain}" = mkMerge [ {
+            /* FIXME: is a listen needed?
+            listen = with builtins;
+              # FIXME: not compatible with IPv6
+              let address = split ":" cfg.settings."builds.sr.ht::worker".name; in
+              [{ addr = elemAt address 0; port = lib.toInt (elemAt address 2); }];
+            */
+            locations."/logs/".alias = cfg.settings."builds.sr.ht::worker".buildlogs + "/";
+          } cfg.nginx.virtualHost ];
+        })
+      ];
+    })
 
+    (import ./service.nix "dispatch" {
+      inherit configIniOfService;
+      port = 5005;
+    })
+
+    (import ./service.nix "git" (let
+      baseService = {
+        path = [ cfg.git.package ];
+        serviceConfig.BindPaths = [ "${cfg.settings."git.sr.ht".repos}:/var/lib/sourcehut/gitsrht/repos" ];
+      };
+      in {
+      inherit configIniOfService;
+      mainService = mkMerge [ baseService {
+        serviceConfig.StateDirectory = [ "sourcehut/gitsrht" "sourcehut/gitsrht/repos" ];
+        preStart = mkIf (!versionAtLeast config.system.stateVersion "22.05") (mkBefore ''
+          # Fix Git hooks of repositories pre-dating https://github.com/NixOS/nixpkgs/pull/133984
+          (
+          set +f
+          shopt -s nullglob
+          for h in /var/lib/sourcehut/gitsrht/repos/~*/*/hooks/{pre-receive,update,post-update}
+          do ln -fnsv /usr/bin/gitsrht-update-hook "$h"; done
+          )
+        '');
+      } ];
+      port = 5001;
+      webhooks = true;
+      extraTimers.gitsrht-periodic = {
+        service = baseService;
+        timerConfig.OnCalendar = ["*:0/20"];
+      };
+      extraConfig = mkMerge [
         {
-          assertion = hasAttrByPath [ "meta.sr.ht" "origin" ] cfgIni && cfgIni."meta.sr.ht".origin != null;
-          message = "meta.sr.ht's origin must be defined.";
+          # https://stackoverflow.com/questions/22314298/git-push-results-in-fatal-protocol-error-bad-line-length-character-this
+          # Probably could use gitsrht-shell if output is restricted to just parameters...
+          users.users.${cfg.git.user}.shell = pkgs.bash;
+          services.sourcehut.settings = {
+            "git.sr.ht::dispatch"."/usr/bin/gitsrht-keys" =
+              mkDefault "${cfg.git.user}:${cfg.git.group}";
+          };
+          systemd.services.sshd = baseService;
         }
+        (mkIf cfg.nginx.enable {
+          services.nginx.virtualHosts."git.${domain}" = {
+            locations."/authorize" = {
+              proxyPass = "http://${cfg.listenAddress}:${toString cfg.git.port}";
+              extraConfig = ''
+                proxy_pass_request_body off;
+                proxy_set_header Content-Length "";
+                proxy_set_header X-Original-URI $request_uri;
+              '';
+            };
+            locations."~ ^/([^/]+)/([^/]+)/(HEAD|info/refs|objects/info/.*|git-upload-pack).*$" = {
+              root = "/var/lib/sourcehut/gitsrht/repos";
+              fastcgiParams = {
+                GIT_HTTP_EXPORT_ALL = "";
+                GIT_PROJECT_ROOT = "$document_root";
+                PATH_INFO = "$uri";
+                SCRIPT_FILENAME = "${cfg.git.package}/bin/git-http-backend";
+              };
+              extraConfig = ''
+                auth_request /authorize;
+                fastcgi_read_timeout 500s;
+                fastcgi_pass unix:/run/gitsrht-fcgiwrap.sock;
+                gzip off;
+              '';
+            };
+          };
+          systemd.sockets.gitsrht-fcgiwrap = {
+            before = [ "nginx.service" ];
+            wantedBy = [ "sockets.target" "gitsrht.service" ];
+            # This path remains accessible to nginx.service, which has no RootDirectory=
+            socketConfig.ListenStream = "/run/gitsrht-fcgiwrap.sock";
+            socketConfig.SocketUser = nginx.user;
+            socketConfig.SocketMode = "600";
+          };
+        })
       ];
+      extraServices.gitsrht-fcgiwrap = mkIf cfg.nginx.enable {
+        serviceConfig = {
+          # Socket is passed by gitsrht-fcgiwrap.socket
+          ExecStart = "${pkgs.fcgiwrap}/sbin/fcgiwrap -c ${toString cfg.git.fcgiwrap.preforkProcess}";
+          # No need for config.ini
+          ExecStartPre = mkForce [];
+          User = null;
+          DynamicUser = true;
+          BindReadOnlyPaths = [ "${cfg.settings."git.sr.ht".repos}:/var/lib/sourcehut/gitsrht/repos" ];
+          IPAddressDeny = "any";
+          InaccessiblePaths = [ "-+/run/postgresql" "-+/run/redis-sourcehut" ];
+          PrivateNetwork = true;
+          RestrictAddressFamilies = mkForce [ "none" ];
+          SystemCallFilter = mkForce [
+            "@system-service"
+            "~@aio" "~@keyring" "~@memlock" "~@privileged" "~@resources" "~@setuid"
+            # @timer is needed for alarm()
+          ];
+        };
+      };
+    }))
+
+    (import ./service.nix "hg" (let
+      baseService = {
+        path = [ cfg.hg.package ];
+        serviceConfig.BindPaths = [ "${cfg.settings."hg.sr.ht".repos}:/var/lib/sourcehut/hgsrht/repos" ];
+      };
+      in {
+      inherit configIniOfService;
+      mainService = mkMerge [ baseService {
+        serviceConfig.StateDirectory = [ "sourcehut/hgsrht" "sourcehut/hgsrht/repos" ];
+      } ];
+      port = 5010;
+      webhooks = true;
+      extraTimers.hgsrht-periodic = {
+        service = baseService;
+        timerConfig.OnCalendar = ["*:0/20"];
+      };
+      extraTimers.hgsrht-clonebundles = mkIf cfg.hg.cloneBundles {
+        service = baseService;
+        timerConfig.OnCalendar = ["daily"];
+        timerConfig.AccuracySec = "1h";
+      };
+      extraConfig = mkMerge [
+        {
+          users.users.${cfg.hg.user}.shell = pkgs.bash;
+          services.sourcehut.settings = {
+            # Note that git.sr.ht::dispatch is not a typo,
+            # gitsrht-dispatch always uses this section.
+            "git.sr.ht::dispatch"."/usr/bin/hgsrht-keys" =
+              mkDefault "${cfg.hg.user}:${cfg.hg.group}";
+          };
+          systemd.services.sshd = baseService;
+        }
+        (mkIf cfg.nginx.enable {
+          # Allow nginx access to repositories
+          users.users.${nginx.user}.extraGroups = [ cfg.hg.group ];
+          services.nginx.virtualHosts."hg.${domain}" = {
+            locations."/authorize" = {
+              proxyPass = "http://${cfg.listenAddress}:${toString cfg.hg.port}";
+              extraConfig = ''
+                proxy_pass_request_body off;
+                proxy_set_header Content-Length "";
+                proxy_set_header X-Original-URI $request_uri;
+              '';
+            };
+            # Let clients reach pull bundles. We don't really need to lock this down even for
+            # private repos because the bundles are named after the revision hashes...
+            # so someone would need to know or guess a SHA value to download anything.
+            # TODO: proxyPass to an hg serve service?
+            locations."~ ^/[~^][a-z0-9_]+/[a-zA-Z0-9_.-]+/\\.hg/bundles/.*$" = {
+              root = "/var/lib/nginx/hgsrht/repos";
+              extraConfig = ''
+                auth_request /authorize;
+                gzip off;
+              '';
+            };
+          };
+          systemd.services.nginx = {
+            serviceConfig.BindReadOnlyPaths = [ "${cfg.settings."hg.sr.ht".repos}:/var/lib/nginx/hgsrht/repos" ];
+          };
+        })
+      ];
+    }))
+
+    (import ./service.nix "hub" {
+      inherit configIniOfService;
+      port = 5014;
+      extraConfig = {
+        services.nginx = mkIf cfg.nginx.enable {
+          virtualHosts."hub.${domain}" = mkMerge [ {
+            serverAliases = [ domain ];
+          } cfg.nginx.virtualHost ];
+        };
+      };
+    })
+
+    (import ./service.nix "lists" (let
+      srvsrht = "listssrht";
+      in {
+      inherit configIniOfService;
+      port = 5006;
+      webhooks = true;
+      # Receive the mail from Postfix and enqueue them into Redis and PostgreSQL
+      extraServices.listssrht-lmtp = {
+        wants = [ "postfix.service" ];
+        unitConfig.JoinsNamespaceOf = optional cfg.postfix.enable "postfix.service";
+        serviceConfig.ExecStart = "${cfg.python}/bin/listssrht-lmtp";
+        # Avoid crashing: os.chown(sock, os.getuid(), sock_gid)
+        serviceConfig.PrivateUsers = mkForce false;
+      };
+      # Dequeue the mails from Redis and dispatch them
+      extraServices.listssrht-process = {
+        serviceConfig = {
+          preStart = ''
+            cp ${pkgs.writeText "${srvsrht}-webhooks-celeryconfig.py" cfg.lists.process.celeryConfig} \
+               /run/sourcehut/${srvsrht}-webhooks/celeryconfig.py
+          '';
+          ExecStart = "${cfg.python}/bin/celery --app listssrht.process worker --hostname listssrht-process@%%h " + concatStringsSep " " cfg.lists.process.extraArgs;
+          # Avoid crashing: os.getloadavg()
+          ProcSubset = mkForce "all";
+        };
+      };
+      extraConfig = mkIf cfg.postfix.enable {
+        users.groups.${postfix.group}.members = [ cfg.lists.user ];
+        services.sourcehut.settings."lists.sr.ht::mail".sock-group = postfix.group;
+        services.postfix = {
+          destination = [ "lists.${domain}" ];
+          # FIXME: an accurate recipient list should be queried
+          # from the lists.sr.ht PostgreSQL database to avoid backscattering.
+          # But usernames are unfortunately not in that database but in meta.sr.ht.
+          # Note that two syntaxes are allowed:
+          # - ~username/list-name@lists.${domain}
+          # - u.username.list-name@lists.${domain}
+          localRecipients = [ "@lists.${domain}" ];
+          transport = ''
+            lists.${domain} lmtp:unix:${cfg.settings."lists.sr.ht::worker".sock}
+          '';
+        };
+      };
+    }))
+
+    (import ./service.nix "man" {
+      inherit configIniOfService;
+      port = 5004;
+    })
+
+    (import ./service.nix "meta" {
+      inherit configIniOfService;
+      port = 5000;
+      webhooks = true;
+      extraServices.metasrht-api = {
+        serviceConfig.Restart = "always";
+        serviceConfig.RestartSec = "2s";
+        preStart = "set -x\n" + concatStringsSep "\n\n" (attrValues (mapAttrs (k: s:
+          let srvMatch = builtins.match "^([a-z]*)\\.sr\\.ht$" k;
+              srv = head srvMatch;
+          in
+          # Configure client(s) as "preauthorized"
+          optionalString (srvMatch != null && cfg.${srv}.enable && ((s.oauth-client-id or null) != null)) ''
+            # Configure ${srv}'s OAuth client as "preauthorized"
+            ${postgresql.package}/bin/psql '${cfg.settings."meta.sr.ht".connection-string}' \
+              -c "UPDATE oauthclient SET preauthorized = true WHERE client_id = '${s.oauth-client-id}'"
+          ''
+          ) cfg.settings));
+        serviceConfig.ExecStart = "${pkgs.sourcehut.metasrht}/bin/metasrht-api -b ${cfg.listenAddress}:${toString (cfg.meta.port + 100)}";
+      };
+      extraTimers.metasrht-daily.timerConfig = {
+        OnCalendar = ["daily"];
+        AccuracySec = "1h";
+      };
+      extraConfig = mkMerge [
+        {
+          assertions = [
+            { assertion = let s = cfg.settings."meta.sr.ht::billing"; in
+                          s.enabled == "yes" -> (s.stripe-public-key != null && s.stripe-secret-key != null);
+              message = "If meta.sr.ht::billing is enabled, the keys must be defined.";
+            }
+          ];
+          environment.systemPackages = optional cfg.meta.enable
+            (pkgs.writeShellScriptBin "metasrht-manageuser" ''
+              set -eux
+              if test "$(${pkgs.coreutils}/bin/id -n -u)" != '${cfg.meta.user}'
+              then exec sudo -u '${cfg.meta.user}' "$0" "$@"
+              else
+                # In order to load config.ini
+                if cd /run/sourcehut/metasrht
+                then exec ${cfg.python}/bin/metasrht-manageuser "$@"
+                else cat <<EOF
+                  Please run: sudo systemctl start metasrht
+              EOF
+                  exit 1
+                fi
+              fi
+            '');
+        }
+        (mkIf cfg.nginx.enable {
+          services.nginx.virtualHosts."meta.${domain}" = {
+            locations."/query" = {
+              proxyPass = cfg.settings."meta.sr.ht".api-origin;
+              extraConfig = ''
+                if ($request_method = 'OPTIONS') {
+                  add_header 'Access-Control-Allow-Origin' '*';
+                  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+                  add_header 'Access-Control-Allow-Headers' 'User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+                  add_header 'Access-Control-Max-Age' 1728000;
+                  add_header 'Content-Type' 'text/plain; charset=utf-8';
+                  add_header 'Content-Length' 0;
+                  return 204;
+                }
+
+                add_header 'Access-Control-Allow-Origin' '*';
+                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+                add_header 'Access-Control-Allow-Headers' 'User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+                add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
+              '';
+            };
+          };
+        })
+      ];
+    })
+
+    (import ./service.nix "pages" {
+      inherit configIniOfService;
+      port = 5112;
+      mainService = let
+        srvsrht = "pagessrht";
+        version = pkgs.sourcehut.${srvsrht}.version;
+        stateDir = "/var/lib/sourcehut/${srvsrht}";
+        iniKey = "pages.sr.ht";
+        in {
+        preStart = mkBefore ''
+          set -x
+          # Use the /run/sourcehut/${srvsrht}/config.ini
+          # installed by a previous ExecStartPre= in baseService
+          cd /run/sourcehut/${srvsrht}
+
+          if test ! -e ${stateDir}/db; then
+            ${postgresql.package}/bin/psql '${cfg.settings.${iniKey}.connection-string}' -f ${pkgs.sourcehut.pagessrht}/share/sql/schema.sql
+            echo ${version} >${stateDir}/db
+          fi
+
+          ${optionalString cfg.settings.${iniKey}.migrate-on-upgrade ''
+            # Just try all the migrations because they're not linked to the version
+            for sql in ${pkgs.sourcehut.pagessrht}/share/sql/migrations/*.sql; do
+              ${postgresql.package}/bin/psql '${cfg.settings.${iniKey}.connection-string}' -f "$sql" || true
+            done
+          ''}
+
+          # Disable webhook
+          touch ${stateDir}/webhook
+        '';
+        serviceConfig = {
+          ExecStart = mkForce "${pkgs.sourcehut.pagessrht}/bin/pages.sr.ht -b ${cfg.listenAddress}:${toString cfg.pages.port}";
+        };
+      };
+    })
+
+    (import ./service.nix "paste" {
+      inherit configIniOfService;
+      port = 5011;
+    })
+
+    (import ./service.nix "todo" {
+      inherit configIniOfService;
+      port = 5003;
+      webhooks = true;
+      extraServices.todosrht-lmtp = {
+        wants = [ "postfix.service" ];
+        unitConfig.JoinsNamespaceOf = optional cfg.postfix.enable "postfix.service";
+        serviceConfig.ExecStart = "${cfg.python}/bin/todosrht-lmtp";
+        # Avoid crashing: os.chown(sock, os.getuid(), sock_gid)
+        serviceConfig.PrivateUsers = mkForce false;
+      };
+      extraConfig = mkIf cfg.postfix.enable {
+        users.groups.${postfix.group}.members = [ cfg.todo.user ];
+        services.sourcehut.settings."todo.sr.ht::mail".sock-group = postfix.group;
+        services.postfix = {
+          destination = [ "todo.${domain}" ];
+          # FIXME: an accurate recipient list should be queried
+          # from the todo.sr.ht PostgreSQL database to avoid backscattering.
+          # But usernames are unfortunately not in that database but in meta.sr.ht.
+          # Note that two syntaxes are allowed:
+          # - ~username/tracker-name@todo.${domain}
+          # - u.username.tracker-name@todo.${domain}
+          localRecipients = [ "@todo.${domain}" ];
+          transport = ''
+            todo.${domain} lmtp:unix:${cfg.settings."todo.sr.ht::mail".sock}
+          '';
+        };
+      };
+    })
+
+    (mkRenamedOptionModule [ "services" "sourcehut" "originBase" ]
+                           [ "services" "sourcehut" "settings" "sr.ht" "global-domain" ])
+    (mkRenamedOptionModule [ "services" "sourcehut" "address" ]
+                           [ "services" "sourcehut" "listenAddress" ])
+
+  ];
 
-    virtualisation.docker.enable = true;
-    environment.etc."sr.ht/config.ini".source =
-      settingsFormat.generate "sourcehut-config.ini" (mapAttrsRecursive
-        (
-          path: v: if v == null then "" else v
-        )
-        cfg.settings);
-
-    environment.systemPackages = [ pkgs.sourcehut.coresrht ];
-
-    # PostgreSQL server
-    services.postgresql.enable = mkOverride 999 true;
-    # Mail server
-    services.postfix.enable = mkOverride 999 true;
-    # Cron daemon
-    services.cron.enable = mkOverride 999 true;
-    # Redis server
-    services.redis.enable = mkOverride 999 true;
-    services.redis.bind = mkOverride 999 "127.0.0.1";
-
-    services.sourcehut.settings = {
-      # The name of your network of sr.ht-based sites
-      "sr.ht".site-name = mkDefault "sourcehut";
-      # The top-level info page for your site
-      "sr.ht".site-info = mkDefault "https://sourcehut.org";
-      # {{ site-name }}, {{ site-blurb }}
-      "sr.ht".site-blurb = mkDefault "the hacker's forge";
-      # If this != production, we add a banner to each page
-      "sr.ht".environment = mkDefault "development";
-      # Contact information for the site owners
-      "sr.ht".owner-name = mkDefault "Drew DeVault";
-      "sr.ht".owner-email = mkDefault "sir@cmpwn.com";
-      # The source code for your fork of sr.ht
-      "sr.ht".source-url = mkDefault "https://git.sr.ht/~sircmpwn/srht";
-      # A secret key to encrypt session cookies with
-      "sr.ht".secret-key = mkDefault null;
-      "sr.ht".global-domain = mkDefault null;
-
-      # Outgoing SMTP settings
-      mail.smtp-host = mkDefault null;
-      mail.smtp-port = mkDefault null;
-      mail.smtp-user = mkDefault null;
-      mail.smtp-password = mkDefault null;
-      mail.smtp-from = mkDefault null;
-      # Application exceptions are emailed to this address
-      mail.error-to = mkDefault null;
-      mail.error-from = mkDefault null;
-      # Your PGP key information (DO NOT mix up pub and priv here)
-      # You must remove the password from your secret key, if present.
-      # You can do this with gpg --edit-key [key-id], then use the passwd
-      # command and do not enter a new password.
-      mail.pgp-privkey = mkDefault null;
-      mail.pgp-pubkey = mkDefault null;
-      mail.pgp-key-id = mkDefault null;
-
-      # base64-encoded Ed25519 key for signing webhook payloads. This should be
-      # consistent for all *.sr.ht sites, as we'll use this key to verify signatures
-      # from other sites in your network.
-      #
-      # Use the srht-webhook-keygen command to generate a key.
-      webhooks.private-key = mkDefault null;
-    };
-  };
   meta.doc = ./sourcehut.xml;
-  meta.maintainers = with maintainers; [ tomberek ];
+  meta.maintainers = with maintainers; [ julm tomberek ];
 }
diff --git a/nixos/modules/services/misc/sourcehut/dispatch.nix b/nixos/modules/services/misc/sourcehut/dispatch.nix
index a9db17bebe8e..292a51d3e1c5 100644
--- a/nixos/modules/services/misc/sourcehut/dispatch.nix
+++ b/nixos/modules/services/misc/sourcehut/dispatch.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   cfgIni = cfg.settings;
   scfg = cfg.dispatch;
   iniKey = "dispatch.sr.ht";
@@ -38,6 +39,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/dispatchsrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/dispatchsrht"'';
       description = ''
         State path for dispatch.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/sourcehut/git.nix b/nixos/modules/services/misc/sourcehut/git.nix
index 2653d77876dc..ff110905d184 100644
--- a/nixos/modules/services/misc/sourcehut/git.nix
+++ b/nixos/modules/services/misc/sourcehut/git.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   scfg = cfg.git;
   iniKey = "git.sr.ht";
 
@@ -41,6 +42,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/gitsrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/gitsrht"'';
       description = ''
         State path for git.sr.ht.
       '';
@@ -205,7 +207,7 @@ in
                 fastcgi_param PATH_INFO $uri;
                 fastcgi_param GIT_PROJECT_ROOT $document_root;
                 fastcgi_read_timeout 500s;
-                include ${pkgs.nginx}/conf/fastcgi_params;
+                include ${config.services.nginx.package}/conf/fastcgi_params;
                 gzip off;
             }
       '';
diff --git a/nixos/modules/services/misc/sourcehut/hg.nix b/nixos/modules/services/misc/sourcehut/hg.nix
index 5cd36bb04550..6ba1df8b6ddb 100644
--- a/nixos/modules/services/misc/sourcehut/hg.nix
+++ b/nixos/modules/services/misc/sourcehut/hg.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   scfg = cfg.hg;
   iniKey = "hg.sr.ht";
 
@@ -40,6 +41,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/hgsrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/hgsrht"'';
       description = ''
         State path for hg.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/sourcehut/hub.nix b/nixos/modules/services/misc/sourcehut/hub.nix
index be3ea21011c7..7d137a765056 100644
--- a/nixos/modules/services/misc/sourcehut/hub.nix
+++ b/nixos/modules/services/misc/sourcehut/hub.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   cfgIni = cfg.settings;
   scfg = cfg.hub;
   iniKey = "hub.sr.ht";
@@ -38,6 +39,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/hubsrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/hubsrht"'';
       description = ''
         State path for hub.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/sourcehut/lists.nix b/nixos/modules/services/misc/sourcehut/lists.nix
index 7b1fe9fd4630..76f155caa05b 100644
--- a/nixos/modules/services/misc/sourcehut/lists.nix
+++ b/nixos/modules/services/misc/sourcehut/lists.nix
@@ -1,11 +1,12 @@
 # Email setup is fairly involved, useful references:
 # https://drewdevault.com/2018/08/05/Local-mail-server.html
 
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   cfgIni = cfg.settings;
   scfg = cfg.lists;
   iniKey = "lists.sr.ht";
@@ -42,6 +43,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/listssrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/listssrht"'';
       description = ''
         State path for lists.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/sourcehut/man.nix b/nixos/modules/services/misc/sourcehut/man.nix
index 7693396d187c..8ca271c32ee3 100644
--- a/nixos/modules/services/misc/sourcehut/man.nix
+++ b/nixos/modules/services/misc/sourcehut/man.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   cfgIni = cfg.settings;
   scfg = cfg.man;
   iniKey = "man.sr.ht";
@@ -38,6 +39,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/mansrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/mansrht"'';
       description = ''
         State path for man.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/sourcehut/meta.nix b/nixos/modules/services/misc/sourcehut/meta.nix
index 56127a824eb4..33e4f2332b53 100644
--- a/nixos/modules/services/misc/sourcehut/meta.nix
+++ b/nixos/modules/services/misc/sourcehut/meta.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   cfgIni = cfg.settings;
   scfg = cfg.meta;
   iniKey = "meta.sr.ht";
@@ -39,6 +40,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/metasrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/metasrht"'';
       description = ''
         State path for meta.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/sourcehut/paste.nix b/nixos/modules/services/misc/sourcehut/paste.nix
index b2d5151969ea..b481ebaf8917 100644
--- a/nixos/modules/services/misc/sourcehut/paste.nix
+++ b/nixos/modules/services/misc/sourcehut/paste.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   cfgIni = cfg.settings;
   scfg = cfg.paste;
   iniKey = "paste.sr.ht";
@@ -39,6 +40,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/pastesrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/pastesrht"'';
       description = ''
         State path for pastesrht.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/sourcehut/service.nix b/nixos/modules/services/misc/sourcehut/service.nix
index 65b4ad020f9a..f1706ad0a6a8 100644
--- a/nixos/modules/services/misc/sourcehut/service.nix
+++ b/nixos/modules/services/misc/sourcehut/service.nix
@@ -1,66 +1,375 @@
-{ config, pkgs, lib }:
-serviceCfg: serviceDrv: iniKey: attrs:
+srv:
+{ configIniOfService
+, srvsrht ? "${srv}srht" # Because "buildsrht" does not follow that pattern (missing an "s").
+, iniKey ? "${srv}.sr.ht"
+, webhooks ? false
+, extraTimers ? {}
+, mainService ? {}
+, extraServices ? {}
+, extraConfig ? {}
+, port
+}:
+{ config, lib, pkgs, ... }:
+
+with lib;
 let
+  inherit (config.services) postgresql;
+  redis = config.services.redis.servers."sourcehut-${srvsrht}";
+  inherit (config.users) users;
   cfg = config.services.sourcehut;
-  cfgIni = cfg.settings."${iniKey}";
-  pgSuperUser = config.services.postgresql.superUser;
-
-  setupDB = pkgs.writeScript "${serviceDrv.pname}-gen-db" ''
-    #! ${cfg.python}/bin/python
-    from ${serviceDrv.pname}.app import db
-    db.create()
-  '';
+  configIni = configIniOfService srv;
+  srvCfg = cfg.${srv};
+  baseService = serviceName: { allowStripe ? false }: extraService: let
+    runDir = "/run/sourcehut/${serviceName}";
+    rootDir = "/run/sourcehut/chroots/${serviceName}";
+    in
+    mkMerge [ extraService {
+    after = [ "network.target" ] ++
+      optional cfg.postgresql.enable "postgresql.service" ++
+      optional cfg.redis.enable "redis-sourcehut-${srvsrht}.service";
+    requires =
+      optional cfg.postgresql.enable "postgresql.service" ++
+      optional cfg.redis.enable "redis-sourcehut-${srvsrht}.service";
+    path = [ pkgs.gawk ];
+    environment.HOME = runDir;
+    serviceConfig = {
+      User = mkDefault srvCfg.user;
+      Group = mkDefault srvCfg.group;
+      RuntimeDirectory = [
+        "sourcehut/${serviceName}"
+        # Used by *srht-keys which reads ../config.ini
+        "sourcehut/${serviceName}/subdir"
+        "sourcehut/chroots/${serviceName}"
+      ];
+      RuntimeDirectoryMode = "2750";
+      # No need for the chroot path once inside the chroot
+      InaccessiblePaths = [ "-+${rootDir}" ];
+      # g+rx is for group members (eg. fcgiwrap or nginx)
+      # to read Git/Mercurial repositories, buildlogs, etc.
+      # o+x is for intermediate directories created by BindPaths= and like,
+      # as they're owned by root:root.
+      UMask = "0026";
+      RootDirectory = rootDir;
+      RootDirectoryStartOnly = true;
+      PrivateTmp = true;
+      MountAPIVFS = true;
+      # config.ini is looked up in there, before /etc/srht/config.ini
+      # Note that it fails to be set in ExecStartPre=
+      WorkingDirectory = mkDefault ("-"+runDir);
+      BindReadOnlyPaths = [
+        builtins.storeDir
+        "/etc"
+        "/run/booted-system"
+        "/run/current-system"
+        "/run/systemd"
+        ] ++
+        optional cfg.postgresql.enable "/run/postgresql" ++
+        optional cfg.redis.enable "/run/redis-sourcehut-${srvsrht}";
+      # LoadCredential= are unfortunately not available in ExecStartPre=
+      # Hence this one is run as root (the +) with RootDirectoryStartOnly=
+      # to reach credentials wherever they are.
+      # Note that each systemd service gets its own ${runDir}/config.ini file.
+      ExecStartPre = mkBefore [("+"+pkgs.writeShellScript "${serviceName}-credentials" ''
+        set -x
+        # Replace values begining with a '<' by the content of the file whose name is after.
+        gawk '{ if (match($0,/^([^=]+=)<(.+)/,m)) { getline f < m[2]; print m[1] f } else print $0 }' ${configIni} |
+        ${optionalString (!allowStripe) "gawk '!/^stripe-secret-key=/' |"}
+        install -o ${srvCfg.user} -g root -m 400 /dev/stdin ${runDir}/config.ini
+      '')];
+      # The following options are only for optimizing:
+      # systemd-analyze security
+      AmbientCapabilities = "";
+      CapabilityBoundingSet = "";
+      # ProtectClock= adds DeviceAllow=char-rtc r
+      DeviceAllow = "";
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      PrivateMounts = true;
+      PrivateNetwork = mkDefault false;
+      PrivateUsers = true;
+      ProcSubset = "pid";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      RemoveIPC = true;
+      RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      #SocketBindAllow = [ "tcp:${toString srvCfg.port}" "tcp:${toString srvCfg.prometheusPort}" ];
+      #SocketBindDeny = "any";
+      SystemCallFilter = [
+        "@system-service"
+        "~@aio" "~@keyring" "~@memlock" "~@privileged" "~@resources" "~@timer"
+        "@chown" "@setuid"
+      ];
+      SystemCallArchitectures = "native";
+    };
+  } ];
 in
-with serviceCfg; with lib; recursiveUpdate
 {
-  environment.HOME = statePath;
-  path = [ config.services.postgresql.package ] ++ (attrs.path or [ ]);
-  restartTriggers = [ config.environment.etc."sr.ht/config.ini".source ];
-  serviceConfig = {
-    Type = "simple";
-    User = user;
-    Group = user;
-    Restart = "always";
-    WorkingDirectory = statePath;
-  } // (if (cfg.statePath == "/var/lib/sourcehut/${serviceDrv.pname}") then {
-          StateDirectory = [ "sourcehut/${serviceDrv.pname}" ];
-        } else {})
-  ;
-
-  preStart = ''
-    if ! test -e ${statePath}/db; then
-      # Setup the initial database
-      ${setupDB}
-
-      # Set the initial state of the database for future database upgrades
-      if test -e ${cfg.python}/bin/${serviceDrv.pname}-migrate; then
-        # Run alembic stamp head once to tell alembic the schema is up-to-date
-        ${cfg.python}/bin/${serviceDrv.pname}-migrate stamp head
-      fi
-
-      printf "%s" "${serviceDrv.version}" > ${statePath}/db
-    fi
-
-    # Update copy of each users' profile to the latest
-    # See https://lists.sr.ht/~sircmpwn/sr.ht-admins/<20190302181207.GA13778%40cirno.my.domain>
-    if ! test -e ${statePath}/webhook; then
-      # Update ${iniKey}'s users' profile copy to the latest
-      ${cfg.python}/bin/srht-update-profiles ${iniKey}
-
-      touch ${statePath}/webhook
-    fi
-
-    ${optionalString (builtins.hasAttr "migrate-on-upgrade" cfgIni && cfgIni.migrate-on-upgrade == "yes") ''
-      if [ "$(cat ${statePath}/db)" != "${serviceDrv.version}" ]; then
-        # Manage schema migrations using alembic
-        ${cfg.python}/bin/${serviceDrv.pname}-migrate -a upgrade head
-
-        # Mark down current package version
-        printf "%s" "${serviceDrv.version}" > ${statePath}/db
-      fi
-    ''}
-
-    ${attrs.preStart or ""}
-  '';
+  options.services.sourcehut.${srv} = {
+    enable = mkEnableOption "${srv} service";
+
+    user = mkOption {
+      type = types.str;
+      default = srvsrht;
+      description = ''
+        User for ${srv}.sr.ht.
+      '';
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = srvsrht;
+      description = ''
+        Group for ${srv}.sr.ht.
+        Membership grants access to the Git/Mercurial repositories by default,
+        but not to the config.ini file (where secrets are).
+      '';
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = port;
+      description = ''
+        Port on which the "${srv}" backend should listen.
+      '';
+    };
+
+    redis = {
+      host = mkOption {
+        type = types.str;
+        default = "unix:/run/redis-sourcehut-${srvsrht}/redis.sock?db=0";
+        example = "redis://shared.wireguard:6379/0";
+        description = ''
+          The redis host URL. This is used for caching and temporary storage, and must
+          be shared between nodes (e.g. git1.sr.ht and git2.sr.ht), but need not be
+          shared between services. It may be shared between services, however, with no
+          ill effect, if this better suits your infrastructure.
+        '';
+      };
+    };
+
+    postgresql = {
+      database = mkOption {
+        type = types.str;
+        default = "${srv}.sr.ht";
+        description = ''
+          PostgreSQL database name for the ${srv}.sr.ht service,
+          used if <xref linkend="opt-services.sourcehut.postgresql.enable"/> is <literal>true</literal>.
+        '';
+      };
+    };
+
+    gunicorn = {
+      extraArgs = mkOption {
+        type = with types; listOf str;
+        default = ["--timeout 120" "--workers 1" "--log-level=info"];
+        description = "Extra arguments passed to Gunicorn.";
+      };
+    };
+  } // optionalAttrs webhooks {
+    webhooks = {
+      extraArgs = mkOption {
+        type = with types; listOf str;
+        default = ["--loglevel DEBUG" "--pool eventlet" "--without-heartbeat"];
+        description = "Extra arguments passed to the Celery responsible for webhooks.";
+      };
+      celeryConfig = mkOption {
+        type = types.lines;
+        default = "";
+        description = "Content of the <literal>celeryconfig.py</literal> used by the Celery responsible for webhooks.";
+      };
+    };
+  };
+
+  config = lib.mkIf (cfg.enable && srvCfg.enable) (mkMerge [ extraConfig {
+    users = {
+      users = {
+        "${srvCfg.user}" = {
+          isSystemUser = true;
+          group = mkDefault srvCfg.group;
+          description = mkDefault "sourcehut user for ${srv}.sr.ht";
+        };
+      };
+      groups = {
+        "${srvCfg.group}" = { };
+      } // optionalAttrs (cfg.postgresql.enable
+        && hasSuffix "0" (postgresql.settings.unix_socket_permissions or "")) {
+        "postgres".members = [ srvCfg.user ];
+      } // optionalAttrs (cfg.redis.enable
+        && hasSuffix "0" (redis.settings.unixsocketperm or "")) {
+        "redis-sourcehut-${srvsrht}".members = [ srvCfg.user ];
+      };
+    };
+
+    services.nginx = mkIf cfg.nginx.enable {
+      virtualHosts."${srv}.${cfg.settings."sr.ht".global-domain}" = mkMerge [ {
+        forceSSL = mkDefault true;
+        locations."/".proxyPass = "http://${cfg.listenAddress}:${toString srvCfg.port}";
+        locations."/static" = {
+          root = "${pkgs.sourcehut.${srvsrht}}/${pkgs.sourcehut.python.sitePackages}/${srvsrht}";
+          extraConfig = mkDefault ''
+            expires 30d;
+          '';
+        };
+      } cfg.nginx.virtualHost ];
+    };
+
+    services.postgresql = mkIf cfg.postgresql.enable {
+      authentication = ''
+        local ${srvCfg.postgresql.database} ${srvCfg.user} trust
+      '';
+      ensureDatabases = [ srvCfg.postgresql.database ];
+      ensureUsers = map (name: {
+          inherit name;
+          ensurePermissions = { "DATABASE \"${srvCfg.postgresql.database}\"" = "ALL PRIVILEGES"; };
+        }) [srvCfg.user];
+    };
+
+    services.sourcehut.services = mkDefault (filter (s: cfg.${s}.enable)
+      [ "builds" "dispatch" "git" "hg" "hub" "lists" "man" "meta" "pages" "paste" "todo" ]);
+
+    services.sourcehut.settings = mkMerge [
+      {
+        "${srv}.sr.ht".origin = mkDefault "https://${srv}.${cfg.settings."sr.ht".global-domain}";
+      }
+
+      (mkIf cfg.postgresql.enable {
+        "${srv}.sr.ht".connection-string = mkDefault "postgresql:///${srvCfg.postgresql.database}?user=${srvCfg.user}&host=/run/postgresql";
+      })
+    ];
+
+    services.redis.servers."sourcehut-${srvsrht}" = mkIf cfg.redis.enable {
+      enable = true;
+      databases = 3;
+      syslog = true;
+      # TODO: set a more informed value
+      save = mkDefault [ [1800 10] [300 100] ];
+      settings = {
+        # TODO: set a more informed value
+        maxmemory = "128MB";
+        maxmemory-policy = "volatile-ttl";
+      };
+    };
+
+    systemd.services = mkMerge [
+      {
+        "${srvsrht}" = baseService srvsrht { allowStripe = srv == "meta"; } (mkMerge [
+        {
+          description = "sourcehut ${srv}.sr.ht website service";
+          before = optional cfg.nginx.enable "nginx.service";
+          wants = optional cfg.nginx.enable "nginx.service";
+          wantedBy = [ "multi-user.target" ];
+          path = optional cfg.postgresql.enable postgresql.package;
+          # Beware: change in credentials' content will not trigger restart.
+          restartTriggers = [ configIni ];
+          serviceConfig = {
+            Type = "simple";
+            Restart = mkDefault "always";
+            #RestartSec = mkDefault "2min";
+            StateDirectory = [ "sourcehut/${srvsrht}" ];
+            StateDirectoryMode = "2750";
+            ExecStart = "${cfg.python}/bin/gunicorn ${srvsrht}.app:app --name ${srvsrht} --bind ${cfg.listenAddress}:${toString srvCfg.port} " + concatStringsSep " " srvCfg.gunicorn.extraArgs;
+          };
+          preStart = let
+            version = pkgs.sourcehut.${srvsrht}.version;
+            stateDir = "/var/lib/sourcehut/${srvsrht}";
+            in mkBefore ''
+            set -x
+            # Use the /run/sourcehut/${srvsrht}/config.ini
+            # installed by a previous ExecStartPre= in baseService
+            cd /run/sourcehut/${srvsrht}
+
+            if test ! -e ${stateDir}/db; then
+              # Setup the initial database.
+              # Note that it stamps the alembic head afterward
+              ${cfg.python}/bin/${srvsrht}-initdb
+              echo ${version} >${stateDir}/db
+            fi
+
+            ${optionalString cfg.settings.${iniKey}.migrate-on-upgrade ''
+              if [ "$(cat ${stateDir}/db)" != "${version}" ]; then
+                # Manage schema migrations using alembic
+                ${cfg.python}/bin/${srvsrht}-migrate -a upgrade head
+                echo ${version} >${stateDir}/db
+              fi
+            ''}
+
+            # Update copy of each users' profile to the latest
+            # See https://lists.sr.ht/~sircmpwn/sr.ht-admins/<20190302181207.GA13778%40cirno.my.domain>
+            if test ! -e ${stateDir}/webhook; then
+              # Update ${iniKey}'s users' profile copy to the latest
+              ${cfg.python}/bin/srht-update-profiles ${iniKey}
+              touch ${stateDir}/webhook
+            fi
+          '';
+        } mainService ]);
+      }
+
+      (mkIf webhooks {
+        "${srvsrht}-webhooks" = baseService "${srvsrht}-webhooks" {}
+          {
+            description = "sourcehut ${srv}.sr.ht webhooks service";
+            after = [ "${srvsrht}.service" ];
+            wantedBy = [ "${srvsrht}.service" ];
+            partOf = [ "${srvsrht}.service" ];
+            preStart = ''
+              cp ${pkgs.writeText "${srvsrht}-webhooks-celeryconfig.py" srvCfg.webhooks.celeryConfig} \
+                 /run/sourcehut/${srvsrht}-webhooks/celeryconfig.py
+            '';
+            serviceConfig = {
+              Type = "simple";
+              Restart = "always";
+              ExecStart = "${cfg.python}/bin/celery --app ${srvsrht}.webhooks worker --hostname ${srvsrht}-webhooks@%%h " + concatStringsSep " " srvCfg.webhooks.extraArgs;
+              # Avoid crashing: os.getloadavg()
+              ProcSubset = mkForce "all";
+            };
+          };
+      })
+
+      (mapAttrs (timerName: timer: (baseService timerName {} (mkMerge [
+        {
+          description = "sourcehut ${timerName} service";
+          after = [ "network.target" "${srvsrht}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            ExecStart = "${cfg.python}/bin/${timerName}";
+          };
+        }
+        (timer.service or {})
+      ]))) extraTimers)
+
+      (mapAttrs (serviceName: extraService: baseService serviceName {} (mkMerge [
+        {
+          description = "sourcehut ${serviceName} service";
+          # So that extraServices have the PostgreSQL database initialized.
+          after = [ "${srvsrht}.service" ];
+          wantedBy = [ "${srvsrht}.service" ];
+          partOf = [ "${srvsrht}.service" ];
+          serviceConfig = {
+            Type = "simple";
+            Restart = mkDefault "always";
+          };
+        }
+        extraService
+      ])) extraServices)
+    ];
+
+    systemd.timers = mapAttrs (timerName: timer:
+      {
+        description = "sourcehut timer for ${timerName}";
+        wantedBy = [ "timers.target" ];
+        inherit (timer) timerConfig;
+      }) extraTimers;
+  } ]);
 }
-  (builtins.removeAttrs attrs [ "path" "preStart" ])
diff --git a/nixos/modules/services/misc/sourcehut/sourcehut.xml b/nixos/modules/services/misc/sourcehut/sourcehut.xml
index ab9a8c6cb4be..41094f65a94d 100644
--- a/nixos/modules/services/misc/sourcehut/sourcehut.xml
+++ b/nixos/modules/services/misc/sourcehut/sourcehut.xml
@@ -14,13 +14,12 @@
   <title>Basic usage</title>
   <para>
    Sourcehut is a Python and Go based set of applications.
-   <literal><link linkend="opt-services.sourcehut.enable">services.sourcehut</link></literal>
-   by default will use
+   This NixOS module also provides basic configuration integrating Sourcehut into locally running
    <literal><link linkend="opt-services.nginx.enable">services.nginx</link></literal>,
-   <literal><link linkend="opt-services.nginx.enable">services.redis</link></literal>,
-   <literal><link linkend="opt-services.nginx.enable">services.cron</link></literal>,
+   <literal><link linkend="opt-services.redis.servers">services.redis.servers.sourcehut</link></literal>,
+   <literal><link linkend="opt-services.postfix.enable">services.postfix</link></literal>
    and
-   <literal><link linkend="opt-services.postgresql.enable">services.postgresql</link></literal>.
+   <literal><link linkend="opt-services.postgresql.enable">services.postgresql</link></literal> services.
   </para>
 
   <para>
@@ -42,18 +41,23 @@ in {
 
   services.sourcehut = {
     <link linkend="opt-services.sourcehut.enable">enable</link> = true;
-    <link linkend="opt-services.sourcehut.originBase">originBase</link> = fqdn;
-    <link linkend="opt-services.sourcehut.services">services</link> = [ "meta" "man" "git" ];
+    <link linkend="opt-services.sourcehut.git.enable">git.enable</link> = true;
+    <link linkend="opt-services.sourcehut.man.enable">man.enable</link> = true;
+    <link linkend="opt-services.sourcehut.meta.enable">meta.enable</link> = true;
+    <link linkend="opt-services.sourcehut.nginx.enable">nginx.enable</link> = true;
+    <link linkend="opt-services.sourcehut.postfix.enable">postfix.enable</link> = true;
+    <link linkend="opt-services.sourcehut.postgresql.enable">postgresql.enable</link> = true;
+    <link linkend="opt-services.sourcehut.redis.enable">redis.enable</link> = true;
     <link linkend="opt-services.sourcehut.settings">settings</link> = {
         "sr.ht" = {
           environment = "production";
           global-domain = fqdn;
           origin = "https://${fqdn}";
           # Produce keys with srht-keygen from <package>sourcehut.coresrht</package>.
-          network-key = "SECRET";
-          service-key = "SECRET";
+          network-key = "/run/keys/path/to/network-key";
+          service-key = "/run/keys/path/to/service-key";
         };
-        webhooks.private-key= "SECRET";
+        webhooks.private-key= "/run/keys/path/to/webhook-key";
     };
   };
 
diff --git a/nixos/modules/services/misc/sourcehut/todo.nix b/nixos/modules/services/misc/sourcehut/todo.nix
index aec773b06692..262fa48f59d4 100644
--- a/nixos/modules/services/misc/sourcehut/todo.nix
+++ b/nixos/modules/services/misc/sourcehut/todo.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.sourcehut;
+  opt = options.services.sourcehut;
   cfgIni = cfg.settings;
   scfg = cfg.todo;
   iniKey = "todo.sr.ht";
@@ -39,6 +40,7 @@ in
     statePath = mkOption {
       type = types.path;
       default = "${cfg.statePath}/todosrht";
+      defaultText = literalExpression ''"''${config.${opt.statePath}}/todosrht"'';
       description = ''
         State path for todo.sr.ht.
       '';
diff --git a/nixos/modules/services/misc/subsonic.nix b/nixos/modules/services/misc/subsonic.nix
index 98b85918ad18..2dda8970dd30 100644
--- a/nixos/modules/services/misc/subsonic.nix
+++ b/nixos/modules/services/misc/subsonic.nix
@@ -1,8 +1,11 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
-let cfg = config.services.subsonic; in {
+let
+  cfg = config.services.subsonic;
+  opt = options.services.subsonic;
+in {
   options = {
     services.subsonic = {
       enable = mkEnableOption "Subsonic daemon";
@@ -97,7 +100,7 @@ let cfg = config.services.subsonic; in {
         description = ''
           List of paths to transcoder executables that should be accessible
           from Subsonic. Symlinks will be created to each executable inside
-          ${cfg.home}/transcoders.
+          ''${config.${opt.home}}/transcoders.
         '';
       };
     };
diff --git a/nixos/modules/services/misc/taskserver/default.nix b/nixos/modules/services/misc/taskserver/default.nix
index a894caed1a34..ff63c41e193c 100644
--- a/nixos/modules/services/misc/taskserver/default.nix
+++ b/nixos/modules/services/misc/taskserver/default.nix
@@ -106,7 +106,7 @@ let
 
   certtool = "${pkgs.gnutls.bin}/bin/certtool";
 
-  nixos-taskserver = pkgs.pythonPackages.buildPythonApplication {
+  nixos-taskserver = with pkgs.python2.pkgs; buildPythonApplication {
     name = "nixos-taskserver";
 
     src = pkgs.runCommand "nixos-taskserver-src" { preferLocalBuild = true; } ''
@@ -129,7 +129,7 @@ let
       EOF
     '';
 
-    propagatedBuildInputs = [ pkgs.pythonPackages.click ];
+    propagatedBuildInputs = [ click ];
   };
 
 in {
@@ -138,12 +138,13 @@ in {
       enable = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = let
+          url = "https://nixos.org/manual/nixos/stable/index.html#module-services-taskserver";
+        in ''
           Whether to enable the Taskwarrior server.
 
           More instructions about NixOS in conjuction with Taskserver can be
-          found in the NixOS manual at
-          <olink targetdoc="manual" targetptr="module-taskserver"/>.
+          found <link xlink:href="${url}">in the NixOS manual</link>.
         '';
       };
 
diff --git a/nixos/modules/services/misc/taskserver/doc.xml b/nixos/modules/services/misc/taskserver/doc.xml
index 5656bb85b373..f6ead7c37857 100644
--- a/nixos/modules/services/misc/taskserver/doc.xml
+++ b/nixos/modules/services/misc/taskserver/doc.xml
@@ -1,7 +1,7 @@
 <chapter xmlns="http://docbook.org/ns/docbook"
     xmlns:xlink="http://www.w3.org/1999/xlink"
     version="5.0"
-    xml:id="module-taskserver">
+    xml:id="module-services-taskserver">
  <title>Taskserver</title>
  <para>
   Taskserver is the server component of
diff --git a/nixos/modules/services/misc/zoneminder.nix b/nixos/modules/services/misc/zoneminder.nix
index 378da7b87442..a557e742b7cf 100644
--- a/nixos/modules/services/misc/zoneminder.nix
+++ b/nixos/modules/services/misc/zoneminder.nix
@@ -171,7 +171,7 @@ in {
         example = "/storage/tank";
         description = ''
           ZoneMinder can generate quite a lot of data, so in case you don't want
-          to use the default ${home}, you can override the path here.
+          to use the default ${defaultDir}, you can override the path here.
         '';
       };
 
@@ -254,7 +254,7 @@ in {
                 location /cgi-bin {
                   gzip off;
 
-                  include ${pkgs.nginx}/conf/fastcgi_params;
+                  include ${config.services.nginx.package}/conf/fastcgi_params;
                   fastcgi_param SCRIPT_FILENAME ${pkg}/libexec/zoneminder/${zms};
                   fastcgi_param HTTP_PROXY "";
                   fastcgi_intercept_errors on;
@@ -270,7 +270,7 @@ in {
                   try_files $uri =404;
                   fastcgi_index index.php;
 
-                  include ${pkgs.nginx}/conf/fastcgi_params;
+                  include ${config.services.nginx.package}/conf/fastcgi_params;
                   fastcgi_param SCRIPT_FILENAME $request_filename;
                   fastcgi_param HTTP_PROXY "";
 
diff --git a/nixos/modules/services/monitoring/collectd.nix b/nixos/modules/services/monitoring/collectd.nix
index 660d108587de..8d81737a3ef0 100644
--- a/nixos/modules/services/monitoring/collectd.nix
+++ b/nixos/modules/services/monitoring/collectd.nix
@@ -5,7 +5,7 @@ with lib;
 let
   cfg = config.services.collectd;
 
-  conf = pkgs.writeText "collectd.conf" ''
+  unvalidated_conf = pkgs.writeText "collectd-unvalidated.conf" ''
     BaseDir "${cfg.dataDir}"
     AutoLoadPlugin ${boolToString cfg.autoLoadPlugin}
     Hostname "${config.networking.hostName}"
@@ -30,6 +30,15 @@ let
     ${cfg.extraConfig}
   '';
 
+  conf = if cfg.validateConfig then
+    pkgs.runCommand "collectd.conf" {} ''
+      echo testing ${unvalidated_conf}
+      # collectd -t fails if BaseDir does not exist.
+      sed '1s/^BaseDir.*$/BaseDir "."/' ${unvalidated_conf} > collectd.conf
+      ${package}/bin/collectd -t -C collectd.conf
+      cp ${unvalidated_conf} $out
+    '' else unvalidated_conf;
+
   package =
     if cfg.buildMinimalPackage
     then minimalPackage
@@ -43,6 +52,16 @@ in {
   options.services.collectd = with types; {
     enable = mkEnableOption "collectd agent";
 
+    validateConfig = mkOption {
+      default = true;
+      description = ''
+        Validate the syntax of collectd configuration file at build time.
+        Disable this if you use the Include directive on files unavailable in
+        the build sandbox, or when cross-compiling.
+      '';
+      type = types.bool;
+    };
+
     package = mkOption {
       default = pkgs.collectd;
       defaultText = literalExpression "pkgs.collectd";
diff --git a/nixos/modules/services/monitoring/grafana.nix b/nixos/modules/services/monitoring/grafana.nix
index 5067047e9690..81fca33f5fec 100644
--- a/nixos/modules/services/monitoring/grafana.nix
+++ b/nixos/modules/services/monitoring/grafana.nix
@@ -404,6 +404,7 @@ in {
       path = mkOption {
         description = "Database path.";
         default = "${cfg.dataDir}/data/grafana.db";
+        defaultText = literalExpression ''"''${config.${opt.dataDir}}/data/grafana.db"'';
         type = types.path;
       };
 
diff --git a/nixos/modules/services/monitoring/graphite.nix b/nixos/modules/services/monitoring/graphite.nix
index 0dbb33530c92..baa943302a00 100644
--- a/nixos/modules/services/monitoring/graphite.nix
+++ b/nixos/modules/services/monitoring/graphite.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.graphite;
+  opt = options.services.graphite;
   writeTextOrNull = f: t: mapNullable (pkgs.writeTextDir f) t;
 
   dataDir = cfg.dataDir;
@@ -171,6 +172,13 @@ in {
             directories:
                 - ${dataDir}/whisper
         '';
+        defaultText = literalExpression ''
+          '''
+            whisper:
+              directories:
+                - ''${config.${opt.dataDir}}/whisper
+          '''
+        '';
         example = ''
           allowed_origins:
             - dashboard.example.com
@@ -312,12 +320,14 @@ in {
 
       seyrenUrl = mkOption {
         default = "http://localhost:${toString cfg.seyren.port}/";
+        defaultText = literalExpression ''"http://localhost:''${toString config.${opt.seyren.port}}/"'';
         description = "Host where seyren is accessible.";
         type = types.str;
       };
 
       graphiteUrl = mkOption {
         default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
+        defaultText = literalExpression ''"http://''${config.${opt.web.listenAddress}}:''${toString config.${opt.web.port}}"'';
         description = "Host where graphite service runs.";
         type = types.str;
       };
diff --git a/nixos/modules/services/monitoring/netdata.nix b/nixos/modules/services/monitoring/netdata.nix
index 00bdd9fcda0d..f528d1830424 100644
--- a/nixos/modules/services/monitoring/netdata.nix
+++ b/nixos/modules/services/monitoring/netdata.nix
@@ -19,8 +19,17 @@ let
     "${wrappedPlugins}/libexec/netdata/plugins.d"
   ] ++ cfg.extraPluginPaths;
 
+  configDirectory = pkgs.runCommand "netdata-config-d" { } ''
+    mkdir $out
+    ${concatStringsSep "\n" (mapAttrsToList (path: file: ''
+        mkdir -p "$out/$(dirname ${path})"
+        ln -s "${file}" "$out/${path}"
+      '') cfg.configDir)}
+  '';
+
   localConfig = {
     global = {
+      "config directory" = "/etc/netdata/conf.d";
       "plugins directory" = concatStringsSep " " plugins;
     };
     web = {
@@ -130,6 +139,26 @@ in {
         '';
       };
 
+      configDir = mkOption {
+        type = types.attrsOf types.path;
+        default = {};
+        description = ''
+          Complete netdata config directory except netdata.conf.
+          The default configuration is merged with changes
+          defined in this option.
+          Each top-level attribute denotes a path in the configuration
+          directory as in environment.etc.
+          Its value is the absolute path and must be readable by netdata.
+          Cannot be combined with configText.
+        '';
+        example = literalExpression ''
+          "health_alarm_notify.conf" = pkgs.writeText "health_alarm_notify.conf" '''
+            sendmail="/path/to/sendmail"
+          ''';
+          "health.d" = "/run/secrets/netdata/health.d";
+        '';
+      };
+
       enableAnalyticsReporting = mkOption {
         type = types.bool;
         default = false;
@@ -150,11 +179,14 @@ in {
         }
       ];
 
+    environment.etc."netdata/netdata.conf".source = configFile;
+    environment.etc."netdata/conf.d".source = configDirectory;
+
     systemd.services.netdata = {
       description = "Real time performance monitoring";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      path = (with pkgs; [ curl gawk iproute2 which ])
+      path = (with pkgs; [ curl gawk iproute2 which procps ])
         ++ lib.optional cfg.python.enable (pkgs.python3.withPackages cfg.python.extraPackages)
         ++ lib.optional config.virtualisation.libvirtd.enable (config.virtualisation.libvirtd.package);
       environment = {
@@ -162,8 +194,12 @@ in {
       } // lib.optionalAttrs (!cfg.enableAnalyticsReporting) {
         DO_NOT_TRACK = "1";
       };
+      restartTriggers = [
+        config.environment.etc."netdata/netdata.conf".source
+        config.environment.etc."netdata/conf.d".source
+      ];
       serviceConfig = {
-        ExecStart = "${cfg.package}/bin/netdata -P /run/netdata/netdata.pid -D -c ${configFile}";
+        ExecStart = "${cfg.package}/bin/netdata -P /run/netdata/netdata.pid -D -c /etc/netdata/netdata.conf";
         ExecReload = "${pkgs.util-linux}/bin/kill -s HUP -s USR1 -s USR2 $MAINPID";
         TimeoutStopSec = 60;
         Restart = "on-failure";
diff --git a/nixos/modules/services/monitoring/parsedmarc.nix b/nixos/modules/services/monitoring/parsedmarc.nix
index 8571e1f01ed6..ec71365ba3c1 100644
--- a/nixos/modules/services/monitoring/parsedmarc.nix
+++ b/nixos/modules/services/monitoring/parsedmarc.nix
@@ -1,7 +1,8 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
   cfg = config.services.parsedmarc;
+  opt = options.services.parsedmarc;
   ini = pkgs.formats.ini {};
 in
 {
@@ -80,6 +81,9 @@ in
         datasource = lib.mkOption {
           type = lib.types.bool;
           default = cfg.provision.elasticsearch && config.services.grafana.enable;
+          defaultText = lib.literalExpression ''
+            config.${opt.provision.elasticsearch} && config.${options.services.grafana.enable}
+          '';
           apply = x: x && cfg.provision.elasticsearch;
           description = ''
             Whether the automatically provisioned Elasticsearch
diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index f20b8dde1abd..f563861b61c0 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -251,6 +251,13 @@ let
 
   promTypes.scrape_config = types.submodule {
     options = {
+      authorization = mkOption {
+        type = types.nullOr types.attrs;
+        default = null;
+        description = ''
+          Sets the `Authorization` header on every scrape request with the configured credentials.
+        '';
+      };
       job_name = mkOption {
         type = types.str;
         description = ''
diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix
index d29d50706ef6..41302d6d3ceb 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -29,6 +29,7 @@ let
     "blackbox"
     "buildkite-agent"
     "collectd"
+    "dmarc"
     "dnsmasq"
     "domain"
     "dovecot"
@@ -55,6 +56,7 @@ let
     "postfix"
     "postgres"
     "process"
+    "pve"
     "py-air-control"
     "redis"
     "rspamd"
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/dmarc.nix b/nixos/modules/services/monitoring/prometheus/exporters/dmarc.nix
new file mode 100644
index 000000000000..330610a15d9e
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/dmarc.nix
@@ -0,0 +1,117 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.dmarc;
+
+  json = builtins.toJSON {
+    inherit (cfg) folders port;
+    listen_addr = cfg.listenAddress;
+    storage_path = "$STATE_DIRECTORY";
+    imap = (builtins.removeAttrs cfg.imap [ "passwordFile" ]) // { password = "$IMAP_PASSWORD"; use_ssl = true; };
+    poll_interval_seconds = cfg.pollIntervalSeconds;
+    deduplication_max_seconds = cfg.deduplicationMaxSeconds;
+    logging = {
+      version = 1;
+      disable_existing_loggers = false;
+    };
+  };
+in {
+  port = 9797;
+  extraOpts = {
+    imap = {
+      host = mkOption {
+        type = types.str;
+        default = "localhost";
+        description = ''
+          Hostname of IMAP server to connect to.
+        '';
+      };
+      port = mkOption {
+        type = types.port;
+        default = 993;
+        description = ''
+          Port of the IMAP server to connect to.
+        '';
+      };
+      username = mkOption {
+        type = types.str;
+        example = "postmaster@example.org";
+        description = ''
+          Login username for the IMAP connection.
+        '';
+      };
+      passwordFile = mkOption {
+        type = types.str;
+        example = "/run/secrets/dovecot_pw";
+        description = ''
+          File containing the login password for the IMAP connection.
+        '';
+      };
+    };
+    folders = {
+      inbox = mkOption {
+        type = types.str;
+        default = "INBOX";
+        description = ''
+          IMAP mailbox that is checked for incoming DMARC aggregate reports
+        '';
+      };
+      done = mkOption {
+        type = types.str;
+        default = "Archive";
+        description = ''
+          IMAP mailbox that successfully processed reports are moved to.
+        '';
+      };
+      error = mkOption {
+        type = types.str;
+        default = "Invalid";
+        description = ''
+          IMAP mailbox that emails are moved to that could not be processed.
+        '';
+      };
+    };
+    pollIntervalSeconds = mkOption {
+      type = types.ints.unsigned;
+      default = 60;
+      description = ''
+        How often to poll the IMAP server in seconds.
+      '';
+    };
+    deduplicationMaxSeconds = mkOption {
+      type = types.ints.unsigned;
+      default = 604800;
+      defaultText = "7 days (in seconds)";
+      description = ''
+        How long individual report IDs will be remembered to avoid
+        counting double delivered reports twice.
+      '';
+    };
+    debug = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to declare enable <literal>--debug</literal>.
+      '';
+    };
+  };
+  serviceOpts = {
+    path = with pkgs; [ envsubst coreutils ];
+    serviceConfig = {
+      StateDirectory = "prometheus-dmarc-exporter";
+      WorkingDirectory = "/var/lib/prometheus-dmarc-exporter";
+      ExecStart = "${pkgs.writeShellScript "setup-cfg" ''
+        export IMAP_PASSWORD="$(<${cfg.imap.passwordFile})"
+        envsubst \
+          -i ${pkgs.writeText "dmarc-exporter.json.template" json} \
+          -o ''${STATE_DIRECTORY}/dmarc-exporter.json
+
+        exec ${pkgs.prometheus-dmarc-exporter}/bin/prometheus-dmarc-exporter \
+          --configuration /var/lib/prometheus-dmarc-exporter/dmarc-exporter.json \
+          ${optionalString cfg.debug "--debug"}
+      ''}";
+    };
+  };
+}
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix b/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix
index 5b35bb29a301..55a61c4949ee 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix
@@ -32,10 +32,10 @@ in
     script = ''
       ${optionalString (cfg.tokenPath != null)
       "export FASTLY_API_TOKEN=$(cat ${toString cfg.tokenPath})"}
-      ${pkgs.fastly-exporter}/bin/fastly-exporter \
-        -endpoint http://${cfg.listenAddress}:${cfg.port}/metrics
+      ${pkgs.prometheus-fastly-exporter}/bin/fastly-exporter \
+        -listen http://${cfg.listenAddress}:${toString cfg.port}
         ${optionalString cfg.debug "-debug true"} \
-        ${optionalString cfg.configFile "-config-file ${cfg.configFile}"}
+        ${optionalString (cfg.configFile != null) "-config-file ${cfg.configFile}"}
     '';
   };
 }
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix b/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix
index 3cdd7866bd4d..6f69f5919d1e 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix
@@ -46,11 +46,11 @@ in
     serviceConfig = {
       ExecStart = ''
         ${pkgs.prometheus-nginx-exporter}/bin/nginx-prometheus-exporter \
-          --nginx.scrape-uri '${cfg.scrapeUri}' \
-          --nginx.ssl-verify ${boolToString cfg.sslVerify} \
-          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
-          --web.telemetry-path ${cfg.telemetryPath} \
-          --prometheus.const-labels ${concatStringsSep "," cfg.constLabels} \
+          --nginx.scrape-uri='${cfg.scrapeUri}' \
+          --nginx.ssl-verify=${boolToString cfg.sslVerify} \
+          --web.listen-address=${cfg.listenAddress}:${toString cfg.port} \
+          --web.telemetry-path=${cfg.telemetryPath} \
+          --prometheus.const-labels=${concatStringsSep "," cfg.constLabels} \
           ${concatStringsSep " \\\n  " cfg.extraFlags}
       '';
     };
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/pve.nix b/nixos/modules/services/monitoring/prometheus/exporters/pve.nix
new file mode 100644
index 000000000000..ef708414c95e
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/pve.nix
@@ -0,0 +1,118 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+let
+  cfg = config.services.prometheus.exporters.pve;
+
+  # pve exporter requires a config file so create an empty one if configFile is not provided
+  emptyConfigFile = pkgs.writeTextFile {
+    name = "pve.yml";
+    text = "default:";
+  };
+
+  computedConfigFile = "${if cfg.configFile == null then emptyConfigFile else cfg.configFile}";
+in
+{
+  port = 9221;
+  extraOpts = {
+    package = mkOption {
+      type = types.package;
+      default = pkgs.prometheus-pve-exporter;
+      defaultText = literalExpression "pkgs.prometheus-pve-exporter";
+      example = literalExpression "pkgs.prometheus-pve-exporter";
+      description = ''
+        The package to use for prometheus-pve-exporter
+      '';
+    };
+
+    environmentFile = mkOption {
+      type = with types; nullOr path;
+      default = null;
+      example = "/etc/prometheus-pve-exporter/pve.env";
+      description = ''
+        Path to the service's environment file. This path can either be a computed path in /nix/store or a path in the local filesystem.
+
+        The environment file should NOT be stored in /nix/store as it contains passwords and/or keys in plain text.
+
+        Environment reference: https://github.com/prometheus-pve/prometheus-pve-exporter#authentication
+      '';
+    };
+
+    configFile = mkOption {
+      type = with types; nullOr path;
+      default = null;
+      example = "/etc/prometheus-pve-exporter/pve.yml";
+      description = ''
+        Path to the service's config file. This path can either be a computed path in /nix/store or a path in the local filesystem.
+
+        The config file should NOT be stored in /nix/store as it will contain passwords and/or keys in plain text.
+
+        If both configFile and environmentFile are provided, the configFile option will be ignored.
+
+        Configuration reference: https://github.com/prometheus-pve/prometheus-pve-exporter/#authentication
+      '';
+    };
+
+    collectors = {
+      status = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Collect Node/VM/CT status
+        '';
+      };
+      version = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Collect PVE version info
+        '';
+      };
+      node = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Collect PVE node info
+        '';
+      };
+      cluster = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Collect PVE cluster info
+        '';
+      };
+      resources = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Collect PVE resources info
+        '';
+      };
+      config = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Collect PVE onboot status
+        '';
+      };
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${cfg.package}/bin/pve_exporter \
+          --${if cfg.collectors.status == true then "" else "no-"}collector.status \
+          --${if cfg.collectors.version == true then "" else "no-"}collector.version \
+          --${if cfg.collectors.node == true then "" else "no-"}collector.node \
+          --${if cfg.collectors.cluster == true then "" else "no-"}collector.cluster \
+          --${if cfg.collectors.resources == true then "" else "no-"}collector.resources \
+          --${if cfg.collectors.config == true then "" else "no-"}collector.config \
+          ${computedConfigFile} \
+          ${toString cfg.port} ${cfg.listenAddress}
+      '';
+    } // optionalAttrs (cfg.environmentFile != null) {
+          EnvironmentFile = cfg.environmentFile;
+    };
+  };
+}
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix b/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix
index b6416b93e69c..bac98364538d 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix
@@ -25,7 +25,8 @@ in {
         [ "/dev/sda", "/dev/nvme0n1" ];
       '';
       description = ''
-        Paths to disks that will be monitored.
+        Paths to the disks that will be monitored. Will autodiscover
+        all disks if none given.
       '';
     };
     maxInterval = mkOption {
@@ -41,13 +42,23 @@ in {
   serviceOpts = {
     serviceConfig = {
       AmbientCapabilities = [
+        "CAP_SYS_RAWIO"
         "CAP_SYS_ADMIN"
       ];
       CapabilityBoundingSet = [
+        "CAP_SYS_RAWIO"
         "CAP_SYS_ADMIN"
       ];
       DevicePolicy = "closed";
-      DeviceAllow = lib.mkForce cfg.devices;
+      DeviceAllow = lib.mkOverride 100 (
+        if cfg.devices != [] then
+          cfg.devices
+        else [
+          "block-blkext rw"
+          "block-sd rw"
+          "char-nvme rw"
+        ]
+      );
       ExecStart = ''
         ${pkgs.prometheus-smartctl-exporter}/bin/smartctl_exporter -config ${configFile}
       '';
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/systemd.nix b/nixos/modules/services/monitoring/prometheus/exporters/systemd.nix
index c0a50f07d717..2edd1de83e1b 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters/systemd.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters/systemd.nix
@@ -11,7 +11,7 @@ in {
     serviceConfig = {
       ExecStart = ''
         ${pkgs.prometheus-systemd-exporter}/bin/systemd_exporter \
-          --web.listen-address ${cfg.listenAddress}:${toString cfg.port}
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} ${concatStringsSep " " cfg.extraFlags}
       '';
       RestrictAddressFamilies = [
         # Need AF_UNIX to collect data
diff --git a/nixos/modules/services/monitoring/smartd.nix b/nixos/modules/services/monitoring/smartd.nix
index 73021b1b4d38..6d39cc3e4e6b 100644
--- a/nixos/modules/services/monitoring/smartd.nix
+++ b/nixos/modules/services/monitoring/smartd.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
@@ -8,6 +8,7 @@ let
        + optionalString (config.networking.domain != null) ".${config.networking.domain}";
 
   cfg = config.services.smartd;
+  opt = options.services.smartd;
 
   nm = cfg.notifications.mail;
   nw = cfg.notifications.wall;
@@ -211,6 +212,7 @@ in
 
         autodetected = mkOption {
           default = cfg.defaults.monitored;
+          defaultText = literalExpression "config.${opt.defaults.monitored}";
           type = types.separatedString " ";
           description = ''
             Like <option>services.smartd.defaults.monitored</option>, but for the
diff --git a/nixos/modules/services/monitoring/thanos.nix b/nixos/modules/services/monitoring/thanos.nix
index da626788d827..9e93d8dbb0ef 100644
--- a/nixos/modules/services/monitoring/thanos.nix
+++ b/nixos/modules/services/monitoring/thanos.nix
@@ -83,6 +83,9 @@ let
   mkArgumentsOption = cmd: mkOption {
     type = types.listOf types.str;
     default = argumentsOf cmd;
+    defaultText = literalDocBook ''
+      calculated from <literal>config.services.thanos.${cmd}</literal>
+    '';
     description = ''
       Arguments to the <literal>thanos ${cmd}</literal> command.
 
diff --git a/nixos/modules/services/monitoring/uptime.nix b/nixos/modules/services/monitoring/uptime.nix
index 245badc3e44f..79b86be6cc71 100644
--- a/nixos/modules/services/monitoring/uptime.nix
+++ b/nixos/modules/services/monitoring/uptime.nix
@@ -1,8 +1,9 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 let
-  inherit (lib) mkOption mkEnableOption mkIf mkMerge types optional;
+  inherit (lib) literalExpression mkOption mkEnableOption mkIf mkMerge types optional;
 
   cfg = config.services.uptime;
+  opt = options.services.uptime;
 
   configDir = pkgs.runCommand "config" { preferLocalBuild = true; }
   (if cfg.configFile != null then ''
@@ -52,7 +53,10 @@ in {
 
     enableWebService = mkEnableOption "the uptime monitoring program web service";
 
-    enableSeparateMonitoringService = mkEnableOption "the uptime monitoring service" // { default = cfg.enableWebService; };
+    enableSeparateMonitoringService = mkEnableOption "the uptime monitoring service" // {
+      default = cfg.enableWebService;
+      defaultText = literalExpression "config.${opt.enableWebService}";
+    };
 
     nodeEnv = mkOption {
       description = "The node environment to run in (development, production, etc.)";
diff --git a/nixos/modules/services/monitoring/zabbix-proxy.nix b/nixos/modules/services/monitoring/zabbix-proxy.nix
index b5009f47f175..0ebd7bcff834 100644
--- a/nixos/modules/services/monitoring/zabbix-proxy.nix
+++ b/nixos/modules/services/monitoring/zabbix-proxy.nix
@@ -1,7 +1,8 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
   cfg = config.services.zabbixProxy;
+  opt = options.services.zabbixProxy;
   pgsql = config.services.postgresql;
   mysql = config.services.mysql;
 
@@ -103,6 +104,11 @@ in
         port = mkOption {
           type = types.int;
           default = if cfg.database.type == "mysql" then mysql.port else pgsql.port;
+          defaultText = literalExpression ''
+            if config.${opt.database.type} == "mysql"
+            then config.${options.services.mysql.port}
+            else config.${options.services.postgresql.port}
+          '';
           description = "Database host port.";
         };
 
diff --git a/nixos/modules/services/monitoring/zabbix-server.nix b/nixos/modules/services/monitoring/zabbix-server.nix
index 0141c073da25..9f960517a81b 100644
--- a/nixos/modules/services/monitoring/zabbix-server.nix
+++ b/nixos/modules/services/monitoring/zabbix-server.nix
@@ -1,7 +1,8 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
   cfg = config.services.zabbixServer;
+  opt = options.services.zabbixServer;
   pgsql = config.services.postgresql;
   mysql = config.services.mysql;
 
@@ -95,6 +96,11 @@ in
         port = mkOption {
           type = types.int;
           default = if cfg.database.type == "mysql" then mysql.port else pgsql.port;
+          defaultText = literalExpression ''
+            if config.${opt.database.type} == "mysql"
+            then config.${options.services.mysql.port}
+            else config.${options.services.postgresql.port}
+          '';
           description = "Database host port.";
         };
 
diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix
index e313589134f1..7a1444decafa 100644
--- a/nixos/modules/services/network-filesystems/ceph.nix
+++ b/nixos/modules/services/network-filesystems/ceph.nix
@@ -181,8 +181,8 @@ in
 
       rgwMimeTypesFile = mkOption {
         type = with types; nullOr path;
-        default = "${pkgs.mime-types}/etc/mime.types";
-        defaultText = literalExpression ''"''${pkgs.mime-types}/etc/mime.types"'';
+        default = "${pkgs.mailcap}/etc/mime.types";
+        defaultText = literalExpression ''"''${pkgs.mailcap}/etc/mime.types"'';
         description = ''
           Path to mime types used by radosgw.
         '';
diff --git a/nixos/modules/services/network-filesystems/ipfs.nix b/nixos/modules/services/network-filesystems/ipfs.nix
index 5482b2aaf88c..17da020bf3e2 100644
--- a/nixos/modules/services/network-filesystems/ipfs.nix
+++ b/nixos/modules/services/network-filesystems/ipfs.nix
@@ -259,25 +259,18 @@ in
         ipfs --offline config Mounts.IPFS ${cfg.ipfsMountDir}
         ipfs --offline config Mounts.IPNS ${cfg.ipnsMountDir}
       '' + optionalString cfg.autoMigrate ''
-        ${pkgs.ipfs-migrator}/bin/fs-repo-migrations -y
-      '' + concatStringsSep "\n" (collect
-        isString
-        (mapAttrsRecursive
-          (path: value:
-            # Using heredoc below so that the value is never improperly quoted
-            ''
-              read value <<EOF
-              ${builtins.toJSON value}
-              EOF
-              ipfs --offline config --json "${concatStringsSep "." path}" "$value"
-            '')
-          ({
-            Addresses.API = cfg.apiAddress;
-            Addresses.Gateway = cfg.gatewayAddress;
-            Addresses.Swarm = cfg.swarmAddress;
-          } //
-          cfg.extraConfig))
-      );
+        ${pkgs.ipfs-migrator}/bin/fs-repo-migrations -to '${cfg.package.repoVersion}' -y
+      '' + ''
+        ipfs --offline config show \
+          | ${pkgs.jq}/bin/jq '. * $extraConfig' --argjson extraConfig ${
+              escapeShellArg (builtins.toJSON ({
+                Addresses.API = cfg.apiAddress;
+                Addresses.Gateway = cfg.gatewayAddress;
+                Addresses.Swarm = cfg.swarmAddress;
+              } // cfg.extraConfig))
+            } \
+          | ipfs --offline config replace -
+      '';
       serviceConfig = {
         ExecStart = [ "" "${cfg.package}/bin/ipfs daemon ${ipfsFlags}" ];
         User = cfg.user;
diff --git a/nixos/modules/services/network-filesystems/moosefs.nix b/nixos/modules/services/network-filesystems/moosefs.nix
new file mode 100644
index 000000000000..88b2ada37e75
--- /dev/null
+++ b/nixos/modules/services/network-filesystems/moosefs.nix
@@ -0,0 +1,249 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.moosefs;
+
+  mfsUser = if cfg.runAsUser then "moosefs" else "root";
+
+  settingsFormat = let
+    listSep = " ";
+    allowedTypes = with types; [ bool int float str ];
+    valueToString = val:
+        if isList val then concatStringsSep listSep (map (x: valueToString x) val)
+        else if isBool val then (if val then "1" else "0")
+        else toString val;
+
+    in {
+      type = with types; let
+        valueType = oneOf ([
+          (listOf valueType)
+        ] ++ allowedTypes) // {
+          description = "Flat key-value file";
+        };
+      in attrsOf valueType;
+
+      generate = name: value:
+        pkgs.writeText name ( lib.concatStringsSep "\n" (
+          lib.mapAttrsToList (key: val: "${key} = ${valueToString val}") value ));
+    };
+
+
+  initTool = pkgs.writeShellScriptBin "mfsmaster-init" ''
+    if [ ! -e ${cfg.master.settings.DATA_PATH}/metadata.mfs ]; then
+      cp ${pkgs.moosefs}/var/mfs/metadata.mfs.empty ${cfg.master.settings.DATA_PATH}
+      chmod +w ${cfg.master.settings.DATA_PATH}/metadata.mfs.empty
+      ${pkgs.moosefs}/bin/mfsmaster -a -c ${masterCfg} start
+      ${pkgs.moosefs}/bin/mfsmaster -c ${masterCfg} stop
+      rm ${cfg.master.settings.DATA_PATH}/metadata.mfs.empty
+    fi
+  '';
+
+  # master config file
+  masterCfg = settingsFormat.generate
+    "mfsmaster.cfg" cfg.master.settings;
+
+  # metalogger config file
+  metaloggerCfg = settingsFormat.generate
+    "mfsmetalogger.cfg" cfg.metalogger.settings;
+
+  # chunkserver config file
+  chunkserverCfg = settingsFormat.generate
+    "mfschunkserver.cfg" cfg.chunkserver.settings;
+
+  # generic template for all deamons
+  systemdService = name: extraConfig: configFile: {
+    wantedBy = [ "multi-user.target" ];
+    wants = [ "network-online.target" ];
+    after = [ "network.target" "network-online.target" ];
+
+    serviceConfig = {
+      Type = "forking";
+      ExecStart  = "${pkgs.moosefs}/bin/mfs${name} -c ${configFile} start";
+      ExecStop   = "${pkgs.moosefs}/bin/mfs${name} -c ${configFile} stop";
+      ExecReload = "${pkgs.moosefs}/bin/mfs${name} -c ${configFile} reload";
+      PIDFile = "${cfg."${name}".settings.DATA_PATH}/.mfs${name}.lock";
+    } // extraConfig;
+  };
+
+in {
+  ###### interface
+
+  options = {
+    services.moosefs = {
+      masterHost = mkOption {
+        type = types.str;
+        default = null;
+        description = "IP or DNS name of master host.";
+      };
+
+      runAsUser = mkOption {
+        type = types.bool;
+        default = true;
+        example = true;
+        description = "Run daemons as user moosefs instead of root.";
+      };
+
+      client.enable = mkEnableOption "Moosefs client.";
+
+      master = {
+        enable = mkOption {
+          type = types.bool;
+          description = ''
+            Enable Moosefs master daemon.
+
+            You need to run <literal>mfsmaster-init</literal> on a freshly installed master server to
+            initialize the <literal>DATA_PATH</literal> direcory.
+          '';
+          default = false;
+        };
+
+        exports = mkOption {
+          type = with types; listOf str;
+          default = null;
+          description = "Paths to export (see mfsexports.cfg).";
+          example = [
+            "* / rw,alldirs,admin,maproot=0:0"
+            "* . rw"
+          ];
+        };
+
+        openFirewall = mkOption {
+          type = types.bool;
+          description = "Whether to automatically open the necessary ports in the firewall.";
+          default = false;
+        };
+
+        settings = mkOption {
+          type = types.submodule {
+            freeformType = settingsFormat.type;
+
+            options.DATA_PATH = mkOption {
+              type = types.str;
+              default = "/var/lib/mfs";
+              description = "Data storage directory.";
+            };
+          };
+
+          description = "Contents of config file (mfsmaster.cfg).";
+        };
+      };
+
+      metalogger = {
+        enable = mkEnableOption "Moosefs metalogger daemon.";
+
+        settings = mkOption {
+          type = types.submodule {
+            freeformType = settingsFormat.type;
+
+            options.DATA_PATH = mkOption {
+              type = types.str;
+              default = "/var/lib/mfs";
+              description = "Data storage directory";
+            };
+          };
+
+          description = "Contents of metalogger config file (mfsmetalogger.cfg).";
+        };
+      };
+
+      chunkserver = {
+        enable = mkEnableOption "Moosefs chunkserver daemon.";
+
+        openFirewall = mkOption {
+          type = types.bool;
+          description = "Whether to automatically open the necessary ports in the firewall.";
+          default = false;
+        };
+
+        hdds = mkOption {
+          type = with types; listOf str;
+          default =  null;
+          description = "Mount points to be used by chunkserver for storage (see mfshdd.cfg).";
+          example = [ "/mnt/hdd1" ];
+        };
+
+        settings = mkOption {
+          type = types.submodule {
+            freeformType = settingsFormat.type;
+
+            options.DATA_PATH = mkOption {
+              type = types.str;
+              default = "/var/lib/mfs";
+              description = "Directory for lock file.";
+            };
+          };
+
+          description = "Contents of chunkserver config file (mfschunkserver.cfg).";
+        };
+      };
+    };
+  };
+
+  ###### implementation
+
+  config =  mkIf ( cfg.client.enable || cfg.master.enable || cfg.metalogger.enable || cfg.chunkserver.enable ) {
+
+    warnings = [ ( mkIf (!cfg.runAsUser) "Running moosefs services as root is not recommended.") ];
+
+    # Service settings
+    services.moosefs = {
+      master.settings = mkIf cfg.master.enable {
+        WORKING_USER = mfsUser;
+        EXPORTS_FILENAME = toString ( pkgs.writeText "mfsexports.cfg"
+          (concatStringsSep "\n" cfg.master.exports));
+      };
+
+      metalogger.settings = mkIf cfg.metalogger.enable {
+        WORKING_USER = mfsUser;
+        MASTER_HOST = cfg.masterHost;
+      };
+
+      chunkserver.settings = mkIf cfg.chunkserver.enable {
+        WORKING_USER = mfsUser;
+        MASTER_HOST = cfg.masterHost;
+        HDD_CONF_FILENAME = toString ( pkgs.writeText "mfshdd.cfg"
+          (concatStringsSep "\n" cfg.chunkserver.hdds));
+      };
+    };
+
+    # Create system user account for daemons
+    users = mkIf ( cfg.runAsUser && ( cfg.master.enable || cfg.metalogger.enable || cfg.chunkserver.enable ) ) {
+      users.moosefs = {
+        isSystemUser = true;
+        description = "moosefs daemon user";
+        group = "moosefs";
+      };
+      groups.moosefs = {};
+    };
+
+    environment.systemPackages =
+      (lib.optional cfg.client.enable pkgs.moosefs) ++
+      (lib.optional cfg.master.enable initTool);
+
+    networking.firewall.allowedTCPPorts =
+      (lib.optionals cfg.master.openFirewall [ 9419 9420 9421 ]) ++
+      (lib.optional cfg.chunkserver.openFirewall 9422);
+
+    # Ensure storage directories exist
+    systemd.tmpfiles.rules =
+         optional cfg.master.enable "d ${cfg.master.settings.DATA_PATH} 0700 ${mfsUser} ${mfsUser}"
+      ++ optional cfg.metalogger.enable "d ${cfg.metalogger.settings.DATA_PATH} 0700 ${mfsUser} ${mfsUser}"
+      ++ optional cfg.chunkserver.enable "d ${cfg.chunkserver.settings.DATA_PATH} 0700 ${mfsUser} ${mfsUser}";
+
+    # Service definitions
+    systemd.services.mfs-master = mkIf cfg.master.enable
+    ( systemdService "master" {
+      TimeoutStartSec = 1800;
+      TimeoutStopSec = 1800;
+      Restart = "no";
+    } masterCfg );
+
+    systemd.services.mfs-metalogger = mkIf cfg.metalogger.enable
+      ( systemdService "metalogger" { Restart = "on-abnormal"; } metaloggerCfg );
+
+    systemd.services.mfs-chunkserver = mkIf cfg.chunkserver.enable
+      ( systemdService "chunkserver" { Restart = "on-abnormal"; } chunkserverCfg );
+    };
+}
diff --git a/nixos/modules/services/network-filesystems/rsyncd.nix b/nixos/modules/services/network-filesystems/rsyncd.nix
index edac86eb0e30..e72f9b54cd6f 100644
--- a/nixos/modules/services/network-filesystems/rsyncd.nix
+++ b/nixos/modules/services/network-filesystems/rsyncd.nix
@@ -79,7 +79,7 @@ in {
     in {
       services.rsync = {
         enable = !cfg.socketActivated;
-        aliases = [ "rsyncd" ];
+        aliases = [ "rsyncd.service" ];
 
         description = "fast remote file copy program daemon";
         after = [ "network.target" ];
diff --git a/nixos/modules/services/network-filesystems/samba.nix b/nixos/modules/services/network-filesystems/samba.nix
index 9ed755d0465c..992f948e8cd5 100644
--- a/nixos/modules/services/network-filesystems/samba.nix
+++ b/nixos/modules/services/network-filesystems/samba.nix
@@ -224,6 +224,7 @@ in
           targets.samba = {
             description = "Samba Server";
             after = [ "network.target" ];
+            wants = [ "network-online.target" ];
             wantedBy = [ "multi-user.target" ];
           };
           # Refer to https://github.com/samba-team/samba/tree/master/packaging/systemd
diff --git a/nixos/modules/services/networking/adguardhome.nix b/nixos/modules/services/networking/adguardhome.nix
index 03f9b9f9bad4..98ddf0716087 100644
--- a/nixos/modules/services/networking/adguardhome.nix
+++ b/nixos/modules/services/networking/adguardhome.nix
@@ -10,12 +10,20 @@ let
     "--pidfile /run/AdGuardHome/AdGuardHome.pid"
     "--work-dir /var/lib/AdGuardHome/"
     "--config /var/lib/AdGuardHome/AdGuardHome.yaml"
-    "--host ${cfg.host}"
-    "--port ${toString cfg.port}"
   ] ++ cfg.extraArgs);
 
-in
-{
+  baseConfig = {
+    bind_host = cfg.host;
+    bind_port = cfg.port;
+  };
+
+  configFile = pkgs.writeTextFile {
+    name = "AdGuardHome.yaml";
+    text = builtins.toJSON (recursiveUpdate cfg.settings baseConfig);
+    checkPhase = "${pkgs.adguardhome}/bin/adguardhome -c $out --check-config";
+  };
+
+in {
   options.services.adguardhome = with types; {
     enable = mkEnableOption "AdGuard Home network-wide ad blocker";
 
@@ -44,6 +52,31 @@ in
       '';
     };
 
+    mutableSettings = mkOption {
+      default = true;
+      type = bool;
+      description = ''
+        Allow changes made on the AdGuard Home web interface to persist between
+        service restarts.
+      '';
+    };
+
+    settings = mkOption {
+      type = (pkgs.formats.yaml { }).type;
+      default = { };
+      description = ''
+        AdGuard Home configuration. Refer to
+        <link xlink:href="https://github.com/AdguardTeam/AdGuardHome/wiki/Configuration#configuration-file"/>
+        for details on supported values.
+
+        <note><para>
+          On start and if <option>mutableSettings</option> is <literal>true</literal>,
+          these options are merged into the configuration file on start, taking
+          precedence over configuration changes made on the web interface.
+        </para></note>
+      '';
+    };
+
     extraArgs = mkOption {
       default = [ ];
       type = listOf str;
@@ -54,6 +87,22 @@ in
   };
 
   config = mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.settings != { }
+          -> (hasAttrByPath [ "dns" "bind_host" ] cfg.settings)
+          || (hasAttrByPath [ "dns" "bind_hosts" ] cfg.settings);
+        message =
+          "AdGuard setting dns.bind_host or dns.bind_hosts needs to be configured for a minimal working configuration";
+      }
+      {
+        assertion = cfg.settings != { }
+          -> hasAttrByPath [ "dns" "bootstrap_dns" ] cfg.settings;
+        message =
+          "AdGuard setting dns.bootstrap_dns needs to be configured for a minimal working configuration";
+      }
+    ];
+
     systemd.services.adguardhome = {
       description = "AdGuard Home: Network-level blocker";
       after = [ "network.target" ];
@@ -62,6 +111,19 @@ in
         StartLimitIntervalSec = 5;
         StartLimitBurst = 10;
       };
+
+      preStart = optionalString (cfg.settings != { }) ''
+        if    [ -e "$STATE_DIRECTORY/AdGuardHome.yaml" ] \
+           && [ "${toString cfg.mutableSettings}" = "1" ]; then
+          # Writing directly to AdGuardHome.yaml results in empty file
+          ${pkgs.yaml-merge}/bin/yaml-merge "$STATE_DIRECTORY/AdGuardHome.yaml" "${configFile}" > "$STATE_DIRECTORY/AdGuardHome.yaml.tmp"
+          mv "$STATE_DIRECTORY/AdGuardHome.yaml.tmp" "$STATE_DIRECTORY/AdGuardHome.yaml"
+        else
+          cp --force "${configFile}" "$STATE_DIRECTORY/AdGuardHome.yaml"
+          chmod 600 "$STATE_DIRECTORY/AdGuardHome.yaml"
+        fi
+      '';
+
       serviceConfig = {
         DynamicUser = true;
         ExecStart = "${pkgs.adguardhome}/bin/adguardhome ${args}";
diff --git a/nixos/modules/services/networking/amuled.nix b/nixos/modules/services/networking/amuled.nix
index 39320643dd5e..aa72a047526b 100644
--- a/nixos/modules/services/networking/amuled.nix
+++ b/nixos/modules/services/networking/amuled.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.amule;
+  opt = options.services.amule;
   user = if cfg.user != null then cfg.user else "amule";
 in
 
@@ -26,6 +27,9 @@ in
       dataDir = mkOption {
         type = types.str;
         default = "/home/${user}/";
+        defaultText = literalExpression ''
+          "/home/''${config.${opt.user}}/"
+        '';
         description = ''
           The directory holding configuration, incoming and temporary files.
         '';
@@ -72,7 +76,7 @@ in
 
       script = ''
         ${pkgs.su}/bin/su -s ${pkgs.runtimeShell} ${user} \
-            -c 'HOME="${cfg.dataDir}" ${pkgs.amuleDaemon}/bin/amuled'
+            -c 'HOME="${cfg.dataDir}" ${pkgs.amule-daemon}/bin/amuled'
       '';
     };
   };
diff --git a/nixos/modules/services/networking/bind.nix b/nixos/modules/services/networking/bind.nix
index e44f8d4cf302..2045612ec054 100644
--- a/nixos/modules/services/networking/bind.nix
+++ b/nixos/modules/services/networking/bind.nix
@@ -59,7 +59,7 @@ let
         listen-on-v6 { ${concatMapStrings (entry: " ${entry}; ") cfg.listenOnIpv6} };
         allow-query { cachenetworks; };
         blackhole { badnetworks; };
-        forward first;
+        forward ${cfg.forward};
         forwarders { ${concatMapStrings (entry: " ${entry}; ") cfg.forwarders} };
         directory "${cfg.directory}";
         pid-file "/run/named/named.pid";
@@ -151,6 +151,14 @@ in
         ";
       };
 
+      forward = mkOption {
+        default = "first";
+        type = types.enum ["first" "only"];
+        description = "
+          Whether to forward 'first' (try forwarding but lookup directly if forwarding fails) or 'only'.
+        ";
+      };
+
       listenOn = mkOption {
         default = [ "any" ];
         type = types.listOf types.str;
diff --git a/nixos/modules/services/networking/bird.nix b/nixos/modules/services/networking/bird.nix
index c14adbda3c5a..3049c4f2bce9 100644
--- a/nixos/modules/services/networking/bird.nix
+++ b/nixos/modules/services/networking/bird.nix
@@ -3,85 +3,101 @@
 let
   inherit (lib) mkEnableOption mkIf mkOption optionalString types;
 
-  generic = variant:
-    let
-      cfg = config.services.${variant};
-      pkg = pkgs.${variant};
-      birdBin = if variant == "bird6" then "bird6" else "bird";
-      birdc = if variant == "bird6" then "birdc6" else "birdc";
-      descr =
-        { bird = "1.6.x with IPv4 support";
-          bird6 = "1.6.x with IPv6 support";
-          bird2 = "2.x";
-        }.${variant};
-    in {
-      ###### interface
-      options = {
-        services.${variant} = {
-          enable = mkEnableOption "BIRD Internet Routing Daemon (${descr})";
-          config = mkOption {
-            type = types.lines;
-            description = ''
-              BIRD Internet Routing Daemon configuration file.
-              <link xlink:href='http://bird.network.cz/'/>
-            '';
-          };
-          checkConfig = mkOption {
-            type = types.bool;
-            default = true;
-            description = ''
-              Whether the config should be checked at build time.
-              Disabling this might become necessary if the config includes files not present during build time.
-            '';
-          };
-        };
+  cfg = config.services.bird2;
+  caps = [ "CAP_NET_ADMIN" "CAP_NET_BIND_SERVICE" "CAP_NET_RAW" ];
+in
+{
+  ###### interface
+  options = {
+    services.bird2 = {
+      enable = mkEnableOption "BIRD Internet Routing Daemon";
+      config = mkOption {
+        type = types.lines;
+        description = ''
+          BIRD Internet Routing Daemon configuration file.
+          <link xlink:href='http://bird.network.cz/'/>
+        '';
       };
+      checkConfig = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Whether the config should be checked at build time.
+          When the config can't be checked during build time, for example when it includes
+          other files, either disable this option or use <code>preCheckConfig</code> to create
+          the included files before checking.
+        '';
+      };
+      preCheckConfig = mkOption {
+        type = types.lines;
+        default = "";
+        example = ''
+          echo "cost 100;" > include.conf
+        '';
+        description = ''
+          Commands to execute before the config file check. The file to be checked will be
+          available as <code>bird2.conf</code> in the current directory.
 
-      ###### implementation
-      config = mkIf cfg.enable {
-        environment.systemPackages = [ pkg ];
-
-        environment.etc."bird/${variant}.conf".source = pkgs.writeTextFile {
-          name = "${variant}.conf";
-          text = cfg.config;
-          checkPhase = optionalString cfg.checkConfig ''
-            ${pkg}/bin/${birdBin} -d -p -c $out
-          '';
-        };
-
-        systemd.services.${variant} = {
-          description = "BIRD Internet Routing Daemon (${descr})";
-          wantedBy = [ "multi-user.target" ];
-          reloadIfChanged = true;
-          restartTriggers = [ config.environment.etc."bird/${variant}.conf".source ];
-          serviceConfig = {
-            Type = "forking";
-            Restart = "on-failure";
-            ExecStart = "${pkg}/bin/${birdBin} -c /etc/bird/${variant}.conf -u ${variant} -g ${variant}";
-            ExecReload = "/bin/sh -c '${pkg}/bin/${birdBin} -c /etc/bird/${variant}.conf -p && ${pkg}/bin/${birdc} configure'";
-            ExecStop = "${pkg}/bin/${birdc} down";
-            CapabilityBoundingSet = [ "CAP_CHOWN" "CAP_FOWNER" "CAP_DAC_OVERRIDE" "CAP_SETUID" "CAP_SETGID"
-                                      # see bird/sysdep/linux/syspriv.h
-                                      "CAP_NET_BIND_SERVICE" "CAP_NET_BROADCAST" "CAP_NET_ADMIN" "CAP_NET_RAW" ];
-            ProtectSystem = "full";
-            ProtectHome = "yes";
-            SystemCallFilter="~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io";
-            MemoryDenyWriteExecute = "yes";
-          };
-        };
-        users = {
-          users.${variant} = {
-            description = "BIRD Internet Routing Daemon user";
-            group = variant;
-            isSystemUser = true;
-          };
-          groups.${variant} = {};
-        };
+          Files created with this option will not be available at service runtime, only during
+          build time checking.
+        '';
       };
     };
+  };
 
-in
 
-{
-  imports = map generic [ "bird" "bird6" "bird2" ];
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "bird" ] "Use services.bird2 instead")
+    (lib.mkRemovedOptionModule [ "services" "bird6" ] "Use services.bird2 instead")
+  ];
+
+  ###### implementation
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.bird ];
+
+    environment.etc."bird/bird2.conf".source = pkgs.writeTextFile {
+      name = "bird2";
+      text = cfg.config;
+      checkPhase = optionalString cfg.checkConfig ''
+        ln -s $out bird2.conf
+        ${cfg.preCheckConfig}
+        ${pkgs.bird}/bin/bird -d -p -c bird2.conf
+      '';
+    };
+
+    systemd.services.bird2 = {
+      description = "BIRD Internet Routing Daemon";
+      wantedBy = [ "multi-user.target" ];
+      reloadIfChanged = true;
+      restartTriggers = [ config.environment.etc."bird/bird2.conf".source ];
+      serviceConfig = {
+        Type = "forking";
+        Restart = "on-failure";
+        User = "bird2";
+        Group = "bird2";
+        ExecStart = "${pkgs.bird}/bin/bird -c /etc/bird/bird2.conf";
+        ExecReload = "${pkgs.bird}/bin/birdc configure";
+        ExecStop = "${pkgs.bird}/bin/birdc down";
+        RuntimeDirectory = "bird";
+        CapabilityBoundingSet = caps;
+        AmbientCapabilities = caps;
+        ProtectSystem = "full";
+        ProtectHome = "yes";
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        SystemCallFilter = "~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io";
+        MemoryDenyWriteExecute = "yes";
+      };
+    };
+    users = {
+      users.bird2 = {
+        description = "BIRD Internet Routing Daemon user";
+        group = "bird2";
+        isSystemUser = true;
+      };
+      groups.bird2 = { };
+    };
+  };
 }
diff --git a/nixos/modules/services/networking/blocky.nix b/nixos/modules/services/networking/blocky.nix
new file mode 100644
index 000000000000..7488e05fc033
--- /dev/null
+++ b/nixos/modules/services/networking/blocky.nix
@@ -0,0 +1,40 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.blocky;
+
+  format = pkgs.formats.yaml { };
+  configFile = format.generate "config.yaml" cfg.settings;
+in
+{
+  options.services.blocky = {
+    enable = mkEnableOption "Fast and lightweight DNS proxy as ad-blocker for local network with many features";
+
+    settings = mkOption {
+      type = format.type;
+      default = { };
+      description = ''
+        Blocky configuration. Refer to
+        <link xlink:href="https://0xerr0r.github.io/blocky/configuration/"/>
+        for details on supported values.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.blocky = {
+      description = "A DNS proxy and ad-blocker for the local network";
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        DynamicUser = true;
+        ExecStart = "${pkgs.blocky}/bin/blocky --config ${configFile}";
+
+        AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+        CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/networking/connman.nix b/nixos/modules/services/networking/connman.nix
index 8886e7a30f1f..9945dc83a279 100644
--- a/nixos/modules/services/networking/connman.nix
+++ b/nixos/modules/services/networking/connman.nix
@@ -127,7 +127,7 @@ in {
       description = "ConnMan VPN service";
       wantedBy = [ "multi-user.target" ];
       after = [ "syslog.target" ];
-      before = [ "connman" ];
+      before = [ "connman.service" ];
       serviceConfig = {
         Type = "dbus";
         BusName = "net.connman.vpn";
@@ -140,7 +140,7 @@ in {
       description = "D-BUS Service";
       serviceConfig = {
         Name = "net.connman.vpn";
-        before = [ "connman" ];
+        before = [ "connman.service" ];
         ExecStart = "${cfg.package}/sbin/connman-vpnd -n";
         User = "root";
         SystemdService = "connman-vpn.service";
diff --git a/nixos/modules/services/networking/croc.nix b/nixos/modules/services/networking/croc.nix
index 9466adf71d8c..d044979e10df 100644
--- a/nixos/modules/services/networking/croc.nix
+++ b/nixos/modules/services/networking/croc.nix
@@ -51,7 +51,7 @@ in
         ProtectKernelLogs = true;
         ProtectKernelModules = true;
         ProtectKernelTunables = true;
-        ProtectProc = "noaccess";
+        ProtectProc = "invisible";
         ProtectSystem = "strict";
         RemoveIPC = true;
         RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
diff --git a/nixos/modules/services/networking/ddclient.nix b/nixos/modules/services/networking/ddclient.nix
index 8a2c0fc7080c..d025c8f8177a 100644
--- a/nixos/modules/services/networking/ddclient.nix
+++ b/nixos/modules/services/networking/ddclient.nix
@@ -13,7 +13,7 @@ let
     foreground=YES
     use=${cfg.use}
     login=${cfg.username}
-    password=
+    password=${lib.optionalString (cfg.protocol == "nsupdate") "/run/${RuntimeDirectory}/ddclient.key"}
     protocol=${cfg.protocol}
     ${lib.optionalString (cfg.script != "") "script=${cfg.script}"}
     ${lib.optionalString (cfg.server != "") "server=${cfg.server}"}
@@ -30,7 +30,9 @@ let
 
   preStart = ''
     install ${configFile} /run/${RuntimeDirectory}/ddclient.conf
-    ${lib.optionalString (cfg.configFile == null) (if (cfg.passwordFile != null) then ''
+    ${lib.optionalString (cfg.configFile == null) (if (cfg.protocol == "nsupdate") then ''
+      install ${cfg.passwordFile} /run/${RuntimeDirectory}/ddclient.key
+    '' else if (cfg.passwordFile != null) then ''
       password=$(printf "%q" "$(head -n 1 "${cfg.passwordFile}")")
       sed -i "s|^password=$|password=$password|" /run/${RuntimeDirectory}/ddclient.conf
     '' else ''
@@ -85,7 +87,9 @@ with lib;
       };
 
       username = mkOption {
-        default = "";
+        # For `nsupdate` username contains the path to the nsupdate executable
+        default = lib.optionalString (config.services.ddclient.protocol == "nsupdate") "${pkgs.bind.dnsutils}/bin/nsupdate";
+        defaultText = "";
         type = str;
         description = ''
           User name.
@@ -96,7 +100,7 @@ with lib;
         default = null;
         type = nullOr str;
         description = ''
-          A file containing the password.
+          A file containing the password or a TSIG key in named format when using the nsupdate protocol.
         '';
       };
 
diff --git a/nixos/modules/services/networking/dhcpcd.nix b/nixos/modules/services/networking/dhcpcd.nix
index 31e4b6ad2988..3eb7ca99eafd 100644
--- a/nixos/modules/services/networking/dhcpcd.nix
+++ b/nixos/modules/services/networking/dhcpcd.nix
@@ -183,6 +183,20 @@ in
 
   config = mkIf enableDHCP {
 
+    assertions = [ {
+      # dhcpcd doesn't start properly with malloc ∉ [ libc scudo ]
+      # see https://github.com/NixOS/nixpkgs/issues/151696
+      assertion =
+        dhcpcd.enablePrivSep
+          -> elem config.environment.memoryAllocator.provider [ "libc" "scudo" ];
+      message = ''
+        dhcpcd with privilege separation is incompatible with chosen system malloc.
+          Currently only the `libc` and `scudo` allocators are known to work.
+          To disable dhcpcd's privilege separation, overlay Nixpkgs and override dhcpcd
+          to set `enablePrivSep = false`.
+      '';
+    } ];
+
     systemd.services.dhcpcd = let
       cfgN = config.networking;
       hasDefaultGatewaySet = (cfgN.defaultGateway != null && cfgN.defaultGateway.address != "")
@@ -207,13 +221,20 @@ in
 
         serviceConfig =
           { Type = "forking";
-            PIDFile = "/run/dhcpcd.pid";
+            PIDFile = "/run/dhcpcd/pid";
+            RuntimeDirectory = "dhcpcd";
             ExecStart = "@${dhcpcd}/sbin/dhcpcd dhcpcd --quiet ${optionalString cfg.persistent "--persistent"} --config ${dhcpcdConf}";
             ExecReload = "${dhcpcd}/sbin/dhcpcd --rebind";
             Restart = "always";
           };
       };
 
+    users.users.dhcpcd = {
+      isSystemUser = true;
+      group = "dhcpcd";
+    };
+    users.groups.dhcpcd = {};
+
     environment.systemPackages = [ dhcpcd ];
 
     environment.etc."dhcpcd.exit-hook".source = exitHook;
diff --git a/nixos/modules/services/networking/dhcpd.nix b/nixos/modules/services/networking/dhcpd.nix
index 54e4f9002859..3c4c0069dfd0 100644
--- a/nixos/modules/services/networking/dhcpd.nix
+++ b/nixos/modules/services/networking/dhcpd.nix
@@ -28,38 +28,45 @@ let
       }
     '';
 
-  dhcpdService = postfix: cfg: optionalAttrs cfg.enable {
-    "dhcpd${postfix}" = {
-      description = "DHCPv${postfix} server";
-      wantedBy = [ "multi-user.target" ];
-      after = [ "network.target" ];
-
-      preStart = ''
-        mkdir -m 755 -p ${cfg.stateDir}
-        chown dhcpd:nogroup ${cfg.stateDir}
-        touch ${cfg.stateDir}/dhcpd.leases
-      '';
-
-      serviceConfig =
-        let
-          configFile = if cfg.configFile != null then cfg.configFile else writeConfig cfg;
-          args = [ "@${pkgs.dhcp}/sbin/dhcpd" "dhcpd${postfix}" "-${postfix}"
-                   "-pf" "/run/dhcpd${postfix}/dhcpd.pid"
-                   "-cf" "${configFile}"
-                   "-lf" "${cfg.stateDir}/dhcpd.leases"
-                   "-user" "dhcpd" "-group" "nogroup"
-                 ] ++ cfg.extraFlags
-                   ++ cfg.interfaces;
-
-        in {
-          ExecStart = concatMapStringsSep " " escapeShellArg args;
-          Type = "forking";
-          Restart = "always";
-          RuntimeDirectory = [ "dhcpd${postfix}" ];
-          PIDFile = "/run/dhcpd${postfix}/dhcpd.pid";
+  dhcpdService = postfix: cfg:
+    let
+      configFile =
+        if cfg.configFile != null
+          then cfg.configFile
+          else writeConfig cfg;
+      leaseFile = "/var/lib/dhcpd${postfix}/dhcpd.leases";
+      args = [
+        "@${pkgs.dhcp}/sbin/dhcpd" "dhcpd${postfix}" "-${postfix}"
+        "-pf" "/run/dhcpd${postfix}/dhcpd.pid"
+        "-cf" configFile
+        "-lf" leaseFile
+      ] ++ cfg.extraFlags
+        ++ cfg.interfaces;
+    in
+      optionalAttrs cfg.enable {
+        "dhcpd${postfix}" = {
+          description = "DHCPv${postfix} server";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+
+          preStart = "touch ${leaseFile}";
+          serviceConfig = {
+            ExecStart = concatMapStringsSep " " escapeShellArg args;
+            Type = "forking";
+            Restart = "always";
+            DynamicUser = true;
+            User = "dhcpd";
+            Group = "dhcpd";
+            AmbientCapabilities = [
+              "CAP_NET_RAW"          # to send ICMP messages
+              "CAP_NET_BIND_SERVICE" # to bind on DHCP port (67)
+            ];
+            StateDirectory   = "dhcpd${postfix}";
+            RuntimeDirectory = "dhcpd${postfix}";
+            PIDFile = "/run/dhcpd${postfix}/dhcpd.pid";
+          };
         };
-    };
-  };
+      };
 
   machineOpts = { ... }: {
 
@@ -102,15 +109,6 @@ let
       '';
     };
 
-    stateDir = mkOption {
-      type = types.path;
-      # We use /var/lib/dhcp for DHCPv4 to save backwards compatibility.
-      default = "/var/lib/dhcp${if postfix == "4" then "" else postfix}";
-      description = ''
-        State directory for the DHCP server.
-      '';
-    };
-
     extraConfig = mkOption {
       type = types.lines;
       default = "";
@@ -194,7 +192,13 @@ in
 
   imports = [
     (mkRenamedOptionModule [ "services" "dhcpd" ] [ "services" "dhcpd4" ])
-  ];
+  ] ++ flip map [ "4" "6" ] (postfix:
+    mkRemovedOptionModule [ "services" "dhcpd${postfix}" "stateDir" ] ''
+      The DHCP server state directory is now managed with the systemd's DynamicUser mechanism.
+      This means the directory is named after the service (dhcpd${postfix}), created under
+      /var/lib/private/ and symlinked to /var/lib/.
+    ''
+  );
 
   ###### interface
 
@@ -210,15 +214,6 @@ in
 
   config = mkIf (cfg4.enable || cfg6.enable) {
 
-    users = {
-      users.dhcpd = {
-        isSystemUser = true;
-        group = "dhcpd";
-        description = "DHCP daemon user";
-      };
-      groups.dhcpd = {};
-    };
-
     systemd.services = dhcpdService "4" cfg4 // dhcpdService "6" cfg6;
 
   };
diff --git a/nixos/modules/services/networking/dnscrypt-proxy2.nix b/nixos/modules/services/networking/dnscrypt-proxy2.nix
index dc6a019e9b77..316e6e37f9da 100644
--- a/nixos/modules/services/networking/dnscrypt-proxy2.nix
+++ b/nixos/modules/services/networking/dnscrypt-proxy2.nix
@@ -118,4 +118,7 @@ in
       };
     };
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/networking/ergo.nix b/nixos/modules/services/networking/ergo.nix
index c52de30dc361..6e55a7cfff6c 100644
--- a/nixos/modules/services/networking/ergo.nix
+++ b/nixos/modules/services/networking/ergo.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
   cfg = config.services.ergo;
+  opt = options.services.ergo;
 
-  inherit (lib) mkEnableOption mkIf mkOption optionalString types;
+  inherit (lib) literalExpression mkEnableOption mkIf mkOption optionalString types;
 
   configFile = pkgs.writeText "ergo.conf" (''
 ergo {
@@ -92,6 +93,7 @@ in {
       group = mkOption {
         type = types.str;
         default = cfg.user;
+        defaultText = literalExpression "config.${opt.user}";
         description = "The group as which to run the Ergo node.";
       };
 
diff --git a/nixos/modules/services/networking/ergochat.nix b/nixos/modules/services/networking/ergochat.nix
new file mode 100644
index 000000000000..cfaf69fc6139
--- /dev/null
+++ b/nixos/modules/services/networking/ergochat.nix
@@ -0,0 +1,155 @@
+{ config, lib, options, pkgs, ... }: let
+  cfg = config.services.ergochat;
+in {
+  options = {
+    services.ergochat = {
+
+      enable = lib.mkEnableOption "Ergo IRC daemon";
+
+      openFilesLimit = lib.mkOption {
+        type = lib.types.int;
+        default = 1024;
+        description = ''
+          Maximum number of open files. Limits the clients and server connections.
+        '';
+      };
+
+      configFile = lib.mkOption {
+        type = lib.types.path;
+        default = (pkgs.formats.yaml {}).generate "ergo.conf" cfg.settings;
+        defaultText = "generated config file from <literal>.settings</literal>";
+        description = ''
+          Path to configuration file.
+          Setting this will skip any configuration done via <literal>.settings</literal>
+        '';
+      };
+
+      settings = lib.mkOption {
+        type = (pkgs.formats.yaml {}).type;
+        description = ''
+          Ergo IRC daemon configuration file.
+          https://raw.githubusercontent.com/ergochat/ergo/master/default.yaml
+        '';
+        default = {
+          network = {
+            name = "testnetwork";
+          };
+          server = {
+            name = "example.com";
+            listeners = {
+              ":6667" = {};
+            };
+            casemapping = "permissive";
+            enforce-utf = true;
+            lookup-hostnames = false;
+            ip-cloaking = {
+              enabled = false;
+            };
+            forward-confirm-hostnames = false;
+            check-ident = false;
+            relaymsg = {
+              enabled = false;
+            };
+            max-sendq = "1M";
+            ip-limits = {
+              count = false;
+              throttle = false;
+            };
+          };
+          datastore = {
+            autoupgrade = true;
+            # this points to the StateDirectory of the systemd service
+            path = "/var/lib/ergo/ircd.db";
+          };
+          accounts = {
+            authentication-enabled = true;
+            registration = {
+              enabled = true;
+              allow-before-connect = true;
+              throttling = {
+                enabled = true;
+                duration = "10m";
+                max-attempts = 30;
+              };
+              bcrypt-cost = 4;
+              email-verification.enabled = false;
+            };
+            multiclient = {
+              enabled = true;
+              allowed-by-default = true;
+              always-on = "opt-out";
+              auto-away = "opt-out";
+            };
+          };
+          channels = {
+            default-modes = "+ntC";
+            registration = {
+              enabled = true;
+            };
+          };
+          limits = {
+            nicklen = 32;
+            identlen = 20;
+            channellen = 64;
+            awaylen = 390;
+            kicklen = 390;
+            topiclen = 390;
+          };
+          history = {
+            enabled = true;
+            channel-length = 2048;
+            client-length = 256;
+            autoresize-window = "3d";
+            autoreplay-on-join = 0;
+            chathistory-maxmessages = 100;
+            znc-maxmessages = 2048;
+            restrictions = {
+              expire-time = "1w";
+              query-cutoff = "none";
+              grace-period = "1h";
+            };
+            retention = {
+              allow-individual-delete = false;
+              enable-account-indexing = false;
+            };
+            tagmsg-storage = {
+              default = false;
+              whitelist = [
+                "+draft/react"
+                "+react"
+              ];
+            };
+          };
+        };
+      };
+
+    };
+  };
+  config = lib.mkIf cfg.enable {
+
+    environment.etc."ergo.yaml".source = cfg.configFile;
+
+    # merge configured values with default values
+    services.ergochat.settings =
+      lib.mapAttrsRecursive (_: lib.mkDefault) options.services.ergochat.settings.default;
+
+    systemd.services.ergochat = {
+      description = "Ergo IRC daemon";
+      wantedBy = [ "multi-user.target" ];
+      # reload is not applying the changed config. further investigation is needed
+      # at some point this should be enabled, since we don't want to restart for
+      # every config change
+      # reloadIfChanged = true;
+      restartTriggers = [ cfg.configFile ];
+      serviceConfig = {
+        ExecStart = "${pkgs.ergochat}/bin/ergo run --conf /etc/ergo.yaml";
+        ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID";
+        DynamicUser = true;
+        StateDirectory = "ergo";
+        LimitNOFILE = toString cfg.openFilesLimit;
+      };
+    };
+
+  };
+  meta.maintainers = with lib.maintainers; [ lassulus tv ];
+}
diff --git a/nixos/modules/services/networking/eternal-terminal.nix b/nixos/modules/services/networking/eternal-terminal.nix
index 88b4cd90540f..0dcf3d28f4e0 100644
--- a/nixos/modules/services/networking/eternal-terminal.nix
+++ b/nixos/modules/services/networking/eternal-terminal.nix
@@ -90,6 +90,6 @@ in
   };
 
   meta = {
-    maintainers = with lib.maintainers; [ pingiun ];
+    maintainers = with lib.maintainers; [ ];
   };
 }
diff --git a/nixos/modules/services/networking/firefox/sync-server.nix b/nixos/modules/services/networking/firefox/sync-server.nix
deleted file mode 100644
index 1ad573abfca3..000000000000
--- a/nixos/modules/services/networking/firefox/sync-server.nix
+++ /dev/null
@@ -1,183 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  cfg = config.services.firefox.syncserver;
-
-  defaultDbLocation = "/var/db/firefox-sync-server/firefox-sync-server.db";
-  defaultSqlUri = "sqlite:///${defaultDbLocation}";
-
-  syncServerIni = pkgs.writeText "syncserver.ini" ''
-    [DEFAULT]
-    overrides = ${cfg.privateConfig}
-
-    [server:main]
-    use = egg:gunicorn
-    host = ${cfg.listen.address}
-    port = ${toString cfg.listen.port}
-
-    [app:main]
-    use = egg:syncserver
-
-    [syncserver]
-    public_url = ${cfg.publicUrl}
-    ${optionalString (cfg.sqlUri != "") "sqluri = ${cfg.sqlUri}"}
-    allow_new_users = ${boolToString cfg.allowNewUsers}
-
-    [browserid]
-    backend = tokenserver.verifiers.LocalVerifier
-    audiences = ${removeSuffix "/" cfg.publicUrl}
-  '';
-
-  user = "syncserver";
-  group = "syncserver";
-in
-
-{
-  meta.maintainers = with lib.maintainers; [ nadrieril ];
-
-  options = {
-    services.firefox.syncserver = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to enable a Firefox Sync Server, this give the opportunity to
-          Firefox users to store all synchronized data on their own server. To use this
-          server, Firefox users should visit the <option>about:config</option>, and
-          replicate the following change
-
-          <screen>
-          services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5
-          </screen>
-
-          where <option>http://localhost:5000/</option> corresponds to the
-          public url of the server.
-        '';
-      };
-
-      listen.address = mkOption {
-        type = types.str;
-        default = "127.0.0.1";
-        example = "0.0.0.0";
-        description = ''
-          Address on which the sync server listen to.
-        '';
-      };
-
-      listen.port = mkOption {
-        type = types.port;
-        default = 5000;
-        description = ''
-          Port on which the sync server listen to.
-        '';
-      };
-
-      publicUrl = mkOption {
-        type = types.str;
-        default = "http://localhost:5000/";
-        example = "http://sync.example.com/";
-        description = ''
-          Public URL with which firefox users can use to access the sync server.
-        '';
-      };
-
-      allowNewUsers = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Whether to allow new-user signups on the server. Only request by
-          existing accounts will be honored.
-        '';
-      };
-
-      sqlUri = mkOption {
-        type = types.str;
-        default = defaultSqlUri;
-        example = "postgresql://scott:tiger@localhost/test";
-        description = ''
-          The location of the database. This URL is composed of
-          <option>dialect[+driver]://user:password@host/dbname[?key=value..]</option>,
-          where <option>dialect</option> is a database name such as
-          <option>mysql</option>, <option>oracle</option>, <option>postgresql</option>,
-          etc., and <option>driver</option> the name of a DBAPI, such as
-          <option>psycopg2</option>, <option>pyodbc</option>, <option>cx_oracle</option>,
-          etc. The <link
-          xlink:href="http://docs.sqlalchemy.org/en/rel_0_9/core/engines.html#database-urls">
-          SQLAlchemy documentation</link> provides more examples and describe the syntax of
-          the expected URL.
-        '';
-      };
-
-      privateConfig = mkOption {
-        type = types.str;
-        default = "/etc/firefox/syncserver-secret.ini";
-        description = ''
-          The private config file is used to extend the generated config with confidential
-          information, such as the <option>syncserver.sqlUri</option> setting if it contains a
-          password, and the <option>syncserver.secret</option> setting is used by the server to
-          generate cryptographically-signed authentication tokens.
-
-          If this file does not exist, then it is created with a generated
-          <option>syncserver.secret</option> settings.
-       '';
-      };
-    };
-  };
-
-  config = mkIf cfg.enable {
-
-    systemd.services.syncserver = {
-      after = [ "network.target" ];
-      description = "Firefox Sync Server";
-      wantedBy = [ "multi-user.target" ];
-      path = [
-        pkgs.coreutils
-        (pkgs.python.withPackages (ps: [ pkgs.syncserver ps.gunicorn ]))
-      ];
-
-      serviceConfig = {
-        User = user;
-        Group = group;
-        PermissionsStartOnly = true;
-      };
-
-      preStart = ''
-        if ! test -e ${cfg.privateConfig}; then
-          mkdir -p $(dirname ${cfg.privateConfig})
-          echo  > ${cfg.privateConfig} '[syncserver]'
-          chmod 600 ${cfg.privateConfig}
-          echo >> ${cfg.privateConfig} "secret = $(head -c 20 /dev/urandom | sha1sum | tr -d ' -')"
-        fi
-        chmod 600 ${cfg.privateConfig}
-        chmod 755 $(dirname ${cfg.privateConfig})
-        chown ${user}:${group} ${cfg.privateConfig}
-
-      '' + optionalString (cfg.sqlUri == defaultSqlUri) ''
-        if ! test -e $(dirname ${defaultDbLocation}); then
-          mkdir -m 700 -p $(dirname ${defaultDbLocation})
-          chown ${user}:${group} $(dirname ${defaultDbLocation})
-        fi
-
-        # Move previous database file if it exists
-        oldDb="/var/db/firefox-sync-server.db"
-        if test -f $oldDb; then
-          mv $oldDb ${defaultDbLocation}
-          chown ${user}:${group} ${defaultDbLocation}
-        fi
-      '';
-
-      script = ''
-        gunicorn --paste ${syncServerIni}
-      '';
-    };
-
-    users.users.${user} = {
-      inherit group;
-      isSystemUser = true;
-    };
-
-    users.groups.${group} = {};
-  };
-}
diff --git a/nixos/modules/services/networking/firewall.nix b/nixos/modules/services/networking/firewall.nix
index b5b46fe6042c..c213a5516a49 100644
--- a/nixos/modules/services/networking/firewall.nix
+++ b/nixos/modules/services/networking/firewall.nix
@@ -179,10 +179,6 @@ let
       ) cfg.allowedUDPPortRanges
     ) allInterfaces)}
 
-    # Accept IPv4 multicast.  Not a big security risk since
-    # probably nobody is listening anyway.
-    #iptables -A nixos-fw -d 224.0.0.0/4 -j nixos-fw-accept
-
     # Optionally respond to ICMPv4 pings.
     ${optionalString cfg.allowPing ''
       iptables -w -A nixos-fw -p icmp --icmp-type echo-request ${optionalString (cfg.pingLimit != null)
@@ -326,7 +322,7 @@ in
         type = types.package;
         default = pkgs.iptables;
         defaultText = literalExpression "pkgs.iptables";
-        example = literalExpression "pkgs.iptables-nftables-compat";
+        example = literalExpression "pkgs.iptables-legacy";
         description =
           ''
             The iptables package to use for running the firewall service."
@@ -421,6 +417,7 @@ in
       checkReversePath = mkOption {
         type = types.either types.bool (types.enum ["strict" "loose"]);
         default = kernelHasRPFilter;
+        defaultText = literalDocBook "<literal>true</literal> if supported by the chosen kernel";
         example = "loose";
         description =
           ''
@@ -436,8 +433,6 @@ in
             drop the packet if the source address is not reachable via any
             interface) or false.  Defaults to the value of
             kernelHasRPFilter.
-
-            (needs kernel 3.3+)
           '';
       };
 
diff --git a/nixos/modules/services/networking/frr.nix b/nixos/modules/services/networking/frr.nix
new file mode 100644
index 000000000000..45a82b9450a4
--- /dev/null
+++ b/nixos/modules/services/networking/frr.nix
@@ -0,0 +1,211 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.services.frr;
+
+  services = [
+    "static"
+    "bgp"
+    "ospf"
+    "ospf6"
+    "rip"
+    "ripng"
+    "isis"
+    "pim"
+    "ldp"
+    "nhrp"
+    "eigrp"
+    "babel"
+    "sharp"
+    "pbr"
+    "bfd"
+    "fabric"
+  ];
+
+  allServices = services ++ [ "zebra" ];
+
+  isEnabled = service: cfg.${service}.enable;
+
+  daemonName = service: if service == "zebra" then service else "${service}d";
+
+  configFile = service:
+    let
+      scfg = cfg.${service};
+    in
+      if scfg.configFile != null then scfg.configFile
+      else pkgs.writeText "${daemonName service}.conf"
+        ''
+          ! FRR ${daemonName service} configuration
+          !
+          hostname ${config.networking.hostName}
+          log syslog
+          service password-encryption
+          !
+          ${scfg.config}
+          !
+          end
+        '';
+
+  serviceOptions = service:
+    {
+      enable = mkEnableOption "the FRR ${toUpper service} routing protocol";
+
+      configFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        example = "/etc/frr/${daemonName service}.conf";
+        description = ''
+          Configuration file to use for FRR ${daemonName service}.
+          By default the NixOS generated files are used.
+        '';
+      };
+
+      config = mkOption {
+        type = types.lines;
+        default = "";
+        example =
+          let
+            examples = {
+              rip = ''
+                router rip
+                  network 10.0.0.0/8
+              '';
+
+              ospf = ''
+                router ospf
+                  network 10.0.0.0/8 area 0
+              '';
+
+              bgp = ''
+                router bgp 65001
+                  neighbor 10.0.0.1 remote-as 65001
+              '';
+            };
+          in
+            examples.${service} or "";
+        description = ''
+          ${daemonName service} configuration statements.
+        '';
+      };
+
+      vtyListenAddress = mkOption {
+        type = types.str;
+        default = "localhost";
+        description = ''
+          Address to bind to for the VTY interface.
+        '';
+      };
+
+      vtyListenPort = mkOption {
+        type = types.nullOr types.int;
+        default = null;
+        description = ''
+          TCP Port to bind to for the VTY interface.
+        '';
+      };
+    };
+
+in
+
+{
+
+  ###### interface
+  imports = [
+    {
+      options.services.frr = {
+        zebra = (serviceOptions "zebra") // {
+          enable = mkOption {
+            type = types.bool;
+            default = any isEnabled services;
+            description = ''
+              Whether to enable the Zebra routing manager.
+
+              The Zebra routing manager is automatically enabled
+              if any routing protocols are configured.
+            '';
+          };
+        };
+      };
+    }
+    { options.services.frr = (genAttrs services serviceOptions); }
+  ];
+
+  ###### implementation
+
+  config = mkIf (any isEnabled allServices) {
+
+    environment.systemPackages = [
+      pkgs.frr # for the vtysh tool
+    ];
+
+    users.users.frr = {
+      description = "FRR daemon user";
+      isSystemUser = true;
+      group = "frr";
+    };
+
+    users.groups = {
+      frr = {};
+      # Members of the frrvty group can use vtysh to inspect the FRR daemons
+      frrvty = { members = [ "frr" ]; };
+    };
+
+    environment.etc = let
+      mkEtcLink = service: {
+        name = "frr/${service}.conf";
+        value.source = configFile service;
+      };
+    in
+      (builtins.listToAttrs
+      (map mkEtcLink (filter isEnabled allServices))) // {
+        "frr/vtysh.conf".text = "";
+      };
+
+    systemd.tmpfiles.rules = [
+      "d /run/frr 0750 frr frr -"
+    ];
+
+    systemd.services =
+      let
+        frrService = service:
+          let
+            scfg = cfg.${service};
+            daemon = daemonName service;
+          in
+            nameValuePair daemon ({
+              wantedBy = [ "multi-user.target" ];
+              after = [ "network-pre.target" "systemd-sysctl.service" ] ++ lib.optionals (service != "zebra") [ "zebra.service" ];
+              bindsTo = lib.optionals (service != "zebra") [ "zebra.service" ];
+              wants = [ "network.target" ];
+
+              description = if service == "zebra" then "FRR Zebra routing manager"
+                else "FRR ${toUpper service} routing daemon";
+
+              unitConfig.Documentation = if service == "zebra" then "man:zebra(8)"
+                else "man:${daemon}(8) man:zebra(8)";
+
+              restartTriggers = [
+                (configFile service)
+              ];
+              reloadIfChanged = true;
+
+              serviceConfig = {
+                PIDFile = "frr/${daemon}.pid";
+                ExecStart = "${pkgs.frr}/libexec/frr/${daemon} -f /etc/frr/${service}.conf"
+                  + optionalString (scfg.vtyListenAddress != "") " -A ${scfg.vtyListenAddress}"
+                  + optionalString (scfg.vtyListenPort != null) " -P ${toString scfg.vtyListenPort}";
+                ExecReload = "${pkgs.python3.interpreter} ${pkgs.frr}/libexec/frr/frr-reload.py --reload --daemon ${daemonName service} --bindir ${pkgs.frr}/bin --rundir /run/frr /etc/frr/${service}.conf";
+                Restart = "on-abnormal";
+              };
+            });
+       in
+         listToAttrs (map frrService (filter isEnabled allServices));
+
+  };
+
+  meta.maintainers = with lib.maintainers; [ woffs ];
+
+}
diff --git a/nixos/modules/services/networking/gogoclient.nix b/nixos/modules/services/networking/gogoclient.nix
deleted file mode 100644
index 1205321818b9..000000000000
--- a/nixos/modules/services/networking/gogoclient.nix
+++ /dev/null
@@ -1,87 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let cfg = config.services.gogoclient;
-in
-
-{
-
-  ###### interface
-
-  options = {
-    services.gogoclient = {
-      enable = mkOption {
-        default = false;
-        type =  types.bool;
-        description = ''
-          Enable the gogoCLIENT IPv6 tunnel.
-        '';
-      };
-      autorun = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Whether to automatically start the tunnel.
-        '';
-      };
-
-      username = mkOption {
-        default = "";
-        type = types.str;
-        description = ''
-          Your Gateway6 login name, if any.
-        '';
-      };
-
-      password = mkOption {
-        default = "";
-        type = types.str;
-        description = ''
-          Path to a file (as a string), containing your gogoNET password, if any.
-        '';
-      };
-
-      server = mkOption {
-        type = types.str;
-        default = "anonymous.freenet6.net";
-        example = "broker.freenet6.net";
-        description = "The Gateway6 server to be used.";
-      };
-    };
-  };
-
-  ###### implementation
-
-  config = mkIf cfg.enable {
-    boot.kernelModules = [ "tun" ];
-
-    networking.enableIPv6 = true;
-
-    systemd.services.gogoclient = {
-      description = "ipv6 tunnel";
-
-      after = [ "network.target" ];
-      requires = [ "network.target" ];
-
-      unitConfig.RequiresMountsFor = "/var/lib/gogoc";
-
-      script = let authMethod = if cfg.password == "" then "anonymous" else "any"; in ''
-        mkdir -p -m 700 /var/lib/gogoc
-        cat ${pkgs.gogoclient}/share/${pkgs.gogoclient.name}/gogoc.conf.sample | \
-          ${pkgs.gnused}/bin/sed \
-            -e "s|^userid=|&${cfg.username}|" \
-            -e "s|^passwd=|&${optionalString (cfg.password != "") "$(cat ${cfg.password})"}|" \
-            -e "s|^server=.*|server=${cfg.server}|" \
-            -e "s|^auth_method=.*|auth_method=${authMethod}|" \
-            -e "s|^#log_file=|log_file=1|" > /var/lib/gogoc/gogoc.conf
-        cd /var/lib/gogoc
-        exec ${pkgs.gogoclient}/bin/gogoc -y -f /var/lib/gogoc/gogoc.conf
-      '';
-    } // optionalAttrs cfg.autorun {
-      wantedBy = [ "multi-user.target" ];
-    };
-
-  };
-
-}
diff --git a/nixos/modules/services/networking/headscale.nix b/nixos/modules/services/networking/headscale.nix
new file mode 100644
index 000000000000..091d2a938cd4
--- /dev/null
+++ b/nixos/modules/services/networking/headscale.nix
@@ -0,0 +1,490 @@
+{ config, lib, pkgs, ... }:
+with lib;
+let
+  cfg = config.services.headscale;
+
+  dataDir = "/var/lib/headscale";
+  runDir = "/run/headscale";
+
+  settingsFormat = pkgs.formats.yaml { };
+  configFile = settingsFormat.generate "headscale.yaml" cfg.settings;
+in
+{
+  options = {
+    services.headscale = {
+      enable = mkEnableOption "headscale, Open Source coordination server for Tailscale";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.headscale;
+        defaultText = literalExpression "pkgs.headscale";
+        description = ''
+          Which headscale package to use for the running server.
+        '';
+      };
+
+      user = mkOption {
+        default = "headscale";
+        type = types.str;
+        description = ''
+          User account under which headscale runs.
+          <note><para>
+          If left as the default value this user will automatically be created
+          on system activation, otherwise you are responsible for
+          ensuring the user exists before the headscale service starts.
+          </para></note>
+        '';
+      };
+
+      group = mkOption {
+        default = "headscale";
+        type = types.str;
+        description = ''
+          Group under which headscale runs.
+          <note><para>
+          If left as the default value this group will automatically be created
+          on system activation, otherwise you are responsible for
+          ensuring the user exists before the headscale service starts.
+          </para></note>
+        '';
+      };
+
+      serverUrl = mkOption {
+        type = types.str;
+        default = "http://127.0.0.1:8080";
+        description = ''
+          The url clients will connect to.
+        '';
+        example = "https://myheadscale.example.com:443";
+      };
+
+      address = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = ''
+          Listening address of headscale.
+        '';
+        example = "0.0.0.0";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 8080;
+        description = ''
+          Listening port of headscale.
+        '';
+        example = 443;
+      };
+
+      privateKeyFile = mkOption {
+        type = types.path;
+        default = "${dataDir}/private.key";
+        description = ''
+          Path to private key file, generated automatically if it does not exist.
+        '';
+      };
+
+      derp = {
+        urls = mkOption {
+          type = types.listOf types.str;
+          default = [ "https://controlplane.tailscale.com/derpmap/default" ];
+          description = ''
+            List of urls containing DERP maps.
+            See <link xlink:href="https://tailscale.com/blog/how-tailscale-works/">How Tailscale works</link> for more information on DERP maps.
+          '';
+        };
+
+        paths = mkOption {
+          type = types.listOf types.path;
+          default = [ ];
+          description = ''
+            List of file paths containing DERP maps.
+            See <link xlink:href="https://tailscale.com/blog/how-tailscale-works/">How Tailscale works</link> for more information on DERP maps.
+          '';
+        };
+
+
+        autoUpdate = mkOption {
+          type = types.bool;
+          default = true;
+          description = ''
+            Whether to automatically update DERP maps on a set frequency.
+          '';
+          example = false;
+        };
+
+        updateFrequency = mkOption {
+          type = types.str;
+          default = "24h";
+          description = ''
+            Frequency to update DERP maps.
+          '';
+          example = "5m";
+        };
+
+      };
+
+      ephemeralNodeInactivityTimeout = mkOption {
+        type = types.str;
+        default = "30m";
+        description = ''
+          Time before an inactive ephemeral node is deleted.
+        '';
+        example = "5m";
+      };
+
+      database = {
+        type = mkOption {
+          type = types.enum [ "sqlite3" "postgres" ];
+          example = "postgres";
+          default = "sqlite3";
+          description = "Database engine to use.";
+        };
+
+        host = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          example = "127.0.0.1";
+          description = "Database host address.";
+        };
+
+        port = mkOption {
+          type = types.nullOr types.port;
+          default = null;
+          example = 3306;
+          description = "Database host port.";
+        };
+
+        name = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          example = "headscale";
+          description = "Database name.";
+        };
+
+        user = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          example = "headscale";
+          description = "Database user.";
+        };
+
+        passwordFile = mkOption {
+          type = types.nullOr types.path;
+          default = null;
+          example = "/run/keys/headscale-dbpassword";
+          description = ''
+            A file containing the password corresponding to
+            <option>database.user</option>.
+          '';
+        };
+
+        path = mkOption {
+          type = types.nullOr types.str;
+          default = "${dataDir}/db.sqlite";
+          description = "Path to the sqlite3 database file.";
+        };
+      };
+
+      logLevel = mkOption {
+        type = types.str;
+        default = "info";
+        description = ''
+          headscale log level.
+        '';
+        example = "debug";
+      };
+
+      dns = {
+        nameservers = mkOption {
+          type = types.listOf types.str;
+          default = [ "1.1.1.1" ];
+          description = ''
+            List of nameservers to pass to Tailscale clients.
+          '';
+        };
+
+        domains = mkOption {
+          type = types.listOf types.str;
+          default = [ ];
+          description = ''
+            Search domains to inject to Tailscale clients.
+          '';
+          example = [ "mydomain.internal" ];
+        };
+
+        magicDns = mkOption {
+          type = types.bool;
+          default = true;
+          description = ''
+            Whether to use [MagicDNS](https://tailscale.com/kb/1081/magicdns/).
+            Only works if there is at least a nameserver defined.
+          '';
+          example = false;
+        };
+
+        baseDomain = mkOption {
+          type = types.str;
+          default = "";
+          description = ''
+            Defines the base domain to create the hostnames for MagicDNS.
+            <option>baseDomain</option> must be a FQDNs, without the trailing dot.
+            The FQDN of the hosts will be
+            <literal>hostname.namespace.base_domain</literal> (e.g.
+            <literal>myhost.mynamespace.example.com</literal>).
+          '';
+        };
+      };
+
+      openIdConnect = {
+        issuer = mkOption {
+          type = types.str;
+          default = "";
+          description = ''
+            URL to OpenID issuer.
+          '';
+          example = "https://openid.example.com";
+        };
+
+        clientId = mkOption {
+          type = types.str;
+          default = "";
+          description = ''
+            OpenID Connect client ID.
+          '';
+        };
+
+        clientSecretFile = mkOption {
+          type = types.nullOr types.path;
+          default = null;
+          description = ''
+            Path to OpenID Connect client secret file.
+          '';
+        };
+
+        domainMap = mkOption {
+          type = types.attrsOf types.str;
+          default = { };
+          description = ''
+            Domain map is used to map incomming users (by their email) to
+            a namespace. The key can be a string, or regex.
+          '';
+          example = {
+            ".*" = "default-namespace";
+          };
+        };
+
+      };
+
+      tls = {
+        letsencrypt = {
+          hostname = mkOption {
+            type = types.nullOr types.str;
+            default = "";
+            description = ''
+              Domain name to request a TLS certificate for.
+            '';
+          };
+          challengeType = mkOption {
+            type = types.enum [ "TLS_ALPN-01" "HTTP-01" ];
+            default = "HTTP-01";
+            description = ''
+              Type of ACME challenge to use, currently supported types:
+              <literal>HTTP-01</literal> or <literal>TLS_ALPN-01</literal>.
+            '';
+          };
+          httpListen = mkOption {
+            type = types.nullOr types.str;
+            default = ":http";
+            description = ''
+              When HTTP-01 challenge is chosen, letsencrypt must set up a
+              verification endpoint, and it will be listening on:
+              <literal>:http = port 80</literal>.
+            '';
+          };
+        };
+
+        certFile = mkOption {
+          type = types.nullOr types.path;
+          default = null;
+          description = ''
+            Path to already created certificate.
+          '';
+        };
+        keyFile = mkOption {
+          type = types.nullOr types.path;
+          default = null;
+          description = ''
+            Path to key for already created certificate.
+          '';
+        };
+      };
+
+      aclPolicyFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        description = ''
+          Path to a file containg ACL policies.
+        '';
+      };
+
+      settings = mkOption {
+        type = settingsFormat.type;
+        default = { };
+        description = ''
+          Overrides to <filename>config.yaml</filename> as a Nix attribute set.
+          This option is ideal for overriding settings not exposed as Nix options.
+          Check the <link xlink:href="https://github.com/juanfont/headscale/blob/main/config-example.yaml">example config</link>
+          for possible options.
+        '';
+      };
+
+
+    };
+
+  };
+  config = mkIf cfg.enable {
+
+    services.headscale.settings = {
+      server_url = mkDefault cfg.serverUrl;
+      listen_addr = mkDefault "${cfg.address}:${toString cfg.port}";
+
+      private_key_path = mkDefault cfg.privateKeyFile;
+
+      derp = {
+        urls = mkDefault cfg.derp.urls;
+        paths = mkDefault cfg.derp.paths;
+        auto_update_enable = mkDefault cfg.derp.autoUpdate;
+        update_frequency = mkDefault cfg.derp.updateFrequency;
+      };
+
+      # Turn off update checks since the origin of our package
+      # is nixpkgs and not Github.
+      disable_check_updates = true;
+
+      ephemeral_node_inactivity_timeout = mkDefault cfg.ephemeralNodeInactivityTimeout;
+
+      db_type = mkDefault cfg.database.type;
+      db_path = mkDefault cfg.database.path;
+
+      log_level = mkDefault cfg.logLevel;
+
+      dns_config = {
+        nameservers = mkDefault cfg.dns.nameservers;
+        domains = mkDefault cfg.dns.domains;
+        magic_dns = mkDefault cfg.dns.magicDns;
+        base_domain = mkDefault cfg.dns.baseDomain;
+      };
+
+      unix_socket = "${runDir}/headscale.sock";
+
+      # OpenID Connect
+      oidc = {
+        issuer = mkDefault cfg.openIdConnect.issuer;
+        client_id = mkDefault cfg.openIdConnect.clientId;
+        domain_map = mkDefault cfg.openIdConnect.domainMap;
+      };
+
+      tls_letsencrypt_cache_dir = "${dataDir}/.cache";
+
+    } // optionalAttrs (cfg.database.host != null) {
+      db_host = mkDefault cfg.database.host;
+    } // optionalAttrs (cfg.database.port != null) {
+      db_port = mkDefault cfg.database.port;
+    } // optionalAttrs (cfg.database.name != null) {
+      db_name = mkDefault cfg.database.name;
+    } // optionalAttrs (cfg.database.user != null) {
+      db_user = mkDefault cfg.database.user;
+    } // optionalAttrs (cfg.tls.letsencrypt.hostname != null) {
+      tls_letsencrypt_hostname = mkDefault cfg.tls.letsencrypt.hostname;
+    } // optionalAttrs (cfg.tls.letsencrypt.challengeType != null) {
+      tls_letsencrypt_challenge_type = mkDefault cfg.tls.letsencrypt.challengeType;
+    } // optionalAttrs (cfg.tls.letsencrypt.httpListen != null) {
+      tls_letsencrypt_listen = mkDefault cfg.tls.letsencrypt.httpListen;
+    } // optionalAttrs (cfg.tls.certFile != null) {
+      tls_cert_path = mkDefault cfg.tls.certFile;
+    } // optionalAttrs (cfg.tls.keyFile != null) {
+      tls_key_path = mkDefault cfg.tls.keyFile;
+    } // optionalAttrs (cfg.aclPolicyFile != null) {
+      acl_policy_path = mkDefault cfg.aclPolicyFile;
+    };
+
+    # Setup the headscale configuration in a known path in /etc to
+    # allow both the Server and the Client use it to find the socket
+    # for communication.
+    environment.etc."headscale/config.yaml".source = configFile;
+
+    users.groups.headscale = mkIf (cfg.group == "headscale") { };
+
+    users.users.headscale = mkIf (cfg.user == "headscale") {
+      description = "headscale user";
+      home = dataDir;
+      group = cfg.group;
+      isSystemUser = true;
+    };
+
+    systemd.services.headscale = {
+      description = "headscale coordination server for Tailscale";
+      after = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      restartTriggers = [ configFile ];
+
+      script = ''
+        ${optionalString (cfg.database.passwordFile != null) ''
+          export HEADSCALE_DB_PASS="$(head -n1 ${escapeShellArg cfg.database.passwordFile})"
+        ''}
+
+        export HEADSCALE_OIDC_CLIENT_SECRET="$(head -n1 ${escapeShellArg cfg.openIdConnect.clientSecretFile})"
+        exec ${cfg.package}/bin/headscale serve
+      '';
+
+      serviceConfig =
+        let
+          capabilityBoundingSet = [ "CAP_CHOWN" ] ++ optional (cfg.port < 1024) "CAP_NET_BIND_SERVICE";
+        in
+        {
+          Restart = "always";
+          Type = "simple";
+          User = cfg.user;
+          Group = cfg.group;
+
+          # Hardening options
+          RuntimeDirectory = "headscale";
+          # Allow headscale group access so users can be added and use the CLI.
+          RuntimeDirectoryMode = "0750";
+
+          StateDirectory = "headscale";
+          StateDirectoryMode = "0750";
+
+          ProtectSystem = "strict";
+          ProtectHome = true;
+          PrivateTmp = true;
+          PrivateDevices = true;
+          ProtectKernelTunables = true;
+          ProtectControlGroups = true;
+          RestrictSUIDSGID = true;
+          PrivateMounts = true;
+          ProtectKernelModules = true;
+          ProtectKernelLogs = true;
+          ProtectHostname = true;
+          ProtectClock = true;
+          ProtectProc = "invisible";
+          ProcSubset = "pid";
+          RestrictNamespaces = true;
+          RemoveIPC = true;
+          UMask = "0077";
+
+          CapabilityBoundingSet = capabilityBoundingSet;
+          AmbientCapabilities = capabilityBoundingSet;
+          NoNewPrivileges = true;
+          LockPersonality = true;
+          RestrictRealtime = true;
+          SystemCallFilter = [ "@system-service" "~@priviledged" "@chown" ];
+          SystemCallArchitectures = "native";
+          RestrictAddressFamilies = "AF_INET AF_INET6 AF_UNIX";
+        };
+    };
+  };
+
+  meta.maintainers = with maintainers; [ kradalby ];
+}
diff --git a/nixos/modules/services/networking/hylafax/options.nix b/nixos/modules/services/networking/hylafax/options.nix
index 8e59c68054d2..8f621b61002f 100644
--- a/nixos/modules/services/networking/hylafax/options.nix
+++ b/nixos/modules/services/networking/hylafax/options.nix
@@ -3,7 +3,7 @@
 let
 
   inherit (lib.options) literalExpression mkEnableOption mkOption;
-  inherit (lib.types) bool enum ints lines attrsOf nullOr path str submodule;
+  inherit (lib.types) bool enum ints lines attrsOf nonEmptyStr nullOr path str submodule;
   inherit (lib.modules) mkDefault mkIf mkMerge;
 
   commonDescr = ''
@@ -17,8 +17,6 @@ let
     configuration to yield an operational system.
   '';
 
-  str1 = lib.types.addCheck str (s: s!="");  # non-empty string
-
   configAttrType =
     # Options in HylaFAX configuration files can be
     # booleans, strings, integers, or list thereof
@@ -37,7 +35,7 @@ let
   modemConfigOptions = { name, config, ... }: {
     options = {
       name = mkOption {
-        type = str1;
+        type = nonEmptyStr;
         example = "ttyS1";
         description = ''
           Name of modem device,
@@ -45,7 +43,7 @@ let
         '';
       };
       type = mkOption {
-        type = str1;
+        type = nonEmptyStr;
         example = "cirrus";
         description = ''
           Name of modem configuration file,
@@ -135,14 +133,14 @@ in
     };
 
     countryCode = mkOption {
-      type = nullOr str1;
+      type = nullOr nonEmptyStr;
       default = null;
       example = "49";
       description = "Country code for server and all modems.";
     };
 
     areaCode = mkOption {
-      type = nullOr str1;
+      type = nullOr nonEmptyStr;
       default = null;
       example = "30";
       description = "Area code for server and all modems.";
@@ -279,7 +277,7 @@ in
       each time the spooling area is initialized.
     '';
     faxcron.enable.frequency = mkOption {
-      type = nullOr str1;
+      type = nullOr nonEmptyStr;
       default = null;
       example = "daily";
       description = ''
@@ -319,7 +317,7 @@ in
       each time the spooling area is initialized.
     '';
     faxqclean.enable.frequency = mkOption {
-      type = nullOr str1;
+      type = nullOr nonEmptyStr;
       default = null;
       example = "daily";
       description = ''
diff --git a/nixos/modules/services/networking/i2pd.nix b/nixos/modules/services/networking/i2pd.nix
index 17828ca44ff2..34fda57b23d2 100644
--- a/nixos/modules/services/networking/i2pd.nix
+++ b/nixos/modules/services/networking/i2pd.nix
@@ -222,14 +222,12 @@ let
         in concatStringsSep "\n" inTunOpts))];
     in pkgs.writeText "i2pd-tunnels.conf" opts;
 
-  i2pdSh = pkgs.writeScriptBin "i2pd" ''
-    #!/bin/sh
-    exec ${pkgs.i2pd}/bin/i2pd \
-      ${if cfg.address == null then "" else "--host="+cfg.address} \
-      --service \
-      --conf=${i2pdConf} \
-      --tunconf=${tunnelConf}
-  '';
+  i2pdFlags = concatStringsSep " " (
+    optional (cfg.address != null) ("--host=" + cfg.address) ++ [
+    "--service"
+    ("--conf=" + i2pdConf)
+    ("--tunconf=" + tunnelConf)
+  ]);
 
 in
 
@@ -253,6 +251,15 @@ in
         '';
       };
 
+      package = mkOption {
+        type = types.package;
+        default = pkgs.i2pd;
+        defaultText = literalExpression "pkgs.i2pd";
+        description = ''
+          i2pd package to use.
+        '';
+      };
+
       logLevel = mkOption {
         type = types.enum ["debug" "info" "warn" "error"];
         default = "error";
@@ -677,7 +684,7 @@ in
         User = "i2pd";
         WorkingDirectory = homeDir;
         Restart = "on-abort";
-        ExecStart = "${i2pdSh}/bin/i2pd";
+        ExecStart = "${cfg.package}/bin/i2pd ${i2pdFlags}";
       };
     };
   };
diff --git a/nixos/modules/services/networking/jibri/default.nix b/nixos/modules/services/networking/jibri/default.nix
index 96832b0eb552..113a7aa4384a 100644
--- a/nixos/modules/services/networking/jibri/default.nix
+++ b/nixos/modules/services/networking/jibri/default.nix
@@ -132,7 +132,7 @@ in
         pkgs.writeScript "finalize_recording.sh" ''''''
         #!/bin/sh
         RECORDINGS_DIR=$1
-        ${pkgs.rclone}/bin/rclone copy $RECORDINGS_DIR RCLONE_REMOTE:jibri-recordings/ -v --log-file=/var/log/jitsi/jibri/recording-upload.txt
+        ''${pkgs.rclone}/bin/rclone copy $RECORDINGS_DIR RCLONE_REMOTE:jibri-recordings/ -v --log-file=/var/log/jitsi/jibri/recording-upload.txt
         exit 0
         '''''';
       '';
diff --git a/nixos/modules/services/networking/kea.nix b/nixos/modules/services/networking/kea.nix
index b11402204aec..17b4eb2e283b 100644
--- a/nixos/modules/services/networking/kea.nix
+++ b/nixos/modules/services/networking/kea.nix
@@ -236,6 +236,7 @@ in
 
       environment = {
         KEA_PIDFILE_DIR = "/run/kea";
+        KEA_LOCKFILE_DIR = "/run/kea";
       };
 
       restartTriggers = [
@@ -271,6 +272,7 @@ in
 
       environment = {
         KEA_PIDFILE_DIR = "/run/kea";
+        KEA_LOCKFILE_DIR = "/run/kea";
       };
 
       restartTriggers = [
@@ -313,6 +315,7 @@ in
 
       environment = {
         KEA_PIDFILE_DIR = "/run/kea";
+        KEA_LOCKFILE_DIR = "/run/kea";
       };
 
       restartTriggers = [
@@ -353,6 +356,7 @@ in
 
       environment = {
         KEA_PIDFILE_DIR = "/run/kea";
+        KEA_LOCKFILE_DIR = "/run/kea";
       };
 
       restartTriggers = [
@@ -361,7 +365,7 @@ in
 
       serviceConfig = {
         ExecStart = "${package}/bin/kea-dhcp-ddns -c /etc/kea/dhcp-ddns.conf ${lib.escapeShellArgs cfg.dhcp-ddns.extraArgs}";
-        AmbientCapabilites = [
+        AmbientCapabilities = [
           "CAP_NET_BIND_SERVICE"
         ];
         CapabilityBoundingSet = [
@@ -374,4 +378,6 @@ in
   ]);
 
   meta.maintainers = with maintainers; [ hexa ];
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/networking/kresd.nix b/nixos/modules/services/networking/kresd.nix
index 3a36ac7e6670..28b8be7a9a0d 100644
--- a/nixos/modules/services/networking/kresd.nix
+++ b/nixos/modules/services/networking/kresd.nix
@@ -7,15 +7,16 @@ let
 
   # Convert systemd-style address specification to kresd config line(s).
   # On Nix level we don't attempt to precisely validate the address specifications.
+  # The optional IPv6 scope spec comes *after* port, perhaps surprisingly.
   mkListen = kind: addr: let
-    al_v4 = builtins.match "([0-9.]+):([0-9]+)" addr;
-    al_v6 = builtins.match "\\[(.+)]:([0-9]+)" addr;
+    al_v4 = builtins.match "([0-9.]+):([0-9]+)($)" addr;
+    al_v6 = builtins.match "\\[(.+)]:([0-9]+)(%.*|$)" addr;
     al_portOnly = builtins.match "([0-9]+)" addr;
     al = findFirst (a: a != null)
       (throw "services.kresd.*: incorrect address specification '${addr}'")
       [ al_v4 al_v6 al_portOnly ];
-    port = last al;
-    addrSpec = if al_portOnly == null then "'${head al}'" else "{'::', '0.0.0.0'}";
+    port = elemAt al 1;
+    addrSpec = if al_portOnly == null then "'${head al}${elemAt al 2}'" else "{'::', '0.0.0.0'}";
     in # freebind is set for compatibility with earlier kresd services;
        # it could be configurable, for example.
       ''
diff --git a/nixos/modules/services/networking/mailpile.nix b/nixos/modules/services/networking/mailpile.nix
deleted file mode 100644
index 4673a2580b60..000000000000
--- a/nixos/modules/services/networking/mailpile.nix
+++ /dev/null
@@ -1,74 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-
-  cfg = config.services.mailpile;
-
-  hostname = cfg.hostname;
-  port = cfg.port;
-
-in
-
-{
-
-  ###### interface
-
-  options = {
-
-    services.mailpile = {
-      enable = mkEnableOption "Mailpile the mail client";
-
-      hostname = mkOption {
-        type = types.str;
-        default = "localhost";
-        description = "Listen to this hostname or ip.";
-      };
-      port = mkOption {
-        type = types.port;
-        default = 33411;
-        description = "Listen on this port.";
-      };
-    };
-
-  };
-
-
-  ###### implementation
-
-  config = mkIf config.services.mailpile.enable {
-
-    users.users.mailpile =
-      { uid = config.ids.uids.mailpile;
-        description = "Mailpile user";
-        createHome = true;
-        home = "/var/lib/mailpile";
-      };
-
-    users.groups.mailpile =
-      { gid = config.ids.gids.mailpile;
-      };
-
-    systemd.services.mailpile =
-      {
-        description = "Mailpile server.";
-        after = [ "network.target" ];
-        wantedBy = [ "multi-user.target" ];
-        serviceConfig = {
-          User = "mailpile";
-          ExecStart = "${pkgs.mailpile}/bin/mailpile --www ${hostname}:${port} --wait";
-          # mixed - first send SIGINT to main process,
-          # then after 2min send SIGKILL to whole group if neccessary
-          KillMode = "mixed";
-          KillSignal = "SIGINT";  # like Ctrl+C - safe mailpile shutdown
-          TimeoutSec = 120;  # wait 2min untill SIGKILL
-        };
-        environment.MAILPILE_HOME = "/var/lib/mailpile/.local/share/Mailpile";
-      };
-
-    environment.systemPackages = [ pkgs.mailpile ];
-
-  };
-
-}
diff --git a/nixos/modules/services/networking/mosquitto.nix b/nixos/modules/services/networking/mosquitto.nix
index 2d498d4dbbcf..b41a2fd27be2 100644
--- a/nixos/modules/services/networking/mosquitto.nix
+++ b/nixos/modules/services/networking/mosquitto.nix
@@ -136,7 +136,7 @@ let
         + concatStringsSep "\n"
           (plainLines
            ++ optional (plainLines != []) ''
-             ${pkgs.mosquitto}/bin/mosquitto_passwd -U "$file"
+             ${cfg.package}/bin/mosquitto_passwd -U "$file"
            ''
            ++ hashedLines));
 
@@ -444,6 +444,15 @@ let
   globalOptions = with types; {
     enable = mkEnableOption "the MQTT Mosquitto broker";
 
+    package = mkOption {
+      type = package;
+      default = pkgs.mosquitto;
+      defaultText = literalExpression "pkgs.mosquitto";
+      description = ''
+        Mosquitto package to use.
+      '';
+    };
+
     bridges = mkOption {
       type = attrsOf bridgeOptions;
       default = {};
@@ -556,7 +565,7 @@ in
     systemd.services.mosquitto = {
       description = "Mosquitto MQTT Broker Daemon";
       wantedBy = [ "multi-user.target" ];
-      after = [ "network.target" ];
+      after = [ "network-online.target" ];
       serviceConfig = {
         Type = "notify";
         NotifyAccess = "main";
@@ -565,7 +574,7 @@ in
         RuntimeDirectory = "mosquitto";
         WorkingDirectory = cfg.dataDir;
         Restart = "on-failure";
-        ExecStart = "${pkgs.mosquitto}/bin/mosquitto -c ${configFile}";
+        ExecStart = "${cfg.package}/bin/mosquitto -c ${configFile}";
         ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
 
         # Hardening
diff --git a/nixos/modules/services/networking/mtr-exporter.nix b/nixos/modules/services/networking/mtr-exporter.nix
new file mode 100644
index 000000000000..ca261074ebde
--- /dev/null
+++ b/nixos/modules/services/networking/mtr-exporter.nix
@@ -0,0 +1,87 @@
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib)
+    maintainers types mkEnableOption mkOption mkIf
+    literalExpression escapeShellArg escapeShellArgs;
+  cfg = config.services.mtr-exporter;
+in {
+  options = {
+    services = {
+      mtr-exporter = {
+        enable = mkEnableOption "a Prometheus exporter for MTR";
+
+        target = mkOption {
+          type = types.str;
+          example = "example.org";
+          description = "Target to check using MTR.";
+        };
+
+        interval = mkOption {
+          type = types.int;
+          default = 60;
+          description = "Interval between MTR checks in seconds.";
+        };
+
+        port = mkOption {
+          type = types.port;
+          default = 8080;
+          description = "Listen port for MTR exporter.";
+        };
+
+        address = mkOption {
+          type = types.str;
+          default = "127.0.0.1";
+          description = "Listen address for MTR exporter.";
+        };
+
+        mtrFlags = mkOption {
+          type = with types; listOf str;
+          default = [];
+          example = ["-G1"];
+          description = "Additional flags to pass to MTR.";
+        };
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.mtr-exporter = {
+      script = ''
+        exec ${pkgs.mtr-exporter}/bin/mtr-exporter \
+          -mtr ${pkgs.mtr}/bin/mtr \
+          -schedule '@every ${toString cfg.interval}s' \
+          -bind ${escapeShellArg cfg.address}:${toString cfg.port} \
+          -- \
+          ${escapeShellArgs (cfg.mtrFlags ++ [ cfg.target ])}
+      '';
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "network.target" ];
+      after = [ "network.target" ];
+      serviceConfig = {
+        Restart = "on-failure";
+        # Hardening
+        CapabilityBoundingSet = [ "" ];
+        DynamicUser = true;
+        LockPersonality = true;
+        ProcSubset = "pid";
+        PrivateDevices = true;
+        PrivateUsers = true;
+        PrivateTmp = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+      };
+    };
+  };
+
+  meta.maintainers = with maintainers; [ jakubgs ];
+}
diff --git a/nixos/modules/services/networking/multipath.nix b/nixos/modules/services/networking/multipath.nix
index 1cc2ad1fc849..1a44184ff6dc 100644
--- a/nixos/modules/services/networking/multipath.nix
+++ b/nixos/modules/services/networking/multipath.nix
@@ -242,21 +242,6 @@ in {
             '';
           };
 
-          retain_attached_hw_handler = mkOption {
-            type = nullOr (enum [ "yes" "no" ]);
-            default = null; # real default: "yes"
-            description = ''
-              (Obsolete for kernels >= 4.3) If set to "yes" and the SCSI layer has
-              already attached a hardware_handler to the device, multipath will not
-              force the device to use the hardware_handler specified by mutipath.conf.
-              If the SCSI layer has not attached a hardware handler, multipath will
-              continue to use its configured hardware handler.
-
-              Important Note: Linux kernel 4.3 or newer always behaves as if
-              "retain_attached_hw_handler yes" was set.
-            '';
-          };
-
           detect_prio = mkOption {
             type = nullOr (enum [ "yes" "no" ]);
             default = null; # real default: "yes"
diff --git a/nixos/modules/services/networking/murmur.nix b/nixos/modules/services/networking/murmur.nix
index bbbe1e181bba..06ec04dbbf16 100644
--- a/nixos/modules/services/networking/murmur.nix
+++ b/nixos/modules/services/networking/murmur.nix
@@ -294,7 +294,7 @@ in
     systemd.services.murmur = {
       description = "Murmur Chat Service";
       wantedBy    = [ "multi-user.target" ];
-      after       = [ "network-online.target "];
+      after       = [ "network-online.target" ];
       preStart    = ''
         ${pkgs.envsubst}/bin/envsubst \
           -o /run/murmur/murmurd.ini \
@@ -306,7 +306,7 @@ in
         Type = if forking then "forking" else "simple";
         PIDFile = mkIf forking "/run/murmur/murmurd.pid";
         EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
-        ExecStart = "${cfg.package}/bin/murmurd -ini /run/murmur/murmurd.ini";
+        ExecStart = "${cfg.package}/bin/mumble-server -ini /run/murmur/murmurd.ini";
         Restart = "always";
         RuntimeDirectory = "murmur";
         RuntimeDirectoryMode = "0700";
diff --git a/nixos/modules/services/networking/nbd.nix b/nixos/modules/services/networking/nbd.nix
new file mode 100644
index 000000000000..87f8c41a8e5c
--- /dev/null
+++ b/nixos/modules/services/networking/nbd.nix
@@ -0,0 +1,146 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.nbd;
+  configFormat = pkgs.formats.ini { };
+  iniFields = with types; attrsOf (oneOf [ bool int float str ]);
+  serverConfig = configFormat.generate "nbd-server-config"
+    ({
+      generic =
+        (cfg.server.extraOptions // {
+          user = "root";
+          group = "root";
+          port = cfg.server.listenPort;
+        } // (optionalAttrs (cfg.server.listenAddress != null) {
+          listenaddr = cfg.server.listenAddress;
+        }));
+    }
+    // (mapAttrs
+      (_: { path, allowAddresses, extraOptions }:
+        extraOptions // {
+          exportname = path;
+        } // (optionalAttrs (allowAddresses != null) {
+          authfile = pkgs.writeText "authfile" (concatStringsSep "\n" allowAddresses);
+        }))
+      cfg.server.exports)
+    );
+  splitLists =
+    partition
+      (path: hasPrefix "/dev/" path)
+      (mapAttrsToList (_: { path, ... }: path) cfg.server.exports);
+  allowedDevices = splitLists.right;
+  boundPaths = splitLists.wrong;
+in
+{
+  options = {
+    services.nbd = {
+      server = {
+        enable = mkEnableOption "the Network Block Device (nbd) server";
+
+        listenPort = mkOption {
+          type = types.port;
+          default = 10809;
+          description = "Port to listen on. The port is NOT automatically opened in the firewall.";
+        };
+
+        extraOptions = mkOption {
+          type = iniFields;
+          default = {
+            allowlist = false;
+          };
+          description = ''
+            Extra options for the server. See
+            <citerefentry><refentrytitle>nbd-server</refentrytitle>
+            <manvolnum>5</manvolnum></citerefentry>.
+          '';
+        };
+
+        exports = mkOption {
+          description = "Files or block devices to make available over the network.";
+          default = { };
+          type = with types; attrsOf
+            (submodule {
+              options = {
+                path = mkOption {
+                  type = str;
+                  description = "File or block device to export.";
+                  example = "/dev/sdb1";
+                };
+
+                allowAddresses = mkOption {
+                  type = nullOr (listOf str);
+                  default = null;
+                  example = [ "10.10.0.0/24" "127.0.0.1" ];
+                  description = "IPs and subnets that are authorized to connect for this device. If not specified, the server will allow all connections.";
+                };
+
+                extraOptions = mkOption {
+                  type = iniFields;
+                  default = {
+                    flush = true;
+                    fua = true;
+                  };
+                  description = ''
+                    Extra options for this export. See
+                    <citerefentry><refentrytitle>nbd-server</refentrytitle>
+                    <manvolnum>5</manvolnum></citerefentry>.
+                  '';
+                };
+              };
+            });
+        };
+
+        listenAddress = mkOption {
+          type = with types; nullOr str;
+          description = "Address to listen on. If not specified, the server will listen on all interfaces.";
+          default = null;
+          example = "10.10.0.1";
+        };
+      };
+    };
+  };
+
+  config = mkIf cfg.server.enable {
+    boot.kernelModules = [ "nbd" ];
+
+    systemd.services.nbd-server = {
+      after = [ "network-online.target" ];
+      before = [ "multi-user.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.nbd}/bin/nbd-server -C ${serverConfig}";
+        Type = "forking";
+
+        DeviceAllow = map (path: "${path} rw") allowedDevices;
+        BindPaths = boundPaths;
+
+        CapabilityBoundingSet = "";
+        DevicePolicy = "closed";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = false;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "noaccess";
+        ProtectSystem = "strict";
+        RestrictAddressFamilies = "AF_INET AF_INET6";
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        UMask = "0077";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/networking/networkmanager.nix b/nixos/modules/services/networking/networkmanager.nix
index 73e63e2ee99b..7a9d9e5428a7 100644
--- a/nixos/modules/services/networking/networkmanager.nix
+++ b/nixos/modules/services/networking/networkmanager.nix
@@ -384,6 +384,17 @@ in {
           so you don't need to to that yourself.
         '';
       };
+
+      enableFccUnlock = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Enable FCC unlock procedures. Since release 1.18.4, the ModemManager daemon no longer
+          automatically performs the FCC unlock procedure by default. See
+          <link xlink:href="https://modemmanager.org/docs/modemmanager/fcc-unlock/">the docs</link>
+          for more details.
+        '';
+      };
     };
   };
 
@@ -438,7 +449,13 @@ in {
 
       "NetworkManager/VPN/nm-sstp-service.name".source =
         "${networkmanager-sstp}/lib/NetworkManager/VPN/nm-sstp-service.name";
+
       }
+      // optionalAttrs cfg.enableFccUnlock
+         {
+           "ModemManager/fcc-unlock.d".source =
+             "${pkgs.modemmanager}/share/ModemManager/fcc-unlock.available.d/*";
+         }
       // optionalAttrs (cfg.appendNameservers != [] || cfg.insertNameservers != [])
          {
            "NetworkManager/dispatcher.d/02overridedns".source = overrideNameserversScript;
@@ -539,6 +556,7 @@ in {
 
     boot.kernelModules = [ "ctr" ];
 
+    security.polkit.enable = true;
     security.polkit.extraConfig = polkitConf;
 
     services.dbus.packages = cfg.packages
diff --git a/nixos/modules/services/networking/nftables.nix b/nixos/modules/services/networking/nftables.nix
index eb74d373b0af..b911f97491eb 100644
--- a/nixos/modules/services/networking/nftables.nix
+++ b/nixos/modules/services/networking/nftables.nix
@@ -25,9 +25,10 @@ in
           for more information.
 
           There are other programs that use iptables internally too, such as
-          libvirt.
+          libvirt. For information on how the two firewalls interact, see [2].
 
           [1]: https://github.com/NixOS/nixpkgs/issues/24318#issuecomment-289216273
+          [2]: https://wiki.nftables.org/wiki-nftables/index.php/Troubleshooting#Question_4._How_do_nftables_and_iptables_interact_when_used_on_the_same_system.3F
         '';
     };
     networking.nftables.ruleset = mkOption {
@@ -118,20 +119,11 @@ in
           flush ruleset
           include "${cfg.rulesetFile}"
         '';
-        checkScript = pkgs.writeScript "nftables-check" ''
-          #! ${pkgs.runtimeShell} -e
-          if $(${pkgs.kmod}/bin/lsmod | grep -q ip_tables); then
-            echo "Unload ip_tables before using nftables!" 1>&2
-            exit 1
-          else
-            ${rulesScript}
-          fi
-        '';
       in {
         Type = "oneshot";
         RemainAfterExit = true;
-        ExecStart = checkScript;
-        ExecReload = checkScript;
+        ExecStart = rulesScript;
+        ExecReload = rulesScript;
         ExecStop = "${pkgs.nftables}/bin/nft flush ruleset";
       };
     };
diff --git a/nixos/modules/services/networking/nix-serve.nix b/nixos/modules/services/networking/nix-serve.nix
index 390f0ddaee83..432938d59d90 100644
--- a/nixos/modules/services/networking/nix-serve.nix
+++ b/nixos/modules/services/networking/nix-serve.nix
@@ -26,6 +26,12 @@ in
         '';
       };
 
+      openFirewall = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Open ports in the firewall for nix-serve.";
+      };
+
       secretKeyFile = mkOption {
         type = types.nullOr types.str;
         default = null;
@@ -77,5 +83,9 @@ in
           "NIX_SECRET_KEY_FILE:${cfg.secretKeyFile}";
       };
     };
+
+    networking.firewall = mkIf cfg.openFirewall {
+      allowedTCPPorts = [ cfg.port ];
+    };
   };
 }
diff --git a/nixos/modules/services/networking/nsd.nix b/nixos/modules/services/networking/nsd.nix
index cf6c9661dc1b..a51fc5345342 100644
--- a/nixos/modules/services/networking/nsd.nix
+++ b/nixos/modules/services/networking/nsd.nix
@@ -194,19 +194,8 @@ let
                        zone.children
       );
 
-  # fighting infinite recursion
-  zoneOptions = zoneOptionsRaw // childConfig zoneOptions1 true;
-  zoneOptions1 = zoneOptionsRaw // childConfig zoneOptions2 false;
-  zoneOptions2 = zoneOptionsRaw // childConfig zoneOptions3 false;
-  zoneOptions3 = zoneOptionsRaw // childConfig zoneOptions4 false;
-  zoneOptions4 = zoneOptionsRaw // childConfig zoneOptions5 false;
-  zoneOptions5 = zoneOptionsRaw // childConfig zoneOptions6 false;
-  zoneOptions6 = zoneOptionsRaw // childConfig null         false;
-
-  childConfig = x: v: { options.children = { type = types.attrsOf x; visible = v; }; };
-
   # options are ordered alphanumerically
-  zoneOptionsRaw = types.submodule {
+  zoneOptions = types.submodule {
     options = {
 
       allowAXFRFallback = mkOption {
@@ -246,6 +235,13 @@ let
       };
 
       children = mkOption {
+        # TODO: This relies on the fact that `types.anything` doesn't set any
+        # values of its own to any defaults, because in the above zoneConfigs',
+        # values from children override ones from parents, but only if the
+        # attributes are defined. Because of this, we can't replace the element
+        # type here with `zoneConfigs`, since that would set all the attributes
+        # to default values, breaking the parent inheriting function.
+        type = types.attrsOf types.anything;
         default = {};
         description = ''
           Children zones inherit all options of their parents. Attributes
diff --git a/nixos/modules/services/networking/ntopng.nix b/nixos/modules/services/networking/ntopng.nix
index c15257117137..022fc923edaa 100644
--- a/nixos/modules/services/networking/ntopng.nix
+++ b/nixos/modules/services/networking/ntopng.nix
@@ -1,11 +1,18 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
 
   cfg = config.services.ntopng;
-  redisCfg = config.services.redis;
+  opt = options.services.ntopng;
+
+  createRedis = cfg.redis.createInstance != null;
+  redisService =
+    if cfg.redis.createInstance == "" then
+      "redis.service"
+    else
+      "redis-${cfg.redis.createInstance}.service";
 
   configFile = if cfg.configText != "" then
     pkgs.writeText "ntopng.conf" ''
@@ -14,8 +21,10 @@ let
     else
     pkgs.writeText "ntopng.conf" ''
       ${concatStringsSep " " (map (e: "--interface=" + e) cfg.interfaces)}
-      --http-port=${toString cfg.http-port}
-      --redis=localhost:${toString redisCfg.port}
+      --http-port=${toString cfg.httpPort}
+      --redis=${cfg.redis.address}
+      --data-dir=/var/lib/ntopng
+      --user=ntopng
       ${cfg.extraConfig}
     '';
 
@@ -23,6 +32,10 @@ in
 
 {
 
+  imports = [
+    (mkRenamedOptionModule [ "services" "ntopng" "http-port" ] [ "services" "ntopng" "httpPort" ])
+  ];
+
   options = {
 
     services.ntopng = {
@@ -35,8 +48,8 @@ in
           collection tool.
 
           With the default configuration, ntopng monitors all network
-          interfaces and displays its findings at http://localhost:${toString
-          cfg.http-port}. Default username and password is admin/admin.
+          interfaces and displays its findings at http://localhost:''${toString
+          config.${opt.http-port}}. Default username and password is admin/admin.
 
           See the ntopng(8) manual page and http://www.ntop.org/products/ntop/
           for more info.
@@ -55,7 +68,7 @@ in
         '';
       };
 
-      http-port = mkOption {
+      httpPort = mkOption {
         default = 3000;
         type = types.int;
         description = ''
@@ -63,6 +76,24 @@ in
         '';
       };
 
+      redis.address = mkOption {
+        type = types.str;
+        example = literalExpression "config.services.redis.ntopng.unixSocket";
+        description = ''
+          Redis address - may be a Unix socket or a network host and port.
+        '';
+      };
+
+      redis.createInstance = mkOption {
+        type = types.nullOr types.str;
+        default = if versionAtLeast config.system.stateVersion "22.05" then "ntopng" else "";
+        description = ''
+          Local Redis instance name. Set to <literal>null</literal> to disable
+          local Redis instance. Defaults to <literal>""</literal> for
+          <literal>system.stateVersion</literal> older than 22.05.
+        '';
+      };
+
       configText = mkOption {
         default = "";
         example = ''
@@ -94,23 +125,36 @@ in
   config = mkIf cfg.enable {
 
     # ntopng uses redis for data storage
-    services.redis.enable = true;
+    services.ntopng.redis.address =
+      mkIf createRedis config.services.redis.servers.${cfg.redis.createInstance}.unixSocket;
+
+    services.redis.servers = mkIf createRedis {
+      ${cfg.redis.createInstance} = {
+        enable = true;
+        user = mkIf (cfg.redis.createInstance == "ntopng") "ntopng";
+      };
+    };
 
     # nice to have manual page and ntopng command in PATH
     environment.systemPackages = [ pkgs.ntopng ];
 
+    systemd.tmpfiles.rules = [ "d /var/lib/ntopng 0700 ntopng ntopng -" ];
+
     systemd.services.ntopng = {
       description = "Ntopng Network Monitor";
-      requires = [ "redis.service" ];
-      after = [ "network.target" "redis.service" ];
+      requires = optional createRedis redisService;
+      after = [ "network.target" ] ++ optional createRedis redisService;
       wantedBy = [ "multi-user.target" ];
-      preStart = "mkdir -p /var/lib/ntopng/";
       serviceConfig.ExecStart = "${pkgs.ntopng}/bin/ntopng ${configFile}";
       unitConfig.Documentation = "man:ntopng(8)";
     };
 
-    # ntopng drops priveleges to user "nobody" and that user is already defined
-    # in users-groups.nix.
+    users.extraUsers.ntopng = {
+      group = "ntopng";
+      isSystemUser = true;
+    };
+
+    users.extraGroups.ntopng = { };
   };
 
 }
diff --git a/nixos/modules/services/networking/pleroma.nix b/nixos/modules/services/networking/pleroma.nix
index 9b8382392c0a..c6d4c14dcb7e 100644
--- a/nixos/modules/services/networking/pleroma.nix
+++ b/nixos/modules/services/networking/pleroma.nix
@@ -1,6 +1,7 @@
 { config, options, lib, pkgs, stdenv, ... }:
 let
   cfg = config.services.pleroma;
+  cookieFile = "/var/lib/pleroma/.cookie";
 in {
   options = {
     services.pleroma = with lib; {
@@ -8,7 +9,7 @@ in {
 
       package = mkOption {
         type = types.package;
-        default = pkgs.pleroma;
+        default = pkgs.pleroma.override { inherit cookieFile; };
         defaultText = literalExpression "pkgs.pleroma";
         description = "Pleroma package to use.";
       };
@@ -100,7 +101,6 @@ in {
       after = [ "network-online.target" "postgresql.service" ];
       wantedBy = [ "multi-user.target" ];
       restartTriggers = [ config.environment.etc."/pleroma/config.exs".source ];
-      environment.RELEASE_COOKIE = "/var/lib/pleroma/.cookie";
       serviceConfig = {
         User = cfg.user;
         Group = cfg.group;
@@ -118,10 +118,10 @@ in {
         # Better be safe than sorry migration-wise.
         ExecStartPre =
           let preScript = pkgs.writers.writeBashBin "pleromaStartPre" ''
-            if [ ! -f /var/lib/pleroma/.cookie ]
+            if [ ! -f "${cookieFile}" ] || [ ! -s "${cookieFile}" ]
             then
               echo "Creating cookie file"
-              dd if=/dev/urandom bs=1 count=16 | hexdump -e '16/1 "%02x"' > /var/lib/pleroma/.cookie
+              dd if=/dev/urandom bs=1 count=16 | ${pkgs.hexdump}/bin/hexdump -e '16/1 "%02x"' > "${cookieFile}"
             fi
             ${cfg.package}/bin/pleroma_ctl migrate
           '';
diff --git a/nixos/modules/services/networking/prosody.xml b/nixos/modules/services/networking/prosody.xml
index 471240cd1475..6358d744ff78 100644
--- a/nixos/modules/services/networking/prosody.xml
+++ b/nixos/modules/services/networking/prosody.xml
@@ -72,7 +72,7 @@ services.prosody = {
    a TLS certificate for the three endponits:
     <programlisting>
 security.acme = {
-  <link linkend="opt-security.acme.email">email</link> = "root@example.org";
+  <link linkend="opt-security.acme.defaults.email">email</link> = "root@example.org";
   <link linkend="opt-security.acme.acceptTerms">acceptTerms</link> = true;
   <link linkend="opt-security.acme.certs">certs</link> = {
     "example.org" = {
diff --git a/nixos/modules/services/networking/quassel.nix b/nixos/modules/services/networking/quassel.nix
index 22940ef7a13a..844c9a6b8b35 100644
--- a/nixos/modules/services/networking/quassel.nix
+++ b/nixos/modules/services/networking/quassel.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.quassel;
+  opt = options.services.quassel;
   quassel = cfg.package;
   user = if cfg.user != null then cfg.user else "quassel";
 in
@@ -63,6 +64,9 @@ in
 
       dataDir = mkOption {
         default = "/home/${user}/.config/quassel-irc.org";
+        defaultText = literalExpression ''
+          "/home/''${config.${opt.user}}/.config/quassel-irc.org"
+        '';
         type = types.str;
         description = ''
           The directory holding configuration files, the SQlite database and the SSL Cert.
diff --git a/nixos/modules/services/networking/quorum.nix b/nixos/modules/services/networking/quorum.nix
index 50148dc314da..bddcd18c7fbe 100644
--- a/nixos/modules/services/networking/quorum.nix
+++ b/nixos/modules/services/networking/quorum.nix
@@ -1,9 +1,10 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 let
 
   inherit (lib) mkEnableOption mkIf mkOption literalExpression types optionalString;
 
   cfg = config.services.quorum;
+  opt = options.services.quorum;
   dataDir = "/var/lib/quorum";
   genesisFile = pkgs.writeText "genesis.json" (builtins.toJSON cfg.genesis);
   staticNodesFile = pkgs.writeText "static-nodes.json" (builtins.toJSON cfg.staticNodes);
@@ -23,6 +24,7 @@ in {
       group = mkOption {
         type = types.str;
         default = cfg.user;
+        defaultText = literalExpression "config.${opt.user}";
         description = "The group as which to run quorum.";
       };
 
diff --git a/nixos/modules/services/networking/racoon.nix b/nixos/modules/services/networking/racoon.nix
deleted file mode 100644
index 328f4cb1497f..000000000000
--- a/nixos/modules/services/networking/racoon.nix
+++ /dev/null
@@ -1,45 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  cfg = config.services.racoon;
-in {
-  options.services.racoon = {
-    enable = mkEnableOption "racoon";
-
-    config = mkOption {
-      description = "Contents of racoon configuration file.";
-      default = "";
-      type = types.str;
-    };
-
-    configPath = mkOption {
-      description = "Location of racoon config if config is not provided.";
-      default = "/etc/racoon/racoon.conf";
-      type = types.path;
-    };
-  };
-
-  config = mkIf cfg.enable {
-    systemd.services.racoon = {
-      description = "Racoon Daemon";
-      wantedBy = [ "multi-user.target" ];
-      after = [ "network.target" ];
-      serviceConfig = {
-        ExecStart = "${pkgs.ipsecTools}/bin/racoon -f ${
-          if (cfg.config != "") then pkgs.writeText "racoon.conf" cfg.config
-          else cfg.configPath
-        }";
-        ExecReload = "${pkgs.ipsecTools}/bin/racoonctl reload-config";
-        PIDFile = "/run/racoon.pid";
-        Type = "forking";
-        Restart = "always";
-      };
-      preStart = ''
-        rm /run/racoon.pid || true
-        mkdir -p /var/racoon
-      '';
-    };
-  };
-}
diff --git a/nixos/modules/services/networking/seafile.nix b/nixos/modules/services/networking/seafile.nix
index d7fb22edebed..2839ffb60a1f 100644
--- a/nixos/modules/services/networking/seafile.nix
+++ b/nixos/modules/services/networking/seafile.nix
@@ -1,7 +1,6 @@
 { config, lib, pkgs, ... }:
 with lib;
 let
-  python = pkgs.python3Packages.python;
   cfg = config.services.seafile;
   settingsFormat = pkgs.formats.ini { };
 
@@ -221,9 +220,7 @@ in {
         '';
       };
 
-      seahub = let
-        penv = (pkgs.python3.withPackages (ps: with ps; [ gunicorn seahub ]));
-      in {
+      seahub = {
         description = "Seafile Server Web Frontend";
         wantedBy = [ "seafile.target" ];
         partOf = [ "seafile.target" ];
@@ -231,8 +228,7 @@ in {
         requires = [ "seaf-server.service" ];
         restartTriggers = [ seahubSettings ];
         environment = {
-          PYTHONPATH =
-            "${pkgs.python3Packages.seahub}/thirdpart:${pkgs.python3Packages.seahub}:${penv}/${python.sitePackages}";
+          PYTHONPATH = "${pkgs.seahub.pythonPath}:${pkgs.seahub}/thirdpart:${pkgs.seahub}";
           DJANGO_SETTINGS_MODULE = "seahub.settings";
           CCNET_CONF_DIR = ccnetDir;
           SEAFILE_CONF_DIR = dataDir;
@@ -249,7 +245,7 @@ in {
           LogsDirectory = "seafile";
           ConfigurationDirectory = "seafile";
           ExecStart = ''
-            ${penv}/bin/gunicorn seahub.wsgi:application \
+            ${pkgs.seahub.python.pkgs.gunicorn}/bin/gunicorn seahub.wsgi:application \
             --name seahub \
             --workers ${toString cfg.workers} \
             --log-level=info \
@@ -262,27 +258,27 @@ in {
         preStart = ''
           mkdir -p ${seahubDir}/media
           # Link all media except avatars
-          for m in `find ${pkgs.python3Packages.seahub}/media/ -maxdepth 1 -not -name "avatars"`; do
+          for m in `find ${pkgs.seahub}/media/ -maxdepth 1 -not -name "avatars"`; do
             ln -sf $m ${seahubDir}/media/
           done
           if [ ! -e "${seafRoot}/.seahubSecret" ]; then
-              ${penv}/bin/python ${pkgs.python3Packages.seahub}/tools/secret_key_generator.py > ${seafRoot}/.seahubSecret
+              ${pkgs.seahub.python}/bin/python ${pkgs.seahub}/tools/secret_key_generator.py > ${seafRoot}/.seahubSecret
               chmod 400 ${seafRoot}/.seahubSecret
           fi
           if [ ! -f "${seafRoot}/seahub-setup" ]; then
               # avatars directory should be writable
-              install -D -t ${seahubDir}/media/avatars/ ${pkgs.python3Packages.seahub}/media/avatars/default.png
-              install -D -t ${seahubDir}/media/avatars/groups ${pkgs.python3Packages.seahub}/media/avatars/groups/default.png
+              install -D -t ${seahubDir}/media/avatars/ ${pkgs.seahub}/media/avatars/default.png
+              install -D -t ${seahubDir}/media/avatars/groups ${pkgs.seahub}/media/avatars/groups/default.png
               # init database
-              ${pkgs.python3Packages.seahub}/manage.py migrate
+              ${pkgs.seahub}/manage.py migrate
               # create admin account
-              ${pkgs.expect}/bin/expect -c 'spawn ${pkgs.python3Packages.seahub}/manage.py createsuperuser --email=${cfg.adminEmail}; expect "Password: "; send "${cfg.initialAdminPassword}\r"; expect "Password (again): "; send "${cfg.initialAdminPassword}\r"; expect "Superuser created successfully."'
-              echo "${pkgs.python3Packages.seahub.version}-sqlite" > "${seafRoot}/seahub-setup"
+              ${pkgs.expect}/bin/expect -c 'spawn ${pkgs.seahub}/manage.py createsuperuser --email=${cfg.adminEmail}; expect "Password: "; send "${cfg.initialAdminPassword}\r"; expect "Password (again): "; send "${cfg.initialAdminPassword}\r"; expect "Superuser created successfully."'
+              echo "${pkgs.seahub.version}-sqlite" > "${seafRoot}/seahub-setup"
           fi
-          if [ $(cat "${seafRoot}/seahub-setup" | cut -d"-" -f1) != "${pkgs.python3Packages.seahub.version}" ]; then
+          if [ $(cat "${seafRoot}/seahub-setup" | cut -d"-" -f1) != "${pkgs.seahub.version}" ]; then
               # update database
-              ${pkgs.python3Packages.seahub}/manage.py migrate
-              echo "${pkgs.python3Packages.seahub.version}-sqlite" > "${seafRoot}/seahub-setup"
+              ${pkgs.seahub}/manage.py migrate
+              echo "${pkgs.seahub.version}-sqlite" > "${seafRoot}/seahub-setup"
           fi
         '';
       };
diff --git a/nixos/modules/services/networking/searx.nix b/nixos/modules/services/networking/searx.nix
index 9fb06af7442e..b73f255eb9dd 100644
--- a/nixos/modules/services/networking/searx.nix
+++ b/nixos/modules/services/networking/searx.nix
@@ -228,5 +228,4 @@ in
   };
 
   meta.maintainers = with maintainers; [ rnhmjoj ];
-
 }
diff --git a/nixos/modules/services/networking/sniproxy.nix b/nixos/modules/services/networking/sniproxy.nix
index 28c201f0565e..adca5398e4ab 100644
--- a/nixos/modules/services/networking/sniproxy.nix
+++ b/nixos/modules/services/networking/sniproxy.nix
@@ -14,6 +14,8 @@ let
 
 in
 {
+  imports = [ (mkRemovedOptionModule [ "services" "sniproxy" "logDir" ] "Now done by LogsDirectory=. Set to a custom path if you log to a different folder in your config.") ];
+
   options = {
     services.sniproxy = {
       enable = mkEnableOption "sniproxy server";
@@ -50,13 +52,6 @@ in
           }
         '';
       };
-
-      logDir = mkOption {
-        type = types.str;
-        default = "/var/log/sniproxy/";
-        description = "Location of the log directory for sniproxy.";
-      };
-
     };
 
   };
@@ -66,18 +61,12 @@ in
       description = "sniproxy server";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        test -d ${cfg.logDir} || {
-          echo "Creating initial log directory for sniproxy in ${cfg.logDir}"
-          mkdir -p ${cfg.logDir}
-          chmod 640 ${cfg.logDir}
-          }
-        chown -R ${cfg.user}:${cfg.group} ${cfg.logDir}
-      '';
 
       serviceConfig = {
         Type = "forking";
         ExecStart = "${pkgs.sniproxy}/bin/sniproxy -c ${configFile}";
+        LogsDirectory = "sniproxy";
+        LogsDirectoryMode = "0640";
         Restart = "always";
       };
     };
diff --git a/nixos/modules/services/networking/snowflake-proxy.nix b/nixos/modules/services/networking/snowflake-proxy.nix
new file mode 100644
index 000000000000..2124644ed9b5
--- /dev/null
+++ b/nixos/modules/services/networking/snowflake-proxy.nix
@@ -0,0 +1,81 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.snowflake-proxy;
+in
+{
+  options = {
+    services.snowflake-proxy = {
+      enable = mkEnableOption "System to defeat internet censorship";
+
+      broker = mkOption {
+        description = "Broker URL (default \"https://snowflake-broker.torproject.net/\")";
+        type = with types; nullOr str;
+        default = null;
+      };
+
+      capacity = mkOption {
+        description = "Limits the amount of maximum concurrent clients allowed.";
+        type = with types; nullOr int;
+        default = null;
+      };
+
+      relay = mkOption {
+        description = "websocket relay URL (default \"wss://snowflake.bamsoftware.com/\")";
+        type = with types; nullOr str;
+        default = null;
+      };
+
+      stun = mkOption {
+        description = "STUN broker URL (default \"stun:stun.stunprotocol.org:3478\")";
+        type = with types; nullOr str;
+        default = null;
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.snowflake-proxy = {
+      wantedBy = [ "network-online.target" ];
+      serviceConfig = {
+        ExecStart =
+          "${pkgs.snowflake}/bin/proxy " + concatStringsSep " " (
+            optional (cfg.broker != null) "-broker ${cfg.broker}"
+            ++ optional (cfg.capacity != null) "-capacity ${builtins.toString cfg.capacity}"
+            ++ optional (cfg.relay != null) "-relay ${cfg.relay}"
+            ++ optional (cfg.stun != null) "-stun ${cfg.stun}"
+          );
+
+        # Security Hardening
+        # Refer to systemd.exec(5) for option descriptions.
+        CapabilityBoundingSet = "";
+
+        # implies RemoveIPC=, PrivateTmp=, NoNewPrivileges=, RestrictSUIDSGID=,
+        # ProtectSystem=strict, ProtectHome=read-only
+        DynamicUser = true;
+        LockPersonality = true;
+        PrivateDevices = true;
+        PrivateUsers = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectProc = "invisible";
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = "~@clock @cpu-emulation @debug @mount @obsolete @reboot @swap @privileged @resources";
+        UMask = "0077";
+      };
+    };
+  };
+
+  meta.maintainers = with maintainers; [ yayayayaka ];
+}
diff --git a/nixos/modules/services/networking/squid.nix b/nixos/modules/services/networking/squid.nix
index 9d063b92aa1e..4f3881af8bbf 100644
--- a/nixos/modules/services/networking/squid.nix
+++ b/nixos/modules/services/networking/squid.nix
@@ -81,7 +81,9 @@ let
     http_access deny all
 
     # Squid normally listens to port 3128
-    http_port ${toString cfg.proxyPort}
+    http_port ${
+      optionalString (cfg.proxyAddress != null) "${cfg.proxyAddress}:"
+    }${toString cfg.proxyPort}
 
     # Leave coredumps in the first cache dir
     coredump_dir /var/cache/squid
@@ -109,6 +111,12 @@ in
         description = "Whether to run squid web proxy.";
       };
 
+      proxyAddress = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "IP address on which squid will listen.";
+      };
+
       proxyPort = mkOption {
         type = types.int;
         default = 3128;
diff --git a/nixos/modules/services/networking/ssh/sshd.nix b/nixos/modules/services/networking/ssh/sshd.nix
index 004b4f99670f..230ab673a976 100644
--- a/nixos/modules/services/networking/ssh/sshd.nix
+++ b/nixos/modules/services/networking/ssh/sshd.nix
@@ -30,7 +30,7 @@ let
 
     options.openssh.authorizedKeys = {
       keys = mkOption {
-        type = types.listOf types.str;
+        type = types.listOf types.singleLineStr;
         default = [];
         description = ''
           A list of verbatim OpenSSH public keys that should be added to the
@@ -81,6 +81,7 @@ in
   imports = [
     (mkAliasOptionModule [ "services" "sshd" "enable" ] [ "services" "openssh" "enable" ])
     (mkAliasOptionModule [ "services" "openssh" "knownHosts" ] [ "programs" "ssh" "knownHosts" ])
+    (mkRenamedOptionModule [ "services" "openssh" "challengeResponseAuthentication" ] [ "services" "openssh" "kbdInteractiveAuthentication" ])
   ];
 
   ###### interface
@@ -218,11 +219,11 @@ in
         '';
       };
 
-      challengeResponseAuthentication = mkOption {
+      kbdInteractiveAuthentication = mkOption {
         type = types.bool;
         default = true;
         description = ''
-          Specifies whether challenge/response authentication is allowed.
+          Specifies whether keyboard-interactive authentication is allowed.
         '';
       };
 
@@ -480,6 +481,8 @@ in
             else
               cfg.ports;
             socketConfig.Accept = true;
+            # Prevent brute-force attacks from shutting down socket
+            socketConfig.TriggerLimitIntervalSec = 0;
           };
 
         services."sshd@" = service;
@@ -532,7 +535,7 @@ in
         PermitRootLogin ${cfg.permitRootLogin}
         GatewayPorts ${cfg.gatewayPorts}
         PasswordAuthentication ${if cfg.passwordAuthentication then "yes" else "no"}
-        ChallengeResponseAuthentication ${if cfg.challengeResponseAuthentication then "yes" else "no"}
+        KbdInteractiveAuthentication ${if cfg.kbdInteractiveAuthentication then "yes" else "no"}
 
         PrintMotd no # handled by pam_motd
 
diff --git a/nixos/modules/services/networking/stubby.nix b/nixos/modules/services/networking/stubby.nix
index c5e0f929a126..78c13798dde2 100644
--- a/nixos/modules/services/networking/stubby.nix
+++ b/nixos/modules/services/networking/stubby.nix
@@ -1,180 +1,51 @@
-{ config, lib, pkgs, ...}:
+{ config, lib, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.stubby;
+  settingsFormat = pkgs.formats.yaml { };
+  confFile = settingsFormat.generate "stubby.yml" cfg.settings;
+in {
+  imports = map (x:
+    (mkRemovedOptionModule [ "services" "stubby" x ]
+      "Stubby configuration moved to services.stubby.settings.")) [
+        "authenticationMode"
+        "fallbackProtocols"
+        "idleTimeout"
+        "listenAddresses"
+        "queryPaddingBlocksize"
+        "roundRobinUpstreams"
+        "subnetPrivate"
+        "upstreamServers"
+      ];
 
-  fallbacks = concatMapStringsSep "\n  " (x: "- ${x}") cfg.fallbackProtocols;
-  listeners = concatMapStringsSep "\n  " (x: "- ${x}") cfg.listenAddresses;
-
-  # By default, the recursive resolvers maintained by the getdns
-  # project itself are enabled. More information about both getdns's servers,
-  # as well as third party options for upstream resolvers, can be found here:
-  # https://dnsprivacy.org/wiki/display/DP/DNS+Privacy+Test+Servers
-  #
-  # You can override these values by supplying a yaml-formatted array of your
-  # preferred upstream resolvers in the following format:
-  #
-  # 106 # - address_data: IPv4 or IPv6 address of the upstream
-  #   port: Port for UDP/TCP (default is 53)
-  #   tls_auth_name: Authentication domain name checked against the server
-  #                  certificate
-  #   tls_pubkey_pinset: An SPKI pinset verified against the keys in the server
-  #                      certificate
-  #     - digest: Only "sha256" is currently supported
-  #       value: Base64 encoded value of the sha256 fingerprint of the public
-  #              key
-  #   tls_port: Port for TLS (default is 853)
-
-  defaultUpstream = ''
-    - address_data: 145.100.185.15
-      tls_auth_name: "dnsovertls.sinodun.com"
-      tls_pubkey_pinset:
-        - digest: "sha256"
-          value: 62lKu9HsDVbyiPenApnc4sfmSYTHOVfFgL3pyB+cBL4=
-    - address_data: 145.100.185.16
-      tls_auth_name: "dnsovertls1.sinodun.com"
-      tls_pubkey_pinset:
-        - digest: "sha256"
-          value: cE2ecALeE5B+urJhDrJlVFmf38cJLAvqekONvjvpqUA=
-    - address_data: 185.49.141.37
-      tls_auth_name: "getdnsapi.net"
-      tls_pubkey_pinset:
-        - digest: "sha256"
-          value: foxZRnIh9gZpWnl+zEiKa0EJ2rdCGroMWm02gaxSc9Q=
-    - address_data: 2001:610:1:40ba:145:100:185:15
-      tls_auth_name: "dnsovertls.sinodun.com"
-      tls_pubkey_pinset:
-        - digest: "sha256"
-          value: 62lKu9HsDVbyiPenApnc4sfmSYTHOVfFgL3pyB+cBL4=
-    - address_data: 2001:610:1:40ba:145:100:185:16
-      tls_auth_name: "dnsovertls1.sinodun.com"
-      tls_pubkey_pinset:
-        - digest: "sha256"
-          value: cE2ecALeE5B+urJhDrJlVFmf38cJLAvqekONvjvpqUA=
-    - address_data: 2a04:b900:0:100::38
-      tls_auth_name: "getdnsapi.net"
-      tls_pubkey_pinset:
-        - digest: "sha256"
-          value: foxZRnIh9gZpWnl+zEiKa0EJ2rdCGroMWm02gaxSc9Q=
-  '';
-
-  # Resolution type is not changeable here because it is required per the
-  # stubby documentation:
-  #
-  # "resolution_type: Work in stub mode only (not recursive mode) - required for Stubby
-  # operation."
-  #
-  # https://dnsprivacy.org/wiki/display/DP/Configuring+Stubby
-
-  confFile = pkgs.writeText "stubby.yml" ''
-    resolution_type: GETDNS_RESOLUTION_STUB
-    dns_transport_list:
-      ${fallbacks}
-    appdata_dir: "/var/cache/stubby"
-    tls_authentication: ${cfg.authenticationMode}
-    tls_query_padding_blocksize: ${toString cfg.queryPaddingBlocksize}
-    edns_client_subnet_private: ${if cfg.subnetPrivate then "1" else "0"}
-    idle_timeout: ${toString cfg.idleTimeout}
-    listen_addresses:
-      ${listeners}
-    round_robin_upstreams: ${if cfg.roundRobinUpstreams then "1" else "0"}
-    ${cfg.extraConfig}
-    upstream_recursive_servers:
-    ${cfg.upstreamServers}
-  '';
-in
-
-{
   options = {
     services.stubby = {
 
       enable = mkEnableOption "Stubby DNS resolver";
 
-      fallbackProtocols = mkOption {
-        default = [ "GETDNS_TRANSPORT_TLS" ];
-        type = with types; listOf (enum [
-          "GETDNS_TRANSPORT_TLS"
-          "GETDNS_TRANSPORT_TCP"
-          "GETDNS_TRANSPORT_UDP"
-        ]);
-        description = ''
-          Ordered list composed of one or more transport protocols.
-          Strict mode should only use <literal>GETDNS_TRANSPORT_TLS</literal>.
-          Other options are <literal>GETDNS_TRANSPORT_UDP</literal> and
-          <literal>GETDNS_TRANSPORT_TCP</literal>.
+      settings = mkOption {
+        type = types.attrsOf settingsFormat.type;
+        example = lib.literalExpression ''
+          pkgs.stubby.passthru.settingsExample // {
+            upstream_recursive_servers = [{
+              address_data = "158.64.1.29";
+              tls_auth_name = "kaitain.restena.lu";
+              tls_pubkey_pinset = [{
+                digest = "sha256";
+                value = "7ftvIkA+UeN/ktVkovd/7rPZ6mbkhVI7/8HnFJIiLa4=";
+              }];
+            }];
+          };
         '';
-      };
-
-      authenticationMode = mkOption {
-        default = "GETDNS_AUTHENTICATION_REQUIRED";
-        type = types.enum [
-          "GETDNS_AUTHENTICATION_REQUIRED"
-          "GETDNS_AUTHENTICATION_NONE"
-        ];
         description = ''
-          Selects the Strict or Opportunistic usage profile.
-          For strict, set to <literal>GETDNS_AUTHENTICATION_REQUIRED</literal>.
-          for opportunistic, use <literal>GETDNS_AUTHENTICATION_NONE</literal>.
-        '';
-      };
-
-      queryPaddingBlocksize = mkOption {
-        default = 128;
-        type = types.int;
-        description = ''
-          EDNS0 option to pad the size of the DNS query to the given blocksize.
-        '';
-      };
-
-      subnetPrivate = mkOption {
-        default = true;
-        type = types.bool;
-        description = ''
-          EDNS0 option for ECS client privacy. Default is
-          <literal>true</literal>. If set, this option prevents the client
-          subnet from being sent to authoritative nameservers.
-        '';
-      };
-
-      idleTimeout = mkOption {
-        default = 10000;
-        type = types.int;
-        description = "EDNS0 option for keepalive idle timeout expressed in
-        milliseconds.";
-      };
-
-      listenAddresses = mkOption {
-        default = [ "127.0.0.1" "0::1" ];
-        type = with types; listOf str;
-        description = ''
-          Sets the listen address for the stubby daemon.
-          Uses port 53 by default.
-          Ise IP@port to specify a different port.
-        '';
-      };
-
-      roundRobinUpstreams = mkOption {
-        default = true;
-        type = types.bool;
-        description = ''
-          Instructs stubby to distribute queries across all available name
-          servers. Default is <literal>true</literal>. Set to
-          <literal>false</literal> in order to use the first available.
-        '';
-      };
-
-      upstreamServers = mkOption {
-        default = defaultUpstream;
-        type = types.lines;
-        description = ''
-          Replace default upstreams. See <citerefentry><refentrytitle>stubby
-          </refentrytitle><manvolnum>1</manvolnum></citerefentry> for an
-          example of the entry formatting. In Strict mode, at least one of the
-          following settings must be supplied for each nameserver:
-          <literal>tls_auth_name</literal> or
-          <literal>tls_pubkey_pinset</literal>.
+          Content of the Stubby configuration file. All Stubby settings may be set or queried
+          here. The default settings are available at
+          <literal>pkgs.stubby.passthru.settingsExample</literal>. See
+          <link xlink:href="https://dnsprivacy.org/wiki/display/DP/Configuring+Stubby"/>.
+          A list of the public recursive servers can be found here:
+          <link xlink:href="https://dnsprivacy.org/wiki/display/DP/DNS+Privacy+Test+Servers"/>.
         '';
       };
 
@@ -184,20 +55,21 @@ in
         description = "Enable or disable debug level logging.";
       };
 
-      extraConfig = mkOption {
-        default = "";
-        type = types.lines;
-        description = ''
-          Add additional configuration options. see <citerefentry>
-          <refentrytitle>stubby</refentrytitle><manvolnum>1</manvolnum>
-          </citerefentry>for more options.
-        '';
-      };
     };
   };
 
   config = mkIf cfg.enable {
-    environment.systemPackages = [ pkgs.stubby ];
+    assertions = [{
+      assertion =
+        (cfg.settings.resolution_type or "") == "GETDNS_RESOLUTION_STUB";
+      message = ''
+        services.stubby.settings.resolution_type must be set to "GETDNS_RESOLUTION_STUB".
+        Is services.stubby.settings unset?
+      '';
+    }];
+
+    services.stubby.settings.appdata_dir = "/var/cache/stubby";
+
     systemd.services.stubby = {
       description = "Stubby local DNS resolver";
       after = [ "network.target" ];
diff --git a/nixos/modules/services/networking/stunnel.nix b/nixos/modules/services/networking/stunnel.nix
index 70d0a7d3c12e..df4908a0fff9 100644
--- a/nixos/modules/services/networking/stunnel.nix
+++ b/nixos/modules/services/networking/stunnel.nix
@@ -25,8 +25,8 @@ let
       };
 
       connect = mkOption {
-        type = types.int;
-        description = "To which port the decrypted connection should be forwarded.";
+        type = types.either types.str types.int;
+        description = "Port or IP:Port to which the decrypted connection should be forwarded.";
       };
 
       cert = mkOption {
diff --git a/nixos/modules/services/networking/syncplay.nix b/nixos/modules/services/networking/syncplay.nix
index 27a16fb2e29f..b6faf2d3f772 100644
--- a/nixos/modules/services/networking/syncplay.nix
+++ b/nixos/modules/services/networking/syncplay.nix
@@ -68,7 +68,7 @@ in
     systemd.services.syncplay = {
       description = "Syncplay Service";
       wantedBy    = [ "multi-user.target" ];
-      after       = [ "network-online.target "];
+      after       = [ "network-online.target" ];
 
       serviceConfig = {
         ExecStart = "${pkgs.syncplay}/bin/syncplay-server ${escapeShellArgs cmdArgs}";
diff --git a/nixos/modules/services/networking/syncthing.nix b/nixos/modules/services/networking/syncthing.nix
index 8c44687a3822..3a3d4c80ecff 100644
--- a/nixos/modules/services/networking/syncthing.nix
+++ b/nixos/modules/services/networking/syncthing.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.syncthing;
+  opt = options.services.syncthing;
   defaultUser = "syncthing";
   defaultGroup = defaultUser;
 
@@ -431,7 +432,26 @@ in {
           The path where the settings and keys will exist.
         '';
         default = cfg.dataDir + optionalString cond "/.config/syncthing";
-        defaultText = literalExpression "dataDir${optionalString cond " + \"/.config/syncthing\""}";
+        defaultText = literalDocBook ''
+          <variablelist>
+            <varlistentry>
+              <term><literal>stateVersion >= 19.03</literal></term>
+              <listitem>
+                <programlisting>
+                  config.${opt.dataDir} + "/.config/syncthing"
+                </programlisting>
+              </listitem>
+            </varlistentry>
+            <varlistentry>
+              <term>otherwise</term>
+              <listitem>
+                <programlisting>
+                  config.${opt.dataDir}
+                </programlisting>
+              </listitem>
+            </varlistentry>
+          </variablelist>
+        '';
       };
 
       extraFlags = mkOption {
@@ -448,7 +468,7 @@ in {
         default = false;
         example = true;
         description = ''
-          Whether to open the default ports in the firewall: TCP 22000 for transfers
+          Whether to open the default ports in the firewall: TCP/UDP 22000 for transfers
           and UDP 21027 for discovery.
 
           If multiple users are running Syncthing on this machine, you will need
@@ -484,7 +504,7 @@ in {
 
     networking.firewall = mkIf cfg.openDefaultPorts {
       allowedTCPPorts = [ 22000 ];
-      allowedUDPPorts = [ 21027 ];
+      allowedUDPPorts = [ 21027 22000 ];
     };
 
     systemd.packages = [ pkgs.syncthing ];
diff --git a/nixos/modules/services/networking/teleport.nix b/nixos/modules/services/networking/teleport.nix
new file mode 100644
index 000000000000..454791621800
--- /dev/null
+++ b/nixos/modules/services/networking/teleport.nix
@@ -0,0 +1,99 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.teleport;
+  settingsYaml = pkgs.formats.yaml { };
+in
+{
+  options = {
+    services.teleport = with lib.types; {
+      enable = mkEnableOption "the Teleport service";
+
+      settings = mkOption {
+        type = settingsYaml.type;
+        default = { };
+        example = literalExpression ''
+          {
+            teleport = {
+              nodename = "client";
+              advertise_ip = "192.168.1.2";
+              auth_token = "60bdc117-8ff4-478d-95e4-9914597847eb";
+              auth_servers = [ "192.168.1.1:3025" ];
+              log.severity = "DEBUG";
+            };
+            ssh_service = {
+              enabled = true;
+              labels = {
+                role = "client";
+              };
+            };
+            proxy_service.enabled = false;
+            auth_service.enabled = false;
+          }
+        '';
+        description = ''
+          Contents of the <literal>teleport.yaml</literal> config file.
+          The <literal>--config</literal> arguments will only be passed if this set is not empty.
+
+          See <link xlink:href="https://goteleport.com/docs/setup/reference/config/"/>.
+        '';
+      };
+
+      insecure.enable = mkEnableOption ''
+        starting teleport in insecure mode.
+
+        This is dangerous!
+        Sensitive information will be logged to console and certificates will not be verified.
+        Proceed with caution!
+
+        Teleport starts with disabled certificate validation on Proxy Service, validation still occurs on Auth Service
+      '';
+
+      diag = {
+        enable = mkEnableOption ''
+          endpoints for monitoring purposes.
+
+          See <link xlink:href="https://goteleport.com/docs/setup/admin/troubleshooting/#troubleshooting/"/>
+        '';
+
+        addr = mkOption {
+          type = str;
+          default = "127.0.0.1";
+          description = "Metrics and diagnostics address.";
+        };
+
+        port = mkOption {
+          type = int;
+          default = 3000;
+          description = "Metrics and diagnostics port.";
+        };
+      };
+    };
+  };
+
+  config = mkIf config.services.teleport.enable {
+    environment.systemPackages = [ pkgs.teleport ];
+
+    systemd.services.teleport = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      serviceConfig = {
+        ExecStart = ''
+          ${pkgs.teleport}/bin/teleport start \
+            ${optionalString cfg.insecure.enable "--insecure"} \
+            ${optionalString cfg.diag.enable "--diag-addr=${cfg.diag.addr}:${toString cfg.diag.port}"} \
+            ${optionalString (cfg.settings != { }) "--config=${settingsYaml.generate "teleport.yaml" cfg.settings}"}
+        '';
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        LimitNOFILE = 65536;
+        Restart = "always";
+        RestartSec = "5s";
+        RuntimeDirectory = "teleport";
+        Type = "simple";
+      };
+    };
+  };
+}
+
diff --git a/nixos/modules/services/networking/tetrd.nix b/nixos/modules/services/networking/tetrd.nix
new file mode 100644
index 000000000000..0801ce129246
--- /dev/null
+++ b/nixos/modules/services/networking/tetrd.nix
@@ -0,0 +1,96 @@
+{ config, lib, pkgs, ... }:
+
+{
+  options.services.tetrd.enable = lib.mkEnableOption "tetrd";
+
+  config = lib.mkIf config.services.tetrd.enable {
+    environment = {
+      systemPackages = [ pkgs.tetrd ];
+      etc."resolv.conf".source = "/etc/tetrd/resolv.conf";
+    };
+
+    systemd = {
+      tmpfiles.rules = [ "f /etc/tetrd/resolv.conf - - -" ];
+
+      services.tetrd = {
+        description = pkgs.tetrd.meta.description;
+        wantedBy = [ "multi-user.target" ];
+
+        serviceConfig = {
+          ExecStart = "${pkgs.tetrd}/opt/Tetrd/bin/tetrd";
+          Restart = "always";
+          RuntimeDirectory = "tetrd";
+          RootDirectory = "/run/tetrd";
+          DynamicUser = true;
+          UMask = "006";
+          DeviceAllow = "usb_device";
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          NoNewPrivileges = true;
+          PrivateMounts = true;
+          PrivateNetwork = lib.mkDefault false;
+          PrivateTmp = true;
+          PrivateUsers = lib.mkDefault false;
+          ProtectClock = lib.mkDefault false;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectProc = "invisible";
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" "AF_NETLINK" ];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+
+          SystemCallFilter = [
+            "@system-service"
+            "~@aio"
+            "~@chown"
+            "~@clock"
+            "~@cpu-emulation"
+            "~@debug"
+            "~@keyring"
+            "~@memlock"
+            "~@module"
+            "~@mount"
+            "~@obsolete"
+            "~@pkey"
+            "~@raw-io"
+            "~@reboot"
+            "~@swap"
+            "~@sync"
+          ];
+
+          BindReadOnlyPaths = [
+            builtins.storeDir
+            "/etc/ssl"
+            "/etc/static/ssl"
+            "${pkgs.nettools}/bin/route:/usr/bin/route"
+            "${pkgs.nettools}/bin/ifconfig:/usr/bin/ifconfig"
+          ];
+
+          BindPaths = [
+            "/etc/tetrd/resolv.conf:/etc/resolv.conf"
+            "/run"
+            "/var/log"
+          ];
+
+          CapabilityBoundingSet = [
+            "CAP_DAC_OVERRIDE"
+            "CAP_NET_ADMIN"
+          ];
+
+          AmbientCapabilities = [
+            "CAP_DAC_OVERRIDE"
+            "CAP_NET_ADMIN"
+          ];
+        };
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/networking/thelounge.nix b/nixos/modules/services/networking/thelounge.nix
index b94491639163..a5118fd8b339 100644
--- a/nixos/modules/services/networking/thelounge.nix
+++ b/nixos/modules/services/networking/thelounge.nix
@@ -6,17 +6,31 @@ let
   cfg = config.services.thelounge;
   dataDir = "/var/lib/thelounge";
   configJsData = "module.exports = " + builtins.toJSON (
-    { private = cfg.private; port = cfg.port; } // cfg.extraConfig
+    { inherit (cfg) public port; } // cfg.extraConfig
   );
-in {
+  pluginManifest = {
+    dependencies = builtins.listToAttrs (builtins.map (pkg: { name = getName pkg; value = getVersion pkg; }) cfg.plugins);
+  };
+  plugins = pkgs.runCommandLocal "thelounge-plugins" { } ''
+    mkdir -p $out/node_modules
+    echo ${escapeShellArg (builtins.toJSON pluginManifest)} >> $out/package.json
+    ${concatMapStringsSep "\n" (pkg: ''
+    ln -s ${pkg}/lib/node_modules/${getName pkg} $out/node_modules/${getName pkg}
+    '') cfg.plugins}
+  '';
+in
+{
+  imports = [ (mkRemovedOptionModule [ "services" "thelounge" "private" ] "The option was renamed to `services.thelounge.public` to follow upstream changes.") ];
+
   options.services.thelounge = {
     enable = mkEnableOption "The Lounge web IRC client";
 
-    private = mkOption {
+    public = mkOption {
       type = types.bool;
       default = false;
       description = ''
-        Make your The Lounge instance private. You will need to configure user
+        Make your The Lounge instance public.
+        Setting this to <literal>false</literal> will require you to configure user
         accounts by using the (<command>thelounge</command>) command or by adding
         entries in <filename>${dataDir}/users</filename>. You might need to restart
         The Lounge after making changes to the state directory.
@@ -30,7 +44,7 @@ in {
     };
 
     extraConfig = mkOption {
-      default = {};
+      default = { };
       type = types.attrs;
       example = literalExpression ''{
         reverseProxy = true;
@@ -50,19 +64,32 @@ in {
         Documentation: <link xlink:href="https://thelounge.chat/docs/server/configuration" />
       '';
     };
+
+    plugins = mkOption {
+      default = [ ];
+      type = types.listOf types.package;
+      example = literalExpression "[ pkgs.theLoungePlugins.themes.solarized ]";
+      description = ''
+        The Lounge plugins to install. Plugins can be found in
+        <literal>pkgs.theLoungePlugins.plugins</literal> and <literal>pkgs.theLoungePlugins.themes</literal>.
+      '';
+    };
   };
 
   config = mkIf cfg.enable {
     users.users.thelounge = {
-      description = "thelounge service user";
+      description = "The Lounge service user";
       group = "thelounge";
       isSystemUser = true;
     };
-    users.groups.thelounge = {};
+
+    users.groups.thelounge = { };
+
     systemd.services.thelounge = {
       description = "The Lounge web IRC client";
       wantedBy = [ "multi-user.target" ];
       preStart = "ln -sf ${pkgs.writeText "config.js" configJsData} ${dataDir}/config.js";
+      environment.THELOUNGE_PACKAGES = mkIf (cfg.plugins != [ ]) "${plugins}";
       serviceConfig = {
         User = "thelounge";
         StateDirectory = baseNameOf dataDir;
@@ -72,4 +99,8 @@ in {
 
     environment.systemPackages = [ pkgs.thelounge ];
   };
+
+  meta = {
+    maintainers = with lib.maintainers; [ winter ];
+  };
 }
diff --git a/nixos/modules/services/networking/tinc.nix b/nixos/modules/services/networking/tinc.nix
index 9db433fa0735..31731b60d484 100644
--- a/nixos/modules/services/networking/tinc.nix
+++ b/nixos/modules/services/networking/tinc.nix
@@ -435,5 +435,5 @@ in
     );
   };
 
-  meta.maintainers = with maintainers; [ minijackson ];
+  meta.maintainers = with maintainers; [ minijackson mic92 ];
 }
diff --git a/nixos/modules/services/networking/tox-node.nix b/nixos/modules/services/networking/tox-node.nix
index c24e7fd12850..c6e5c2d6e819 100644
--- a/nixos/modules/services/networking/tox-node.nix
+++ b/nixos/modules/services/networking/tox-node.nix
@@ -8,12 +8,7 @@ let
   homeDir = "/var/lib/tox-node";
 
   configFile = let
-    # fetchurl should be switched to getting this file from tox-node.src once
-    # the dpkg directory is in a release
-    src = pkgs.fetchurl {
-      url = "https://raw.githubusercontent.com/tox-rs/tox-node/master/dpkg/config.yml";
-      sha256 = "1431wzpzm786mcvyzk1rp7ar418n45dr75hdggxvlm7pkpam31xa";
-    };
+    src = "${pkg.src}/dpkg/config.yml";
     confJSON = pkgs.writeText "config.json" (
       builtins.toJSON {
         log-type = cfg.logType;
diff --git a/nixos/modules/services/networking/unbound.nix b/nixos/modules/services/networking/unbound.nix
index f6e963490924..87873c8c1e83 100644
--- a/nixos/modules/services/networking/unbound.nix
+++ b/nixos/modules/services/networking/unbound.nix
@@ -62,6 +62,7 @@ in {
       };
 
       stateDir = mkOption {
+        type = types.path;
         default = "/var/lib/unbound";
         description = "Directory holding all state for unbound to run.";
       };
diff --git a/nixos/modules/services/networking/unifi.nix b/nixos/modules/services/networking/unifi.nix
index 174e919f988b..a683c537f05b 100644
--- a/nixos/modules/services/networking/unifi.nix
+++ b/nixos/modules/services/networking/unifi.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, utils, ... }:
+{ config, options, lib, pkgs, utils, ... }:
 with lib;
 let
   cfg = config.services.unifi;
@@ -7,7 +7,6 @@ let
     @${cfg.jrePackage}/bin/java java \
         ${optionalString (cfg.initialJavaHeapSize != null) "-Xms${(toString cfg.initialJavaHeapSize)}m"} \
         ${optionalString (cfg.maximumJavaHeapSize != null) "-Xmx${(toString cfg.maximumJavaHeapSize)}m"} \
-        ${optionalString (lib.versionOlder cfg.unifiPackage.version "6.5.54") "-Dlog4j2.formatMsgNoLookups=true"} \
         -jar ${stateDir}/lib/ace.jar
   '';
 in
@@ -50,7 +49,7 @@ in
       '';
     };
 
-    services.unifi.openPorts = mkOption {
+    services.unifi.openFirewall = mkOption {
       type = types.bool;
       default = true;
       description = ''
@@ -86,6 +85,10 @@ in
 
   config = mkIf cfg.enable {
 
+    warnings = optional
+      (options.services.unifi.openFirewall.highestPrio >= (mkOptionDefault null).priority)
+      "The current services.unifi.openFirewall = true default is deprecated and will change to false in 22.11. Set it explicitly to silence this warning.";
+
     users.users.unifi = {
       isSystemUser = true;
       group = "unifi";
@@ -94,7 +97,7 @@ in
     };
     users.groups.unifi = {};
 
-    networking.firewall = mkIf cfg.openPorts {
+    networking.firewall = mkIf cfg.openFirewall {
       # https://help.ubnt.com/hc/en-us/articles/218506997
       allowedTCPPorts = [
         8080  # Port for UAP to inform controller.
@@ -192,6 +195,7 @@ in
   };
   imports = [
     (mkRemovedOptionModule [ "services" "unifi" "dataDir" ] "You should move contents of dataDir to /var/lib/unifi/data" )
+    (mkRenamedOptionModule [ "services" "unifi" "openPorts" ] [ "services" "unifi" "openFirewall" ])
   ];
 
   meta.maintainers = with lib.maintainers; [ erictapen pennae ];
diff --git a/nixos/modules/services/networking/vsftpd.nix b/nixos/modules/services/networking/vsftpd.nix
index 710c2d9ca17b..d205302051e1 100644
--- a/nixos/modules/services/networking/vsftpd.nix
+++ b/nixos/modules/services/networking/vsftpd.nix
@@ -153,6 +153,7 @@ in
 
       userlist = mkOption {
         default = [];
+        type = types.listOf types.str;
         description = "See <option>userlistFile</option>.";
       };
 
diff --git a/nixos/modules/services/networking/wasabibackend.nix b/nixos/modules/services/networking/wasabibackend.nix
index 8482823e197f..b6dcd940915a 100644
--- a/nixos/modules/services/networking/wasabibackend.nix
+++ b/nixos/modules/services/networking/wasabibackend.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
   cfg = config.services.wasabibackend;
+  opt = options.services.wasabibackend;
 
-  inherit (lib) mkEnableOption mkIf mkOption optionalAttrs optionalString types;
+  inherit (lib) literalExpression mkEnableOption mkIf mkOption optionalAttrs optionalString types;
 
   confOptions = {
       BitcoinRpcConnectionString = "${cfg.rpc.user}:${cfg.rpc.password}";
@@ -103,6 +104,7 @@ in {
       group = mkOption {
         type = types.str;
         default = cfg.user;
+        defaultText = literalExpression "config.${opt.user}";
         description = "The group as which to run the wasabibackend node.";
       };
     };
diff --git a/nixos/modules/services/networking/wg-netmanager.nix b/nixos/modules/services/networking/wg-netmanager.nix
new file mode 100644
index 000000000000..493ff7ceba9f
--- /dev/null
+++ b/nixos/modules/services/networking/wg-netmanager.nix
@@ -0,0 +1,42 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.wg-netmanager;
+in
+{
+
+  options = {
+    services.wg-netmanager = {
+      enable = mkEnableOption "Wireguard network manager";
+    };
+  };
+
+  ###### implementation
+  config = mkIf cfg.enable {
+    # NOTE: wg-netmanager runs as root
+    systemd.services.wg-netmanager = {
+      description = "Wireguard network manager";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      path = with pkgs; [ wireguard-tools iproute2 wireguard-go ];
+      serviceConfig = {
+        Type = "simple";
+        Restart = "on-failure";
+        ExecStart = "${pkgs.wg-netmanager}/bin/wg_netmanager";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        ExecStop = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+
+        ReadWritePaths = [
+          "/tmp"  # wg-netmanager creates files in /tmp before deleting them after use
+        ];
+      };
+      unitConfig =  {
+        ConditionPathExists = ["/etc/wg_netmanager/network.yaml" "/etc/wg_netmanager/peer.yaml"];
+      };
+    };
+  };
+
+  meta.maintainers = with maintainers; [ gin66 ];
+}
diff --git a/nixos/modules/services/networking/wireguard.nix b/nixos/modules/services/networking/wireguard.nix
index 55b84935b6cb..7cd44b2f8a0a 100644
--- a/nixos/modules/services/networking/wireguard.nix
+++ b/nixos/modules/services/networking/wireguard.nix
@@ -1,10 +1,11 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
 
   cfg = config.networking.wireguard;
+  opt = options.networking.wireguard;
 
   kernel = config.boot.kernelPackages;
 
@@ -438,6 +439,7 @@ in
         type = types.bool;
         # 2019-05-25: Backwards compatibility.
         default = cfg.interfaces != {};
+        defaultText = literalExpression "config.${opt.interfaces} != { }";
         example = true;
       };
 
diff --git a/nixos/modules/services/networking/wpa_supplicant.nix b/nixos/modules/services/networking/wpa_supplicant.nix
index 4aa350d21a2b..c2e1d37e28bf 100644
--- a/nixos/modules/services/networking/wpa_supplicant.nix
+++ b/nixos/modules/services/networking/wpa_supplicant.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, utils, ... }:
+{ config, lib, options, pkgs, utils, ... }:
 
 with lib;
 
@@ -8,15 +8,47 @@ let
     else pkgs.wpa_supplicant;
 
   cfg = config.networking.wireless;
+  opt = options.networking.wireless;
+
+  wpa3Protocols = [ "SAE" "FT-SAE" ];
+  hasMixedWPA = opts:
+    let
+      hasWPA3 = !mutuallyExclusive opts.authProtocols wpa3Protocols;
+      others = subtractLists wpa3Protocols opts.authProtocols;
+    in hasWPA3 && others != [];
+
+  # Gives a WPA3 network higher priority
+  increaseWPA3Priority = opts:
+    opts // optionalAttrs (hasMixedWPA opts)
+      { priority = if opts.priority == null
+                     then 1
+                     else opts.priority + 1;
+      };
+
+  # Creates a WPA2 fallback network
+  mkWPA2Fallback = opts:
+    opts // { authProtocols = subtractLists wpa3Protocols opts.authProtocols; };
+
+  # Networks attrset as a list
+  networkList = mapAttrsToList (ssid: opts: opts // { inherit ssid; })
+                cfg.networks;
+
+  # List of all networks (normal + generated fallbacks)
+  allNetworks =
+    if cfg.fallbackToWPA2
+      then map increaseWPA3Priority networkList
+           ++ map mkWPA2Fallback (filter hasMixedWPA networkList)
+      else networkList;
 
   # Content of wpa_supplicant.conf
   generatedConfig = concatStringsSep "\n" (
-    (mapAttrsToList mkNetwork cfg.networks)
+    (map mkNetwork allNetworks)
     ++ optional cfg.userControlled.enable (concatStringsSep "\n"
       [ "ctrl_interface=/run/wpa_supplicant"
         "ctrl_interface_group=${cfg.userControlled.group}"
         "update_config=1"
       ])
+    ++ [ "pmf=1" ]
     ++ optional cfg.scanOnLowSignal ''bgscan="simple:30:-70:3600"''
     ++ optional (cfg.extraConfig != "") cfg.extraConfig);
 
@@ -32,7 +64,7 @@ let
   finalConfig = ''"$RUNTIME_DIRECTORY"/wpa_supplicant.conf'';
 
   # Creates a network block for wpa_supplicant.conf
-  mkNetwork = ssid: opts:
+  mkNetwork = opts:
   let
     quote = x: ''"${x}"'';
     indent = x: "  " + x;
@@ -42,7 +74,7 @@ let
       else opts.pskRaw;
 
     options = [
-      "ssid=${quote ssid}"
+      "ssid=${quote opts.ssid}"
       (if pskString != null || opts.auth != null
         then "key_mgmt=${concatStringsSep " " opts.authProtocols}"
         else "key_mgmt=NONE")
@@ -174,6 +206,18 @@ in {
         '';
       };
 
+      fallbackToWPA2 = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Whether to fall back to WPA2 authentication protocols if WPA3 failed.
+          This allows old wireless cards (that lack recent features required by
+          WPA3) to connect to mixed WPA2/WPA3 access points.
+
+          To avoid possible downgrade attacks, disable this options.
+        '';
+      };
+
       environmentFile = mkOption {
         type = types.nullOr types.path;
         default = null;
@@ -421,6 +465,7 @@ in {
       dbusControlled = mkOption {
         type = types.bool;
         default = lib.length cfg.interfaces < 2;
+        defaultText = literalExpression "length config.${opt.interfaces} < 2";
         description = ''
           Whether to enable the DBus control interface.
           This is only needed when using NetworkManager or connman.
diff --git a/nixos/modules/services/networking/xrdp.nix b/nixos/modules/services/networking/xrdp.nix
index e9f123a181ae..747fb7a1f9c4 100644
--- a/nixos/modules/services/networking/xrdp.nix
+++ b/nixos/modules/services/networking/xrdp.nix
@@ -100,6 +100,7 @@ in
       confDir = mkOption {
         type = types.path;
         default = confDir;
+        defaultText = literalDocBook "generated from configuration";
         description = "The location of the config files for xrdp.";
       };
     };
diff --git a/nixos/modules/services/networking/yggdrasil.xml b/nixos/modules/services/networking/yggdrasil.xml
index c012cd4a9294..a341d5d8153b 100644
--- a/nixos/modules/services/networking/yggdrasil.xml
+++ b/nixos/modules/services/networking/yggdrasil.xml
@@ -84,7 +84,6 @@ in {
       interface eth0
       {
         AdvSendAdvert on;
-        AdvDefaultLifetime 0;
         prefix ${prefix}::/64 {
           AdvOnLink on;
           AdvAutonomous on;
diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix
index 6df147be0c49..041d0b3c43fd 100644
--- a/nixos/modules/services/search/elasticsearch.nix
+++ b/nixos/modules/services/search/elasticsearch.nix
@@ -143,6 +143,17 @@ in
       example = lib.literalExpression "[ pkgs.elasticsearchPlugins.discovery-ec2 ]";
     };
 
+    restartIfChanged  = mkOption {
+      type = types.bool;
+      description = ''
+        Automatically restart the service on config change.
+        This can be set to false to defer restarts on a server or cluster.
+        Please consider the security implications of inadvertently running an older version,
+        and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+      '';
+      default = true;
+    };
+
   };
 
   ###### implementation
@@ -153,6 +164,7 @@ in
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
       path = [ pkgs.inetutils ];
+      inherit (cfg) restartIfChanged;
       environment = {
         ES_HOME = cfg.dataDir;
         ES_JAVA_OPTS = toString cfg.extraJavaOptions;
@@ -163,6 +175,8 @@ in
         User = "elasticsearch";
         PermissionsStartOnly = true;
         LimitNOFILE = "1024000";
+        Restart = "always";
+        TimeoutStartSec = "infinity";
       };
       preStart = ''
         ${optionalString (!config.boot.isContainer) ''
@@ -204,7 +218,7 @@ in
       postStart = ''
         # Make sure elasticsearch is up and running before dependents
         # are started
-        while ! ${pkgs.curl}/bin/curl -sS -f http://localhost:${toString cfg.port} 2>/dev/null; do
+        while ! ${pkgs.curl}/bin/curl -sS -f http://${cfg.listenAddress}:${toString cfg.port} 2>/dev/null; do
           sleep 1
         done
       '';
diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix
index 381f5156ceb6..e4ab85be9ef1 100644
--- a/nixos/modules/services/search/kibana.nix
+++ b/nixos/modules/services/search/kibana.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.kibana;
+  opt = options.services.kibana;
 
   ge7 = builtins.compareVersions cfg.package.version "7" >= 0;
   lt6_6 = builtins.compareVersions cfg.package.version "6.6" < 0;
@@ -130,6 +131,9 @@ in {
           This defaults to the singleton list [ca] when the <option>ca</option> option is defined.
         '';
         default = if cfg.elasticsearch.ca == null then [] else [ca];
+        defaultText = literalExpression ''
+          if config.${opt.elasticsearch.ca} == null then [ ] else [ ca ]
+        '';
         type = types.listOf types.path;
       };
 
diff --git a/nixos/modules/services/security/aesmd.nix b/nixos/modules/services/security/aesmd.nix
new file mode 100644
index 000000000000..8268b034a15e
--- /dev/null
+++ b/nixos/modules/services/security/aesmd.nix
@@ -0,0 +1,236 @@
+{ config, options, pkgs, lib, ... }:
+with lib;
+let
+  cfg = config.services.aesmd;
+  opt = options.services.aesmd;
+
+  sgx-psw = pkgs.sgx-psw.override { inherit (cfg) debug; };
+
+  configFile = with cfg.settings; pkgs.writeText "aesmd.conf" (
+    concatStringsSep "\n" (
+      optional (whitelistUrl != null) "whitelist url = ${whitelistUrl}" ++
+      optional (proxy != null) "aesm proxy = ${proxy}" ++
+      optional (proxyType != null) "proxy type = ${proxyType}" ++
+      optional (defaultQuotingType != null) "default quoting type = ${defaultQuotingType}" ++
+      # Newline at end of file
+      [ "" ]
+    )
+  );
+in
+{
+  options.services.aesmd = {
+    enable = mkEnableOption "Intel's Architectural Enclave Service Manager (AESM) for Intel SGX";
+    debug = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Whether to build the PSW package in debug mode.";
+    };
+    settings = mkOption {
+      description = "AESM configuration";
+      default = { };
+      type = types.submodule {
+        options.whitelistUrl = mkOption {
+          type = with types; nullOr str;
+          default = null;
+          example = "http://whitelist.trustedservices.intel.com/SGX/LCWL/Linux/sgx_white_list_cert.bin";
+          description = "URL to retrieve authorized Intel SGX enclave signers.";
+        };
+        options.proxy = mkOption {
+          type = with types; nullOr str;
+          default = null;
+          example = "http://proxy_url:1234";
+          description = "HTTP network proxy.";
+        };
+        options.proxyType = mkOption {
+          type = with types; nullOr (enum [ "default" "direct" "manual" ]);
+          default = if (cfg.settings.proxy != null) then "manual" else null;
+          defaultText = literalExpression ''
+            if (config.${opt.settings}.proxy != null) then "manual" else null
+          '';
+          example = "default";
+          description = ''
+            Type of proxy to use. The <literal>default</literal> uses the system's default proxy.
+            If <literal>direct</literal> is given, uses no proxy.
+            A value of <literal>manual</literal> uses the proxy from
+            <option>services.aesmd.settings.proxy</option>.
+          '';
+        };
+        options.defaultQuotingType = mkOption {
+          type = with types; nullOr (enum [ "ecdsa_256" "epid_linkable" "epid_unlinkable" ]);
+          default = null;
+          example = "ecdsa_256";
+          description = "Attestation quote type.";
+        };
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [{
+      assertion = !(config.boot.specialFileSystems."/dev".options ? "noexec");
+      message = "SGX requires exec permission for /dev";
+    }];
+
+    hardware.cpu.intel.sgx.provision.enable = true;
+
+    # Make sure the AESM service can find the SGX devices until
+    # https://github.com/intel/linux-sgx/issues/772 is resolved
+    # and updated in nixpkgs.
+    hardware.cpu.intel.sgx.enableDcapCompat = mkForce true;
+
+    systemd.services.aesmd =
+      let
+        storeAesmFolder = "${sgx-psw}/aesm";
+        # Hardcoded path AESM_DATA_FOLDER in psw/ae/aesm_service/source/oal/linux/aesm_util.cpp
+        aesmDataFolder = "/var/opt/aesmd/data";
+        aesmStateDirSystemd = "%S/aesmd";
+      in
+      {
+        description = "Intel Architectural Enclave Service Manager";
+        wantedBy = [ "multi-user.target" ];
+
+        after = [
+          "auditd.service"
+          "network.target"
+          "syslog.target"
+        ];
+
+        environment = {
+          NAME = "aesm_service";
+          AESM_PATH = storeAesmFolder;
+          LD_LIBRARY_PATH = storeAesmFolder;
+        };
+
+        # Make sure any of the SGX application enclave devices is available
+        unitConfig.AssertPathExists = [
+          # legacy out-of-tree driver
+          "|/dev/isgx"
+          # DCAP driver
+          "|/dev/sgx/enclave"
+          # in-tree driver
+          "|/dev/sgx_enclave"
+        ];
+
+        serviceConfig = rec {
+          ExecStartPre = pkgs.writeShellScript "copy-aesmd-data-files.sh" ''
+            set -euo pipefail
+            whiteListFile="${aesmDataFolder}/white_list_cert_to_be_verify.bin"
+            if [[ ! -f "$whiteListFile" ]]; then
+              ${pkgs.coreutils}/bin/install -m 644 -D \
+                "${storeAesmFolder}/data/white_list_cert_to_be_verify.bin" \
+                "$whiteListFile"
+            fi
+          '';
+          ExecStart = "${sgx-psw}/bin/aesm_service --no-daemon";
+          ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"'';
+
+          Restart = "on-failure";
+          RestartSec = "15s";
+
+          DynamicUser = true;
+          Group = "sgx";
+          SupplementaryGroups = [
+            config.hardware.cpu.intel.sgx.provision.group
+          ];
+
+          Type = "simple";
+
+          WorkingDirectory = storeAesmFolder;
+          StateDirectory = "aesmd";
+          StateDirectoryMode = "0700";
+          RuntimeDirectory = "aesmd";
+          RuntimeDirectoryMode = "0750";
+
+          # Hardening
+
+          # chroot into the runtime directory
+          RootDirectory = "%t/aesmd";
+          BindReadOnlyPaths = [
+            builtins.storeDir
+            # Hardcoded path AESM_CONFIG_FILE in psw/ae/aesm_service/source/utils/aesm_config.cpp
+            "${configFile}:/etc/aesmd.conf"
+          ];
+          BindPaths = [
+            # Hardcoded path CONFIG_SOCKET_PATH in psw/ae/aesm_service/source/core/ipc/SocketConfig.h
+            "%t/aesmd:/var/run/aesmd"
+            "%S/aesmd:/var/opt/aesmd"
+          ];
+
+          # PrivateDevices=true will mount /dev noexec which breaks AESM
+          PrivateDevices = false;
+          DevicePolicy = "closed";
+          DeviceAllow = [
+            # legacy out-of-tree driver
+            "/dev/isgx rw"
+            # DCAP driver
+            "/dev/sgx rw"
+            # in-tree driver
+            "/dev/sgx_enclave rw"
+            "/dev/sgx_provision rw"
+          ];
+
+          # Requires Internet access for attestation
+          PrivateNetwork = false;
+
+          RestrictAddressFamilies = [
+            # Allocates the socket /var/run/aesmd/aesm.socket
+            "AF_UNIX"
+            # Uses the HTTP protocol to initialize some services
+            "AF_INET"
+            "AF_INET6"
+          ];
+
+          # True breaks stuff
+          MemoryDenyWriteExecute = false;
+
+          # needs the ipc syscall in order to run
+          SystemCallFilter = [
+            "@system-service"
+            "~@aio"
+            "~@chown"
+            "~@clock"
+            "~@cpu-emulation"
+            "~@debug"
+            "~@keyring"
+            "~@memlock"
+            "~@module"
+            "~@mount"
+            "~@privileged"
+            "~@raw-io"
+            "~@reboot"
+            "~@resources"
+            "~@setuid"
+            "~@swap"
+            "~@sync"
+            "~@timer"
+          ];
+          SystemCallArchitectures = "native";
+          SystemCallErrorNumber = "EPERM";
+
+          CapabilityBoundingSet = "";
+          KeyringMode = "private";
+          LockPersonality = true;
+          NoNewPrivileges = true;
+          NotifyAccess = "none";
+          PrivateMounts = true;
+          PrivateTmp = true;
+          PrivateUsers = true;
+          ProcSubset = "pid";
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectProc = "invisible";
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          UMask = "0066";
+        };
+      };
+  };
+}
diff --git a/nixos/modules/services/security/cfssl.nix b/nixos/modules/services/security/cfssl.nix
index e5bed0a9987c..6df2343b84d2 100644
--- a/nixos/modules/services/security/cfssl.nix
+++ b/nixos/modules/services/security/cfssl.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, options, lib, pkgs, ... }:
 
 with lib;
 
@@ -11,7 +11,16 @@ in {
     dataDir = mkOption {
       default = "/var/lib/cfssl";
       type = types.path;
-      description = "Cfssl work directory.";
+      description = ''
+        The work directory for CFSSL.
+
+        <note><para>
+          If left as the default value this directory will automatically be
+          created before the CFSSL server starts, otherwise you are
+          responsible for ensuring the directory exists with appropriate
+          ownership and permissions.
+        </para></note>
+      '';
     };
 
     address = mkOption {
@@ -22,7 +31,7 @@ in {
 
     port = mkOption {
       default = 8888;
-      type = types.ints.u16;
+      type = types.port;
       description = "Port to bind.";
     };
 
@@ -147,13 +156,12 @@ in {
   };
 
   config = mkIf cfg.enable {
-    users.extraGroups.cfssl = {
+    users.groups.cfssl = {
       gid = config.ids.gids.cfssl;
     };
 
-    users.extraUsers.cfssl = {
+    users.users.cfssl = {
       description = "cfssl user";
-      createHome = true;
       home = cfg.dataDir;
       group = "cfssl";
       uid = config.ids.uids.cfssl;
@@ -164,41 +172,46 @@ in {
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
 
-      serviceConfig = {
-        WorkingDirectory = cfg.dataDir;
-        StateDirectory = cfg.dataDir;
-        StateDirectoryMode = 700;
-        Restart = "always";
-        User = "cfssl";
-
-        ExecStart = with cfg; let
-          opt = n: v: optionalString (v != null) ''-${n}="${v}"'';
-        in
-          lib.concatStringsSep " \\\n" [
-            "${pkgs.cfssl}/bin/cfssl serve"
-            (opt "address" address)
-            (opt "port" (toString port))
-            (opt "ca" ca)
-            (opt "ca-key" caKey)
-            (opt "ca-bundle" caBundle)
-            (opt "int-bundle" intBundle)
-            (opt "int-dir" intDir)
-            (opt "metadata" metadata)
-            (opt "remote" remote)
-            (opt "config" configFile)
-            (opt "responder" responder)
-            (opt "responder-key" responderKey)
-            (opt "tls-key" tlsKey)
-            (opt "tls-cert" tlsCert)
-            (opt "mutual-tls-ca" mutualTlsCa)
-            (opt "mutual-tls-cn" mutualTlsCn)
-            (opt "mutual-tls-client-key" mutualTlsClientKey)
-            (opt "mutual-tls-client-cert" mutualTlsClientCert)
-            (opt "tls-remote-ca" tlsRemoteCa)
-            (opt "db-config" dbConfig)
-            (opt "loglevel" (toString logLevel))
-          ];
-      };
+      serviceConfig = lib.mkMerge [
+        {
+          WorkingDirectory = cfg.dataDir;
+          Restart = "always";
+          User = "cfssl";
+          Group = "cfssl";
+
+          ExecStart = with cfg; let
+            opt = n: v: optionalString (v != null) ''-${n}="${v}"'';
+          in
+            lib.concatStringsSep " \\\n" [
+              "${pkgs.cfssl}/bin/cfssl serve"
+              (opt "address" address)
+              (opt "port" (toString port))
+              (opt "ca" ca)
+              (opt "ca-key" caKey)
+              (opt "ca-bundle" caBundle)
+              (opt "int-bundle" intBundle)
+              (opt "int-dir" intDir)
+              (opt "metadata" metadata)
+              (opt "remote" remote)
+              (opt "config" configFile)
+              (opt "responder" responder)
+              (opt "responder-key" responderKey)
+              (opt "tls-key" tlsKey)
+              (opt "tls-cert" tlsCert)
+              (opt "mutual-tls-ca" mutualTlsCa)
+              (opt "mutual-tls-cn" mutualTlsCn)
+              (opt "mutual-tls-client-key" mutualTlsClientKey)
+              (opt "mutual-tls-client-cert" mutualTlsClientCert)
+              (opt "tls-remote-ca" tlsRemoteCa)
+              (opt "db-config" dbConfig)
+              (opt "loglevel" (toString logLevel))
+            ];
+        }
+        (mkIf (cfg.dataDir == options.services.cfssl.dataDir.default) {
+          StateDirectory = baseNameOf cfg.dataDir;
+          StateDirectoryMode = 700;
+        })
+      ];
     };
 
     services.cfssl = {
diff --git a/nixos/modules/services/security/clamav.nix b/nixos/modules/services/security/clamav.nix
index 340cbbf02fb4..95a0ad8770e2 100644
--- a/nixos/modules/services/security/clamav.nix
+++ b/nixos/modules/services/security/clamav.nix
@@ -9,7 +9,7 @@ let
   pkg = pkgs.clamav;
 
   toKeyValue = generators.toKeyValue {
-    mkKeyValue = generators.mkKeyValueDefault {} " ";
+    mkKeyValue = generators.mkKeyValueDefault { } " ";
     listsAsDuplicateKeys = true;
   };
 
@@ -30,7 +30,7 @@ in
 
         settings = mkOption {
           type = with types; attrsOf (oneOf [ bool int str (listOf str) ]);
-          default = {};
+          default = { };
           description = ''
             ClamAV configuration. Refer to <link xlink:href="https://linux.die.net/man/5/clamd.conf"/>,
             for details on supported values.
@@ -59,7 +59,7 @@ in
 
         settings = mkOption {
           type = with types; attrsOf (oneOf [ bool int str (listOf str) ]);
-          default = {};
+          default = { };
           description = ''
             freshclam configuration. Refer to <link xlink:href="https://linux.die.net/man/5/freshclam.conf"/>,
             for details on supported values.
@@ -104,7 +104,6 @@ in
     systemd.services.clamav-daemon = mkIf cfg.daemon.enable {
       description = "ClamAV daemon (clamd)";
       after = optional cfg.updater.enable "clamav-freshclam.service";
-      requires = optional cfg.updater.enable "clamav-freshclam.service";
       wantedBy = [ "multi-user.target" ];
       restartTriggers = [ clamdConfigFile ];
 
@@ -134,7 +133,7 @@ in
     systemd.services.clamav-freshclam = mkIf cfg.updater.enable {
       description = "ClamAV virus database updater (freshclam)";
       restartTriggers = [ freshclamConfigFile ];
-
+      after = [ "network-online.target" ];
       preStart = ''
         mkdir -m 0755 -p ${stateDir}
         chown ${clamavUser}:${clamavGroup} ${stateDir}
diff --git a/nixos/modules/services/security/fprot.nix b/nixos/modules/services/security/fprot.nix
deleted file mode 100644
index df60d553e85b..000000000000
--- a/nixos/modules/services/security/fprot.nix
+++ /dev/null
@@ -1,82 +0,0 @@
-{ config, lib, pkgs, ... }:
-with lib;
-let
-  fprotUser = "fprot";
-  stateDir = "/var/lib/fprot";
-  fprotGroup = fprotUser;
-  cfg = config.services.fprot;
-in {
-  options = {
-
-    services.fprot = {
-      updater = {
-        enable = mkEnableOption "automatic F-Prot virus definitions database updates";
-
-        productData = mkOption {
-          description = ''
-            product.data file. Defaults to the one supplied with installation package.
-          '';
-          type = types.path;
-        };
-
-        frequency = mkOption {
-          default = 30;
-          type = types.int;
-          description = ''
-            Update virus definitions every X minutes.
-          '';
-        };
-
-        licenseKeyfile = mkOption {
-          type = types.path;
-          description = ''
-            License keyfile. Defaults to the one supplied with installation package.
-          '';
-        };
-
-      };
-    };
-  };
-
-  ###### implementation
-
-  config = mkIf cfg.updater.enable {
-
-    services.fprot.updater.productData = mkDefault "${pkgs.fprot}/opt/f-prot/product.data";
-    services.fprot.updater.licenseKeyfile = mkDefault "${pkgs.fprot}/opt/f-prot/license.key";
-
-    environment.systemPackages = [ pkgs.fprot ];
-    environment.etc."f-prot.conf" = {
-      source = "${pkgs.fprot}/opt/f-prot/f-prot.conf";
-    };
-
-    users.users.${fprotUser} =
-      { uid = config.ids.uids.fprot;
-        description = "F-Prot daemon user";
-        home = stateDir;
-      };
-
-    users.groups.${fprotGroup} =
-      { gid = config.ids.gids.fprot; };
-
-    services.cron.systemCronJobs = [ "*/${toString cfg.updater.frequency} * * * * root start fprot-updater" ];
-
-    systemd.services.fprot-updater = {
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = false;
-      };
-      wantedBy = [ "multi-user.target" ];
-
-      # have to copy fpupdate executable because it insists on storing the virus database in the same dir
-      preStart = ''
-        mkdir -m 0755 -p ${stateDir}
-        chown ${fprotUser}:${fprotGroup} ${stateDir}
-        cp ${pkgs.fprot}/opt/f-prot/fpupdate ${stateDir}
-        ln -sf ${cfg.updater.productData} ${stateDir}/product.data
-      '';
-
-      script = "/var/lib/fprot/fpupdate --keyfile ${cfg.updater.licenseKeyfile}";
-    };
- };
-}
diff --git a/nixos/modules/services/security/haveged.nix b/nixos/modules/services/security/haveged.nix
index 22ece1883446..57cef7e44d50 100644
--- a/nixos/modules/services/security/haveged.nix
+++ b/nixos/modules/services/security/haveged.nix
@@ -3,12 +3,10 @@
 with lib;
 
 let
-
   cfg = config.services.haveged;
 
 in
 
-
 {
 
   ###### interface
@@ -17,14 +15,11 @@ in
 
     services.haveged = {
 
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to enable to haveged entropy daemon, which refills
-          /dev/random when low.
-        '';
-      };
+      enable = mkEnableOption ''
+        haveged entropy daemon, which refills /dev/random when low.
+        NOTE: does nothing on kernels newer than 5.6.
+      '';
+      # source for the note https://github.com/jirka-h/haveged/issues/57
 
       refill_threshold = mkOption {
         type = types.int;
@@ -39,29 +34,44 @@ in
 
   };
 
-
-  ###### implementation
-
   config = mkIf cfg.enable {
 
-    systemd.services.haveged =
-      { description = "Entropy Harvesting Daemon";
-        unitConfig.Documentation = "man:haveged(8)";
-        wantedBy = [ "multi-user.target" ];
-
-        path = [ pkgs.haveged ];
-
-        serviceConfig = {
-          ExecStart = "${pkgs.haveged}/bin/haveged -F -w ${toString cfg.refill_threshold} -v 1";
-          SuccessExitStatus = 143;
-          PrivateTmp = true;
-          PrivateDevices = true;
-          PrivateNetwork = true;
-          ProtectSystem = "full";
-          ProtectHome = true;
-        };
+    # https://github.com/jirka-h/haveged/blob/a4b69d65a8dfc5a9f52ff8505c7f58dcf8b9234f/contrib/Fedora/haveged.service
+    systemd.services.haveged = {
+      description = "Entropy Daemon based on the HAVEGE algorithm";
+      unitConfig = {
+        Documentation = "man:haveged(8)";
+        DefaultDependencies = false;
+        ConditionKernelVersion = "<5.6";
+      };
+      wantedBy = [ "sysinit.target" ];
+      after = [ "systemd-tmpfiles-setup-dev.service" ];
+      before = [ "sysinit.target" "shutdown.target" "systemd-journald.service" ];
+
+      serviceConfig = {
+        ExecStart = "${pkgs.haveged}/bin/haveged -w ${toString cfg.refill_threshold} --Foreground -v 1";
+        Restart = "always";
+        SuccessExitStatus = "137 143";
+        SecureBits = "noroot-locked";
+        CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_SYS_CHROOT" ];
+        # We can *not* set PrivateTmp=true as it can cause an ordering cycle.
+        PrivateTmp = false;
+        PrivateDevices = true;
+        ProtectSystem = "full";
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [ "@system-service" "newuname" "~@mount" ];
+        SystemCallErrorNumber = "EPERM";
       };
 
+    };
   };
 
 }
diff --git a/nixos/modules/services/security/oauth2_proxy.nix b/nixos/modules/services/security/oauth2_proxy.nix
index 4d3562424170..ce295bd4ba3b 100644
--- a/nixos/modules/services/security/oauth2_proxy.nix
+++ b/nixos/modules/services/security/oauth2_proxy.nix
@@ -102,17 +102,19 @@ in
     # Taken from: https://github.com/oauth2-proxy/oauth2-proxy/blob/master/providers/providers.go
     provider = mkOption {
       type = types.enum [
-        "google"
+        "adfs"
         "azure"
+        "bitbucket"
+        "digitalocean"
         "facebook"
         "github"
-        "keycloak"
         "gitlab"
+        "google"
+        "keycloak"
+        "keycloak-oidc"
         "linkedin"
         "login.gov"
-        "bitbucket"
         "nextcloud"
-        "digitalocean"
         "oidc"
       ];
       default = "google";
diff --git a/nixos/modules/services/security/opensnitch.nix b/nixos/modules/services/security/opensnitch.nix
index 919346cf2bb1..f9b4985e1991 100644
--- a/nixos/modules/services/security/opensnitch.nix
+++ b/nixos/modules/services/security/opensnitch.nix
@@ -3,22 +3,123 @@
 with lib;
 
 let
-  name = "opensnitch";
   cfg = config.services.opensnitch;
+  format = pkgs.formats.json {};
 in {
   options = {
     services.opensnitch = {
       enable = mkEnableOption "Opensnitch application firewall";
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = format.type;
+
+          options = {
+            Server = {
+
+              Address = mkOption {
+                type = types.str;
+                description = ''
+                  Unix socket path (unix:///tmp/osui.sock, the "unix:///" part is
+                  mandatory) or TCP socket (192.168.1.100:50051).
+                '';
+              };
+
+              LogFile = mkOption {
+                type = types.path;
+                description = ''
+                  File to write logs to (use /dev/stdout to write logs to standard
+                  output).
+                '';
+              };
+
+            };
+
+            DefaultAction = mkOption {
+              type = types.enum [ "allow" "deny" ];
+              description = ''
+                Default action whether to block or allow application internet
+                access.
+              '';
+            };
+
+            DefaultDuration = mkOption {
+              type = types.enum [
+                "once" "always" "until restart" "30s" "5m" "15m" "30m" "1h"
+              ];
+              description = ''
+                Default duration of firewall rule.
+              '';
+            };
+
+            InterceptUnknown = mkOption {
+              type = types.bool;
+              description = ''
+                Wheter to intercept spare connections.
+              '';
+            };
+
+            ProcMonitorMethod = mkOption {
+              type = types.enum [ "ebpf" "proc" "ftrace" "audit" ];
+              description = ''
+                Which process monitoring method to use.
+              '';
+            };
+
+            LogLevel = mkOption {
+              type = types.enum [ 0 1 2 3 4 ];
+              description = ''
+                Default log level from 0 to 4 (debug, info, important, warning,
+                error).
+              '';
+            };
+
+            Firewall = mkOption {
+              type = types.enum [ "iptables" "nftables" ];
+              description = ''
+                Which firewall backend to use.
+              '';
+            };
+
+            Stats = {
+
+              MaxEvents = mkOption {
+                type = types.int;
+                description = ''
+                  Max events to send to the GUI.
+                '';
+              };
+
+              MaxStats = mkOption {
+                type = types.int;
+                description = ''
+                  Max stats per item to keep in backlog.
+                '';
+              };
+
+            };
+          };
+        };
+        description = ''
+          opensnitchd configuration. Refer to
+          <link xlink:href="https://github.com/evilsocket/opensnitch/wiki/Configurations"/>
+          for details on supported values.
+        '';
+      };
     };
   };
 
   config = mkIf cfg.enable {
 
+    # pkg.opensnitch is referred to elsewhere in the module so we don't need to worry about it being garbage collected
+    services.opensnitch.settings = mapAttrs (_: v: mkDefault v) (builtins.fromJSON (builtins.unsafeDiscardStringContext (builtins.readFile "${pkgs.opensnitch}/etc/default-config.json")));
+
     systemd = {
       packages = [ pkgs.opensnitch ];
       services.opensnitchd.wantedBy = [ "multi-user.target" ];
     };
 
+    environment.etc."opensnitchd/default-config.json".source = format.generate "default-config.json" cfg.settings;
+
   };
 }
 
diff --git a/nixos/modules/services/security/privacyidea.nix b/nixos/modules/services/security/privacyidea.nix
index 05f4995cc416..b8e2d9a8b0df 100644
--- a/nixos/modules/services/security/privacyidea.nix
+++ b/nixos/modules/services/security/privacyidea.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.privacyidea;
+  opt = options.services.privacyidea;
 
   uwsgi = pkgs.uwsgi.override { plugins = [ "python3" ]; };
   python = uwsgi.python3;
@@ -112,6 +113,7 @@ in
       encFile = mkOption {
         type = types.str;
         default = "${cfg.stateDir}/enckey";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/enckey"'';
         description = ''
           This is used to encrypt the token data and token passwords
         '';
@@ -120,6 +122,7 @@ in
       auditKeyPrivate = mkOption {
         type = types.str;
         default = "${cfg.stateDir}/private.pem";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/private.pem"'';
         description = ''
           Private Key for signing the audit log.
         '';
@@ -128,6 +131,7 @@ in
       auditKeyPublic = mkOption {
         type = types.str;
         default = "${cfg.stateDir}/public.pem";
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/public.pem"'';
         description = ''
           Public key for checking signatures of the audit log.
         '';
@@ -200,6 +204,7 @@ in
       systemd.services.privacyidea = let
         piuwsgi = pkgs.writeText "uwsgi.json" (builtins.toJSON {
           uwsgi = {
+            buffer-size = 8192;
             plugins = [ "python3" ];
             pythonpath = "${penv}/${uwsgi.python3.sitePackages}";
             socket = "/run/privacyidea/socket";
diff --git a/nixos/modules/services/security/step-ca.nix b/nixos/modules/services/security/step-ca.nix
index 27b2ceed1a43..95183078d7b6 100644
--- a/nixos/modules/services/security/step-ca.nix
+++ b/nixos/modules/services/security/step-ca.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, nixosTests, ... }:
+{ config, lib, pkgs, ... }:
 let
   cfg = config.services.step-ca;
   settingsFormat = (pkgs.formats.json { });
@@ -82,8 +82,6 @@ in
       });
     in
     {
-      passthru.tests.step-ca = nixosTests.step-ca;
-
       assertions =
         [
           {
@@ -108,6 +106,9 @@ in
           ConditionFileNotEmpty = ""; # override upstream
         };
         serviceConfig = {
+          User = "step-ca";
+          Group = "step-ca";
+          UMask = "0077";
           Environment = "HOME=%S/step-ca";
           WorkingDirectory = ""; # override upstream
           ReadWriteDirectories = ""; # override upstream
@@ -129,6 +130,14 @@ in
         };
       };
 
+      users.users.step-ca = {
+        home = "/var/lib/step-ca";
+        group = "step-ca";
+        isSystemUser = true;
+      };
+
+      users.groups.step-ca = {};
+
       networking.firewall = lib.mkIf cfg.openFirewall {
         allowedTCPPorts = [ cfg.port ];
       };
diff --git a/nixos/modules/services/security/tor.nix b/nixos/modules/services/security/tor.nix
index c3e3248ee8ab..a5822c02794d 100644
--- a/nixos/modules/services/security/tor.nix
+++ b/nixos/modules/services/security/tor.nix
@@ -1,10 +1,11 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with builtins;
 with lib;
 
 let
   cfg = config.services.tor;
+  opt = options.services.tor;
   stateDir = "/var/lib/tor";
   runDir = "/run/tor";
   descriptionGeneric = option: ''
@@ -793,12 +794,22 @@ in
               };
             }));
           };
+          options.ShutdownWaitLength = mkOption {
+            type = types.int;
+            default = 30;
+            description = descriptionGeneric "ShutdownWaitLength";
+          };
           options.SocksPolicy = optionStrings "SocksPolicy" // {
             example = ["accept *:*"];
           };
           options.SOCKSPort = mkOption {
             description = descriptionGeneric "SOCKSPort";
             default = if cfg.settings.HiddenServiceNonAnonymousMode == true then [{port = 0;}] else [];
+            defaultText = literalExpression ''
+              if config.${opt.settings}.HiddenServiceNonAnonymousMode == true
+              then [ { port = 0; } ]
+              else [ ]
+            '';
             example = [{port = 9090;}];
             type = types.listOf (optionSOCKSPort true);
           };
@@ -899,6 +910,11 @@ in
         ORPort = mkForce [];
         PublishServerDescriptor = mkForce false;
       })
+      (mkIf (!cfg.client.enable) {
+        # Make sure application connections via SOCKS are disabled
+        # when services.tor.client.enable is false
+        SOCKSPort = mkForce [ 0 ];
+      })
       (mkIf cfg.client.enable (
         { SOCKSPort = [ cfg.client.socksListenAddress ];
         } // optionalAttrs cfg.client.transparentProxy.enable {
@@ -951,7 +967,7 @@ in
               '') onion.authorizedClients ++
               optional (onion.secretKey != null) ''
                 install -d -o tor -g tor -m 0700 ${escapeShellArg onion.path}
-                key="$(cut -f1 -d: ${escapeShellArg onion.secretKey})"
+                key="$(cut -f1 -d: ${escapeShellArg onion.secretKey} | head -1)"
                 case "$key" in
                  ("== ed25519v"*"-secret")
                   install -o tor -g tor -m 0400 ${escapeShellArg onion.secretKey} ${escapeShellArg onion.path}/hs_ed25519_secret_key;;
@@ -971,7 +987,7 @@ in
         ExecStart = "${cfg.package}/bin/tor -f ${torrc}";
         ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
         KillSignal = "SIGINT";
-        TimeoutSec = 30;
+        TimeoutSec = cfg.settings.ShutdownWaitLength + 30; # Wait a bit longer than ShutdownWaitLength before actually timing out
         Restart = "on-failure";
         LimitNOFILE = 32768;
         RuntimeDirectory = [
@@ -997,7 +1013,11 @@ in
         #InaccessiblePaths = [ "-+${runDir}/root" ];
         UMask = "0066";
         BindPaths = [ stateDir ];
-        BindReadOnlyPaths = [ storeDir "/etc" ];
+        BindReadOnlyPaths = [ storeDir "/etc" ] ++
+          optionals config.services.resolved.enable [
+            "/run/systemd/resolve/stub-resolv.conf"
+            "/run/systemd/resolve/resolv.conf"
+          ];
         AmbientCapabilities   = [""] ++ lib.optional bindsPrivilegedPort "CAP_NET_BIND_SERVICE";
         CapabilityBoundingSet = [""] ++ lib.optional bindsPrivilegedPort "CAP_NET_BIND_SERVICE";
         # ProtectClock= adds DeviceAllow=char-rtc r
diff --git a/nixos/modules/services/security/vault.nix b/nixos/modules/services/security/vault.nix
index b0ade62d97c9..d48bc472cb82 100644
--- a/nixos/modules/services/security/vault.nix
+++ b/nixos/modules/services/security/vault.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.vault;
+  opt = options.services.vault;
 
   configFile = pkgs.writeText "vault.hcl" ''
     listener "tcp" {
@@ -83,6 +84,11 @@ in
       storagePath = mkOption {
         type = types.nullOr types.path;
         default = if cfg.storageBackend == "file" then "/var/lib/vault" else null;
+        defaultText = literalExpression ''
+          if config.${opt.storageBackend} == "file"
+          then "/var/lib/vault"
+          else null
+        '';
         description = "Data directory for file backend";
       };
 
diff --git a/nixos/modules/services/security/vaultwarden/default.nix b/nixos/modules/services/security/vaultwarden/default.nix
index 5b951bc85ec0..8277f493639c 100644
--- a/nixos/modules/services/security/vaultwarden/default.nix
+++ b/nixos/modules/services/security/vaultwarden/default.nix
@@ -131,7 +131,7 @@ in {
     users.groups.vaultwarden = { };
 
     systemd.services.vaultwarden = {
-      aliases = [ "bitwarden_rs" ];
+      aliases = [ "bitwarden_rs.service" ];
       after = [ "network.target" ];
       path = with pkgs; [ openssl ];
       serviceConfig = {
@@ -151,7 +151,7 @@ in {
     };
 
     systemd.services.backup-vaultwarden = mkIf (cfg.backupDir != null) {
-      aliases = [ "backup-bitwarden_rs" ];
+      aliases = [ "backup-bitwarden_rs.service" ];
       description = "Backup vaultwarden";
       environment = {
         DATA_FOLDER = "/var/lib/bitwarden_rs";
@@ -169,7 +169,7 @@ in {
     };
 
     systemd.timers.backup-vaultwarden = mkIf (cfg.backupDir != null) {
-      aliases = [ "backup-bitwarden_rs" ];
+      aliases = [ "backup-bitwarden_rs.service" ];
       description = "Backup vaultwarden on time";
       timerConfig = {
         OnCalendar = mkDefault "23:00";
@@ -179,4 +179,7 @@ in {
       wantedBy = [ "multi-user.target" ];
     };
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/system/cachix-agent/default.nix b/nixos/modules/services/system/cachix-agent/default.nix
new file mode 100644
index 000000000000..496e0b90355b
--- /dev/null
+++ b/nixos/modules/services/system/cachix-agent/default.nix
@@ -0,0 +1,57 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.cachix-agent;
+in {
+  meta.maintainers = [ lib.maintainers.domenkozar ];
+
+  options.services.cachix-agent = {
+    enable = mkEnableOption "Cachix Deploy Agent: https://docs.cachix.org/deploy/";
+
+    name = mkOption {
+      type = types.str;
+      description = "Agent name, usually same as the hostname";
+      default = config.networking.hostName;
+      defaultText = "config.networking.hostName";
+    };
+
+    profile = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = "Profile name, defaults to 'system' (NixOS).";
+    };
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.cachix;
+      defaultText = literalExpression "pkgs.cachix";
+      description = "Cachix Client package to use.";
+    };
+
+    credentialsFile = mkOption {
+      type = types.path;
+      default = "/etc/cachix-agent.token";
+      description = ''
+        Required file that needs to contain CACHIX_AGENT_TOKEN=...
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.cachix-agent = {
+      description = "Cachix Deploy Agent";
+      after = ["network-online.target"];
+      path = [ config.nix.package ];
+      wantedBy = [ "multi-user.target" ];
+      # don't restart while changing
+      reloadIfChanged = true;
+      serviceConfig = {
+        Restart = "on-failure";
+        EnvironmentFile = cfg.credentialsFile;
+        ExecStart = "${cfg.package}/bin/cachix deploy agent ${cfg.name} ${if cfg.profile != null then profile else ""}";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/system/cloud-init.nix b/nixos/modules/services/system/cloud-init.nix
index eb82b738e492..8c6a6e294ebb 100644
--- a/nixos/modules/services/system/cloud-init.nix
+++ b/nixos/modules/services/system/cloud-init.nix
@@ -52,11 +52,22 @@ in
         '';
       };
 
+      network.enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Allow the cloud-init service to configure network interfaces
+          through systemd-networkd.
+        '';
+      };
+
       config = mkOption {
         type = types.str;
         default = ''
           system_info:
             distro: nixos
+            network:
+              renderers: [ 'networkd' ]
           users:
              - root
 
@@ -109,9 +120,12 @@ in
 
     environment.etc."cloud/cloud.cfg".text = cfg.config;
 
+    systemd.network.enable = cfg.network.enable;
+
     systemd.services.cloud-init-local =
       { description = "Initial cloud-init job (pre-networking)";
         wantedBy = [ "multi-user.target" ];
+        before = ["systemd-networkd.service"];
         path = path;
         serviceConfig =
           { Type = "oneshot";
@@ -129,7 +143,7 @@ in
                   "sshd.service" "sshd-keygen.service" ];
         after = [ "network-online.target" "cloud-init-local.service" ];
         before = [ "sshd.service" "sshd-keygen.service" ];
-        requires = [ "network.target "];
+        requires = [ "network.target"];
         path = path;
         serviceConfig =
           { Type = "oneshot";
diff --git a/nixos/modules/services/system/earlyoom.nix b/nixos/modules/services/system/earlyoom.nix
index 452efc736439..ddd5bcebcdd5 100644
--- a/nixos/modules/services/system/earlyoom.nix
+++ b/nixos/modules/services/system/earlyoom.nix
@@ -1,124 +1,104 @@
 { config, lib, pkgs, ... }:
 
-with lib;
-
 let
-  ecfg = config.services.earlyoom;
+  cfg = config.services.earlyoom;
+
+  inherit (lib)
+    mkDefault mkEnableOption mkIf mkOption types
+    mkRemovedOptionModule
+    concatStringsSep optional;
+
 in
 {
-  options = {
-    services.earlyoom = {
-
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enable early out of memory killing.
-        '';
-      };
+  options.services.earlyoom = {
+    enable = mkEnableOption "Early out of memory killing";
 
-      freeMemThreshold = mkOption {
-        type = types.int;
-        default = 10;
-        description = ''
-          Minimum of availabe memory (in percent).
-          If the free memory falls below this threshold and the analog is true for
-          <option>services.earlyoom.freeSwapThreshold</option>
-          the killing begins.
-        '';
-      };
+    freeMemThreshold = mkOption {
+      type = types.ints.between 1 100;
+      default = 10;
+      description = ''
+        Minimum of availabe memory (in percent).
+        If the free memory falls below this threshold and the analog is true for
+        <option>services.earlyoom.freeSwapThreshold</option>
+        the killing begins.
+      '';
+    };
 
-      freeSwapThreshold = mkOption {
-        type = types.int;
-        default = 10;
-        description = ''
-          Minimum of availabe swap space (in percent).
-          If the available swap space falls below this threshold and the analog
-          is true for <option>services.earlyoom.freeMemThreshold</option>
-          the killing begins.
-        '';
-      };
+    freeSwapThreshold = mkOption {
+      type = types.ints.between 1 100;
+      default = 10;
+      description = ''
+        Minimum of availabe swap space (in percent).
+        If the available swap space falls below this threshold and the analog
+        is true for <option>services.earlyoom.freeMemThreshold</option>
+        the killing begins.
+      '';
+    };
 
-      useKernelOOMKiller= mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Use kernel OOM killer instead of own user-space implementation.
-        '';
-      };
+    # TODO: remove or warn after 1.7 (https://github.com/rfjakob/earlyoom/commit/7ebc4554)
+    ignoreOOMScoreAdjust = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Ignore oom_score_adjust values of processes.
+      '';
+    };
 
-      ignoreOOMScoreAdjust = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Ignore oom_score_adjust values of processes.
-          User-space implementation only.
-        '';
-      };
+    enableDebugInfo = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enable debugging messages.
+      '';
+    };
 
-      enableDebugInfo = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enable debugging messages.
-        '';
-      };
+    enableNotifications = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Send notifications about killed processes via the system d-bus.
 
-      notificationsCommand = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          This option is deprecated and ignored by earlyoom since 1.6.
-          Use <option>services.earlyoom.enableNotifications</option> instead.
-        '';
-      };
+        WARNING: enabling this option (while convenient) should *not* be done on a
+        machine where you do not trust the other users as it allows any other
+        local user to DoS your session by spamming notifications.
 
-      enableNotifications = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Send notifications about killed processes via the system d-bus.
-          To actually see the notifications in your GUI session, you need to have
-          <literal>systembus-notify</literal> running as your user.
+        To actually see the notifications in your GUI session, you need to have
+        <literal>systembus-notify</literal> running as your user which this
+        option handles.
 
-          See <link xlink:href="https://github.com/rfjakob/earlyoom#notifications">README</link> for details.
-        '';
-      };
+        See <link xlink:href="https://github.com/rfjakob/earlyoom#notifications">README</link> for details.
+      '';
     };
   };
 
-  config = mkIf ecfg.enable {
-    assertions = [
-      { assertion = ecfg.freeMemThreshold > 0 && ecfg.freeMemThreshold <= 100;
-        message = "Needs to be a positive percentage"; }
-      { assertion = ecfg.freeSwapThreshold > 0 && ecfg.freeSwapThreshold <= 100;
-        message = "Needs to be a positive percentage"; }
-      { assertion = !ecfg.useKernelOOMKiller || !ecfg.ignoreOOMScoreAdjust;
-        message = "Both options in conjunction do not make sense"; }
-    ];
+  imports = [
+    (mkRemovedOptionModule [ "services" "earlyoom" "useKernelOOMKiller" ] ''
+      This option is deprecated and ignored by earlyoom since 1.2.
+    '')
+    (mkRemovedOptionModule [ "services" "earlyoom" "notificationsCommand" ] ''
+      This option is deprecated and ignored by earlyoom since 1.6.
+    '')
+  ];
 
-    warnings = optional (ecfg.notificationsCommand != null)
-      "`services.earlyoom.notificationsCommand` is deprecated and ignored by earlyoom since 1.6.";
+  config = mkIf cfg.enable {
+    services.systembus-notify.enable = mkDefault cfg.enableNotifications;
 
     systemd.services.earlyoom = {
       description = "Early OOM Daemon for Linux";
       wantedBy = [ "multi-user.target" ];
-      path = optional ecfg.enableNotifications pkgs.dbus;
+      path = optional cfg.enableNotifications pkgs.dbus;
       serviceConfig = {
-        StandardOutput = "null";
         StandardError = "journal";
-        ExecStart = ''
-          ${pkgs.earlyoom}/bin/earlyoom \
-          -m ${toString ecfg.freeMemThreshold} \
-          -s ${toString ecfg.freeSwapThreshold} \
-          ${optionalString ecfg.useKernelOOMKiller "-k"} \
-          ${optionalString ecfg.ignoreOOMScoreAdjust "-i"} \
-          ${optionalString ecfg.enableDebugInfo "-d"} \
-          ${optionalString ecfg.enableNotifications "-n"}
-        '';
+        ExecStart = concatStringsSep " " ([
+          "${pkgs.earlyoom}/bin/earlyoom"
+          "-m ${toString cfg.freeMemThreshold}"
+          "-s ${toString cfg.freeSwapThreshold}"
+        ]
+        ++ optional cfg.ignoreOOMScoreAdjust "-i"
+        ++ optional cfg.enableDebugInfo "-d"
+        ++ optional cfg.enableNotifications "-n"
+        );
       };
     };
-
-    environment.systemPackages = optional ecfg.enableNotifications pkgs.systembus-notify;
   };
 }
diff --git a/nixos/modules/services/system/nscd.nix b/nixos/modules/services/system/nscd.nix
index d720f254b813..00a87e788dc4 100644
--- a/nixos/modules/services/system/nscd.nix
+++ b/nixos/modules/services/system/nscd.nix
@@ -50,7 +50,9 @@ in
     systemd.services.nscd =
       { description = "Name Service Cache Daemon";
 
-        wantedBy = [ "nss-lookup.target" "nss-user-lookup.target" ];
+        before = [ "nss-lookup.target" "nss-user-lookup.target" ];
+        wants = [ "nss-lookup.target" "nss-user-lookup.target" ];
+        wantedBy = [ "multi-user.target" ];
 
         environment = { LD_LIBRARY_PATH = nssModulesPath; };
 
diff --git a/nixos/modules/services/system/self-deploy.nix b/nixos/modules/services/system/self-deploy.nix
index 33d15e08f4aa..d7130a13c731 100644
--- a/nixos/modules/services/system/self-deploy.nix
+++ b/nixos/modules/services/system/self-deploy.nix
@@ -126,6 +126,8 @@ in
 
   config = lib.mkIf cfg.enable {
     systemd.services.self-deploy = {
+      inherit (cfg) startAt;
+
       wantedBy = [ "multi-user.target" ];
 
       requires = lib.mkIf (!(isPathType cfg.repository)) [ "network-online.target" ];
@@ -138,8 +140,7 @@ in
       path = with pkgs; [
         git
         nix
-        systemd
-      ];
+      ] ++ lib.optionals (cfg.switchCommand == "boot") [ systemd ];
 
       script = ''
         if [ ! -e ${repositoryDirectory} ]; then
diff --git a/nixos/modules/services/system/systembus-notify.nix b/nixos/modules/services/system/systembus-notify.nix
new file mode 100644
index 000000000000..e918bc552ece
--- /dev/null
+++ b/nixos/modules/services/system/systembus-notify.nix
@@ -0,0 +1,27 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.systembus-notify;
+
+  inherit (lib) mkEnableOption mkIf;
+
+in
+{
+  options.services.systembus-notify = {
+    enable = mkEnableOption ''
+      System bus notification support
+
+      WARNING: enabling this option (while convenient) should *not* be done on a
+      machine where you do not trust the other users as it allows any other
+      local user to DoS your session by spamming notifications.
+    '';
+  };
+
+  config = mkIf cfg.enable {
+    systemd = {
+      packages = with pkgs; [ systembus-notify ];
+
+      user.services.systembus-notify.wantedBy = [ "graphical-session.target" ];
+    };
+  };
+}
diff --git a/nixos/modules/services/torrent/peerflix.nix b/nixos/modules/services/torrent/peerflix.nix
index 3e5f80960dc7..821c829f6b4a 100644
--- a/nixos/modules/services/torrent/peerflix.nix
+++ b/nixos/modules/services/torrent/peerflix.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.peerflix;
+  opt = options.services.peerflix;
 
   configFile = pkgs.writeText "peerflix-config.json" ''
     {
@@ -32,6 +33,7 @@ in {
     downloadDir = mkOption {
       description = "Peerflix temporary download directory.";
       default = "${cfg.stateDir}/torrents";
+      defaultText = literalExpression ''"''${config.${opt.stateDir}}/torrents"'';
       type = types.path;
     };
   };
diff --git a/nixos/modules/services/torrent/rtorrent.nix b/nixos/modules/services/torrent/rtorrent.nix
index dd7df623c739..759dcfe2e6c5 100644
--- a/nixos/modules/services/torrent/rtorrent.nix
+++ b/nixos/modules/services/torrent/rtorrent.nix
@@ -1,10 +1,11 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 
 with lib;
 
 let
 
   cfg = config.services.rtorrent;
+  opt = options.services.rtorrent;
 
 in {
   options.services.rtorrent = {
@@ -21,6 +22,7 @@ in {
     downloadDir = mkOption {
       type = types.str;
       default = "${cfg.dataDir}/download";
+      defaultText = literalExpression ''"''${config.${opt.dataDir}}/download"'';
       description = ''
         Where to put downloaded files.
       '';
diff --git a/nixos/modules/services/torrent/transmission.nix b/nixos/modules/services/torrent/transmission.nix
index 9e440e49b504..d12d8aa23980 100644
--- a/nixos/modules/services/torrent/transmission.nix
+++ b/nixos/modules/services/torrent/transmission.nix
@@ -4,6 +4,7 @@ with lib;
 
 let
   cfg = config.services.transmission;
+  opt = options.services.transmission;
   inherit (config.environment) etc;
   apparmor = config.security.apparmor;
   rootDir = "/run/transmission";
@@ -47,11 +48,13 @@ in
           options.download-dir = mkOption {
             type = types.path;
             default = "${cfg.home}/${downloadsDir}";
+            defaultText = literalExpression ''"''${config.${opt.home}}/${downloadsDir}"'';
             description = "Directory where to download torrents.";
           };
           options.incomplete-dir = mkOption {
             type = types.path;
             default = "${cfg.home}/${incompleteDir}";
+            defaultText = literalExpression ''"''${config.${opt.home}}/${incompleteDir}"'';
             description = ''
               When enabled with
               services.transmission.home
@@ -147,6 +150,7 @@ in
           options.watch-dir = mkOption {
             type = types.path;
             default = "${cfg.home}/${watchDir}";
+            defaultText = literalExpression ''"''${config.${opt.home}}/${watchDir}"'';
             description = "Watch a directory for torrent files and add them to transmission.";
           };
           options.watch-dir-enabled = mkOption {
diff --git a/nixos/modules/services/video/epgstation/default.nix b/nixos/modules/services/video/epgstation/default.nix
index 56bd9d9eeeca..191f6eb52e57 100644
--- a/nixos/modules/services/video/epgstation/default.nix
+++ b/nixos/modules/services/video/epgstation/default.nix
@@ -1,29 +1,40 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
+{ config, lib, options, pkgs, ... }:
 
 let
   cfg = config.services.epgstation;
+  opt = options.services.epgstation;
+
+  description = "EPGStation: DVR system for Mirakurun-managed TV tuners";
 
   username = config.users.users.epgstation.name;
   groupname = config.users.users.epgstation.group;
+  mirakurun = {
+    sock = config.services.mirakurun.unixSocket;
+    option = options.services.mirakurun.unixSocket;
+  };
 
-  settingsFmt = pkgs.formats.json {};
-  settingsTemplate = settingsFmt.generate "config.json" cfg.settings;
+  yaml = pkgs.formats.yaml { };
+  settingsTemplate = yaml.generate "config.yml" cfg.settings;
   preStartScript = pkgs.writeScript "epgstation-prestart" ''
     #!${pkgs.runtimeShell}
 
-    PASSWORD="$(head -n1 "${cfg.basicAuth.passwordFile}")"
-    DB_PASSWORD="$(head -n1 "${cfg.database.passwordFile}")"
+    DB_PASSWORD_FILE=${lib.escapeShellArg cfg.database.passwordFile}
+
+    if [[ ! -f "$DB_PASSWORD_FILE" ]]; then
+      printf "[FATAL] File containing the DB password was not found in '%s'. Double check the NixOS option '%s'." \
+        "$DB_PASSWORD_FILE" ${lib.escapeShellArg opt.database.passwordFile} >&2
+      exit 1
+    fi
+
+    DB_PASSWORD="$(head -n1 ${lib.escapeShellArg cfg.database.passwordFile})"
 
     # setup configuration
-    touch /etc/epgstation/config.json
-    chmod 640 /etc/epgstation/config.json
+    touch /etc/epgstation/config.yml
+    chmod 640 /etc/epgstation/config.yml
     sed \
-      -e "s,@password@,$PASSWORD,g" \
       -e "s,@dbPassword@,$DB_PASSWORD,g" \
-      ${settingsTemplate} > /etc/epgstation/config.json
-    chown "${username}:${groupname}" /etc/epgstation/config.json
+      ${settingsTemplate} > /etc/epgstation/config.yml
+    chown "${username}:${groupname}" /etc/epgstation/config.yml
 
     # NOTE: Use password authentication, since mysqljs does not yet support auth_socket
     if [ ! -e /var/lib/epgstation/db-created ]; then
@@ -34,7 +45,7 @@ let
   '';
 
   streamingConfig = lib.importJSON ./streaming.json;
-  logConfig = {
+  logConfig = yaml.generate "logConfig.yml" {
     appenders.stdout.type = "stdout";
     categories = {
       default = { appenders = [ "stdout" ]; level = "info"; };
@@ -44,51 +55,51 @@ let
     };
   };
 
-  defaultPassword = "INSECURE_GO_CHECK_CONFIGURATION_NIX\n";
+  # Deprecate top level options that are redundant.
+  deprecateTopLevelOption = config:
+    lib.mkRenamedOptionModule
+      ([ "services" "epgstation" ] ++ config)
+      ([ "services" "epgstation" "settings" ] ++ config);
+
+  removeOption = config: instruction:
+    lib.mkRemovedOptionModule
+      ([ "services" "epgstation" ] ++ config)
+      instruction;
 in
 {
-  options.services.epgstation = {
-    enable = mkEnableOption "EPGStation: DTV Software in Japan";
+  meta.maintainers = with lib.maintainers; [ midchildan ];
 
-    usePreconfiguredStreaming = mkOption {
-      type = types.bool;
-      default = true;
-      description = ''
-        Use preconfigured default streaming options.
+  imports = [
+    (deprecateTopLevelOption [ "port" ])
+    (deprecateTopLevelOption [ "socketioPort" ])
+    (deprecateTopLevelOption [ "clientSocketioPort" ])
+    (removeOption [ "basicAuth" ]
+      "Use a TLS-terminated reverse proxy with authentication instead.")
+  ];
 
-        Upstream defaults:
-        <link xlink:href="https://github.com/l3tnun/EPGStation/blob/master/config/config.sample.json"/>
-      '';
-    };
+  options.services.epgstation = {
+    enable = lib.mkEnableOption description;
 
-    port = mkOption {
-      type = types.port;
-      default = 20772;
-      description = ''
-        HTTP port for EPGStation to listen on.
-      '';
+    package = lib.mkOption {
+      default = pkgs.epgstation;
+      type = lib.types.package;
+      defaultText = lib.literalExpression "pkgs.epgstation";
+      description = "epgstation package to use";
     };
 
-    socketioPort = mkOption {
-      type = types.port;
-      default = cfg.port + 1;
+    usePreconfiguredStreaming = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
       description = ''
-        Socket.io port for EPGStation to listen on.
-      '';
-    };
+        Use preconfigured default streaming options.
 
-    clientSocketioPort = mkOption {
-      type = types.port;
-      default = cfg.socketioPort;
-      description = ''
-        Socket.io port that the web client is going to connect to. This may be
-        different from <option>socketioPort</option> if EPGStation is hidden
-        behind a reverse proxy.
+        Upstream defaults:
+        <link xlink:href="https://github.com/l3tnun/EPGStation/blob/master/config/config.yml.template"/>
       '';
     };
 
-    openFirewall = mkOption {
-      type = types.bool;
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
       default = false;
       description = ''
         Open ports in the firewall for the EPGStation web interface.
@@ -103,50 +114,17 @@ in
       '';
     };
 
-    basicAuth = {
-      user = mkOption {
-        type = with types; nullOr str;
-        default = null;
-        example = "epgstation";
-        description = ''
-          Basic auth username for EPGStation. If <literal>null</literal>, basic
-          auth will be disabled.
-
-          <warning>
-            <para>
-              Basic authentication has known weaknesses, the most critical being
-              that it sends passwords over the network in clear text. Use this
-              feature to control access to EPGStation within your family and
-              friends, but don't rely on it for security.
-            </para>
-          </warning>
-        '';
-      };
-
-      passwordFile = mkOption {
-        type = types.path;
-        default = pkgs.writeText "epgstation-password" defaultPassword;
-        defaultText = literalDocBook ''a file containing <literal>${defaultPassword}</literal>'';
-        example = "/run/keys/epgstation-password";
-        description = ''
-          A file containing the password for <option>basicAuth.user</option>.
-        '';
-      };
-    };
-
-    database =  {
-      name = mkOption {
-        type = types.str;
+    database = {
+      name = lib.mkOption {
+        type = lib.types.str;
         default = "epgstation";
         description = ''
           Name of the MySQL database that holds EPGStation's data.
         '';
       };
 
-      passwordFile = mkOption {
-        type = types.path;
-        default = pkgs.writeText "epgstation-db-password" defaultPassword;
-        defaultText = literalDocBook ''a file containing <literal>${defaultPassword}</literal>'';
+      passwordFile = lib.mkOption {
+        type = lib.types.path;
         example = "/run/keys/epgstation-db-password";
         description = ''
           A file containing the password for the database named
@@ -155,66 +133,106 @@ in
       };
     };
 
-    settings = mkOption {
+    # The defaults for some options come from the upstream template
+    # configuration, which is the one that users would get if they follow the
+    # upstream instructions. This is, in some cases, different from the
+    # application defaults. Some options like encodeProcessNum and
+    # concurrentEncodeNum doesn't have an optimal default value that works for
+    # all hardware setups and/or performance requirements. For those kind of
+    # options, the application default wouldn't always result in the expected
+    # out-of-the-box behavior because it's the responsibility of the user to
+    # configure them according to their needs. In these cases, the value in the
+    # upstream template configuration should serve as a "good enough" default.
+    settings = lib.mkOption {
       description = ''
-        Options to add to config.json.
+        Options to add to config.yml.
 
         Documentation:
         <link xlink:href="https://github.com/l3tnun/EPGStation/blob/master/doc/conf-manual.md"/>
       '';
 
-      default = {};
+      default = { };
       example = {
         recPriority = 20;
         conflictPriority = 10;
       };
 
-      type = types.submodule {
-        freeformType = settingsFmt.type;
+      type = lib.types.submodule {
+        freeformType = yaml.type;
+
+        options.port = lib.mkOption {
+          type = lib.types.port;
+          default = 20772;
+          description = ''
+            HTTP port for EPGStation to listen on.
+          '';
+        };
+
+        options.socketioPort = lib.mkOption {
+          type = lib.types.port;
+          default = cfg.settings.port + 1;
+          defaultText = lib.literalExpression "config.${opt.settings}.port + 1";
+          description = ''
+            Socket.io port for EPGStation to listen on. It is valid to share
+            ports with <option>${opt.settings}.port</option>.
+          '';
+        };
 
-        options.readOnlyOnce = mkOption {
-          type = types.bool;
-          default = false;
-          description = "Don't reload configuration files at runtime.";
+        options.clientSocketioPort = lib.mkOption {
+          type = lib.types.port;
+          default = cfg.settings.socketioPort;
+          defaultText = lib.literalExpression "config.${opt.settings}.socketioPort";
+          description = ''
+            Socket.io port that the web client is going to connect to. This may
+            be different from <option>${opt.settings}.socketioPort</option> if
+            EPGStation is hidden behind a reverse proxy.
+          '';
         };
 
-        options.mirakurunPath = mkOption (let
-          sockPath = config.services.mirakurun.unixSocket;
-        in {
-          type = types.str;
-          default = "http+unix://${replaceStrings ["/"] ["%2F"] sockPath}";
+        options.mirakurunPath = with mirakurun; lib.mkOption {
+          type = lib.types.str;
+          default = "http+unix://${lib.replaceStrings ["/"] ["%2F"] sock}";
+          defaultText = lib.literalExpression ''
+            "http+unix://''${lib.replaceStrings ["/"] ["%2F"] config.${option}}"
+          '';
           example = "http://localhost:40772";
           description = "URL to connect to Mirakurun.";
-        });
+        };
+
+        options.encodeProcessNum = lib.mkOption {
+          type = lib.types.ints.positive;
+          default = 4;
+          description = ''
+            The maximum number of processes that EPGStation would allow to run
+            at the same time for encoding or streaming videos.
+          '';
+        };
+
+        options.concurrentEncodeNum = lib.mkOption {
+          type = lib.types.ints.positive;
+          default = 1;
+          description = ''
+            The maximum number of encoding jobs that EPGStation would run at the
+            same time.
+          '';
+        };
 
-        options.encode = mkOption {
-          type = with types; listOf attrs;
+        options.encode = lib.mkOption {
+          type = with lib.types; listOf attrs;
           description = "Encoding presets for recorded videos.";
           default = [
             {
-              name = "H264";
-              cmd = "${pkgs.epgstation}/libexec/enc.sh main";
+              name = "H.264";
+              cmd = "%NODE% ${cfg.package}/libexec/enc.js";
               suffix = ".mp4";
-              default = true;
-            }
-            {
-              name = "H264-sub";
-              cmd = "${pkgs.epgstation}/libexec/enc.sh sub";
-              suffix = "-sub.mp4";
             }
           ];
-          defaultText = literalExpression ''
+          defaultText = lib.literalExpression ''
             [
               {
-                name = "H264";
-                cmd = "''${pkgs.epgstation}/libexec/enc.sh main";
+                name = "H.264";
+                cmd = "%NODE% config.${opt.package}/libexec/enc.js";
                 suffix = ".mp4";
-                default = true;
-              }
-              {
-                name = "H264-sub";
-                cmd = "''${pkgs.epgstation}/libexec/enc.sh sub";
-                suffix = "-sub.mp4";
               }
             ]
           '';
@@ -223,14 +241,25 @@ in
     };
   };
 
-  config = mkIf cfg.enable {
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !(lib.hasAttr "readOnlyOnce" cfg.settings);
+        message = ''
+          The option config.${opt.settings}.readOnlyOnce can no longer be used
+          since it's been removed. No replacements are available.
+        '';
+      }
+    ];
+
     environment.etc = {
-      "epgstation/operatorLogConfig.json".text = builtins.toJSON logConfig;
-      "epgstation/serviceLogConfig.json".text = builtins.toJSON logConfig;
+      "epgstation/epgUpdaterLogConfig.yml".source = logConfig;
+      "epgstation/operatorLogConfig.yml".source = logConfig;
+      "epgstation/serviceLogConfig.yml".source = logConfig;
     };
 
-    networking.firewall = mkIf cfg.openFirewall {
-      allowedTCPPorts = with cfg; [ port socketioPort ];
+    networking.firewall = lib.mkIf cfg.openFirewall {
+      allowedTCPPorts = with cfg.settings; [ port socketioPort ];
     };
 
     users.users.epgstation = {
@@ -239,13 +268,13 @@ in
       isSystemUser = true;
     };
 
-    users.groups.epgstation = {};
+    users.groups.epgstation = { };
 
-    services.mirakurun.enable = mkDefault true;
+    services.mirakurun.enable = lib.mkDefault true;
 
     services.mysql = {
-      enable = mkDefault true;
-      package = mkDefault pkgs.mariadb;
+      enable = lib.mkDefault true;
+      package = lib.mkDefault pkgs.mariadb;
       ensureDatabases = [ cfg.database.name ];
       # FIXME: enable once mysqljs supports auth_socket
       # ensureUsers = [ {
@@ -254,39 +283,28 @@ in
       # } ];
     };
 
-    services.epgstation.settings = let
-      defaultSettings = {
-        serverPort = cfg.port;
-        socketioPort = cfg.socketioPort;
-        clientSocketioPort = cfg.clientSocketioPort;
-
-        dbType = mkDefault "mysql";
-        mysql = {
-          user = username;
-          database = cfg.database.name;
-          socketPath = mkDefault "/run/mysqld/mysqld.sock";
-          password = mkDefault "@dbPassword@";
-          connectTimeout = mkDefault 1000;
-          connectionLimit = mkDefault 10;
+    services.epgstation.settings =
+      let
+        defaultSettings = {
+          dbtype = lib.mkDefault "mysql";
+          mysql = {
+            socketPath = lib.mkDefault "/run/mysqld/mysqld.sock";
+            user = username;
+            password = lib.mkDefault "@dbPassword@";
+            database = cfg.database.name;
+          };
+
+          ffmpeg = lib.mkDefault "${pkgs.ffmpeg-full}/bin/ffmpeg";
+          ffprobe = lib.mkDefault "${pkgs.ffmpeg-full}/bin/ffprobe";
+
+          # for disambiguation with TypeScript files
+          recordedFileExtension = lib.mkDefault ".m2ts";
         };
-
-        basicAuth = mkIf (cfg.basicAuth.user != null) {
-          user = mkDefault cfg.basicAuth.user;
-          password = mkDefault "@password@";
-        };
-
-        ffmpeg = mkDefault "${pkgs.ffmpeg-full}/bin/ffmpeg";
-        ffprobe = mkDefault "${pkgs.ffmpeg-full}/bin/ffprobe";
-
-        fileExtension = mkDefault ".m2ts";
-        maxEncode = mkDefault 2;
-        maxStreaming = mkDefault 2;
-      };
-    in
-    mkMerge [
-      defaultSettings
-      (mkIf cfg.usePreconfiguredStreaming streamingConfig)
-    ];
+      in
+      lib.mkMerge [
+        defaultSettings
+        (lib.mkIf cfg.usePreconfiguredStreaming streamingConfig)
+      ];
 
     systemd.tmpfiles.rules = [
       "d '/var/lib/epgstation/streamfiles' - ${username} ${groupname} - -"
@@ -295,15 +313,15 @@ in
     ];
 
     systemd.services.epgstation = {
-      description = pkgs.epgstation.meta.description;
+      inherit description;
+
       wantedBy = [ "multi-user.target" ];
-      after = [
-        "network.target"
-      ] ++ optional config.services.mirakurun.enable "mirakurun.service"
-        ++ optional config.services.mysql.enable "mysql.service";
+      after = [ "network.target" ]
+        ++ lib.optional config.services.mirakurun.enable "mirakurun.service"
+        ++ lib.optional config.services.mysql.enable "mysql.service";
 
       serviceConfig = {
-        ExecStart = "${pkgs.epgstation}/bin/epgstation start";
+        ExecStart = "${cfg.package}/bin/epgstation start";
         ExecStartPre = "+${preStartScript}";
         User = username;
         Group = groupname;
diff --git a/nixos/modules/services/video/epgstation/streaming.json b/nixos/modules/services/video/epgstation/streaming.json
index 8eb99cf85584..7f8df0817fc3 100644
--- a/nixos/modules/services/video/epgstation/streaming.json
+++ b/nixos/modules/services/video/epgstation/streaming.json
@@ -1,119 +1,140 @@
 {
-  "liveHLS": [
-    {
-      "name": "720p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -map 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 17 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -preset veryfast -flags +loop-global_header %OUTPUT%"
+  "urlscheme": {
+    "m2ts": {
+      "ios": "vlc-x-callback://x-callback-url/stream?url=PROTOCOL://ADDRESS",
+      "android": "intent://ADDRESS#Intent;package=org.videolan.vlc;type=video;scheme=PROTOCOL;end"
     },
-    {
-      "name": "480p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -map 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 17 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -preset veryfast -flags +loop-global_header %OUTPUT%"
+    "video": {
+      "ios": "infuse://x-callback-url/play?url=PROTOCOL://ADDRESS",
+      "android": "intent://ADDRESS#Intent;package=com.mxtech.videoplayer.ad;type=video;scheme=PROTOCOL;end"
     },
-    {
-      "name": "180p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -map 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 17 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -c:a aac -ar 48000 -b:a 48k -ac 2 -c:v libx264 -vf yadif,scale=-2:180 -b:v 100k -preset veryfast -maxrate 110k -bufsize 1000k -flags +loop-global_header %OUTPUT%"
+    "download": {
+      "ios": "vlc-x-callback://x-callback-url/download?url=PROTOCOL://ADDRESS&filename=FILENAME"
     }
-  ],
-  "liveMP4": [
-    {
-      "name": "720p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
-    },
-    {
-      "name": "480p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
-    }
-  ],
-  "liveWebM": [
-    {
-      "name": "720p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 3 -c:a libvorbis -ar 48000 -b:a 192k -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:720 -b:v 3000k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
-    },
-    {
-      "name": "480p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 2 -c:a libvorbis -ar 48000 -b:a 128k -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:480 -b:v 1500k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
-    }
-  ],
-  "mpegTsStreaming": [
-    {
-      "name": "720p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -preset veryfast -y -f mpegts pipe:1"
-    },
-    {
-      "name": "480p",
-      "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -preset veryfast -y -f mpegts pipe:1"
-    },
-    {
-      "name": "Original"
-    }
-  ],
-  "mpegTsViewer": {
-    "ios": "vlc-x-callback://x-callback-url/stream?url=http://ADDRESS",
-    "android": "intent://ADDRESS#Intent;package=com.mxtech.videoplayer.ad;type=video;scheme=http;end"
-  },
-  "recordedDownloader": {
-    "ios": "vlc-x-callback://x-callback-url/download?url=http://ADDRESS&filename=FILENAME",
-    "android": "intent://ADDRESS#Intent;package=com.dv.adm;type=video;scheme=http;end"
   },
-  "recordedStreaming": {
-    "webm": [
-      {
-        "name": "720p",
-        "cmd": "%FFMPEG% -dual_mono_mode main %RE% -i pipe:0 -sn -threads 3 -c:a libvorbis -ar 48000 -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:720 %VB% %VBUFFER% %AB% %ABUFFER% -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1",
-        "vb": "3000k",
-        "ab": "192k"
-      },
-      {
-        "name": "360p",
-        "cmd": "%FFMPEG% -dual_mono_mode main %RE% -i pipe:0 -sn -threads 2 -c:a libvorbis -ar 48000 -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:360 %VB% %VBUFFER% %AB% %ABUFFER% -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1",
-        "vb": "1500k",
-        "ab": "128k"
-      }
-    ],
-    "mp4": [
-      {
-        "name": "720p",
-        "cmd": "%FFMPEG% -dual_mono_mode main %RE% -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -ac 2 -c:v libx264 -vf yadif,scale=-2:720 %VB% %VBUFFER% %AB% %ABUFFER% -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1",
-        "vb": "3000k",
-        "ab": "192k"
-      },
-      {
-        "name": "360p",
-        "cmd": "%FFMPEG% -dual_mono_mode main %RE% -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -ac 2 -c:v libx264 -vf yadif,scale=-2:360 %VB% %VBUFFER% %AB% %ABUFFER% -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1",
-        "vb": "1500k",
-        "ab": "128k"
+  "stream": {
+    "live": {
+      "ts": {
+        "m2ts": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -preset veryfast -y -f mpegts pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -preset veryfast -y -f mpegts pipe:1"
+          },
+          {
+            "name": "無変換"
+          }
+        ],
+        "m2tsll": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -f mpegts -analyzeduration 500000 -i pipe:0 -map 0 -c:s copy -c:d copy -ignore_unknown -fflags nobuffer -flags low_delay -max_delay 250000 -max_interleave_delta 1 -threads 0 -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -flags +cgop -vf yadif,scale=-2:720 -b:v 3000k -preset veryfast -y -f mpegts pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -f mpegts -analyzeduration 500000 -i pipe:0 -map 0 -c:s copy -c:d copy -ignore_unknown -fflags nobuffer -flags low_delay -max_delay 250000 -max_interleave_delta 1 -threads 0 -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -flags +cgop -vf yadif,scale=-2:480 -b:v 1500k -preset veryfast -y -f mpegts pipe:1"
+          }
+        ],
+        "webm": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 3 -c:a libvorbis -ar 48000 -b:a 192k -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:720 -b:v 3000k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 2 -c:a libvorbis -ar 48000 -b:a 128k -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:480 -b:v 1500k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
+          }
+        ],
+        "mp4": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
+          }
+        ],
+        "hls": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -map 0 -threads 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 17 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -hls_flags delete_segments -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -preset veryfast -flags +loop-global_header %OUTPUT%"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -re -dual_mono_mode main -i pipe:0 -sn -map 0 -threads 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 17 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -hls_flags delete_segments -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -preset veryfast -flags +loop-global_header %OUTPUT%"
+          }
+        ]
       }
-    ],
-    "mpegTs": [
-      {
-        "name": "720p (H.264)",
-        "cmd": "%FFMPEG% -dual_mono_mode main %RE% -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -ac 2 -c:v libx264 -vf yadif,scale=-2:720 %VB% %VBUFFER% %AB% %ABUFFER% -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -y -f mpegts pipe:1",
-        "vb": "3000k",
-        "ab": "192k"
+    },
+    "recorded": {
+      "ts": {
+        "webm": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -i pipe:0 -sn -threads 3 -c:a libvorbis -ar 48000 -b:a 192k -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:720 -b:v 3000k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -i pipe:0 -sn -threads 3 -c:a libvorbis -ar 48000 -b:a 128k -ac 2 -c:v libvpx-vp9 -vf yadif,scale=-2:480 -b:v 1500k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
+          }
+        ],
+        "mp4": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
+          }
+        ],
+        "hls": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -i pipe:0 -sn -map 0 -threads 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 0 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -hls_flags delete_segments -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -preset veryfast -flags +loop-global_header %OUTPUT%"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -i pipe:0 -sn -map 0 -threads 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 0 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -hls_flags delete_segments -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -preset veryfast -flags +loop-global_header %OUTPUT%"
+          }
+        ]
       },
-      {
-        "name": "360p (H.264)",
-        "cmd": "%FFMPEG% -dual_mono_mode main %RE% -i pipe:0 -sn -threads 0 -c:a aac -ar 48000 -ac 2 -c:v libx264 -vf yadif,scale=-2:360 %VB% %VBUFFER% %AB% %ABUFFER% -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -y -f mpegts pipe:1",
-        "vb": "1500k",
-        "ab": "128k"
+      "encoded": {
+        "webm": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -ss %SS% -i %INPUT% -sn -threads 3 -c:a libvorbis -ar 48000 -b:a 192k -ac 2 -c:v libvpx-vp9 -vf scale=-2:720 -b:v 3000k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -ss %SS% -i %INPUT% -sn -threads 3 -c:a libvorbis -ar 48000 -b:a 128k -ac 2 -c:v libvpx-vp9 -vf scale=-2:480 -b:v 1500k -deadline realtime -speed 4 -cpu-used -8 -y -f webm pipe:1"
+          }
+        ],
+        "mp4": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -ss %SS% -i %INPUT% -sn -threads 0 -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf scale=-2:720 -b:v 3000k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -ss %SS% -i %INPUT% -sn -threads 0 -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf scale=-2:480 -b:v 1500k -profile:v baseline -preset veryfast -tune fastdecode,zerolatency -movflags frag_keyframe+empty_moov+faststart+default_base_moof -y -f mp4 pipe:1"
+          }
+        ],
+        "hls": [
+          {
+            "name": "720p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -ss %SS% -i %INPUT% -sn -threads 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 0 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -hls_flags delete_segments -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf scale=-2:720 -b:v 3000k -preset veryfast -flags +loop-global_header %OUTPUT%"
+          },
+          {
+            "name": "480p",
+            "cmd": "%FFMPEG% -dual_mono_mode main -ss %SS% -i %INPUT% -sn -threads 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 0 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -hls_flags delete_segments -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf scale=-2:480 -b:v 3000k -preset veryfast -flags +loop-global_header %OUTPUT%"
+          }
+        ]
       }
-    ]
-  },
-  "recordedHLS": [
-    {
-      "name": "720p",
-      "cmd": "%FFMPEG% -dual_mono_mode main -i %INPUT% -sn -threads 0 -map 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 0 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -c:a aac -ar 48000 -b:a 192k -ac 2 -c:v libx264 -vf yadif,scale=-2:720 -b:v 3000k -preset veryfast -flags +loop-global_header %OUTPUT%"
-    },
-    {
-      "name": "480p",
-      "cmd": "%FFMPEG% -dual_mono_mode main -i %INPUT% -sn -threads 0 -map 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 0 -hls_allow_cache 1 -hls_segment_filename %streamFileDir%/stream%streamNum%-%09d.ts -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx264 -vf yadif,scale=-2:480 -b:v 1500k -preset veryfast -flags +loop-global_header %OUTPUT%"
-    },
-    {
-      "name": "480p(h265)",
-      "cmd": "%FFMPEG% -dual_mono_mode main -i %INPUT% -sn -map 0 -ignore_unknown -max_muxing_queue_size 1024 -f hls -hls_time 3 -hls_list_size 0 -hls_allow_cache 1 -hls_segment_type fmp4 -hls_fmp4_init_filename stream%streamNum%-init.mp4 -hls_segment_filename stream%streamNum%-%09d.m4s -c:a aac -ar 48000 -b:a 128k -ac 2 -c:v libx265 -vf yadif,scale=-2:480 -b:v 350k -preset veryfast -tag:v hvc1 %OUTPUT%"
     }
-  ],
-  "recordedViewer": {
-    "ios": "infuse://x-callback-url/play?url=http://ADDRESS",
-    "android": "intent://ADDRESS#Intent;package=com.mxtech.videoplayer.ad;type=video;scheme=http;end"
   }
 }
diff --git a/nixos/modules/services/video/unifi-video.nix b/nixos/modules/services/video/unifi-video.nix
index 17971b23db82..43208a9fe4cf 100644
--- a/nixos/modules/services/video/unifi-video.nix
+++ b/nixos/modules/services/video/unifi-video.nix
@@ -1,7 +1,8 @@
-{ config, lib, pkgs, utils, ... }:
+{ config, lib, options, pkgs, utils, ... }:
 with lib;
 let
   cfg = config.services.unifi-video;
+  opt = options.services.unifi-video;
   mainClass = "com.ubnt.airvision.Main";
   cmd = ''
     ${pkgs.jsvc}/bin/jsvc \
@@ -164,6 +165,7 @@ in
       pidFile = mkOption {
         type = types.path;
         default = "${cfg.dataDir}/unifi-video.pid";
+        defaultText = literalExpression ''"''${config.${opt.dataDir}}/unifi-video.pid"'';
         description = "Location of unifi-video pid file.";
       };
 
diff --git a/nixos/modules/services/wayland/cage.nix b/nixos/modules/services/wayland/cage.nix
index 273693a3b2fe..a32b81a916fc 100644
--- a/nixos/modules/services/wayland/cage.nix
+++ b/nixos/modules/services/wayland/cage.nix
@@ -74,11 +74,15 @@ in {
         TTYVTDisallocate = "yes";
         # Fail to start if not controlling the virtual terminal.
         StandardInput = "tty-fail";
+        StandardOutput = "journal";
+        StandardError = "journal";
         # Set up a full (custom) user session for the user, required by Cage.
         PAMName = "cage";
       };
     };
 
+    security.polkit.enable = true;
+
     security.pam.services.cage.text = ''
       auth    required pam_unix.so nullok
       account required pam_unix.so
diff --git a/nixos/modules/services/web-apps/baget.nix b/nixos/modules/services/web-apps/baget.nix
new file mode 100644
index 000000000000..3007dd4fbb26
--- /dev/null
+++ b/nixos/modules/services/web-apps/baget.nix
@@ -0,0 +1,170 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.services.baget;
+
+  defaultConfig = {
+    "PackageDeletionBehavior" = "Unlist";
+    "AllowPackageOverwrites" = false;
+
+    "Database" = {
+      "Type" = "Sqlite";
+      "ConnectionString" = "Data Source=baget.db";
+    };
+
+    "Storage" = {
+      "Type" = "FileSystem";
+      "Path" = "";
+    };
+
+    "Search" = {
+      "Type" = "Database";
+    };
+
+    "Mirror" = {
+      "Enabled" = false;
+      "PackageSource" = "https://api.nuget.org/v3/index.json";
+    };
+
+    "Logging" = {
+      "IncludeScopes" = false;
+      "Debug" = {
+        "LogLevel" = {
+          "Default" = "Warning";
+        };
+      };
+      "Console" = {
+        "LogLevel" = {
+          "Microsoft.Hosting.Lifetime" = "Information";
+          "Default" = "Warning";
+        };
+      };
+    };
+  };
+
+  configAttrs = recursiveUpdate defaultConfig cfg.extraConfig;
+
+  configFormat = pkgs.formats.json {};
+  configFile = configFormat.generate "appsettings.json" configAttrs;
+
+in
+{
+  options.services.baget = {
+    enable = mkEnableOption "BaGet NuGet-compatible server";
+
+    apiKeyFile = mkOption {
+      type = types.path;
+      example = "/root/baget.key";
+      description = ''
+        Private API key for BaGet.
+      '';
+    };
+
+    extraConfig = mkOption {
+      type = configFormat.type;
+      default = {};
+      example = {
+        "Database" = {
+          "Type" = "PostgreSql";
+          "ConnectionString" = "Server=/run/postgresql;Port=5432;";
+        };
+      };
+      defaultText = literalExpression ''
+        {
+          "PackageDeletionBehavior" = "Unlist";
+          "AllowPackageOverwrites" = false;
+
+          "Database" = {
+            "Type" = "Sqlite";
+            "ConnectionString" = "Data Source=baget.db";
+          };
+
+          "Storage" = {
+            "Type" = "FileSystem";
+            "Path" = "";
+          };
+
+          "Search" = {
+            "Type" = "Database";
+          };
+
+          "Mirror" = {
+            "Enabled" = false;
+            "PackageSource" = "https://api.nuget.org/v3/index.json";
+          };
+
+          "Logging" = {
+            "IncludeScopes" = false;
+            "Debug" = {
+              "LogLevel" = {
+                "Default" = "Warning";
+              };
+            };
+            "Console" = {
+              "LogLevel" = {
+                "Microsoft.Hosting.Lifetime" = "Information";
+                "Default" = "Warning";
+              };
+            };
+          };
+        }
+      '';
+      description = ''
+        Extra configuration options for BaGet. Refer to <link xlink:href="https://loic-sharma.github.io/BaGet/configuration/"/> for details.
+        Default value is merged with values from here.
+      '';
+    };
+  };
+
+  # implementation
+
+  config = mkIf cfg.enable {
+
+    systemd.services.baget = {
+      description = "BaGet server";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network.target" "network-online.target" ];
+      path = [ pkgs.jq ];
+      serviceConfig = {
+        WorkingDirectory = "/var/lib/baget";
+        DynamicUser = true;
+        StateDirectory = "baget";
+        StateDirectoryMode = "0700";
+        LoadCredential = "api_key:${cfg.apiKeyFile}";
+
+        CapabilityBoundingSet = "";
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        PrivateMounts = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        SystemCallFilter = [ "@system-service" "~@privileged" ];
+      };
+      script = ''
+        jq --slurpfile apiKeys <(jq -R . "$CREDENTIALS_DIRECTORY/api_key") '.ApiKey = $apiKeys[0]' ${configFile} > appsettings.json
+        ln -snf ${pkgs.baget}/lib/BaGet/wwwroot wwwroot
+        exec ${pkgs.baget}/bin/BaGet
+      '';
+    };
+
+  };
+}
diff --git a/nixos/modules/services/web-apps/bookstack.nix b/nixos/modules/services/web-apps/bookstack.nix
index 54c491f8b176..64a2767fab6e 100644
--- a/nixos/modules/services/web-apps/bookstack.nix
+++ b/nixos/modules/services/web-apps/bookstack.nix
@@ -24,8 +24,14 @@ let
     $sudo ${pkgs.php}/bin/php artisan $*
   '';
 
+  tlsEnabled = cfg.nginx.addSSL || cfg.nginx.forceSSL || cfg.nginx.onlySSL || cfg.nginx.enableACME;
 
 in {
+  imports = [
+    (mkRemovedOptionModule [ "services" "bookstack" "extraConfig" ] "Use services.bookstack.config instead.")
+    (mkRemovedOptionModule [ "services" "bookstack" "cacheDir" ] "The cache directory is now handled automatically.")
+  ];
+
   options.services.bookstack = {
 
     enable = mkEnableOption "BookStack";
@@ -44,28 +50,38 @@ in {
 
     appKeyFile = mkOption {
       description = ''
-        A file containing the AppKey.
-        Used for encryption where needed. Can be generated with <code>head -c 32 /dev/urandom| base64</code> and must be prefixed with <literal>base64:</literal>.
+        A file containing the Laravel APP_KEY - a 32 character long,
+        base64 encoded key used for encryption where needed. Can be
+        generated with <code>head -c 32 /dev/urandom | base64</code>.
       '';
       example = "/run/keys/bookstack-appkey";
       type = types.path;
     };
 
+    hostname = lib.mkOption {
+      type = lib.types.str;
+      default = if config.networking.domain != null then
+                  config.networking.fqdn
+                else
+                  config.networking.hostName;
+      defaultText = lib.literalExpression "config.networking.fqdn";
+      example = "bookstack.example.com";
+      description = ''
+        The hostname to serve BookStack on.
+      '';
+    };
+
     appURL = mkOption {
       description = ''
         The root URL that you want to host BookStack on. All URLs in BookStack will be generated using this value.
         If you change this in the future you may need to run a command to update stored URLs in the database. Command example: <code>php artisan bookstack:update-url https://old.example.com https://new.example.com</code>
       '';
+      default = "http${lib.optionalString tlsEnabled "s"}://${cfg.hostname}";
+      defaultText = ''http''${lib.optionalString tlsEnabled "s"}://''${cfg.hostname}'';
       example = "https://example.com";
       type = types.str;
     };
 
-    cacheDir = mkOption {
-      description = "BookStack cache directory";
-      default = "/var/cache/bookstack";
-      type = types.path;
-    };
-
     dataDir = mkOption {
       description = "BookStack data directory";
       default = "/var/lib/bookstack";
@@ -202,16 +218,59 @@ in {
       '';
     };
 
-    extraConfig = mkOption {
-      type = types.nullOr types.lines;
-      default = null;
-      example = ''
-        ALLOWED_IFRAME_HOSTS="https://example.com"
-        WKHTMLTOPDF=/home/user/bins/wkhtmltopdf
+    config = mkOption {
+      type = with types;
+        attrsOf
+          (nullOr
+            (either
+              (oneOf [
+                bool
+                int
+                port
+                path
+                str
+              ])
+              (submodule {
+                options = {
+                  _secret = mkOption {
+                    type = nullOr str;
+                    description = ''
+                      The path to a file containing the value the
+                      option should be set to in the final
+                      configuration file.
+                    '';
+                  };
+                };
+              })));
+      default = {};
+      example = literalExpression ''
+        {
+          ALLOWED_IFRAME_HOSTS = "https://example.com";
+          WKHTMLTOPDF = "/home/user/bins/wkhtmltopdf";
+          AUTH_METHOD = "oidc";
+          OIDC_NAME = "MyLogin";
+          OIDC_DISPLAY_NAME_CLAIMS = "name";
+          OIDC_CLIENT_ID = "bookstack";
+          OIDC_CLIENT_SECRET = {_secret = "/run/keys/oidc_secret"};
+          OIDC_ISSUER = "https://keycloak.example.com/auth/realms/My%20Realm";
+          OIDC_ISSUER_DISCOVER = true;
+        }
       '';
       description = ''
-        Lines to be appended verbatim to the BookStack configuration.
-        Refer to <link xlink:href="https://www.bookstackapp.com/docs/"/> for details on supported values.
+        BookStack configuration options to set in the
+        <filename>.env</filename> file.
+
+        Refer to <link xlink:href="https://www.bookstackapp.com/docs/"/>
+        for details on supported values.
+
+        Settings containing secret data should be set to an attribute
+        set containing the attribute <literal>_secret</literal> - a
+        string pointing to a file containing the value the option
+        should be set to. See the example to get a better picture of
+        this: in the resulting <filename>.env</filename> file, the
+        <literal>OIDC_CLIENT_SECRET</literal> key will be set to the
+        contents of the <filename>/run/keys/oidc_secret</filename>
+        file.
       '';
     };
 
@@ -228,6 +287,30 @@ in {
       }
     ];
 
+    services.bookstack.config = {
+      APP_KEY._secret = cfg.appKeyFile;
+      APP_URL = cfg.appURL;
+      DB_HOST = db.host;
+      DB_PORT = db.port;
+      DB_DATABASE = db.name;
+      DB_USERNAME = db.user;
+      MAIL_DRIVER = mail.driver;
+      MAIL_FROM_NAME = mail.fromName;
+      MAIL_FROM = mail.from;
+      MAIL_HOST = mail.host;
+      MAIL_PORT = mail.port;
+      MAIL_USERNAME = mail.user;
+      MAIL_ENCRYPTION = mail.encryption;
+      DB_PASSWORD._secret = db.passwordFile;
+      MAIL_PASSWORD._secret = mail.passwordFile;
+      APP_SERVICES_CACHE = "/run/bookstack/cache/services.php";
+      APP_PACKAGES_CACHE = "/run/bookstack/cache/packages.php";
+      APP_CONFIG_CACHE = "/run/bookstack/cache/config.php";
+      APP_ROUTES_CACHE = "/run/bookstack/cache/routes-v7.php";
+      APP_EVENTS_CACHE = "/run/bookstack/cache/events.php";
+      SESSION_SECURE_COOKIE = tlsEnabled;
+    };
+
     environment.systemPackages = [ artisan ];
 
     services.mysql = mkIf db.createLocally {
@@ -258,24 +341,19 @@ in {
 
     services.nginx = {
       enable = mkDefault true;
-      virtualHosts.bookstack = mkMerge [ cfg.nginx {
+      recommendedTlsSettings = true;
+      recommendedOptimisation = true;
+      recommendedGzipSettings = true;
+      virtualHosts.${cfg.hostname} = mkMerge [ cfg.nginx {
         root = mkForce "${bookstack}/public";
-        extraConfig = optionalString (cfg.nginx.addSSL || cfg.nginx.forceSSL || cfg.nginx.onlySSL || cfg.nginx.enableACME) "fastcgi_param HTTPS on;";
         locations = {
           "/" = {
             index = "index.php";
-            extraConfig = ''try_files $uri $uri/ /index.php?$query_string;'';
-          };
-          "~ \.php$" = {
-            extraConfig = ''
-              try_files $uri $uri/ /index.php?$query_string;
-              include ${pkgs.nginx}/conf/fastcgi_params;
-              fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-              fastcgi_param REDIRECT_STATUS 200;
-              fastcgi_pass unix:${config.services.phpfpm.pools."bookstack".socket};
-              ${optionalString (cfg.nginx.addSSL || cfg.nginx.forceSSL || cfg.nginx.onlySSL || cfg.nginx.enableACME) "fastcgi_param HTTPS on;"}
-            '';
+            tryFiles = "$uri $uri/ /index.php?$query_string";
           };
+          "~ \.php$".extraConfig = ''
+            fastcgi_pass unix:${config.services.phpfpm.pools."bookstack".socket};
+          '';
           "~ \.(js|css|gif|png|ico|jpg|jpeg)$" = {
             extraConfig = "expires 365d;";
           };
@@ -290,53 +368,54 @@ in {
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
         Type = "oneshot";
+        RemainAfterExit = true;
         User = user;
         WorkingDirectory = "${bookstack}";
+        RuntimeDirectory = "bookstack/cache";
+        RuntimeDirectoryMode = 0700;
       };
-      script = ''
+      path = [ pkgs.replace-secret ];
+      script =
+        let
+          isSecret = v: isAttrs v && v ? _secret && isString v._secret;
+          bookstackEnvVars = lib.generators.toKeyValue {
+            mkKeyValue = lib.flip lib.generators.mkKeyValueDefault "=" {
+              mkValueString = v: with builtins;
+                if isInt         v then toString v
+                else if isString v then v
+                else if true  == v then "true"
+                else if false == v then "false"
+                else if isSecret v then hashString "sha256" v._secret
+                else throw "unsupported type ${typeOf v}: ${(lib.generators.toPretty {}) v}";
+            };
+          };
+          secretPaths = lib.mapAttrsToList (_: v: v._secret) (lib.filterAttrs (_: isSecret) cfg.config);
+          mkSecretReplacement = file: ''
+            replace-secret ${escapeShellArgs [ (builtins.hashString "sha256" file) file "${cfg.dataDir}/.env" ]}
+          '';
+          secretReplacements = lib.concatMapStrings mkSecretReplacement secretPaths;
+          filteredConfig = lib.converge (lib.filterAttrsRecursive (_: v: ! elem v [ {} null ])) cfg.config;
+          bookstackEnv = pkgs.writeText "bookstack.env" (bookstackEnvVars filteredConfig);
+        in ''
+        # error handling
+        set -euo pipefail
+
         # set permissions
         umask 077
+
         # create .env file
-        echo "
-        APP_KEY=base64:$(head -n1 ${cfg.appKeyFile})
-        APP_URL=${cfg.appURL}
-        DB_HOST=${db.host}
-        DB_PORT=${toString db.port}
-        DB_DATABASE=${db.name}
-        DB_USERNAME=${db.user}
-        MAIL_DRIVER=${mail.driver}
-        MAIL_FROM_NAME=\"${mail.fromName}\"
-        MAIL_FROM=${mail.from}
-        MAIL_HOST=${mail.host}
-        MAIL_PORT=${toString mail.port}
-        ${optionalString (mail.user != null) "MAIL_USERNAME=${mail.user};"}
-        ${optionalString (mail.encryption != null) "MAIL_ENCRYPTION=${mail.encryption};"}
-        ${optionalString (db.passwordFile != null) "DB_PASSWORD=$(head -n1 ${db.passwordFile})"}
-        ${optionalString (mail.passwordFile != null) "MAIL_PASSWORD=$(head -n1 ${mail.passwordFile})"}
-        APP_SERVICES_CACHE=${cfg.cacheDir}/services.php
-        APP_PACKAGES_CACHE=${cfg.cacheDir}/packages.php
-        APP_CONFIG_CACHE=${cfg.cacheDir}/config.php
-        APP_ROUTES_CACHE=${cfg.cacheDir}/routes-v7.php
-        APP_EVENTS_CACHE=${cfg.cacheDir}/events.php
-        ${optionalString (cfg.nginx.addSSL || cfg.nginx.forceSSL || cfg.nginx.onlySSL || cfg.nginx.enableACME) "SESSION_SECURE_COOKIE=true"}
-        ${toString cfg.extraConfig}
-        " > "${cfg.dataDir}/.env"
+        install -T -m 0600 -o ${user} ${bookstackEnv} "${cfg.dataDir}/.env"
+        ${secretReplacements}
+        if ! grep 'APP_KEY=base64:' "${cfg.dataDir}/.env" >/dev/null; then
+            sed -i 's/APP_KEY=/APP_KEY=base64:/' "${cfg.dataDir}/.env"
+        fi
 
         # migrate db
         ${pkgs.php}/bin/php artisan migrate --force
-
-        # clear & create caches (needed in case of update)
-        ${pkgs.php}/bin/php artisan cache:clear
-        ${pkgs.php}/bin/php artisan config:clear
-        ${pkgs.php}/bin/php artisan view:clear
-        ${pkgs.php}/bin/php artisan config:cache
-        ${pkgs.php}/bin/php artisan route:cache
-        ${pkgs.php}/bin/php artisan view:cache
       '';
     };
 
     systemd.tmpfiles.rules = [
-      "d ${cfg.cacheDir}                           0700 ${user} ${group} - -"
       "d ${cfg.dataDir}                            0710 ${user} ${group} - -"
       "d ${cfg.dataDir}/public                     0750 ${user} ${group} - -"
       "d ${cfg.dataDir}/public/uploads             0750 ${user} ${group} - -"
diff --git a/nixos/modules/services/web-apps/dex.nix b/nixos/modules/services/web-apps/dex.nix
index f08dd65bdb0f..4d4689a4cf24 100644
--- a/nixos/modules/services/web-apps/dex.nix
+++ b/nixos/modules/services/web-apps/dex.nix
@@ -112,4 +112,7 @@ in
       };
     };
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/web-apps/discourse.nix b/nixos/modules/services/web-apps/discourse.nix
index 18b61200aa1b..2c2911aada3f 100644
--- a/nixos/modules/services/web-apps/discourse.nix
+++ b/nixos/modules/services/web-apps/discourse.nix
@@ -4,6 +4,7 @@ let
   json = pkgs.formats.json {};
 
   cfg = config.services.discourse;
+  opt = options.services.discourse;
 
   # Keep in sync with https://github.com/discourse/discourse_docker/blob/master/image/base/Dockerfile#L5
   upstreamPostgresqlVersion = lib.getVersion pkgs.postgresql_13;
@@ -327,6 +328,7 @@ in
         useSSL = lib.mkOption {
           type = lib.types.bool;
           default = cfg.redis.host != "localhost";
+          defaultText = lib.literalExpression ''config.${opt.redis.host} != "localhost"'';
           description = ''
             Connect to Redis with SSL.
           '';
@@ -399,6 +401,7 @@ in
           domain = lib.mkOption {
             type = lib.types.str;
             default = cfg.hostname;
+            defaultText = lib.literalExpression "config.${opt.hostname}";
             description = ''
               HELO domain to use for outgoing mail.
             '';
diff --git a/nixos/modules/services/web-apps/discourse.xml b/nixos/modules/services/web-apps/discourse.xml
index e91d3eac422d..ad9b65abf51e 100644
--- a/nixos/modules/services/web-apps/discourse.xml
+++ b/nixos/modules/services/web-apps/discourse.xml
@@ -25,7 +25,7 @@ services.discourse = {
   };
   <link linkend="opt-services.discourse.secretKeyBaseFile">secretKeyBaseFile</link> = "/path/to/secret_key_base_file";
 };
-<link linkend="opt-security.acme.email">security.acme.email</link> = "me@example.com";
+<link linkend="opt-security.acme.defaults.email">security.acme.email</link> = "me@example.com";
 <link linkend="opt-security.acme.acceptTerms">security.acme.acceptTerms</link> = true;
 </programlisting>
    </para>
diff --git a/nixos/modules/services/web-apps/dokuwiki.nix b/nixos/modules/services/web-apps/dokuwiki.nix
index 9b9ae931f9a7..1f8ca742db95 100644
--- a/nixos/modules/services/web-apps/dokuwiki.nix
+++ b/nixos/modules/services/web-apps/dokuwiki.nix
@@ -1,20 +1,14 @@
 { config, pkgs, lib, ... }:
 
-let
-  inherit (lib) mkDefault mkEnableOption mkForce mkIf mkMerge mkOption types maintainers recursiveUpdate;
-  inherit (lib) any attrValues concatMapStrings concatMapStringsSep flatten literalExpression;
-  inherit (lib) filterAttrs mapAttrs mapAttrs' mapAttrsToList nameValuePair optional optionalAttrs optionalString;
+with lib;
 
-  cfg = migrateOldAttrs config.services.dokuwiki;
+let
+  cfg = config.services.dokuwiki;
   eachSite = cfg.sites;
   user = "dokuwiki";
   webserver = config.services.${cfg.webserver};
   stateDir = hostName: "/var/lib/dokuwiki/${hostName}/data";
 
-  # Migrate config.services.dokuwiki.<hostName> to config.services.dokuwiki.sites.<hostName>
-  oldSites = filterAttrs (o: _: o != "sites" && o != "webserver");
-  migrateOldAttrs = cfg: cfg // { sites = cfg.sites // oldSites cfg; };
-
   dokuwikiAclAuthConfig = hostName: cfg: pkgs.writeText "acl.auth-${hostName}.php" ''
     # acl.auth.php
     # <?php exit()?>
@@ -255,36 +249,29 @@ in
 {
   # interface
   options = {
-    services.dokuwiki = mkOption {
-      type = types.submodule {
-        # Used to support old interface
-        freeformType = types.attrsOf (types.submodule siteOpts);
-
-        # New interface
-        options.sites = mkOption {
-          type = types.attrsOf (types.submodule siteOpts);
-          default = {};
-          description = "Specification of one or more DokuWiki sites to serve";
-        };
+    services.dokuwiki = {
 
-        options.webserver = mkOption {
-          type = types.enum [ "nginx" "caddy" ];
-          default = "nginx";
-          description = ''
-            Whether to use nginx or caddy for virtual host management.
+      sites = mkOption {
+        type = types.attrsOf (types.submodule siteOpts);
+        default = {};
+        description = "Specification of one or more DokuWiki sites to serve";
+      };
 
-            Further nginx configuration can be done by adapting <literal>services.nginx.virtualHosts.&lt;name&gt;</literal>.
-            See <xref linkend="opt-services.nginx.virtualHosts"/> for further information.
+      webserver = mkOption {
+        type = types.enum [ "nginx" "caddy" ];
+        default = "nginx";
+        description = ''
+          Whether to use nginx or caddy for virtual host management.
 
-            Further apache2 configuration can be done by adapting <literal>services.httpd.virtualHosts.&lt;name&gt;</literal>.
-            See <xref linkend="opt-services.httpd.virtualHosts"/> for further information.
-          '';
-        };
+          Further nginx configuration can be done by adapting <literal>services.nginx.virtualHosts.&lt;name&gt;</literal>.
+          See <xref linkend="opt-services.nginx.virtualHosts"/> for further information.
+
+          Further apache2 configuration can be done by adapting <literal>services.httpd.virtualHosts.&lt;name&gt;</literal>.
+          See <xref linkend="opt-services.httpd.virtualHosts"/> for further information.
+        '';
       };
-      default = {};
-      description = "DokuWiki configuration";
-    };
 
+    };
   };
 
   # implementation
@@ -301,8 +288,6 @@ in
     }
     ]) eachSite);
 
-    warnings = mapAttrsToList (hostName: _: ''services.dokuwiki."${hostName}" is deprecated use services.dokuwiki.sites."${hostName}"'') (oldSites cfg);
-
     services.phpfpm.pools = mapAttrs' (hostName: cfg: (
       nameValuePair "dokuwiki-${hostName}" {
         inherit user;
@@ -391,7 +376,7 @@ in
           "~ \\.php$" = {
             extraConfig = ''
               try_files $uri $uri/ /doku.php;
-              include ${pkgs.nginx}/conf/fastcgi_params;
+              include ${config.services.nginx.package}/conf/fastcgi_params;
               fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
               fastcgi_param REDIRECT_STATUS 200;
               fastcgi_pass unix:${config.services.phpfpm.pools."dokuwiki-${hostName}".socket};
diff --git a/nixos/modules/services/web-apps/ethercalc.nix b/nixos/modules/services/web-apps/ethercalc.nix
new file mode 100644
index 000000000000..d74def59c6c3
--- /dev/null
+++ b/nixos/modules/services/web-apps/ethercalc.nix
@@ -0,0 +1,62 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.ethercalc;
+in {
+  options = {
+    services.ethercalc = {
+      enable = mkOption {
+        default = false;
+        type = types.bool;
+        description = ''
+          ethercalc, an online collaborative spreadsheet server.
+
+          Persistent state will be maintained under
+          <filename>/var/lib/ethercalc</filename>. Upstream supports using a
+          redis server for storage and recommends the redis backend for
+          intensive use; however, the Nix module doesn't currently support
+          redis.
+
+          Note that while ethercalc is a good and robust project with an active
+          issue tracker, there haven't been new commits since the end of 2020.
+        '';
+      };
+
+      package = mkOption {
+        default = pkgs.ethercalc;
+        defaultText = literalExpression "pkgs.ethercalc";
+        type = types.package;
+        description = "Ethercalc package to use.";
+      };
+
+      host = mkOption {
+        type = types.str;
+        default = "0.0.0.0";
+        description = "Address to listen on (use 0.0.0.0 to allow access from any address).";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 8000;
+        description = "Port to bind to.";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.ethercalc = {
+      description = "Ethercalc service";
+      wantedBy    = [ "multi-user.target" ];
+      after       = [ "network.target" ];
+      serviceConfig = {
+        DynamicUser    =   true;
+        ExecStart        = "${cfg.package}/bin/ethercalc --host ${cfg.host} --port ${toString cfg.port}";
+        Restart          = "always";
+        StateDirectory   = "ethercalc";
+        WorkingDirectory = "/var/lib/ethercalc";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/web-apps/galene.nix b/nixos/modules/services/web-apps/galene.nix
index db9dfeb47499..1d0a620585b0 100644
--- a/nixos/modules/services/web-apps/galene.nix
+++ b/nixos/modules/services/web-apps/galene.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 let
   cfg = config.services.galene;
+  opt = options.services.galene;
   defaultstateDir = "/var/lib/galene";
   defaultrecordingsDir = "${cfg.stateDir}/recordings";
   defaultgroupsDir = "${cfg.stateDir}/groups";
@@ -88,6 +89,7 @@ in
       recordingsDir = mkOption {
         type = types.str;
         default = defaultrecordingsDir;
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/recordings"'';
         example = "/var/lib/galene/recordings";
         description = "Recordings directory.";
       };
@@ -95,6 +97,7 @@ in
       dataDir = mkOption {
         type = types.str;
         default = defaultdataDir;
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/data"'';
         example = "/var/lib/galene/data";
         description = "Data directory.";
       };
@@ -102,6 +105,7 @@ in
       groupsDir = mkOption {
         type = types.str;
         default = defaultgroupsDir;
+        defaultText = literalExpression ''"''${config.${opt.stateDir}}/groups"'';
         example = "/var/lib/galene/groups";
         description = "Web server directory.";
       };
diff --git a/nixos/modules/services/web-apps/gerrit.nix b/nixos/modules/services/web-apps/gerrit.nix
index 9ee9dbf1aa49..6bfc67368dd5 100644
--- a/nixos/modules/services/web-apps/gerrit.nix
+++ b/nixos/modules/services/web-apps/gerrit.nix
@@ -237,4 +237,6 @@ in
   };
 
   meta.maintainers = with lib.maintainers; [ edef zimbatm ];
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/web-apps/hedgedoc.nix b/nixos/modules/services/web-apps/hedgedoc.nix
index e0c00fe67ea3..9eeabb9d5662 100644
--- a/nixos/modules/services/web-apps/hedgedoc.nix
+++ b/nixos/modules/services/web-apps/hedgedoc.nix
@@ -33,7 +33,7 @@ in
       type = types.listOf types.str;
       default = [];
       description = ''
-        Groups to which the user ${name} should be added.
+        Groups to which the service user should be added.
       '';
     };
 
diff --git a/nixos/modules/services/web-apps/invidious.nix b/nixos/modules/services/web-apps/invidious.nix
index 50370629e47c..10b30bf1fd1d 100644
--- a/nixos/modules/services/web-apps/invidious.nix
+++ b/nixos/modules/services/web-apps/invidious.nix
@@ -225,6 +225,7 @@ in
       port = lib.mkOption {
         type = types.port;
         default = options.services.postgresql.port.default;
+        defaultText = lib.literalExpression "options.services.postgresql.port.default";
         description = ''
           The port of the database Invidious should use.
 
diff --git a/nixos/modules/services/web-apps/invoiceplane.nix b/nixos/modules/services/web-apps/invoiceplane.nix
new file mode 100644
index 000000000000..095eec36dec3
--- /dev/null
+++ b/nixos/modules/services/web-apps/invoiceplane.nix
@@ -0,0 +1,305 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.invoiceplane;
+  eachSite = cfg.sites;
+  user = "invoiceplane";
+  webserver = config.services.${cfg.webserver};
+
+  invoiceplane-config = hostName: cfg: pkgs.writeText "ipconfig.php" ''
+    IP_URL=http://${hostName}
+    ENABLE_DEBUG=false
+    DISABLE_SETUP=false
+    REMOVE_INDEXPHP=false
+    DB_HOSTNAME=${cfg.database.host}
+    DB_USERNAME=${cfg.database.user}
+    # NOTE: file_get_contents adds newline at the end of returned string
+    DB_PASSWORD=${if cfg.database.passwordFile == null then "" else "trim(file_get_contents('${cfg.database.passwordFile}'), \"\\r\\n\")"}
+    DB_DATABASE=${cfg.database.name}
+    DB_PORT=${toString cfg.database.port}
+    SESS_EXPIRATION=864000
+    ENABLE_INVOICE_DELETION=false
+    DISABLE_READ_ONLY=false
+    ENCRYPTION_KEY=
+    ENCRYPTION_CIPHER=AES-256
+    SETUP_COMPLETED=false
+  '';
+
+  extraConfig = hostName: cfg: pkgs.writeText "extraConfig.php" ''
+    ${toString cfg.extraConfig}
+  '';
+
+  pkg = hostName: cfg: pkgs.stdenv.mkDerivation rec {
+    pname = "invoiceplane-${hostName}";
+    version = src.version;
+    src = pkgs.invoiceplane;
+
+    patchPhase = ''
+      # Patch index.php file to load additional config file
+      substituteInPlace index.php \
+        --replace "require('vendor/autoload.php');" "require('vendor/autoload.php'); \$dotenv = new \Dotenv\Dotenv(__DIR__, 'extraConfig.php'); \$dotenv->load();";
+    '';
+
+    installPhase = ''
+      mkdir -p $out
+      cp -r * $out/
+
+      # symlink uploads and log directories
+      rm -r $out/uploads $out/application/logs $out/vendor/mpdf/mpdf/tmp
+      ln -sf ${cfg.stateDir}/uploads $out/
+      ln -sf ${cfg.stateDir}/logs $out/application/
+      ln -sf ${cfg.stateDir}/tmp $out/vendor/mpdf/mpdf/
+
+      # symlink the InvoicePlane config
+      ln -s ${cfg.stateDir}/ipconfig.php $out/ipconfig.php
+
+      # symlink the extraConfig file
+      ln -s ${extraConfig hostName cfg} $out/extraConfig.php
+
+      # symlink additional templates
+      ${concatMapStringsSep "\n" (template: "cp -r ${template}/. $out/application/views/invoice_templates/pdf/") cfg.invoiceTemplates}
+    '';
+  };
+
+  siteOpts = { lib, name, ... }:
+    {
+      options = {
+
+        enable = mkEnableOption "InvoicePlane web application";
+
+        stateDir = mkOption {
+          type = types.path;
+          default = "/var/lib/invoiceplane/${name}";
+          description = ''
+            This directory is used for uploads of attachements and cache.
+            The directory passed here is automatically created and permissions
+            adjusted as required.
+          '';
+        };
+
+        database = {
+          host = mkOption {
+            type = types.str;
+            default = "localhost";
+            description = "Database host address.";
+          };
+
+          port = mkOption {
+            type = types.port;
+            default = 3306;
+            description = "Database host port.";
+          };
+
+          name = mkOption {
+            type = types.str;
+            default = "invoiceplane";
+            description = "Database name.";
+          };
+
+          user = mkOption {
+            type = types.str;
+            default = "invoiceplane";
+            description = "Database user.";
+          };
+
+          passwordFile = mkOption {
+            type = types.nullOr types.path;
+            default = null;
+            example = "/run/keys/invoiceplane-dbpassword";
+            description = ''
+              A file containing the password corresponding to
+              <option>database.user</option>.
+            '';
+          };
+
+          createLocally = mkOption {
+            type = types.bool;
+            default = true;
+            description = "Create the database and database user locally.";
+          };
+        };
+
+        invoiceTemplates = mkOption {
+          type = types.listOf types.path;
+          default = [];
+          description = ''
+            List of path(s) to respective template(s) which are copied from the 'invoice_templates/pdf' directory.
+            <note><para>These templates need to be packaged before use, see example.</para></note>
+          '';
+          example = literalExpression ''
+            let
+              # Let's package an example template
+              template-vtdirektmarketing = pkgs.stdenv.mkDerivation {
+                name = "vtdirektmarketing";
+                # Download the template from a public repository
+                src = pkgs.fetchgit {
+                  url = "https://git.project-insanity.org/onny/invoiceplane-vtdirektmarketing.git";
+                  sha256 = "1hh0q7wzsh8v8x03i82p6qrgbxr4v5fb05xylyrpp975l8axyg2z";
+                };
+                sourceRoot = ".";
+                # Installing simply means copying template php file to the output directory
+                installPhase = ""
+                  mkdir -p $out
+                  cp invoiceplane-vtdirektmarketing/vtdirektmarketing.php $out/
+                "";
+              };
+            # And then pass this package to the template list like this:
+            in [ template-vtdirektmarketing ]
+          '';
+        };
+
+        poolConfig = mkOption {
+          type = with types; attrsOf (oneOf [ str int bool ]);
+          default = {
+            "pm" = "dynamic";
+            "pm.max_children" = 32;
+            "pm.start_servers" = 2;
+            "pm.min_spare_servers" = 2;
+            "pm.max_spare_servers" = 4;
+            "pm.max_requests" = 500;
+          };
+          description = ''
+            Options for the InvoicePlane PHP pool. See the documentation on <literal>php-fpm.conf</literal>
+            for details on configuration directives.
+          '';
+        };
+
+        extraConfig = mkOption {
+          type = types.nullOr types.lines;
+          default = null;
+          example = ''
+            SETUP_COMPLETED=true
+            DISABLE_SETUP=true
+            IP_URL=https://invoice.example.com
+          '';
+          description = ''
+            InvoicePlane configuration. Refer to
+            <link xlink:href="https://github.com/InvoicePlane/InvoicePlane/blob/master/ipconfig.php.example"/>
+            for details on supported values.
+          '';
+        };
+
+      };
+
+    };
+in
+{
+  # interface
+  options = {
+    services.invoiceplane = mkOption {
+      type = types.submodule {
+
+        options.sites = mkOption {
+          type = types.attrsOf (types.submodule siteOpts);
+          default = {};
+          description = "Specification of one or more WordPress sites to serve";
+        };
+
+        options.webserver = mkOption {
+          type = types.enum [ "caddy" ];
+          default = "caddy";
+          description = ''
+            Which webserver to use for virtual host management. Currently only
+            caddy is supported.
+          '';
+        };
+      };
+      default = {};
+      description = "InvoicePlane configuration.";
+    };
+
+  };
+
+  # implementation
+  config = mkIf (eachSite != {}) (mkMerge [{
+
+    assertions = flatten (mapAttrsToList (hostName: cfg:
+      [{ assertion = cfg.database.createLocally -> cfg.database.user == user;
+        message = ''services.invoiceplane.sites."${hostName}".database.user must be ${user} if the database is to be automatically provisioned'';
+      }
+      { assertion = cfg.database.createLocally -> cfg.database.passwordFile == null;
+        message = ''services.invoiceplane.sites."${hostName}".database.passwordFile cannot be specified if services.invoiceplane.sites."${hostName}".database.createLocally is set to true.'';
+      }]
+    ) eachSite);
+
+    services.mysql = mkIf (any (v: v.database.createLocally) (attrValues eachSite)) {
+      enable = true;
+      package = mkDefault pkgs.mariadb;
+      ensureDatabases = mapAttrsToList (hostName: cfg: cfg.database.name) eachSite;
+      ensureUsers = mapAttrsToList (hostName: cfg:
+        { name = cfg.database.user;
+          ensurePermissions = { "${cfg.database.name}.*" = "ALL PRIVILEGES"; };
+        }
+      ) eachSite;
+    };
+
+    services.phpfpm = {
+      phpPackage = pkgs.php74;
+      pools = mapAttrs' (hostName: cfg: (
+        nameValuePair "invoiceplane-${hostName}" {
+          inherit user;
+          group = webserver.group;
+          settings = {
+            "listen.owner" = webserver.user;
+            "listen.group" = webserver.group;
+          } // cfg.poolConfig;
+        }
+      )) eachSite;
+    };
+
+  }
+
+  {
+    systemd.tmpfiles.rules = flatten (mapAttrsToList (hostName: cfg: [
+      "d ${cfg.stateDir} 0750 ${user} ${webserver.group} - -"
+      "f ${cfg.stateDir}/ipconfig.php 0750 ${user} ${webserver.group} - -"
+      "d ${cfg.stateDir}/logs 0750 ${user} ${webserver.group} - -"
+      "d ${cfg.stateDir}/uploads 0750 ${user} ${webserver.group} - -"
+      "d ${cfg.stateDir}/uploads/archive 0750 ${user} ${webserver.group} - -"
+      "d ${cfg.stateDir}/uploads/customer_files 0750 ${user} ${webserver.group} - -"
+      "d ${cfg.stateDir}/uploads/temp 0750 ${user} ${webserver.group} - -"
+      "d ${cfg.stateDir}/uploads/temp/mpdf 0750 ${user} ${webserver.group} - -"
+      "d ${cfg.stateDir}/tmp 0750 ${user} ${webserver.group} - -"
+    ]) eachSite);
+
+    systemd.services.invoiceplane-config = {
+      serviceConfig.Type = "oneshot";
+      script = concatStrings (mapAttrsToList (hostName: cfg:
+        ''
+          mkdir -p ${cfg.stateDir}/logs \
+                   ${cfg.stateDir}/uploads
+          if ! grep -q IP_URL "${cfg.stateDir}/ipconfig.php"; then
+            cp "${invoiceplane-config hostName cfg}" "${cfg.stateDir}/ipconfig.php"
+          fi
+        '') eachSite);
+      wantedBy = [ "multi-user.target" ];
+    };
+
+    users.users.${user} = {
+      group = webserver.group;
+      isSystemUser = true;
+    };
+  }
+
+  (mkIf (cfg.webserver == "caddy") {
+    services.caddy = {
+      enable = true;
+      virtualHosts = mapAttrs' (hostName: cfg: (
+        nameValuePair "http://${hostName}" {
+          extraConfig = ''
+            root    * ${pkg hostName cfg}
+            file_server
+
+            php_fastcgi unix/${config.services.phpfpm.pools."invoiceplane-${hostName}".socket}
+          '';
+        }
+      )) eachSite;
+    };
+  })
+
+
+  ]);
+}
+
diff --git a/nixos/modules/services/web-apps/jirafeau.nix b/nixos/modules/services/web-apps/jirafeau.nix
index 83cf224f7d27..328c61c8e646 100644
--- a/nixos/modules/services/web-apps/jirafeau.nix
+++ b/nixos/modules/services/web-apps/jirafeau.nix
@@ -136,7 +136,7 @@ in
               '';
             locations = {
               "~ \\.php$".extraConfig = ''
-                include ${pkgs.nginx}/conf/fastcgi_params;
+                include ${config.services.nginx.package}/conf/fastcgi_params;
                 fastcgi_split_path_info ^(.+\.php)(/.+)$;
                 fastcgi_index index.php;
                 fastcgi_pass unix:${config.services.phpfpm.pools.jirafeau.socket};
@@ -167,4 +167,7 @@ in
       "d ${cfg.dataDir}/async/ 0750 ${user} ${group} - -"
     ];
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/services/web-apps/jitsi-meet.xml b/nixos/modules/services/web-apps/jitsi-meet.xml
index 97373bc6d9a8..ff44c724adf4 100644
--- a/nixos/modules/services/web-apps/jitsi-meet.xml
+++ b/nixos/modules/services/web-apps/jitsi-meet.xml
@@ -20,7 +20,7 @@
   };
   <link linkend="opt-services.jitsi-videobridge.openFirewall">services.jitsi-videobridge.openFirewall</link> = true;
   <link linkend="opt-networking.firewall.allowedTCPPorts">networking.firewall.allowedTCPPorts</link> = [ 80 443 ];
-  <link linkend="opt-security.acme.email">security.acme.email</link> = "me@example.com";
+  <link linkend="opt-security.acme.defaults.email">security.acme.email</link> = "me@example.com";
   <link linkend="opt-security.acme.acceptTerms">security.acme.acceptTerms</link> = true;
 }</programlisting>
    </para>
@@ -46,7 +46,7 @@
   };
   <link linkend="opt-services.jitsi-videobridge.openFirewall">services.jitsi-videobridge.openFirewall</link> = true;
   <link linkend="opt-networking.firewall.allowedTCPPorts">networking.firewall.allowedTCPPorts</link> = [ 80 443 ];
-  <link linkend="opt-security.acme.email">security.acme.email</link> = "me@example.com";
+  <link linkend="opt-security.acme.defaults.email">security.acme.email</link> = "me@example.com";
   <link linkend="opt-security.acme.acceptTerms">security.acme.acceptTerms</link> = true;
 }</programlisting>
    </para>
diff --git a/nixos/modules/services/web-apps/keycloak.nix b/nixos/modules/services/web-apps/keycloak.nix
index 699c88bc2395..88cba0febfc0 100644
--- a/nixos/modules/services/web-apps/keycloak.nix
+++ b/nixos/modules/services/web-apps/keycloak.nix
@@ -1,287 +1,321 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 
 let
   cfg = config.services.keycloak;
+  opt = options.services.keycloak;
+
+  inherit (lib) types mkOption concatStringsSep mapAttrsToList
+    escapeShellArg recursiveUpdate optionalAttrs boolToString mkOrder
+    sort filterAttrs concatMapStringsSep concatStrings mkIf
+    optionalString optionals mkDefault literalExpression hasSuffix
+    foldl' isAttrs filter attrNames elem literalDocBook
+    maintainers;
+
+  inherit (builtins) match typeOf;
 in
 {
-  options.services.keycloak = {
-
-    enable = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      example = true;
-      description = ''
-        Whether to enable the Keycloak identity and access management
-        server.
-      '';
-    };
-
-    bindAddress = lib.mkOption {
-      type = lib.types.str;
-      default = "\${jboss.bind.address:0.0.0.0}";
-      example = "127.0.0.1";
-      description = ''
-        On which address Keycloak should accept new connections.
+  options.services.keycloak =
+    let
+      inherit (types) bool str nullOr attrsOf path enum anything
+        package port;
+    in
+    {
+      enable = mkOption {
+        type = bool;
+        default = false;
+        example = true;
+        description = ''
+          Whether to enable the Keycloak identity and access management
+          server.
+        '';
+      };
 
-        A special syntax can be used to allow command line Java system
-        properties to override the value: ''${property.name:value}
-      '';
-    };
+      bindAddress = mkOption {
+        type = str;
+        default = "\${jboss.bind.address:0.0.0.0}";
+        example = "127.0.0.1";
+        description = ''
+          On which address Keycloak should accept new connections.
 
-    httpPort = lib.mkOption {
-      type = lib.types.str;
-      default = "\${jboss.http.port:80}";
-      example = "8080";
-      description = ''
-        On which port Keycloak should listen for new HTTP connections.
+          A special syntax can be used to allow command line Java system
+          properties to override the value: ''${property.name:value}
+        '';
+      };
 
-        A special syntax can be used to allow command line Java system
-        properties to override the value: ''${property.name:value}
-      '';
-    };
+      httpPort = mkOption {
+        type = str;
+        default = "\${jboss.http.port:80}";
+        example = "8080";
+        description = ''
+          On which port Keycloak should listen for new HTTP connections.
 
-    httpsPort = lib.mkOption {
-      type = lib.types.str;
-      default = "\${jboss.https.port:443}";
-      example = "8443";
-      description = ''
-        On which port Keycloak should listen for new HTTPS connections.
+          A special syntax can be used to allow command line Java system
+          properties to override the value: ''${property.name:value}
+        '';
+      };
 
-        A special syntax can be used to allow command line Java system
-        properties to override the value: ''${property.name:value}
-      '';
-    };
+      httpsPort = mkOption {
+        type = str;
+        default = "\${jboss.https.port:443}";
+        example = "8443";
+        description = ''
+          On which port Keycloak should listen for new HTTPS connections.
 
-    frontendUrl = lib.mkOption {
-      type = lib.types.str;
-      apply = x: if lib.hasSuffix "/" x then x else x + "/";
-      example = "keycloak.example.com/auth";
-      description = ''
-        The public URL used as base for all frontend requests. Should
-        normally include a trailing <literal>/auth</literal>.
-
-        See <link xlink:href="https://www.keycloak.org/docs/latest/server_installation/#_hostname">the
-        Hostname section of the Keycloak server installation
-        manual</link> for more information.
-      '';
-    };
+          A special syntax can be used to allow command line Java system
+          properties to override the value: ''${property.name:value}
+        '';
+      };
 
-    forceBackendUrlToFrontendUrl = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      example = true;
-      description = ''
-        Whether Keycloak should force all requests to go through the
-        frontend URL configured in <xref
-        linkend="opt-services.keycloak.frontendUrl" />. By default,
-        Keycloak allows backend requests to instead use its local
-        hostname or IP address and may also advertise it to clients
-        through its OpenID Connect Discovery endpoint.
-
-        See <link
-        xlink:href="https://www.keycloak.org/docs/latest/server_installation/#_hostname">the
-        Hostname section of the Keycloak server installation
-        manual</link> for more information.
-      '';
-    };
+      frontendUrl = mkOption {
+        type = str;
+        apply = x:
+          if x == "" || hasSuffix "/" x then
+            x
+          else
+            x + "/";
+        example = "keycloak.example.com/auth";
+        description = ''
+          The public URL used as base for all frontend requests. Should
+          normally include a trailing <literal>/auth</literal>.
 
-    sslCertificate = lib.mkOption {
-      type = lib.types.nullOr lib.types.path;
-      default = null;
-      example = "/run/keys/ssl_cert";
-      description = ''
-        The path to a PEM formatted certificate to use for TLS/SSL
-        connections.
+          See <link xlink:href="https://www.keycloak.org/docs/latest/server_installation/#_hostname">the
+          Hostname section of the Keycloak server installation
+          manual</link> for more information.
+        '';
+      };
 
-        This should be a string, not a Nix path, since Nix paths are
-        copied into the world-readable Nix store.
-      '';
-    };
+      forceBackendUrlToFrontendUrl = mkOption {
+        type = bool;
+        default = false;
+        example = true;
+        description = ''
+          Whether Keycloak should force all requests to go through the
+          frontend URL configured in <xref
+          linkend="opt-services.keycloak.frontendUrl" />. By default,
+          Keycloak allows backend requests to instead use its local
+          hostname or IP address and may also advertise it to clients
+          through its OpenID Connect Discovery endpoint.
+
+          See <link
+          xlink:href="https://www.keycloak.org/docs/latest/server_installation/#_hostname">the
+          Hostname section of the Keycloak server installation
+          manual</link> for more information.
+        '';
+      };
 
-    sslCertificateKey = lib.mkOption {
-      type = lib.types.nullOr lib.types.path;
-      default = null;
-      example = "/run/keys/ssl_key";
-      description = ''
-        The path to a PEM formatted private key to use for TLS/SSL
-        connections.
+      sslCertificate = mkOption {
+        type = nullOr path;
+        default = null;
+        example = "/run/keys/ssl_cert";
+        description = ''
+          The path to a PEM formatted certificate to use for TLS/SSL
+          connections.
 
-        This should be a string, not a Nix path, since Nix paths are
-        copied into the world-readable Nix store.
-      '';
-    };
+          This should be a string, not a Nix path, since Nix paths are
+          copied into the world-readable Nix store.
+        '';
+      };
 
-    database = {
-      type = lib.mkOption {
-        type = lib.types.enum [ "mysql" "postgresql" ];
-        default = "postgresql";
-        example = "mysql";
+      sslCertificateKey = mkOption {
+        type = nullOr path;
+        default = null;
+        example = "/run/keys/ssl_key";
         description = ''
-          The type of database Keycloak should connect to.
+          The path to a PEM formatted private key to use for TLS/SSL
+          connections.
+
+          This should be a string, not a Nix path, since Nix paths are
+          copied into the world-readable Nix store.
         '';
       };
 
-      host = lib.mkOption {
-        type = lib.types.str;
-        default = "localhost";
+      plugins = lib.mkOption {
+        type = lib.types.listOf lib.types.path;
+        default = [];
         description = ''
-          Hostname of the database to connect to.
+          Keycloak plugin jar, ear files or derivations with them
         '';
       };
 
-      port =
-        let
-          dbPorts = {
-            postgresql = 5432;
-            mysql = 3306;
-          };
-        in
-          lib.mkOption {
-            type = lib.types.port;
+      database = {
+        type = mkOption {
+          type = enum [ "mysql" "postgresql" ];
+          default = "postgresql";
+          example = "mysql";
+          description = ''
+            The type of database Keycloak should connect to.
+          '';
+        };
+
+        host = mkOption {
+          type = str;
+          default = "localhost";
+          description = ''
+            Hostname of the database to connect to.
+          '';
+        };
+
+        port =
+          let
+            dbPorts = {
+              postgresql = 5432;
+              mysql = 3306;
+            };
+          in
+          mkOption {
+            type = port;
             default = dbPorts.${cfg.database.type};
+            defaultText = literalDocBook "default port of selected database";
             description = ''
               Port of the database to connect to.
             '';
           };
 
-      useSSL = lib.mkOption {
-        type = lib.types.bool;
-        default = cfg.database.host != "localhost";
-        description = ''
-          Whether the database connection should be secured by SSL /
-          TLS.
-        '';
-      };
+        useSSL = mkOption {
+          type = bool;
+          default = cfg.database.host != "localhost";
+          defaultText = literalExpression ''config.${opt.database.host} != "localhost"'';
+          description = ''
+            Whether the database connection should be secured by SSL /
+            TLS.
+          '';
+        };
 
-      caCert = lib.mkOption {
-        type = lib.types.nullOr lib.types.path;
-        default = null;
-        description = ''
-          The SSL / TLS CA certificate that verifies the identity of the
-          database server.
+        caCert = mkOption {
+          type = nullOr path;
+          default = null;
+          description = ''
+            The SSL / TLS CA certificate that verifies the identity of the
+            database server.
 
-          Required when PostgreSQL is used and SSL is turned on.
+            Required when PostgreSQL is used and SSL is turned on.
 
-          For MySQL, if left at <literal>null</literal>, the default
-          Java keystore is used, which should suffice if the server
-          certificate is issued by an official CA.
-        '';
+            For MySQL, if left at <literal>null</literal>, the default
+            Java keystore is used, which should suffice if the server
+            certificate is issued by an official CA.
+          '';
+        };
+
+        createLocally = mkOption {
+          type = bool;
+          default = true;
+          description = ''
+            Whether a database should be automatically created on the
+            local host. Set this to false if you plan on provisioning a
+            local database yourself. This has no effect if
+            services.keycloak.database.host is customized.
+          '';
+        };
+
+        username = mkOption {
+          type = str;
+          default = "keycloak";
+          description = ''
+            Username to use when connecting to an external or manually
+            provisioned database; has no effect when a local database is
+            automatically provisioned.
+
+            To use this with a local database, set <xref
+            linkend="opt-services.keycloak.database.createLocally" /> to
+            <literal>false</literal> and create the database and user
+            manually. The database should be called
+            <literal>keycloak</literal>.
+          '';
+        };
+
+        passwordFile = mkOption {
+          type = path;
+          example = "/run/keys/db_password";
+          description = ''
+            File containing the database password.
+
+            This should be a string, not a Nix path, since Nix paths are
+            copied into the world-readable Nix store.
+          '';
+        };
       };
 
-      createLocally = lib.mkOption {
-        type = lib.types.bool;
-        default = true;
+      package = mkOption {
+        type = package;
+        default = pkgs.keycloak;
+        defaultText = literalExpression "pkgs.keycloak";
         description = ''
-          Whether a database should be automatically created on the
-          local host. Set this to false if you plan on provisioning a
-          local database yourself. This has no effect if
-          services.keycloak.database.host is customized.
+          Keycloak package to use.
         '';
       };
 
-      username = lib.mkOption {
-        type = lib.types.str;
-        default = "keycloak";
+      initialAdminPassword = mkOption {
+        type = str;
+        default = "changeme";
         description = ''
-          Username to use when connecting to an external or manually
-          provisioned database; has no effect when a local database is
-          automatically provisioned.
-
-          To use this with a local database, set <xref
-          linkend="opt-services.keycloak.database.createLocally" /> to
-          <literal>false</literal> and create the database and user
-          manually. The database should be called
-          <literal>keycloak</literal>.
+          Initial password set for the <literal>admin</literal>
+          user. The password is not stored safely and should be changed
+          immediately in the admin panel.
         '';
       };
 
-      passwordFile = lib.mkOption {
-        type = lib.types.path;
-        example = "/run/keys/db_password";
+      themes = mkOption {
+        type = attrsOf package;
+        default = { };
         description = ''
-          File containing the database password.
+          Additional theme packages for Keycloak. Each theme is linked into
+          subdirectory with a corresponding attribute name.
 
-          This should be a string, not a Nix path, since Nix paths are
-          copied into the world-readable Nix store.
+          Theme packages consist of several subdirectories which provide
+          different theme types: for example, <literal>account</literal>,
+          <literal>login</literal> etc. After adding a theme to this option you
+          can select it by its name in Keycloak administration console.
         '';
       };
-    };
-
-    package = lib.mkOption {
-      type = lib.types.package;
-      default = pkgs.keycloak;
-      defaultText = lib.literalExpression "pkgs.keycloak";
-      description = ''
-        Keycloak package to use.
-      '';
-    };
-
-    plugins = lib.mkOption {
-      type = lib.types.listOf lib.types.path;
-      default = [];
-      description = ''
-        Keycloak plugin jar, ear files or derivations with them
-      '';
-    };
-
-    initialAdminPassword = lib.mkOption {
-      type = lib.types.str;
-      default = "changeme";
-      description = ''
-        Initial password set for the <literal>admin</literal>
-        user. The password is not stored safely and should be changed
-        immediately in the admin panel.
-      '';
-    };
 
-    extraConfig = lib.mkOption {
-      type = lib.types.attrs;
-      default = { };
-      example = lib.literalExpression ''
-        {
-          "subsystem=keycloak-server" = {
-            "spi=hostname" = {
-              "provider=default" = null;
-              "provider=fixed" = {
-                enabled = true;
-                properties.hostname = "keycloak.example.com";
+      extraConfig = mkOption {
+        type = attrsOf anything;
+        default = { };
+        example = literalExpression ''
+          {
+            "subsystem=keycloak-server" = {
+              "spi=hostname" = {
+                "provider=default" = null;
+                "provider=fixed" = {
+                  enabled = true;
+                  properties.hostname = "keycloak.example.com";
+                };
+                default-provider = "fixed";
               };
-              default-provider = "fixed";
             };
-          };
-        }
-      '';
-      description = ''
-        Additional Keycloak configuration options to set in
-        <literal>standalone.xml</literal>.
-
-        Options are expressed as a Nix attribute set which matches the
-        structure of the jboss-cli configuration. The configuration is
-        effectively overlayed on top of the default configuration
-        shipped with Keycloak. To remove existing nodes and undefine
-        attributes from the default configuration, set them to
-        <literal>null</literal>.
-
-        The example configuration does the equivalent of the following
-        script, which removes the hostname provider
-        <literal>default</literal>, adds the deprecated hostname
-        provider <literal>fixed</literal> and defines it the default:
-
-        <programlisting>
-        /subsystem=keycloak-server/spi=hostname/provider=default:remove()
-        /subsystem=keycloak-server/spi=hostname/provider=fixed:add(enabled = true, properties = { hostname = "keycloak.example.com" })
-        /subsystem=keycloak-server/spi=hostname:write-attribute(name=default-provider, value="fixed")
-        </programlisting>
-
-        You can discover available options by using the <link
-        xlink:href="http://docs.wildfly.org/21/Admin_Guide.html#Command_Line_Interface">jboss-cli.sh</link>
-        program and by referring to the <link
-        xlink:href="https://www.keycloak.org/docs/latest/server_installation/index.html">Keycloak
-        Server Installation and Configuration Guide</link>.
-      '';
-    };
+          }
+        '';
+        description = ''
+          Additional Keycloak configuration options to set in
+          <literal>standalone.xml</literal>.
+
+          Options are expressed as a Nix attribute set which matches the
+          structure of the jboss-cli configuration. The configuration is
+          effectively overlayed on top of the default configuration
+          shipped with Keycloak. To remove existing nodes and undefine
+          attributes from the default configuration, set them to
+          <literal>null</literal>.
+
+          The example configuration does the equivalent of the following
+          script, which removes the hostname provider
+          <literal>default</literal>, adds the deprecated hostname
+          provider <literal>fixed</literal> and defines it the default:
+
+          <programlisting>
+          /subsystem=keycloak-server/spi=hostname/provider=default:remove()
+          /subsystem=keycloak-server/spi=hostname/provider=fixed:add(enabled = true, properties = { hostname = "keycloak.example.com" })
+          /subsystem=keycloak-server/spi=hostname:write-attribute(name=default-provider, value="fixed")
+          </programlisting>
+
+          You can discover available options by using the <link
+          xlink:href="http://docs.wildfly.org/21/Admin_Guide.html#Command_Line_Interface">jboss-cli.sh</link>
+          program and by referring to the <link
+          xlink:href="https://www.keycloak.org/docs/latest/server_installation/index.html">Keycloak
+          Server Installation and Configuration Guide</link>.
+        '';
+      };
 
-  };
+    };
 
   config =
     let
@@ -290,28 +324,58 @@ in
       createLocalPostgreSQL = databaseActuallyCreateLocally && cfg.database.type == "postgresql";
       createLocalMySQL = databaseActuallyCreateLocally && cfg.database.type == "mysql";
 
-      mySqlCaKeystore = pkgs.runCommand "mysql-ca-keystore" {} ''
+      mySqlCaKeystore = pkgs.runCommand "mysql-ca-keystore" { } ''
         ${pkgs.jre}/bin/keytool -importcert -trustcacerts -alias MySQLCACert -file ${cfg.database.caCert} -keystore $out -storepass notsosecretpassword -noprompt
       '';
 
-      keycloakConfig' = builtins.foldl' lib.recursiveUpdate {
-        "interface=public".inet-address = cfg.bindAddress;
-        "socket-binding-group=standard-sockets"."socket-binding=http".port = cfg.httpPort;
-        "subsystem=keycloak-server"."spi=hostname" = {
-          "provider=default" = {
-            enabled = true;
-            properties = {
-              inherit (cfg) frontendUrl forceBackendUrlToFrontendUrl;
+      # Both theme and theme type directories need to be actual directories in one hierarchy to pass Keycloak checks.
+      themesBundle = pkgs.runCommand "keycloak-themes" { } ''
+        linkTheme() {
+          theme="$1"
+          name="$2"
+
+          mkdir "$out/$name"
+          for typeDir in "$theme"/*; do
+            if [ -d "$typeDir" ]; then
+              type="$(basename "$typeDir")"
+              mkdir "$out/$name/$type"
+              for file in "$typeDir"/*; do
+                ln -sn "$file" "$out/$name/$type/$(basename "$file")"
+              done
+            fi
+          done
+        }
+
+        mkdir -p "$out"
+        for theme in ${cfg.package}/themes/*; do
+          if [ -d "$theme" ]; then
+            linkTheme "$theme" "$(basename "$theme")"
+          fi
+        done
+
+        ${concatStringsSep "\n" (mapAttrsToList (name: theme: "linkTheme ${theme} ${escapeShellArg name}") cfg.themes)}
+      '';
+
+      keycloakConfig' = foldl' recursiveUpdate
+        {
+          "interface=public".inet-address = cfg.bindAddress;
+          "socket-binding-group=standard-sockets"."socket-binding=http".port = cfg.httpPort;
+          "subsystem=keycloak-server" = {
+            "spi=hostname"."provider=default" = {
+              enabled = true;
+              properties = {
+                inherit (cfg) frontendUrl forceBackendUrlToFrontendUrl;
+              };
             };
+            "theme=defaults".dir = toString themesBundle;
           };
-        };
-        "subsystem=datasources"."data-source=KeycloakDS" = {
-          max-pool-size = "20";
-          user-name = if databaseActuallyCreateLocally then "keycloak" else cfg.database.username;
-          password = "@db-password@";
-        };
-      } [
-        (lib.optionalAttrs (cfg.database.type == "postgresql") {
+          "subsystem=datasources"."data-source=KeycloakDS" = {
+            max-pool-size = "20";
+            user-name = if databaseActuallyCreateLocally then "keycloak" else cfg.database.username;
+            password = "@db-password@";
+          };
+        } [
+        (optionalAttrs (cfg.database.type == "postgresql") {
           "subsystem=datasources" = {
             "jdbc-driver=postgresql" = {
               driver-module-name = "org.postgresql";
@@ -319,16 +383,16 @@ in
               driver-xa-datasource-class-name = "org.postgresql.xa.PGXADataSource";
             };
             "data-source=KeycloakDS" = {
-              connection-url = "jdbc:postgresql://${cfg.database.host}:${builtins.toString cfg.database.port}/keycloak";
+              connection-url = "jdbc:postgresql://${cfg.database.host}:${toString cfg.database.port}/keycloak";
               driver-name = "postgresql";
-              "connection-properties=ssl".value = lib.boolToString cfg.database.useSSL;
-            } // (lib.optionalAttrs (cfg.database.caCert != null) {
+              "connection-properties=ssl".value = boolToString cfg.database.useSSL;
+            } // (optionalAttrs (cfg.database.caCert != null) {
               "connection-properties=sslrootcert".value = cfg.database.caCert;
               "connection-properties=sslmode".value = "verify-ca";
             });
           };
         })
-        (lib.optionalAttrs (cfg.database.type == "mysql") {
+        (optionalAttrs (cfg.database.type == "mysql") {
           "subsystem=datasources" = {
             "jdbc-driver=mysql" = {
               driver-module-name = "com.mysql";
@@ -336,28 +400,40 @@ in
               driver-class-name = "com.mysql.jdbc.Driver";
             };
             "data-source=KeycloakDS" = {
-              connection-url = "jdbc:mysql://${cfg.database.host}:${builtins.toString cfg.database.port}/keycloak";
+              connection-url = "jdbc:mysql://${cfg.database.host}:${toString cfg.database.port}/keycloak";
               driver-name = "mysql";
-              "connection-properties=useSSL".value = lib.boolToString cfg.database.useSSL;
-              "connection-properties=requireSSL".value = lib.boolToString cfg.database.useSSL;
-              "connection-properties=verifyServerCertificate".value = lib.boolToString cfg.database.useSSL;
+              "connection-properties=useSSL".value = boolToString cfg.database.useSSL;
+              "connection-properties=requireSSL".value = boolToString cfg.database.useSSL;
+              "connection-properties=verifyServerCertificate".value = boolToString cfg.database.useSSL;
               "connection-properties=characterEncoding".value = "UTF-8";
               valid-connection-checker-class-name = "org.jboss.jca.adapters.jdbc.extensions.mysql.MySQLValidConnectionChecker";
               validate-on-match = true;
               exception-sorter-class-name = "org.jboss.jca.adapters.jdbc.extensions.mysql.MySQLExceptionSorter";
-            } // (lib.optionalAttrs (cfg.database.caCert != null) {
+            } // (optionalAttrs (cfg.database.caCert != null) {
               "connection-properties=trustCertificateKeyStoreUrl".value = "file:${mySqlCaKeystore}";
               "connection-properties=trustCertificateKeyStorePassword".value = "notsosecretpassword";
             });
           };
         })
-        (lib.optionalAttrs (cfg.sslCertificate != null && cfg.sslCertificateKey != null) {
+        (optionalAttrs (cfg.sslCertificate != null && cfg.sslCertificateKey != null) {
           "socket-binding-group=standard-sockets"."socket-binding=https".port = cfg.httpsPort;
-          "core-service=management"."security-realm=UndertowRealm"."server-identity=ssl" = {
-            keystore-path = "/run/keycloak/ssl/certificate_private_key_bundle.p12";
-            keystore-password = "notsosecretpassword";
+          "subsystem=elytron" = mkOrder 900 {
+            "key-store=httpsKS" = mkOrder 900 {
+              path = "/run/keycloak/ssl/certificate_private_key_bundle.p12";
+              credential-reference.clear-text = "notsosecretpassword";
+              type = "JKS";
+            };
+            "key-manager=httpsKM" = mkOrder 901 {
+              key-store = "httpsKS";
+              credential-reference.clear-text = "notsosecretpassword";
+            };
+            "server-ssl-context=httpsSSC" = mkOrder 902 {
+              key-manager = "httpsKM";
+            };
+          };
+          "subsystem=undertow" = mkOrder 901 {
+            "server=default-server"."https-listener=https".ssl-context = "httpsSSC";
           };
-          "subsystem=undertow"."server=default-server"."https-listener=https".security-realm = "UndertowRealm";
         })
         cfg.extraConfig
       ];
@@ -446,41 +522,42 @@ in
               # with `expression` to evaluate.
               prefixExpression = string:
                 let
-                  match = (builtins.match ''"\$\{.*}"'' string);
+                  matchResult = match ''"\$\{.*}"'' string;
                 in
-                  if match != null then
-                    "expression " + string
-                  else
-                    string;
+                if matchResult != null then
+                  "expression " + string
+                else
+                  string;
 
               writeAttribute = attribute: value:
                 let
-                  type = builtins.typeOf value;
+                  type = typeOf value;
                 in
-                  if type == "set" then
-                    let
-                      names = builtins.attrNames value;
-                    in
-                      builtins.foldl' (text: name: text + (writeAttribute "${attribute}.${name}" value.${name})) "" names
-                  else if value == null then ''
-                    if (outcome == success) of ${path}:read-attribute(name="${attribute}")
-                        ${path}:undefine-attribute(name="${attribute}")
+                if type == "set" then
+                  let
+                    names = attrNames value;
+                  in
+                  foldl' (text: name: text + (writeAttribute "${attribute}.${name}" value.${name})) "" names
+                else if value == null then ''
+                  if (outcome == success) of ${path}:read-attribute(name="${attribute}")
+                      ${path}:undefine-attribute(name="${attribute}")
+                  end-if
+                ''
+                else if elem type [ "string" "path" "bool" ] then
+                  let
+                    value' = if type == "bool" then boolToString value else ''"${value}"'';
+                  in
+                  ''
+                    if (result != ${prefixExpression value'}) of ${path}:read-attribute(name="${attribute}")
+                      ${path}:write-attribute(name=${attribute}, value=${value'})
                     end-if
                   ''
-                  else if builtins.elem type [ "string" "path" "bool" ] then
-                    let
-                      value' = if type == "bool" then lib.boolToString value else ''"${value}"'';
-                    in ''
-                      if (result != ${prefixExpression value'}) of ${path}:read-attribute(name="${attribute}")
-                        ${path}:write-attribute(name=${attribute}, value=${value'})
-                      end-if
-                    ''
-                  else throw "Unsupported type '${type}' for path '${path}'!";
+                else throw "Unsupported type '${type}' for path '${path}'!";
             in
-              lib.concatStrings
-                (lib.mapAttrsToList
-                  (attribute: value: (writeAttribute attribute value))
-                  set);
+            concatStrings
+              (mapAttrsToList
+                (attribute: value: (writeAttribute attribute value))
+                set);
 
 
           /* Produces an argument list for the JBoss `add()` function,
@@ -503,98 +580,108 @@ in
             let
               makeArg = attribute: value:
                 let
-                  type = builtins.typeOf value;
+                  type = typeOf value;
                 in
-                  if type == "set" then
-                    "${attribute} = { " + (makeArgList value) + " }"
-                  else if builtins.elem type [ "string" "path" "bool" ] then
-                    "${attribute} = ${if type == "bool" then lib.boolToString value else ''"${value}"''}"
-                  else if value == null then
-                    ""
-                  else
-                    throw "Unsupported type '${type}' for attribute '${attribute}'!";
+                if type == "set" then
+                  "${attribute} = { " + (makeArgList value) + " }"
+                else if elem type [ "string" "path" "bool" ] then
+                  "${attribute} = ${if type == "bool" then boolToString value else ''"${value}"''}"
+                else if value == null then
+                  ""
+                else
+                  throw "Unsupported type '${type}' for attribute '${attribute}'!";
+
             in
-              lib.concatStringsSep ", " (lib.mapAttrsToList makeArg set);
+            concatStringsSep ", " (mapAttrsToList makeArg set);
 
 
-          /* Recurses into the `attrs` attrset, beginning at the path
-             resolved from `state.path ++ node`; if `node` is `null`,
-             starts from `state.path`. Only subattrsets that are JBoss
-             paths, i.e. follows the `key=value` format, are recursed
+          /* Recurses into the `nodeValue` attrset. Only subattrsets that
+             are JBoss paths, i.e. follows the `key=value` format, are recursed
              into - the rest are considered JBoss attributes / maps.
           */
-          recurse = state: node:
+          recurse = nodePath: nodeValue:
             let
-              path = state.path ++ (lib.optional (node != null) node);
+              nodeContent =
+                if isAttrs nodeValue && nodeValue._type or "" == "order" then
+                  nodeValue.content
+                else
+                  nodeValue;
               isPath = name:
                 let
-                  value = lib.getAttrFromPath (path ++ [ name ]) attrs;
+                  value = nodeContent.${name};
                 in
-                  if (builtins.match ".*([=]).*" name) == [ "=" ] then
-                    if builtins.isAttrs value || value == null then
-                      true
-                    else
-                      throw "Parsing path '${lib.concatStringsSep "." (path ++ [ name ])}' failed: JBoss attributes cannot contain '='!"
+                if (match ".*([=]).*" name) == [ "=" ] then
+                  if isAttrs value || value == null then
+                    true
                   else
-                    false;
-              jbossPath = "/" + (lib.concatStringsSep "/" path);
-              nodeValue = lib.getAttrFromPath path attrs;
-              children = if !builtins.isAttrs nodeValue then {} else nodeValue;
-              subPaths = builtins.filter isPath (builtins.attrNames children);
-              jbossAttrs = lib.filterAttrs (name: _: !(isPath name)) children;
-            in
-              state // {
-                text = state.text + (
-                  if nodeValue != null then ''
+                    throw "Parsing path '${concatStringsSep "." (nodePath ++ [ name ])}' failed: JBoss attributes cannot contain '='!"
+                else
+                  false;
+              jbossPath = "/" + concatStringsSep "/" nodePath;
+              children = if !isAttrs nodeContent then { } else nodeContent;
+              subPaths = filter isPath (attrNames children);
+              getPriority = name:
+                let
+                  value = children.${name};
+                in
+                if value._type or "" == "order" then value.priority else 1000;
+              orderedSubPaths = sort (a: b: getPriority a < getPriority b) subPaths;
+              jbossAttrs = filterAttrs (name: _: !(isPath name)) children;
+              text =
+                if nodeContent != null then
+                  ''
                     if (outcome != success) of ${jbossPath}:read-resource()
                         ${jbossPath}:add(${makeArgList jbossAttrs})
                     end-if
-                  '' + (writeAttributes jbossPath jbossAttrs)
-                  else ''
+                  '' + writeAttributes jbossPath jbossAttrs
+                else
+                  ''
                     if (outcome == success) of ${jbossPath}:read-resource()
                         ${jbossPath}:remove()
                     end-if
-                  '') + (builtins.foldl' recurse { text = ""; inherit path; } subPaths).text;
-              };
+                  '';
+            in
+            text + concatMapStringsSep "\n" (name: recurse (nodePath ++ [ name ]) children.${name}) orderedSubPaths;
         in
-          (recurse { text = ""; path = []; } null).text;
-
+        recurse [ ] attrs;
 
       jbossCliScript = pkgs.writeText "jboss-cli-script" (mkJbossScript keycloakConfig');
 
-      keycloakConfig = pkgs.runCommand "keycloak-config" {
-        nativeBuildInputs = [ cfg.package ];
-      } ''
-        export JBOSS_BASE_DIR="$(pwd -P)";
-        export JBOSS_MODULEPATH="${cfg.package}/modules";
-        export JBOSS_LOG_DIR="$JBOSS_BASE_DIR/log";
+      keycloakConfig = pkgs.runCommand "keycloak-config"
+        {
+          nativeBuildInputs = [ cfg.package ];
+        }
+        ''
+          export JBOSS_BASE_DIR="$(pwd -P)";
+          export JBOSS_MODULEPATH="${cfg.package}/modules";
+          export JBOSS_LOG_DIR="$JBOSS_BASE_DIR/log";
 
-        cp -r ${cfg.package}/standalone/configuration .
-        chmod -R u+rwX ./configuration
+          cp -r ${cfg.package}/standalone/configuration .
+          chmod -R u+rwX ./configuration
 
-        mkdir -p {deployments,ssl}
+          mkdir -p {deployments,ssl}
 
-        standalone.sh&
+          standalone.sh&
 
-        attempt=1
-        max_attempts=30
-        while ! jboss-cli.sh --connect ':read-attribute(name=server-state)'; do
-            if [[ "$attempt" == "$max_attempts" ]]; then
-                echo "ERROR: Could not connect to Keycloak after $attempt attempts! Failing.." >&2
-                exit 1
-            fi
-            echo "Keycloak not fully started yet, retrying.. ($attempt/$max_attempts)"
-            sleep 1
-            (( attempt++ ))
-        done
+          attempt=1
+          max_attempts=30
+          while ! jboss-cli.sh --connect ':read-attribute(name=server-state)'; do
+              if [[ "$attempt" == "$max_attempts" ]]; then
+                  echo "ERROR: Could not connect to Keycloak after $attempt attempts! Failing.." >&2
+                  exit 1
+              fi
+              echo "Keycloak not fully started yet, retrying.. ($attempt/$max_attempts)"
+              sleep 1
+              (( attempt++ ))
+          done
 
-        jboss-cli.sh --connect --file=${jbossCliScript} --echo-command
+          jboss-cli.sh --connect --file=${jbossCliScript} --echo-command
 
-        cp configuration/standalone.xml $out
-      '';
+          cp configuration/standalone.xml $out
+        '';
     in
-      lib.mkIf cfg.enable {
-
+    mkIf cfg.enable
+      {
         assertions = [
           {
             assertion = (cfg.database.useSSL && cfg.database.type == "postgresql") -> (cfg.database.caCert != null);
@@ -604,7 +691,7 @@ in
 
         environment.systemPackages = [ cfg.package ];
 
-        systemd.services.keycloakPostgreSQLInit = lib.mkIf createLocalPostgreSQL {
+        systemd.services.keycloakPostgreSQLInit = mkIf createLocalPostgreSQL {
           after = [ "postgresql.service" ];
           before = [ "keycloak.service" ];
           bindsTo = [ "postgresql.service" ];
@@ -614,6 +701,7 @@ in
             RemainAfterExit = true;
             User = "postgres";
             Group = "postgres";
+            LoadCredential = [ "db_password:${cfg.database.passwordFile}" ];
           };
           script = ''
             set -o errexit -o pipefail -o nounset -o errtrace
@@ -622,13 +710,14 @@ in
             create_role="$(mktemp)"
             trap 'rm -f "$create_role"' ERR EXIT
 
-            echo "CREATE ROLE keycloak WITH LOGIN PASSWORD '$(<'${cfg.database.passwordFile}')' CREATEDB" > "$create_role"
+            db_password="$(<"$CREDENTIALS_DIRECTORY/db_password")"
+            echo "CREATE ROLE keycloak WITH LOGIN PASSWORD '$db_password' CREATEDB" > "$create_role"
             psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='keycloak'" | grep -q 1 || psql -tA --file="$create_role"
             psql -tAc "SELECT 1 FROM pg_database WHERE datname = 'keycloak'" | grep -q 1 || psql -tAc 'CREATE DATABASE "keycloak" OWNER "keycloak"'
           '';
         };
 
-        systemd.services.keycloakMySQLInit = lib.mkIf createLocalMySQL {
+        systemd.services.keycloakMySQLInit = mkIf createLocalMySQL {
           after = [ "mysql.service" ];
           before = [ "keycloak.service" ];
           bindsTo = [ "mysql.service" ];
@@ -638,14 +727,14 @@ in
             RemainAfterExit = true;
             User = config.services.mysql.user;
             Group = config.services.mysql.group;
+            LoadCredential = [ "db_password:${cfg.database.passwordFile}" ];
           };
           script = ''
             set -o errexit -o pipefail -o nounset -o errtrace
             shopt -s inherit_errexit
-
-            db_password="$(<'${cfg.database.passwordFile}')"
+            db_password="$(<"$CREDENTIALS_DIRECTORY/db_password")"
             ( echo "CREATE USER IF NOT EXISTS 'keycloak'@'localhost' IDENTIFIED BY '$db_password';"
-              echo "CREATE DATABASE keycloak CHARACTER SET utf8 COLLATE utf8_unicode_ci;"
+              echo "CREATE DATABASE IF NOT EXISTS keycloak CHARACTER SET utf8 COLLATE utf8_unicode_ci;"
               echo "GRANT ALL PRIVILEGES ON keycloak.* TO 'keycloak'@'localhost';"
             ) | mysql -N
           '';
@@ -655,13 +744,16 @@ in
           let
             databaseServices =
               if createLocalPostgreSQL then [
-                "keycloakPostgreSQLInit.service" "postgresql.service"
+                "keycloakPostgreSQLInit.service"
+                "postgresql.service"
               ]
               else if createLocalMySQL then [
-                "keycloakMySQLInit.service" "mysql.service"
+                "keycloakMySQLInit.service"
+                "mysql.service"
               ]
               else [ ];
-          in {
+          in
+          {
             after = databaseServices;
             bindsTo = databaseServices;
             wantedBy = [ "multi-user.target" ];
@@ -676,62 +768,16 @@ in
               JBOSS_MODULEPATH = "${cfg.package}/modules";
             };
             serviceConfig = {
-              ExecStartPre = let
-                startPreFullPrivileges = ''
-                  set -o errexit -o pipefail -o nounset -o errtrace
-                  shopt -s inherit_errexit
-
-                  umask u=rwx,g=,o=
-
-                  install_plugin() {
-                    if [ -d "$1" ]; then
-                      find "$1" -type f \( -iname \*.ear -o -iname \*.jar \) -exec install -m 0500 -o keycloak -g keycloak "{}" "/run/keycloak/deployments/" \;
-                    else
-                      install -m 0500 -o keycloak -g keycloak "$1" "/run/keycloak/deployments/"
-                    fi
-                  }
-
-                  install -T -m 0400 -o keycloak -g keycloak '${cfg.database.passwordFile}' /run/keycloak/secrets/db_password
-                '' + lib.optionalString (cfg.plugins != []) (lib.concatStringsSep "\n" (map (pl: "install_plugin ${lib.escapeShellArg pl}") cfg.plugins))
-                + lib.optionalString (cfg.sslCertificate != null && cfg.sslCertificateKey != null) ''
-
-                  install -T -m 0400 -o keycloak -g keycloak '${cfg.sslCertificate}' /run/keycloak/secrets/ssl_cert
-                  install -T -m 0400 -o keycloak -g keycloak '${cfg.sslCertificateKey}' /run/keycloak/secrets/ssl_key
-                '';
-                startPre = ''
-                  set -o errexit -o pipefail -o nounset -o errtrace
-                  shopt -s inherit_errexit
-
-                  umask u=rwx,g=,o=
-
-                  install -m 0600 ${cfg.package}/standalone/configuration/*.properties /run/keycloak/configuration
-                  install -T -m 0600 ${keycloakConfig} /run/keycloak/configuration/standalone.xml
-
-                  replace-secret '@db-password@' '/run/keycloak/secrets/db_password' /run/keycloak/configuration/standalone.xml
-
-                  export JAVA_OPTS=-Djboss.server.config.user.dir=/run/keycloak/configuration
-                  add-user-keycloak.sh -u admin -p '${cfg.initialAdminPassword}'
-                '' + lib.optionalString (cfg.sslCertificate != null && cfg.sslCertificateKey != null) ''
-                  pushd /run/keycloak/ssl/
-                  cat /run/keycloak/secrets/ssl_cert <(echo) \
-                      /run/keycloak/secrets/ssl_key <(echo) \
-                      /etc/ssl/certs/ca-certificates.crt \
-                      > allcerts.pem
-                  openssl pkcs12 -export -in /run/keycloak/secrets/ssl_cert -inkey /run/keycloak/secrets/ssl_key -chain \
-                                 -name "${cfg.frontendUrl}" -out certificate_private_key_bundle.p12 \
-                                 -CAfile allcerts.pem -passout pass:notsosecretpassword
-                  popd
-                '';
-              in [
-                "+${pkgs.writeShellScript "keycloak-start-pre-full-privileges" startPreFullPrivileges}"
-                "${pkgs.writeShellScript "keycloak-start-pre" startPre}"
+              LoadCredential = [
+                "db_password:${cfg.database.passwordFile}"
+              ] ++ optionals (cfg.sslCertificate != null && cfg.sslCertificateKey != null) [
+                "ssl_cert:${cfg.sslCertificate}"
+                "ssl_key:${cfg.sslCertificateKey}"
               ];
-              ExecStart = "${cfg.package}/bin/standalone.sh";
               User = "keycloak";
               Group = "keycloak";
               DynamicUser = true;
               RuntimeDirectory = map (p: "keycloak/" + p) [
-                "secrets"
                 "configuration"
                 "deployments"
                 "data"
@@ -743,13 +789,49 @@ in
               LogsDirectory = "keycloak";
               AmbientCapabilities = "CAP_NET_BIND_SERVICE";
             };
+            script = ''
+              set -o errexit -o pipefail -o nounset -o errtrace
+              shopt -s inherit_errexit
+
+              umask u=rwx,g=,o=
+
+              install_plugin() {
+                if [ -d "$1" ]; then
+                  find "$1" -type f \( -iname \*.ear -o -iname \*.jar \) -exec install -m 0500 -o keycloak -g keycloak "{}" "/run/keycloak/deployments/" \;
+                else
+                  install -m 0500 -o keycloak -g keycloak "$1" "/run/keycloak/deployments/"
+                fi
+              }
+
+              install -m 0600 ${cfg.package}/standalone/configuration/*.properties /run/keycloak/configuration
+              install -T -m 0600 ${keycloakConfig} /run/keycloak/configuration/standalone.xml
+
+              replace-secret '@db-password@' "$CREDENTIALS_DIRECTORY/db_password" /run/keycloak/configuration/standalone.xml
+
+              export JAVA_OPTS=-Djboss.server.config.user.dir=/run/keycloak/configuration
+              add-user-keycloak.sh -u admin -p '${cfg.initialAdminPassword}'
+            ''
+            + lib.optionalString (cfg.plugins != []) (lib.concatStringsSep "\n" (map (pl: "install_plugin ${lib.escapeShellArg pl}") cfg.plugins))
+            + optionalString (cfg.sslCertificate != null && cfg.sslCertificateKey != null) ''
+              pushd /run/keycloak/ssl/
+              cat "$CREDENTIALS_DIRECTORY/ssl_cert" <(echo) \
+                  "$CREDENTIALS_DIRECTORY/ssl_key" <(echo) \
+                  /etc/ssl/certs/ca-certificates.crt \
+                  > allcerts.pem
+              openssl pkcs12 -export -in "$CREDENTIALS_DIRECTORY/ssl_cert" -inkey "$CREDENTIALS_DIRECTORY/ssl_key" -chain \
+                             -name "${cfg.frontendUrl}" -out certificate_private_key_bundle.p12 \
+                             -CAfile allcerts.pem -passout pass:notsosecretpassword
+              popd
+            '' + ''
+              ${cfg.package}/bin/standalone.sh
+            '';
           };
 
-        services.postgresql.enable = lib.mkDefault createLocalPostgreSQL;
-        services.mysql.enable = lib.mkDefault createLocalMySQL;
-        services.mysql.package = lib.mkIf createLocalMySQL pkgs.mariadb;
+        services.postgresql.enable = mkDefault createLocalPostgreSQL;
+        services.mysql.enable = mkDefault createLocalMySQL;
+        services.mysql.package = mkIf createLocalMySQL pkgs.mariadb;
       };
 
   meta.doc = ./keycloak.xml;
-  meta.maintainers = [ lib.maintainers.talyz ];
+  meta.maintainers = [ maintainers.talyz ];
 }
diff --git a/nixos/modules/services/web-apps/keycloak.xml b/nixos/modules/services/web-apps/keycloak.xml
index 7ba656c20f16..cb706932f48f 100644
--- a/nixos/modules/services/web-apps/keycloak.xml
+++ b/nixos/modules/services/web-apps/keycloak.xml
@@ -85,7 +85,12 @@
        The frontend URL is used as base for all frontend requests and
        must be configured through <xref linkend="opt-services.keycloak.frontendUrl" />.
        It should normally include a trailing <literal>/auth</literal>
-       (the default web context).
+       (the default web context). If you use a reverse proxy, you need
+       to set this option to <literal>""</literal>, so that frontend URL
+       is derived from HTTP headers. <literal>X-Forwarded-*</literal> headers
+       support also should be enabled, using <link
+       xlink:href="https://www.keycloak.org/docs/latest/server_installation/index.html#identifying-client-ip-addresses">
+       respective guidelines</link>.
      </para>
 
      <para>
@@ -131,6 +136,17 @@
      </warning>
    </section>
 
+   <section xml:id="module-services-keycloak-themes">
+     <title>Themes</title>
+     <para>
+        You can package custom themes and make them visible to Keycloak via
+        <xref linkend="opt-services.keycloak.themes" />
+        option. See the <link xlink:href="https://www.keycloak.org/docs/latest/server_development/#_themes">
+        Themes section of the Keycloak Server Development Guide</link>
+        and respective NixOS option description for more information.
+     </para>
+   </section>
+
    <section xml:id="module-services-keycloak-extra-config">
      <title>Additional configuration</title>
      <para>
diff --git a/nixos/modules/services/web-apps/mastodon.nix b/nixos/modules/services/web-apps/mastodon.nix
index 1e3c7e53c175..8208c85bfd70 100644
--- a/nixos/modules/services/web-apps/mastodon.nix
+++ b/nixos/modules/services/web-apps/mastodon.nix
@@ -92,6 +92,7 @@ let
 
   mastodonEnv = pkgs.writeShellScriptBin "mastodon-env" ''
     set -a
+    export RAILS_ROOT="${cfg.package}"
     source "${envFile}"
     source /var/lib/mastodon/.secrets_env
     eval -- "\$@"
diff --git a/nixos/modules/services/web-apps/matomo.nix b/nixos/modules/services/web-apps/matomo.nix
index eba55e7e9bef..c6d4ed6d39de 100644
--- a/nixos/modules/services/web-apps/matomo.nix
+++ b/nixos/modules/services/web-apps/matomo.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 with lib;
 let
   cfg = config.services.matomo;
@@ -12,10 +12,7 @@ let
   phpExecutionUnit = "phpfpm-${pool}";
   databaseService = "mysql.service";
 
-  fqdn =
-    let
-      join = hostName: domain: hostName + optionalString (domain != null) ".${domain}";
-     in join config.networking.hostName config.networking.domain;
+  fqdn = if config.networking.domain != null then config.networking.fqdn else config.networking.hostName;
 
 in {
   imports = [
@@ -81,9 +78,14 @@ in {
       hostname = mkOption {
         type = types.str;
         default = "${user}.${fqdn}";
+        defaultText = literalExpression ''
+          if config.${options.networking.domain} != null
+          then "${user}.''${config.${options.networking.fqdn}}"
+          else "${user}.''${config.${options.networking.hostName}}"
+        '';
         example = "matomo.yourdomain.org";
         description = ''
-          URL of the host, without https prefix. By default, this is ${user}.${fqdn}, but you may want to change it if you
+          URL of the host, without https prefix. You may want to change it if you
           run Matomo on a different URL than matomo.yourdomain.
         '';
       };
@@ -190,6 +192,7 @@ in {
             # Copy config folder
             chmod g+s "${dataDir}"
             cp -r "${cfg.package}/share/config" "${dataDir}/"
+            mkdir -p "${dataDir}/misc"
             chmod -R u+rwX,g+rwX,o-rwx "${dataDir}"
 
             # check whether user setup has already been done
diff --git a/nixos/modules/services/web-apps/mattermost.nix b/nixos/modules/services/web-apps/mattermost.nix
index f5c2c356afce..2901f307dc5a 100644
--- a/nixos/modules/services/web-apps/mattermost.nix
+++ b/nixos/modules/services/web-apps/mattermost.nix
@@ -6,23 +6,95 @@ let
 
   cfg = config.services.mattermost;
 
-  defaultConfig = builtins.fromJSON (builtins.replaceStrings [ "\\u0026" ] [ "&" ]
-    (readFile "${pkgs.mattermost}/config/config.json")
-  );
-
   database = "postgres://${cfg.localDatabaseUser}:${cfg.localDatabasePassword}@localhost:5432/${cfg.localDatabaseName}?sslmode=disable&connect_timeout=10";
 
-  mattermostConf = foldl recursiveUpdate defaultConfig
-    [ { ServiceSettings.SiteURL = cfg.siteUrl;
-        ServiceSettings.ListenAddress = cfg.listenAddress;
-        TeamSettings.SiteName = cfg.siteName;
-        SqlSettings.DriverName = "postgres";
-        SqlSettings.DataSource = database;
+  postgresPackage = config.services.postgresql.package;
+
+  createDb = {
+    statePath ? cfg.statePath,
+    localDatabaseUser ? cfg.localDatabaseUser,
+    localDatabasePassword ? cfg.localDatabasePassword,
+    localDatabaseName ? cfg.localDatabaseName,
+    useSudo ? true
+  }: ''
+    if ! test -e ${escapeShellArg "${statePath}/.db-created"}; then
+      ${lib.optionalString useSudo "${pkgs.sudo}/bin/sudo -u ${escapeShellArg config.services.postgresql.superUser} \\"}
+        ${postgresPackage}/bin/psql postgres -c \
+          "CREATE ROLE ${localDatabaseUser} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${localDatabasePassword}'"
+      ${lib.optionalString useSudo "${pkgs.sudo}/bin/sudo -u ${escapeShellArg config.services.postgresql.superUser} \\"}
+        ${postgresPackage}/bin/createdb \
+          --owner ${escapeShellArg localDatabaseUser} ${escapeShellArg localDatabaseName}
+      touch ${escapeShellArg "${statePath}/.db-created"}
+    fi
+  '';
+
+  mattermostPluginDerivations = with pkgs;
+    map (plugin: stdenv.mkDerivation {
+      name = "mattermost-plugin";
+      installPhase = ''
+        mkdir -p $out/share
+        cp ${plugin} $out/share/plugin.tar.gz
+      '';
+      dontUnpack = true;
+      dontPatch = true;
+      dontConfigure = true;
+      dontBuild = true;
+      preferLocalBuild = true;
+    }) cfg.plugins;
+
+  mattermostPlugins = with pkgs;
+    if mattermostPluginDerivations == [] then null
+    else stdenv.mkDerivation {
+      name = "${cfg.package.name}-plugins";
+      nativeBuildInputs = [
+        autoPatchelfHook
+      ] ++ mattermostPluginDerivations;
+      buildInputs = [
+        cfg.package
+      ];
+      installPhase = ''
+        mkdir -p $out/data/plugins
+        plugins=(${escapeShellArgs (map (plugin: "${plugin}/share/plugin.tar.gz") mattermostPluginDerivations)})
+        for plugin in "''${plugins[@]}"; do
+          hash="$(sha256sum "$plugin" | cut -d' ' -f1)"
+          mkdir -p "$hash"
+          tar -C "$hash" -xzf "$plugin"
+          autoPatchelf "$hash"
+          GZIP_OPT=-9 tar -C "$hash" -cvzf "$out/data/plugins/$hash.tar.gz" .
+          rm -rf "$hash"
+        done
+      '';
+
+      dontUnpack = true;
+      dontPatch = true;
+      dontConfigure = true;
+      dontBuild = true;
+      preferLocalBuild = true;
+    };
+
+  mattermostConfWithoutPlugins = recursiveUpdate
+    { ServiceSettings.SiteURL = cfg.siteUrl;
+      ServiceSettings.ListenAddress = cfg.listenAddress;
+      TeamSettings.SiteName = cfg.siteName;
+      SqlSettings.DriverName = "postgres";
+      SqlSettings.DataSource = database;
+      PluginSettings.Directory = "${cfg.statePath}/plugins/server";
+      PluginSettings.ClientDirectory = "${cfg.statePath}/plugins/client";
+    }
+    cfg.extraConfig;
+
+  mattermostConf = recursiveUpdate
+    mattermostConfWithoutPlugins
+    (
+      if mattermostPlugins == null then {}
+      else {
+        PluginSettings = {
+          Enable = true;
+        };
       }
-      cfg.extraConfig
-    ];
+    );
 
-  mattermostConfJSON = pkgs.writeText "mattermost-config-raw.json" (builtins.toJSON mattermostConf);
+  mattermostConfJSON = pkgs.writeText "mattermost-config.json" (builtins.toJSON mattermostConf);
 
 in
 
@@ -31,6 +103,13 @@ in
     services.mattermost = {
       enable = mkEnableOption "Mattermost chat server";
 
+      package = mkOption {
+        type = types.package;
+        default = pkgs.mattermost;
+        defaultText = "pkgs.mattermost";
+        description = "Mattermost derivation to use.";
+      };
+
       statePath = mkOption {
         type = types.str;
         default = "/var/lib/mattermost";
@@ -77,6 +156,16 @@ in
         '';
       };
 
+      preferNixConfig = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          If both mutableConfig and this option are set, the Nix configuration
+          will take precedence over any settings configured in the server
+          console.
+        '';
+      };
+
       extraConfig = mkOption {
         type = types.attrs;
         default = { };
@@ -85,6 +174,17 @@ in
         '';
       };
 
+      plugins = mkOption {
+        type = types.listOf (types.oneOf [types.path types.package]);
+        default = [];
+        example = "[ ./com.github.moussetc.mattermost.plugin.giphy-2.0.0.tar.gz ]";
+        description = ''
+          Plugins to add to the configuration. Overrides any installed if non-null.
+          This is a list of paths to .tar.gz files or derivations evaluating to
+          .tar.gz files.
+        '';
+      };
+
       localDatabaseCreate = mkOption {
         type = types.bool;
         default = true;
@@ -135,6 +235,12 @@ in
 
       matterircd = {
         enable = mkEnableOption "Mattermost IRC bridge";
+        package = mkOption {
+          type = types.package;
+          default = pkgs.matterircd;
+          defaultText = "pkgs.matterircd";
+          description = "matterircd derivation to use.";
+        };
         parameters = mkOption {
           type = types.listOf types.str;
           default = [ ];
@@ -167,7 +273,7 @@ in
       # The systemd service will fail to execute the preStart hook
       # if the WorkingDirectory does not exist
       system.activationScripts.mattermost = ''
-        mkdir -p ${cfg.statePath}
+        mkdir -p "${cfg.statePath}"
       '';
 
       systemd.services.mattermost = {
@@ -176,39 +282,41 @@ in
         after = [ "network.target" "postgresql.service" ];
 
         preStart = ''
-          mkdir -p ${cfg.statePath}/{data,config,logs}
-          ln -sf ${pkgs.mattermost}/{bin,fonts,i18n,templates,client} ${cfg.statePath}
+          mkdir -p "${cfg.statePath}"/{data,config,logs,plugins}
+          mkdir -p "${cfg.statePath}/plugins"/{client,server}
+          ln -sf ${cfg.package}/{bin,fonts,i18n,templates,client} "${cfg.statePath}"
+        '' + lib.optionalString (mattermostPlugins != null) ''
+          rm -rf "${cfg.statePath}/data/plugins"
+          ln -sf ${mattermostPlugins}/data/plugins "${cfg.statePath}/data"
         '' + lib.optionalString (!cfg.mutableConfig) ''
-          rm -f ${cfg.statePath}/config/config.json
-          cp ${mattermostConfJSON} ${cfg.statePath}/config/config.json
-          ${pkgs.mattermost}/bin/mattermost config migrate ${cfg.statePath}/config/config.json ${database}
+          rm -f "${cfg.statePath}/config/config.json"
+          ${pkgs.jq}/bin/jq -s '.[0] * .[1]' ${cfg.package}/config/config.json ${mattermostConfJSON} > "${cfg.statePath}/config/config.json"
         '' + lib.optionalString cfg.mutableConfig ''
           if ! test -e "${cfg.statePath}/config/.initial-created"; then
             rm -f ${cfg.statePath}/config/config.json
-            cp ${mattermostConfJSON} ${cfg.statePath}/config/config.json
-            touch ${cfg.statePath}/config/.initial-created
+            ${pkgs.jq}/bin/jq -s '.[0] * .[1]' ${cfg.package}/config/config.json ${mattermostConfJSON} > "${cfg.statePath}/config/config.json"
+            touch "${cfg.statePath}/config/.initial-created"
           fi
-        '' + lib.optionalString cfg.localDatabaseCreate ''
-          if ! test -e "${cfg.statePath}/.db-created"; then
-            ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} \
-              ${config.services.postgresql.package}/bin/psql postgres -c \
-                "CREATE ROLE ${cfg.localDatabaseUser} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${cfg.localDatabasePassword}'"
-            ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} \
-              ${config.services.postgresql.package}/bin/createdb \
-                --owner ${cfg.localDatabaseUser} ${cfg.localDatabaseName}
-            touch ${cfg.statePath}/.db-created
-          fi
-        '' + ''
-          chown ${cfg.user}:${cfg.group} -R ${cfg.statePath}
-          chmod u+rw,g+r,o-rwx -R ${cfg.statePath}
+        '' + lib.optionalString (cfg.mutableConfig && cfg.preferNixConfig) ''
+          new_config="$(${pkgs.jq}/bin/jq -s '.[0] * .[1]' "${cfg.statePath}/config/config.json" ${mattermostConfJSON})"
+
+          rm -f "${cfg.statePath}/config/config.json"
+          echo "$new_config" > "${cfg.statePath}/config/config.json"
+        '' + lib.optionalString cfg.localDatabaseCreate (createDb {}) + ''
+          # Don't change permissions recursively on the data, current, and symlinked directories (see ln -sf command above).
+          # This dramatically decreases startup times for installations with a lot of files.
+          find . -maxdepth 1 -not -name data -not -name client -not -name templates -not -name i18n -not -name fonts -not -name bin -not -name . \
+            -exec chown "${cfg.user}:${cfg.group}" -R {} \; -exec chmod u+rw,g+r,o-rwx -R {} \;
+
+          chown "${cfg.user}:${cfg.group}" "${cfg.statePath}/data" .
+          chmod u+rw,g+r,o-rwx "${cfg.statePath}/data" .
         '';
 
         serviceConfig = {
           PermissionsStartOnly = true;
           User = cfg.user;
           Group = cfg.group;
-          ExecStart = "${pkgs.mattermost}/bin/mattermost" +
-            (lib.optionalString (!cfg.mutableConfig) " -c ${database}");
+          ExecStart = "${cfg.package}/bin/mattermost";
           WorkingDirectory = "${cfg.statePath}";
           Restart = "always";
           RestartSec = "10";
@@ -224,7 +332,7 @@ in
         serviceConfig = {
           User = "nobody";
           Group = "nogroup";
-          ExecStart = "${pkgs.matterircd}/bin/matterircd ${concatStringsSep " " cfg.matterircd.parameters}";
+          ExecStart = "${cfg.matterircd.package}/bin/matterircd ${escapeShellArgs cfg.matterircd.parameters}";
           WorkingDirectory = "/tmp";
           PrivateTmp = true;
           Restart = "always";
diff --git a/nixos/modules/services/web-apps/miniflux.nix b/nixos/modules/services/web-apps/miniflux.nix
index 026bde2a92df..641c9be85d8c 100644
--- a/nixos/modules/services/web-apps/miniflux.nix
+++ b/nixos/modules/services/web-apps/miniflux.nix
@@ -4,34 +4,22 @@ with lib;
 let
   cfg = config.services.miniflux;
 
+  defaultAddress = "localhost:8080";
+
   dbUser = "miniflux";
-  dbPassword = "miniflux";
-  dbHost = "localhost";
   dbName = "miniflux";
 
-  defaultCredentials = pkgs.writeText "miniflux-admin-credentials" ''
-    ADMIN_USERNAME=admin
-    ADMIN_PASSWORD=password
-  '';
-
   pgbin = "${config.services.postgresql.package}/bin";
   preStart = pkgs.writeScript "miniflux-pre-start" ''
     #!${pkgs.runtimeShell}
-    db_exists() {
-      [ "$(${pgbin}/psql -Atc "select 1 from pg_database where datname='$1'")" == "1" ]
-    }
-    if ! db_exists "${dbName}"; then
-      ${pgbin}/psql postgres -c "CREATE ROLE ${dbUser} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${dbPassword}'"
-      ${pgbin}/createdb --owner "${dbUser}" "${dbName}"
-      ${pgbin}/psql "${dbName}" -c "CREATE EXTENSION IF NOT EXISTS hstore"
-    fi
+    ${pgbin}/psql "${dbName}" -c "CREATE EXTENSION IF NOT EXISTS hstore"
   '';
 in
 
 {
   options = {
     services.miniflux = {
-      enable = mkEnableOption "miniflux";
+      enable = mkEnableOption "miniflux and creates a local postgres database for it";
 
       config = mkOption {
         type = types.attrsOf types.str;
@@ -45,15 +33,17 @@ in
           Configuration for Miniflux, refer to
           <link xlink:href="https://miniflux.app/docs/configuration.html"/>
           for documentation on the supported values.
+
+          Correct configuration for the database is already provided.
+          By default, listens on ${defaultAddress}.
         '';
       };
 
       adminCredentialsFile = mkOption  {
-        type = types.nullOr types.path;
-        default = null;
+        type = types.path;
         description = ''
-          File containing the ADMIN_USERNAME, default is "admin", and
-          ADMIN_PASSWORD (length >= 6), default is "password"; in the format of
+          File containing the ADMIN_USERNAME and
+          ADMIN_PASSWORD (length >= 6) in the format of
           an EnvironmentFile=, as described by systemd.exec(5).
         '';
         example = "/etc/nixos/miniflux-admin-credentials";
@@ -64,17 +54,25 @@ in
   config = mkIf cfg.enable {
 
     services.miniflux.config =  {
-      LISTEN_ADDR = mkDefault "localhost:8080";
-      DATABASE_URL = "postgresql://${dbUser}:${dbPassword}@${dbHost}/${dbName}?sslmode=disable";
+      LISTEN_ADDR = mkDefault defaultAddress;
+      DATABASE_URL = "user=${dbUser} host=/run/postgresql dbname=${dbName}";
       RUN_MIGRATIONS = "1";
       CREATE_ADMIN = "1";
     };
 
-    services.postgresql.enable = true;
+    services.postgresql = {
+      enable = true;
+      ensureUsers = [ {
+        name = dbUser;
+        ensurePermissions = {
+          "DATABASE ${dbName}" = "ALL PRIVILEGES";
+        };
+      } ];
+      ensureDatabases = [ dbName ];
+    };
 
     systemd.services.miniflux-dbsetup = {
       description = "Miniflux database setup";
-      wantedBy = [ "multi-user.target" ];
       requires = [ "postgresql.service" ];
       after = [ "network.target" "postgresql.service" ];
       serviceConfig = {
@@ -87,17 +85,16 @@ in
     systemd.services.miniflux = {
       description = "Miniflux service";
       wantedBy = [ "multi-user.target" ];
-      requires = [ "postgresql.service" ];
+      requires = [ "miniflux-dbsetup.service" ];
       after = [ "network.target" "postgresql.service" "miniflux-dbsetup.service" ];
 
       serviceConfig = {
         ExecStart = "${pkgs.miniflux}/bin/miniflux";
+        User = dbUser;
         DynamicUser = true;
         RuntimeDirectory = "miniflux";
         RuntimeDirectoryMode = "0700";
-        EnvironmentFile = if cfg.adminCredentialsFile == null
-        then defaultCredentials
-        else cfg.adminCredentialsFile;
+        EnvironmentFile = cfg.adminCredentialsFile;
         # Hardening
         CapabilityBoundingSet = [ "" ];
         DeviceAllow = [ "" ];
@@ -114,7 +111,7 @@ in
         ProtectKernelModules = true;
         ProtectKernelTunables = true;
         ProtectProc = "invisible";
-        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
         RestrictNamespaces = true;
         RestrictRealtime = true;
         RestrictSUIDSGID = true;
diff --git a/nixos/modules/services/web-apps/moinmoin.nix b/nixos/modules/services/web-apps/moinmoin.nix
deleted file mode 100644
index efb73124a237..000000000000
--- a/nixos/modules/services/web-apps/moinmoin.nix
+++ /dev/null
@@ -1,304 +0,0 @@
-{ config, lib, pkgs, ... }:
-with lib;
-
-let
-  cfg = config.services.moinmoin;
-  python = pkgs.python27;
-  pkg = python.pkgs.moinmoin;
-  dataDir = "/var/lib/moin";
-  usingGunicorn = cfg.webServer == "nginx-gunicorn" || cfg.webServer == "gunicorn";
-  usingNginx = cfg.webServer == "nginx-gunicorn";
-  user = "moin";
-  group = "moin";
-
-  uLit = s: ''u"${s}"'';
-  indentLines = n: str: concatMapStrings (line: "${fixedWidthString n " " " "}${line}\n") (splitString "\n" str);
-
-  moinCliWrapper = wikiIdent: pkgs.writeShellScriptBin "moin-${wikiIdent}" ''
-    ${pkgs.su}/bin/su -s ${pkgs.runtimeShell} -c "${pkg}/bin/moin --config-dir=/var/lib/moin/${wikiIdent}/config $*" ${user}
-  '';
-
-  wikiConfig = wikiIdent: w: ''
-    # -*- coding: utf-8 -*-
-
-    from MoinMoin.config import multiconfig, url_prefix_static
-
-    class Config(multiconfig.DefaultConfig):
-        ${optionalString (w.webLocation != "/") ''
-          url_prefix_static = '${w.webLocation}' + url_prefix_static
-        ''}
-
-        sitename = u'${w.siteName}'
-        page_front_page = u'${w.frontPage}'
-
-        data_dir = '${dataDir}/${wikiIdent}/data'
-        data_underlay_dir = '${dataDir}/${wikiIdent}/underlay'
-
-        language_default = u'${w.languageDefault}'
-        ${optionalString (w.superUsers != []) ''
-          superuser = [${concatMapStringsSep ", " uLit w.superUsers}]
-        ''}
-
-    ${indentLines 4 w.extraConfig}
-  '';
-  wikiConfigFile = name: wiki: pkgs.writeText "${name}.py" (wikiConfig name wiki);
-
-in
-{
-  options.services.moinmoin = with types; {
-    enable = mkEnableOption "MoinMoin Wiki Engine";
-
-    webServer = mkOption {
-      type = enum [ "nginx-gunicorn" "gunicorn" "none" ];
-      default = "nginx-gunicorn";
-      example = "none";
-      description = ''
-        Which web server to use to serve the wiki.
-        Use <literal>none</literal> if you want to configure this yourself.
-      '';
-    };
-
-    gunicorn.workers = mkOption {
-      type = ints.positive;
-      default = 3;
-      example = 10;
-      description = ''
-        The number of worker processes for handling requests.
-      '';
-    };
-
-    wikis = mkOption {
-      type = attrsOf (submodule ({ name, ... }: {
-        options = {
-          siteName = mkOption {
-            type = str;
-            default = "Untitled Wiki";
-            example = "ExampleWiki";
-            description = ''
-              Short description of your wiki site, displayed below the logo on each page, and
-              used in RSS documents as the channel title.
-            '';
-          };
-
-          webHost = mkOption {
-            type = str;
-            description = "Host part of the wiki URL. If undefined, the name of the attribute set will be used.";
-            example = "wiki.example.org";
-          };
-
-          webLocation = mkOption {
-            type = str;
-            default = "/";
-            example = "/moin";
-            description = "Location part of the wiki URL.";
-          };
-
-          frontPage = mkOption {
-            type = str;
-            default = "LanguageSetup";
-            example = "FrontPage";
-            description = ''
-              Front page name. Set this to something like <literal>FrontPage</literal> once languages are
-              configured.
-            '';
-          };
-
-          superUsers = mkOption {
-            type = listOf str;
-            default = [];
-            example = [ "elvis" ];
-            description = ''
-              List of trusted user names with wiki system administration super powers.
-
-              Please note that accounts for these users need to be created using the <command>moin</command> command-line utility, e.g.:
-              <command>moin-<replaceable>WIKINAME</replaceable> account create --name=<replaceable>NAME</replaceable> --email=<replaceable>EMAIL</replaceable> --password=<replaceable>PASSWORD</replaceable></command>.
-            '';
-          };
-
-          languageDefault = mkOption {
-            type = str;
-            default = "en";
-            example = "de";
-            description = "The ISO-639-1 name of the main wiki language. Languages that MoinMoin does not support are ignored.";
-          };
-
-          extraConfig = mkOption {
-            type = lines;
-            default = "";
-            example = ''
-              show_hosts = True
-              search_results_per_page = 100
-              acl_rights_default = u"Known:read,write,delete,revert All:read"
-              logo_string = u"<h2>\U0001f639</h2>"
-              theme_default = u"modernized"
-
-              user_checkbox_defaults = {'show_page_trail': 0, 'edit_on_doubleclick': 0}
-              navi_bar = [u'SomePage'] + multiconfig.DefaultConfig.navi_bar
-              actions_excluded = multiconfig.DefaultConfig.actions_excluded + ['newaccount']
-
-              mail_smarthost = "mail.example.org"
-              mail_from = u"Example.Org Wiki <wiki@example.org>"
-            '';
-            description = ''
-              Additional configuration to be appended verbatim to this wiki's config.
-
-              See <link xlink:href='http://moinmo.in/HelpOnConfiguration' /> for documentation.
-            '';
-          };
-
-        };
-        config = {
-          webHost = mkDefault name;
-        };
-      }));
-      example = literalExpression ''
-        {
-          "mywiki" = {
-            siteName = "Example Wiki";
-            webHost = "wiki.example.org";
-            superUsers = [ "admin" ];
-            frontPage = "Index";
-            extraConfig = "page_category_regex = ur'(?P<all>(Category|Kategorie)(?P<key>(?!Template)\S+))'"
-          };
-        }
-      '';
-      description = ''
-        Configurations of the individual wikis. Attribute names must be valid Python
-        identifiers of the form <literal>[A-Za-z_][A-Za-z0-9_]*</literal>.
-
-        For every attribute <replaceable>WIKINAME</replaceable>, a helper script
-        moin-<replaceable>WIKINAME</replaceable> is created which runs the
-        <command>moin</command> command under the <literal>moin</literal> user (to avoid
-        file ownership issues) and with the right configuration directory passed to it.
-      '';
-    };
-  };
-
-  config = mkIf cfg.enable {
-    assertions = forEach (attrNames cfg.wikis) (wname:
-      { assertion = builtins.match "[A-Za-z_][A-Za-z0-9_]*" wname != null;
-        message = "${wname} is not valid Python identifier";
-      }
-    );
-
-    users.users = {
-      moin = {
-        description = "MoinMoin wiki";
-        home = dataDir;
-        group = group;
-        isSystemUser = true;
-      };
-    };
-
-    users.groups = {
-      moin = {
-        members = mkIf usingNginx [ config.services.nginx.user ];
-      };
-    };
-
-    environment.systemPackages = [ pkg ] ++ map moinCliWrapper (attrNames cfg.wikis);
-
-    systemd.services = mkIf usingGunicorn
-      (flip mapAttrs' cfg.wikis (wikiIdent: wiki:
-        nameValuePair "moin-${wikiIdent}"
-          {
-            description = "MoinMoin wiki ${wikiIdent} - gunicorn process";
-            wantedBy = [ "multi-user.target" ];
-            after = [ "network.target" ];
-            restartIfChanged = true;
-            restartTriggers = [ (wikiConfigFile wikiIdent wiki) ];
-
-            environment = let
-              penv = python.buildEnv.override {
-                # setuptools: https://github.com/benoitc/gunicorn/issues/1716
-                extraLibs = [ python.pkgs.eventlet python.pkgs.setuptools pkg ];
-              };
-            in {
-              PYTHONPATH = "${dataDir}/${wikiIdent}/config:${penv}/${python.sitePackages}";
-            };
-
-            preStart = ''
-              umask 0007
-              rm -rf ${dataDir}/${wikiIdent}/underlay
-              cp -r ${pkg}/share/moin/underlay ${dataDir}/${wikiIdent}/
-              chmod -R u+w ${dataDir}/${wikiIdent}/underlay
-            '';
-
-            startLimitIntervalSec = 30;
-
-            serviceConfig = {
-              User = user;
-              Group = group;
-              WorkingDirectory = "${dataDir}/${wikiIdent}";
-              ExecStart = ''${python.pkgs.gunicorn}/bin/gunicorn moin_wsgi \
-                --name gunicorn-${wikiIdent} \
-                --workers ${toString cfg.gunicorn.workers} \
-                --worker-class eventlet \
-                --bind unix:/run/moin/${wikiIdent}/gunicorn.sock
-              '';
-
-              Restart = "on-failure";
-              RestartSec = "2s";
-
-              StateDirectory = "moin/${wikiIdent}";
-              StateDirectoryMode = "0750";
-              RuntimeDirectory = "moin/${wikiIdent}";
-              RuntimeDirectoryMode = "0750";
-
-              NoNewPrivileges = true;
-              ProtectSystem = "strict";
-              ProtectHome = true;
-              PrivateTmp = true;
-              PrivateDevices = true;
-              PrivateNetwork = true;
-              ProtectKernelTunables = true;
-              ProtectKernelModules = true;
-              ProtectControlGroups = true;
-              RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
-              RestrictNamespaces = true;
-              LockPersonality = true;
-              MemoryDenyWriteExecute = true;
-              RestrictRealtime = true;
-            };
-          }
-      ));
-
-    services.nginx = mkIf usingNginx {
-      enable = true;
-      virtualHosts = flip mapAttrs' cfg.wikis (name: w: nameValuePair w.webHost {
-        forceSSL = mkDefault true;
-        enableACME = mkDefault true;
-        locations."${w.webLocation}" = {
-          extraConfig = ''
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            proxy_set_header X-Forwarded-Host $host;
-            proxy_set_header X-Forwarded-Server $host;
-
-            proxy_pass http://unix:/run/moin/${name}/gunicorn.sock;
-          '';
-        };
-      });
-    };
-
-    systemd.tmpfiles.rules = [
-      "d  /run/moin            0750 ${user} ${group} - -"
-      "d  ${dataDir}           0550 ${user} ${group} - -"
-    ]
-    ++ (concatLists (flip mapAttrsToList cfg.wikis (wikiIdent: wiki: [
-      "d  ${dataDir}/${wikiIdent}                      0750 ${user} ${group} - -"
-      "d  ${dataDir}/${wikiIdent}/config               0550 ${user} ${group} - -"
-      "L+ ${dataDir}/${wikiIdent}/config/wikiconfig.py -    -       -        - ${wikiConfigFile wikiIdent wiki}"
-      # needed in order to pass module name to gunicorn
-      "L+ ${dataDir}/${wikiIdent}/config/moin_wsgi.py  -    -       -        - ${pkg}/share/moin/server/moin.wsgi"
-      # seed data files
-      "C  ${dataDir}/${wikiIdent}/data                 0770 ${user} ${group} - ${pkg}/share/moin/data"
-      # fix nix store permissions
-      "Z  ${dataDir}/${wikiIdent}/data                 0770 ${user} ${group} - -"
-    ])));
-  };
-
-  meta.maintainers = with lib.maintainers; [ mmilata ];
-}
diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix
index b1a536e519db..b32220a5e579 100644
--- a/nixos/modules/services/web-apps/nextcloud.nix
+++ b/nixos/modules/services/web-apps/nextcloud.nix
@@ -153,7 +153,7 @@ in {
     package = mkOption {
       type = types.package;
       description = "Which package to use for the Nextcloud instance.";
-      relatedPackages = [ "nextcloud21" "nextcloud22" "nextcloud23" ];
+      relatedPackages = [ "nextcloud22" "nextcloud23" ];
     };
     phpPackage = mkOption {
       type = types.package;
@@ -499,11 +499,18 @@ in {
     occ = mkOption {
       type = types.package;
       default = occ;
+      defaultText = literalDocBook "generated script";
       internal = true;
       description = ''
         The nextcloud-occ program preconfigured to target this Nextcloud instance.
       '';
     };
+
+    nginx.recommendedHttpHeaders = mkOption {
+      type = types.bool;
+      default = true;
+      description = "Enable additional recommended HTTP response headers";
+    };
   };
 
   config = mkIf cfg.enable (mkMerge [
@@ -526,8 +533,8 @@ in {
         # FIXME(@Ma27) remove as soon as nextcloud properly supports
         # mariadb >=10.6.
         isUnsupportedMariadb =
-          # All currently supported Nextcloud versions are affected.
-          (versionOlder cfg.package.version "23")
+          # All currently supported Nextcloud versions are affected (https://github.com/nextcloud/server/issues/25436).
+          (versionOlder cfg.package.version "24")
           # This module uses mysql
           && (cfg.config.dbtype == "mysql")
           # MySQL is managed via NixOS
@@ -564,15 +571,6 @@ in {
               nextcloud defined in an overlay, please set `services.nextcloud.package` to
               `pkgs.nextcloud`.
             ''
-          # 21.03 will not be an official release - it was instead 21.05.
-          # This versionOlder statement remains set to 21.03 for backwards compatibility.
-          # See https://github.com/NixOS/nixpkgs/pull/108899 and
-          # https://github.com/NixOS/rfcs/blob/master/rfcs/0080-nixos-release-schedule.md.
-          # FIXME(@Ma27) remove this else-if as soon as 21.05 is EOL! This is only here
-          # to ensure that users who are on Nextcloud 19 with a stateVersion <21.05 with
-          # no explicit services.nextcloud.package don't upgrade to v21 by accident (
-          # nextcloud20 throws an eval-error because it's dropped).
-          else if versionOlder stateVersion "21.03" then nextcloud20
           else if versionOlder stateVersion "21.11" then nextcloud21
           else if versionOlder stateVersion "22.05" then nextcloud22
           else nextcloud23
@@ -592,6 +590,8 @@ in {
         timerConfig.Unit = "nextcloud-cron.service";
       };
 
+      systemd.tmpfiles.rules = ["d ${cfg.home} 0750 nextcloud nextcloud"];
+
       systemd.services = {
         # When upgrading the Nextcloud package, Nextcloud can report errors such as
         # "The files of the app [all apps in /var/lib/nextcloud/apps] were not replaced correctly"
@@ -713,8 +713,6 @@ in {
           before = [ "phpfpm-nextcloud.service" ];
           path = [ occ ];
           script = ''
-            chmod og+x ${cfg.home}
-
             ${optionalString (c.dbpassFile != null) ''
               if [ ! -r "${c.dbpassFile}" ]; then
                 echo "dbpassFile ${c.dbpassFile} is not readable by nextcloud:nextcloud! Aborting..."
@@ -807,7 +805,6 @@ in {
       users.users.nextcloud = {
         home = "${cfg.home}";
         group = "nextcloud";
-        createHome = true;
         isSystemUser = true;
       };
       users.groups.nextcloud.members = [ "nextcloud" config.services.nginx.user ];
@@ -903,14 +900,16 @@ in {
         };
         extraConfig = ''
           index index.php index.html /index.php$request_uri;
-          add_header X-Content-Type-Options nosniff;
-          add_header X-XSS-Protection "1; mode=block";
-          add_header X-Robots-Tag none;
-          add_header X-Download-Options noopen;
-          add_header X-Permitted-Cross-Domain-Policies none;
-          add_header X-Frame-Options sameorigin;
-          add_header Referrer-Policy no-referrer;
-          add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+          ${optionalString (cfg.nginx.recommendedHttpHeaders) ''
+            add_header X-Content-Type-Options nosniff;
+            add_header X-XSS-Protection "1; mode=block";
+            add_header X-Robots-Tag none;
+            add_header X-Download-Options noopen;
+            add_header X-Permitted-Cross-Domain-Policies none;
+            add_header X-Frame-Options sameorigin;
+            add_header Referrer-Policy no-referrer;
+            add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+          ''}
           client_max_body_size ${cfg.maxUploadSize};
           fastcgi_buffers 64 4K;
           fastcgi_hide_header X-Powered-By;
diff --git a/nixos/modules/services/web-apps/peertube.nix b/nixos/modules/services/web-apps/peertube.nix
index 932ddcfef198..e195e6e6e824 100644
--- a/nixos/modules/services/web-apps/peertube.nix
+++ b/nixos/modules/services/web-apps/peertube.nix
@@ -1,7 +1,8 @@
-{ lib, pkgs, config, ... }:
+{ lib, pkgs, config, options, ... }:
 
 let
   cfg = config.services.peertube;
+  opt = options.services.peertube;
 
   settingsFormat = pkgs.formats.json {};
   configFile = settingsFormat.generate "production.json" cfg.settings;
@@ -153,6 +154,11 @@ in {
       host = lib.mkOption {
         type = lib.types.str;
         default = if cfg.database.createLocally then "/run/postgresql" else null;
+        defaultText = lib.literalExpression ''
+          if config.${opt.database.createLocally}
+          then "/run/postgresql"
+          else null
+        '';
         example = "192.168.15.47";
         description = "Database host address or unix socket.";
       };
@@ -193,12 +199,22 @@ in {
       host = lib.mkOption {
         type = lib.types.nullOr lib.types.str;
         default = if cfg.redis.createLocally && !cfg.redis.enableUnixSocket then "127.0.0.1" else null;
+        defaultText = lib.literalExpression ''
+          if config.${opt.redis.createLocally} && !config.${opt.redis.enableUnixSocket}
+          then "127.0.0.1"
+          else null
+        '';
         description = "Redis host.";
       };
 
       port = lib.mkOption {
         type = lib.types.nullOr lib.types.port;
         default = if cfg.redis.createLocally && cfg.redis.enableUnixSocket then null else 6379;
+        defaultText = lib.literalExpression ''
+          if config.${opt.redis.createLocally} && config.${opt.redis.enableUnixSocket}
+          then null
+          else 6379
+        '';
         description = "Redis port.";
       };
 
@@ -212,6 +228,7 @@ in {
       enableUnixSocket = lib.mkOption {
         type = lib.types.bool;
         default = cfg.redis.createLocally;
+        defaultText = lib.literalExpression "config.${opt.redis.createLocally}";
         description = "Use Unix socket.";
       };
     };
@@ -303,6 +320,7 @@ in {
         };
         storage = {
           tmp = lib.mkDefault "/var/lib/peertube/storage/tmp/";
+          bin = lib.mkDefault "/var/lib/peertube/storage/bin/";
           avatars = lib.mkDefault "/var/lib/peertube/storage/avatars/";
           videos = lib.mkDefault "/var/lib/peertube/storage/videos/";
           streaming_playlists = lib.mkDefault "/var/lib/peertube/storage/streaming-playlists/";
@@ -316,6 +334,15 @@ in {
           plugins = lib.mkDefault "/var/lib/peertube/storage/plugins/";
           client_overrides = lib.mkDefault "/var/lib/peertube/storage/client-overrides/";
         };
+        import = {
+          videos = {
+            http = {
+              youtube_dl_release = {
+                python_path = "${pkgs.python3}/bin/python";
+              };
+            };
+          };
+        };
       }
       (lib.mkIf cfg.redis.enableUnixSocket { redis = { socket = "/run/redis/redis.sock"; }; })
     ];
@@ -363,7 +390,7 @@ in {
 
       environment = env;
 
-      path = with pkgs; [ bashInteractive ffmpeg nodejs-16_x openssl yarn youtube-dl ];
+      path = with pkgs; [ bashInteractive ffmpeg nodejs-16_x openssl yarn python3 ];
 
       script = ''
         #!/bin/sh
diff --git a/nixos/modules/services/web-apps/pgpkeyserver-lite.nix b/nixos/modules/services/web-apps/pgpkeyserver-lite.nix
index 5642627d397d..faf0ce13238e 100644
--- a/nixos/modules/services/web-apps/pgpkeyserver-lite.nix
+++ b/nixos/modules/services/web-apps/pgpkeyserver-lite.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
@@ -6,6 +6,7 @@ let
 
   cfg = config.services.pgpkeyserver-lite;
   sksCfg = config.services.sks;
+  sksOpt = options.services.sks;
 
   webPkg = cfg.package;
 
@@ -37,6 +38,7 @@ in
 
       hkpAddress = mkOption {
         default = builtins.head sksCfg.hkpAddress;
+        defaultText = literalExpression "head config.${sksOpt.hkpAddress}";
         type = types.str;
         description = "
           Wich ip address the sks-keyserver is listening on.
@@ -45,6 +47,7 @@ in
 
       hkpPort = mkOption {
         default = sksCfg.hkpPort;
+        defaultText = literalExpression "config.${sksOpt.hkpPort}";
         type = types.int;
         description = "
           Which port the sks-keyserver is listening on.
diff --git a/nixos/modules/services/web-apps/plantuml-server.nix b/nixos/modules/services/web-apps/plantuml-server.nix
index f4bf43f56b98..9ea37b8a4cad 100644
--- a/nixos/modules/services/web-apps/plantuml-server.nix
+++ b/nixos/modules/services/web-apps/plantuml-server.nix
@@ -20,6 +20,21 @@ in
         description = "PlantUML server package to use";
       };
 
+      packages = {
+        jdk = mkOption {
+          type = types.package;
+          default = pkgs.jdk;
+          defaultText = literalExpression "pkgs.jdk";
+          description = "JDK package to use for the server";
+        };
+        jetty = mkOption {
+          type = types.package;
+          default = pkgs.jetty;
+          defaultText = literalExpression "pkgs.jetty";
+          description = "Jetty package to use for the server";
+        };
+      };
+
       user = mkOption {
         type = types.str;
         default = "plantuml";
@@ -105,10 +120,10 @@ in
         ALLOW_PLANTUML_INCLUDE = if cfg.allowPlantumlInclude then "true" else "false";
       };
       script = ''
-      ${pkgs.jre}/bin/java \
-        -jar ${pkgs.jetty}/start.jar \
+      ${cfg.packages.jdk}/bin/java \
+        -jar ${cfg.packages.jetty}/start.jar \
           --module=deploy,http,jsp \
-          jetty.home=${pkgs.jetty} \
+          jetty.home=${cfg.packages.jetty} \
           jetty.base=${cfg.package} \
           jetty.http.host=${cfg.listenHost} \
           jetty.http.port=${builtins.toString cfg.listenPort}
diff --git a/nixos/modules/services/web-apps/plausible.nix b/nixos/modules/services/web-apps/plausible.nix
index b6c48186a1d3..5d550ae5ca86 100644
--- a/nixos/modules/services/web-apps/plausible.nix
+++ b/nixos/modules/services/web-apps/plausible.nix
@@ -10,8 +10,7 @@ in {
     enable = mkEnableOption "plausible";
 
     releaseCookiePath = mkOption {
-      default = null;
-      type = with types; nullOr (either str path);
+      type = with types; either str path;
       description = ''
         The path to the file with release cookie. (used for remote connection to the running node).
       '';
@@ -235,6 +234,8 @@ in {
           script = ''
             export CONFIG_DIR=$CREDENTIALS_DIRECTORY
 
+            export RELEASE_COOKIE="$(< $CREDENTIALS_DIRECTORY/RELEASE_COOKIE )"
+
             # setup
             ${pkgs.plausible}/createdb.sh
             ${pkgs.plausible}/migrate.sh
@@ -243,10 +244,8 @@ in {
                 psql -d plausible <<< "UPDATE users SET email_verified=true;"
               fi
             ''}
-            ${optionalString (cfg.releaseCookiePath != null) ''
-              export RELEASE_COOKIE="$(< $CREDENTIALS_DIRECTORY/RELEASE_COOKIE )"
-            ''}
-            plausible start
+
+            exec plausible start
           '';
 
           serviceConfig = {
@@ -257,8 +256,8 @@ in {
             LoadCredential = [
               "ADMIN_USER_PWD:${cfg.adminUser.passwordFile}"
               "SECRET_KEY_BASE:${cfg.server.secretKeybaseFile}"
-            ] ++ lib.optionals (cfg.mail.smtp.passwordFile != null) [ "SMTP_USER_PWD:${cfg.mail.smtp.passwordFile}"]
-            ++ lib.optionals (cfg.releaseCookiePath != null) [ "RELEASE_COOKIE:${cfg.releaseCookiePath}"];
+              "RELEASE_COOKIE:${cfg.releaseCookiePath}"
+            ] ++ lib.optionals (cfg.mail.smtp.passwordFile != null) [ "SMTP_USER_PWD:${cfg.mail.smtp.passwordFile}"];
           };
         };
       }
diff --git a/nixos/modules/services/web-apps/powerdns-admin.nix b/nixos/modules/services/web-apps/powerdns-admin.nix
new file mode 100644
index 000000000000..4661ba80c5d6
--- /dev/null
+++ b/nixos/modules/services/web-apps/powerdns-admin.nix
@@ -0,0 +1,152 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.powerdns-admin;
+
+  configText = ''
+    ${cfg.config}
+  ''
+  + optionalString (cfg.secretKeyFile != null) ''
+    with open('${cfg.secretKeyFile}') as file:
+      SECRET_KEY = file.read()
+  ''
+  + optionalString (cfg.saltFile != null) ''
+    with open('${cfg.saltFile}') as file:
+      SALT = file.read()
+  '';
+in
+{
+  options.services.powerdns-admin = {
+    enable = mkEnableOption "the PowerDNS web interface";
+
+    extraArgs = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = literalExpression ''
+        [ "-b" "127.0.0.1:8000" ]
+      '';
+      description = ''
+        Extra arguments passed to powerdns-admin.
+      '';
+    };
+
+    config = mkOption {
+      type = types.str;
+      default = "";
+      example = ''
+        BIND_ADDRESS = '127.0.0.1'
+        PORT = 8000
+        SQLALCHEMY_DATABASE_URI = 'postgresql://powerdnsadmin@/powerdnsadmin?host=/run/postgresql'
+      '';
+      description = ''
+        Configuration python file.
+        See <link xlink:href="https://github.com/ngoduykhanh/PowerDNS-Admin/blob/v${pkgs.powerdns-admin.version}/configs/development.py">the example configuration</link>
+        for options.
+      '';
+    };
+
+    secretKeyFile = mkOption {
+      type = types.nullOr types.path;
+      example = "/etc/powerdns-admin/secret";
+      description = ''
+        The secret used to create cookies.
+        This needs to be set, otherwise the default is used and everyone can forge valid login cookies.
+        Set this to null to ignore this setting and configure it through another way.
+      '';
+    };
+
+    saltFile = mkOption {
+      type = types.nullOr types.path;
+      example = "/etc/powerdns-admin/salt";
+      description = ''
+        The salt used for serialization.
+        This should be set, otherwise the default is used.
+        Set this to null to ignore this setting and configure it through another way.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.powerdns-admin = {
+      description = "PowerDNS web interface";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "networking.target" ];
+
+      environment.FLASK_CONF = builtins.toFile "powerdns-admin-config.py" configText;
+      environment.PYTHONPATH = pkgs.powerdns-admin.pythonPath;
+      serviceConfig = {
+        ExecStart = "${pkgs.powerdns-admin}/bin/powerdns-admin --pid /run/powerdns-admin/pid ${escapeShellArgs cfg.extraArgs}";
+        ExecStartPre = "${pkgs.coreutils}/bin/env FLASK_APP=${pkgs.powerdns-admin}/share/powerdnsadmin/__init__.py ${pkgs.python3Packages.flask}/bin/flask db upgrade -d ${pkgs.powerdns-admin}/share/migrations";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        ExecStop = "${pkgs.coreutils}/bin/kill -TERM $MAINPID";
+        PIDFile = "/run/powerdns-admin/pid";
+        RuntimeDirectory = "powerdns-admin";
+        User = "powerdnsadmin";
+        Group = "powerdnsadmin";
+
+        AmbientCapabilities = "CAP_NET_BIND_SERVICE";
+        BindReadOnlyPaths = [
+          "/nix/store"
+          "-/etc/resolv.conf"
+          "-/etc/nsswitch.conf"
+          "-/etc/hosts"
+          "-/etc/localtime"
+        ]
+        ++ (optional (cfg.secretKeyFile != null) cfg.secretKeyFile)
+        ++ (optional (cfg.saltFile != null) cfg.saltFile);
+        CapabilityBoundingSet = "CAP_NET_BIND_SERVICE";
+        # ProtectClock= adds DeviceAllow=char-rtc r
+        DeviceAllow = "";
+        # Implies ProtectSystem=strict, which re-mounts all paths
+        #DynamicUser = true;
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        # Needs to start a server
+        #PrivateNetwork = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        # Would re-mount paths ignored by temporary root
+        #ProtectSystem = "strict";
+        ProtectControlGroups = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        # gunicorn needs setuid
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged @resources @keyring"
+          # These got removed by the line above but are needed
+          "@setuid @chown"
+        ];
+        TemporaryFileSystem = "/:ro";
+        # Does not work well with the temporary root
+        #UMask = "0066";
+      };
+    };
+
+    users.groups.powerdnsadmin = { };
+    users.users.powerdnsadmin = {
+      description = "PowerDNS web interface user";
+      isSystemUser = true;
+      group = "powerdnsadmin";
+    };
+  };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
+}
diff --git a/nixos/modules/services/web-apps/prosody-filer.nix b/nixos/modules/services/web-apps/prosody-filer.nix
new file mode 100644
index 000000000000..a901a95fd5f9
--- /dev/null
+++ b/nixos/modules/services/web-apps/prosody-filer.nix
@@ -0,0 +1,86 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+
+  cfg = config.services.prosody-filer;
+
+  settingsFormat = pkgs.formats.toml { };
+  configFile = settingsFormat.generate "prosody-filer.toml" cfg.settings;
+in {
+
+  options = {
+    services.prosody-filer = {
+      enable = mkEnableOption "Prosody Filer XMPP upload file server";
+
+      settings = mkOption {
+        description = ''
+          Configuration for Prosody Filer.
+          Refer to <link xlink:href="https://github.com/ThomasLeister/prosody-filer#configure-prosody-filer"/> for details on supported values.
+        '';
+
+        type = settingsFormat.type;
+
+        example = {
+          secret = "mysecret";
+          storeDir = "/srv/http/nginx/prosody-upload";
+        };
+
+        defaultText = literalExpression ''
+          {
+            listenport = mkDefault "127.0.0.1:5050";
+            uploadSubDir = mkDefault "upload/";
+          }
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.prosody-filer.settings = {
+      listenport = mkDefault "127.0.0.1:5050";
+      uploadSubDir = mkDefault "upload/";
+    };
+
+    users.users.prosody-filer = {
+      group = "prosody-filer";
+      isSystemUser = true;
+    };
+
+    users.groups.prosody-filer = { };
+
+    systemd.services.prosody-filer = {
+      description = "Prosody file upload server";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      serviceConfig = {
+        User = "prosody-filer";
+        Group = "prosody-filer";
+        ExecStart = "${pkgs.prosody-filer}/bin/prosody-filer -config ${configFile}";
+        Restart = "on-failure";
+        CapabilityBoundingSet = "";
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        PrivateMounts = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/web-apps/restya-board.nix b/nixos/modules/services/web-apps/restya-board.nix
index fd97ab76a5f6..4b36cc8754c6 100644
--- a/nixos/modules/services/web-apps/restya-board.nix
+++ b/nixos/modules/services/web-apps/restya-board.nix
@@ -235,7 +235,7 @@ in
       locations."~ \\.php$" = {
         tryFiles = "$uri =404";
         extraConfig = ''
-          include ${pkgs.nginx}/conf/fastcgi_params;
+          include ${config.services.nginx.package}/conf/fastcgi_params;
           fastcgi_pass    unix:${fpm.socket};
           fastcgi_index   index.php;
           fastcgi_param   SCRIPT_FILENAME $document_root$fastcgi_script_name;
diff --git a/nixos/modules/services/web-apps/rss-bridge.nix b/nixos/modules/services/web-apps/rss-bridge.nix
index 456ca00416fe..f2b6d9559823 100644
--- a/nixos/modules/services/web-apps/rss-bridge.nix
+++ b/nixos/modules/services/web-apps/rss-bridge.nix
@@ -111,7 +111,7 @@ in
 
           locations."~ ^/index.php(/|$)" = {
             extraConfig = ''
-              include ${pkgs.nginx}/conf/fastcgi_params;
+              include ${config.services.nginx.package}/conf/fastcgi_params;
               fastcgi_split_path_info ^(.+\.php)(/.+)$;
               fastcgi_pass unix:${config.services.phpfpm.pools.${cfg.pool}.socket};
               fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
diff --git a/nixos/modules/services/web-apps/timetagger.nix b/nixos/modules/services/web-apps/timetagger.nix
new file mode 100644
index 000000000000..373f4fcd52f8
--- /dev/null
+++ b/nixos/modules/services/web-apps/timetagger.nix
@@ -0,0 +1,80 @@
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkEnableOption mkIf mkOption types literalExpression;
+
+  cfg = config.services.timetagger;
+in {
+
+  options = {
+    services.timetagger = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Tag your time, get the insight
+
+          <note><para>
+            This app does not do authentication.
+            You must setup authentication yourself or run it in an environment where
+            only allowed users have access.
+          </para></note>
+        '';
+      };
+
+      bindAddr = mkOption {
+        description = "Address to bind to.";
+        type = types.str;
+        default = "127.0.0.1";
+      };
+
+      port = mkOption {
+        description = "Port to bind to.";
+        type = types.port;
+        default = 8080;
+      };
+
+      package = mkOption {
+        description = ''
+          Use own package for starting timetagger web application.
+
+          The ${literalExpression ''pkgs.timetagger''} package only provides a
+          "run.py" script for the actual package
+          ${literalExpression ''pkgs.python3Packages.timetagger''}.
+
+          If you want to provide a "run.py" script for starting timetagger
+          yourself, you can do so with this option.
+          If you do so, the 'bindAddr' and 'port' options are ignored.
+        '';
+
+        default = pkgs.timetagger.override { addr = cfg.bindAddr; port = cfg.port; };
+        defaultText = literalExpression ''
+          pkgs.timetagger.override {
+            addr = ${cfg.bindAddr};
+            port = ${cfg.port};
+          };
+        '';
+        type = types.package;
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.timetagger = {
+      description = "Timetagger service";
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        User = "timetagger";
+        Group = "timetagger";
+        StateDirectory = "timetagger";
+
+        ExecStart = "${cfg.package}/bin/timetagger";
+
+        Restart = "on-failure";
+        RestartSec = 1;
+      };
+    };
+  };
+}
+
diff --git a/nixos/modules/services/web-apps/trac.nix b/nixos/modules/services/web-apps/trac.nix
deleted file mode 100644
index 207fb857438a..000000000000
--- a/nixos/modules/services/web-apps/trac.nix
+++ /dev/null
@@ -1,79 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  cfg = config.services.trac;
-
-  inherit (lib) mkEnableOption mkIf mkOption types;
-
-in {
-
-  options = {
-
-    services.trac = {
-      enable = mkEnableOption "Trac service";
-
-      listen = {
-        ip = mkOption {
-          type = types.str;
-          default = "0.0.0.0";
-          description = ''
-            IP address that Trac should listen on.
-          '';
-        };
-
-        port = mkOption {
-          type = types.port;
-          default = 8000;
-          description = ''
-            Listen port for Trac.
-          '';
-        };
-      };
-
-      dataDir = mkOption {
-        default = "/var/lib/trac";
-        type = types.path;
-        description = ''
-            The directory for storing the Trac data.
-        '';
-      };
-
-      openFirewall = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Open ports in the firewall for Trac.
-        '';
-      };
-    };
-
-  };
-
-  config = mkIf cfg.enable {
-
-    systemd.services.trac = {
-      description = "Trac server";
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        DynamicUser = true;
-        StateDirectory = baseNameOf cfg.dataDir;
-        ExecStart = ''
-          ${pkgs.trac}/bin/tracd -s \
-            -b ${toString cfg.listen.ip} \
-            -p ${toString cfg.listen.port} \
-            ${cfg.dataDir}
-        '';
-      };
-      preStart = ''
-        if [ ! -e ${cfg.dataDir}/VERSION ]; then
-          ${pkgs.trac}/bin/trac-admin ${cfg.dataDir} initenv Trac "sqlite:db/trac.db"
-        fi
-      '';
-    };
-
-    networking.firewall = mkIf cfg.openFirewall {
-      allowedTCPPorts = [ cfg.listen.port ];
-    };
-
-  };
-}
diff --git a/nixos/modules/services/web-apps/wordpress.nix b/nixos/modules/services/web-apps/wordpress.nix
index 8ebb72296627..59471a739cbb 100644
--- a/nixos/modules/services/web-apps/wordpress.nix
+++ b/nixos/modules/services/web-apps/wordpress.nix
@@ -1,20 +1,14 @@
 { config, pkgs, lib, ... }:
 
-let
-  inherit (lib) mkDefault mkEnableOption mkForce mkIf mkMerge mkOption types;
-  inherit (lib) any attrValues concatMapStringsSep flatten literalExpression;
-  inherit (lib) filterAttrs mapAttrs mapAttrs' mapAttrsToList nameValuePair optional optionalAttrs optionalString;
+with lib;
 
-  cfg = migrateOldAttrs config.services.wordpress;
+let
+  cfg = config.services.wordpress;
   eachSite = cfg.sites;
   user = "wordpress";
   webserver = config.services.${cfg.webserver};
   stateDir = hostName: "/var/lib/wordpress/${hostName}";
 
-  # Migrate config.services.wordpress.<hostName> to config.services.wordpress.sites.<hostName>
-  oldSites = filterAttrs (o: _: o != "sites" && o != "webserver");
-  migrateOldAttrs = cfg: cfg // { sites = cfg.sites // oldSites cfg; };
-
   pkg = hostName: cfg: pkgs.stdenv.mkDerivation rec {
     pname = "wordpress-${hostName}";
     version = src.version;
@@ -266,48 +260,44 @@ in
 {
   # interface
   options = {
-    services.wordpress = mkOption {
-      type = types.submodule {
-        # Used to support old interface
-        freeformType = types.attrsOf (types.submodule siteOpts);
-
-        # New interface
-        options.sites = mkOption {
-          type = types.attrsOf (types.submodule siteOpts);
-          default = {};
-          description = "Specification of one or more WordPress sites to serve";
-        };
+    services.wordpress = {
 
-        options.webserver = mkOption {
-          type = types.enum [ "httpd" "nginx" "caddy" ];
-          default = "httpd";
-          description = ''
-            Whether to use apache2 or nginx for virtual host management.
+      sites = mkOption {
+        type = types.attrsOf (types.submodule siteOpts);
+        default = {};
+        description = "Specification of one or more WordPress sites to serve";
+      };
 
-            Further nginx configuration can be done by adapting <literal>services.nginx.virtualHosts.&lt;name&gt;</literal>.
-            See <xref linkend="opt-services.nginx.virtualHosts"/> for further information.
+      webserver = mkOption {
+        type = types.enum [ "httpd" "nginx" "caddy" ];
+        default = "httpd";
+        description = ''
+          Whether to use apache2 or nginx for virtual host management.
 
-            Further apache2 configuration can be done by adapting <literal>services.httpd.virtualHosts.&lt;name&gt;</literal>.
-            See <xref linkend="opt-services.httpd.virtualHosts"/> for further information.
-          '';
-        };
+          Further nginx configuration can be done by adapting <literal>services.nginx.virtualHosts.&lt;name&gt;</literal>.
+          See <xref linkend="opt-services.nginx.virtualHosts"/> for further information.
+
+          Further apache2 configuration can be done by adapting <literal>services.httpd.virtualHosts.&lt;name&gt;</literal>.
+          See <xref linkend="opt-services.httpd.virtualHosts"/> for further information.
+        '';
       };
-      default = {};
-      description = "Wordpress configuration";
-    };
 
+    };
   };
 
   # implementation
   config = mkIf (eachSite != {}) (mkMerge [{
 
-    assertions = mapAttrsToList (hostName: cfg:
-      { assertion = cfg.database.createLocally -> cfg.database.user == user;
-        message = ''services.wordpress.sites."${hostName}".database.user must be ${user} if the database is to be automatically provisioned'';
-      }
-    ) eachSite;
+    assertions =
+      (mapAttrsToList (hostName: cfg:
+        { assertion = cfg.database.createLocally -> cfg.database.user == user;
+          message = ''services.wordpress.sites."${hostName}".database.user must be ${user} if the database is to be automatically provisioned'';
+        }) eachSite) ++
+      (mapAttrsToList (hostName: cfg:
+        { assertion = cfg.database.createLocally -> cfg.database.passwordFile == null;
+          message = ''services.wordpress.sites."${hostName}".database.passwordFile cannot be specified if services.wordpress.sites."${hostName}".database.createLocally is set to true.'';
+        }) eachSite);
 
-    warnings = mapAttrsToList (hostName: _: ''services.wordpress."${hostName}" is deprecated use services.wordpress.sites."${hostName}"'') (oldSites cfg);
 
     services.mysql = mkIf (any (v: v.database.createLocally) (attrValues eachSite)) {
       enable = true;
@@ -359,7 +349,7 @@ in
 
             DirectoryIndex index.php
             Require all granted
-            Options +FollowSymLinks
+            Options +FollowSymLinks -Indexes
           </Directory>
 
           # https://wordpress.org/support/article/hardening-wordpress/#securing-wp-config-php
diff --git a/nixos/modules/services/web-apps/youtrack.nix b/nixos/modules/services/web-apps/youtrack.nix
index 7a70ae6cd523..b83265ffeab6 100644
--- a/nixos/modules/services/web-apps/youtrack.nix
+++ b/nixos/modules/services/web-apps/youtrack.nix
@@ -128,6 +128,7 @@ in
         Type = "simple";
         User = "youtrack";
         Group = "youtrack";
+        Restart = "on-failure";
         ExecStart = ''${cfg.package}/bin/youtrack --J-Xmx${cfg.maxMemory} --J-XX:MaxMetaspaceSize=${cfg.maxMetaspaceSize} ${cfg.jvmOpts} ${cfg.address}:${toString cfg.port}'';
       };
     };
diff --git a/nixos/modules/services/web-apps/zabbix.nix b/nixos/modules/services/web-apps/zabbix.nix
index ff50b95254f9..538dac0d5be2 100644
--- a/nixos/modules/services/web-apps/zabbix.nix
+++ b/nixos/modules/services/web-apps/zabbix.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 let
 
@@ -6,6 +6,7 @@ let
   inherit (lib) literalExpression mapAttrs optionalString versionAtLeast;
 
   cfg = config.services.zabbixWeb;
+  opt = options.services.zabbixWeb;
   fpm = config.services.phpfpm.pools.zabbix;
 
   user = "zabbix";
@@ -82,6 +83,11 @@ in
             if cfg.database.type == "mysql" then config.services.mysql.port
             else if cfg.database.type == "pgsql" then config.services.postgresql.port
             else 1521;
+          defaultText = literalExpression ''
+            if config.${opt.database.type} == "mysql" then config.${options.services.mysql.port}
+            else if config.${opt.database.type} == "pgsql" then config.${options.services.postgresql.port}
+            else 1521
+          '';
           description = "Database host port.";
         };
 
diff --git a/nixos/modules/services/web-servers/agate.nix b/nixos/modules/services/web-servers/agate.nix
new file mode 100644
index 000000000000..3afdb561c0b0
--- /dev/null
+++ b/nixos/modules/services/web-servers/agate.nix
@@ -0,0 +1,148 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.agate;
+in
+{
+  options = {
+    services.agate = {
+      enable = mkEnableOption "Agate Server";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.agate;
+        defaultText = literalExpression "pkgs.agate";
+        description = "The package to use";
+      };
+
+      addresses = mkOption {
+        type = types.listOf types.str;
+        default = [ "0.0.0.0:1965" ];
+        description = ''
+          Addresses to listen on, IP:PORT, if you haven't disabled forwarding
+          only set IPv4.
+        '';
+      };
+
+      contentDir = mkOption {
+        default = "/var/lib/agate/content";
+        type = types.path;
+        description = "Root of the content directory.";
+      };
+
+      certificatesDir = mkOption {
+        default = "/var/lib/agate/certificates";
+        type = types.path;
+        description = "Root of the certificate directory.";
+      };
+
+      hostnames = mkOption {
+        default = [ ];
+        type = types.listOf types.str;
+        description = ''
+          Domain name of this Gemini server, enables checking hostname and port
+          in requests. (multiple occurences means basic vhosts)
+        '';
+      };
+
+      language = mkOption {
+        default = null;
+        type = types.nullOr types.str;
+        description = "RFC 4646 Language code for text/gemini documents.";
+      };
+
+      onlyTls_1_3 = mkOption {
+        default = false;
+        type = types.bool;
+        description = "Only use TLSv1.3 (default also allows TLSv1.2).";
+      };
+
+      extraArgs = mkOption {
+        type = types.listOf types.str;
+        default = [ "" ];
+        example = [ "--log-ip" ];
+        description = "Extra arguments to use running agate.";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # available for generating certs by hand
+    # it can be a bit arduous with openssl
+    environment.systemPackages = [ cfg.package ];
+
+    systemd.services.agate = {
+      description = "Agate";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" "network-online.target" ];
+
+      script =
+        let
+          prefixKeyList = key: list: concatMap (v: [ key v ]) list;
+          addresses = prefixKeyList "--addr" cfg.addresses;
+          hostnames = prefixKeyList "--hostname" cfg.hostnames;
+        in
+        ''
+          exec ${cfg.package}/bin/agate ${
+            escapeShellArgs (
+              [
+                "--content" "${cfg.contentDir}"
+                "--certs" "${cfg.certificatesDir}"
+              ] ++
+              addresses ++
+              (optionals (cfg.hostnames != []) hostnames) ++
+              (optionals (cfg.language != null) [ "--lang" cfg.language ]) ++
+              (optionals cfg.onlyTls_1_3 [ "--only-tls13" ]) ++
+              (optionals (cfg.extraArgs != []) cfg.extraArgs)
+            )
+          }
+        '';
+
+      serviceConfig = {
+        Restart = "always";
+        RestartSec = "5s";
+        DynamicUser = true;
+        StateDirectory = "agate";
+
+        # Security options:
+        AmbientCapabilities = "";
+        CapabilityBoundingSet = "";
+
+        # ProtectClock= adds DeviceAllow=char-rtc r
+        DeviceAllow = "";
+
+        LockPersonality = true;
+
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateUsers = true;
+
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+
+        RestrictNamespaces = true;
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        RestrictRealtime = true;
+
+        SystemCallArchitectures = "native";
+        SystemCallErrorNumber = "EPERM";
+        SystemCallFilter = [
+          "@system-service"
+          "~@cpu-emulation"
+          "~@debug"
+          "~@keyring"
+          "~@memlock"
+          "~@obsolete"
+          "~@privileged"
+          "~@setuid"
+        ];
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/web-servers/apache-httpd/default.nix b/nixos/modules/services/web-servers/apache-httpd/default.nix
index 992a58875e43..d817ff6019a3 100644
--- a/nixos/modules/services/web-servers/apache-httpd/default.nix
+++ b/nixos/modules/services/web-servers/apache-httpd/default.nix
@@ -154,7 +154,7 @@ let
       sslServerKey = if useACME then "${sslCertDir}/key.pem" else hostOpts.sslServerKey;
       sslServerChain = if useACME then "${sslCertDir}/chain.pem" else hostOpts.sslServerChain;
 
-      acmeChallenge = optionalString useACME ''
+      acmeChallenge = optionalString (useACME && hostOpts.acmeRoot != null) ''
         Alias /.well-known/acme-challenge/ "${hostOpts.acmeRoot}/.well-known/acme-challenge/"
         <Directory "${hostOpts.acmeRoot}">
             AllowOverride None
@@ -370,6 +370,8 @@ let
       cat ${php.phpIni} > $out
       echo "$options" >> $out
     '';
+
+  mkCertOwnershipAssertion = import ../../../security/acme/mk-cert-ownership-assertion.nix;
 in
 
 
@@ -657,7 +659,11 @@ in
           `services.httpd.virtualHosts.<name>.useACMEHost` are mutually exclusive.
         '';
       }
-    ];
+    ] ++ map (name: mkCertOwnershipAssertion {
+      inherit (cfg) group user;
+      cert = config.security.acme.certs.${name};
+      groups = config.users.groups;
+    }) dependentCertNames;
 
     warnings =
       mapAttrsToList (name: hostOpts: ''
@@ -677,9 +683,16 @@ in
     };
 
     security.acme.certs = let
-      acmePairs = map (hostOpts: nameValuePair hostOpts.hostName {
+      acmePairs = map (hostOpts: let
+        hasRoot = hostOpts.acmeRoot != null;
+      in nameValuePair hostOpts.hostName {
         group = mkDefault cfg.group;
-        webroot = hostOpts.acmeRoot;
+        # if acmeRoot is null inherit config.security.acme
+        # Since config.security.acme.certs.<cert>.webroot's own default value
+        # should take precedence set priority higher than mkOptionDefault
+        webroot = mkOverride (if hasRoot then 1000 else 2000) hostOpts.acmeRoot;
+        # Also nudge dnsProvider to null in case it is inherited
+        dnsProvider = mkOverride (if hasRoot then 1000 else 2000) null;
         extraDomainNames = hostOpts.serverAliases;
         # Use the vhost-specific email address if provided, otherwise let
         # security.acme.email or security.acme.certs.<cert>.email be used.
diff --git a/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix b/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix
index 8bb7e91ec9cd..c52ab2c596e0 100644
--- a/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix
+++ b/nixos/modules/services/web-servers/apache-httpd/vhost-options.nix
@@ -128,9 +128,12 @@ in
     };
 
     acmeRoot = mkOption {
-      type = types.str;
+      type = types.nullOr types.str;
       default = "/var/lib/acme/acme-challenge";
-      description = "Directory for the acme challenge which is PUBLIC, don't put certs or keys in here";
+      description = ''
+        Directory for the acme challenge which is PUBLIC, don't put certs or keys in here.
+        Set to null to inherit from config.security.acme.
+      '';
     };
 
     sslServerCert = mkOption {
diff --git a/nixos/modules/services/web-servers/caddy/default.nix b/nixos/modules/services/web-servers/caddy/default.nix
index ed27dd375c86..2b8c6f2e308b 100644
--- a/nixos/modules/services/web-servers/caddy/default.nix
+++ b/nixos/modules/services/web-servers/caddy/default.nix
@@ -4,111 +4,159 @@ with lib;
 
 let
   cfg = config.services.caddy;
-  vhostToConfig = vhostName: vhostAttrs: ''
-    ${vhostName} ${builtins.concatStringsSep " " vhostAttrs.serverAliases} {
-      ${vhostAttrs.extraConfig}
-    }
-  '';
-  configFile = pkgs.writeText "Caddyfile" (builtins.concatStringsSep "\n"
-    ([ cfg.config ] ++ (mapAttrsToList vhostToConfig cfg.virtualHosts)));
-
-  formattedConfig = pkgs.runCommand "formattedCaddyFile" { } ''
-    ${cfg.package}/bin/caddy fmt ${configFile} > $out
-  '';
-
-  tlsConfig = {
-    apps.tls.automation.policies = [{
-      issuers = [{
-        inherit (cfg) ca email;
-        module = "acme";
-      }];
-    }];
-  };
 
-  adaptedConfig = pkgs.runCommand "caddy-config-adapted.json" { } ''
-    ${cfg.package}/bin/caddy adapt \
-      --config ${formattedConfig} --adapter ${cfg.adapter} > $out
-  '';
-  tlsJSON = pkgs.writeText "tls.json" (builtins.toJSON tlsConfig);
-
-  # merge the TLS config options we expose with the ones originating in the Caddyfile
-  configJSON =
-    if cfg.ca != null then
-      let tlsConfigMerge = ''
-        {"apps":
-          {"tls":
-            {"automation":
-              {"policies":
-                (if .[0].apps.tls.automation.policies == .[1]?.apps.tls.automation.policies
-                 then .[0].apps.tls.automation.policies
-                 else (.[0].apps.tls.automation.policies + .[1]?.apps.tls.automation.policies)
-                 end)
-              }
-            }
-          }
-        }'';
-      in
-      pkgs.runCommand "caddy-config.json" { } ''
-        ${pkgs.jq}/bin/jq -s '.[0] * ${tlsConfigMerge}' ${adaptedConfig} ${tlsJSON} > $out
+  virtualHosts = attrValues cfg.virtualHosts;
+  acmeVHosts = filter (hostOpts: hostOpts.useACMEHost != null) virtualHosts;
+
+  mkVHostConf = hostOpts:
+    let
+      sslCertDir = config.security.acme.certs.${hostOpts.useACMEHost}.directory;
+    in
       ''
-    else
-      adaptedConfig;
+        ${hostOpts.hostName} ${concatStringsSep " " hostOpts.serverAliases} {
+          bind ${concatStringsSep " " hostOpts.listenAddresses}
+          ${optionalString (hostOpts.useACMEHost != null) "tls ${sslCertDir}/cert.pem ${sslCertDir}/key.pem"}
+          log {
+            ${hostOpts.logFormat}
+          }
+
+          ${hostOpts.extraConfig}
+        }
+      '';
+
+  configFile =
+    let
+      Caddyfile = pkgs.writeText "Caddyfile" ''
+        {
+          ${cfg.globalConfig}
+        }
+        ${cfg.extraConfig}
+      '';
+
+      Caddyfile-formatted = pkgs.runCommand "Caddyfile-formatted" { nativeBuildInputs = [ cfg.package ]; } ''
+        ${cfg.package}/bin/caddy fmt ${Caddyfile} > $out
+      '';
+    in
+      if pkgs.stdenv.buildPlatform == pkgs.stdenv.hostPlatform then Caddyfile-formatted else Caddyfile;
+
+  acmeHosts = unique (catAttrs "useACMEHost" acmeVHosts);
+
+  mkCertOwnershipAssertion = import ../../../security/acme/mk-cert-ownership-assertion.nix;
 in
 {
   imports = [
     (mkRemovedOptionModule [ "services" "caddy" "agree" ] "this option is no longer necessary for Caddy 2")
+    (mkRenamedOptionModule [ "services" "caddy" "ca" ] [ "services" "caddy" "acmeCA" ])
+    (mkRenamedOptionModule [ "services" "caddy" "config" ] [ "services" "caddy" "extraConfig" ])
   ];
 
+  # interface
   options.services.caddy = {
     enable = mkEnableOption "Caddy web server";
 
-    config = mkOption {
-      default = "";
-      example = ''
-        example.com {
-          encode gzip
-          log
-          root /srv/http
-        }
+    user = mkOption {
+      default = "caddy";
+      type = types.str;
+      description = ''
+        User account under which caddy runs.
+
+        <note><para>
+          If left as the default value this user will automatically be created
+          on system activation, otherwise you are responsible for
+          ensuring the user exists before the Caddy service starts.
+        </para></note>
       '';
-      type = types.lines;
+    };
+
+    group = mkOption {
+      default = "caddy";
+      type = types.str;
       description = ''
-        Verbatim Caddyfile to use.
-        Caddy v2 supports multiple config formats via adapters (see <option>services.caddy.adapter</option>).
+        Group account under which caddy runs.
+
+        <note><para>
+          If left as the default value this user will automatically be created
+          on system activation, otherwise you are responsible for
+          ensuring the user exists before the Caddy service starts.
+        </para></note>
       '';
     };
 
-    virtualHosts = mkOption {
-      type = types.attrsOf (types.submodule (import ./vhost-options.nix {
-        inherit config lib;
-      }));
-      default = { };
-      example = literalExpression ''
-        {
-          "hydra.example.com" = {
-            serverAliases = [ "www.hydra.example.com" ];
-            extraConfig = ''''''
-              encode gzip
-              log
-              root /srv/http
-            '''''';
-          };
-        };
+    package = mkOption {
+      default = pkgs.caddy;
+      defaultText = literalExpression "pkgs.caddy";
+      type = types.package;
+      description = ''
+        Caddy package to use.
       '';
-      description = "Declarative vhost config";
     };
 
+    dataDir = mkOption {
+      type = types.path;
+      default = "/var/lib/caddy";
+      description = ''
+        The data directory for caddy.
 
-    user = mkOption {
-      default = "caddy";
-      type = types.str;
-      description = "User account under which caddy runs.";
+        <note>
+          <para>
+            If left as the default value this directory will automatically be created
+            before the Caddy server starts, otherwise you are responsible for ensuring
+            the directory exists with appropriate ownership and permissions.
+          </para>
+          <para>
+            Caddy v2 replaced <literal>CADDYPATH</literal> with XDG directories.
+            See <link xlink:href="https://caddyserver.com/docs/conventions#file-locations"/>.
+          </para>
+        </note>
+      '';
     };
 
-    group = mkOption {
-      default = "caddy";
-      type = types.str;
-      description = "Group account under which caddy runs.";
+    logDir = mkOption {
+      type = types.path;
+      default = "/var/log/caddy";
+      description = ''
+        Directory for storing Caddy access logs.
+
+        <note><para>
+          If left as the default value this directory will automatically be created
+          before the Caddy server starts, otherwise the sysadmin is responsible for
+          ensuring the directory exists with appropriate ownership and permissions.
+        </para></note>
+      '';
+    };
+
+    logFormat = mkOption {
+      type = types.lines;
+      default = ''
+        level ERROR
+      '';
+      example = literalExpression ''
+        mkForce "level INFO";
+      '';
+      description = ''
+        Configuration for the default logger. See
+        <link xlink:href="https://caddyserver.com/docs/caddyfile/options#log"/>
+        for details.
+      '';
+    };
+
+    configFile = mkOption {
+      type = types.path;
+      default = configFile;
+      defaultText = "A Caddyfile automatically generated by values from services.caddy.*";
+      example = literalExpression ''
+        pkgs.writeText "Caddyfile" '''
+          example.com
+
+          root * /var/www/wordpress
+          php_fastcgi unix//run/php/php-version-fpm.sock
+          file_server
+        ''';
+      '';
+      description = ''
+        Override the configuration file used by Caddy. By default,
+        NixOS generates one automatically.
+      '';
     };
 
     adapter = mkOption {
@@ -117,7 +165,13 @@ in
       type = types.str;
       description = ''
         Name of the config adapter to use.
-        See https://caddyserver.com/docs/config-adapters for the full list.
+        See <link xlink:href="https://caddyserver.com/docs/config-adapters"/>
+        for the full list.
+
+        <note><para>
+          Any value other than <literal>caddyfile</literal> is only valid when
+          providing your own <option>configFile</option>.
+        </para></note>
       '';
     };
 
@@ -125,54 +179,118 @@ in
       default = false;
       type = types.bool;
       description = ''
-        Use saved config, if any (and prefer over configuration passed with <option>services.caddy.config</option>).
+        Use saved config, if any (and prefer over any specified configuration passed with <literal>--config</literal>).
       '';
     };
 
-    ca = mkOption {
-      default = "https://acme-v02.api.letsencrypt.org/directory";
-      example = "https://acme-staging-v02.api.letsencrypt.org/directory";
-      type = types.nullOr types.str;
+    globalConfig = mkOption {
+      type = types.lines;
+      default = "";
+      example = ''
+        debug
+        servers {
+          protocol {
+            experimental_http3
+          }
+        }
+      '';
       description = ''
-        Certificate authority ACME server. The default (Let's Encrypt
-        production server) should be fine for most people. Set it to null if
-        you don't want to include any authority (or if you want to write a more
-        fine-graned configuration manually)
+        Additional lines of configuration appended to the global config section
+        of the <literal>Caddyfile</literal>.
+
+        Refer to <link xlink:href="https://caddyserver.com/docs/caddyfile/options#global-options"/>
+        for details on supported values.
       '';
     };
 
-    email = mkOption {
+    extraConfig = mkOption {
+      type = types.lines;
       default = "";
-      type = types.str;
-      description = "Email address (for Let's Encrypt certificate)";
+      example = ''
+        example.com {
+          encode gzip
+          log
+          root /srv/http
+        }
+      '';
+      description = ''
+        Additional lines of configuration appended to the automatically
+        generated <literal>Caddyfile</literal>.
+      '';
     };
 
-    dataDir = mkOption {
-      default = "/var/lib/caddy";
-      type = types.path;
+    virtualHosts = mkOption {
+      type = with types; attrsOf (submodule (import ./vhost-options.nix { inherit cfg; }));
+      default = {};
+      example = literalExpression ''
+        {
+          "hydra.example.com" = {
+            serverAliases = [ "www.hydra.example.com" ];
+            extraConfig = '''
+              encode gzip
+              root /srv/http
+            ''';
+          };
+        };
+      '';
       description = ''
-        The data directory, for storing certificates. Before 17.09, this
-        would create a .caddy directory. With 17.09 the contents of the
-        .caddy directory are in the specified data directory instead.
+        Declarative specification of virtual hosts served by Caddy.
+      '';
+    };
 
-        Caddy v2 replaced CADDYPATH with XDG directories.
-        See https://caddyserver.com/docs/conventions#file-locations.
+    acmeCA = mkOption {
+      default = "https://acme-v02.api.letsencrypt.org/directory";
+      example = "https://acme-staging-v02.api.letsencrypt.org/directory";
+      type = with types; nullOr str;
+      description = ''
+        The URL to the ACME CA's directory. It is strongly recommended to set
+        this to Let's Encrypt's staging endpoint for testing or development.
+
+        Set it to <literal>null</literal> if you want to write a more
+        fine-grained configuration manually.
       '';
     };
 
-    package = mkOption {
-      default = pkgs.caddy;
-      defaultText = literalExpression "pkgs.caddy";
-      type = types.package;
+    email = mkOption {
+      default = null;
+      type = with types; nullOr str;
       description = ''
-        Caddy package to use.
+        Your email address. Mainly used when creating an ACME account with your
+        CA, and is highly recommended in case there are problems with your
+        certificates.
       '';
     };
+
   };
 
+  # implementation
   config = mkIf cfg.enable {
+
+    assertions = [
+      { assertion = cfg.adapter != "caddyfile" -> cfg.configFile != configFile;
+        message = "Any value other than 'caddyfile' is only valid when providing your own `services.caddy.configFile`";
+      }
+    ] ++ map (name: mkCertOwnershipAssertion {
+      inherit (cfg) group user;
+      cert = config.security.acme.certs.${name};
+      groups = config.users.groups;
+    }) acmeHosts;
+
+    services.caddy.extraConfig = concatMapStringsSep "\n" mkVHostConf virtualHosts;
+    services.caddy.globalConfig = ''
+      ${optionalString (cfg.email != null) "email ${cfg.email}"}
+      ${optionalString (cfg.acmeCA != null) "acme_ca ${cfg.acmeCA}"}
+      log {
+        ${cfg.logFormat}
+      }
+    '';
+
     systemd.packages = [ cfg.package ];
     systemd.services.caddy = {
+      wants = map (hostOpts: "acme-finished-${hostOpts.useACMEHost}.target") acmeVHosts;
+      after = map (hostOpts: "acme-selfsigned-${hostOpts.useACMEHost}.service") acmeVHosts;
+      before = map (hostOpts: "acme-${hostOpts.useACMEHost}.service") acmeVHosts;
+
       wantedBy = [ "multi-user.target" ];
       startLimitIntervalSec = 14400;
       startLimitBurst = 10;
@@ -180,13 +298,17 @@ in
       serviceConfig = {
         # https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStart=
         # If the empty string is assigned to this option, the list of commands to start is reset, prior assignments of this option will have no effect.
-        ExecStart = [ "" "${cfg.package}/bin/caddy run ${optionalString cfg.resume "--resume"} --config ${configJSON}" ];
-        ExecReload = [ "" "${cfg.package}/bin/caddy reload --config ${configJSON}" ];
+        ExecStart = [ "" "${cfg.package}/bin/caddy run --config ${cfg.configFile} --adapter ${cfg.adapter} ${optionalString cfg.resume "--resume"}" ];
+        ExecReload = [ "" "${cfg.package}/bin/caddy reload --config ${cfg.configFile} --adapter ${cfg.adapter}" ];
 
+        ExecStartPre = "${cfg.package}/bin/caddy validate --config ${cfg.configFile} --adapter ${cfg.adapter}";
         User = cfg.user;
         Group = cfg.group;
         ReadWriteDirectories = cfg.dataDir;
+        StateDirectory = mkIf (cfg.dataDir == "/var/lib/caddy") [ "caddy" ];
+        LogsDirectory = mkIf (cfg.logDir == "/var/log/caddy") [ "caddy" ];
         Restart = "on-abnormal";
+        SupplementaryGroups = mkIf (length acmeVHosts != 0) [ "acme" ];
 
         # TODO: attempt to upstream these options
         NoNewPrivileges = true;
@@ -200,7 +322,6 @@ in
         group = cfg.group;
         uid = config.ids.uids.caddy;
         home = cfg.dataDir;
-        createHome = true;
       };
     };
 
@@ -208,5 +329,11 @@ in
       caddy.gid = config.ids.gids.caddy;
     };
 
+    security.acme.certs =
+      let
+        reloads = map (useACMEHost: nameValuePair useACMEHost { reloadServices = [ "caddy.service" ]; }) acmeHosts;
+      in
+        listToAttrs reloads;
+
   };
 }
diff --git a/nixos/modules/services/web-servers/caddy/vhost-options.nix b/nixos/modules/services/web-servers/caddy/vhost-options.nix
index 1f74295fc9a2..f240ec605c29 100644
--- a/nixos/modules/services/web-servers/caddy/vhost-options.nix
+++ b/nixos/modules/services/web-servers/caddy/vhost-options.nix
@@ -1,15 +1,19 @@
-# This file defines the options that can be used both for the Nginx
-# main server configuration, and for the virtual hosts.  (The latter
-# has additional options that affect the web server as a whole, like
-# the user/group to run under.)
-
-{ lib, ... }:
-
-with lib;
+{ cfg }:
+{ config, lib, name, ... }:
+let
+  inherit (lib) literalExpression mkOption types;
+in
 {
   options = {
+
+    hostName = mkOption {
+      type = types.str;
+      default = name;
+      description = "Canonical hostname for the server.";
+    };
+
     serverAliases = mkOption {
-      type = types.listOf types.str;
+      type = with types; listOf str;
       default = [ ];
       example = [ "www.example.org" "example.org" ];
       description = ''
@@ -17,12 +21,59 @@ with lib;
       '';
     };
 
+    listenAddresses = mkOption {
+      type = with types; listOf str;
+      description = ''
+        A list of host interfaces to bind to for this virtual host.
+      '';
+      default = [ ];
+      example = [ "127.0.0.1" "::1" ];
+    };
+
+    useACMEHost = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        A host of an existing Let's Encrypt certificate to use.
+        This is mostly useful if you use DNS challenges but Caddy does not
+        currently support your provider.
+
+        <emphasis>Note that this option does not create any certificates, nor
+        does it add subdomains to existing ones – you will need to create them
+        manually using <xref linkend="opt-security.acme.certs"/>. Additionally,
+        you should probably add the <literal>caddy</literal> user to the
+        <literal>acme</literal> group to grant access to the certificates.</emphasis>
+      '';
+    };
+
+    logFormat = mkOption {
+      type = types.lines;
+      default = ''
+        output file ${cfg.logDir}/access-${config.hostName}.log
+      '';
+      defaultText = ''
+        output file ''${config.services.caddy.logDir}/access-''${hostName}.log
+      '';
+      example = literalExpression ''
+        mkForce '''
+          output discard
+        ''';
+      '';
+      description = ''
+        Configuration for HTTP request logging (also known as access logs). See
+        <link xlink:href="https://caddyserver.com/docs/caddyfile/directives/log#log"/>
+        for details.
+      '';
+    };
+
     extraConfig = mkOption {
       type = types.lines;
       default = "";
       description = ''
-        These lines go into the vhost verbatim
+        Additional lines of configuration appended to this virtual host in the
+        automatically generated <literal>Caddyfile</literal>.
       '';
     };
+
   };
 }
diff --git a/nixos/modules/services/web-servers/lighttpd/collectd.nix b/nixos/modules/services/web-servers/lighttpd/collectd.nix
index 3f262451c2cb..5f091591daf9 100644
--- a/nixos/modules/services/web-servers/lighttpd/collectd.nix
+++ b/nixos/modules/services/web-servers/lighttpd/collectd.nix
@@ -1,9 +1,10 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
   cfg = config.services.lighttpd.collectd;
+  opt = options.services.lighttpd.collectd;
 
   collectionConf = pkgs.writeText "collection.conf" ''
     datadir: "${config.services.collectd.dataDir}"
@@ -29,6 +30,9 @@ in
     collectionCgi = mkOption {
       type = types.path;
       default = defaultCollectionCgi;
+      defaultText = literalDocBook ''
+        <literal>config.${options.services.collectd.package}</literal> configured for lighttpd
+      '';
       description = ''
         Path to collection.cgi script from (collectd sources)/contrib/collection.cgi
         This option allows to use a customized version
diff --git a/nixos/modules/services/web-servers/nginx/default.nix b/nixos/modules/services/web-servers/nginx/default.nix
index 96e45cfc4f77..e046c28dd6bb 100644
--- a/nixos/modules/services/web-servers/nginx/default.nix
+++ b/nixos/modules/services/web-servers/nginx/default.nix
@@ -245,12 +245,9 @@ let
         defaultListen =
           if vhost.listen != [] then vhost.listen
           else
-            let addrs = if vhost.listenAddresses != [] then vhost.listenAddresses else (
-              [ "0.0.0.0" ] ++ optional enableIPv6 "[::0]"
-            );
-            in
-          optionals (hasSSL || vhost.rejectSSL) (map (addr: { inherit addr; port = 443; ssl = true; }) addrs)
-          ++ optionals (!onlySSL) (map (addr: { inherit addr; port = 80; ssl = false; }) addrs);
+            let addrs = if vhost.listenAddresses != [] then vhost.listenAddresses else cfg.defaultListenAddresses;
+            in optionals (hasSSL || vhost.rejectSSL) (map (addr: { inherit addr; port = 443; ssl = true; }) addrs)
+              ++ optionals (!onlySSL) (map (addr: { inherit addr; port = 80; ssl = false; }) addrs);
 
         hostListen =
           if vhost.forceSSL
@@ -278,7 +275,7 @@ let
         acmeLocation = optionalString (vhost.enableACME || vhost.useACMEHost != null) ''
           location /.well-known/acme-challenge {
             ${optionalString (vhost.acmeFallbackHost != null) "try_files $uri @acme-fallback;"}
-            root ${vhost.acmeRoot};
+            ${optionalString (vhost.acmeRoot != null) "root ${vhost.acmeRoot};"}
             auth_basic off;
           }
           ${optionalString (vhost.acmeFallbackHost != null) ''
@@ -320,6 +317,9 @@ let
           ${optionalString vhost.rejectSSL ''
             ssl_reject_handshake on;
           ''}
+          ${optionalString (hasSSL && vhost.kTLS) ''
+            ssl_conf_command Options KTLS;
+          ''}
 
           ${mkBasicAuth vhostName vhost}
 
@@ -371,6 +371,8 @@ let
       ${user}:{PLAIN}${password}
     '') authDef)
   );
+
+  mkCertOwnershipAssertion = import ../../../security/acme/mk-cert-ownership-assertion.nix;
 in
 
 {
@@ -427,6 +429,16 @@ in
         ";
       };
 
+      defaultListenAddresses = mkOption {
+        type = types.listOf types.str;
+        default = [ "0.0.0.0" ] ++ optional enableIPv6 "[::0]";
+        defaultText = literalExpression ''[ "0.0.0.0" ] ++ lib.optional config.networking.enableIPv6 "[::0]"'';
+        example = literalExpression ''[ "10.0.0.12" "[2002:a00:1::]" ]'';
+        description = "
+          If vhosts do not specify listenAddresses, use these addresses by default.
+        ";
+      };
+
       package = mkOption {
         default = pkgs.nginxStable;
         defaultText = literalExpression "pkgs.nginxStable";
@@ -825,13 +837,25 @@ in
       }
 
       {
+        assertion = any (host: host.kTLS) (attrValues virtualHosts) -> versionAtLeast cfg.package.version "1.21.4";
+        message = ''
+          services.nginx.virtualHosts.<name>.kTLS requires nginx version
+          1.21.4 or above; see the documentation for services.nginx.package.
+        '';
+      }
+
+      {
         assertion = all (host: !(host.enableACME && host.useACMEHost != null)) (attrValues virtualHosts);
         message = ''
           Options services.nginx.service.virtualHosts.<name>.enableACME and
           services.nginx.virtualHosts.<name>.useACMEHost are mutually exclusive.
         '';
       }
-    ];
+    ] ++ map (name: mkCertOwnershipAssertion {
+      inherit (cfg) group user;
+      cert = config.security.acme.certs.${name};
+      groups = config.users.groups;
+    }) dependentCertNames;
 
     systemd.services.nginx = {
       description = "Nginx Web Server";
@@ -900,7 +924,8 @@ in
         PrivateMounts = true;
         # System Call Filtering
         SystemCallArchitectures = "native";
-        SystemCallFilter = "~@cpu-emulation @debug @keyring @ipc @mount @obsolete @privileged @setuid @mincore";
+        SystemCallFilter = [ "~@cpu-emulation @debug @keyring @mount @obsolete @privileged @setuid" ]
+          ++ optionals ((cfg.package != pkgs.tengine) && (!lib.any (mod: (mod.disableIPC or false)) cfg.package.modules)) [ "~@ipc" ];
       };
     };
 
@@ -937,9 +962,16 @@ in
     };
 
     security.acme.certs = let
-      acmePairs = map (vhostConfig: nameValuePair vhostConfig.serverName {
+      acmePairs = map (vhostConfig: let
+        hasRoot = vhostConfig.acmeRoot != null;
+      in nameValuePair vhostConfig.serverName {
         group = mkDefault cfg.group;
-        webroot = vhostConfig.acmeRoot;
+        # if acmeRoot is null inherit config.security.acme
+        # Since config.security.acme.certs.<cert>.webroot's own default value
+        # should take precedence set priority higher than mkOptionDefault
+        webroot = mkOverride (if hasRoot then 1000 else 2000) vhostConfig.acmeRoot;
+        # Also nudge dnsProvider to null in case it is inherited
+        dnsProvider = mkOverride (if hasRoot then 1000 else 2000) null;
         extraDomainNames = vhostConfig.serverAliases;
       # Filter for enableACME-only vhosts. Don't want to create dud certs
       }) (filter (vhostConfig: vhostConfig.useACMEHost == null) acmeEnabledVhosts);
@@ -957,5 +989,17 @@ in
       nginx.gid = config.ids.gids.nginx;
     };
 
+    services.logrotate.paths.nginx = mapAttrs (_: mkDefault) {
+      path = "/var/log/nginx/*.log";
+      frequency = "weekly";
+      keep = 26;
+      extraConfig = ''
+        compress
+        delaycompress
+        postrotate
+          [ ! -f /var/run/nginx/nginx.pid ] || kill -USR1 `cat /var/run/nginx/nginx.pid`
+        endscript
+      '';
+    };
   };
 }
diff --git a/nixos/modules/services/web-servers/nginx/gitweb.nix b/nixos/modules/services/web-servers/nginx/gitweb.nix
index 11bf2a309ea8..db45577a46d1 100644
--- a/nixos/modules/services/web-servers/nginx/gitweb.nix
+++ b/nixos/modules/services/web-servers/nginx/gitweb.nix
@@ -79,7 +79,7 @@ in
         };
         locations."${cfg.location}/" = {
           extraConfig = ''
-            include ${pkgs.nginx}/conf/fastcgi_params;
+            include ${config.services.nginx.package}/conf/fastcgi_params;
             fastcgi_param GITWEB_CONFIG ${gitwebConfig.gitwebConfigFile};
             fastcgi_pass unix:/run/gitweb/gitweb.sock;
           '';
diff --git a/nixos/modules/services/web-servers/nginx/vhost-options.nix b/nixos/modules/services/web-servers/nginx/vhost-options.nix
index 7ee041d37211..c4e8285dc48b 100644
--- a/nixos/modules/services/web-servers/nginx/vhost-options.nix
+++ b/nixos/modules/services/web-servers/nginx/vhost-options.nix
@@ -3,7 +3,7 @@
 # has additional options that affect the web server as a whole, like
 # the user/group to run under.)
 
-{ lib, ... }:
+{ config, lib, ... }:
 
 with lib;
 {
@@ -85,9 +85,12 @@ with lib;
     };
 
     acmeRoot = mkOption {
-      type = types.str;
+      type = types.nullOr types.str;
       default = "/var/lib/acme/acme-challenge";
-      description = "Directory for the acme challenge which is PUBLIC, don't put certs or keys in here";
+      description = ''
+        Directory for the acme challenge which is PUBLIC, don't put certs or keys in here.
+        Set to null to inherit from config.security.acme.
+      '';
     };
 
     acmeFallbackHost = mkOption {
@@ -147,6 +150,17 @@ with lib;
       '';
     };
 
+    kTLS = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to enable kTLS support.
+        Implementing TLS in the kernel (kTLS) improves performance by significantly
+        reducing the need for copying operations between user space and the kernel.
+        Required Nginx version 1.21.4 or later.
+      '';
+    };
+
     sslCertificate = mkOption {
       type = types.path;
       example = "/var/host.cert";
diff --git a/nixos/modules/services/web-servers/pomerium.nix b/nixos/modules/services/web-servers/pomerium.nix
index 2bc7d01c7c28..0b460755f50e 100644
--- a/nixos/modules/services/web-servers/pomerium.nix
+++ b/nixos/modules/services/web-servers/pomerium.nix
@@ -69,11 +69,16 @@ in
         CERTIFICATE_KEY_FILE = "key.pem";
       };
       startLimitIntervalSec = 60;
+      script = ''
+        if [[ -v CREDENTIALS_DIRECTORY ]]; then
+          cd "$CREDENTIALS_DIRECTORY"
+        fi
+        exec "${pkgs.pomerium}/bin/pomerium" -config "${cfgFile}"
+      '';
 
       serviceConfig = {
         DynamicUser = true;
         StateDirectory = [ "pomerium" ];
-        ExecStart = "${pkgs.pomerium}/bin/pomerium -config ${cfgFile}";
 
         PrivateUsers = false;  # breaks CAP_NET_BIND_SERVICE
         MemoryDenyWriteExecute = false;  # breaks LuaJIT
@@ -99,7 +104,6 @@ in
         AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
         CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
 
-        WorkingDirectory = mkIf (cfg.useACMEHost != null) "$CREDENTIALS_DIRECTORY";
         LoadCredential = optionals (cfg.useACMEHost != null) [
           "fullchain.pem:/var/lib/acme/${cfg.useACMEHost}/fullchain.pem"
           "key.pem:/var/lib/acme/${cfg.useACMEHost}/key.pem"
@@ -124,7 +128,7 @@ in
         Type = "oneshot";
         TimeoutSec = 60;
         ExecCondition = "/run/current-system/systemd/bin/systemctl -q is-active pomerium.service";
-        ExecStart = "/run/current-system/systemd/bin/systemctl restart pomerium.service";
+        ExecStart = "/run/current-system/systemd/bin/systemctl --no-block restart pomerium.service";
       };
     };
   });
diff --git a/nixos/modules/services/web-servers/shellinabox.nix b/nixos/modules/services/web-servers/shellinabox.nix
deleted file mode 100644
index c7c51f873eba..000000000000
--- a/nixos/modules/services/web-servers/shellinabox.nix
+++ /dev/null
@@ -1,122 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-
-  cfg = config.services.shellinabox;
-
-  # If a certificate file is specified, shellinaboxd requires
-  # a file descriptor to retrieve it
-  fd = "3";
-  createFd = optionalString (cfg.certFile != null) "${fd}<${cfg.certFile}";
-
-  # Command line arguments for the shellinabox daemon
-  args = [ "--background" ]
-   ++ optional (! cfg.enableSSL) "--disable-ssl"
-   ++ optional (cfg.certFile != null) "--cert-fd=${fd}"
-   ++ optional (cfg.certDirectory != null) "--cert=${cfg.certDirectory}"
-   ++ cfg.extraOptions;
-
-  # Command to start shellinaboxd
-  cmd = "${pkgs.shellinabox}/bin/shellinaboxd ${concatStringsSep " " args}";
-
-  # Command to start shellinaboxd if certFile is specified
-  wrappedCmd = "${pkgs.bash}/bin/bash -c 'exec ${createFd} && ${cmd}'";
-
-in
-
-{
-
-  ###### interface
-
-  options = {
-    services.shellinabox = {
-      enable = mkEnableOption "shellinabox daemon";
-
-      user = mkOption {
-        type = types.str;
-        default = "root";
-        description = ''
-          User to run shellinaboxd as. If started as root, the server drops
-          privileges by changing to nobody, unless overridden by the
-          <literal>--user</literal> option.
-        '';
-      };
-
-      enableSSL = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether or not to enable SSL (https) support.
-        '';
-      };
-
-      certDirectory = mkOption {
-        type = types.nullOr types.path;
-        default = null;
-        example = "/var/certs";
-        description = ''
-          The daemon will look in this directory far any certificates.
-          If the browser negotiated a Server Name Identification the daemon
-          will look for a matching certificate-SERVERNAME.pem file. If no SNI
-          handshake takes place, it will fall back on using the certificate in the
-          certificate.pem file.
-
-          If no suitable certificate is installed, shellinaboxd will attempt to
-          create a new self-signed certificate. This will only succeed if, after
-          dropping privileges, shellinaboxd has write permissions for this
-          directory.
-        '';
-      };
-
-      certFile = mkOption {
-        type = types.nullOr types.path;
-        default = null;
-        example = "/var/certificate.pem";
-        description = "Path to server SSL certificate.";
-      };
-
-      extraOptions = mkOption {
-        type = types.listOf types.str;
-        default = [ ];
-        example = [ "--port=443" "--service /:LOGIN" ];
-        description = ''
-          A list of strings to be appended to the command line arguments
-          for shellinaboxd. Please see the manual page
-          <link xlink:href="https://code.google.com/p/shellinabox/wiki/shellinaboxd_man"/>
-          for a full list of available arguments.
-        '';
-      };
-
-    };
-  };
-
-  ###### implementation
-
-  config = mkIf cfg.enable {
-
-    assertions =
-      [ { assertion = cfg.enableSSL == true
-            -> cfg.certDirectory != null || cfg.certFile != null;
-          message = "SSL is enabled for shellinabox, but no certDirectory or certFile has been specefied."; }
-        { assertion = ! (cfg.certDirectory != null && cfg.certFile != null);
-          message = "Cannot set both certDirectory and certFile for shellinabox."; }
-      ];
-
-    systemd.services.shellinaboxd = {
-      description = "Shellinabox Web Server Daemon";
-
-      wantedBy = [ "multi-user.target" ];
-      requires = [ "sshd.service" ];
-      after = [ "sshd.service" ];
-
-      serviceConfig = {
-        Type = "forking";
-        User = "${cfg.user}";
-        ExecStart = "${if cfg.certFile == null then "${cmd}" else "${wrappedCmd}"}";
-        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
-      };
-    };
-  };
-}
diff --git a/nixos/modules/services/web-servers/tomcat.nix b/nixos/modules/services/web-servers/tomcat.nix
index f9446fe125a3..877097cf3781 100644
--- a/nixos/modules/services/web-servers/tomcat.nix
+++ b/nixos/modules/services/web-servers/tomcat.nix
@@ -23,8 +23,8 @@ in
 
       package = mkOption {
         type = types.package;
-        default = pkgs.tomcat85;
-        defaultText = literalExpression "pkgs.tomcat85";
+        default = pkgs.tomcat9;
+        defaultText = literalExpression "pkgs.tomcat9";
         example = lib.literalExpression "pkgs.tomcat9";
         description = ''
           Which tomcat package to use.
@@ -127,7 +127,7 @@ in
       webapps = mkOption {
         type = types.listOf types.path;
         default = [ tomcat.webapps ];
-        defaultText = literalExpression "[ pkgs.tomcat85.webapps ]";
+        defaultText = literalExpression "[ config.services.tomcat.package.webapps ]";
         description = "List containing WAR files or directories with WAR files which are web applications to be deployed on Tomcat";
       };
 
@@ -201,6 +201,7 @@ in
       { uid = config.ids.uids.tomcat;
         description = "Tomcat user";
         home = "/homeless-shelter";
+        group = "tomcat";
         extraGroups = cfg.extraGroups;
       };
 
diff --git a/nixos/modules/services/web-servers/uwsgi.nix b/nixos/modules/services/web-servers/uwsgi.nix
index a1cad17336d8..1b3474f2f521 100644
--- a/nixos/modules/services/web-servers/uwsgi.nix
+++ b/nixos/modules/services/web-servers/uwsgi.nix
@@ -20,10 +20,11 @@ let
 
   buildCfg = name: c:
     let
-      plugins =
+      plugins' =
         if any (n: !any (m: m == n) cfg.plugins) (c.plugins or [])
         then throw "`plugins` attribute in uWSGI configuration contains plugins not in config.services.uwsgi.plugins"
         else c.plugins or cfg.plugins;
+      plugins = unique plugins';
 
       hasPython = v: filter (n: n == "python${v}") plugins != [];
       hasPython2 = hasPython "2";
@@ -48,13 +49,10 @@ let
                 pyhome = "${pythonEnv}";
                 env =
                   # Argh, uwsgi expects list of key-values there instead of a dictionary.
-                  let env' = c.env or [];
-                      getPath =
-                        x: if hasPrefix "PATH=" x
-                           then substring (stringLength "PATH=") (stringLength x) x
-                           else null;
-                      oldPaths = filter (x: x != null) (map getPath env');
-                  in env' ++ [ "PATH=${optionalString (oldPaths != []) "${last oldPaths}:"}${pythonEnv}/bin" ];
+                  let envs = partition (hasPrefix "PATH=") (c.env or []);
+                      oldPaths = map (x: substring (stringLength "PATH=") (stringLength x) x) envs.right;
+                      paths = oldPaths ++ [ "${pythonEnv}/bin" ];
+                  in [ "PATH=${concatStringsSep ":" paths}" ] ++ envs.wrong;
               }
           else if isEmperor
             then {
@@ -225,7 +223,7 @@ in {
     };
 
     services.uwsgi.package = pkgs.uwsgi.override {
-      inherit (cfg) plugins;
+      plugins = unique cfg.plugins;
     };
   };
 }
diff --git a/nixos/modules/services/x11/desktop-managers/cinnamon.nix b/nixos/modules/services/x11/desktop-managers/cinnamon.nix
index 82b07206a8b6..3a78a526460c 100644
--- a/nixos/modules/services/x11/desktop-managers/cinnamon.nix
+++ b/nixos/modules/services/x11/desktop-managers/cinnamon.nix
@@ -136,7 +136,7 @@ in
         # session requirements
         cinnamon-screensaver
         # cinnamon-killer-daemon: provided by cinnamon-common
-        gnome.networkmanagerapplet # session requirement - also nm-applet not needed
+        networkmanagerapplet # session requirement - also nm-applet not needed
 
         # For a polkit authentication agent
         polkit_gnome
@@ -145,7 +145,7 @@ in
         nemo
         cinnamon-control-center
         cinnamon-settings-daemon
-        gnome.libgnomekbd
+        libgnomekbd
         orca
 
         # theme
diff --git a/nixos/modules/services/x11/desktop-managers/default.nix b/nixos/modules/services/x11/desktop-managers/default.nix
index 6ee5b0fc54f7..8247a7e381c9 100644
--- a/nixos/modules/services/x11/desktop-managers/default.nix
+++ b/nixos/modules/services/x11/desktop-managers/default.nix
@@ -19,7 +19,7 @@ in
   # E.g., if Plasma 5 is enabled, it supersedes xterm.
   imports = [
     ./none.nix ./xterm.nix ./xfce.nix ./plasma5.nix ./lumina.nix
-    ./lxqt.nix ./enlightenment.nix ./gnome.nix ./kodi.nix
+    ./lxqt.nix ./enlightenment.nix ./gnome.nix ./retroarch.nix ./kodi.nix
     ./mate.nix ./pantheon.nix ./surf-display.nix ./cde.nix
     ./cinnamon.nix
   ];
diff --git a/nixos/modules/services/x11/desktop-managers/enlightenment.nix b/nixos/modules/services/x11/desktop-managers/enlightenment.nix
index e3d876e82fdd..d1513a596b9f 100644
--- a/nixos/modules/services/x11/desktop-managers/enlightenment.nix
+++ b/nixos/modules/services/x11/desktop-managers/enlightenment.nix
@@ -36,6 +36,7 @@ in
       enlightenment.econnman
       enlightenment.efl
       enlightenment.enlightenment
+      enlightenment.ecrire
       enlightenment.ephoto
       enlightenment.rage
       enlightenment.terminology
diff --git a/nixos/modules/services/x11/desktop-managers/gnome.nix b/nixos/modules/services/x11/desktop-managers/gnome.nix
index efc9bd39b366..e2323785149a 100644
--- a/nixos/modules/services/x11/desktop-managers/gnome.nix
+++ b/nixos/modules/services/x11/desktop-managers/gnome.nix
@@ -569,6 +569,7 @@ in
         atomix
         five-or-more
         four-in-a-row
+        pkgs.gnome-2048
         gnome-chess
         gnome-klotski
         gnome-mahjongg
diff --git a/nixos/modules/services/x11/desktop-managers/gnome.xml b/nixos/modules/services/x11/desktop-managers/gnome.xml
index e5da7740196e..807c9d64e204 100644
--- a/nixos/modules/services/x11/desktop-managers/gnome.xml
+++ b/nixos/modules/services/x11/desktop-managers/gnome.xml
@@ -249,14 +249,5 @@ services.xserver.desktopManager.gnome = {
     See <link xlink:href="https://github.com/NixOS/nixpkgs/issues/56342">this issue.</link>
    </para>
   </section>
-
-  <section xml:id="sec-gnome-faq-nixos-rebuild-switch-kills-session">
-   <title>Why does <literal>nixos-rebuild switch</literal> sometimes kill my session?</title>
-
-   <para>
-    This is a known <link xlink:href="https://github.com/NixOS/nixpkgs/issues/44344">issue</link> without any workarounds.
-    If you are doing a fairly large upgrade, it is probably safer to use <literal>nixos-rebuild boot</literal>.
-   </para>
-  </section>
  </section>
 </chapter>
diff --git a/nixos/modules/services/x11/desktop-managers/lumina.nix b/nixos/modules/services/x11/desktop-managers/lumina.nix
index 1ab61953e735..419f5055d8be 100644
--- a/nixos/modules/services/x11/desktop-managers/lumina.nix
+++ b/nixos/modules/services/x11/desktop-managers/lumina.nix
@@ -38,11 +38,5 @@ in
       "/share"
     ];
 
-    security.wrappers.lumina-checkpass-wrapped = {
-      source = "${pkgs.lumina.lumina}/bin/lumina-checkpass";
-      owner = "root";
-      group = "root";
-    };
-
   };
 }
diff --git a/nixos/modules/services/x11/desktop-managers/mate.nix b/nixos/modules/services/x11/desktop-managers/mate.nix
index f8f47a061452..a7fda4be9796 100644
--- a/nixos/modules/services/x11/desktop-managers/mate.nix
+++ b/nixos/modules/services/x11/desktop-managers/mate.nix
@@ -74,11 +74,9 @@ in
     # Debugging
     environment.sessionVariables.MATE_SESSION_DEBUG = mkIf cfg.debug "1";
 
-    environment.systemPackages =
-      pkgs.mate.basePackages ++
-      (pkgs.gnome.removePackagesByName
-        pkgs.mate.extraPackages
-        config.environment.mate.excludePackages) ++
+    environment.systemPackages = pkgs.gnome.removePackagesByName
+      (pkgs.mate.basePackages ++
+      pkgs.mate.extraPackages ++
       [
         pkgs.desktop-file-utils
         pkgs.glib
@@ -87,7 +85,8 @@ in
         pkgs.xdg-user-dirs # Update user dirs as described in https://freedesktop.org/wiki/Software/xdg-user-dirs/
         pkgs.mate.mate-settings-daemon
         pkgs.yelp # for 'Contents' in 'Help' menus
-      ];
+      ])
+      config.environment.mate.excludePackages;
 
     programs.dconf.enable = true;
     # Shell integration for VTE terminals
diff --git a/nixos/modules/services/x11/desktop-managers/pantheon.nix b/nixos/modules/services/x11/desktop-managers/pantheon.nix
index 3296b7220485..8ff9b0b756d3 100644
--- a/nixos/modules/services/x11/desktop-managers/pantheon.nix
+++ b/nixos/modules/services/x11/desktop-managers/pantheon.nix
@@ -135,6 +135,7 @@ in
       services.bamf.enable = true;
       services.colord.enable = mkDefault true;
       services.fwupd.enable = mkDefault true;
+      services.packagekit.enable = mkDefault true;
       services.touchegg.enable = mkDefault true;
       services.touchegg.package = pkgs.pantheon.touchegg;
       services.tumbler.enable = mkDefault true;
@@ -224,12 +225,12 @@ in
       programs.file-roller.package = pkgs.pantheon.file-roller;
 
       # Settings from elementary-default-settings
-      environment.sessionVariables.GTK_CSD = "1";
       environment.etc."gtk-3.0/settings.ini".source = "${pkgs.pantheon.elementary-default-settings}/etc/gtk-3.0/settings.ini";
 
-      xdg.portal.extraPortals = with pkgs; [
-        pantheon.elementary-files
-        pantheon.elementary-settings-daemon
+      xdg.portal.enable = true;
+      xdg.portal.extraPortals = with pkgs.pantheon; [
+        elementary-files
+        elementary-settings-daemon
         xdg-desktop-portal-pantheon
       ];
 
@@ -273,7 +274,7 @@ in
     })
 
     (mkIf serviceCfg.apps.enable {
-      environment.systemPackages = (with pkgs.pantheon; pkgs.gnome.removePackagesByName [
+      environment.systemPackages = with pkgs.pantheon; pkgs.gnome.removePackagesByName ([
         elementary-calculator
         elementary-calendar
         elementary-camera
@@ -287,7 +288,11 @@ in
         elementary-terminal
         elementary-videos
         epiphany
-      ] config.environment.pantheon.excludePackages);
+      ] ++ lib.optionals config.services.flatpak.enable [
+        # Only install appcenter if flatpak is enabled before
+        # https://github.com/NixOS/nixpkgs/issues/15932 is resolved.
+        appcenter
+      ]) config.environment.pantheon.excludePackages;
 
       # needed by screenshot
       fonts.fonts = [
diff --git a/nixos/modules/services/x11/desktop-managers/pantheon.xml b/nixos/modules/services/x11/desktop-managers/pantheon.xml
index fe0a1c496223..202909d398f0 100644
--- a/nixos/modules/services/x11/desktop-managers/pantheon.xml
+++ b/nixos/modules/services/x11/desktop-managers/pantheon.xml
@@ -105,10 +105,10 @@ switchboard-with-plugs.override {
     </term>
     <listitem>
      <para>
-      AppCenter has been available since 20.03, but it is of little use. This is because there is no functioning PackageKit backend for Nix 2.0. Starting from 21.11, the Flatpak backend should work so you can install some Flatpak applications using it. See this <link xlink:href="https://github.com/NixOS/nixpkgs/issues/70214">issue</link>.
+      AppCenter has been available since 20.03. Starting from 21.11, the Flatpak backend should work so you can install some Flatpak applications using it. However, due to missing appstream metadata, the Packagekit backend does not function currently. See this <link xlink:href="https://github.com/NixOS/nixpkgs/issues/15932">issue</link>.
      </para>
      <para>
-      To use AppCenter on NixOS, add <literal>pantheon.appcenter</literal> to <xref linkend="opt-environment.systemPackages" />, <link linkend="module-services-flatpak">enable Flatpak support</link> and optionally add the <literal>appcenter</literal> Flatpak remote:
+      If you are using Pantheon, AppCenter should be installed by default if you have <link linkend="module-services-flatpak">Flatpak support</link> enabled. If you also wish to add the <literal>appcenter</literal> Flatpak remote:
      </para>
 <screen>
 <prompt>$ </prompt>flatpak remote-add --if-not-exists appcenter https://flatpak.elementary.io/repo.flatpakrepo
diff --git a/nixos/modules/services/x11/desktop-managers/plasma5.nix b/nixos/modules/services/x11/desktop-managers/plasma5.nix
index 9bacdaa9be98..b7aa2eba81cf 100644
--- a/nixos/modules/services/x11/desktop-managers/plasma5.nix
+++ b/nixos/modules/services/x11/desktop-managers/plasma5.nix
@@ -394,7 +394,8 @@ in
 
       # Extra UDEV rules used by Solid
       services.udev.packages = [
-        pkgs.libmtp
+        # libmtp has "bin", "dev", "out" outputs. UDEV rules file is in "out".
+        pkgs.libmtp.out
         pkgs.media-player-info
       ];
 
diff --git a/nixos/modules/services/x11/desktop-managers/retroarch.nix b/nixos/modules/services/x11/desktop-managers/retroarch.nix
new file mode 100644
index 000000000000..d471673d4521
--- /dev/null
+++ b/nixos/modules/services/x11/desktop-managers/retroarch.nix
@@ -0,0 +1,40 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let cfg = config.services.xserver.desktopManager.retroarch;
+
+in {
+  options.services.xserver.desktopManager.retroarch = {
+    enable = mkEnableOption "RetroArch";
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.retroarch;
+      defaultText = literalExpression "pkgs.retroarch";
+      example = literalExpression "pkgs.retroarch-full";
+      description = "RetroArch package to use.";
+    };
+
+    extraArgs = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "--verbose" "--host" ];
+      description = "Extra arguments to pass to RetroArch.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.xserver.desktopManager.session = [{
+      name = "RetroArch";
+      start = ''
+        ${cfg.package}/bin/retroarch -f ${escapeShellArgs cfg.extraArgs} &
+        waitPID=$!
+      '';
+    }];
+
+    environment.systemPackages = [ cfg.package ];
+  };
+
+  meta.maintainers = with maintainers; [ j0hax ];
+}
diff --git a/nixos/modules/services/x11/display-managers/default.nix b/nixos/modules/services/x11/display-managers/default.nix
index bdc46faa7fd0..a5db3dd5dd45 100644
--- a/nixos/modules/services/x11/display-managers/default.nix
+++ b/nixos/modules/services/x11/display-managers/default.nix
@@ -7,13 +7,14 @@
 # (e.g., KDE, Gnome or a plain xterm), and optionally the *window
 # manager* (e.g. kwin or twm).
 
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
 
   cfg = config.services.xserver;
+  opt = options.services.xserver;
   xorg = pkgs.xorg;
 
   fontconfig = config.fonts.fontconfig;
@@ -147,6 +148,7 @@ in
       xauthBin = mkOption {
         internal = true;
         default = "${xorg.xauth}/bin/xauth";
+        defaultText = literalExpression ''"''${pkgs.xorg.xauth}/bin/xauth"'';
         description = "Path to the <command>xauth</command> program used by display managers.";
       };
 
@@ -217,6 +219,7 @@ in
 
       session = mkOption {
         default = [];
+        type = types.listOf types.attrs;
         example = literalExpression
           ''
             [ { manage = "desktop";
@@ -278,6 +281,9 @@ in
             defaultSessionFromLegacyOptions
           else
             null;
+        defaultText = literalDocBook ''
+          Taken from display manager settings or window manager settings, if either is set.
+        '';
         example = "gnome";
         description = ''
           Graphical session to pre-select in the session chooser (only effective for GDM, LightDM and SDDM).
@@ -337,11 +343,12 @@ in
 
       # Configuration for automatic login. Common for all DM.
       autoLogin = mkOption {
-        type = types.submodule {
+        type = types.submodule ({ config, options, ... }: {
           options = {
             enable = mkOption {
               type = types.bool;
-              default = cfg.displayManager.autoLogin.user != null;
+              default = config.user != null;
+              defaultText = literalExpression "config.${options.user} != null";
               description = ''
                 Automatically log in as <option>autoLogin.user</option>.
               '';
@@ -355,7 +362,7 @@ in
               '';
             };
           };
-        };
+        });
 
         default = {};
         description = ''
diff --git a/nixos/modules/services/x11/display-managers/gdm.nix b/nixos/modules/services/x11/display-managers/gdm.nix
index 6f0d645725e9..b1dc6643be82 100644
--- a/nixos/modules/services/x11/display-managers/gdm.nix
+++ b/nixos/modules/services/x11/display-managers/gdm.nix
@@ -53,6 +53,8 @@ in
       "autoLogin"
       "user"
     ])
+
+    (mkRemovedOptionModule [ "services" "xserver" "displayManager" "gdm" "nvidiaWayland" ] "We defer to GDM whether Wayland should be enabled.")
   ];
 
   meta = {
@@ -83,17 +85,6 @@ in
         default = true;
         description = ''
           Allow GDM to run on Wayland instead of Xserver.
-          Note to enable Wayland with Nvidia the <option>nvidiaWayland</option>
-          must not be disabled.
-        '';
-      };
-
-      nvidiaWayland = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Whether to allow wayland to be used with the proprietary
-          NVidia graphics driver.
         '';
       };
 
@@ -149,7 +140,8 @@ in
         environment = {
           GDM_X_SERVER_EXTRA_ARGS = toString
             (filter (arg: arg != "-terminate") cfg.xserverArgs);
-          XDG_DATA_DIRS = "${cfg.sessionData.desktops}/share/";
+          # GDM is needed for gnome-login.session
+          XDG_DATA_DIRS = "${gdm}/share:${cfg.sessionData.desktops}/share";
         } // optionalAttrs (xSessionWrapper != null) {
           # Make GDM use this wrapper before running the session, which runs the
           # configured setupCommands. This relies on a patched GDM which supports
@@ -230,19 +222,6 @@ in
 
     services.dbus.packages = [ gdm ];
 
-    # We duplicate upstream's udev rules manually to make wayland with nvidia configurable
-    services.udev.extraRules = ''
-      # disable Wayland on Cirrus chipsets
-      ATTR{vendor}=="0x1013", ATTR{device}=="0x00b8", ATTR{subsystem_vendor}=="0x1af4", ATTR{subsystem_device}=="0x1100", RUN+="${gdm}/libexec/gdm-runtime-config set daemon WaylandEnable false"
-      # disable Wayland on Hi1710 chipsets
-      ATTR{vendor}=="0x19e5", ATTR{device}=="0x1711", RUN+="${gdm}/libexec/gdm-runtime-config set daemon WaylandEnable false"
-      ${optionalString (!cfg.gdm.nvidiaWayland) ''
-        DRIVER=="nvidia", RUN+="${gdm}/libexec/gdm-runtime-config set daemon WaylandEnable false"
-      ''}
-      # disable Wayland when modesetting is disabled
-      IMPORT{cmdline}="nomodeset", RUN+="${gdm}/libexec/gdm-runtime-config set daemon WaylandEnable false"
-    '';
-
     systemd.user.services.dbus.wantedBy = [ "default.target" ];
 
     programs.dconf.profiles.gdm =
diff --git a/nixos/modules/services/x11/display-managers/lightdm.nix b/nixos/modules/services/x11/display-managers/lightdm.nix
index 84b75c83aeab..27dfed3cc14c 100644
--- a/nixos/modules/services/x11/display-managers/lightdm.nix
+++ b/nixos/modules/services/x11/display-managers/lightdm.nix
@@ -267,6 +267,8 @@ in
     # Enable the accounts daemon to find lightdm's dbus interface
     environment.systemPackages = [ lightdm ];
 
+    security.polkit.enable = true;
+
     security.pam.services.lightdm.text = ''
         auth      substack      login
         account   include       login
diff --git a/nixos/modules/services/x11/hardware/synaptics.nix b/nixos/modules/services/x11/hardware/synaptics.nix
index 22af869f1f8a..93dd560bca40 100644
--- a/nixos/modules/services/x11/hardware/synaptics.nix
+++ b/nixos/modules/services/x11/hardware/synaptics.nix
@@ -1,8 +1,9 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let cfg = config.services.xserver.synaptics;
+    opt = options.services.xserver.synaptics;
     tapConfig = if cfg.tapButtons then enabledTapConfig else disabledTapConfig;
     enabledTapConfig = ''
       Option "MaxTapTime" "180"
@@ -77,24 +78,28 @@ in {
       horizTwoFingerScroll = mkOption {
         type = types.bool;
         default = cfg.twoFingerScroll;
+        defaultText = literalExpression "config.${opt.twoFingerScroll}";
         description = "Whether to enable horizontal two-finger drag-scrolling.";
       };
 
       vertTwoFingerScroll = mkOption {
         type = types.bool;
         default = cfg.twoFingerScroll;
+        defaultText = literalExpression "config.${opt.twoFingerScroll}";
         description = "Whether to enable vertical two-finger drag-scrolling.";
       };
 
       horizEdgeScroll = mkOption {
         type = types.bool;
         default = ! cfg.horizTwoFingerScroll;
+        defaultText = literalExpression "! config.${opt.horizTwoFingerScroll}";
         description = "Whether to enable horizontal edge drag-scrolling.";
       };
 
       vertEdgeScroll = mkOption {
         type = types.bool;
         default = ! cfg.vertTwoFingerScroll;
+        defaultText = literalExpression "! config.${opt.vertTwoFingerScroll}";
         description = "Whether to enable vertical edge drag-scrolling.";
       };
 
diff --git a/nixos/modules/services/x11/picom.nix b/nixos/modules/services/x11/picom.nix
index dbd4b1cefef1..b40e20bcd357 100644
--- a/nixos/modules/services/x11/picom.nix
+++ b/nixos/modules/services/x11/picom.nix
@@ -1,10 +1,11 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
 let
 
   cfg = config.services.picom;
+  opt = options.services.picom;
 
   pairOf = x: with types;
     addCheck (listOf x) (y: length y == 2)
@@ -178,7 +179,16 @@ in {
 
     wintypes = mkOption {
       type = types.attrs;
-      default = { popup_menu = { opacity = cfg.menuOpacity; }; dropdown_menu = { opacity = cfg.menuOpacity; }; };
+      default = {
+        popup_menu = { opacity = cfg.menuOpacity; };
+        dropdown_menu = { opacity = cfg.menuOpacity; };
+      };
+      defaultText = literalExpression ''
+        {
+          popup_menu = { opacity = config.${opt.menuOpacity}; };
+          dropdown_menu = { opacity = config.${opt.menuOpacity}; };
+        }
+      '';
       example = {};
       description = ''
         Rules for specific window types.
diff --git a/nixos/modules/services/x11/window-managers/xmonad.nix b/nixos/modules/services/x11/window-managers/xmonad.nix
index ecad411ff683..68f97c2f504b 100644
--- a/nixos/modules/services/x11/window-managers/xmonad.nix
+++ b/nixos/modules/services/x11/window-managers/xmonad.nix
@@ -2,7 +2,7 @@
 
 with lib;
 let
-  inherit (lib) mkOption mkIf optionals literalExpression;
+  inherit (lib) mkOption mkIf optionals literalExpression optionalString;
   cfg = config.services.xserver.windowManager.xmonad;
 
   ghcWithPackages = cfg.haskellPackages.ghcWithPackages;
@@ -26,11 +26,14 @@ let
     in
       pkgs.runCommandLocal "xmonad" {
         nativeBuildInputs = [ pkgs.makeWrapper ];
-      } ''
+      } (''
         install -D ${xmonadEnv}/share/man/man1/xmonad.1.gz $out/share/man/man1/xmonad.1.gz
         makeWrapper ${configured}/bin/xmonad $out/bin/xmonad \
+      '' + optionalString cfg.enableConfiguredRecompile ''
+          --set NIX_GHC "${xmonadEnv}/bin/ghc" \
+      '' + ''
           --set XMONAD_XMESSAGE "${pkgs.xorg.xmessage}/bin/xmessage"
-      '';
+      '');
 
   xmonad = if (cfg.config != null) then xmonad-config else xmonad-vanilla;
 in {
@@ -95,12 +98,14 @@ in {
           xmonad from PATH. This allows e.g. switching to the new xmonad binary
           after rebuilding your system with nixos-rebuild.
           For the same reason, ghc is not added to the environment when this
-          option is set.
+          option is set, unless <option>enableConfiguredRecompile</option> is
+          set to <literal>true</literal>.
 
           If you actually want to run xmonad with a config specified here, but
           also be able to recompile and restart it from a copy of that source in
-          $HOME/.xmonad on the fly, you will have to implement that yourself
-          using something like "compileRestart" from the example.
+          $HOME/.xmonad on the fly, set <option>enableConfiguredRecompile</option>
+          to <literal>true</literal> and implement something like "compileRestart"
+          from the example.
           This should allow you to switch at will between the local xmonad and
           the one NixOS puts in your PATH.
         '';
@@ -116,6 +121,29 @@ in {
 
           compiledConfig = printf "xmonad-%s-%s" arch os
 
+          myConfig = defaultConfig
+            { modMask = mod4Mask -- Use Super instead of Alt
+            , terminal = "urxvt" }
+            `additionalKeys`
+            [ ( (mod4Mask,xK_r), compileRestart True)
+            , ( (mod4Mask,xK_q), restart "xmonad" True ) ]
+
+          --------------------------------------------
+          {- version 0.17.0 -}
+          --------------------------------------------
+          -- compileRestart resume =
+          --   dirs <- io getDirectories
+          --   whenX (recompile dirs True) $
+          --     when resume writeStateToFile
+          --       *> catchIO
+          --         ( do
+          --             args <- getArgs
+          --             executeFile (cacheDir dirs </> compiledConfig) False args Nothing
+          --         )
+          --
+          -- main = getDirectories >>= launch myConfig
+          --------------------------------------------
+
           compileRestart resume =
             whenX (recompile True) $
               when resume writeStateToFile
@@ -126,12 +154,17 @@ in {
                       executeFile (dir </> compiledConfig) False args Nothing
                   )
 
-          main = launch defaultConfig
-              { modMask = mod4Mask -- Use Super instead of Alt
-              , terminal = "urxvt" }
-              `additionalKeys`
-              [ ( (mod4Mask,xK_r), compileRestart True)
-              , ( (mod4Mask,xK_q), restart "xmonad" True ) ]
+          main = launch myConfig
+        '';
+      };
+
+      enableConfiguredRecompile = mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Enable recompilation even if <option>config</option> is set to a
+          non-null value. This adds the necessary Haskell dependencies (GHC with
+          packages) to the xmonad binary's environment.
         '';
       };
 
diff --git a/nixos/modules/services/x11/xserver.nix b/nixos/modules/services/x11/xserver.nix
index 24d925734423..0c50d82b23be 100644
--- a/nixos/modules/services/x11/xserver.nix
+++ b/nixos/modules/services/x11/xserver.nix
@@ -620,9 +620,6 @@ in
       in optional (driver != null) ({ inherit name; modules = []; driverName = name; display = true; } // driver));
 
     assertions = [
-      { assertion = config.security.polkit.enable;
-        message = "X11 requires Polkit to be enabled (‘security.polkit.enable = true’).";
-      }
       (let primaryHeads = filter (x: x.primary) cfg.xrandrHeads; in {
         assertion = length primaryHeads < 2;
         message = "Only one head is allowed to be primary in "
@@ -703,7 +700,7 @@ in
 
         environment =
           optionalAttrs config.hardware.opengl.setLdLibraryPath
-            { LD_LIBRARY_PATH = pkgs.addOpenGLRunpath.driverLink; }
+            { LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.addOpenGLRunpath.driverLink ]; }
           // cfg.displayManager.job.environment;
 
         preStart =
@@ -865,4 +862,6 @@ in
 
   };
 
+  # uses relatedPackages
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/system/activation/activation-script.nix b/nixos/modules/system/activation/activation-script.nix
index 4a32387db8da..c04d0fc16b24 100644
--- a/nixos/modules/system/activation/activation-script.nix
+++ b/nixos/modules/system/activation/activation-script.nix
@@ -56,6 +56,7 @@ let
       ln -sfn "$(readlink -f "$systemConfig")" /run/current-system
 
       # Prevent the current configuration from being garbage-collected.
+      mkdir -p /nix/var/nix/gcroots
       ln -sfn /run/current-system /nix/var/nix/gcroots/current-system
 
       exit $_status
@@ -142,6 +143,7 @@ in
       readOnly = true;
       internal = true;
       default = systemActivationScript (removeAttrs config.system.activationScripts [ "script" ]) true;
+      defaultText = literalDocBook "generated activation script";
     };
 
     system.userActivationScripts = mkOption {
diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl
index d3e9a97c2120..9e5b760434a0 100644..100755
--- a/nixos/modules/system/activation/switch-to-configuration.pl
+++ b/nixos/modules/system/activation/switch-to-configuration.pl
@@ -2,38 +2,63 @@
 
 use strict;
 use warnings;
+use Config::IniFiles;
 use File::Path qw(make_path);
 use File::Basename;
-use File::Slurp;
+use File::Slurp qw(read_file write_file edit_file);
 use Net::DBus;
 use Sys::Syslog qw(:standard :macros);
-use Cwd 'abs_path';
+use Cwd qw(abs_path);
 
-my $out = "@out@";
+## no critic(ControlStructures::ProhibitDeepNests)
+## no critic(ErrorHandling::RequireCarping)
+## no critic(CodeLayout::ProhibitParensWithBuiltins)
+## no critic(Variables::ProhibitPunctuationVars, Variables::RequireLocalizedPunctuationVars)
+## no critic(InputOutput::RequireCheckedSyscalls, InputOutput::RequireBracedFileHandleWithPrint, InputOutput::RequireBriefOpen)
+## no critic(ValuesAndExpressions::ProhibitNoisyQuotes, ValuesAndExpressions::ProhibitMagicNumbers, ValuesAndExpressions::ProhibitEmptyQuotes, ValuesAndExpressions::ProhibitInterpolationOfLiterals)
+## no critic(RegularExpressions::ProhibitEscapedMetacharacters)
 
-my $curSystemd = abs_path("/run/current-system/sw/bin");
+# System closure path to switch to
+my $out = "@out@";
+# Path to the directory containing systemd tools of the old system
+my $cur_systemd = abs_path("/run/current-system/sw/bin");
+# Path to the systemd store path of the new system
+my $new_systemd = "@systemd@";
 
 # To be robust against interruption, record what units need to be started etc.
-my $startListFile = "/run/nixos/start-list";
-my $restartListFile = "/run/nixos/restart-list";
-my $reloadListFile = "/run/nixos/reload-list";
-
-# Parse restart/reload requests by the activation script
-my $restartByActivationFile = "/run/nixos/activation-restart-list";
-my $reloadByActivationFile = "/run/nixos/activation-reload-list";
-my $dryRestartByActivationFile = "/run/nixos/dry-activation-restart-list";
-my $dryReloadByActivationFile = "/run/nixos/dry-activation-reload-list";
-
-make_path("/run/nixos", { mode => oct(755) });
-
-my $action = shift @ARGV;
+# We read these files again every time this script starts to make sure we continue
+# where the old (interrupted) script left off.
+my $start_list_file = "/run/nixos/start-list";
+my $restart_list_file = "/run/nixos/restart-list";
+my $reload_list_file = "/run/nixos/reload-list";
+
+# Parse restart/reload requests by the activation script.
+# Activation scripts may write newline-separated units to the restart
+# file and switch-to-configuration will handle them. While
+# `stopIfChanged = true` is ignored, switch-to-configuration will
+# handle `restartIfChanged = false` and `reloadIfChanged = true`.
+# This is the same as specifying a restart trigger in the NixOS module.
+#
+# The reload file asks the script to reload a unit. This is the same as
+# specifying a reload trigger in the NixOS module and can be ignored if
+# the unit is restarted in this activation.
+my $restart_by_activation_file = "/run/nixos/activation-restart-list";
+my $reload_by_activation_file = "/run/nixos/activation-reload-list";
+my $dry_restart_by_activation_file = "/run/nixos/dry-activation-restart-list";
+my $dry_reload_by_activation_file = "/run/nixos/dry-activation-reload-list";
+
+# The action that is to be performed (like switch, boot, test, dry-activate)
+# Also exposed via environment variable from now on
+my $action = shift(@ARGV);
+$ENV{NIXOS_ACTION} = $action;
 
+# Expose the locale archive as an environment variable for systemctl and the activation script
 if ("@localeArchive@" ne "") {
     $ENV{LOCALE_ARCHIVE} = "@localeArchive@";
 }
 
-if (!defined $action || ($action ne "switch" && $action ne "boot" && $action ne "test" && $action ne "dry-activate")) {
-    print STDERR <<EOF;
+if (!defined($action) || ($action ne "switch" && $action ne "boot" && $action ne "test" && $action ne "dry-activate")) {
+    print STDERR <<"EOF";
 Usage: $0 [switch|boot|test]
 
 switch:       make the configuration the boot default and activate now
@@ -41,67 +66,102 @@ boot:         make the configuration the boot default
 test:         activate the configuration, but don\'t make it the boot default
 dry-activate: show what would be done if this configuration were activated
 EOF
-    exit 1;
+    exit(1);
 }
 
-$ENV{NIXOS_ACTION} = $action;
-
 # This is a NixOS installation if it has /etc/NIXOS or a proper
 # /etc/os-release.
-die "This is not a NixOS installation!\n" unless
-    -f "/etc/NIXOS" || (read_file("/etc/os-release", err_mode => 'quiet') // "") =~ /ID=nixos/s;
+if (!-f "/etc/NIXOS" && (read_file("/etc/os-release", err_mode => "quiet") // "") !~ /^ID="?nixos"?/msx) {
+    die("This is not a NixOS installation!\n");
+}
 
+make_path("/run/nixos", { mode => oct(755) });
 openlog("nixos", "", LOG_USER);
 
 # Install or update the bootloader.
 if ($action eq "switch" || $action eq "boot") {
-    system("@installBootLoader@ $out") == 0 or exit 1;
+    chomp(my $install_boot_loader = <<'EOFBOOTLOADER');
+@installBootLoader@
+EOFBOOTLOADER
+    system("$install_boot_loader $out") == 0 or exit 1;
 }
 
 # Just in case the new configuration hangs the system, do a sync now.
-system("@coreutils@/bin/sync", "-f", "/nix/store") unless ($ENV{"NIXOS_NO_SYNC"} // "") eq "1";
+if (($ENV{"NIXOS_NO_SYNC"} // "") ne "1") {
+    system("@coreutils@/bin/sync", "-f", "/nix/store");
+}
 
-exit 0 if $action eq "boot";
+if ($action eq "boot") {
+    exit(0);
+}
 
 # Check if we can activate the new configuration.
-my $oldVersion = read_file("/run/current-system/init-interface-version", err_mode => 'quiet') // "";
-my $newVersion = read_file("$out/init-interface-version");
+my $cur_init_interface_version = read_file("/run/current-system/init-interface-version", err_mode => "quiet") // "";
+my $new_init_interface_version = read_file("$out/init-interface-version");
 
-if ($newVersion ne $oldVersion) {
-    print STDERR <<EOF;
+if ($new_init_interface_version ne $cur_init_interface_version) {
+    print STDERR <<'EOF';
 Warning: the new NixOS configuration has an ‘init’ that is
 incompatible with the current configuration.  The new configuration
-won\'t take effect until you reboot the system.
+won't take effect until you reboot the system.
 EOF
-    exit 100;
+    exit(100);
 }
 
 # Ignore SIGHUP so that we're not killed if we're running on (say)
 # virtual console 1 and we restart the "tty1" unit.
 $SIG{PIPE} = "IGNORE";
 
-sub getActiveUnits {
+# Asks the currently running systemd instance via dbus which units are active.
+# Returns a hash where the key is the name of each unit and the value a hash
+# of load, state, substate.
+sub get_active_units {
     my $mgr = Net::DBus->system->get_service("org.freedesktop.systemd1")->get_object("/org/freedesktop/systemd1");
     my $units = $mgr->ListUnitsByPatterns([], []);
     my $res = {};
-    for my $item (@$units) {
+    for my $item (@{$units}) {
         my ($id, $description, $load_state, $active_state, $sub_state,
-            $following, $unit_path, $job_id, $job_type, $job_path) = @$item;
-        next unless $following eq '';
-        next if $job_id == 0 and $active_state eq 'inactive';
+            $following, $unit_path, $job_id, $job_type, $job_path) = @{$item};
+        if ($following ne "") {
+            next;
+        }
+        if ($job_id == 0 and $active_state eq "inactive") {
+            next;
+        }
         $res->{$id} = { load => $load_state, state => $active_state, substate => $sub_state };
     }
     return $res;
 }
 
-sub parseFstab {
+# Asks the currently running systemd instance whether a unit is currently active.
+# Takes the name of the unit as an argument and returns a bool whether the unit is active or not.
+sub unit_is_active {
+    my ($unit_name) = @_;
+
+    my $mgr = Net::DBus->system->get_service("org.freedesktop.systemd1")->get_object("/org/freedesktop/systemd1");
+    my $units = $mgr->ListUnitsByNames([$unit_name]);
+    if (scalar(@{$units}) == 0) {
+        return 0;
+    }
+    my $active_state = $units->[0]->[3];
+    return $active_state eq "active" || $active_state eq "activating";
+}
+
+# Parse a fstab file, given its path.
+# Returns a tuple of filesystems and swaps.
+#
+# Filesystems is a hash of mountpoint and { device, fsType, options }
+# Swaps is a hash of device and { options }
+sub parse_fstab {
     my ($filename) = @_;
     my ($fss, $swaps);
-    foreach my $line (read_file($filename, err_mode => 'quiet')) {
-        chomp $line;
-        $line =~ s/^\s*#.*//;
-        next if $line =~ /^\s*$/;
-        my @xs = split / /, $line;
+    foreach my $line (read_file($filename, err_mode => "quiet")) {
+        chomp($line);
+        $line =~ s/^\s*\#.*//msx;
+        if ($line =~ /^\s*$/msx) {
+            next;
+        }
+        my @xs = split(/\s+/msx, $line);
         if ($xs[2] eq "swap") {
             $swaps->{$xs[0]} = { options => $xs[3] // "" };
         } else {
@@ -111,80 +171,368 @@ sub parseFstab {
     return ($fss, $swaps);
 }
 
-sub parseUnit {
-    my ($filename) = @_;
-    my $info = {};
-    parseKeyValues($info, read_file($filename)) if -f $filename;
-    parseKeyValues($info, read_file("${filename}.d/overrides.conf")) if -f "${filename}.d/overrides.conf";
-    return $info;
+# This subroutine takes a single ini file that specified systemd configuration
+# like unit configuration and parses it into a hash where the keys are the sections
+# of the unit file and the values are hashes themselves. These hashes have the unit file
+# keys as their keys (left side of =) and an array of all values that were set as their
+# values. If a value is empty (for example `ExecStart=`), then all current definitions are
+# removed.
+#
+# Instead of returning the hash, this subroutine takes a hashref to return the data in. This
+# allows calling the subroutine multiple times with the same hash to parse override files.
+sub parse_systemd_ini {
+    my ($unit_contents, $path) = @_;
+    # Tie the ini file to a hash for easier access
+    tie(my %file_contents, "Config::IniFiles", (-file => $path, -allowempty => 1, -allowcontinue => 1)); ## no critic(Miscellanea::ProhibitTies)
+
+    # Copy over all sections
+    foreach my $section_name (keys(%file_contents)) {
+        if ($section_name eq "Install") {
+            # Skip the [Install] section because it has no relevant keys for us
+            next;
+        }
+        # Copy over all keys
+        foreach my $ini_key (keys(%{$file_contents{$section_name}})) {
+            # Ensure the value is an array so it's easier to work with
+            my $ini_value = $file_contents{$section_name}{$ini_key};
+            my @ini_values;
+            if (ref($ini_value) eq "ARRAY") {
+                @ini_values = @{$ini_value};
+            } else {
+                @ini_values = $ini_value;
+            }
+            # Go over all values
+            for my $ini_value (@ini_values) {
+                # If a value is empty, it's an override that tells us to clean the value
+                if ($ini_value eq "") {
+                    delete $unit_contents->{$section_name}->{$ini_key};
+                    next;
+                }
+                push(@{$unit_contents->{$section_name}->{$ini_key}}, $ini_value);
+            }
+        }
+    }
+    return;
 }
 
-sub parseKeyValues {
-    my $info = shift;
-    foreach my $line (@_) {
-        # FIXME: not quite correct.
-        $line =~ /^([^=]+)=(.*)$/ or next;
-        $info->{$1} = $2;
+# This subroutine takes the path to a systemd configuration file (like a unit configuration),
+# parses it, and returns a hash that contains the contents. The contents of this hash are
+# explained in the `parse_systemd_ini` subroutine. Neither the sections nor the keys inside
+# the sections are consistently sorted.
+#
+# If a directory with the same basename ending in .d exists next to the unit file, it will be
+# assumed to contain override files which will be parsed as well and handled properly.
+sub parse_unit {
+    my ($unit_path) = @_;
+
+    # Parse the main unit and all overrides
+    my %unit_data;
+    # Replace \ with \\ so glob() still works with units that have a \ in them
+    # Valid characters in unit names are ASCII letters, digits, ":", "-", "_", ".", and "\"
+    $unit_path =~ s/\\/\\\\/gmsx;
+    foreach (glob("${unit_path}{,.d/*.conf}")) {
+        parse_systemd_ini(\%unit_data, "$_")
     }
+    return %unit_data;
 }
 
-sub boolIsTrue {
-    my ($s) = @_;
-    return $s eq "yes" || $s eq "true";
+# Checks whether a specified boolean in a systemd unit is true
+# or false, with a default that is applied when the value is not set.
+sub parse_systemd_bool {
+    my ($unit_config, $section_name, $bool_name, $default) = @_;
+
+    my @values = @{$unit_config->{$section_name}{$bool_name} // []};
+    # Return default if value is not set
+    if ((scalar(@values) < 1) || (not defined($values[-1]))) {
+        return $default;
+    }
+    # If value is defined multiple times, use the last definition
+    my $last_value = $values[-1];
+    # These are valid values as of systemd.syntax(7)
+    return $last_value eq "1" || $last_value eq "yes" || $last_value eq "true" || $last_value eq "on";
 }
 
-sub recordUnit {
+# Writes a unit name into a given file to be more resilient against
+# crashes of the script. Does nothing when the action is dry-activate.
+sub record_unit {
     my ($fn, $unit) = @_;
-    write_file($fn, { append => 1 }, "$unit\n") if $action ne "dry-activate";
+    if ($action ne "dry-activate") {
+        write_file($fn, { append => 1 }, "$unit\n");
+    }
+    return;
+}
+
+# The opposite of record_unit, removes a unit name from a file
+sub unrecord_unit {
+    my ($fn, $unit) = @_;
+    if ($action ne "dry-activate") {
+        edit_file(sub { s/^$unit\n//msx }, $fn);
+    }
+    return;
+}
+
+# Compare the contents of two unit files and return whether the unit
+# needs to be restarted or reloaded. If the units differ, the service
+# is restarted unless the only difference is `X-Reload-Triggers` in the
+# `Unit` section. If this is the only modification, the unit is reloaded
+# instead of restarted.
+# Returns:
+# - 0 if the units are equal
+# - 1 if the units are different and a restart action is required
+# - 2 if the units are different and a reload action is required
+sub compare_units { ## no critic(Subroutines::ProhibitExcessComplexity)
+    my ($cur_unit, $new_unit) = @_;
+    my $ret = 0;
+    # Keys to ignore in the [Unit] section
+    my %unit_section_ignores = map { $_ => 1 } qw(
+        X-Reload-Triggers
+        Description Documentation
+        OnFailure OnSuccess OnFailureJobMode
+        IgnoreOnIsolate StopWhenUnneeded
+        RefuseManualStart RefuseManualStop
+        AllowIsolate CollectMode
+        SourcePath
+    );
+
+    my $comp_array = sub {
+      my ($a, $b) = @_;
+      return join("\0", @{$a}) eq join("\0", @{$b});
+    };
+
+    # Comparison hash for the sections
+    my %section_cmp = map { $_ => 1 } keys(%{$new_unit});
+    # Iterate over the sections
+    foreach my $section_name (keys(%{$cur_unit})) {
+        # Missing section in the new unit?
+        if (not exists($section_cmp{$section_name})) {
+            # If the [Unit] section was removed, make sure that only keys
+            # were in it that are ignored
+            if ($section_name eq "Unit") {
+                foreach my $ini_key (keys(%{$cur_unit->{"Unit"}})) {
+                    if (not defined($unit_section_ignores{$ini_key})) {
+                        return 1;
+                    }
+                }
+                next; # check the next section
+            } else {
+                return 1;
+            }
+            if ($section_name eq "Unit" and %{$cur_unit->{"Unit"}} == 1 and defined(%{$cur_unit->{"Unit"}}{"X-Reload-Triggers"})) {
+                # If a new [Unit] section was removed that only contained X-Reload-Triggers,
+                # do nothing.
+                next;
+            } else {
+                return 1;
+            }
+        }
+        delete $section_cmp{$section_name};
+        # Comparison hash for the section contents
+        my %ini_cmp = map { $_ => 1 } keys(%{$new_unit->{$section_name}});
+        # Iterate over the keys of the section
+        foreach my $ini_key (keys(%{$cur_unit->{$section_name}})) {
+            delete $ini_cmp{$ini_key};
+            my @cur_value = @{$cur_unit->{$section_name}{$ini_key}};
+            # If the key is missing in the new unit, they are different...
+            if (not $new_unit->{$section_name}{$ini_key}) {
+                # ... unless the key that is now missing is one of the ignored keys
+                if ($section_name eq "Unit" and defined($unit_section_ignores{$ini_key})) {
+                    next;
+                }
+                return 1;
+            }
+            my @new_value = @{$new_unit->{$section_name}{$ini_key}};
+            # If the contents are different, the units are different
+            if (not $comp_array->(\@cur_value, \@new_value)) {
+                # Check if only the reload triggers changed or one of the ignored keys
+                if ($section_name eq "Unit") {
+                    if ($ini_key eq "X-Reload-Triggers") {
+                        $ret = 2;
+                        next;
+                    } elsif (defined($unit_section_ignores{$ini_key})) {
+                        next;
+                    }
+                }
+                return 1;
+            }
+        }
+        # A key was introduced that was missing in the previous unit
+        if (%ini_cmp) {
+            if ($section_name eq "Unit") {
+                foreach my $ini_key (keys(%ini_cmp)) {
+                    if ($ini_key eq "X-Reload-Triggers") {
+                        $ret = 2;
+                    } elsif (defined($unit_section_ignores{$ini_key})) {
+                        next;
+                    } else {
+                        return 1;
+                    }
+                }
+            } else {
+                return 1;
+            }
+        };
+    }
+    # A section was introduced that was missing in the previous unit
+    if (%section_cmp) {
+        if (%section_cmp == 1 and defined($section_cmp{"Unit"})) {
+            foreach my $ini_key (keys(%{$new_unit->{"Unit"}})) {
+                if (not defined($unit_section_ignores{$ini_key})) {
+                    return 1;
+                } elsif ($ini_key eq "X-Reload-Triggers") {
+                    $ret = 2;
+                }
+            }
+        } else {
+            return 1;
+        }
+    }
+
+    return $ret;
 }
 
-# As a fingerprint for determining whether a unit has changed, we use
-# its absolute path. If it has an override file, we append *its*
-# absolute path as well.
-sub fingerprintUnit {
-    my ($s) = @_;
-    return abs_path($s) . (-f "${s}.d/overrides.conf" ? " " . abs_path "${s}.d/overrides.conf" : "");
+# Called when a unit exists in both the old systemd and the new system and the units
+# differ. This figures out of what units are to be stopped, restarted, reloaded, started, and skipped.
+sub handle_modified_unit { ## no critic(Subroutines::ProhibitManyArgs, Subroutines::ProhibitExcessComplexity)
+    my ($unit, $base_name, $new_unit_file, $new_unit_info, $active_cur, $units_to_stop, $units_to_start, $units_to_reload, $units_to_restart, $units_to_skip) = @_;
+
+    if ($unit eq "sysinit.target" || $unit eq "basic.target" || $unit eq "multi-user.target" || $unit eq "graphical.target" || $unit =~ /\.path$/msx || $unit =~ /\.slice$/msx) {
+        # Do nothing.  These cannot be restarted directly.
+
+        # Slices and Paths don't have to be restarted since
+        # properties (resource limits and inotify watches)
+        # seem to get applied on daemon-reload.
+    } elsif ($unit =~ /\.mount$/msx) {
+        # Reload the changed mount unit to force a remount.
+        # FIXME: only reload when Options= changed, restart otherwise
+        $units_to_reload->{$unit} = 1;
+        record_unit($reload_list_file, $unit);
+    } elsif ($unit =~ /\.socket$/msx) {
+        # FIXME: do something?
+        # Attempt to fix this: https://github.com/NixOS/nixpkgs/pull/141192
+        # Revert of the attempt: https://github.com/NixOS/nixpkgs/pull/147609
+        # More details: https://github.com/NixOS/nixpkgs/issues/74899#issuecomment-981142430
+    } else {
+        my %new_unit_info = $new_unit_info ? %{$new_unit_info} : parse_unit($new_unit_file);
+        if (parse_systemd_bool(\%new_unit_info, "Service", "X-ReloadIfChanged", 0) and not $units_to_restart->{$unit} and not $units_to_stop->{$unit}) {
+            $units_to_reload->{$unit} = 1;
+            record_unit($reload_list_file, $unit);
+        }
+        elsif (!parse_systemd_bool(\%new_unit_info, "Service", "X-RestartIfChanged", 1) || parse_systemd_bool(\%new_unit_info, "Unit", "RefuseManualStop", 0) || parse_systemd_bool(\%new_unit_info, "Unit", "X-OnlyManualStart", 0)) {
+            $units_to_skip->{$unit} = 1;
+        } else {
+            # It doesn't make sense to stop and start non-services because
+            # they can't have ExecStop=
+            if (!parse_systemd_bool(\%new_unit_info, "Service", "X-StopIfChanged", 1) || $unit !~ /\.service$/msx) {
+                # This unit should be restarted instead of
+                # stopped and started.
+                $units_to_restart->{$unit} = 1;
+                record_unit($restart_list_file, $unit);
+                # Remove from units to reload so we don't restart and reload
+                if ($units_to_reload->{$unit}) {
+                    delete $units_to_reload->{$unit};
+                    unrecord_unit($reload_list_file, $unit);
+                }
+            } else {
+                # If this unit is socket-activated, then stop the
+                # socket unit(s) as well, and restart the
+                # socket(s) instead of the service.
+                my $socket_activated = 0;
+                if ($unit =~ /\.service$/msx) {
+                    my @sockets = split(/\s+/msx, join(" ", @{$new_unit_info{Service}{Sockets} // []}));
+                    if (scalar(@sockets) == 0) {
+                        @sockets = ("$base_name.socket");
+                    }
+                    foreach my $socket (@sockets) {
+                        if (defined($active_cur->{$socket})) {
+                            # We can now be sure this is a socket-activate unit
+
+                            $units_to_stop->{$socket} = 1;
+                            # Only restart sockets that actually
+                            # exist in new configuration:
+                            if (-e "$out/etc/systemd/system/$socket") {
+                                $units_to_start->{$socket} = 1;
+                                if ($units_to_start eq $units_to_restart) {
+                                    record_unit($restart_list_file, $socket);
+                                } else {
+                                    record_unit($start_list_file, $socket);
+                                }
+                                $socket_activated = 1;
+                            }
+                            # Remove from units to reload so we don't restart and reload
+                            if ($units_to_reload->{$unit}) {
+                                delete $units_to_reload->{$unit};
+                                unrecord_unit($reload_list_file, $unit);
+                            }
+                        }
+                    }
+                }
+
+                # If the unit is not socket-activated, record
+                # that this unit needs to be started below.
+                # We write this to a file to ensure that the
+                # service gets restarted if we're interrupted.
+                if (!$socket_activated) {
+                    $units_to_start->{$unit} = 1;
+                    if ($units_to_start eq $units_to_restart) {
+                        record_unit($restart_list_file, $unit);
+                    } else {
+                        record_unit($start_list_file, $unit);
+                    }
+                }
+
+                $units_to_stop->{$unit} = 1;
+                # Remove from units to reload so we don't restart and reload
+                if ($units_to_reload->{$unit}) {
+                    delete $units_to_reload->{$unit};
+                    unrecord_unit($reload_list_file, $unit);
+                }
+            }
+        }
+    }
+    return;
 }
 
 # Figure out what units need to be stopped, started, restarted or reloaded.
-my (%unitsToStop, %unitsToSkip, %unitsToStart, %unitsToRestart, %unitsToReload);
+my (%units_to_stop, %units_to_skip, %units_to_start, %units_to_restart, %units_to_reload);
 
-my %unitsToFilter; # units not shown
+my %units_to_filter; # units not shown
 
-$unitsToStart{$_} = 1 foreach
-    split('\n', read_file($startListFile, err_mode => 'quiet') // "");
+%units_to_start = map { $_ => 1 }
+    split(/\n/msx, read_file($start_list_file, err_mode => "quiet") // "");
 
-$unitsToRestart{$_} = 1 foreach
-    split('\n', read_file($restartListFile, err_mode => 'quiet') // "");
+%units_to_restart = map { $_ => 1 }
+    split(/\n/msx, read_file($restart_list_file, err_mode => "quiet") // "");
 
-$unitsToReload{$_} = 1 foreach
-    split('\n', read_file($reloadListFile, err_mode => 'quiet') // "");
+%units_to_reload = map { $_ => 1 }
+    split(/\n/msx, read_file($reload_list_file, err_mode => "quiet") // "");
 
-my $activePrev = getActiveUnits;
-while (my ($unit, $state) = each %{$activePrev}) {
-    my $baseUnit = $unit;
+my $active_cur = get_active_units();
+while (my ($unit, $state) = each(%{$active_cur})) {
+    my $base_unit = $unit;
 
-    my $prevUnitFile = "/etc/systemd/system/$baseUnit";
-    my $newUnitFile = "$out/etc/systemd/system/$baseUnit";
+    my $cur_unit_file = "/etc/systemd/system/$base_unit";
+    my $new_unit_file = "$out/etc/systemd/system/$base_unit";
 
     # Detect template instances.
-    if (!-e $prevUnitFile && !-e $newUnitFile && $unit =~ /^(.*)@[^\.]*\.(.*)$/) {
-      $baseUnit = "$1\@.$2";
-      $prevUnitFile = "/etc/systemd/system/$baseUnit";
-      $newUnitFile = "$out/etc/systemd/system/$baseUnit";
+    if (!-e $cur_unit_file && !-e $new_unit_file && $unit =~ /^(.*)@[^\.]*\.(.*)$/msx) {
+      $base_unit = "$1\@.$2";
+      $cur_unit_file = "/etc/systemd/system/$base_unit";
+      $new_unit_file = "$out/etc/systemd/system/$base_unit";
     }
 
-    my $baseName = $baseUnit;
-    $baseName =~ s/\.[a-z]*$//;
+    my $base_name = $base_unit;
+    $base_name =~ s/\.[[:lower:]]*$//msx;
 
-    if (-e $prevUnitFile && ($state->{state} eq "active" || $state->{state} eq "activating")) {
-        if (! -e $newUnitFile || abs_path($newUnitFile) eq "/dev/null") {
-            my $unitInfo = parseUnit($prevUnitFile);
-            $unitsToStop{$unit} = 1 if boolIsTrue($unitInfo->{'X-StopOnRemoval'} // "yes");
+    if (-e $cur_unit_file && ($state->{state} eq "active" || $state->{state} eq "activating")) {
+        if (! -e $new_unit_file || abs_path($new_unit_file) eq "/dev/null") {
+            my %cur_unit_info = parse_unit($cur_unit_file);
+            if (parse_systemd_bool(\%cur_unit_info, "Unit", "X-StopOnRemoval", 1)) {
+                $units_to_stop{$unit} = 1;
+            }
         }
 
-        elsif ($unit =~ /\.target$/) {
-            my $unitInfo = parseUnit($newUnitFile);
+        elsif ($unit =~ /\.target$/msx) {
+            my %new_unit_info = parse_unit($new_unit_file);
 
             # Cause all active target units to be restarted below.
             # This should start most changed units we stop here as
@@ -193,11 +541,11 @@ while (my ($unit, $state) = each %{$activePrev}) {
             # active after the system has resumed, which probably
             # should not be the case.  Just ignore it.
             if ($unit ne "suspend.target" && $unit ne "hibernate.target" && $unit ne "hybrid-sleep.target") {
-                unless (boolIsTrue($unitInfo->{'RefuseManualStart'} // "no") || boolIsTrue($unitInfo->{'X-OnlyManualStart'} // "no")) {
-                    $unitsToStart{$unit} = 1;
-                    recordUnit($startListFile, $unit);
+                if (!(parse_systemd_bool(\%new_unit_info, "Unit", "RefuseManualStart", 0) || parse_systemd_bool(\%new_unit_info, "Unit", "X-OnlyManualStart", 0))) {
+                    $units_to_start{$unit} = 1;
+                    record_unit($start_list_file, $unit);
                     # Don't spam the user with target units that always get started.
-                    $unitsToFilter{$unit} = 1;
+                    $units_to_filter{$unit} = 1;
                 }
             }
 
@@ -212,208 +560,188 @@ while (my ($unit, $state) = each %{$activePrev}) {
             # Stopping a target generally has no effect on other units
             # (unless there is a PartOf dependency), so this is just a
             # bookkeeping thing to get systemd to do the right thing.
-            if (boolIsTrue($unitInfo->{'X-StopOnReconfiguration'} // "no")) {
-                $unitsToStop{$unit} = 1;
+            if (parse_systemd_bool(\%new_unit_info, "Unit", "X-StopOnReconfiguration", 0)) {
+                $units_to_stop{$unit} = 1;
             }
         }
 
-        elsif (fingerprintUnit($prevUnitFile) ne fingerprintUnit($newUnitFile)) {
-            if ($unit eq "sysinit.target" || $unit eq "basic.target" || $unit eq "multi-user.target" || $unit eq "graphical.target" || $unit =~ /\.path$/ || $unit =~ /\.slice$/) {
-                # Do nothing.  These cannot be restarted directly.
-
-                # Slices and Paths don't have to be restarted since
-                # properties (resource limits and inotify watches)
-                # seem to get applied on daemon-reload.
-            } elsif ($unit =~ /\.mount$/) {
-                # Reload the changed mount unit to force a remount.
-                $unitsToReload{$unit} = 1;
-                recordUnit($reloadListFile, $unit);
-            } elsif ($unit =~ /\.socket$/) {
-                # FIXME: do something?
-            } else {
-                my $unitInfo = parseUnit($newUnitFile);
-                if (boolIsTrue($unitInfo->{'X-ReloadIfChanged'} // "no")) {
-                    $unitsToReload{$unit} = 1;
-                    recordUnit($reloadListFile, $unit);
-                }
-                elsif (!boolIsTrue($unitInfo->{'X-RestartIfChanged'} // "yes") || boolIsTrue($unitInfo->{'RefuseManualStop'} // "no") || boolIsTrue($unitInfo->{'X-OnlyManualStart'} // "no")) {
-                    $unitsToSkip{$unit} = 1;
-                } else {
-                    if (!boolIsTrue($unitInfo->{'X-StopIfChanged'} // "yes")) {
-                        # This unit should be restarted instead of
-                        # stopped and started.
-                        $unitsToRestart{$unit} = 1;
-                        recordUnit($restartListFile, $unit);
-                    } else {
-                        # If this unit is socket-activated, then stop the
-                        # socket unit(s) as well, and restart the
-                        # socket(s) instead of the service.
-                        my $socketActivated = 0;
-                        if ($unit =~ /\.service$/) {
-                            my @sockets = split / /, ($unitInfo->{Sockets} // "");
-                            if (scalar @sockets == 0) {
-                                @sockets = ("$baseName.socket");
-                            }
-                            foreach my $socket (@sockets) {
-                                if (defined $activePrev->{$socket}) {
-                                    $unitsToStop{$socket} = 1;
-                                    # Only restart sockets that actually
-                                    # exist in new configuration:
-                                    if (-e "$out/etc/systemd/system/$socket") {
-                                        $unitsToStart{$socket} = 1;
-                                        recordUnit($startListFile, $socket);
-                                        $socketActivated = 1;
-                                    }
-                                }
-                            }
-                        }
-
-                        # If the unit is not socket-activated, record
-                        # that this unit needs to be started below.
-                        # We write this to a file to ensure that the
-                        # service gets restarted if we're interrupted.
-                        if (!$socketActivated) {
-                            $unitsToStart{$unit} = 1;
-                            recordUnit($startListFile, $unit);
-                        }
-
-                        $unitsToStop{$unit} = 1;
-                    }
-                }
+        else {
+            my %cur_unit_info = parse_unit($cur_unit_file);
+            my %new_unit_info = parse_unit($new_unit_file);
+            my $diff = compare_units(\%cur_unit_info, \%new_unit_info);
+            if ($diff == 1) {
+                handle_modified_unit($unit, $base_name, $new_unit_file, \%new_unit_info, $active_cur, \%units_to_stop, \%units_to_start, \%units_to_reload, \%units_to_restart, \%units_to_skip);
+            } elsif ($diff == 2 and not $units_to_restart{$unit}) {
+                $units_to_reload{$unit} = 1;
+                record_unit($reload_list_file, $unit);
             }
         }
     }
 }
 
-sub pathToUnitName {
+# Converts a path to the name of a systemd mount unit that would be responsible
+# for mounting this path.
+sub path_to_unit_name {
     my ($path) = @_;
     # Use current version of systemctl binary before daemon is reexeced.
-    open my $cmd, "-|", "$curSystemd/systemd-escape", "--suffix=mount", "-p", $path
+    open(my $cmd, "-|", "$cur_systemd/systemd-escape", "--suffix=mount", "-p", $path)
         or die "Unable to escape $path!\n";
-    my $escaped = join "", <$cmd>;
-    chomp $escaped;
-    close $cmd or die;
+    my $escaped = do { local $/ = undef; <$cmd> };
+    chomp($escaped);
+    close($cmd) or die("Unable to close systemd-escape pipe");
     return $escaped;
 }
 
-sub unique {
-    my %seen;
-    my @res;
-    foreach my $name (@_) {
-        next if $seen{$name};
-        $seen{$name} = 1;
-        push @res, $name;
-    }
-    return @res;
-}
-
 # Compare the previous and new fstab to figure out which filesystems
 # need a remount or need to be unmounted.  New filesystems are mounted
 # automatically by starting local-fs.target.  FIXME: might be nicer if
 # we generated units for all mounts; then we could unify this with the
 # unit checking code above.
-my ($prevFss, $prevSwaps) = parseFstab "/etc/fstab";
-my ($newFss, $newSwaps) = parseFstab "$out/etc/fstab";
-foreach my $mountPoint (keys %$prevFss) {
-    my $prev = $prevFss->{$mountPoint};
-    my $new = $newFss->{$mountPoint};
-    my $unit = pathToUnitName($mountPoint);
-    if (!defined $new) {
+my ($cur_fss, $cur_swaps) = parse_fstab("/etc/fstab");
+my ($new_fss, $new_swaps) = parse_fstab("$out/etc/fstab");
+foreach my $mount_point (keys(%{$cur_fss})) {
+    my $cur = $cur_fss->{$mount_point};
+    my $new = $new_fss->{$mount_point};
+    my $unit = path_to_unit_name($mount_point);
+    if (!defined($new)) {
         # Filesystem entry disappeared, so unmount it.
-        $unitsToStop{$unit} = 1;
-    } elsif ($prev->{fsType} ne $new->{fsType} || $prev->{device} ne $new->{device}) {
+        $units_to_stop{$unit} = 1;
+    } elsif ($cur->{fsType} ne $new->{fsType} || $cur->{device} ne $new->{device}) {
         # Filesystem type or device changed, so unmount and mount it.
-        $unitsToStop{$unit} = 1;
-        $unitsToStart{$unit} = 1;
-        recordUnit($startListFile, $unit);
-    } elsif ($prev->{options} ne $new->{options}) {
+        $units_to_stop{$unit} = 1;
+        $units_to_start{$unit} = 1;
+        record_unit($start_list_file, $unit);
+    } elsif ($cur->{options} ne $new->{options}) {
         # Mount options changes, so remount it.
-        $unitsToReload{$unit} = 1;
-        recordUnit($reloadListFile, $unit);
+        $units_to_reload{$unit} = 1;
+        record_unit($reload_list_file, $unit);
     }
 }
 
 # Also handles swap devices.
-foreach my $device (keys %$prevSwaps) {
-    my $prev = $prevSwaps->{$device};
-    my $new = $newSwaps->{$device};
-    if (!defined $new) {
+foreach my $device (keys(%{$cur_swaps})) {
+    my $cur = $cur_swaps->{$device};
+    my $new = $new_swaps->{$device};
+    if (!defined($new)) {
         # Swap entry disappeared, so turn it off.  Can't use
         # "systemctl stop" here because systemd has lots of alias
         # units that prevent a stop from actually calling
         # "swapoff".
-        print STDERR "stopping swap device: $device\n";
-        system("@utillinux@/sbin/swapoff", $device);
+        if ($action ne "dry-activate") {
+            print STDERR "would stop swap device: $device\n";
+        } else {
+            print STDERR "stopping swap device: $device\n";
+            system("@utillinux@/sbin/swapoff", $device);
+        }
     }
     # FIXME: update swap options (i.e. its priority).
 }
 
 
 # Should we have systemd re-exec itself?
-my $prevSystemd = abs_path("/proc/1/exe") // "/unknown";
-my $prevSystemdSystemConfig = abs_path("/etc/systemd/system.conf") // "/unknown";
-my $newSystemd = abs_path("@systemd@/lib/systemd/systemd") or die;
-my $newSystemdSystemConfig = abs_path("$out/etc/systemd/system.conf") // "/unknown";
-
-my $restartSystemd = $prevSystemd ne $newSystemd;
-if ($prevSystemdSystemConfig ne $newSystemdSystemConfig) {
-    $restartSystemd = 1;
+my $cur_pid1_path = abs_path("/proc/1/exe") // "/unknown";
+my $cur_systemd_system_config = abs_path("/etc/systemd/system.conf") // "/unknown";
+my $new_pid1_path = abs_path("$new_systemd/lib/systemd/systemd") or die;
+my $new_systemd_system_config = abs_path("$out/etc/systemd/system.conf") // "/unknown";
+
+my $restart_systemd = $cur_pid1_path ne $new_pid1_path;
+if ($cur_systemd_system_config ne $new_systemd_system_config) {
+    $restart_systemd = 1;
 }
 
-
-sub filterUnits {
+# Takes an array of unit names and returns an array with the same elements,
+# except all units that are also in the global variable `unitsToFilter`.
+sub filter_units {
     my ($units) = @_;
     my @res;
-    foreach my $unit (sort(keys %{$units})) {
-        push @res, $unit if !defined $unitsToFilter{$unit};
+    foreach my $unit (sort(keys(%{$units}))) {
+        if (!defined($units_to_filter{$unit})) {
+            push(@res, $unit);
+        }
     }
     return @res;
 }
 
-my @unitsToStopFiltered = filterUnits(\%unitsToStop);
-my @unitsToStartFiltered = filterUnits(\%unitsToStart);
+my @units_to_stop_filtered = filter_units(\%units_to_stop);
 
 
 # Show dry-run actions.
 if ($action eq "dry-activate") {
-    print STDERR "would stop the following units: ", join(", ", @unitsToStopFiltered), "\n"
-        if scalar @unitsToStopFiltered > 0;
-    print STDERR "would NOT stop the following changed units: ", join(", ", sort(keys %unitsToSkip)), "\n"
-        if scalar(keys %unitsToSkip) > 0;
+    if (scalar(@units_to_stop_filtered) > 0) {
+        print STDERR "would stop the following units: ", join(", ", @units_to_stop_filtered), "\n";
+    }
+    if (scalar(keys(%units_to_skip)) > 0) {
+        print STDERR "would NOT stop the following changed units: ", join(", ", sort(keys(%units_to_skip))), "\n";
+    }
 
     print STDERR "would activate the configuration...\n";
     system("$out/dry-activate", "$out");
 
-    $unitsToRestart{$_} = 1 foreach
-        split('\n', read_file($dryRestartByActivationFile, err_mode => 'quiet') // "");
-
-    $unitsToReload{$_} = 1 foreach
-        split('\n', read_file($dryReloadByActivationFile, err_mode => 'quiet') // "");
-
-    print STDERR "would restart systemd\n" if $restartSystemd;
-    print STDERR "would reload the following units: ", join(", ", sort(keys %unitsToReload)), "\n"
-        if scalar(keys %unitsToReload) > 0;
-    print STDERR "would restart the following units: ", join(", ", sort(keys %unitsToRestart)), "\n"
-        if scalar(keys %unitsToRestart) > 0;
-    print STDERR "would start the following units: ", join(", ", @unitsToStartFiltered), "\n"
-        if scalar @unitsToStartFiltered;
-    unlink($dryRestartByActivationFile);
-    unlink($dryReloadByActivationFile);
+    # Handle the activation script requesting the restart or reload of a unit.
+    foreach (split(/\n/msx, read_file($dry_restart_by_activation_file, err_mode => "quiet") // "")) {
+        my $unit = $_;
+        my $base_unit = $unit;
+        my $new_unit_file = "$out/etc/systemd/system/$base_unit";
+
+        # Detect template instances.
+        if (!-e $new_unit_file && $unit =~ /^(.*)@[^\.]*\.(.*)$/msx) {
+          $base_unit = "$1\@.$2";
+          $new_unit_file = "$out/etc/systemd/system/$base_unit";
+        }
+
+        my $base_name = $base_unit;
+        $base_name =~ s/\.[[:lower:]]*$//msx;
+
+        # Start units if they were not active previously
+        if (not defined($active_cur->{$unit})) {
+            $units_to_start{$unit} = 1;
+            next;
+        }
+
+        handle_modified_unit($unit, $base_name, $new_unit_file, undef, $active_cur, \%units_to_restart, \%units_to_restart, \%units_to_reload, \%units_to_restart, \%units_to_skip);
+    }
+    unlink($dry_restart_by_activation_file);
+
+    foreach (split(/\n/msx, read_file($dry_reload_by_activation_file, err_mode => "quiet") // "")) {
+        my $unit = $_;
+
+        if (defined($active_cur->{$unit}) and not $units_to_restart{$unit} and not $units_to_stop{$unit}) {
+            $units_to_reload{$unit} = 1;
+            record_unit($reload_list_file, $unit);
+        }
+    }
+    unlink($dry_reload_by_activation_file);
+
+    if ($restart_systemd) {
+        print STDERR "would restart systemd\n";
+    }
+    if (scalar(keys(%units_to_reload)) > 0) {
+        print STDERR "would reload the following units: ", join(", ", sort(keys(%units_to_reload))), "\n";
+    }
+    if (scalar(keys(%units_to_restart)) > 0) {
+        print STDERR "would restart the following units: ", join(", ", sort(keys(%units_to_restart))), "\n";
+    }
+    my @units_to_start_filtered = filter_units(\%units_to_start);
+    if (scalar(@units_to_start_filtered)) {
+        print STDERR "would start the following units: ", join(", ", @units_to_start_filtered), "\n";
+    }
     exit 0;
 }
 
 
 syslog(LOG_NOTICE, "switching to system configuration $out");
 
-if (scalar (keys %unitsToStop) > 0) {
-    print STDERR "stopping the following units: ", join(", ", @unitsToStopFiltered), "\n"
-        if scalar @unitsToStopFiltered;
+if (scalar(keys(%units_to_stop)) > 0) {
+    if (scalar(@units_to_stop_filtered)) {
+        print STDERR "stopping the following units: ", join(", ", @units_to_stop_filtered), "\n";
+    }
     # Use current version of systemctl binary before daemon is reexeced.
-    system("$curSystemd/systemctl", "stop", "--", sort(keys %unitsToStop));
+    system("$cur_systemd/systemctl", "stop", "--", sort(keys(%units_to_stop)));
 }
 
-print STDERR "NOT restarting the following changed units: ", join(", ", sort(keys %unitsToSkip)), "\n"
-    if scalar(keys %unitsToSkip) > 0;
+if (scalar(keys(%units_to_skip)) > 0) {
+    print STDERR "NOT restarting the following changed units: ", join(", ", sort(keys(%units_to_skip))), "\n";
+}
 
 # Activate the new configuration (i.e., update /etc, make accounts,
 # and so on).
@@ -422,63 +750,110 @@ print STDERR "activating the configuration...\n";
 system("$out/activate", "$out") == 0 or $res = 2;
 
 # Handle the activation script requesting the restart or reload of a unit.
-# We can only restart and reload (not stop/start) because the units to be
-# stopped are already stopped before the activation script is run.
-$unitsToRestart{$_} = 1 foreach
-    split('\n', read_file($restartByActivationFile, err_mode => 'quiet') // "");
+foreach (split(/\n/msx, read_file($restart_by_activation_file, err_mode => "quiet") // "")) {
+    my $unit = $_;
+    my $base_unit = $unit;
+    my $new_unit_file = "$out/etc/systemd/system/$base_unit";
 
-$unitsToReload{$_} = 1 foreach
-    split('\n', read_file($reloadByActivationFile, err_mode => 'quiet') // "");
+    # Detect template instances.
+    if (!-e $new_unit_file && $unit =~ /^(.*)@[^\.]*\.(.*)$/msx) {
+      $base_unit = "$1\@.$2";
+      $new_unit_file = "$out/etc/systemd/system/$base_unit";
+    }
+
+    my $base_name = $base_unit;
+    $base_name =~ s/\.[[:lower:]]*$//msx;
+
+    # Start units if they were not active previously
+    if (not defined($active_cur->{$unit})) {
+        $units_to_start{$unit} = 1;
+        record_unit($start_list_file, $unit);
+        next;
+    }
+
+    handle_modified_unit($unit, $base_name, $new_unit_file, undef, $active_cur, \%units_to_restart, \%units_to_restart, \%units_to_reload, \%units_to_restart, \%units_to_skip);
+}
+# We can remove the file now because it has been propagated to the other restart/reload files
+unlink($restart_by_activation_file);
+
+foreach (split(/\n/msx, read_file($reload_by_activation_file, err_mode => "quiet") // "")) {
+    my $unit = $_;
+
+    if (defined($active_cur->{$unit}) and not $units_to_restart{$unit} and not $units_to_stop{$unit}) {
+        $units_to_reload{$unit} = 1;
+        record_unit($reload_list_file, $unit);
+    }
+}
+# We can remove the file now because it has been propagated to the other reload file
+unlink($reload_by_activation_file);
 
 # Restart systemd if necessary. Note that this is done using the
 # current version of systemd, just in case the new one has trouble
 # communicating with the running pid 1.
-if ($restartSystemd) {
+if ($restart_systemd) {
     print STDERR "restarting systemd...\n";
-    system("$curSystemd/systemctl", "daemon-reexec") == 0 or $res = 2;
+    system("$cur_systemd/systemctl", "daemon-reexec") == 0 or $res = 2;
 }
 
 # Forget about previously failed services.
-system("@systemd@/bin/systemctl", "reset-failed");
+system("$new_systemd/bin/systemctl", "reset-failed");
 
 # Make systemd reload its units.
-system("@systemd@/bin/systemctl", "daemon-reload") == 0 or $res = 3;
+system("$new_systemd/bin/systemctl", "daemon-reload") == 0 or $res = 3;
 
 # Reload user units
-open my $listActiveUsers, '-|', '@systemd@/bin/loginctl', 'list-users', '--no-legend';
-while (my $f = <$listActiveUsers>) {
-    next unless $f =~ /^\s*(?<uid>\d+)\s+(?<user>\S+)/;
+open(my $list_active_users, "-|", "$new_systemd/bin/loginctl", "list-users", "--no-legend") || die("Unable to call loginctl");
+while (my $f = <$list_active_users>) {
+    if ($f !~ /^\s*(?<uid>\d+)\s+(?<user>\S+)/msx) {
+        next;
+    }
     my ($uid, $name) = ($+{uid}, $+{user});
     print STDERR "reloading user units for $name...\n";
 
     system("@su@", "-s", "@shell@", "-l", $name, "-c",
            "export XDG_RUNTIME_DIR=/run/user/$uid; " .
-           "$curSystemd/systemctl --user daemon-reexec; " .
-           "@systemd@/bin/systemctl --user start nixos-activation.service");
+           "$cur_systemd/systemctl --user daemon-reexec; " .
+           "$new_systemd/bin/systemctl --user start nixos-activation.service");
 }
 
-close $listActiveUsers;
+close($list_active_users) || die("Unable to close the file handle to loginctl");
 
 # Set the new tmpfiles
 print STDERR "setting up tmpfiles\n";
-system("@systemd@/bin/systemd-tmpfiles", "--create", "--remove", "--exclude-prefix=/dev") == 0 or $res = 3;
-
+system("$new_systemd/bin/systemd-tmpfiles", "--create", "--remove", "--exclude-prefix=/dev") == 0 or $res = 3;
+
+# Before reloading we need to ensure that the units are still active. They may have been
+# deactivated because one of their requirements got stopped. If they are inactive
+# but should have been reloaded, the user probably expects them to be started.
+if (scalar(keys(%units_to_reload)) > 0) {
+    for my $unit (keys(%units_to_reload)) {
+        if (!unit_is_active($unit)) {
+            # Figure out if we need to start the unit
+            my %unit_info = parse_unit("$out/etc/systemd/system/$unit");
+            if (!(parse_systemd_bool(\%unit_info, "Unit", "RefuseManualStart", 0) || parse_systemd_bool(\%unit_info, "Unit", "X-OnlyManualStart", 0))) {
+                $units_to_start{$unit} = 1;
+                record_unit($start_list_file, $unit);
+            }
+            # Don't reload the unit, reloading would fail
+            delete %units_to_reload{$unit};
+            unrecord_unit($reload_list_file, $unit);
+        }
+    }
+}
 # Reload units that need it. This includes remounting changed mount
 # units.
-if (scalar(keys %unitsToReload) > 0) {
-    print STDERR "reloading the following units: ", join(", ", sort(keys %unitsToReload)), "\n";
-    system("@systemd@/bin/systemctl", "reload", "--", sort(keys %unitsToReload)) == 0 or $res = 4;
-    unlink($reloadListFile);
-    unlink($reloadByActivationFile);
+if (scalar(keys(%units_to_reload)) > 0) {
+    print STDERR "reloading the following units: ", join(", ", sort(keys(%units_to_reload))), "\n";
+    system("$new_systemd/bin/systemctl", "reload", "--", sort(keys(%units_to_reload))) == 0 or $res = 4;
+    unlink($reload_list_file);
 }
 
 # Restart changed services (those that have to be restarted rather
 # than stopped and started).
-if (scalar(keys %unitsToRestart) > 0) {
-    print STDERR "restarting the following units: ", join(", ", sort(keys %unitsToRestart)), "\n";
-    system("@systemd@/bin/systemctl", "restart", "--", sort(keys %unitsToRestart)) == 0 or $res = 4;
-    unlink($restartListFile);
-    unlink($restartByActivationFile);
+if (scalar(keys(%units_to_restart)) > 0) {
+    print STDERR "restarting the following units: ", join(", ", sort(keys(%units_to_restart))), "\n";
+    system("$new_systemd/bin/systemctl", "restart", "--", sort(keys(%units_to_restart))) == 0 or $res = 4;
+    unlink($restart_list_file);
 }
 
 # Start all active targets, as well as changed units we stopped above.
@@ -487,45 +862,52 @@ if (scalar(keys %unitsToRestart) > 0) {
 # that are symlinks to other units.  We shouldn't start both at the
 # same time because we'll get a "Failed to add path to set" error from
 # systemd.
-print STDERR "starting the following units: ", join(", ", @unitsToStartFiltered), "\n"
-    if scalar @unitsToStartFiltered;
-system("@systemd@/bin/systemctl", "start", "--", sort(keys %unitsToStart)) == 0 or $res = 4;
-unlink($startListFile);
+my @units_to_start_filtered = filter_units(\%units_to_start);
+if (scalar(@units_to_start_filtered)) {
+    print STDERR "starting the following units: ", join(", ", @units_to_start_filtered), "\n"
+}
+system("$new_systemd/bin/systemctl", "start", "--", sort(keys(%units_to_start))) == 0 or $res = 4;
+unlink($start_list_file);
 
 
 # Print failed and new units.
 my (@failed, @new);
-my $activeNew = getActiveUnits;
-while (my ($unit, $state) = each %{$activeNew}) {
+my $active_new = get_active_units();
+while (my ($unit, $state) = each(%{$active_new})) {
     if ($state->{state} eq "failed") {
-        push @failed, $unit;
+        push(@failed, $unit);
+        next;
     }
-    elsif ($state->{state} eq "auto-restart") {
-        # A unit in auto-restart state is a failure *if* it previously failed to start
-        my $lines = `@systemd@/bin/systemctl show '$unit'`;
-        my $info = {};
-        parseKeyValues($info, split("\n", $lines));
 
-        if ($info->{ExecMainStatus} ne '0') {
-            push @failed, $unit;
+    if ($state->{substate} eq "auto-restart") {
+        # A unit in auto-restart substate is a failure *if* it previously failed to start
+        open(my $main_status_fd, "-|", "$new_systemd/bin/systemctl", "show", "--value", "--property=ExecMainStatus", $unit) || die("Unable to call 'systemctl show'");
+        my $main_status = do { local $/ = undef; <$main_status_fd> };
+        close($main_status_fd) || die("Unable to close 'systemctl show' fd");
+        chomp($main_status);
+
+        if ($main_status ne "0") {
+            push(@failed, $unit);
+            next;
         }
     }
+
     # Ignore scopes since they are not managed by this script but rather
     # created and managed by third-party services via the systemd dbus API.
-    elsif ($state->{state} ne "failed" && !defined $activePrev->{$unit} && $unit !~ /\.scope$/) {
-        push @new, $unit;
+    # This only lists units that are not failed (including ones that are in auto-restart but have not failed previously)
+    if ($state->{state} ne "failed" && !defined($active_cur->{$unit}) && $unit !~ /\.scope$/msx) {
+        push(@new, $unit);
     }
 }
 
-print STDERR "the following new units were started: ", join(", ", sort(@new)), "\n"
-    if scalar @new > 0;
+if (scalar(@new) > 0) {
+    print STDERR "the following new units were started: ", join(", ", sort(@new)), "\n"
+}
 
-if (scalar @failed > 0) {
-    print STDERR "warning: the following units failed: ", join(", ", sort(@failed)), "\n";
-    foreach my $unit (@failed) {
-        print STDERR "\n";
-        system("COLUMNS=1000 @systemd@/bin/systemctl status --no-pager '$unit' >&2");
-    }
+if (scalar(@failed) > 0) {
+    my @failed_sorted = sort(@failed);
+    print STDERR "warning: the following units failed: ", join(", ", @failed_sorted), "\n\n";
+    system("$new_systemd/bin/systemctl status --no-pager --full '" . join("' '", @failed_sorted) . "' >&2");
     $res = 4;
 }
 
@@ -535,4 +917,4 @@ if ($res == 0) {
     syslog(LOG_ERR, "switching to system configuration $out failed (status $res)");
 }
 
-exit $res;
+exit($res);
diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix
index 501998fa399e..b8aeee8c11b3 100644
--- a/nixos/modules/system/activation/top-level.nix
+++ b/nixos/modules/system/activation/top-level.nix
@@ -55,8 +55,8 @@ let
       substituteInPlace $out/dry-activate --subst-var out
       chmod u+x $out/activate $out/dry-activate
       unset activationScript dryActivationScript
-      ${pkgs.stdenv.shell} -n $out/activate
-      ${pkgs.stdenv.shell} -n $out/dry-activate
+      ${pkgs.stdenv.shellDryRun} $out/activate
+      ${pkgs.stdenv.shellDryRun} $out/dry-activate
 
       cp ${config.system.build.bootStage2} $out/init
       substituteInPlace $out/init --subst-var-by systemConfig $out
@@ -109,9 +109,7 @@ let
     utillinux = pkgs.util-linux;
 
     kernelParams = config.boot.kernelParams;
-    installBootLoader =
-      config.system.build.installBootLoader
-      or "echo 'Warning: do not know how to make this configuration bootable; please enable a boot loader.' 1>&2; true";
+    installBootLoader = config.system.build.installBootLoader;
     activationScript = config.system.activationScripts.script;
     dryActivationScript = config.system.dryActivationScript;
     nixosLabel = config.system.nixos.label;
@@ -119,7 +117,7 @@ let
     configurationName = config.boot.loader.grub.configurationName;
 
     # Needed by switch-to-configuration.
-    perl = pkgs.perl.withPackages (p: with p; [ FileSlurp NetDBus XMLParser XMLTwig ]);
+    perl = pkgs.perl.withPackages (p: with p; [ ConfigIniFiles FileSlurp NetDBus ]);
   };
 
   # Handle assertions and warnings
@@ -135,28 +133,30 @@ let
       pkgs.replaceDependency { inherit oldDependency newDependency drv; }
     ) baseSystemAssertWarn config.system.replaceRuntimeDependencies;
 
+  /* Workaround until https://github.com/NixOS/nixpkgs/pull/156533
+     Call can be replaced by argument when that's merged.
+  */
+  tmpFixupSubmoduleBoundary = subopts:
+    lib.mkOption {
+      type = lib.types.submoduleWith {
+        modules = [ { options = subopts; } ];
+      };
+    };
+
 in
 
 {
   imports = [
+    ../build.nix
     (mkRemovedOptionModule [ "nesting" "clone" ] "Use `specialisation.«name» = { inheritParentConfig = true; configuration = { ... }; }` instead.")
     (mkRemovedOptionModule [ "nesting" "children" ] "Use `specialisation.«name».configuration = { ... }` instead.")
   ];
 
   options = {
 
-    system.build = mkOption {
-      internal = true;
-      default = {};
-      type = types.attrs;
-      description = ''
-        Attribute set of derivations used to setup the system.
-      '';
-    };
-
     specialisation = mkOption {
       default = {};
-      example = lib.literalExpression "{ fewJobsManyCores.configuration = { nix.buildCores = 0; nix.maxJobs = 1; }; }";
+      example = lib.literalExpression "{ fewJobsManyCores.configuration = { nix.settings = { core = 0; max-jobs = 1; }; }";
       description = ''
         Additional configurations to build. If
         <literal>inheritParentConfig</literal> is true, the system
@@ -224,6 +224,39 @@ in
       '';
     };
 
+    system.build = tmpFixupSubmoduleBoundary {
+      installBootLoader = mkOption {
+        internal = true;
+        # "; true" => make the `$out` argument from switch-to-configuration.pl
+        #             go to `true` instead of `echo`, hiding the useless path
+        #             from the log.
+        default = "echo 'Warning: do not know how to make this configuration bootable; please enable a boot loader.' 1>&2; true";
+        description = ''
+          A program that writes a bootloader installation script to the path passed in the first command line argument.
+
+          See <literal>nixos/modules/system/activation/switch-to-configuration.pl</literal>.
+        '';
+        type = types.unique {
+          message = ''
+            Only one bootloader can be enabled at a time. This requirement has not
+            been checked until NixOS 22.05. Earlier versions defaulted to the last
+            definition. Change your configuration to enable only one bootloader.
+          '';
+        } (types.either types.str types.package);
+      };
+
+      toplevel = mkOption {
+        type = types.package;
+        readOnly = true;
+        description = ''
+          This option contains the store path that typically represents a NixOS system.
+
+          You can read this path in a custom deployment tool for example.
+        '';
+      };
+    };
+
+
     system.copySystemConfiguration = mkOption {
       type = types.bool;
       default = false;
@@ -317,4 +350,6 @@ in
 
   };
 
+  # uses extendModules to generate a type
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/system/boot/binfmt.nix b/nixos/modules/system/boot/binfmt.nix
index fdb4d0e4c7fb..33748358e45b 100644
--- a/nixos/modules/system/boot/binfmt.nix
+++ b/nixos/modules/system/boot/binfmt.nix
@@ -20,16 +20,20 @@ let
                  optionalString fixBinary "F";
   in ":${name}:${type}:${offset'}:${magicOrExtension}:${mask'}:${interpreter}:${flags}";
 
-  activationSnippet = name: { interpreter, ... }: ''
+  activationSnippet = name: { interpreter, wrapInterpreterInShell, ... }: if wrapInterpreterInShell then ''
     rm -f /run/binfmt/${name}
     cat > /run/binfmt/${name} << 'EOF'
     #!${pkgs.bash}/bin/sh
     exec -- ${interpreter} "$@"
     EOF
     chmod +x /run/binfmt/${name}
+  '' else ''
+    rm -f /run/binfmt/${name}
+    ln -s ${interpreter} /run/binfmt/${name}
   '';
 
   getEmulator = system: (lib.systems.elaborate { inherit system; }).emulator pkgs;
+  getQemuArch = system: (lib.systems.elaborate { inherit system; }).qemuArch;
 
   # Mapping of systems to “magicOrExtension” and “mask”. Mostly taken from:
   # - https://github.com/cleverca22/nixos-configs/blob/master/qemu.nix
@@ -238,6 +242,25 @@ in {
               '';
               type = types.bool;
             };
+
+            wrapInterpreterInShell = mkOption {
+              default = true;
+              description = ''
+                Whether to wrap the interpreter in a shell script.
+
+                This allows a shell command to be set as the interpreter.
+              '';
+              type = types.bool;
+            };
+
+            interpreterSandboxPath = mkOption {
+              internal = true;
+              default = null;
+              description = ''
+                Path of the interpreter to expose in the build sandbox.
+              '';
+              type = types.nullOr types.path;
+            };
           };
         }));
       };
@@ -258,16 +281,34 @@ in {
   config = {
     boot.binfmt.registrations = builtins.listToAttrs (map (system: {
       name = system;
-      value = {
+      value = let
         interpreter = getEmulator system;
+        qemuArch = getQemuArch system;
+
+        preserveArgvZero = "qemu-${qemuArch}" == baseNameOf interpreter;
+        interpreterReg = let
+          wrapperName = "qemu-${qemuArch}-binfmt-P";
+          wrapper = pkgs.wrapQemuBinfmtP wrapperName interpreter;
+        in
+          if preserveArgvZero then "${wrapper}/bin/${wrapperName}"
+          else interpreter;
+      in {
+        inherit preserveArgvZero;
+
+        interpreter = interpreterReg;
+        wrapInterpreterInShell = !preserveArgvZero;
+        interpreterSandboxPath = dirOf (dirOf interpreterReg);
       } // (magics.${system} or (throw "Cannot create binfmt registration for system ${system}"));
     }) cfg.emulatedSystems);
-    # TODO: add a nix.extraPlatforms option to NixOS!
-    nix.extraOptions = lib.mkIf (cfg.emulatedSystems != []) ''
-      extra-platforms = ${toString (cfg.emulatedSystems ++ lib.optional pkgs.stdenv.hostPlatform.isx86_64 "i686-linux")}
-    '';
-    nix.sandboxPaths = lib.mkIf (cfg.emulatedSystems != [])
-      ([ "/run/binfmt" "${pkgs.bash}" ] ++ (map (system: dirOf (dirOf (getEmulator system))) cfg.emulatedSystems));
+    nix.settings = lib.mkIf (cfg.emulatedSystems != []) {
+      extra-platforms = cfg.emulatedSystems ++ lib.optional pkgs.stdenv.hostPlatform.isx86_64 "i686-linux";
+      extra-sandbox-paths = let
+        ruleFor = system: cfg.registrations.${system};
+        hasWrappedRule = lib.any (system: (ruleFor system).wrapInterpreterInShell) cfg.emulatedSystems;
+      in [ "/run/binfmt" ]
+        ++ lib.optional hasWrappedRule "${pkgs.bash}"
+        ++ (map (system: (ruleFor system).interpreterSandboxPath) cfg.emulatedSystems);
+    };
 
     environment.etc."binfmt.d/nixos.conf".source = builtins.toFile "binfmt_nixos.conf"
       (lib.concatStringsSep "\n" (lib.mapAttrsToList makeBinfmtLine config.boot.binfmt.registrations));
diff --git a/nixos/modules/system/boot/kernel.nix b/nixos/modules/system/boot/kernel.nix
index d147155d796c..db00244ca0af 100644
--- a/nixos/modules/system/boot/kernel.nix
+++ b/nixos/modules/system/boot/kernel.nix
@@ -36,7 +36,7 @@ in
 
     boot.kernelPackages = mkOption {
       default = pkgs.linuxPackages;
-      type = types.unspecified // { merge = mergeEqualOption; };
+      type = types.raw;
       apply = kernelPackages: kernelPackages.extend (self: super: {
         kernel = super.kernel.override (originalArgs: {
           inherit randstructSeed;
diff --git a/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix b/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix
index bd508bbe8eaa..545b594674f3 100644
--- a/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix
+++ b/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix
@@ -30,6 +30,21 @@ in
         '';
       };
 
+      useGenerationDeviceTree = mkOption {
+        default = true;
+        type = types.bool;
+        description = ''
+          Whether to generate Device Tree-related directives in the
+          extlinux configuration.
+
+          When enabled, the bootloader will attempt to load the device
+          tree binaries from the generation's kernel.
+
+          Note that this affects all generations, regardless of the
+          setting value used in their configurations.
+        '';
+      };
+
       configurationLimit = mkOption {
         default = 20;
         example = 10;
@@ -54,7 +69,9 @@ in
   };
 
   config = let
-    builderArgs = "-g ${toString cfg.configurationLimit} -t ${timeoutStr}" + lib.optionalString (dtCfg.name != null) " -n ${dtCfg.name}";
+    builderArgs = "-g ${toString cfg.configurationLimit} -t ${timeoutStr}"
+      + lib.optionalString (dtCfg.name != null) " -n ${dtCfg.name}"
+      + lib.optionalString (!cfg.useGenerationDeviceTree) " -r";
   in
     mkIf cfg.enable {
       system.build.installBootLoader = "${builder} ${builderArgs} -c";
diff --git a/nixos/modules/system/boot/loader/generic-extlinux-compatible/extlinux-conf-builder.sh b/nixos/modules/system/boot/loader/generic-extlinux-compatible/extlinux-conf-builder.sh
index 5ffffb95edb1..1a0da0050291 100644
--- a/nixos/modules/system/boot/loader/generic-extlinux-compatible/extlinux-conf-builder.sh
+++ b/nixos/modules/system/boot/loader/generic-extlinux-compatible/extlinux-conf-builder.sh
@@ -6,7 +6,7 @@ export PATH=/empty
 for i in @path@; do PATH=$PATH:$i/bin; done
 
 usage() {
-    echo "usage: $0 -t <timeout> -c <path-to-default-configuration> [-d <boot-dir>] [-g <num-generations>] [-n <dtbName>]" >&2
+    echo "usage: $0 -t <timeout> -c <path-to-default-configuration> [-d <boot-dir>] [-g <num-generations>] [-n <dtbName>] [-r]" >&2
     exit 1
 }
 
@@ -15,7 +15,7 @@ default=                # Default configuration
 target=/boot            # Target directory
 numGenerations=0        # Number of other generations to include in the menu
 
-while getopts "t:c:d:g:n:" opt; do
+while getopts "t:c:d:g:n:r" opt; do
     case "$opt" in
         t) # U-Boot interprets '0' as infinite and negative as instant boot
             if [ "$OPTARG" -lt 0 ]; then
@@ -30,6 +30,7 @@ while getopts "t:c:d:g:n:" opt; do
         d) target="$OPTARG" ;;
         g) numGenerations="$OPTARG" ;;
         n) dtbName="$OPTARG" ;;
+        r) noDeviceTree=1 ;;
         \?) usage ;;
     esac
 done
@@ -96,6 +97,12 @@ addEntry() {
     fi
     echo "  LINUX ../nixos/$(basename $kernel)"
     echo "  INITRD ../nixos/$(basename $initrd)"
+    echo "  APPEND init=$path/init $extraParams"
+
+    if [ -n "$noDeviceTree" ]; then
+        return
+    fi
+
     if [ -d "$dtbDir" ]; then
         # if a dtbName was specified explicitly, use that, else use FDTDIR
         if [ -n "$dtbName" ]; then
@@ -109,7 +116,6 @@ addEntry() {
             exit 1
         fi
     fi
-    echo "  APPEND init=$path/init $extraParams"
 }
 
 tmpFile="$target/extlinux/extlinux.conf.tmp.$$"
diff --git a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py
index e9697b5f0e64..fa879437fd81 100644
--- a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py
+++ b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py
@@ -15,9 +15,12 @@ import re
 import datetime
 import glob
 import os.path
-from typing import Tuple, List, Optional
+from typing import NamedTuple, List, Optional
 
-SystemIdentifier = Tuple[Optional[str], int, Optional[str]]
+class SystemIdentifier(NamedTuple):
+    profile: Optional[str]
+    generation: int
+    specialisation: Optional[str]
 
 
 def copy_if_not_exists(source: str, dest: str) -> None:
@@ -45,16 +48,6 @@ initrd {initrd}
 options {kernel_params}
 """
 
-# The boot loader entry for memtest86.
-#
-# TODO: This is hard-coded to use the 64-bit EFI app, but it could probably
-# be updated to use the 32-bit EFI app on 32-bit systems.  The 32-bit EFI
-# app filename is BOOTIA32.efi.
-MEMTEST_BOOT_ENTRY = """title MemTest86
-efi /efi/memtest86/BOOTX64.efi
-"""
-
-
 def generation_conf_filename(profile: Optional[str], generation: int, specialisation: Optional[str]) -> str:
     pieces = [
         "nixos",
@@ -161,7 +154,14 @@ def get_generations(profile: Optional[str] = None) -> List[SystemIdentifier]:
     gen_lines.pop()
 
     configurationLimit = @configurationLimit@
-    configurations: List[SystemIdentifier] = [ (profile, int(line.split()[0]), None) for line in gen_lines ]
+    configurations = [
+        SystemIdentifier(
+            profile=profile,
+            generation=int(line.split()[0]),
+            specialisation=None
+        )
+        for line in gen_lines
+    ]
     return configurations[-configurationLimit:]
 
 
@@ -170,7 +170,7 @@ def get_specialisations(profile: Optional[str], generation: int, _: Optional[str
             system_dir(profile, generation, None), "specialisation")
     if not os.path.exists(specialisations_dir):
         return []
-    return [(profile, generation, spec) for spec in os.listdir(specialisations_dir)]
+    return [SystemIdentifier(profile, generation, spec) for spec in os.listdir(specialisations_dir)]
 
 
 def remove_old_entries(gens: List[SystemIdentifier]) -> None:
@@ -281,25 +281,27 @@ def main() -> None:
             if os.readlink(system_dir(*gen)) == args.default_config:
                 write_loader_conf(*gen)
         except OSError as e:
-            print("ignoring profile '{}' in the list of boot entries because of the following error:\n{}".format(profile, e), file=sys.stderr)
-
-    memtest_entry_file = "@efiSysMountPoint@/loader/entries/memtest86.conf"
-    if os.path.exists(memtest_entry_file):
-        os.unlink(memtest_entry_file)
-    shutil.rmtree("@efiSysMountPoint@/efi/memtest86", ignore_errors=True)
-    if "@memtest86@" != "":
-        mkdir_p("@efiSysMountPoint@/efi/memtest86")
-        for path in glob.iglob("@memtest86@/*"):
-            if os.path.isdir(path):
-                shutil.copytree(path, os.path.join("@efiSysMountPoint@/efi/memtest86", os.path.basename(path)))
-            else:
-                shutil.copy(path, "@efiSysMountPoint@/efi/memtest86/")
+            profile = f"profile '{gen.profile}'" if gen.profile else "default profile"
+            print("ignoring {} in the list of boot entries because of the following error:\n{}".format(profile, e), file=sys.stderr)
+
+    for root, _, files in os.walk('@efiSysMountPoint@/efi/nixos/.extra-files', topdown=False):
+        relative_root = root.removeprefix("@efiSysMountPoint@/efi/nixos/.extra-files").removeprefix("/")
+        actual_root = os.path.join("@efiSysMountPoint@", relative_root)
+
+        for file in files:
+            actual_file = os.path.join(actual_root, file)
+
+            if os.path.exists(actual_file):
+                os.unlink(actual_file)
+            os.unlink(os.path.join(root, file))
+
+        if not len(os.listdir(actual_root)):
+            os.rmdir(actual_root)
+        os.rmdir(root)
+
+    mkdir_p("@efiSysMountPoint@/efi/nixos/.extra-files")
 
-        memtest_entry_file = "@efiSysMountPoint@/loader/entries/memtest86.conf"
-        memtest_entry_file_tmp_path = "%s.tmp" % memtest_entry_file
-        with open(memtest_entry_file_tmp_path, 'w') as f:
-            f.write(MEMTEST_BOOT_ENTRY)
-        os.rename(memtest_entry_file_tmp_path, memtest_entry_file)
+    subprocess.check_call("@copyExtraFiles@")
 
     # Since fat32 provides little recovery facilities after a crash,
     # it can leave the system in an unbootable state, when a crash/outage
diff --git a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix
index 0f76d7d6b24a..c07567ec82ea 100644
--- a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix
+++ b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix
@@ -29,6 +29,22 @@ let
     inherit (efi) efiSysMountPoint canTouchEfiVariables;
 
     memtest86 = if cfg.memtest86.enable then pkgs.memtest86-efi else "";
+
+    netbootxyz = if cfg.netbootxyz.enable then pkgs.netbootxyz-efi else "";
+
+    copyExtraFiles = pkgs.writeShellScript "copy-extra-files" ''
+      empty_file=$(mktemp)
+
+      ${concatStrings (mapAttrsToList (n: v: ''
+        ${pkgs.coreutils}/bin/install -Dp "${v}" "${efi.efiSysMountPoint}/"${escapeShellArg n}
+        ${pkgs.coreutils}/bin/install -D $empty_file "${efi.efiSysMountPoint}/efi/nixos/.extra-files/"${escapeShellArg n}
+      '') cfg.extraFiles)}
+
+      ${concatStrings (mapAttrsToList (n: v: ''
+        ${pkgs.coreutils}/bin/install -Dp "${pkgs.writeText n v}" "${efi.efiSysMountPoint}/loader/entries/"${escapeShellArg n}
+        ${pkgs.coreutils}/bin/install -D $empty_file "${efi.efiSysMountPoint}/efi/nixos/.extra-files/loader/entries/"${escapeShellArg n}
+      '') cfg.extraEntries)}
+    '';
   };
 
   checkedSystemdBootBuilder = pkgs.runCommand "systemd-boot" {
@@ -125,6 +141,74 @@ in {
           <literal>true</literal>.
         '';
       };
+
+      entryFilename = mkOption {
+        default = "memtest86.conf";
+        type = types.str;
+        description = ''
+          <literal>systemd-boot</literal> orders the menu entries by the config file names,
+          so if you want something to appear after all the NixOS entries,
+          it should start with <filename>o</filename> or onwards.
+        '';
+      };
+    };
+
+    netbootxyz = {
+      enable = mkOption {
+        default = false;
+        type = types.bool;
+        description = ''
+          Make <literal>netboot.xyz</literal> available from the
+          <literal>systemd-boot</literal> menu. <literal>netboot.xyz</literal>
+          is a menu system that allows you to boot OS installers and
+          utilities over the network.
+        '';
+      };
+
+      entryFilename = mkOption {
+        default = "o_netbootxyz.conf";
+        type = types.str;
+        description = ''
+          <literal>systemd-boot</literal> orders the menu entries by the config file names,
+          so if you want something to appear after all the NixOS entries,
+          it should start with <filename>o</filename> or onwards.
+        '';
+      };
+    };
+
+    extraEntries = mkOption {
+      type = types.attrsOf types.lines;
+      default = {};
+      example = literalExpression ''
+        { "memtest86.conf" = '''
+          title MemTest86
+          efi /efi/memtest86/memtest86.efi
+        '''; }
+      '';
+      description = ''
+        Any additional entries you want added to the <literal>systemd-boot</literal> menu.
+        These entries will be copied to <filename>/boot/loader/entries</filename>.
+        Each attribute name denotes the destination file name,
+        and the corresponding attribute value is the contents of the entry.
+
+        <literal>systemd-boot</literal> orders the menu entries by the config file names,
+        so if you want something to appear after all the NixOS entries,
+        it should start with <filename>o</filename> or onwards.
+      '';
+    };
+
+    extraFiles = mkOption {
+      type = types.attrsOf types.path;
+      default = {};
+      example = literalExpression ''
+        { "efi/memtest86/memtest86.efi" = "''${pkgs.memtest86-efi}/BOOTX64.efi"; }
+      '';
+      description = ''
+        A set of files to be copied to <filename>/boot</filename>.
+        Each attribute name denotes the destination file name in
+        <filename>/boot</filename>, while the corresponding
+        attribute value specifies the source file.
+      '';
     };
 
     graceful = mkOption {
@@ -148,15 +232,64 @@ in {
     assertions = [
       {
         assertion = (config.boot.kernelPackages.kernel.features or { efiBootStub = true; }) ? efiBootStub;
-
         message = "This kernel does not support the EFI boot stub";
       }
-    ];
+    ] ++ concatMap (filename: [
+      {
+        assertion = !(hasInfix "/" filename);
+        message = "boot.loader.systemd-boot.extraEntries.${lib.strings.escapeNixIdentifier filename} is invalid: entries within folders are not supported";
+      }
+      {
+        assertion = hasSuffix ".conf" filename;
+        message = "boot.loader.systemd-boot.extraEntries.${lib.strings.escapeNixIdentifier filename} is invalid: entries must have a .conf file extension";
+      }
+    ]) (builtins.attrNames cfg.extraEntries)
+      ++ concatMap (filename: [
+        {
+          assertion = !(hasPrefix "/" filename);
+          message = "boot.loader.systemd-boot.extraFiles.${lib.strings.escapeNixIdentifier filename} is invalid: paths must not begin with a slash";
+        }
+        {
+          assertion = !(hasInfix ".." filename);
+          message = "boot.loader.systemd-boot.extraFiles.${lib.strings.escapeNixIdentifier filename} is invalid: paths must not reference the parent directory";
+        }
+        {
+          assertion = !(hasInfix "nixos/.extra-files" (toLower filename));
+          message = "boot.loader.systemd-boot.extraFiles.${lib.strings.escapeNixIdentifier filename} is invalid: files cannot be placed in the nixos/.extra-files directory";
+        }
+      ]) (builtins.attrNames cfg.extraFiles);
 
     boot.loader.grub.enable = mkDefault false;
 
     boot.loader.supportsInitrdSecrets = true;
 
+    boot.loader.systemd-boot.extraFiles = mkMerge [
+      # TODO: This is hard-coded to use the 64-bit EFI app, but it could probably
+      # be updated to use the 32-bit EFI app on 32-bit systems.  The 32-bit EFI
+      # app filename is BOOTIA32.efi.
+      (mkIf cfg.memtest86.enable {
+        "efi/memtest86/BOOTX64.efi" = "${pkgs.memtest86-efi}/BOOTX64.efi";
+      })
+      (mkIf cfg.netbootxyz.enable {
+        "efi/netbootxyz/netboot.xyz.efi" = "${pkgs.netbootxyz-efi}";
+      })
+    ];
+
+    boot.loader.systemd-boot.extraEntries = mkMerge [
+      (mkIf cfg.memtest86.enable {
+        "${cfg.memtest86.entryFilename}" = ''
+          title  MemTest86
+          efi    /efi/memtest86/BOOTX64.efi
+        '';
+      })
+      (mkIf cfg.netbootxyz.enable {
+        "${cfg.netbootxyz.entryFilename}" = ''
+          title  netboot.xyz
+          efi    /efi/netbootxyz/netboot.xyz.efi
+        '';
+      })
+    ];
+
     system = {
       build.installBootLoader = checkedSystemdBootBuilder;
 
diff --git a/nixos/modules/system/boot/modprobe.nix b/nixos/modules/system/boot/modprobe.nix
index c75f32c4d99a..e683d1817297 100644
--- a/nixos/modules/system/boot/modprobe.nix
+++ b/nixos/modules/system/boot/modprobe.nix
@@ -52,6 +52,8 @@ with lib;
       '';
     environment.etc."modprobe.d/debian.conf".source = pkgs.kmod-debian-aliases;
 
+    environment.etc."modprobe.d/systemd.conf".source = "${pkgs.systemd}/lib/modprobe.d/systemd.conf";
+
     environment.systemPackages = [ pkgs.kmod ];
 
     system.activationScripts.modprobe = stringAfter ["specialfs"]
diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix
index 1145831ee2ea..ac1e4ef34b46 100644
--- a/nixos/modules/system/boot/networkd.nix
+++ b/nixos/modules/system/boot/networkd.nix
@@ -513,7 +513,7 @@ let
         (assertValueOneOf "EmitLLDP" (boolValues ++ ["nearest-bridge" "non-tpmr-bridge" "customer-bridge"]))
         (assertValueOneOf "DNSDefaultRoute" boolValues)
         (assertValueOneOf "IPForward" (boolValues ++ ["ipv4" "ipv6"]))
-        (assertValueOneOf "IPMasquerade" boolValues)
+        (assertValueOneOf "IPMasquerade" (boolValues ++ ["ipv4" "ipv6" "both"]))
         (assertValueOneOf "IPv6PrivacyExtensions" (boolValues ++ ["prefer-public" "kernel"]))
         (assertValueOneOf "IPv6AcceptRA" boolValues)
         (assertInt "IPv6DuplicateAddressDetection")
diff --git a/nixos/modules/system/boot/plymouth.nix b/nixos/modules/system/boot/plymouth.nix
index 4b8194d2f85c..78ae8e9d20b7 100644
--- a/nixos/modules/system/boot/plymouth.nix
+++ b/nixos/modules/system/boot/plymouth.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, options, pkgs, ... }:
 
 with lib;
 
@@ -7,6 +7,7 @@ let
   inherit (pkgs) plymouth nixos-icons;
 
   cfg = config.boot.plymouth;
+  opt = options.boot.plymouth;
 
   nixosBreezePlymouth = pkgs.plasma5Packages.breeze-plymouth.override {
     logoFile = cfg.logo;
@@ -71,6 +72,11 @@ in
 
       themePackages = mkOption {
         default = lib.optional (cfg.theme == "breeze") nixosBreezePlymouth;
+        defaultText = literalDocBook ''
+          A NixOS branded variant of the breeze theme when
+          <literal>config.${opt.theme} == "breeze"</literal>, otherwise
+          <literal>[ ]</literal>.
+        '';
         type = types.listOf types.package;
         description = ''
           Extra theme packages for plymouth.
diff --git a/nixos/modules/system/boot/stage-1-init.sh b/nixos/modules/system/boot/stage-1-init.sh
index e8e32bab6e3c..8fcc1f029723 100644
--- a/nixos/modules/system/boot/stage-1-init.sh
+++ b/nixos/modules/system/boot/stage-1-init.sh
@@ -81,30 +81,35 @@ ln -s /proc/mounts /etc/mtab # to shut up mke2fs
 touch /etc/udev/hwdb.bin # to shut up udev
 touch /etc/initrd-release
 
-# Function for waiting a device to appear.
+# Function for waiting for device(s) to appear.
 waitDevice() {
     local device="$1"
+    # Split device string using ':' as a delimiter as bcachefs
+    # uses this for multi-device filesystems, i.e. /dev/sda1:/dev/sda2:/dev/sda3
+    local IFS=':'
 
     # USB storage devices tend to appear with some delay.  It would be
     # great if we had a way to synchronously wait for them, but
     # alas...  So just wait for a few seconds for the device to
     # appear.
-    if test ! -e $device; then
-        echo -n "waiting for device $device to appear..."
-        try=20
-        while [ $try -gt 0 ]; do
-            sleep 1
-            # also re-try lvm activation now that new block devices might have appeared
-            lvm vgchange -ay
-            # and tell udev to create nodes for the new LVs
-            udevadm trigger --action=add
-            if test -e $device; then break; fi
-            echo -n "."
-            try=$((try - 1))
-        done
-        echo
-        [ $try -ne 0 ]
-    fi
+    for dev in $device; do
+        if test ! -e $dev; then
+            echo -n "waiting for device $dev to appear..."
+            try=20
+            while [ $try -gt 0 ]; do
+                sleep 1
+                # also re-try lvm activation now that new block devices might have appeared
+                lvm vgchange -ay
+                # and tell udev to create nodes for the new LVs
+                udevadm trigger --action=add
+                if test -e $dev; then break; fi
+                echo -n "."
+                try=$((try - 1))
+            done
+            echo
+            [ $try -ne 0 ]
+        fi
+    done
 }
 
 # Mount special file systems.
@@ -263,15 +268,6 @@ if test -n "$debug1devices"; then fail; fi
 @postDeviceCommands@
 
 
-# Return true if the machine is on AC power, or if we can't determine
-# whether it's on AC power.
-onACPower() {
-    ! test -d "/proc/acpi/battery" ||
-    ! ls /proc/acpi/battery/BAT[0-9]* > /dev/null 2>&1 ||
-    ! cat /proc/acpi/battery/BAT*/state | grep "^charging state" | grep -q "discharg"
-}
-
-
 # Check the specified file system, if appropriate.
 checkFS() {
     local device="$1"
@@ -286,6 +282,9 @@ checkFS() {
     # Don't check resilient COWs as they validate the fs structures at mount time
     if [ "$fsType" = btrfs -o "$fsType" = zfs -o "$fsType" = bcachefs ]; then return 0; fi
 
+    # Skip fsck for apfs as the fsck utility does not support repairing the filesystem (no -a option)
+    if [ "$fsType" = apfs ]; then return 0; fi
+
     # Skip fsck for nilfs2 - not needed by design and no fsck tool for this filesystem.
     if [ "$fsType" = nilfs2 ]; then return 0; fi
 
@@ -316,13 +315,6 @@ checkFS() {
         return 0
     fi
 
-    # Don't run `fsck' if the machine is on battery power.  !!! Is
-    # this a good idea?
-    if ! onACPower; then
-        echo "on battery power, so no \`fsck' will be performed on \`$device'"
-        return 0
-    fi
-
     echo "checking $device..."
 
     fsckFlags=
diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix
index 409424a5b0f6..8b011d91563f 100644
--- a/nixos/modules/system/boot/stage-1.nix
+++ b/nixos/modules/system/boot/stage-1.nix
@@ -347,6 +347,9 @@ let
             '';
           symlink = "/etc/modprobe.d/ubuntu.conf";
         }
+        { object = config.environment.etc."modprobe.d/nixos.conf".source;
+          symlink = "/etc/modprobe.d/nixos.conf";
+        }
         { object = pkgs.kmod-debian-aliases;
           symlink = "/etc/modprobe.d/debian.conf";
         }
@@ -575,7 +578,7 @@ in
         else "gzip"
       );
       defaultText = literalDocBook "<literal>zstd</literal> if the kernel supports it (5.9+), <literal>gzip</literal> if not";
-      type = types.unspecified; # We don't have a function type...
+      type = types.either types.str (types.functionTo types.str);
       description = ''
         The compressor to use on the initrd image. May be any of:
 
@@ -633,7 +636,7 @@ in
 
           <itemizedlist>
             <listitem><para><literal>boot.consoleLogLevel = 0;</literal></para></listitem>
-            <listitem><para><literal>boot.kernelParams = [ "quiet" "udev.log_priority=3" ];</literal></para></listitem>
+            <listitem><para><literal>boot.kernelParams = [ "quiet" "udev.log_level=3" ];</literal></para></listitem>
           </itemizedlist>
         '';
     };
diff --git a/nixos/modules/system/boot/stage-2-init.sh b/nixos/modules/system/boot/stage-2-init.sh
index afaca2e4158d..a90f58042d2d 100644..100755
--- a/nixos/modules/system/boot/stage-2-init.sh
+++ b/nixos/modules/system/boot/stage-2-init.sh
@@ -172,4 +172,5 @@ echo "starting systemd..."
 
 PATH=/run/current-system/systemd/lib/systemd:@fsPackagesPath@ \
     LOCALE_ARCHIVE=/run/current-system/sw/lib/locale/locale-archive @systemdUnitPathEnvVar@ \
+    TZDIR=/etc/zoneinfo \
     exec @systemdExecutable@
diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix
index ec5dea075bbc..057474c607ac 100644
--- a/nixos/modules/system/boot/systemd.nix
+++ b/nixos/modules/system/boot/systemd.nix
@@ -2,7 +2,6 @@
 
 with utils;
 with systemdUtils.unitOptions;
-with systemdUtils.lib;
 with lib;
 
 let
@@ -11,6 +10,24 @@ let
 
   systemd = cfg.package;
 
+  inherit (systemdUtils.lib)
+    makeUnit
+    generateUnits
+    makeJobScript
+    unitConfig
+    serviceConfig
+    mountConfig
+    automountConfig
+    commonUnitText
+    targetToUnit
+    serviceToUnit
+    socketToUnit
+    timerToUnit
+    pathToUnit
+    mountToUnit
+    automountToUnit
+    sliceToUnit;
+
   upstreamSystemUnits =
     [ # Targets.
       "basic.target"
@@ -25,9 +42,11 @@ let
       "nss-lookup.target"
       "nss-user-lookup.target"
       "time-sync.target"
+    ] ++ (optionals cfg.package.withCryptsetup [
       "cryptsetup.target"
       "cryptsetup-pre.target"
       "remote-cryptsetup.target"
+    ]) ++ [
       "sigpwr.target"
       "timers.target"
       "paths.target"
@@ -61,32 +80,6 @@ let
       "printer.target"
       "smartcard.target"
 
-      # Login stuff.
-      "systemd-logind.service"
-      "autovt@.service"
-      "systemd-user-sessions.service"
-      "dbus-org.freedesktop.import1.service"
-      "dbus-org.freedesktop.machine1.service"
-      "dbus-org.freedesktop.login1.service"
-      "user@.service"
-      "user-runtime-dir@.service"
-
-      # Journal.
-      "systemd-journald.socket"
-      "systemd-journald@.socket"
-      "systemd-journald-varlink@.socket"
-      "systemd-journald.service"
-      "systemd-journald@.service"
-      "systemd-journal-flush.service"
-      "systemd-journal-catalog-update.service"
-      ] ++ (optional (!config.boot.isContainer) "systemd-journald-audit.socket") ++ [
-      "systemd-journald-dev-log.socket"
-      "syslog.socket"
-
-      # Coredumps.
-      "systemd-coredump.socket"
-      "systemd-coredump@.service"
-
       # Kernel module loading.
       "systemd-modules-load.service"
       "kmod-static-nodes.service"
@@ -147,19 +140,12 @@ let
 
       # Slices / containers.
       "slices.target"
-      "user.slice"
       "machine.slice"
       "machines.target"
       "systemd-importd.service"
       "systemd-machined.service"
       "systemd-nspawn@.service"
 
-      # Temporary file creation / cleanup.
-      "systemd-tmpfiles-clean.service"
-      "systemd-tmpfiles-clean.timer"
-      "systemd-tmpfiles-setup.service"
-      "systemd-tmpfiles-setup-dev.service"
-
       # Misc.
       "systemd-sysctl.service"
       "dbus-org.freedesktop.timedate1.service"
@@ -170,9 +156,6 @@ let
       "systemd-hostnamed.service"
       "systemd-exit.service"
       "systemd-update-done.service"
-    ] ++ optionals config.services.journald.enableHttpGateway [
-      "systemd-journal-gatewayd.socket"
-      "systemd-journal-gatewayd.service"
     ] ++ cfg.additionalUpstreamSystemUnits;
 
   upstreamSystemWants =
@@ -183,241 +166,6 @@ let
       "timers.target.wants"
     ];
 
-    upstreamUserUnits = [
-      "app.slice"
-      "background.slice"
-      "basic.target"
-      "bluetooth.target"
-      "default.target"
-      "exit.target"
-      "graphical-session-pre.target"
-      "graphical-session.target"
-      "paths.target"
-      "printer.target"
-      "session.slice"
-      "shutdown.target"
-      "smartcard.target"
-      "sockets.target"
-      "sound.target"
-      "systemd-exit.service"
-      "systemd-tmpfiles-clean.service"
-      "systemd-tmpfiles-clean.timer"
-      "systemd-tmpfiles-setup.service"
-      "timers.target"
-      "xdg-desktop-autostart.target"
-    ];
-
-  makeJobScript = name: text:
-    let
-      scriptName = replaceChars [ "\\" "@" ] [ "-" "_" ] (shellEscape name);
-      out = pkgs.writeTextFile {
-        # The derivation name is different from the script file name
-        # to keep the script file name short to avoid cluttering logs.
-        name = "unit-script-${scriptName}";
-        executable = true;
-        destination = "/bin/${scriptName}";
-        text = ''
-          #!${pkgs.runtimeShell} -e
-          ${text}
-        '';
-        checkPhase = ''
-          ${pkgs.stdenv.shell} -n "$out/bin/${scriptName}"
-        '';
-      };
-    in "${out}/bin/${scriptName}";
-
-  unitConfig = { config, options, ... }: {
-    config = {
-      unitConfig =
-        optionalAttrs (config.requires != [])
-          { Requires = toString config.requires; }
-        // optionalAttrs (config.wants != [])
-          { Wants = toString config.wants; }
-        // optionalAttrs (config.after != [])
-          { After = toString config.after; }
-        // optionalAttrs (config.before != [])
-          { Before = toString config.before; }
-        // optionalAttrs (config.bindsTo != [])
-          { BindsTo = toString config.bindsTo; }
-        // optionalAttrs (config.partOf != [])
-          { PartOf = toString config.partOf; }
-        // optionalAttrs (config.conflicts != [])
-          { Conflicts = toString config.conflicts; }
-        // optionalAttrs (config.requisite != [])
-          { Requisite = toString config.requisite; }
-        // optionalAttrs (config.restartTriggers != [])
-          { X-Restart-Triggers = toString config.restartTriggers; }
-        // optionalAttrs (config.description != "") {
-          Description = config.description; }
-        // optionalAttrs (config.documentation != []) {
-          Documentation = toString config.documentation; }
-        // optionalAttrs (config.onFailure != []) {
-          OnFailure = toString config.onFailure; }
-        // optionalAttrs (options.startLimitIntervalSec.isDefined) {
-          StartLimitIntervalSec = toString config.startLimitIntervalSec;
-        } // optionalAttrs (options.startLimitBurst.isDefined) {
-          StartLimitBurst = toString config.startLimitBurst;
-        };
-    };
-  };
-
-  serviceConfig = { name, config, ... }: {
-    config = mkMerge
-      [ { # Default path for systemd services.  Should be quite minimal.
-          path = mkAfter
-            [ pkgs.coreutils
-              pkgs.findutils
-              pkgs.gnugrep
-              pkgs.gnused
-              systemd
-            ];
-          environment.PATH = "${makeBinPath config.path}:${makeSearchPathOutput "bin" "sbin" config.path}";
-        }
-        (mkIf (config.preStart != "")
-          { serviceConfig.ExecStartPre =
-              [ (makeJobScript "${name}-pre-start" config.preStart) ];
-          })
-        (mkIf (config.script != "")
-          { serviceConfig.ExecStart =
-              makeJobScript "${name}-start" config.script + " " + config.scriptArgs;
-          })
-        (mkIf (config.postStart != "")
-          { serviceConfig.ExecStartPost =
-              [ (makeJobScript "${name}-post-start" config.postStart) ];
-          })
-        (mkIf (config.reload != "")
-          { serviceConfig.ExecReload =
-              makeJobScript "${name}-reload" config.reload;
-          })
-        (mkIf (config.preStop != "")
-          { serviceConfig.ExecStop =
-              makeJobScript "${name}-pre-stop" config.preStop;
-          })
-        (mkIf (config.postStop != "")
-          { serviceConfig.ExecStopPost =
-              makeJobScript "${name}-post-stop" config.postStop;
-          })
-      ];
-  };
-
-  mountConfig = { config, ... }: {
-    config = {
-      mountConfig =
-        { What = config.what;
-          Where = config.where;
-        } // optionalAttrs (config.type != "") {
-          Type = config.type;
-        } // optionalAttrs (config.options != "") {
-          Options = config.options;
-        };
-    };
-  };
-
-  automountConfig = { config, ... }: {
-    config = {
-      automountConfig =
-        { Where = config.where;
-        };
-    };
-  };
-
-  commonUnitText = def: ''
-      [Unit]
-      ${attrsToSection def.unitConfig}
-    '';
-
-  targetToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text =
-        ''
-          [Unit]
-          ${attrsToSection def.unitConfig}
-        '';
-    };
-
-  serviceToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text = commonUnitText def +
-        ''
-          [Service]
-          ${let env = cfg.globalEnvironment // def.environment;
-            in concatMapStrings (n:
-              let s = optionalString (env.${n} != null)
-                "Environment=${builtins.toJSON "${n}=${env.${n}}"}\n";
-              # systemd max line length is now 1MiB
-              # https://github.com/systemd/systemd/commit/e6dde451a51dc5aaa7f4d98d39b8fe735f73d2af
-              in if stringLength s >= 1048576 then throw "The value of the environment variable ‘${n}’ in systemd service ‘${name}.service’ is too long." else s) (attrNames env)}
-          ${if def.reloadIfChanged then ''
-            X-ReloadIfChanged=true
-          '' else if !def.restartIfChanged then ''
-            X-RestartIfChanged=false
-          '' else ""}
-          ${optionalString (!def.stopIfChanged) "X-StopIfChanged=false"}
-          ${attrsToSection def.serviceConfig}
-        '';
-    };
-
-  socketToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text = commonUnitText def +
-        ''
-          [Socket]
-          ${attrsToSection def.socketConfig}
-          ${concatStringsSep "\n" (map (s: "ListenStream=${s}") def.listenStreams)}
-          ${concatStringsSep "\n" (map (s: "ListenDatagram=${s}") def.listenDatagrams)}
-        '';
-    };
-
-  timerToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text = commonUnitText def +
-        ''
-          [Timer]
-          ${attrsToSection def.timerConfig}
-        '';
-    };
-
-  pathToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text = commonUnitText def +
-        ''
-          [Path]
-          ${attrsToSection def.pathConfig}
-        '';
-    };
-
-  mountToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text = commonUnitText def +
-        ''
-          [Mount]
-          ${attrsToSection def.mountConfig}
-        '';
-    };
-
-  automountToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text = commonUnitText def +
-        ''
-          [Automount]
-          ${attrsToSection def.automountConfig}
-        '';
-    };
-
-  sliceToUnit = name: def:
-    { inherit (def) aliases wantedBy requiredBy enable;
-      text = commonUnitText def +
-        ''
-          [Slice]
-          ${attrsToSection def.sliceConfig}
-        '';
-    };
-
-  logindHandlerType = types.enum [
-    "ignore" "poweroff" "reboot" "halt" "kexec" "suspend"
-    "hibernate" "hybrid-sleep" "suspend-then-hibernate" "lock"
-  ];
-
   proxy_env = config.networking.proxy.envVars;
 
 in
@@ -570,26 +318,6 @@ in
       '';
     };
 
-    systemd.coredump.enable = mkOption {
-      default = true;
-      type = types.bool;
-      description = ''
-        Whether core dumps should be processed by
-        <command>systemd-coredump</command>. If disabled, core dumps
-        appear in the current directory of the crashing process.
-      '';
-    };
-
-    systemd.coredump.extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      example = "Storage=journal";
-      description = ''
-        Extra config options for systemd-coredump. See coredump.conf(5) man page
-        for available options.
-      '';
-    };
-
     systemd.extraConfig = mkOption {
       default = "";
       type = types.lines;
@@ -600,142 +328,6 @@ in
       '';
     };
 
-    services.journald.console = mkOption {
-      default = "";
-      type = types.str;
-      description = "If non-empty, write log messages to the specified TTY device.";
-    };
-
-    services.journald.rateLimitInterval = mkOption {
-      default = "30s";
-      type = types.str;
-      description = ''
-        Configures the rate limiting interval that is applied to all
-        messages generated on the system. This rate limiting is applied
-        per-service, so that two services which log do not interfere with
-        each other's limit. The value may be specified in the following
-        units: s, min, h, ms, us. To turn off any kind of rate limiting,
-        set either value to 0.
-
-        See <option>services.journald.rateLimitBurst</option> for important
-        considerations when setting this value.
-      '';
-    };
-
-    services.journald.rateLimitBurst = mkOption {
-      default = 10000;
-      type = types.int;
-      description = ''
-        Configures the rate limiting burst limit (number of messages per
-        interval) that is applied to all messages generated on the system.
-        This rate limiting is applied per-service, so that two services
-        which log do not interfere with each other's limit.
-
-        Note that the effective rate limit is multiplied by a factor derived
-        from the available free disk space for the journal as described on
-        <link xlink:href="https://www.freedesktop.org/software/systemd/man/journald.conf.html">
-        journald.conf(5)</link>.
-
-        Note that the total amount of logs stored is limited by journald settings
-        such as <literal>SystemMaxUse</literal>, which defaults to a 4 GB cap.
-
-        It is thus recommended to compute what period of time that you will be
-        able to store logs for when an application logs at full burst rate.
-        With default settings for log lines that are 100 Bytes long, this can
-        amount to just a few hours.
-      '';
-    };
-
-    services.journald.extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      example = "Storage=volatile";
-      description = ''
-        Extra config options for systemd-journald. See man journald.conf
-        for available options.
-      '';
-    };
-
-    services.journald.enableHttpGateway = mkOption {
-      default = false;
-      type = types.bool;
-      description = ''
-        Whether to enable the HTTP gateway to the journal.
-      '';
-    };
-
-    services.journald.forwardToSyslog = mkOption {
-      default = config.services.rsyslogd.enable || config.services.syslog-ng.enable;
-      defaultText = literalExpression "services.rsyslogd.enable || services.syslog-ng.enable";
-      type = types.bool;
-      description = ''
-        Whether to forward log messages to syslog.
-      '';
-    };
-
-    services.logind.extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      example = "IdleAction=lock";
-      description = ''
-        Extra config options for systemd-logind. See
-        <link xlink:href="https://www.freedesktop.org/software/systemd/man/logind.conf.html">
-        logind.conf(5)</link> for available options.
-      '';
-    };
-
-    services.logind.killUserProcesses = mkOption {
-      default = false;
-      type = types.bool;
-      description = ''
-        Specifies whether the processes of a user should be killed
-        when the user logs out.  If true, the scope unit corresponding
-        to the session and all processes inside that scope will be
-        terminated.  If false, the scope is "abandoned" (see
-        <link xlink:href="https://www.freedesktop.org/software/systemd/man/systemd.scope.html#">
-        systemd.scope(5)</link>), and processes are not killed.
-        </para>
-
-        <para>
-        See <link xlink:href="https://www.freedesktop.org/software/systemd/man/logind.conf.html#KillUserProcesses=">logind.conf(5)</link>
-        for more details.
-      '';
-    };
-
-    services.logind.lidSwitch = mkOption {
-      default = "suspend";
-      example = "ignore";
-      type = logindHandlerType;
-
-      description = ''
-        Specifies what to be done when the laptop lid is closed.
-      '';
-    };
-
-    services.logind.lidSwitchDocked = mkOption {
-      default = "ignore";
-      example = "suspend";
-      type = logindHandlerType;
-
-      description = ''
-        Specifies what to be done when the laptop lid is closed
-        and another screen is added.
-      '';
-    };
-
-    services.logind.lidSwitchExternalPower = mkOption {
-      default = config.services.logind.lidSwitch;
-      defaultText = literalExpression "services.logind.lidSwitch";
-      example = "ignore";
-      type = logindHandlerType;
-
-      description = ''
-        Specifies what to do when the laptop lid is closed and the system is
-        on external power. By default use the same action as specified in
-        services.logind.lidSwitch.
-      '';
-    };
-
     systemd.sleep.extraConfig = mkOption {
       default = "";
       type = types.lines;
@@ -746,95 +338,6 @@ in
       '';
     };
 
-    systemd.user.extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      example = "DefaultCPUAccounting=yes";
-      description = ''
-        Extra config options for systemd user instances. See man systemd-user.conf for
-        available options.
-      '';
-    };
-
-    systemd.tmpfiles.rules = mkOption {
-      type = types.listOf types.str;
-      default = [];
-      example = [ "d /tmp 1777 root root 10d" ];
-      description = ''
-        Rules for creation, deletion and cleaning of volatile and temporary files
-        automatically. See
-        <citerefentry><refentrytitle>tmpfiles.d</refentrytitle><manvolnum>5</manvolnum></citerefentry>
-        for the exact format.
-      '';
-    };
-
-    systemd.tmpfiles.packages = mkOption {
-      type = types.listOf types.package;
-      default = [];
-      example = literalExpression "[ pkgs.lvm2 ]";
-      apply = map getLib;
-      description = ''
-        List of packages containing <command>systemd-tmpfiles</command> rules.
-
-        All files ending in .conf found in
-        <filename><replaceable>pkg</replaceable>/lib/tmpfiles.d</filename>
-        will be included.
-        If this folder does not exist or does not contain any files an error will be returned instead.
-
-        If a <filename>lib</filename> output is available, rules are searched there and only there.
-        If there is no <filename>lib</filename> output it will fall back to <filename>out</filename>
-        and if that does not exist either, the default output will be used.
-      '';
-    };
-
-    systemd.user.units = mkOption {
-      description = "Definition of systemd per-user units.";
-      default = {};
-      type = with types; attrsOf (submodule (
-        { name, config, ... }:
-        { options = concreteUnitOptions;
-          config = {
-            unit = mkDefault (makeUnit name config);
-          };
-        }));
-    };
-
-    systemd.user.paths = mkOption {
-      default = {};
-      type = with types; attrsOf (submodule [ { options = pathOptions; } unitConfig ]);
-      description = "Definition of systemd per-user path units.";
-    };
-
-    systemd.user.services = mkOption {
-      default = {};
-      type = with types; attrsOf (submodule [ { options = serviceOptions; } unitConfig serviceConfig ] );
-      description = "Definition of systemd per-user service units.";
-    };
-
-    systemd.user.slices = mkOption {
-      default = {};
-      type = with types; attrsOf (submodule [ { options = sliceOptions; } unitConfig ] );
-      description = "Definition of systemd per-user slice units.";
-    };
-
-    systemd.user.sockets = mkOption {
-      default = {};
-      type = with types; attrsOf (submodule [ { options = socketOptions; } unitConfig ] );
-      description = "Definition of systemd per-user socket units.";
-    };
-
-    systemd.user.targets = mkOption {
-      default = {};
-      type = with types; attrsOf (submodule [ { options = targetOptions; } unitConfig] );
-      description = "Definition of systemd per-user target units.";
-    };
-
-    systemd.user.timers = mkOption {
-      default = {};
-      type = with types; attrsOf (submodule [ { options = timerOptions; } unitConfig ] );
-      description = "Definition of systemd per-user timer units.";
-    };
-
     systemd.additionalUpstreamSystemUnits = mkOption {
       default = [ ];
       type = types.listOf types.str;
@@ -921,6 +424,9 @@ in
               (optional hasDeprecated
                 "Service '${name}.service' uses the attribute 'StartLimitInterval' in the Service section, which is deprecated. See https://github.com/NixOS/nixpkgs/issues/45786."
               )
+              (optional (service.reloadIfChanged && service.reloadTriggers != [])
+                "Service '${name}.service' has both 'reloadIfChanged' and 'reloadTriggers' set. This is probably not what you want, because 'reloadTriggers' behave the same whay as 'restartTriggers' if 'reloadIfChanged' is set."
+              )
             ]
         )
         cfg.services
@@ -967,8 +473,6 @@ in
     in ({
       "systemd/system".source = generateUnits "system" enabledUnits enabledUpstreamSystemUnits upstreamSystemWants;
 
-      "systemd/user".source = generateUnits "user" cfg.user.units upstreamUserUnits [];
-
       "systemd/system.conf".text = ''
         [Manager]
         ${optionalString config.systemd.enableCgroupAccounting ''
@@ -994,76 +498,17 @@ in
         ${config.systemd.extraConfig}
       '';
 
-      "systemd/user.conf".text = ''
-        [Manager]
-        ${config.systemd.user.extraConfig}
-      '';
-
-      "systemd/journald.conf".text = ''
-        [Journal]
-        Storage=persistent
-        RateLimitInterval=${config.services.journald.rateLimitInterval}
-        RateLimitBurst=${toString config.services.journald.rateLimitBurst}
-        ${optionalString (config.services.journald.console != "") ''
-          ForwardToConsole=yes
-          TTYPath=${config.services.journald.console}
-        ''}
-        ${optionalString (config.services.journald.forwardToSyslog) ''
-          ForwardToSyslog=yes
-        ''}
-        ${config.services.journald.extraConfig}
-      '';
-
-      "systemd/coredump.conf".text =
-        ''
-          [Coredump]
-          ${config.systemd.coredump.extraConfig}
-        '';
-
-      "systemd/logind.conf".text = ''
-        [Login]
-        KillUserProcesses=${if config.services.logind.killUserProcesses then "yes" else "no"}
-        HandleLidSwitch=${config.services.logind.lidSwitch}
-        HandleLidSwitchDocked=${config.services.logind.lidSwitchDocked}
-        HandleLidSwitchExternalPower=${config.services.logind.lidSwitchExternalPower}
-        ${config.services.logind.extraConfig}
-      '';
-
       "systemd/sleep.conf".text = ''
         [Sleep]
         ${config.systemd.sleep.extraConfig}
       '';
 
-      # install provided sysctl snippets
-      "sysctl.d/50-coredump.conf".source = "${systemd}/example/sysctl.d/50-coredump.conf";
-      "sysctl.d/50-default.conf".source = "${systemd}/example/sysctl.d/50-default.conf";
-
-      "tmpfiles.d".source = (pkgs.symlinkJoin {
-        name = "tmpfiles.d";
-        paths = map (p: p + "/lib/tmpfiles.d") cfg.tmpfiles.packages;
-        postBuild = ''
-          for i in $(cat $pathsPath); do
-            (test -d "$i" && test $(ls "$i"/*.conf | wc -l) -ge 1) || (
-              echo "ERROR: The path '$i' from systemd.tmpfiles.packages contains no *.conf files."
-              exit 1
-            )
-          done
-        '' + concatMapStrings (name: optionalString (hasPrefix "tmpfiles.d/" name) ''
-          rm -f $out/${removePrefix "tmpfiles.d/" name}
-        '') config.system.build.etc.passthru.targets;
-      }) + "/*";
-
       "systemd/system-generators" = { source = hooks "generators" cfg.generators; };
       "systemd/system-shutdown" = { source = hooks "shutdown" cfg.shutdown; };
     });
 
     services.dbus.enable = true;
 
-    users.users.systemd-coredump = {
-      uid = config.ids.uids.systemd-coredump;
-      group = "systemd-coredump";
-    };
-    users.groups.systemd-coredump = {};
     users.users.systemd-network = {
       uid = config.ids.uids.systemd-network;
       group = "systemd-network";
@@ -1083,36 +528,6 @@ in
         unitConfig.X-StopOnReconfiguration = true;
       };
 
-    systemd.tmpfiles.packages = [
-      # Default tmpfiles rules provided by systemd
-      (pkgs.runCommand "systemd-default-tmpfiles" {} ''
-        mkdir -p $out/lib/tmpfiles.d
-        cd $out/lib/tmpfiles.d
-
-        ln -s "${systemd}/example/tmpfiles.d/home.conf"
-        ln -s "${systemd}/example/tmpfiles.d/journal-nocow.conf"
-        ln -s "${systemd}/example/tmpfiles.d/static-nodes-permissions.conf"
-        ln -s "${systemd}/example/tmpfiles.d/systemd.conf"
-        ln -s "${systemd}/example/tmpfiles.d/systemd-nologin.conf"
-        ln -s "${systemd}/example/tmpfiles.d/systemd-nspawn.conf"
-        ln -s "${systemd}/example/tmpfiles.d/systemd-tmp.conf"
-        ln -s "${systemd}/example/tmpfiles.d/tmp.conf"
-        ln -s "${systemd}/example/tmpfiles.d/var.conf"
-        ln -s "${systemd}/example/tmpfiles.d/x11.conf"
-      '')
-      # User-specified tmpfiles rules
-      (pkgs.writeTextFile {
-        name = "nixos-tmpfiles.d";
-        destination = "/lib/tmpfiles.d/00-nixos.conf";
-        text = ''
-          # This file is created automatically and should not be modified.
-          # Please change the option ‘systemd.tmpfiles.rules’ instead.
-
-          ${concatStringsSep "\n" cfg.tmpfiles.rules}
-        '';
-      })
-    ];
-
     systemd.units =
          mapAttrs' (n: v: nameValuePair "${n}.path"    (pathToUnit    n v)) cfg.paths
       // mapAttrs' (n: v: nameValuePair "${n}.service" (serviceToUnit n v)) cfg.services
@@ -1127,14 +542,6 @@ in
                    (v: let n = escapeSystemdPath v.where;
                        in nameValuePair "${n}.automount" (automountToUnit n v)) cfg.automounts);
 
-    systemd.user.units =
-         mapAttrs' (n: v: nameValuePair "${n}.path"    (pathToUnit    n v)) cfg.user.paths
-      // mapAttrs' (n: v: nameValuePair "${n}.service" (serviceToUnit n v)) cfg.user.services
-      // mapAttrs' (n: v: nameValuePair "${n}.slice"   (sliceToUnit   n v)) cfg.user.slices
-      // mapAttrs' (n: v: nameValuePair "${n}.socket"  (socketToUnit  n v)) cfg.user.sockets
-      // mapAttrs' (n: v: nameValuePair "${n}.target"  (targetToUnit  n v)) cfg.user.targets
-      // mapAttrs' (n: v: nameValuePair "${n}.timer"   (timerToUnit   n v)) cfg.user.timers;
-
     system.requiredKernelConfig = map config.lib.kernelConfig.isEnabled
       [ "DEVTMPFS" "CGROUPS" "INOTIFY_USER" "SIGNALFD" "TIMERFD" "EPOLL" "NET"
         "SYSFS" "PROC_FS" "FHANDLE" "CRYPTO_USER_API_HASH" "CRYPTO_HMAC"
@@ -1142,11 +549,6 @@ in
         "TMPFS_XATTR" "SECCOMP"
       ];
 
-    users.groups.systemd-journal.gid = config.ids.gids.systemd-journal;
-    users.users.systemd-journal-gateway.uid = config.ids.uids.systemd-journal-gateway;
-    users.users.systemd-journal-gateway.group = "systemd-journal-gateway";
-    users.groups.systemd-journal-gateway.gid = config.ids.gids.systemd-journal-gateway;
-
     # Generate timer units for all services that have a ‘startAt’ value.
     systemd.timers =
       mapAttrs (name: service:
@@ -1163,42 +565,14 @@ in
         })
         (filterAttrs (name: service: service.startAt != []) cfg.user.services);
 
-    systemd.sockets.systemd-journal-gatewayd.wantedBy =
-      optional config.services.journald.enableHttpGateway "sockets.target";
-
-    # Provide the systemd-user PAM service, required to run systemd
-    # user instances.
-    security.pam.services.systemd-user =
-      { # Ensure that pam_systemd gets included. This is special-cased
-        # in systemd to provide XDG_RUNTIME_DIR.
-        startSession = true;
-      };
-
     # Some overrides to upstream units.
     systemd.services."systemd-backlight@".restartIfChanged = false;
     systemd.services."systemd-fsck@".restartIfChanged = false;
     systemd.services."systemd-fsck@".path = [ config.system.path ];
-    systemd.services."user@".restartIfChanged = false;
-    systemd.services.systemd-journal-flush.restartIfChanged = false;
     systemd.services.systemd-random-seed.restartIfChanged = false;
     systemd.services.systemd-remount-fs.restartIfChanged = false;
     systemd.services.systemd-update-utmp.restartIfChanged = false;
-    systemd.services.systemd-user-sessions.restartIfChanged = false; # Restart kills all active sessions.
     systemd.services.systemd-udev-settle.restartIfChanged = false; # Causes long delays in nixos-rebuild
-    # Restarting systemd-logind breaks X11
-    # - upstream commit: https://cgit.freedesktop.org/xorg/xserver/commit/?id=dc48bd653c7e101
-    # - systemd announcement: https://github.com/systemd/systemd/blob/22043e4317ecd2bc7834b48a6d364de76bb26d91/NEWS#L103-L112
-    # - this might be addressed in the future by xorg
-    #systemd.services.systemd-logind.restartTriggers = [ config.environment.etc."systemd/logind.conf".source ];
-    systemd.services.systemd-logind.restartIfChanged = false;
-    systemd.services.systemd-logind.stopIfChanged = false;
-    # The user-runtime-dir@ service is managed by systemd-logind we should not touch it or else we break the users' sessions.
-    systemd.services."user-runtime-dir@".stopIfChanged = false;
-    systemd.services."user-runtime-dir@".restartIfChanged = false;
-    systemd.services.systemd-journald.restartTriggers = [ config.environment.etc."systemd/journald.conf".source ];
-    systemd.services.systemd-journald.stopIfChanged = false;
-    systemd.services."systemd-journald@".restartTriggers = [ config.environment.etc."systemd/journald.conf".source ];
-    systemd.services."systemd-journald@".stopIfChanged = false;
     systemd.targets.local-fs.unitConfig.X-StopOnReconfiguration = true;
     systemd.targets.remote-fs.unitConfig.X-StopOnReconfiguration = true;
     systemd.targets.network-online.wantedBy = [ "multi-user.target" ];
@@ -1209,13 +583,30 @@ in
     systemd.services.systemd-remount-fs.unitConfig.ConditionVirtualization = "!container";
     systemd.services.systemd-random-seed.unitConfig.ConditionVirtualization = "!container";
 
-    boot.kernel.sysctl."kernel.core_pattern" = mkIf (!cfg.coredump.enable) "core";
-
     # Increase numeric PID range (set directly instead of copying a one-line file from systemd)
     # https://github.com/systemd/systemd/pull/12226
     boot.kernel.sysctl."kernel.pid_max" = mkIf pkgs.stdenv.is64bit (lib.mkDefault 4194304);
 
     boot.kernelParams = optional (!cfg.enableUnifiedCgroupHierarchy) "systemd.unified_cgroup_hierarchy=0";
+
+    services.logrotate.paths = {
+      "/var/log/btmp" = mapAttrs (_: mkDefault) {
+        frequency = "monthly";
+        keep = 1;
+        extraConfig = ''
+          create 0660 root ${config.users.groups.utmp.name}
+          minsize 1M
+        '';
+      };
+      "/var/log/wtmp" = mapAttrs (_: mkDefault) {
+        frequency = "monthly";
+        keep = 1;
+        extraConfig = ''
+          create 0664 root ${config.users.groups.utmp.name}
+          minsize 1M
+        '';
+      };
+    };
   };
 
   # FIXME: Remove these eventually.
diff --git a/nixos/modules/system/boot/systemd/coredump.nix b/nixos/modules/system/boot/systemd/coredump.nix
new file mode 100644
index 000000000000..b6ee2cff1f9a
--- /dev/null
+++ b/nixos/modules/system/boot/systemd/coredump.nix
@@ -0,0 +1,57 @@
+{ config, lib, pkgs, utils, ... }:
+
+with lib;
+
+let
+  cfg = config.systemd.coredump;
+  systemd = config.systemd.package;
+in {
+  options = {
+    systemd.coredump.enable = mkOption {
+      default = true;
+      type = types.bool;
+      description = ''
+        Whether core dumps should be processed by
+        <command>systemd-coredump</command>. If disabled, core dumps
+        appear in the current directory of the crashing process.
+      '';
+    };
+
+    systemd.coredump.extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = "Storage=journal";
+      description = ''
+        Extra config options for systemd-coredump. See coredump.conf(5) man page
+        for available options.
+      '';
+    };
+  };
+
+  config = {
+    systemd.additionalUpstreamSystemUnits = [
+      "systemd-coredump.socket"
+      "systemd-coredump@.service"
+    ];
+
+    environment.etc = {
+      "systemd/coredump.conf".text =
+      ''
+        [Coredump]
+        ${cfg.extraConfig}
+      '';
+
+      # install provided sysctl snippets
+      "sysctl.d/50-coredump.conf".source = "${systemd}/example/sysctl.d/50-coredump.conf";
+      "sysctl.d/50-default.conf".source = "${systemd}/example/sysctl.d/50-default.conf";
+    };
+
+    users.users.systemd-coredump = {
+      uid = config.ids.uids.systemd-coredump;
+      group = "systemd-coredump";
+    };
+    users.groups.systemd-coredump = {};
+
+    boot.kernel.sysctl."kernel.core_pattern" = mkIf (!cfg.enable) "core";
+  };
+}
diff --git a/nixos/modules/system/boot/systemd/journald.nix b/nixos/modules/system/boot/systemd/journald.nix
new file mode 100644
index 000000000000..7e14c8ae4077
--- /dev/null
+++ b/nixos/modules/system/boot/systemd/journald.nix
@@ -0,0 +1,131 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.journald;
+in {
+  options = {
+    services.journald.console = mkOption {
+      default = "";
+      type = types.str;
+      description = "If non-empty, write log messages to the specified TTY device.";
+    };
+
+    services.journald.rateLimitInterval = mkOption {
+      default = "30s";
+      type = types.str;
+      description = ''
+        Configures the rate limiting interval that is applied to all
+        messages generated on the system. This rate limiting is applied
+        per-service, so that two services which log do not interfere with
+        each other's limit. The value may be specified in the following
+        units: s, min, h, ms, us. To turn off any kind of rate limiting,
+        set either value to 0.
+
+        See <option>services.journald.rateLimitBurst</option> for important
+        considerations when setting this value.
+      '';
+    };
+
+    services.journald.rateLimitBurst = mkOption {
+      default = 10000;
+      type = types.int;
+      description = ''
+        Configures the rate limiting burst limit (number of messages per
+        interval) that is applied to all messages generated on the system.
+        This rate limiting is applied per-service, so that two services
+        which log do not interfere with each other's limit.
+
+        Note that the effective rate limit is multiplied by a factor derived
+        from the available free disk space for the journal as described on
+        <link xlink:href="https://www.freedesktop.org/software/systemd/man/journald.conf.html">
+        journald.conf(5)</link>.
+
+        Note that the total amount of logs stored is limited by journald settings
+        such as <literal>SystemMaxUse</literal>, which defaults to a 4 GB cap.
+
+        It is thus recommended to compute what period of time that you will be
+        able to store logs for when an application logs at full burst rate.
+        With default settings for log lines that are 100 Bytes long, this can
+        amount to just a few hours.
+      '';
+    };
+
+    services.journald.extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = "Storage=volatile";
+      description = ''
+        Extra config options for systemd-journald. See man journald.conf
+        for available options.
+      '';
+    };
+
+    services.journald.enableHttpGateway = mkOption {
+      default = false;
+      type = types.bool;
+      description = ''
+        Whether to enable the HTTP gateway to the journal.
+      '';
+    };
+
+    services.journald.forwardToSyslog = mkOption {
+      default = config.services.rsyslogd.enable || config.services.syslog-ng.enable;
+      defaultText = literalExpression "services.rsyslogd.enable || services.syslog-ng.enable";
+      type = types.bool;
+      description = ''
+        Whether to forward log messages to syslog.
+      '';
+    };
+  };
+
+  config = {
+    systemd.additionalUpstreamSystemUnits = [
+      "systemd-journald.socket"
+      "systemd-journald@.socket"
+      "systemd-journald-varlink@.socket"
+      "systemd-journald.service"
+      "systemd-journald@.service"
+      "systemd-journal-flush.service"
+      "systemd-journal-catalog-update.service"
+      ] ++ (optional (!config.boot.isContainer) "systemd-journald-audit.socket") ++ [
+      "systemd-journald-dev-log.socket"
+      "syslog.socket"
+      ] ++ optionals cfg.enableHttpGateway [
+      "systemd-journal-gatewayd.socket"
+      "systemd-journal-gatewayd.service"
+      ];
+
+    environment.etc = {
+      "systemd/journald.conf".text = ''
+        [Journal]
+        Storage=persistent
+        RateLimitInterval=${cfg.rateLimitInterval}
+        RateLimitBurst=${toString cfg.rateLimitBurst}
+        ${optionalString (cfg.console != "") ''
+          ForwardToConsole=yes
+          TTYPath=${cfg.console}
+        ''}
+        ${optionalString (cfg.forwardToSyslog) ''
+          ForwardToSyslog=yes
+        ''}
+        ${cfg.extraConfig}
+      '';
+    };
+
+    users.groups.systemd-journal.gid = config.ids.gids.systemd-journal;
+    users.users.systemd-journal-gateway.uid = config.ids.uids.systemd-journal-gateway;
+    users.users.systemd-journal-gateway.group = "systemd-journal-gateway";
+    users.groups.systemd-journal-gateway.gid = config.ids.gids.systemd-journal-gateway;
+
+    systemd.sockets.systemd-journal-gatewayd.wantedBy =
+      optional cfg.enableHttpGateway "sockets.target";
+
+    systemd.services.systemd-journal-flush.restartIfChanged = false;
+    systemd.services.systemd-journald.restartTriggers = [ config.environment.etc."systemd/journald.conf".source ];
+    systemd.services.systemd-journald.stopIfChanged = false;
+    systemd.services."systemd-journald@".restartTriggers = [ config.environment.etc."systemd/journald.conf".source ];
+    systemd.services."systemd-journald@".stopIfChanged = false;
+  };
+}
diff --git a/nixos/modules/system/boot/systemd/logind.nix b/nixos/modules/system/boot/systemd/logind.nix
new file mode 100644
index 000000000000..c1e6cfe61d04
--- /dev/null
+++ b/nixos/modules/system/boot/systemd/logind.nix
@@ -0,0 +1,114 @@
+{ config, lib, pkgs, utils, ... }:
+
+with lib;
+
+let
+  cfg = config.services.logind;
+
+  logindHandlerType = types.enum [
+    "ignore" "poweroff" "reboot" "halt" "kexec" "suspend"
+    "hibernate" "hybrid-sleep" "suspend-then-hibernate" "lock"
+  ];
+in
+{
+  options = {
+    services.logind.extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = "IdleAction=lock";
+      description = ''
+        Extra config options for systemd-logind. See
+        <link xlink:href="https://www.freedesktop.org/software/systemd/man/logind.conf.html">
+        logind.conf(5)</link> for available options.
+      '';
+    };
+
+    services.logind.killUserProcesses = mkOption {
+      default = false;
+      type = types.bool;
+      description = ''
+        Specifies whether the processes of a user should be killed
+        when the user logs out.  If true, the scope unit corresponding
+        to the session and all processes inside that scope will be
+        terminated.  If false, the scope is "abandoned" (see
+        <link xlink:href="https://www.freedesktop.org/software/systemd/man/systemd.scope.html#">
+        systemd.scope(5)</link>), and processes are not killed.
+        </para>
+
+        <para>
+        See <link xlink:href="https://www.freedesktop.org/software/systemd/man/logind.conf.html#KillUserProcesses=">logind.conf(5)</link>
+        for more details.
+      '';
+    };
+
+    services.logind.lidSwitch = mkOption {
+      default = "suspend";
+      example = "ignore";
+      type = logindHandlerType;
+
+      description = ''
+        Specifies what to be done when the laptop lid is closed.
+      '';
+    };
+
+    services.logind.lidSwitchDocked = mkOption {
+      default = "ignore";
+      example = "suspend";
+      type = logindHandlerType;
+
+      description = ''
+        Specifies what to be done when the laptop lid is closed
+        and another screen is added.
+      '';
+    };
+
+    services.logind.lidSwitchExternalPower = mkOption {
+      default = cfg.lidSwitch;
+      defaultText = literalExpression "services.logind.lidSwitch";
+      example = "ignore";
+      type = logindHandlerType;
+
+      description = ''
+        Specifies what to do when the laptop lid is closed and the system is
+        on external power. By default use the same action as specified in
+        services.logind.lidSwitch.
+      '';
+    };
+  };
+
+  config = {
+    systemd.additionalUpstreamSystemUnits = [
+      "systemd-logind.service"
+      "autovt@.service"
+      "systemd-user-sessions.service"
+      "dbus-org.freedesktop.import1.service"
+      "dbus-org.freedesktop.machine1.service"
+      "dbus-org.freedesktop.login1.service"
+      "user@.service"
+      "user-runtime-dir@.service"
+    ];
+
+    environment.etc = {
+      "systemd/logind.conf".text = ''
+        [Login]
+        KillUserProcesses=${if cfg.killUserProcesses then "yes" else "no"}
+        HandleLidSwitch=${cfg.lidSwitch}
+        HandleLidSwitchDocked=${cfg.lidSwitchDocked}
+        HandleLidSwitchExternalPower=${cfg.lidSwitchExternalPower}
+        ${cfg.extraConfig}
+      '';
+    };
+
+    # Restarting systemd-logind breaks X11
+    # - upstream commit: https://cgit.freedesktop.org/xorg/xserver/commit/?id=dc48bd653c7e101
+    # - systemd announcement: https://github.com/systemd/systemd/blob/22043e4317ecd2bc7834b48a6d364de76bb26d91/NEWS#L103-L112
+    # - this might be addressed in the future by xorg
+    #systemd.services.systemd-logind.restartTriggers = [ config.environment.etc."systemd/logind.conf".source ];
+    systemd.services.systemd-logind.restartIfChanged = false;
+    systemd.services.systemd-logind.stopIfChanged = false;
+
+    # The user-runtime-dir@ service is managed by systemd-logind we should not touch it or else we break the users' sessions.
+    systemd.services."user-runtime-dir@".stopIfChanged = false;
+    systemd.services."user-runtime-dir@".restartIfChanged = false;
+  };
+}
diff --git a/nixos/modules/system/boot/systemd-nspawn.nix b/nixos/modules/system/boot/systemd/nspawn.nix
index 02d2660add89..0c6822319a5b 100644
--- a/nixos/modules/system/boot/systemd-nspawn.nix
+++ b/nixos/modules/system/boot/systemd/nspawn.nix
@@ -120,14 +120,6 @@ in {
         })
         {
           systemd.targets.multi-user.wants = [ "machines.target" ];
-
-          # Workaround for https://github.com/NixOS/nixpkgs/pull/67232#issuecomment-531315437 and https://github.com/systemd/systemd/issues/13622
-          # Once systemd fixes this upstream, we can re-enable -U
-          systemd.services."systemd-nspawn@".serviceConfig.ExecStart = [
-            ""  # deliberately empty. signals systemd to override the ExecStart
-            # Only difference between upstream is that we do not pass the -U flag
-            "${config.systemd.package}/bin/systemd-nspawn --quiet --keep-unit --boot --link-journal=try-guest --network-veth --settings=override --machine=%i"
-          ];
         }
       ];
 }
diff --git a/nixos/modules/system/boot/systemd/tmpfiles.nix b/nixos/modules/system/boot/systemd/tmpfiles.nix
new file mode 100644
index 000000000000..57d44c8591ed
--- /dev/null
+++ b/nixos/modules/system/boot/systemd/tmpfiles.nix
@@ -0,0 +1,104 @@
+{ config, lib, pkgs, utils, ... }:
+
+with lib;
+
+let
+  cfg = config.systemd.tmpfiles;
+  systemd = config.systemd.package;
+in
+{
+  options = {
+    systemd.tmpfiles.rules = mkOption {
+      type = types.listOf types.str;
+      default = [];
+      example = [ "d /tmp 1777 root root 10d" ];
+      description = ''
+        Rules for creation, deletion and cleaning of volatile and temporary files
+        automatically. See
+        <citerefentry><refentrytitle>tmpfiles.d</refentrytitle><manvolnum>5</manvolnum></citerefentry>
+        for the exact format.
+      '';
+    };
+
+    systemd.tmpfiles.packages = mkOption {
+      type = types.listOf types.package;
+      default = [];
+      example = literalExpression "[ pkgs.lvm2 ]";
+      apply = map getLib;
+      description = ''
+        List of packages containing <command>systemd-tmpfiles</command> rules.
+
+        All files ending in .conf found in
+        <filename><replaceable>pkg</replaceable>/lib/tmpfiles.d</filename>
+        will be included.
+        If this folder does not exist or does not contain any files an error will be returned instead.
+
+        If a <filename>lib</filename> output is available, rules are searched there and only there.
+        If there is no <filename>lib</filename> output it will fall back to <filename>out</filename>
+        and if that does not exist either, the default output will be used.
+      '';
+    };
+  };
+
+  config = {
+    systemd.additionalUpstreamSystemUnits = [
+      "systemd-tmpfiles-clean.service"
+      "systemd-tmpfiles-clean.timer"
+      "systemd-tmpfiles-setup.service"
+      "systemd-tmpfiles-setup-dev.service"
+    ];
+
+    systemd.additionalUpstreamUserUnits = [
+      "systemd-tmpfiles-clean.service"
+      "systemd-tmpfiles-clean.timer"
+      "systemd-tmpfiles-setup.service"
+    ];
+
+    environment.etc = {
+      "tmpfiles.d".source = (pkgs.symlinkJoin {
+        name = "tmpfiles.d";
+        paths = map (p: p + "/lib/tmpfiles.d") cfg.packages;
+        postBuild = ''
+          for i in $(cat $pathsPath); do
+            (test -d "$i" && test $(ls "$i"/*.conf | wc -l) -ge 1) || (
+              echo "ERROR: The path '$i' from systemd.tmpfiles.packages contains no *.conf files."
+              exit 1
+            )
+          done
+        '' + concatMapStrings (name: optionalString (hasPrefix "tmpfiles.d/" name) ''
+          rm -f $out/${removePrefix "tmpfiles.d/" name}
+        '') config.system.build.etc.passthru.targets;
+      }) + "/*";
+    };
+
+    systemd.tmpfiles.packages = [
+      # Default tmpfiles rules provided by systemd
+      (pkgs.runCommand "systemd-default-tmpfiles" {} ''
+        mkdir -p $out/lib/tmpfiles.d
+        cd $out/lib/tmpfiles.d
+
+        ln -s "${systemd}/example/tmpfiles.d/home.conf"
+        ln -s "${systemd}/example/tmpfiles.d/journal-nocow.conf"
+        ln -s "${systemd}/example/tmpfiles.d/static-nodes-permissions.conf"
+        ln -s "${systemd}/example/tmpfiles.d/systemd.conf"
+        ln -s "${systemd}/example/tmpfiles.d/systemd-nologin.conf"
+        ln -s "${systemd}/example/tmpfiles.d/systemd-nspawn.conf"
+        ln -s "${systemd}/example/tmpfiles.d/systemd-tmp.conf"
+        ln -s "${systemd}/example/tmpfiles.d/tmp.conf"
+        ln -s "${systemd}/example/tmpfiles.d/var.conf"
+        ln -s "${systemd}/example/tmpfiles.d/x11.conf"
+      '')
+      # User-specified tmpfiles rules
+      (pkgs.writeTextFile {
+        name = "nixos-tmpfiles.d";
+        destination = "/lib/tmpfiles.d/00-nixos.conf";
+        text = ''
+          # This file is created automatically and should not be modified.
+          # Please change the option ‘systemd.tmpfiles.rules’ instead.
+
+          ${concatStringsSep "\n" cfg.rules}
+        '';
+      })
+    ];
+  };
+}
diff --git a/nixos/modules/system/boot/systemd/user.nix b/nixos/modules/system/boot/systemd/user.nix
new file mode 100644
index 000000000000..e30f83f3457f
--- /dev/null
+++ b/nixos/modules/system/boot/systemd/user.nix
@@ -0,0 +1,158 @@
+{ config, lib, pkgs, utils, ... }:
+with utils;
+with systemdUtils.unitOptions;
+with lib;
+
+let
+  cfg = config.systemd.user;
+
+  systemd = config.systemd.package;
+
+  inherit
+    (systemdUtils.lib)
+    makeUnit
+    generateUnits
+    makeJobScript
+    unitConfig
+    serviceConfig
+    commonUnitText
+    targetToUnit
+    serviceToUnit
+    socketToUnit
+    timerToUnit
+    pathToUnit;
+
+  upstreamUserUnits = [
+    "app.slice"
+    "background.slice"
+    "basic.target"
+    "bluetooth.target"
+    "default.target"
+    "exit.target"
+    "graphical-session-pre.target"
+    "graphical-session.target"
+    "paths.target"
+    "printer.target"
+    "session.slice"
+    "shutdown.target"
+    "smartcard.target"
+    "sockets.target"
+    "sound.target"
+    "systemd-exit.service"
+    "timers.target"
+    "xdg-desktop-autostart.target"
+  ] ++ config.systemd.additionalUpstreamUserUnits;
+in {
+  options = {
+    systemd.user.extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = "DefaultCPUAccounting=yes";
+      description = ''
+        Extra config options for systemd user instances. See man systemd-user.conf for
+        available options.
+      '';
+    };
+
+    systemd.user.units = mkOption {
+      description = "Definition of systemd per-user units.";
+      default = {};
+      type = with types; attrsOf (submodule (
+        { name, config, ... }:
+        { options = concreteUnitOptions;
+          config = {
+            unit = mkDefault (makeUnit name config);
+          };
+        }));
+    };
+
+    systemd.user.paths = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule [ { options = pathOptions; } unitConfig ]);
+      description = "Definition of systemd per-user path units.";
+    };
+
+    systemd.user.services = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule [ { options = serviceOptions; } unitConfig serviceConfig ] );
+      description = "Definition of systemd per-user service units.";
+    };
+
+    systemd.user.slices = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule [ { options = sliceOptions; } unitConfig ] );
+      description = "Definition of systemd per-user slice units.";
+    };
+
+    systemd.user.sockets = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule [ { options = socketOptions; } unitConfig ] );
+      description = "Definition of systemd per-user socket units.";
+    };
+
+    systemd.user.targets = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule [ { options = targetOptions; } unitConfig] );
+      description = "Definition of systemd per-user target units.";
+    };
+
+    systemd.user.timers = mkOption {
+      default = {};
+      type = with types; attrsOf (submodule [ { options = timerOptions; } unitConfig ] );
+      description = "Definition of systemd per-user timer units.";
+    };
+
+    systemd.additionalUpstreamUserUnits = mkOption {
+      default = [];
+      type = types.listOf types.str;
+      example = [];
+      description = ''
+        Additional units shipped with systemd that should be enabled for per-user systemd instances.
+      '';
+      internal = true;
+    };
+  };
+
+  config = {
+    systemd.additionalUpstreamSystemUnits = [
+      "user.slice"
+    ];
+
+    environment.etc = {
+      "systemd/user".source = generateUnits "user" cfg.units upstreamUserUnits [];
+
+      "systemd/user.conf".text = ''
+        [Manager]
+        ${cfg.extraConfig}
+      '';
+    };
+
+    systemd.user.units =
+         mapAttrs' (n: v: nameValuePair "${n}.path"    (pathToUnit    n v)) cfg.paths
+      // mapAttrs' (n: v: nameValuePair "${n}.service" (serviceToUnit n v)) cfg.services
+      // mapAttrs' (n: v: nameValuePair "${n}.slice"   (sliceToUnit   n v)) cfg.slices
+      // mapAttrs' (n: v: nameValuePair "${n}.socket"  (socketToUnit  n v)) cfg.sockets
+      // mapAttrs' (n: v: nameValuePair "${n}.target"  (targetToUnit  n v)) cfg.targets
+      // mapAttrs' (n: v: nameValuePair "${n}.timer"   (timerToUnit   n v)) cfg.timers;
+
+    # Generate timer units for all services that have a ‘startAt’ value.
+    systemd.user.timers =
+      mapAttrs (name: service: {
+        wantedBy = ["timers.target"];
+        timerConfig.OnCalendar = service.startAt;
+      })
+      (filterAttrs (name: service: service.startAt != []) cfg.services);
+
+    # Provide the systemd-user PAM service, required to run systemd
+    # user instances.
+    security.pam.services.systemd-user =
+      { # Ensure that pam_systemd gets included. This is special-cased
+        # in systemd to provide XDG_RUNTIME_DIR.
+        startSession = true;
+      };
+
+    # Some overrides to upstream units.
+    systemd.services."user@".restartIfChanged = false;
+    systemd.services.systemd-user-sessions.restartIfChanged = false; # Restart kills all active sessions.
+  };
+}
diff --git a/nixos/modules/system/boot/tmp.nix b/nixos/modules/system/boot/tmp.nix
index 6edafd6695b6..cf6d19eb5f0e 100644
--- a/nixos/modules/system/boot/tmp.nix
+++ b/nixos/modules/system/boot/tmp.nix
@@ -48,7 +48,12 @@ in
         what = "tmpfs";
         where = "/tmp";
         type = "tmpfs";
-        mountConfig.Options = [ "mode=1777" "strictatime" "rw" "nosuid" "nodev" "size=${toString cfg.tmpOnTmpfsSize}" ];
+        mountConfig.Options = concatStringsSep "," [ "mode=1777"
+                                                     "strictatime"
+                                                     "rw"
+                                                     "nosuid"
+                                                     "nodev"
+                                                     "size=${toString cfg.tmpOnTmpfsSize}" ];
       }
     ];
 
diff --git a/nixos/modules/system/build.nix b/nixos/modules/system/build.nix
new file mode 100644
index 000000000000..58dc3f0d4113
--- /dev/null
+++ b/nixos/modules/system/build.nix
@@ -0,0 +1,21 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption types;
+in
+{
+  options = {
+
+    system.build = mkOption {
+      default = {};
+      description = ''
+        Attribute set of derivations used to set up the system.
+      '';
+      type = types.submoduleWith {
+        modules = [{
+          freeformType = with types; lazyAttrsOf (uniq unspecified);
+        }];
+      };
+    };
+
+  };
+}
diff --git a/nixos/modules/system/etc/etc-activation.nix b/nixos/modules/system/etc/etc-activation.nix
new file mode 100644
index 000000000000..780104950186
--- /dev/null
+++ b/nixos/modules/system/etc/etc-activation.nix
@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+let
+  inherit (lib) stringAfter;
+in {
+
+  imports = [ ./etc.nix ];
+
+  config = {
+    system.activationScripts.etc =
+      stringAfter [ "users" "groups" ] config.system.build.etcActivationCommands;
+  };
+}
diff --git a/nixos/modules/system/etc/etc.nix b/nixos/modules/system/etc/etc.nix
index 6cc8c341e6df..ed552fecec53 100644
--- a/nixos/modules/system/etc/etc.nix
+++ b/nixos/modules/system/etc/etc.nix
@@ -66,6 +66,8 @@ in
 
 {
 
+  imports = [ ../build.nix ];
+
   ###### interface
 
   options = {
@@ -188,14 +190,12 @@ in
   config = {
 
     system.build.etc = etc;
-
-    system.activationScripts.etc = stringAfter [ "users" "groups" ]
+    system.build.etcActivationCommands =
       ''
         # Set up the statically computed bits of /etc.
         echo "setting up /etc..."
         ${pkgs.perl.withPackages (p: [ p.FileSlurp ])}/bin/perl ${./setup-etc.pl} ${etc}/etc
       '';
-
   };
 
 }
diff --git a/nixos/modules/system/etc/test.nix b/nixos/modules/system/etc/test.nix
new file mode 100644
index 000000000000..5e43b155038d
--- /dev/null
+++ b/nixos/modules/system/etc/test.nix
@@ -0,0 +1,70 @@
+{ lib
+, coreutils
+, fakechroot
+, fakeroot
+, evalMinimalConfig
+, pkgsModule
+, runCommand
+, util-linux
+, vmTools
+, writeText
+}:
+let
+  node = evalMinimalConfig ({ config, ... }: {
+    imports = [ pkgsModule ../etc/etc.nix ];
+    environment.etc."passwd" = {
+      text = passwdText;
+    };
+    environment.etc."hosts" = {
+      text = hostsText;
+      mode = "0751";
+    };
+  });
+  passwdText = ''
+    root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash
+  '';
+  hostsText = ''
+    127.0.0.1 localhost
+    ::1 localhost
+    # testing...
+  '';
+in
+lib.recurseIntoAttrs {
+  test-etc-vm =
+    vmTools.runInLinuxVM (runCommand "test-etc-vm" { } ''
+      mkdir -p /etc
+      ${node.config.system.build.etcActivationCommands}
+      set -x
+      [[ -L /etc/passwd ]]
+      diff /etc/passwd ${writeText "expected-passwd" passwdText}
+      [[ 751 = $(stat --format %a /etc/hosts) ]]
+      diff /etc/hosts ${writeText "expected-hosts" hostsText}
+      set +x
+      touch $out
+    '');
+
+  # fakeroot is behaving weird
+  test-etc-fakeroot =
+    runCommand "test-etc"
+      {
+        nativeBuildInputs = [
+          fakeroot
+          fakechroot
+          # for chroot
+          coreutils
+          # fakechroot needs getopt, which is provided by util-linux
+          util-linux
+        ];
+        fakeRootCommands = ''
+          mkdir -p /etc
+          ${node.config.system.build.etcActivationCommands}
+          diff /etc/hosts ${writeText "expected-hosts" hostsText}
+          touch $out
+        '';
+      } ''
+      mkdir fake-root
+      export FAKECHROOT_EXCLUDE_PATH=/dev:/proc:/sys:${builtins.storeDir}:$out
+      fakechroot fakeroot chroot $PWD/fake-root bash -c 'source $stdenv/setup; eval "$fakeRootCommands"'
+    '';
+
+}
diff --git a/nixos/modules/tasks/auto-upgrade.nix b/nixos/modules/tasks/auto-upgrade.nix
index b931b27ad817..1404dcbaf7c0 100644
--- a/nixos/modules/tasks/auto-upgrade.nix
+++ b/nixos/modules/tasks/auto-upgrade.nix
@@ -80,6 +80,7 @@ in {
           Reboot the system into the new generation instead of a switch
           if the new generation uses a different kernel, kernel modules
           or initrd than the booted system.
+          See <option>rebootWindow</option> for configuring the times at which a reboot is allowed.
         '';
       };
 
@@ -96,6 +97,32 @@ in {
         '';
       };
 
+      rebootWindow = mkOption {
+        description = ''
+          Define a lower and upper time value (in HH:MM format) which
+          constitute a time window during which reboots are allowed after an upgrade.
+          This option only has an effect when <option>allowReboot</option> is enabled.
+          The default value of <literal>null</literal> means that reboots are allowed at any time.
+        '';
+        default = null;
+        example = { lower = "01:00"; upper = "05:00"; };
+        type = with types; nullOr (submodule {
+          options = {
+            lower = mkOption {
+              description = "Lower limit of the reboot window";
+              type = types.strMatching "[[:digit:]]{2}:[[:digit:]]{2}";
+              example = "01:00";
+            };
+
+            upper = mkOption {
+              description = "Upper limit of the reboot window";
+              type = types.strMatching "[[:digit:]]{2}:[[:digit:]]{2}";
+              example = "05:00";
+            };
+          };
+        });
+      };
+
     };
 
   };
@@ -110,12 +137,10 @@ in {
     }];
 
     system.autoUpgrade.flags = (if cfg.flake == null then
-        [ "--no-build-output" ] ++ (if cfg.channel == null then
-          [ "--upgrade" ]
-        else [
+        [ "--no-build-output" ] ++ optionals (cfg.channel != null) [
           "-I"
           "nixpkgs=${cfg.channel}/nixexprs.tar.xz"
-        ])
+        ]
       else
         [ "--flake ${cfg.flake}" ]);
 
@@ -143,19 +168,52 @@ in {
       ];
 
       script = let
-        nixos-rebuild =
-          "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
+        nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
+        date     = "${pkgs.coreutils}/bin/date";
+        readlink = "${pkgs.coreutils}/bin/readlink";
+        shutdown = "${pkgs.systemd}/bin/shutdown";
+        upgradeFlag = optional (cfg.channel == null) "--upgrade";
       in if cfg.allowReboot then ''
-        ${nixos-rebuild} boot ${toString cfg.flags}
-        booted="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})"
-        built="$(readlink /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
-        if [ "$booted" = "$built" ]; then
+        ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)}
+        booted="$(${readlink} /run/booted-system/{initrd,kernel,kernel-modules})"
+        built="$(${readlink} /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
+
+        ${optionalString (cfg.rebootWindow != null) ''
+          current_time="$(${date} +%H:%M)"
+
+          lower="${cfg.rebootWindow.lower}"
+          upper="${cfg.rebootWindow.upper}"
+
+          if [[ "''${lower}" < "''${upper}" ]]; then
+            if [[ "''${current_time}" > "''${lower}" ]] && \
+               [[ "''${current_time}" < "''${upper}" ]]; then
+              do_reboot="true"
+            else
+              do_reboot="false"
+            fi
+          else
+            # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
+            # we want to reboot if cur > 23h or cur < 6h
+            if [[ "''${current_time}" < "''${upper}" ]] || \
+               [[ "''${current_time}" > "''${lower}" ]]; then
+              do_reboot="true"
+            else
+              do_reboot="false"
+            fi
+          fi
+        ''}
+
+        if [ "''${booted}" = "''${built}" ]; then
           ${nixos-rebuild} switch ${toString cfg.flags}
+        ${optionalString (cfg.rebootWindow != null) ''
+          elif [ "''${do_reboot}" != true ]; then
+            echo "Outside of configured reboot window, skipping."
+        ''}
         else
-          /run/current-system/sw/bin/shutdown -r +1
+          ${shutdown} -r +1
         fi
       '' else ''
-        ${nixos-rebuild} switch ${toString cfg.flags}
+        ${nixos-rebuild} switch ${toString (cfg.flags ++ upgradeFlag)}
       '';
 
       startAt = cfg.dates;
@@ -167,3 +225,4 @@ in {
   };
 
 }
+
diff --git a/nixos/modules/tasks/filesystems.nix b/nixos/modules/tasks/filesystems.nix
index 225bcbe58e01..f3da6771197e 100644
--- a/nixos/modules/tasks/filesystems.nix
+++ b/nixos/modules/tasks/filesystems.nix
@@ -250,7 +250,7 @@ in
 
     environment.etc.fstab.text =
       let
-        fsToSkipCheck = [ "none" "bindfs" "btrfs" "zfs" "tmpfs" "nfs" "vboxsf" "glusterfs" ];
+        fsToSkipCheck = [ "none" "bindfs" "btrfs" "zfs" "tmpfs" "nfs" "vboxsf" "glusterfs" "apfs" ];
         skipCheck = fs: fs.noCheck || fs.device == "none" || builtins.elem fs.fsType fsToSkipCheck;
         # https://wiki.archlinux.org/index.php/fstab#Filepath_spaces
         escape = string: builtins.replaceStrings [ " " "\t" ] [ "\\040" "\\011" ] string;
diff --git a/nixos/modules/tasks/filesystems/apfs.nix b/nixos/modules/tasks/filesystems/apfs.nix
new file mode 100644
index 000000000000..2f2be351df61
--- /dev/null
+++ b/nixos/modules/tasks/filesystems/apfs.nix
@@ -0,0 +1,22 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  inInitrd = any (fs: fs == "apfs") config.boot.initrd.supportedFilesystems;
+
+in
+
+{
+  config = mkIf (any (fs: fs == "apfs") config.boot.supportedFilesystems) {
+
+    system.fsPackages = [ pkgs.apfsprogs ];
+
+    boot.extraModulePackages = [ config.boot.kernelPackages.apfs ];
+
+    boot.initrd.kernelModules = mkIf inInitrd [ "apfs" ];
+
+    # Don't copy apfsck into the initramfs since it does not support repairing the filesystem
+  };
+}
diff --git a/nixos/modules/tasks/filesystems/zfs.nix b/nixos/modules/tasks/filesystems/zfs.nix
index 65364801c32a..3bc0dedec00e 100644
--- a/nixos/modules/tasks/filesystems/zfs.nix
+++ b/nixos/modules/tasks/filesystems/zfs.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, utils, ... }:
+{ config, lib, options, pkgs, utils, ... }:
 #
 # TODO: zfs tunables
 
@@ -8,6 +8,7 @@ with lib;
 let
 
   cfgZfs = config.boot.zfs;
+  optZfs = options.boot.zfs;
   cfgExpandOnBoot = config.services.zfs.expandOnBoot;
   cfgSnapshots = config.services.zfs.autoSnapshot;
   cfgSnapFlags = cfgSnapshots.flags;
@@ -112,6 +113,7 @@ in
         readOnly = true;
         type = types.bool;
         default = inInitrd || inSystem;
+        defaultText = literalDocBook "<literal>true</literal> if ZFS filesystem support is enabled";
         description = "True if ZFS filesystem support is enabled";
       };
 
@@ -346,6 +348,7 @@ in
     services.zfs.zed = {
       enableMail = mkEnableOption "ZED's ability to send emails" // {
         default = cfgZfs.package.enableMail;
+        defaultText = literalExpression "config.${optZfs.package}.enableMail";
       };
 
       settings = mkOption {
diff --git a/nixos/modules/tasks/network-interfaces-scripted.nix b/nixos/modules/tasks/network-interfaces-scripted.nix
index e8e2de090b32..19f2be2c4a25 100644
--- a/nixos/modules/tasks/network-interfaces-scripted.nix
+++ b/nixos/modules/tasks/network-interfaces-scripted.nix
@@ -532,6 +532,33 @@ let
             '';
           });
 
+        createGreDevice = n: v: nameValuePair "${n}-netdev"
+          (let
+            deps = deviceDependency v.dev;
+          in
+          { description = "GRE Tunnel Interface ${n}";
+            wantedBy = [ "network-setup.service" (subsystemDevice n) ];
+            bindsTo = deps;
+            partOf = [ "network-setup.service" ];
+            after = [ "network-pre.target" ] ++ deps;
+            before = [ "network-setup.service" ];
+            serviceConfig.Type = "oneshot";
+            serviceConfig.RemainAfterExit = true;
+            path = [ pkgs.iproute2 ];
+            script = ''
+              # Remove Dead Interfaces
+              ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
+              ip link add name "${n}" type ${v.type} \
+                ${optionalString (v.remote != null) "remote \"${v.remote}\""} \
+                ${optionalString (v.local != null) "local \"${v.local}\""} \
+                ${optionalString (v.dev != null) "dev \"${v.dev}\""}
+              ip link set "${n}" up
+            '';
+            postStop = ''
+              ip link delete "${n}" || true
+            '';
+          });
+
         createVlanDevice = n: v: nameValuePair "${n}-netdev"
           (let
             deps = deviceDependency v.interface;
@@ -570,6 +597,7 @@ let
          // mapAttrs' createMacvlanDevice cfg.macvlans
          // mapAttrs' createFouEncapsulation cfg.fooOverUDP
          // mapAttrs' createSitDevice cfg.sits
+         // mapAttrs' createGreDevice cfg.greTunnels
          // mapAttrs' createVlanDevice cfg.vlans
          // {
            network-setup = networkSetup;
diff --git a/nixos/modules/tasks/network-interfaces-systemd.nix b/nixos/modules/tasks/network-interfaces-systemd.nix
index ccfd7fd4132b..8a5e1b5af114 100644
--- a/nixos/modules/tasks/network-interfaces-systemd.nix
+++ b/nixos/modules/tasks/network-interfaces-systemd.nix
@@ -12,12 +12,17 @@ let
     i.ipv4.addresses
     ++ optionals cfg.enableIPv6 i.ipv6.addresses;
 
+  interfaceRoutes = i:
+    i.ipv4.routes
+    ++ optionals cfg.enableIPv6 i.ipv6.routes;
+
   dhcpStr = useDHCP: if useDHCP == true || useDHCP == null then "yes" else "no";
 
   slaves =
     concatLists (map (bond: bond.interfaces) (attrValues cfg.bonds))
     ++ concatLists (map (bridge: bridge.interfaces) (attrValues cfg.bridges))
     ++ map (sit: sit.dev) (attrValues cfg.sits)
+    ++ map (gre: gre.dev) (attrValues cfg.greTunnels)
     ++ map (vlan: vlan.interface) (attrValues cfg.vlans)
     # add dependency to physical or independently created vswitch member interface
     # TODO: warn the user that any address configured on those interfaces will be useless
@@ -87,12 +92,69 @@ in
             };
           };
         });
-        networks."40-${i.name}" = mkMerge [ (genericNetwork mkDefault) {
+        networks."40-${i.name}" = mkMerge [ (genericNetwork id) {
           name = mkDefault i.name;
           DHCP = mkForce (dhcpStr
             (if i.useDHCP != null then i.useDHCP else false));
           address = forEach (interfaceIps i)
             (ip: "${ip.address}/${toString ip.prefixLength}");
+          routes = forEach (interfaceRoutes i)
+            (route: {
+              # Most of these route options have not been tested.
+              # Please fix or report any mistakes you may find.
+              routeConfig =
+                optionalAttrs (route.prefixLength > 0) {
+                  Destination = "${route.address}/${toString route.prefixLength}";
+                } //
+                optionalAttrs (route.options ? fastopen_no_cookie) {
+                  FastOpenNoCookie = route.options.fastopen_no_cookie;
+                } //
+                optionalAttrs (route.via != null) {
+                  Gateway = route.via;
+                } //
+                optionalAttrs (route.options ? onlink) {
+                  GatewayOnLink = true;
+                } //
+                optionalAttrs (route.options ? initrwnd) {
+                  InitialAdvertisedReceiveWindow = route.options.initrwnd;
+                } //
+                optionalAttrs (route.options ? initcwnd) {
+                  InitialCongestionWindow = route.options.initcwnd;
+                } //
+                optionalAttrs (route.options ? pref) {
+                  IPv6Preference = route.options.pref;
+                } //
+                optionalAttrs (route.options ? mtu) {
+                  MTUBytes = route.options.mtu;
+                } //
+                optionalAttrs (route.options ? metric) {
+                  Metric = route.options.metric;
+                } //
+                optionalAttrs (route.options ? src) {
+                  PreferredSource = route.options.src;
+                } //
+                optionalAttrs (route.options ? protocol) {
+                  Protocol = route.options.protocol;
+                } //
+                optionalAttrs (route.options ? quickack) {
+                  QuickAck = route.options.quickack;
+                } //
+                optionalAttrs (route.options ? scope) {
+                  Scope = route.options.scope;
+                } //
+                optionalAttrs (route.options ? from) {
+                  Source = route.options.from;
+                } //
+                optionalAttrs (route.options ? table) {
+                  Table = route.options.table;
+                } //
+                optionalAttrs (route.options ? advmss) {
+                  TCPAdvertisedMaximumSegmentSize = route.options.advmss;
+                } //
+                optionalAttrs (route.options ? ttl-propagate) {
+                  TTLPropagate = route.options.ttl-propagate == "enabled";
+                };
+            });
           networkConfig.IPv6PrivacyExtensions = "kernel";
           linkConfig = optionalAttrs (i.macAddress != null) {
             MACAddress = i.macAddress;
@@ -245,6 +307,25 @@ in
           } ]);
         };
       })))
+      (mkMerge (flip mapAttrsToList cfg.greTunnels (name: gre: {
+        netdevs."40-${name}" = {
+          netdevConfig = {
+            Name = name;
+            Kind = gre.type;
+          };
+          tunnelConfig =
+            (optionalAttrs (gre.remote != null) {
+              Remote = gre.remote;
+            }) // (optionalAttrs (gre.local != null) {
+              Local = gre.local;
+            });
+        };
+        networks = mkIf (gre.dev != null) {
+          "40-${gre.dev}" = (mkMerge [ (genericNetwork (mkOverride 999)) {
+            tunnel = [ name ];
+          } ]);
+        };
+      })))
       (mkMerge (flip mapAttrsToList cfg.vlans (name: vlan: {
         netdevs."40-${name}" = {
           netdevConfig = {
diff --git a/nixos/modules/tasks/network-interfaces.nix b/nixos/modules/tasks/network-interfaces.nix
index 49901cda848d..01980b80f1cf 100644
--- a/nixos/modules/tasks/network-interfaces.nix
+++ b/nixos/modules/tasks/network-interfaces.nix
@@ -6,9 +6,11 @@ with utils;
 let
 
   cfg = config.networking;
+  opt = options.networking;
   interfaces = attrValues cfg.interfaces;
   hasVirtuals = any (i: i.virtual) interfaces;
   hasSits = cfg.sits != { };
+  hasGres = cfg.greTunnels != { };
   hasBonds = cfg.bonds != { };
   hasFous = cfg.fooOverUDP != { }
     || filterAttrs (_: s: s.encapsulation != null) cfg.sits != { };
@@ -101,6 +103,11 @@ let
         description = ''
           Other route options. See the symbol <literal>OPTIONS</literal>
           in the <literal>ip-route(8)</literal> manual page for the details.
+          You may also specify <literal>metric</literal>,
+          <literal>src</literal>, <literal>protocol</literal>,
+          <literal>scope</literal>, <literal>from</literal>
+          and <literal>table</literal>, which are technically
+          not route options, in the sense used in the manual.
         '';
       };
 
@@ -206,6 +213,14 @@ let
         type = with types; listOf (submodule (routeOpts 4));
         description = ''
           List of extra IPv4 static routes that will be assigned to the interface.
+          <warning><para>If the route type is the default <literal>unicast</literal>, then the scope
+          is set differently depending on the value of <option>networking.useNetworkd</option>:
+          the script-based backend sets it to <literal>link</literal>, while networkd sets
+          it to <literal>global</literal>.</para></warning>
+          If you want consistency between the two implementations,
+          set the scope of the route manually with
+          <literal>networking.interfaces.eth0.ipv4.routes = [{ options.scope = "global"; }]</literal>
+          for example.
         '';
       };
 
@@ -290,7 +305,7 @@ let
         enable = mkOption {
           type = types.bool;
           default = false;
-          description = "Wether to enable wol on this interface.";
+          description = "Whether to enable wol on this interface.";
         };
       };
     };
@@ -996,6 +1011,76 @@ in
       });
     };
 
+    networking.greTunnels = mkOption {
+      default = { };
+      example = literalExpression ''
+        {
+          greBridge = {
+            remote = "10.0.0.1";
+            local = "10.0.0.22";
+            dev = "enp4s0f0";
+            type = "tap";
+          };
+          gre6Tunnel = {
+            remote = "fd7a:5634::1";
+            local = "fd7a:5634::2";
+            dev = "enp4s0f0";
+            type = "tun6";
+          };
+        }
+      '';
+      description = ''
+        This option allows you to define Generic Routing Encapsulation (GRE) tunnels.
+      '';
+      type = with types; attrsOf (submodule {
+        options = {
+
+          remote = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+            example = "10.0.0.1";
+            description = ''
+              The address of the remote endpoint to forward traffic over.
+            '';
+          };
+
+          local = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+            example = "10.0.0.22";
+            description = ''
+              The address of the local endpoint which the remote
+              side should send packets to.
+            '';
+          };
+
+          dev = mkOption {
+            type = types.nullOr types.str;
+            default = null;
+            example = "enp4s0f0";
+            description = ''
+              The underlying network device on which the tunnel resides.
+            '';
+          };
+
+          type = mkOption {
+            type = with types; enum [ "tun" "tap" "tun6" "tap6" ];
+            default = "tap";
+            example = "tap";
+            apply = v: {
+              tun = "gre";
+              tap = "gretap";
+              tun6 = "ip6gre";
+              tap6 = "ip6gretap";
+            }.${v};
+            description = ''
+              Whether the tunnel routes layer 2 (tap) or layer 3 (tun) traffic.
+            '';
+          };
+        };
+      });
+    };
+
     networking.vlans = mkOption {
       default = { };
       example = literalExpression ''
@@ -1169,6 +1254,9 @@ in
 
     networking.tempAddresses = mkOption {
       default = if cfg.enableIPv6 then "default" else "disabled";
+      defaultText = literalExpression ''
+        if ''${config.${opt.enableIPv6}} then "default" else "disabled"
+      '';
       type = types.enum (lib.attrNames tempaddrValues);
       description = ''
         Whether to enable IPv6 Privacy Extensions for interfaces not
@@ -1225,6 +1313,7 @@ in
     boot.kernelModules = [ ]
       ++ optional hasVirtuals "tun"
       ++ optional hasSits "sit"
+      ++ optional hasGres "gre"
       ++ optional hasBonds "bonding"
       ++ optional hasFous "fou";
 
@@ -1247,22 +1336,13 @@ in
           val = tempaddrValues.${opt}.sysctl;
          in nameValuePair "net.ipv6.conf.${replaceChars ["."] ["/"] i.name}.use_tempaddr" val));
 
-    # Capabilities won't work unless we have at-least a 4.3 Linux
-    # kernel because we need the ambient capability
-    security.wrappers = if (versionAtLeast (getVersion config.boot.kernelPackages.kernel) "4.3") then {
+    security.wrappers = {
       ping = {
         owner = "root";
         group = "root";
         capabilities = "cap_net_raw+p";
         source = "${pkgs.iputils.out}/bin/ping";
       };
-    } else {
-      ping = {
-        setuid = true;
-        owner = "root";
-        group = "root";
-        source = "${pkgs.iputils.out}/bin/ping";
-      };
     };
     security.apparmor.policies."bin.ping".profile = lib.mkIf config.security.apparmor.policies."bin.ping".enable (lib.mkAfter ''
       /run/wrappers/bin/ping {
diff --git a/nixos/modules/testing/test-instrumentation.nix b/nixos/modules/testing/test-instrumentation.nix
index a7011be7e042..01447e6ada87 100644
--- a/nixos/modules/testing/test-instrumentation.nix
+++ b/nixos/modules/testing/test-instrumentation.nix
@@ -109,6 +109,10 @@ in
       # Allow very slow start
       DefaultTimeoutStartSec=300
     '';
+    systemd.user.extraConfig = ''
+      # Allow very slow start
+      DefaultTimeoutStartSec=300
+    '';
 
     boot.consoleLogLevel = 7;
 
diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix
index fe248a94488b..9a56b6950155 100644
--- a/nixos/modules/virtualisation/amazon-image.nix
+++ b/nixos/modules/virtualisation/amazon-image.nix
@@ -37,8 +37,13 @@ in
       { assertion = cfg.efi -> cfg.hvm;
         message = "EC2 instances using EFI must be HVM instances.";
       }
+      { assertion = versionOlder config.boot.kernelPackages.kernel.version "5.15";
+        message = "ENA driver fails to build with kernel >= 5.15";
+      }
     ];
 
+    boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
+
     boot.growPartition = cfg.hvm;
 
     fileSystems."/" = mkIf (!cfg.zfs.enable) {
@@ -155,7 +160,7 @@ in
     systemd.services."serial-getty@ttyS0".enable = true;
 
     # Creates symlinks for block device names.
-    services.udev.packages = [ pkgs.ec2-utils ];
+    services.udev.packages = [ pkgs.amazon-ec2-utils ];
 
     # Force getting the hostname from EC2.
     networking.hostName = mkDefault "";
diff --git a/nixos/modules/virtualisation/build-vm.nix b/nixos/modules/virtualisation/build-vm.nix
new file mode 100644
index 000000000000..4a4694950f98
--- /dev/null
+++ b/nixos/modules/virtualisation/build-vm.nix
@@ -0,0 +1,58 @@
+{ config, extendModules, lib, ... }:
+let
+
+  inherit (lib)
+    mkOption
+    ;
+
+  vmVariant = extendModules {
+    modules = [ ./qemu-vm.nix ];
+  };
+
+  vmVariantWithBootLoader = vmVariant.extendModules {
+    modules = [
+      ({ config, ... }: {
+        _file = "nixos/default.nix##vmWithBootLoader";
+        virtualisation.useBootLoader = true;
+        virtualisation.useEFIBoot =
+          config.boot.loader.systemd-boot.enable ||
+          config.boot.loader.efi.canTouchEfiVariables;
+      })
+    ];
+  };
+in
+{
+  options = {
+
+    virtualisation.vmVariant = mkOption {
+      description = ''
+        Machine configuration to be added for the vm script produced by <literal>nixos-rebuild build-vm</literal>.
+      '';
+      inherit (vmVariant) type;
+      default = {};
+      visible = "shallow";
+    };
+
+    virtualisation.vmVariantWithBootLoader = mkOption {
+      description = ''
+        Machine configuration to be added for the vm script produced by <literal>nixos-rebuild build-vm-with-bootloader</literal>.
+      '';
+      inherit (vmVariantWithBootLoader) type;
+      default = {};
+      visible = "shallow";
+    };
+
+  };
+
+  config = {
+
+    system.build = {
+      vm = lib.mkDefault config.virtualisation.vmVariant.system.build.vm;
+      vmWithBootLoader = lib.mkDefault config.virtualisation.vmVariantWithBootLoader.system.build.vm;
+    };
+
+  };
+
+  # uses extendModules
+  meta.buildDocsInSandbox = false;
+}
diff --git a/nixos/modules/virtualisation/container-config.nix b/nixos/modules/virtualisation/container-config.nix
index 6ff6bdd30c20..0966ef84827f 100644
--- a/nixos/modules/virtualisation/container-config.nix
+++ b/nixos/modules/virtualisation/container-config.nix
@@ -18,7 +18,7 @@ with lib;
     services.openssh.startWhenNeeded = mkDefault true;
 
     # Shut up warnings about not having a boot loader.
-    system.build.installBootLoader = "${pkgs.coreutils}/bin/true";
+    system.build.installBootLoader = lib.mkDefault "${pkgs.coreutils}/bin/true";
 
     # Not supported in systemd-nspawn containers.
     security.audit.enable = false;
diff --git a/nixos/modules/virtualisation/containerd.nix b/nixos/modules/virtualisation/containerd.nix
index 898a66e7b04e..ea89a994b172 100644
--- a/nixos/modules/virtualisation/containerd.nix
+++ b/nixos/modules/virtualisation/containerd.nix
@@ -53,6 +53,7 @@ in
     virtualisation.containerd = {
       args.config = toString containerdConfigChecked;
       settings = {
+        version = 2;
         plugins."io.containerd.grpc.v1.cri" = {
          containerd.snapshotter =
            lib.mkIf config.boot.zfs.enabled (lib.mkOptionDefault "zfs");
diff --git a/nixos/modules/virtualisation/cri-o.nix b/nixos/modules/virtualisation/cri-o.nix
index 38766113f391..cf5110001503 100644
--- a/nixos/modules/virtualisation/cri-o.nix
+++ b/nixos/modules/virtualisation/cri-o.nix
@@ -71,6 +71,10 @@ in
     package = mkOption {
       type = types.package;
       default = crioPackage;
+      defaultText = literalDocBook ''
+        <literal>pkgs.cri-o</literal> built with
+        <literal>config.${opt.extraPackages}</literal>.
+      '';
       internal = true;
       description = ''
         The final CRI-O package (including extra packages).
diff --git a/nixos/modules/virtualisation/docker-rootless.nix b/nixos/modules/virtualisation/docker-rootless.nix
new file mode 100644
index 000000000000..d371f67ecdc8
--- /dev/null
+++ b/nixos/modules/virtualisation/docker-rootless.nix
@@ -0,0 +1,102 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.virtualisation.docker.rootless;
+  proxy_env = config.networking.proxy.envVars;
+  settingsFormat = pkgs.formats.json {};
+  daemonSettingsFile = settingsFormat.generate "daemon.json" cfg.daemon.settings;
+
+in
+
+{
+  ###### interface
+
+  options.virtualisation.docker.rootless = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        This option enables docker in a rootless mode, a daemon that manages
+        linux containers. To interact with the daemon, one needs to set
+        <command>DOCKER_HOST=unix://$XDG_RUNTIME_DIR/docker.sock</command>.
+      '';
+    };
+
+    setSocketVariable = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Point <command>DOCKER_HOST</command> to rootless Docker instance for
+        normal users by default.
+      '';
+    };
+
+    daemon.settings = mkOption {
+      type = settingsFormat.type;
+      default = { };
+      example = {
+        ipv6 = true;
+        "fixed-cidr-v6" = "fd00::/80";
+      };
+      description = ''
+        Configuration for docker daemon. The attributes are serialized to JSON used as daemon.conf.
+        See https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file
+      '';
+    };
+
+    package = mkOption {
+      default = pkgs.docker;
+      defaultText = literalExpression "pkgs.docker";
+      type = types.package;
+      example = literalExpression "pkgs.docker-edge";
+      description = ''
+        Docker package to be used in the module.
+      '';
+    };
+  };
+
+  ###### implementation
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    environment.extraInit = optionalString cfg.setSocketVariable ''
+      if [ -z "$DOCKER_HOST" -a -n "$XDG_RUNTIME_DIR" ]; then
+        export DOCKER_HOST="unix://$XDG_RUNTIME_DIR/docker.sock"
+      fi
+    '';
+
+    # Taken from https://github.com/moby/moby/blob/master/contrib/dockerd-rootless-setuptool.sh
+    systemd.user.services.docker = {
+      wantedBy = [ "default.target" ];
+      description = "Docker Application Container Engine (Rootless)";
+      # needs newuidmap from pkgs.shadow
+      path = [ "/run/wrappers" ];
+      environment = proxy_env;
+      unitConfig = {
+        # docker-rootless doesn't support running as root.
+        ConditionUser = "!root";
+        StartLimitInterval = "60s";
+      };
+      serviceConfig = {
+        Type = "notify";
+        ExecStart = "${cfg.package}/bin/dockerd-rootless --config-file=${daemonSettingsFile}";
+        ExecReload = "${pkgs.procps}/bin/kill -s HUP $MAINPID";
+        TimeoutSec = 0;
+        RestartSec = 2;
+        Restart = "always";
+        StartLimitBurst = 3;
+        LimitNOFILE = "infinity";
+        LimitNPROC = "infinity";
+        LimitCORE = "infinity";
+        Delegate = true;
+        NotifyAccess = "all";
+        KillMode = "mixed";
+      };
+    };
+  };
+
+}
diff --git a/nixos/modules/virtualisation/docker.nix b/nixos/modules/virtualisation/docker.nix
index 06858e150309..a69cbe55c784 100644
--- a/nixos/modules/virtualisation/docker.nix
+++ b/nixos/modules/virtualisation/docker.nix
@@ -8,7 +8,8 @@ let
 
   cfg = config.virtualisation.docker;
   proxy_env = config.networking.proxy.envVars;
-
+  settingsFormat = pkgs.formats.json {};
+  daemonSettingsFile = settingsFormat.generate "daemon.json" cfg.daemon.settings;
 in
 
 {
@@ -52,6 +53,20 @@ in
           '';
       };
 
+    daemon.settings =
+      mkOption {
+        type = settingsFormat.type;
+        default = { };
+        example = {
+          ipv6 = true;
+          "fixed-cidr-v6" = "fd00::/80";
+        };
+        description = ''
+          Configuration for docker daemon. The attributes are serialized to JSON used as daemon.conf.
+          See https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file
+        '';
+      };
+
     enableNvidia =
       mkOption {
         type = types.bool;
@@ -171,12 +186,7 @@ in
             ""
             ''
               ${cfg.package}/bin/dockerd \
-                --group=docker \
-                --host=fd:// \
-                --log-driver=${cfg.logDriver} \
-                ${optionalString (cfg.storageDriver != null) "--storage-driver=${cfg.storageDriver}"} \
-                ${optionalString cfg.liveRestore "--live-restore" } \
-                ${optionalString cfg.enableNvidia "--add-runtime nvidia=${pkgs.nvidia-docker}/bin/nvidia-container-runtime" } \
+                --config-file=${daemonSettingsFile} \
                 ${cfg.extraOptions}
             ''];
           ExecReload=[
@@ -219,6 +229,19 @@ in
         { assertion = cfg.enableNvidia -> config.hardware.opengl.driSupport32Bit or false;
           message = "Option enableNvidia requires 32bit support libraries";
         }];
+
+      virtualisation.docker.daemon.settings = {
+        group = "docker";
+        hosts = [ "fd://" ];
+        log-driver = mkDefault cfg.logDriver;
+        storage-driver = mkIf (cfg.storageDriver != null) (mkDefault cfg.storageDriver);
+        live-restore = mkDefault cfg.liveRestore;
+        runtimes = mkIf cfg.enableNvidia {
+          nvidia = {
+            path = "${pkgs.nvidia-docker}/bin/nvidia-container-runtime";
+          };
+        };
+      };
     }
   ]);
 
diff --git a/nixos/modules/virtualisation/fetch-instance-ssh-keys.bash b/nixos/modules/virtualisation/fetch-instance-ssh-keys.bash
deleted file mode 100644
index 4a8601961115..000000000000
--- a/nixos/modules/virtualisation/fetch-instance-ssh-keys.bash
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-WGET() {
-    wget --retry-connrefused -t 15 --waitretry=10 --header='Metadata-Flavor: Google' "$@"
-}
-
-# When dealing with cryptographic keys, we want to keep things private.
-umask 077
-mkdir -p /root/.ssh
-
-echo "Fetching authorized keys..."
-WGET -O /tmp/auth_keys http://metadata.google.internal/computeMetadata/v1/instance/attributes/sshKeys
-
-# Read keys one by one, split in case Google decided
-# to append metadata (it does sometimes) and add to
-# authorized_keys if not already present.
-touch /root/.ssh/authorized_keys
-while IFS='' read -r line || [[ -n "$line" ]]; do
-    keyLine=$(echo -n "$line" | cut -d ':' -f2)
-    IFS=' ' read -r -a array <<<"$keyLine"
-    if [[ ${#array[@]} -ge 3 ]]; then
-        echo "${array[@]:0:3}" >>/tmp/new_keys
-        echo "Added ${array[*]:2} to authorized_keys"
-    fi
-done </tmp/auth_keys
-mv /tmp/new_keys /root/.ssh/authorized_keys
-chmod 600 /root/.ssh/authorized_keys
-
-echo "Fetching host keys..."
-WGET -O /tmp/ssh_host_ed25519_key http://metadata.google.internal/computeMetadata/v1/instance/attributes/ssh_host_ed25519_key
-WGET -O /tmp/ssh_host_ed25519_key.pub http://metadata.google.internal/computeMetadata/v1/instance/attributes/ssh_host_ed25519_key_pub
-mv -f /tmp/ssh_host_ed25519_key* /etc/ssh/
-chmod 600 /etc/ssh/ssh_host_ed25519_key
-chmod 644 /etc/ssh/ssh_host_ed25519_key.pub
diff --git a/nixos/modules/virtualisation/google-compute-config.nix b/nixos/modules/virtualisation/google-compute-config.nix
index cff48d20b2b9..44d2a589511f 100644
--- a/nixos/modules/virtualisation/google-compute-config.nix
+++ b/nixos/modules/virtualisation/google-compute-config.nix
@@ -1,8 +1,5 @@
 { config, lib, pkgs, ... }:
 with lib;
-let
-  gce = pkgs.google-compute-engine;
-in
 {
   imports = [
     ../profiles/headless.nix
@@ -40,7 +37,8 @@ in
   security.googleOsLogin.enable = true;
 
   # Use GCE udev rules for dynamic disk volumes
-  services.udev.packages = [ gce ];
+  services.udev.packages = [ pkgs.google-guest-configs ];
+  services.udev.path = [ pkgs.google-guest-configs ];
 
   # Force getting the hostname from Google Compute.
   networking.hostName = mkDefault "";
@@ -48,12 +46,6 @@ in
   # Always include cryptsetup so that NixOps can use it.
   environment.systemPackages = [ pkgs.cryptsetup ];
 
-  # Make sure GCE image does not replace host key that NixOps sets
-  environment.etc."default/instance_configs.cfg".text = lib.mkDefault ''
-    [InstanceSetup]
-    set_host_keys = false
-  '';
-
   # Rely on GCP's firewall instead
   networking.firewall.enable = mkDefault false;
 
@@ -69,105 +61,42 @@ in
   # GC has 1460 MTU
   networking.interfaces.eth0.mtu = 1460;
 
-  # Used by NixOps
-  systemd.services.fetch-instance-ssh-keys = {
-    description = "Fetch host keys and authorized_keys for root user";
-
-    wantedBy = [ "sshd.service" ];
-    before = [ "sshd.service" ];
-    after = [ "network-online.target" ];
-    wants = [ "network-online.target" ];
-    path = [ pkgs.wget ];
-
-    serviceConfig = {
-      Type = "oneshot";
-      ExecStart = pkgs.runCommand "fetch-instance-ssh-keys" { } ''
-        cp ${./fetch-instance-ssh-keys.bash} $out
-        chmod +x $out
-        ${pkgs.shfmt}/bin/shfmt -i 4 -d $out
-        ${pkgs.shellcheck}/bin/shellcheck $out
-        patchShebangs $out
-      '';
-      PrivateTmp = true;
-      StandardError = "journal+console";
-      StandardOutput = "journal+console";
-    };
+  systemd.packages = [ pkgs.google-guest-agent ];
+  systemd.services.google-guest-agent = {
+    wantedBy = [ "multi-user.target" ];
+    restartTriggers = [ config.environment.etc."default/instance_configs.cfg".source ];
+    path = lib.optional config.users.mutableUsers pkgs.shadow;
   };
+  systemd.services.google-startup-scripts.wantedBy = [ "multi-user.target" ];
+  systemd.services.google-shutdown-scripts.wantedBy = [ "multi-user.target" ];
 
-  systemd.services.google-instance-setup = {
-    description = "Google Compute Engine Instance Setup";
-    after = [ "network-online.target" "network.target" "rsyslog.service" ];
-    before = [ "sshd.service" ];
-    path = with pkgs; [ coreutils ethtool openssh ];
-    serviceConfig = {
-      ExecStart = "${gce}/bin/google_instance_setup";
-      StandardOutput="journal+console";
-      Type = "oneshot";
-    };
-    wantedBy = [ "sshd.service" "multi-user.target" ];
-  };
+  security.sudo.extraRules = mkIf config.users.mutableUsers [
+    { groups = [ "google-sudoers" ]; commands = [ { command = "ALL"; options = [ "NOPASSWD" ]; } ]; }
+  ];
 
-  systemd.services.google-network-daemon = {
-    description = "Google Compute Engine Network Daemon";
-    after = [ "network-online.target" "network.target" "google-instance-setup.service" ];
-    path = with pkgs; [ iproute2 ];
-    serviceConfig = {
-      ExecStart = "${gce}/bin/google_network_daemon";
-      StandardOutput="journal+console";
-      Type="simple";
-    };
-    wantedBy = [ "multi-user.target" ];
-  };
+  users.groups.google-sudoers = mkIf config.users.mutableUsers { };
 
-  systemd.services.google-clock-skew-daemon = {
-    description = "Google Compute Engine Clock Skew Daemon";
-    after = [ "network.target" "google-instance-setup.service" "google-network-daemon.service" ];
-    serviceConfig = {
-      ExecStart = "${gce}/bin/google_clock_skew_daemon";
-      StandardOutput="journal+console";
-      Type = "simple";
-    };
-    wantedBy = ["multi-user.target"];
-  };
+  boot.extraModprobeConfig = lib.readFile "${pkgs.google-guest-configs}/etc/modprobe.d/gce-blacklist.conf";
 
+  environment.etc."sysctl.d/60-gce-network-security.conf".source = "${pkgs.google-guest-configs}/etc/sysctl.d/60-gce-network-security.conf";
 
-  systemd.services.google-shutdown-scripts = {
-    description = "Google Compute Engine Shutdown Scripts";
-    after = [
-      "network-online.target"
-      "network.target"
-      "rsyslog.service"
-      "google-instance-setup.service"
-      "google-network-daemon.service"
-    ];
-    serviceConfig = {
-      ExecStart = "${pkgs.coreutils}/bin/true";
-      ExecStop = "${gce}/bin/google_metadata_script_runner --script-type shutdown";
-      RemainAfterExit = true;
-      StandardOutput="journal+console";
-      TimeoutStopSec = "0";
-      Type = "oneshot";
-    };
-    wantedBy = [ "multi-user.target" ];
-  };
+  environment.etc."default/instance_configs.cfg".text = ''
+    [Accounts]
+    useradd_cmd = useradd -m -s /run/current-system/sw/bin/bash -p * {user}
 
-  systemd.services.google-startup-scripts = {
-    description = "Google Compute Engine Startup Scripts";
-    after = [
-      "network-online.target"
-      "network.target"
-      "rsyslog.service"
-      "google-instance-setup.service"
-      "google-network-daemon.service"
-    ];
-    serviceConfig = {
-      ExecStart = "${gce}/bin/google_metadata_script_runner --script-type startup";
-      KillMode = "process";
-      StandardOutput = "journal+console";
-      Type = "oneshot";
-    };
-    wantedBy = [ "multi-user.target" ];
-  };
+    [Daemons]
+    accounts_daemon = ${boolToString config.users.mutableUsers}
 
-  environment.etc."sysctl.d/11-gce-network-security.conf".source = "${gce}/sysctl.d/11-gce-network-security.conf";
+    [InstanceSetup]
+    # Make sure GCE image does not replace host key that NixOps sets.
+    set_host_keys = false
+
+    [MetadataScripts]
+    default_shell = ${pkgs.stdenv.shell}
+
+    [NetworkInterfaces]
+    dhclient_script = ${pkgs.google-guest-configs}/bin/google-dhclient-script
+    # We set up network interfaces declaratively.
+    setup = false
+  '';
 }
diff --git a/nixos/modules/virtualisation/kubevirt.nix b/nixos/modules/virtualisation/kubevirt.nix
new file mode 100644
index 000000000000..408822b6af0b
--- /dev/null
+++ b/nixos/modules/virtualisation/kubevirt.nix
@@ -0,0 +1,30 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    ../profiles/qemu-guest.nix
+  ];
+
+  config = {
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+      autoResize = true;
+    };
+
+    boot.growPartition = true;
+    boot.kernelParams = [ "console=ttyS0" ];
+    boot.loader.grub.device = "/dev/vda";
+    boot.loader.timeout = 0;
+
+    services.qemuGuest.enable = true;
+    services.openssh.enable = true;
+    services.cloud-init.enable = true;
+    systemd.services."serial-getty@ttyS0".enable = true;
+
+    system.build.kubevirtImage = import ../../lib/make-disk-image.nix {
+      inherit lib config pkgs;
+      format = "qcow2";
+    };
+  };
+}
diff --git a/nixos/modules/virtualisation/kvmgt.nix b/nixos/modules/virtualisation/kvmgt.nix
index 72bd2c24e566..5e7a73bec902 100644
--- a/nixos/modules/virtualisation/kvmgt.nix
+++ b/nixos/modules/virtualisation/kvmgt.nix
@@ -82,5 +82,5 @@ in {
     };
   };
 
-  meta.maintainers = with maintainers; [ ];
+  meta.maintainers = with maintainers; [ patryk27 ];
 }
diff --git a/nixos/modules/virtualisation/oci-containers.nix b/nixos/modules/virtualisation/oci-containers.nix
index 5af9baff8bc1..f40481727830 100644
--- a/nixos/modules/virtualisation/oci-containers.nix
+++ b/nixos/modules/virtualisation/oci-containers.nix
@@ -22,11 +22,13 @@ let
           type = with types; nullOr package;
           default = null;
           description = ''
-            Path to an image file to load instead of pulling from a registry.
-            If defined, do not pull from registry.
+            Path to an image file to load before running the image. This can
+            be used to bypass pulling the image from the registry.
 
-            You still need to set the <literal>image</literal> attribute, as it
-            will be used as the image name for docker to start a container.
+            The <literal>image</literal> attribute must match the name and
+            tag of the image contained in this file, as they will be used to
+            run the container with that image. If they do not match, the
+            image will be pulled from the registry as usual.
           '';
           example = literalExpression "pkgs.dockerTools.buildImage {...};";
         };
diff --git a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix
index 133cd4c0e9f9..25104bb47667 100644
--- a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix
+++ b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix
@@ -15,7 +15,8 @@
   }
 
   wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
-  (umask 077 && wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data)
+  # When no user-data is provided, the OpenStack metadata server doesn't expose the user-data route.
+  (umask 077 && wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data || rm -f "$metaDir/user-data")
   wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
   wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
 ''
diff --git a/nixos/modules/virtualisation/openvswitch.nix b/nixos/modules/virtualisation/openvswitch.nix
index 325f6f5b43f4..436a375fb5eb 100644
--- a/nixos/modules/virtualisation/openvswitch.nix
+++ b/nixos/modules/virtualisation/openvswitch.nix
@@ -36,17 +36,6 @@ in {
         Open vSwitch package to use.
       '';
     };
-
-    ipsec = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Whether to start racoon service for openvswitch.
-        Supported only if openvswitch version is less than 2.6.0.
-        Use <literal>virtualisation.vswitch.package = pkgs.openvswitch-lts</literal>
-        for a version that supports ipsec over GRE.
-      '';
-    };
   };
 
   config = mkIf cfg.enable (let
@@ -65,7 +54,7 @@ in {
       installPhase = "mkdir -p $out";
     };
 
-  in (mkMerge [{
+  in {
     environment.systemPackages = [ cfg.package ];
     boot.kernelModules = [ "tun" "openvswitch" ];
 
@@ -142,48 +131,14 @@ in {
       };
     };
 
-  }
-  (mkIf (cfg.ipsec && (versionOlder cfg.package.version "2.6.0")) {
-    environment.systemPackages = [ pkgs.ipsecTools ];
-
-    services.racoon.enable = true;
-    services.racoon.configPath = "${runDir}/ipsec/etc/racoon/racoon.conf";
+  });
 
-    networking.firewall.extraCommands = ''
-      iptables -I INPUT -t mangle -p esp -j MARK --set-mark 1/1
-      iptables -I INPUT -t mangle -p udp --dport 4500 -j MARK --set-mark 1/1
-    '';
-
-    systemd.services.ovs-monitor-ipsec = {
-      description = "Open_vSwitch Ipsec Daemon";
-      wantedBy = [ "multi-user.target" ];
-      requires = [ "ovsdb.service" ];
-      before = [ "vswitchd.service" "racoon.service" ];
-      environment.UNIXCTLPATH = "/tmp/ovsdb.ctl.sock";
-      serviceConfig = {
-        ExecStart = ''
-          ${cfg.package}/bin/ovs-monitor-ipsec \
-            --root-prefix ${runDir}/ipsec \
-            --pidfile /run/openvswitch/ovs-monitor-ipsec.pid \
-            --monitor --detach \
-            unix:/run/openvswitch/db.sock
-        '';
-        PIDFile = "/run/openvswitch/ovs-monitor-ipsec.pid";
-        # Use service type 'forking' to correctly determine when ovs-monitor-ipsec is ready.
-        Type = "forking";
-      };
-
-      preStart = ''
-        rm -r ${runDir}/ipsec/etc/racoon/certs || true
-        mkdir -p ${runDir}/ipsec/{etc/racoon,etc/init.d/,usr/sbin/}
-        ln -fs ${pkgs.ipsecTools}/bin/setkey ${runDir}/ipsec/usr/sbin/setkey
-        ln -fs ${pkgs.writeScript "racoon-restart" ''
-        #!${pkgs.runtimeShell}
-        /run/current-system/sw/bin/systemctl $1 racoon
-        ''} ${runDir}/ipsec/etc/init.d/racoon
-      '';
-    };
-  })]));
+  imports = [
+    (mkRemovedOptionModule [ "virtualisation" "vswitch" "ipsec" ] ''
+      OpenVSwitch IPSec functionality has been removed, because it depended on racoon,
+      which was removed from nixpkgs, because it was abanoded upstream.
+    '')
+  ];
 
   meta.maintainers = with maintainers; [ netixx ];
 
diff --git a/nixos/modules/virtualisation/proxmox-lxc.nix b/nixos/modules/virtualisation/proxmox-lxc.nix
new file mode 100644
index 000000000000..3913b474afbe
--- /dev/null
+++ b/nixos/modules/virtualisation/proxmox-lxc.nix
@@ -0,0 +1,64 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+{
+  options.proxmoxLXC = {
+    privileged = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to enable privileged mounts
+      '';
+    };
+    manageNetwork = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to manage network interfaces through nix options
+        When false, systemd-networkd is enabled to accept network
+        configuration from proxmox.
+      '';
+    };
+  };
+
+  config =
+    let
+      cfg = config.proxmoxLXC;
+    in
+    {
+      system.build.tarball = pkgs.callPackage ../../lib/make-system-tarball.nix {
+        storeContents = [{
+          object = config.system.build.toplevel;
+          symlink = "none";
+        }];
+
+        contents = [{
+          source = config.system.build.toplevel + "/init";
+          target = "/sbin/init";
+        }];
+
+        extraCommands = "mkdir -p root etc/systemd/network";
+      };
+
+      boot = {
+        isContainer = true;
+        loader.initScript.enable = true;
+      };
+
+      networking = mkIf (!cfg.manageNetwork) {
+        useDHCP = false;
+        useHostResolvConf = false;
+        useNetworkd = true;
+      };
+
+      services.openssh = {
+        enable = mkDefault true;
+        startWhenNeeded = mkDefault true;
+      };
+
+      systemd.mounts = mkIf (!cfg.privileged)
+        [{ where = "/sys/kernel/debug"; enable = false; }];
+
+    };
+}
diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix
index fa3e25afb03e..514389358947 100644
--- a/nixos/modules/virtualisation/qemu-vm.nix
+++ b/nixos/modules/virtualisation/qemu-vm.nix
@@ -632,6 +632,15 @@ in
             Enable the Qemu guest agent.
           '';
         };
+
+      virtioKeyboard =
+        mkOption {
+          type = types.bool;
+          default = true;
+          description = ''
+            Enable the virtio-keyboard device.
+          '';
+        };
     };
 
     virtualisation.useNixStoreImage =
@@ -835,7 +844,9 @@ in
 
     # FIXME: Consolidate this one day.
     virtualisation.qemu.options = mkMerge [
-      [ "-device virtio-keyboard" ]
+      (mkIf cfg.qemu.virtioKeyboard [
+        "-device virtio-keyboard"
+      ])
       (mkIf pkgs.stdenv.hostPlatform.isx86 [
         "-usb" "-device usb-tablet,bus=usb-bus.0"
       ])
@@ -999,4 +1010,7 @@ in
       ];
 
   };
+
+  # uses types of services/x11/xserver.nix
+  meta.buildDocsInSandbox = false;
 }
diff --git a/nixos/modules/virtualisation/virtualbox-guest.nix b/nixos/modules/virtualisation/virtualbox-guest.nix
index f702fb4e525c..7b55b3b9759e 100644
--- a/nixos/modules/virtualisation/virtualbox-guest.nix
+++ b/nixos/modules/virtualisation/virtualbox-guest.nix
@@ -68,7 +68,7 @@ in
         SUBSYSTEM=="misc", KERNEL=="vboxguest", TAG+="systemd"
       '';
   } (mkIf cfg.x11 {
-    services.xserver.videoDrivers = mkOverride 50 [ "vmware" "virtualbox" "modesetting" ];
+    services.xserver.videoDrivers = [ "vmware" "virtualbox" "modesetting" ];
 
     services.xserver.config =
       ''
diff --git a/nixos/modules/virtualisation/vmware-guest.nix b/nixos/modules/virtualisation/vmware-guest.nix
index 481dedf84054..3caed746ca91 100644
--- a/nixos/modules/virtualisation/vmware-guest.nix
+++ b/nixos/modules/virtualisation/vmware-guest.nix
@@ -27,6 +27,7 @@ in
       message = "VMWare guest is not currently supported on ${pkgs.stdenv.hostPlatform.system}";
     } ];
 
+    boot.initrd.availableKernelModules = [ "mptspi" ];
     boot.initrd.kernelModules = [ "vmw_pvscsi" ];
 
     environment.systemPackages = [ open-vm-tools ];
diff --git a/nixos/modules/virtualisation/xen-dom0.nix b/nixos/modules/virtualisation/xen-dom0.nix
index f8f4af4f6b85..975eed10cd26 100644
--- a/nixos/modules/virtualisation/xen-dom0.nix
+++ b/nixos/modules/virtualisation/xen-dom0.nix
@@ -450,5 +450,4 @@ in
     };
 
   };
-
 }
diff --git a/nixos/release-combined.nix b/nixos/release-combined.nix
index ee3f3d19174e..fd8a39cfb92b 100644
--- a/nixos/release-combined.nix
+++ b/nixos/release-combined.nix
@@ -98,7 +98,6 @@ in rec {
         (onFullSupported "nixos.tests.login")
         (onFullSupported "nixos.tests.misc")
         (onFullSupported "nixos.tests.mutableUsers")
-        (onFullSupported "nixos.tests.nano")
         (onFullSupported "nixos.tests.nat.firewall-conntrack")
         (onFullSupported "nixos.tests.nat.firewall")
         (onFullSupported "nixos.tests.nat.standalone")
@@ -130,7 +129,8 @@ in rec {
         (onFullSupported "nixos.tests.networking.networkd.virtual")
         (onFullSupported "nixos.tests.networking.networkd.vlan")
         (onFullSupported "nixos.tests.systemd-networkd-ipv6-prefix-delegation")
-        (onFullSupported "nixos.tests.nfs3.simple")
+        # fails with kernel >= 5.15 https://github.com/NixOS/nixpkgs/pull/152505#issuecomment-1005049314
+        #(onFullSupported "nixos.tests.nfs3.simple")
         (onFullSupported "nixos.tests.nfs4.simple")
         (onFullSupported "nixos.tests.openssh")
         (onFullSupported "nixos.tests.pantheon")
diff --git a/nixos/release-small.nix b/nixos/release-small.nix
index 996db54c9a40..1d51b4e7f28f 100644
--- a/nixos/release-small.nix
+++ b/nixos/release-small.nix
@@ -38,7 +38,8 @@ in rec {
         login
         misc
         nat
-        nfs3
+        # fails with kernel >= 5.15 https://github.com/NixOS/nixpkgs/pull/152505#issuecomment-1005049314
+        #nfs3
         openssh
         php
         predictable-interface-names
@@ -107,7 +108,8 @@ in rec {
         "nixos.tests.nat.firewall-conntrack.x86_64-linux"
         "nixos.tests.nat.firewall.x86_64-linux"
         "nixos.tests.nat.standalone.x86_64-linux"
-        "nixos.tests.nfs3.simple.x86_64-linux"
+        # fails with kernel >= 5.15 https://github.com/NixOS/nixpkgs/pull/152505#issuecomment-1005049314
+        #"nixos.tests.nfs3.simple.x86_64-linux"
         "nixos.tests.openssh.x86_64-linux"
         "nixos.tests.php.fpm.x86_64-linux"
         "nixos.tests.php.pcre.x86_64-linux"
diff --git a/nixos/tests/acme.nix b/nixos/tests/acme.nix
index 72b7bb8a396a..2dd06a50f40b 100644
--- a/nixos/tests/acme.nix
+++ b/nixos/tests/acme.nix
@@ -1,9 +1,9 @@
-let
+import ./make-test-python.nix ({ pkgs, lib, ... }: let
   commonConfig = ./common/acme/client;
 
   dnsServerIP = nodes: nodes.dnsserver.config.networking.primaryIPAddress;
 
-  dnsScript = {pkgs, nodes}: let
+  dnsScript = nodes: let
     dnsAddress = dnsServerIP nodes;
   in pkgs.writeShellScript "dns-hook.sh" ''
     set -euo pipefail
@@ -15,30 +15,137 @@ let
     fi
   '';
 
-  documentRoot = pkgs: pkgs.runCommand "docroot" {} ''
+  dnsConfig = nodes: {
+    dnsProvider = "exec";
+    dnsPropagationCheck = false;
+    credentialsFile = pkgs.writeText "wildcard.env" ''
+      EXEC_PATH=${dnsScript nodes}
+      EXEC_POLLING_INTERVAL=1
+      EXEC_PROPAGATION_TIMEOUT=1
+      EXEC_SEQUENCE_INTERVAL=1
+    '';
+  };
+
+  documentRoot = pkgs.runCommand "docroot" {} ''
     mkdir -p "$out"
     echo hello world > "$out/index.html"
   '';
 
-  vhostBase = pkgs: {
+  vhostBase = {
     forceSSL = true;
-    locations."/".root = documentRoot pkgs;
+    locations."/".root = documentRoot;
+  };
+
+  vhostBaseHttpd = {
+    forceSSL = true;
+    inherit documentRoot;
+  };
+
+  # Base specialisation config for testing general ACME features
+  webserverBasicConfig = {
+    services.nginx.enable = true;
+    services.nginx.virtualHosts."a.example.test" = vhostBase // {
+      enableACME = true;
+    };
   };
 
-in import ./make-test-python.nix ({ lib, ... }: {
+  # Generate specialisations for testing a web server
+  mkServerConfigs = { server, group, vhostBaseData, extraConfig ? {} }: let
+    baseConfig = { nodes, config, specialConfig ? {} }: lib.mkMerge [
+      {
+        security.acme = {
+          defaults = (dnsConfig nodes);
+          # One manual wildcard cert
+          certs."example.test" = {
+            domain = "*.example.test";
+          };
+        };
+
+        users.users."${config.services."${server}".user}".extraGroups = ["acme"];
+
+        services."${server}" = {
+          enable = true;
+          virtualHosts = {
+            # Run-of-the-mill vhost using HTTP-01 validation
+            "${server}-http.example.test" = vhostBaseData // {
+              serverAliases = [ "${server}-http-alias.example.test" ];
+              enableACME = true;
+            };
+
+            # Another which inherits the DNS-01 config
+            "${server}-dns.example.test" = vhostBaseData // {
+              serverAliases = [ "${server}-dns-alias.example.test" ];
+              enableACME = true;
+              # Set acmeRoot to null instead of using the default of "/var/lib/acme/acme-challenge"
+              # webroot + dnsProvider are mutually exclusive.
+              acmeRoot = null;
+            };
+
+            # One using the wildcard certificate
+            "${server}-wildcard.example.test" = vhostBaseData // {
+              serverAliases = [ "${server}-wildcard-alias.example.test" ];
+              useACMEHost = "example.test";
+            };
+          };
+        };
+
+        # Used to determine if service reload was triggered
+        systemd.targets."test-renew-${server}" = {
+          wants = [ "acme-${server}-http.example.test.service" ];
+          after = [ "acme-${server}-http.example.test.service" "${server}-config-reload.service" ];
+        };
+      }
+      specialConfig
+      extraConfig
+    ];
+  in {
+    "${server}".configuration = { nodes, config, ... }: baseConfig {
+      inherit nodes config;
+    };
+
+    # Test that server reloads when an alias is removed (and subsequently test removal works in acme)
+    "${server}-remove-alias".configuration = { nodes, config, ... }: baseConfig {
+      inherit nodes config;
+      specialConfig = {
+        # Remove an alias, but create a standalone vhost in its place for testing.
+        # This configuration results in certificate errors as useACMEHost does not imply
+        # append extraDomains, and thus we can validate the SAN is removed.
+        services."${server}" = {
+          virtualHosts."${server}-http.example.test".serverAliases = lib.mkForce [];
+          virtualHosts."${server}-http-alias.example.test" = vhostBaseData // {
+            useACMEHost = "${server}-http.example.test";
+          };
+        };
+      };
+    };
+
+    # Test that the server reloads when only the acme configuration is changed.
+    "${server}-change-acme-conf".configuration = { nodes, config, ... }: baseConfig {
+      inherit nodes config;
+      specialConfig = {
+        security.acme.certs."${server}-http.example.test" = {
+          keyType = "ec384";
+          # Also test that postRun is exec'd as root
+          postRun = "id | grep root";
+        };
+      };
+    };
+  };
+
+in {
   name = "acme";
   meta.maintainers = lib.teams.acme.members;
 
   nodes = {
     # The fake ACME server which will respond to client requests
-    acme = { nodes, lib, ... }: {
+    acme = { nodes, ... }: {
       imports = [ ./common/acme/server ];
       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
     };
 
     # A fake DNS server which can be configured with records as desired
     # Used to test DNS-01 challenge
-    dnsserver = { nodes, pkgs, ... }: {
+    dnsserver = { nodes, ... }: {
       networking.firewall.allowedTCPPorts = [ 8055 53 ];
       networking.firewall.allowedUDPPorts = [ 53 ];
       systemd.services.pebble-challtestsrv = {
@@ -54,7 +161,7 @@ in import ./make-test-python.nix ({ lib, ... }: {
     };
 
     # A web server which will be the node requesting certs
-    webserver = { pkgs, nodes, lib, config, ... }: {
+    webserver = { nodes, config, ... }: {
       imports = [ commonConfig ];
       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
       networking.firewall.allowedTCPPorts = [ 80 443 ];
@@ -63,130 +170,142 @@ in import ./make-test-python.nix ({ lib, ... }: {
       environment.systemPackages = [ pkgs.openssl ];
 
       # Set log level to info so that we can see when the service is reloaded
-      services.nginx.enable = true;
       services.nginx.logError = "stderr info";
 
-      # First tests configure a basic cert and run a bunch of openssl checks
-      services.nginx.virtualHosts."a.example.test" = (vhostBase pkgs) // {
-        enableACME = true;
-      };
-
-      # Used to determine if service reload was triggered
-      systemd.targets.test-renew-nginx = {
-        wants = [ "acme-a.example.test.service" ];
-        after = [ "acme-a.example.test.service" "nginx-config-reload.service" ];
-      };
-
-      # Test that account creation is collated into one service
-      specialisation.account-creation.configuration = { nodes, pkgs, lib, ... }: let
-        email = "newhostmaster@example.test";
-        caDomain = nodes.acme.config.test-support.acme.caDomain;
-        # Exit 99 to make it easier to track if this is the reason a renew failed
-        testScript = ''
-          test -e accounts/${caDomain}/${email}/account.json || exit 99
-        '';
-      in {
-        security.acme.email = lib.mkForce email;
-        systemd.services."b.example.test".preStart = testScript;
-        systemd.services."c.example.test".preStart = testScript;
-
-        services.nginx.virtualHosts."b.example.test" = (vhostBase pkgs) // {
-          enableACME = true;
-        };
-        services.nginx.virtualHosts."c.example.test" = (vhostBase pkgs) // {
-          enableACME = true;
-        };
-      };
-
-      # Cert config changes will not cause the nginx configuration to change.
-      # This tests that the reload service is correctly triggered.
-      # It also tests that postRun is exec'd as root
-      specialisation.cert-change.configuration = { pkgs, ... }: {
-        security.acme.certs."a.example.test".keyType = "ec384";
-        security.acme.certs."a.example.test".postRun = ''
-          set -euo pipefail
-          touch /home/test
-          chown root:root /home/test
-          echo testing > /home/test
-        '';
-      };
-
-      # Now adding an alias to ensure that the certs are updated
-      specialisation.nginx-aliases.configuration = { pkgs, ... }: {
-        services.nginx.virtualHosts."a.example.test" = {
-          serverAliases = [ "b.example.test" ];
-        };
-      };
-
-      # Test OCSP Stapling
-      specialisation.ocsp-stapling.configuration = { pkgs, ... }: {
-        security.acme.certs."a.example.test" = {
-          ocspMustStaple = true;
-        };
-        services.nginx.virtualHosts."a.example.com" = {
-          extraConfig = ''
-            ssl_stapling on;
-            ssl_stapling_verify on;
+      specialisation = {
+        # First derivation used to test general ACME features
+        general.configuration = { ... }: let
+          caDomain = nodes.acme.config.test-support.acme.caDomain;
+          email = config.security.acme.defaults.email;
+          # Exit 99 to make it easier to track if this is the reason a renew failed
+          accountCreateTester = ''
+            test -e accounts/${caDomain}/${email}/account.json || exit 99
           '';
-        };
-      };
-
-      # Test using Apache HTTPD
-      specialisation.httpd-aliases.configuration = { pkgs, config, lib, ... }: {
-        services.nginx.enable = lib.mkForce false;
-        services.httpd.enable = true;
-        services.httpd.adminAddr = config.security.acme.email;
-        services.httpd.virtualHosts."c.example.test" = {
-          serverAliases = [ "d.example.test" ];
-          forceSSL = true;
-          enableACME = true;
-          documentRoot = documentRoot pkgs;
-        };
-
-        # Used to determine if service reload was triggered
-        systemd.targets.test-renew-httpd = {
-          wants = [ "acme-c.example.test.service" ];
-          after = [ "acme-c.example.test.service" "httpd-config-reload.service" ];
-        };
-      };
-
-      # Validation via DNS-01 challenge
-      specialisation.dns-01.configuration = { pkgs, config, nodes, ... }: {
-        security.acme.certs."example.test" = {
-          domain = "*.example.test";
-          group = config.services.nginx.group;
-          dnsProvider = "exec";
-          dnsPropagationCheck = false;
-          credentialsFile = pkgs.writeText "wildcard.env" ''
-            EXEC_PATH=${dnsScript { inherit pkgs nodes; }}
-          '';
-        };
-
-        services.nginx.virtualHosts."dns.example.test" = (vhostBase pkgs) // {
-          useACMEHost = "example.test";
-        };
-      };
-
-      # Validate service relationships by adding a slow start service to nginx' wants.
-      # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
-      specialisation.slow-startup.configuration = { pkgs, config, nodes, lib, ... }: {
-        systemd.services.my-slow-service = {
-          wantedBy = [ "multi-user.target" "nginx.service" ];
-          before = [ "nginx.service" ];
-          preStart = "sleep 5";
-          script = "${pkgs.python3}/bin/python -m http.server";
+        in lib.mkMerge [
+          webserverBasicConfig
+          {
+            # Used to test that account creation is collated into one service.
+            # These should not run until after acme-finished-a.example.test.target
+            systemd.services."b.example.test".preStart = accountCreateTester;
+            systemd.services."c.example.test".preStart = accountCreateTester;
+
+            services.nginx.virtualHosts."b.example.test" = vhostBase // {
+              enableACME = true;
+            };
+            services.nginx.virtualHosts."c.example.test" = vhostBase // {
+              enableACME = true;
+            };
+          }
+        ];
+
+        # Test OCSP Stapling
+        ocsp-stapling.configuration = { ... }: lib.mkMerge [
+          webserverBasicConfig
+          {
+            security.acme.certs."a.example.test".ocspMustStaple = true;
+            services.nginx.virtualHosts."a.example.test" = {
+              extraConfig = ''
+                ssl_stapling on;
+                ssl_stapling_verify on;
+              '';
+            };
+          }
+        ];
+
+        # Validate service relationships by adding a slow start service to nginx' wants.
+        # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
+        slow-startup.configuration = { ... }: lib.mkMerge [
+          webserverBasicConfig
+          {
+            systemd.services.my-slow-service = {
+              wantedBy = [ "multi-user.target" "nginx.service" ];
+              before = [ "nginx.service" ];
+              preStart = "sleep 5";
+              script = "${pkgs.python3}/bin/python -m http.server";
+            };
+
+            services.nginx.virtualHosts."slow.example.test" = {
+              forceSSL = true;
+              enableACME = true;
+              locations."/".proxyPass = "http://localhost:8000";
+            };
+          }
+        ];
+
+        # Test lego internal server (listenHTTP option)
+        # Also tests useRoot option
+        lego-server.configuration = { ... }: {
+          security.acme.useRoot = true;
+          security.acme.certs."lego.example.test" = {
+            listenHTTP = ":80";
+            group = "nginx";
+          };
+          services.nginx.enable = true;
+          services.nginx.virtualHosts."lego.example.test" = {
+            useACMEHost = "lego.example.test";
+            onlySSL = true;
+          };
         };
 
-        services.nginx.virtualHosts."slow.example.com" = {
-          forceSSL = true;
-          enableACME = true;
-          locations."/".proxyPass = "http://localhost:8000";
+      # Test compatiblity with Caddy
+      # It only supports useACMEHost, hence not using mkServerConfigs
+      } // (let
+        baseCaddyConfig = { nodes, config, ... }: {
+          security.acme = {
+            defaults = (dnsConfig nodes);
+            # One manual wildcard cert
+            certs."example.test" = {
+              domain = "*.example.test";
+            };
+          };
+
+          users.users."${config.services.caddy.user}".extraGroups = ["acme"];
+
+          services.caddy = {
+            enable = true;
+            virtualHosts."a.exmaple.test" = {
+              useACMEHost = "example.test";
+              extraConfig = ''
+                root * ${documentRoot}
+              '';
+            };
+          };
         };
-      };
+      in {
+        caddy.configuration = baseCaddyConfig;
+
+        # Test that the server reloads when only the acme configuration is changed.
+        "caddy-change-acme-conf".configuration = { nodes, config, ... }: lib.mkMerge [
+          (baseCaddyConfig {
+            inherit nodes config;
+          })
+          {
+            security.acme.certs."example.test" = {
+              keyType = "ec384";
+            };
+          }
+        ];
+
+      # Test compatibility with Nginx
+      }) // (mkServerConfigs {
+          server = "nginx";
+          group = "nginx";
+          vhostBaseData = vhostBase;
+        })
+
+      # Test compatibility with Apache HTTPD
+        // (mkServerConfigs {
+          server = "httpd";
+          group = "wwwrun";
+          vhostBaseData = vhostBaseHttpd;
+          extraConfig = {
+            services.httpd.adminAddr = config.security.acme.defaults.email;
+          };
+        });
     };
 
     # The client will be used to curl the webserver to validate configuration
-    client = {nodes, lib, pkgs, ...}: {
+    client = { nodes, ... }: {
       imports = [ commonConfig ];
       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 
@@ -195,7 +314,7 @@ in import ./make-test-python.nix ({ lib, ... }: {
     };
   };
 
-  testScript = {nodes, ...}:
+  testScript = { nodes, ... }:
     let
       caDomain = nodes.acme.config.test-support.acme.caDomain;
       newServerSystem = nodes.webserver.config.system.build.toplevel;
@@ -204,23 +323,26 @@ in import ./make-test-python.nix ({ lib, ... }: {
     # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true,
     # this is because a oneshot goes from inactive => activating => inactive, and never
     # reaches the active state. Targets do not have this issue.
-
     ''
       import time
 
 
-      has_switched = False
+      def switch_to(node, name):
+          # On first switch, this will create a symlink to the current system so that we can
+          # quickly switch between derivations
+          root_specs = "/tmp/specialisation"
+          node.execute(
+            f"test -e {root_specs}"
+            f" || ln -s $(readlink /run/current-system)/specialisation {root_specs}"
+          )
 
+          switcher_path = f"/run/current-system/specialisation/{name}/bin/switch-to-configuration"
+          rc, _ = node.execute(f"test -e '{switcher_path}'")
+          if rc > 0:
+              switcher_path = f"/tmp/specialisation/{name}/bin/switch-to-configuration"
 
-      def switch_to(node, name):
-          global has_switched
-          if has_switched:
-              node.succeed(
-                  "${switchToNewServer}"
-              )
-          has_switched = True
           node.succeed(
-              f"/run/current-system/specialisation/{name}/bin/switch-to-configuration test"
+              f"{switcher_path} test"
           )
 
 
@@ -310,8 +432,7 @@ in import ./make-test-python.nix ({ lib, ... }: {
               return download_ca_certs(node, retries - 1)
 
 
-      client.start()
-      dnsserver.start()
+      start_all()
 
       dnsserver.wait_for_unit("pebble-challtestsrv.service")
       client.wait_for_unit("default.target")
@@ -320,19 +441,30 @@ in import ./make-test-python.nix ({ lib, ... }: {
           'curl --data \'{"host": "${caDomain}", "addresses": ["${nodes.acme.config.networking.primaryIPAddress}"]}\' http://${dnsServerIP nodes}:8055/add-a'
       )
 
-      acme.start()
-      webserver.start()
-
       acme.wait_for_unit("network-online.target")
       acme.wait_for_unit("pebble.service")
 
       download_ca_certs(client)
 
-      with subtest("Can request certificate with HTTPS-01 challenge"):
+      # Perform general tests first
+      switch_to(webserver, "general")
+
+      with subtest("Can request certificate with HTTP-01 challenge"):
           webserver.wait_for_unit("acme-finished-a.example.test.target")
+          check_fullchain(webserver, "a.example.test")
+          check_issuer(webserver, "a.example.test", "pebble")
+          webserver.wait_for_unit("nginx.service")
+          check_connection(client, "a.example.test")
+
+      with subtest("Runs 1 cert for account creation before others"):
+          webserver.wait_for_unit("acme-finished-b.example.test.target")
+          webserver.wait_for_unit("acme-finished-c.example.test.target")
+          check_connection(client, "b.example.test")
+          check_connection(client, "c.example.test")
 
       with subtest("Certificates and accounts have safe + valid permissions"):
-          group = "${nodes.webserver.config.security.acme.certs."a.example.test".group}"
+          # Nginx will set the group appropriately when enableACME is used
+          group = "nginx"
           webserver.succeed(
               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
           )
@@ -346,12 +478,6 @@ in import ./make-test-python.nix ({ lib, ... }: {
               f"test $(find /var/lib/acme/accounts -type f -exec stat -L -c '%a %U %G' {{}} \\; | tee /dev/stderr | grep -v '600 acme {group}' | wc -l) -eq 0"
           )
 
-      with subtest("Certs are accepted by web server"):
-          webserver.succeed("systemctl start nginx.service")
-          check_fullchain(webserver, "a.example.test")
-          check_issuer(webserver, "a.example.test", "pebble")
-          check_connection(client, "a.example.test")
-
       # Selfsigned certs tests happen late so we aren't fighting the system init triggering cert renewal
       with subtest("Can generate valid selfsigned certs"):
           webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
@@ -365,77 +491,107 @@ in import ./make-test-python.nix ({ lib, ... }: {
           # Will succeed if nginx can load the certs
           webserver.succeed("systemctl start nginx-config-reload.service")
 
-      with subtest("Can reload nginx when timer triggers renewal"):
-          webserver.succeed("systemctl start test-renew-nginx.target")
-          check_issuer(webserver, "a.example.test", "pebble")
-          check_connection(client, "a.example.test")
-
-      with subtest("Runs 1 cert for account creation before others"):
-          switch_to(webserver, "account-creation")
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          check_connection(client, "a.example.test")
-          webserver.wait_for_unit("acme-finished-b.example.test.target")
-          webserver.wait_for_unit("acme-finished-c.example.test.target")
-          check_connection(client, "b.example.test")
-          check_connection(client, "c.example.test")
-
-      with subtest("Can reload web server when cert configuration changes"):
-          switch_to(webserver, "cert-change")
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          check_connection_key_bits(client, "a.example.test", "384")
-          webserver.succeed("grep testing /home/test")
-          # Clean to remove the testing file (and anything else messy we did)
-          webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
-
       with subtest("Correctly implements OCSP stapling"):
           switch_to(webserver, "ocsp-stapling")
           webserver.wait_for_unit("acme-finished-a.example.test.target")
           check_stapling(client, "a.example.test")
 
-      with subtest("Can request certificate with HTTPS-01 when nginx startup is delayed"):
+      with subtest("Can request certificate with HTTP-01 using lego's internal web server"):
+          switch_to(webserver, "lego-server")
+          webserver.wait_for_unit("acme-finished-lego.example.test.target")
+          webserver.wait_for_unit("nginx.service")
+          webserver.succeed("echo HENLO && systemctl cat nginx.service")
+          webserver.succeed("test \"$(stat -c '%U' /var/lib/acme/* | uniq)\" = \"root\"")
+          check_connection(client, "a.example.test")
+          check_connection(client, "lego.example.test")
+
+      with subtest("Can request certificate with HTTP-01 when nginx startup is delayed"):
+          webserver.execute("systemctl stop nginx")
           switch_to(webserver, "slow-startup")
-          webserver.wait_for_unit("acme-finished-slow.example.com.target")
-          check_issuer(webserver, "slow.example.com", "pebble")
-          check_connection(client, "slow.example.com")
+          webserver.wait_for_unit("acme-finished-slow.example.test.target")
+          check_issuer(webserver, "slow.example.test", "pebble")
+          webserver.wait_for_unit("nginx.service")
+          check_connection(client, "slow.example.test")
 
-      with subtest("Can request certificate for vhost + aliases (nginx)"):
-          # Check the key hash before and after adding an alias. It should not change.
-          # The previous test reverts the ed384 change
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          switch_to(webserver, "nginx-aliases")
-          webserver.wait_for_unit("acme-finished-a.example.test.target")
-          check_issuer(webserver, "a.example.test", "pebble")
+      with subtest("Works with caddy"):
+          switch_to(webserver, "caddy")
+          webserver.wait_for_unit("acme-finished-example.test.target")
+          webserver.wait_for_unit("caddy.service")
+          # FIXME reloading caddy is not sufficient to load new certs.
+          # Restart it manually until this is fixed.
+          webserver.succeed("systemctl restart caddy.service")
           check_connection(client, "a.example.test")
-          check_connection(client, "b.example.test")
 
-      with subtest("Can request certificates for vhost + aliases (apache-httpd)"):
-          try:
-              switch_to(webserver, "httpd-aliases")
-              webserver.wait_for_unit("acme-finished-c.example.test.target")
-          except Exception as err:
-              _, output = webserver.execute(
-                  "cat /var/log/httpd/*.log && ls -al /var/lib/acme/acme-challenge"
-              )
-              print(output)
-              raise err
-          check_issuer(webserver, "c.example.test", "pebble")
-          check_connection(client, "c.example.test")
-          check_connection(client, "d.example.test")
-
-      with subtest("Can reload httpd when timer triggers renewal"):
-          # Switch to selfsigned first
-          webserver.succeed("systemctl clean acme-c.example.test.service --what=state")
-          webserver.succeed("systemctl start acme-selfsigned-c.example.test.service")
-          check_issuer(webserver, "c.example.test", "minica")
-          webserver.succeed("systemctl start httpd-config-reload.service")
-          webserver.succeed("systemctl start test-renew-httpd.target")
-          check_issuer(webserver, "c.example.test", "pebble")
-          check_connection(client, "c.example.test")
-
-      with subtest("Can request wildcard certificates using DNS-01 challenge"):
-          switch_to(webserver, "dns-01")
+      with subtest("security.acme changes reflect on caddy"):
+          switch_to(webserver, "caddy-change-acme-conf")
           webserver.wait_for_unit("acme-finished-example.test.target")
-          check_issuer(webserver, "example.test", "pebble")
-          check_connection(client, "dns.example.test")
+          webserver.wait_for_unit("caddy.service")
+          # FIXME reloading caddy is not sufficient to load new certs.
+          # Restart it manually until this is fixed.
+          webserver.succeed("systemctl restart caddy.service")
+          check_connection_key_bits(client, "a.example.test", "384")
+
+      domains = ["http", "dns", "wildcard"]
+      for server, logsrc in [
+          ("nginx", "journalctl -n 30 -u nginx.service"),
+          ("httpd", "tail -n 30 /var/log/httpd/*.log"),
+      ]:
+          wait_for_server = lambda: webserver.wait_for_unit(f"{server}.service")
+          with subtest(f"Works with {server}"):
+              try:
+                  switch_to(webserver, server)
+                  # Skip wildcard domain for this check ([:-1])
+                  for domain in domains[:-1]:
+                      webserver.wait_for_unit(
+                          f"acme-finished-{server}-{domain}.example.test.target"
+                      )
+              except Exception as err:
+                  _, output = webserver.execute(
+                      f"{logsrc} && ls -al /var/lib/acme/acme-challenge"
+                  )
+                  print(output)
+                  raise err
+
+              wait_for_server()
+
+              for domain in domains[:-1]:
+                  check_issuer(webserver, f"{server}-{domain}.example.test", "pebble")
+              for domain in domains:
+                  check_connection(client, f"{server}-{domain}.example.test")
+                  check_connection(client, f"{server}-{domain}-alias.example.test")
+
+          test_domain = f"{server}-{domains[0]}.example.test"
+
+          with subtest(f"Can reload {server} when timer triggers renewal"):
+              # Switch to selfsigned first
+              webserver.succeed(f"systemctl clean acme-{test_domain}.service --what=state")
+              webserver.succeed(f"systemctl start acme-selfsigned-{test_domain}.service")
+              check_issuer(webserver, test_domain, "minica")
+              webserver.succeed(f"systemctl start {server}-config-reload.service")
+              webserver.succeed(f"systemctl start test-renew-{server}.target")
+              check_issuer(webserver, test_domain, "pebble")
+              check_connection(client, test_domain)
+
+          with subtest("Can remove an alias from a domain + cert is updated"):
+              test_alias = f"{server}-{domains[0]}-alias.example.test"
+              switch_to(webserver, f"{server}-remove-alias")
+              webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
+              wait_for_server()
+              check_connection(client, test_domain)
+              rc, _ = client.execute(
+                  f"openssl s_client -CAfile /tmp/ca.crt -connect {test_alias}:443"
+                  " </dev/null 2>/dev/null | openssl x509 -noout -text"
+                  f" | grep DNS: | grep {test_alias}"
+              )
+              assert rc > 0, "Removed extraDomainName was not removed from the cert"
+
+          with subtest("security.acme changes reflect on web server"):
+              # Switch back to normal server config first, reset everything.
+              switch_to(webserver, server)
+              wait_for_server()
+              switch_to(webserver, f"{server}-change-acme-conf")
+              webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
+              wait_for_server()
+              check_connection_key_bits(client, test_domain, "384")
     '';
 })
diff --git a/nixos/tests/adguardhome.nix b/nixos/tests/adguardhome.nix
new file mode 100644
index 000000000000..ddbe8ff9c117
--- /dev/null
+++ b/nixos/tests/adguardhome.nix
@@ -0,0 +1,57 @@
+import ./make-test-python.nix {
+  name = "adguardhome";
+
+  nodes = {
+    minimalConf = { ... }: {
+      services.adguardhome = { enable = true; };
+    };
+
+    declarativeConf = { ... }: {
+      services.adguardhome = {
+        enable = true;
+
+        mutableSettings = false;
+        settings = {
+          dns = {
+            bind_host = "0.0.0.0";
+            bootstrap_dns = "127.0.0.1";
+          };
+        };
+      };
+    };
+
+    mixedConf = { ... }: {
+      services.adguardhome = {
+        enable = true;
+
+        mutableSettings = true;
+        settings = {
+          dns = {
+            bind_host = "0.0.0.0";
+            bootstrap_dns = "127.0.0.1";
+          };
+        };
+      };
+    };
+  };
+
+  testScript = ''
+    with subtest("Minimal config test"):
+        minimalConf.wait_for_unit("adguardhome.service")
+        minimalConf.wait_for_open_port(3000)
+
+    with subtest("Declarative config test, DNS will be reachable"):
+        declarativeConf.wait_for_unit("adguardhome.service")
+        declarativeConf.wait_for_open_port(53)
+        declarativeConf.wait_for_open_port(3000)
+
+    with subtest("Mixed config test, check whether merging works"):
+        mixedConf.wait_for_unit("adguardhome.service")
+        mixedConf.wait_for_open_port(53)
+        mixedConf.wait_for_open_port(3000)
+        # Test whether merging works properly, even if nothing is changed
+        mixedConf.systemctl("restart adguardhome.service")
+        mixedConf.wait_for_unit("adguardhome.service")
+        mixedConf.wait_for_open_port(3000)
+  '';
+}
diff --git a/nixos/tests/aesmd.nix b/nixos/tests/aesmd.nix
new file mode 100644
index 000000000000..59c04fe7e96a
--- /dev/null
+++ b/nixos/tests/aesmd.nix
@@ -0,0 +1,62 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }: {
+  name = "aesmd";
+  meta = {
+    maintainers = with lib.maintainers; [ veehaitch ];
+  };
+
+  machine = { lib, ... }: {
+    services.aesmd = {
+      enable = true;
+      settings = {
+        defaultQuotingType = "ecdsa_256";
+        proxyType = "direct";
+        whitelistUrl = "http://nixos.org";
+      };
+    };
+
+    # Should have access to the AESM socket
+    users.users."sgxtest" = {
+      isNormalUser = true;
+      extraGroups = [ "sgx" ];
+    };
+
+    # Should NOT have access to the AESM socket
+    users.users."nosgxtest".isNormalUser = true;
+
+    # We don't have a real SGX machine in NixOS tests
+    systemd.services.aesmd.unitConfig.AssertPathExists = lib.mkForce [ ];
+  };
+
+  testScript = ''
+    with subtest("aesmd.service starts"):
+      machine.wait_for_unit("aesmd.service")
+      status, main_pid = machine.systemctl("show --property MainPID --value aesmd.service")
+      assert status == 0, "Could not get MainPID of aesmd.service"
+      main_pid = main_pid.strip()
+
+    with subtest("aesmd.service runtime directory permissions"):
+      runtime_dir = "/run/aesmd";
+      res = machine.succeed(f"stat -c '%a %U %G' {runtime_dir}").strip()
+      assert "750 aesmd sgx" == res, f"{runtime_dir} does not have the expected permissions: {res}"
+
+    with subtest("aesm.socket available on host"):
+      socket_path = "/var/run/aesmd/aesm.socket"
+      machine.wait_until_succeeds(f"test -S {socket_path}")
+      machine.succeed(f"test 777 -eq $(stat -c '%a' {socket_path})")
+      for op in [ "-r", "-w", "-x" ]:
+        machine.succeed(f"sudo -u sgxtest test {op} {socket_path}")
+        machine.fail(f"sudo -u nosgxtest test {op} {socket_path}")
+
+    with subtest("Copies white_list_cert_to_be_verify.bin"):
+      whitelist_path = "/var/opt/aesmd/data/white_list_cert_to_be_verify.bin"
+      whitelist_perms = machine.succeed(
+        f"nsenter -m -t {main_pid} ${pkgs.coreutils}/bin/stat -c '%a' {whitelist_path}"
+      ).strip()
+      assert "644" == whitelist_perms, f"white_list_cert_to_be_verify.bin has permissions {whitelist_perms}"
+
+    with subtest("Writes and binds aesm.conf in service namespace"):
+      aesmd_config = machine.succeed(f"nsenter -m -t {main_pid} ${pkgs.coreutils}/bin/cat /etc/aesmd.conf")
+
+      assert aesmd_config == "whitelist url = http://nixos.org\nproxy type = direct\ndefault quoting type = ecdsa_256\n", "aesmd.conf differs"
+  '';
+})
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 06305460c6ac..b6311f856fa8 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -19,13 +19,24 @@ let
   handleTestOn = systems: path: args:
     if elem system systems then handleTest path args
     else {};
+
+  nixosLib = import ../lib {
+    # Experimental features need testing too, but there's no point in warning
+    # about it, so we enable the feature flag.
+    featureFlags.minimalModules = {};
+  };
+  evalMinimalConfig = module: nixosLib.evalModules { modules = [ module ]; };
 in
 {
   _3proxy = handleTest ./3proxy.nix {};
   acme = handleTest ./acme.nix {};
+  adguardhome = handleTest ./adguardhome.nix {};
+  aesmd = handleTest ./aesmd.nix {};
+  agate = handleTest ./web-servers/agate.nix {};
   agda = handleTest ./agda.nix {};
   airsonic = handleTest ./airsonic.nix {};
   amazon-init-shell = handleTest ./amazon-init-shell.nix {};
+  apfs = handleTest ./apfs.nix {};
   apparmor = handleTest ./apparmor.nix {};
   atd = handleTest ./atd.nix {};
   atop = handleTest ./atop.nix {};
@@ -37,14 +48,18 @@ in
   beanstalkd = handleTest ./beanstalkd.nix {};
   bees = handleTest ./bees.nix {};
   bind = handleTest ./bind.nix {};
+  bird = handleTest ./bird.nix {};
   bitcoind = handleTest ./bitcoind.nix {};
   bittorrent = handleTest ./bittorrent.nix {};
   blockbook-frontend = handleTest ./blockbook-frontend.nix {};
+  blocky = handleTest ./blocky.nix {};
   boot = handleTestOn ["x86_64-linux" "aarch64-linux"] ./boot.nix {};
   boot-stage1 = handleTest ./boot-stage1.nix {};
   borgbackup = handleTest ./borgbackup.nix {};
   botamusique = handleTest ./botamusique.nix {};
   bpf = handleTestOn ["x86_64-linux" "aarch64-linux"] ./bpf.nix {};
+  breitbandmessung = handleTest ./breitbandmessung.nix {};
+  brscan5 = handleTest ./brscan5.nix {};
   btrbk = handleTest ./btrbk.nix {};
   buildbot = handleTest ./buildbot.nix {};
   buildkite-agents = handleTest ./buildkite-agents.nix {};
@@ -69,6 +84,7 @@ in
   cloud-init = handleTest ./cloud-init.nix {};
   cntr = handleTest ./cntr.nix {};
   cockroachdb = handleTestOn ["x86_64-linux"] ./cockroachdb.nix {};
+  collectd = handleTest ./collectd.nix {};
   consul = handleTest ./consul.nix {};
   containers-bridge = handleTest ./containers-bridge.nix {};
   containers-custom-pkgs.nix = handleTest ./containers-custom-pkgs.nix {};
@@ -101,14 +117,17 @@ in
   discourse = handleTest ./discourse.nix {};
   dnscrypt-proxy2 = handleTestOn ["x86_64-linux"] ./dnscrypt-proxy2.nix {};
   dnscrypt-wrapper = handleTestOn ["x86_64-linux"] ./dnscrypt-wrapper {};
+  dnsdist = handleTest ./dnsdist.nix {};
   doas = handleTest ./doas.nix {};
   docker = handleTestOn ["x86_64-linux"] ./docker.nix {};
+  docker-rootless = handleTestOn ["x86_64-linux"] ./docker-rootless.nix {};
   docker-edge = handleTestOn ["x86_64-linux"] ./docker-edge.nix {};
   docker-registry = handleTest ./docker-registry.nix {};
   docker-tools = handleTestOn ["x86_64-linux"] ./docker-tools.nix {};
   docker-tools-cross = handleTestOn ["x86_64-linux" "aarch64-linux"] ./docker-tools-cross.nix {};
   docker-tools-overlay = handleTestOn ["x86_64-linux"] ./docker-tools-overlay.nix {};
   documize = handleTest ./documize.nix {};
+  doh-proxy-rust = handleTest ./doh-proxy-rust.nix {};
   dokuwiki = handleTest ./dokuwiki.nix {};
   domination = handleTest ./domination.nix {};
   dovecot = handleTest ./dovecot.nix {};
@@ -118,16 +137,20 @@ in
   ecryptfs = handleTest ./ecryptfs.nix {};
   ejabberd = handleTest ./xmpp/ejabberd.nix {};
   elk = handleTestOn ["x86_64-linux"] ./elk.nix {};
+  emacs-daemon = handleTest ./emacs-daemon.nix {};
   engelsystem = handleTest ./engelsystem.nix {};
   enlightenment = handleTest ./enlightenment.nix {};
   env = handleTest ./env.nix {};
   ergo = handleTest ./ergo.nix {};
+  ergochat = handleTest ./ergochat.nix {};
+  etc = pkgs.callPackage ../modules/system/etc/test.nix { inherit evalMinimalConfig; };
   etcd = handleTestOn ["x86_64-linux"] ./etcd.nix {};
   etcd-cluster = handleTestOn ["x86_64-linux"] ./etcd-cluster.nix {};
   etebase-server = handleTest ./etebase-server.nix {};
   etesync-dav = handleTest ./etesync-dav.nix {};
   fancontrol = handleTest ./fancontrol.nix {};
   fcitx = handleTest ./fcitx {};
+  fenics = handleTest ./fenics.nix {};
   ferm = handleTest ./ferm.nix {};
   firefox = handleTest ./firefox.nix { firefoxPackage = pkgs.firefox; };
   firefox-esr    = handleTest ./firefox.nix { firefoxPackage = pkgs.firefox-esr; }; # used in `tested` job
@@ -140,9 +163,11 @@ in
   fluidd = handleTest ./fluidd.nix {};
   fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {};
   freeswitch = handleTest ./freeswitch.nix {};
+  frr = handleTest ./frr.nix {};
   fsck = handleTest ./fsck.nix {};
   ft2-clone = handleTest ./ft2-clone.nix {};
   gerrit = handleTest ./gerrit.nix {};
+  geth = handleTest ./geth.nix {};
   ghostunnel = handleTest ./ghostunnel.nix {};
   gitdaemon = handleTest ./gitdaemon.nix {};
   gitea = handleTest ./gitea.nix {};
@@ -164,10 +189,10 @@ in
   grocy = handleTest ./grocy.nix {};
   grub = handleTest ./grub.nix {};
   gvisor = handleTest ./gvisor.nix {};
-  hadoop.all = handleTestOn [ "x86_64-linux" ] ./hadoop/hadoop.nix {};
-  hadoop.hdfs = handleTestOn [ "x86_64-linux" ] ./hadoop/hdfs.nix {};
-  hadoop.yarn = handleTestOn [ "x86_64-linux" ] ./hadoop/yarn.nix {};
-  handbrake = handleTestOn ["x86_64-linux"] ./handbrake.nix {};
+  hadoop = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop; };
+  hadoop_3_2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop_3_2; };
+  hadoop2 = import ./hadoop { inherit handleTestOn; package=pkgs.hadoop2; };
+  haka = handleTest ./haka.nix {};
   haproxy = handleTest ./haproxy.nix {};
   hardened = handleTest ./hardened.nix {};
   hedgedoc = handleTest ./hedgedoc.nix {};
@@ -199,19 +224,25 @@ in
   initrd-network-ssh = handleTest ./initrd-network-ssh {};
   initrdNetwork = handleTest ./initrd-network.nix {};
   initrd-secrets = handleTest ./initrd-secrets.nix {};
+  input-remapper = handleTest ./input-remapper.nix {};
   inspircd = handleTest ./inspircd.nix {};
   installer = handleTest ./installer.nix {};
+  invoiceplane = handleTest ./invoiceplane.nix {};
   iodine = handleTest ./iodine.nix {};
   ipfs = handleTest ./ipfs.nix {};
   ipv6 = handleTest ./ipv6.nix {};
+  iscsi-multipath-root = handleTest ./iscsi-multipath-root.nix {};
   iscsi-root = handleTest ./iscsi-root.nix {};
+  isso = handleTest ./isso.nix {};
   jackett = handleTest ./jackett.nix {};
   jellyfin = handleTest ./jellyfin.nix {};
   jenkins = handleTest ./jenkins.nix {};
+  jenkins-cli = handleTest ./jenkins-cli.nix {};
   jibri = handleTest ./jibri.nix {};
   jirafeau = handleTest ./jirafeau.nix {};
   jitsi-meet = handleTest ./jitsi-meet.nix {};
-  k3s = handleTest ./k3s.nix {};
+  k3s-single-node = handleTest ./k3s-single-node.nix {};
+  k3s-single-node-docker = handleTest ./k3s-single-node-docker.nix {};
   kafka = handleTest ./kafka.nix {};
   kbd-setfont-decompress = handleTest ./kbd-setfont-decompress.nix {};
   kbd-update-search-paths-patch = handleTest ./kbd-update-search-paths-patch.nix {};
@@ -240,6 +271,7 @@ in
   litestream = handleTest ./litestream.nix {};
   locate = handleTest ./locate.nix {};
   login = handleTest ./login.nix {};
+  logrotate = handleTest ./logrotate.nix {};
   loki = handleTest ./loki.nix {};
   lxd = handleTest ./lxd.nix {};
   lxd-image = handleTest ./lxd-image.nix {};
@@ -252,11 +284,15 @@ in
   magnetico = handleTest ./magnetico.nix {};
   mailcatcher = handleTest ./mailcatcher.nix {};
   mailhog = handleTest ./mailhog.nix {};
-  mariadb-galera-mariabackup = handleTest ./mysql/mariadb-galera-mariabackup.nix {};
-  mariadb-galera-rsync = handleTest ./mysql/mariadb-galera-rsync.nix {};
+  man = handleTest ./man.nix {};
+  mariadb-galera = handleTest ./mysql/mariadb-galera.nix {};
+  mastodon = handleTestOn ["x86_64-linux" "i686-linux" "aarch64-linux"] ./web-apps/mastodon.nix {};
   matomo = handleTest ./matomo.nix {};
   matrix-appservice-irc = handleTest ./matrix-appservice-irc.nix {};
+  matrix-conduit = handleTest ./matrix-conduit.nix {};
   matrix-synapse = handleTest ./matrix-synapse.nix {};
+  mattermost = handleTest ./mattermost.nix {};
+  mediatomb = handleTest ./mediatomb.nix {};
   mediawiki = handleTest ./mediawiki.nix {};
   meilisearch = handleTest ./meilisearch.nix {};
   memcached = handleTest ./memcached.nix {};
@@ -269,11 +305,12 @@ in
   misc = handleTest ./misc.nix {};
   mjolnir = handleTest ./matrix/mjolnir.nix {};
   mod_perl = handleTest ./mod_perl.nix {};
-  moinmoin = handleTest ./moinmoin.nix {};
+  molly-brown = handleTest ./molly-brown.nix {};
   mongodb = handleTest ./mongodb.nix {};
   moodle = handleTest ./moodle.nix {};
   morty = handleTest ./morty.nix {};
   mosquitto = handleTest ./mosquitto.nix {};
+  moosefs = handleTest ./moosefs.nix {};
   mpd = handleTest ./mpd.nix {};
   mpv = handleTest ./mpv.nix {};
   mumble = handleTest ./mumble.nix {};
@@ -287,13 +324,13 @@ in
   mysql-replication = handleTest ./mysql/mysql-replication.nix {};
   n8n = handleTest ./n8n.nix {};
   nagios = handleTest ./nagios.nix {};
-  nano = handleTest ./nano.nix {};
   nar-serve = handleTest ./nar-serve.nix {};
   nat.firewall = handleTest ./nat.nix { withFirewall = true; };
   nat.firewall-conntrack = handleTest ./nat.nix { withFirewall = true; withConntrackHelpers = true; };
   nat.standalone = handleTest ./nat.nix { withFirewall = false; };
   nats = handleTest ./nats.nix {};
   navidrome = handleTest ./navidrome.nix {};
+  nbd = handleTest ./nbd.nix {};
   ncdns = handleTest ./ncdns.nix {};
   ndppd = handleTest ./ndppd.nix {};
   nebula = handleTest ./nebula.nix {};
@@ -313,17 +350,21 @@ in
   nginx = handleTest ./nginx.nix {};
   nginx-auth = handleTest ./nginx-auth.nix {};
   nginx-etag = handleTest ./nginx-etag.nix {};
+  nginx-modsecurity = handleTest ./nginx-modsecurity.nix {};
   nginx-pubhtml = handleTest ./nginx-pubhtml.nix {};
   nginx-sandbox = handleTestOn ["x86_64-linux"] ./nginx-sandbox.nix {};
   nginx-sso = handleTest ./nginx-sso.nix {};
   nginx-variants = handleTest ./nginx-variants.nix {};
   nitter = handleTest ./nitter.nix {};
+  nix-ld = handleTest ./nix-ld {};
   nix-serve = handleTest ./nix-serve.nix {};
   nix-serve-ssh = handleTest ./nix-serve-ssh.nix {};
   nixops = handleTest ./nixops/default.nix {};
   nixos-generate-config = handleTest ./nixos-generate-config.nix {};
+  nixpkgs = pkgs.callPackage ../modules/misc/nixpkgs/test.nix { inherit evalMinimalConfig; };
   node-red = handleTest ./node-red.nix {};
   nomad = handleTest ./nomad.nix {};
+  noto-fonts = handleTest ./noto-fonts.nix {};
   novacomd = handleTestOn ["x86_64-linux"] ./novacomd.nix {};
   nsd = handleTest ./nsd.nix {};
   nzbget = handleTest ./nzbget.nix {};
@@ -345,6 +386,7 @@ in
   os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {};
   osrm-backend = handleTest ./osrm-backend.nix {};
   overlayfs = handleTest ./overlayfs.nix {};
+  pacemaker = handleTest ./pacemaker.nix {};
   packagekit = handleTest ./packagekit.nix {};
   pam-file-contents = handleTest ./pam/pam-file-contents.nix {};
   pam-oath-login = handleTest ./pam/pam-oath-login.nix {};
@@ -356,11 +398,15 @@ in
   pdns-recursor = handleTest ./pdns-recursor.nix {};
   peerflix = handleTest ./peerflix.nix {};
   peertube = handleTestOn ["x86_64-linux"] ./web-apps/peertube.nix {};
+  pgadmin4 = handleTest ./pgadmin4.nix {};
+  pgadmin4-standalone = handleTest ./pgadmin4-standalone.nix {};
   pgjwt = handleTest ./pgjwt.nix {};
   pgmanage = handleTest ./pgmanage.nix {};
   php = handleTest ./php {};
   php74 = handleTest ./php { php = pkgs.php74; };
   php80 = handleTest ./php { php = pkgs.php80; };
+  php81 = handleTest ./php { php = pkgs.php81; };
+  pict-rs = handleTest ./pict-rs.nix {};
   pinnwand = handleTest ./pinnwand.nix {};
   plasma5 = handleTest ./plasma5.nix {};
   plasma5-systemd-start = handleTest ./plasma5-systemd-start.nix {};
@@ -380,6 +426,7 @@ in
   postgresql = handleTest ./postgresql.nix {};
   postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {};
   powerdns = handleTest ./powerdns.nix {};
+  powerdns-admin = handleTest ./powerdns-admin.nix {};
   power-profiles-daemon = handleTest ./power-profiles-daemon.nix {};
   pppd = handleTest ./pppd.nix {};
   predictable-interface-names = handleTest ./predictable-interface-names.nix {};
@@ -389,10 +436,11 @@ in
   prometheus = handleTest ./prometheus.nix {};
   prometheus-exporters = handleTest ./prometheus-exporters.nix {};
   prosody = handleTest ./xmpp/prosody.nix {};
-  prosodyMysql = handleTest ./xmpp/prosody-mysql.nix {};
+  prosody-mysql = handleTest ./xmpp/prosody-mysql.nix {};
   proxy = handleTest ./proxy.nix {};
   prowlarr = handleTest ./prowlarr.nix {};
   pt2-clone = handleTest ./pt2-clone.nix {};
+  pulseaudio = discoverTests (import ./pulseaudio.nix);
   qboot = handleTestOn ["x86_64-linux" "i686-linux"] ./qboot.nix {};
   quorum = handleTest ./quorum.nix {};
   rabbitmq = handleTest ./rabbitmq.nix {};
@@ -401,12 +449,17 @@ in
   rasdaemon = handleTest ./rasdaemon.nix {};
   redis = handleTest ./redis.nix {};
   redmine = handleTest ./redmine.nix {};
+  resolv = handleTest ./resolv.nix {};
   restartByActivationScript = handleTest ./restart-by-activation-script.nix {};
   restic = handleTest ./restic.nix {};
+  retroarch = handleTest ./retroarch.nix {};
+  riak = handleTest ./riak.nix {};
   robustirc-bridge = handleTest ./robustirc-bridge.nix {};
   roundcube = handleTest ./roundcube.nix {};
   rspamd = handleTest ./rspamd.nix {};
   rss2email = handleTest ./rss2email.nix {};
+  rstudio-server = handleTest ./rstudio-server.nix {};
+  rsyncd = handleTest ./rsyncd.nix {};
   rsyslogd = handleTest ./rsyslogd.nix {};
   rxe = handleTest ./rxe.nix {};
   sabnzbd = handleTest ./sabnzbd.nix {};
@@ -427,15 +480,18 @@ in
   smokeping = handleTest ./smokeping.nix {};
   snapcast = handleTest ./snapcast.nix {};
   snapper = handleTest ./snapper.nix {};
+  soapui = handleTest ./soapui.nix {};
   sogo = handleTest ./sogo.nix {};
   solanum = handleTest ./solanum.nix {};
   solr = handleTest ./solr.nix {};
   sonarr = handleTest ./sonarr.nix {};
+  sourcehut = handleTest ./sourcehut.nix {};
   spacecookie = handleTest ./spacecookie.nix {};
-  spark = handleTestOn ["x86_64-linux"] ./spark {};
+  spark = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./spark {};
   sslh = handleTest ./sslh.nix {};
   sssd = handleTestOn ["x86_64-linux"] ./sssd.nix {};
   sssd-ldap = handleTestOn ["x86_64-linux"] ./sssd-ldap.nix {};
+  starship = handleTest ./starship.nix {};
   step-ca = handleTestOn ["x86_64-linux"] ./step-ca.nix {};
   strongswan-swanctl = handleTest ./strongswan-swanctl.nix {};
   sudo = handleTest ./sudo.nix {};
@@ -451,7 +507,9 @@ in
   systemd-boot = handleTest ./systemd-boot.nix {};
   systemd-confinement = handleTest ./systemd-confinement.nix {};
   systemd-cryptenroll = handleTest ./systemd-cryptenroll.nix {};
+  systemd-escaping = handleTest ./systemd-escaping.nix {};
   systemd-journal = handleTest ./systemd-journal.nix {};
+  systemd-machinectl = handleTest ./systemd-machinectl.nix {};
   systemd-networkd = handleTest ./systemd-networkd.nix {};
   systemd-networkd-dhcpserver = handleTest ./systemd-networkd-dhcpserver.nix {};
   systemd-networkd-dhcpserver-static-leases = handleTest ./systemd-networkd-dhcpserver-static-leases.nix {};
@@ -461,14 +519,19 @@ in
   systemd-timesyncd = handleTest ./systemd-timesyncd.nix {};
   systemd-unit-path = handleTest ./systemd-unit-path.nix {};
   taskserver = handleTest ./taskserver.nix {};
+  teeworlds = handleTest ./teeworlds.nix {};
   telegraf = handleTest ./telegraf.nix {};
+  teleport = handleTest ./teleport.nix {};
+  thelounge = handleTest ./thelounge.nix {};
+  terminal-emulators = handleTest ./terminal-emulators.nix {};
   tiddlywiki = handleTest ./tiddlywiki.nix {};
   tigervnc = handleTest ./tigervnc.nix {};
   timezone = handleTest ./timezone.nix {};
   tinc = handleTest ./tinc {};
   tinydns = handleTest ./tinydns.nix {};
+  tinywl = handleTest ./tinywl.nix {};
+  tomcat = handleTest ./tomcat.nix {};
   tor = handleTest ./tor.nix {};
-  trac = handleTest ./trac.nix {};
   # traefik test relies on docker-containers
   traefik = handleTestOn ["x86_64-linux"] ./traefik.nix {};
   trafficserver = handleTest ./trafficserver.nix {};
@@ -476,6 +539,7 @@ in
   trezord = handleTest ./trezord.nix {};
   trickster = handleTest ./trickster.nix {};
   trilium-server = handleTestOn ["x86_64-linux"] ./trilium-server.nix {};
+  tsm-client-gui = handleTest ./tsm-client-gui.nix {};
   txredisapi = handleTest ./txredisapi.nix {};
   tuptime = handleTest ./tuptime.nix {};
   turbovnc-headless-server = handleTest ./turbovnc-headless-server.nix {};
@@ -483,6 +547,7 @@ in
   ucarp = handleTest ./ucarp.nix {};
   udisks2 = handleTest ./udisks2.nix {};
   unbound = handleTest ./unbound.nix {};
+  unifi = handleTest ./unifi.nix {};
   unit-php = handleTest ./web-servers/unit-php.nix {};
   upnp = handleTest ./upnp.nix {};
   usbguard = handleTest ./usbguard.nix {};
@@ -498,8 +563,10 @@ in
   vikunja = handleTest ./vikunja.nix {};
   virtualbox = handleTestOn ["x86_64-linux"] ./virtualbox.nix {};
   vscodium = discoverTests (import ./vscodium.nix);
+  vsftpd = handleTest ./vsftpd.nix {};
   wasabibackend = handleTest ./wasabibackend.nix {};
   wiki-js = handleTest ./wiki-js.nix {};
+  wine = handleTest ./wine.nix {};
   wireguard = handleTest ./wireguard {};
   without-nix = handleTest ./without-nix.nix {};
   wmderland = handleTest ./wmderland.nix {};
@@ -512,8 +579,10 @@ in
   xrdp = handleTest ./xrdp.nix {};
   xss-lock = handleTest ./xss-lock.nix {};
   xterm = handleTest ./xterm.nix {};
+  xxh = handleTest ./xxh.nix {};
   yabar = handleTest ./yabar.nix {};
   yggdrasil = handleTest ./yggdrasil.nix {};
+  zammad = handleTest ./zammad.nix {};
   zfs = handleTest ./zfs.nix {};
   zigbee2mqtt = handleTest ./zigbee2mqtt.nix {};
   zoneminder = handleTest ./zoneminder.nix {};
diff --git a/nixos/tests/apfs.nix b/nixos/tests/apfs.nix
new file mode 100644
index 000000000000..a82886cbe731
--- /dev/null
+++ b/nixos/tests/apfs.nix
@@ -0,0 +1,54 @@
+import ./make-test-python.nix ({ pkgs, ... }: {
+  name = "apfs";
+  meta.maintainers = with pkgs.lib.maintainers; [ Luflosi ];
+
+  machine = { pkgs, ... }: {
+    virtualisation.emptyDiskImages = [ 1024 ];
+
+    boot.supportedFilesystems = [ "apfs" ];
+  };
+
+  testScript = ''
+    machine.wait_for_unit("basic.target")
+    machine.succeed("mkdir /tmp/mnt")
+
+    with subtest("mkapfs refuses to work with a label that is too long"):
+      machine.fail( "mkapfs -L '000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F' /dev/vdb")
+
+    with subtest("mkapfs works with the maximum label length"):
+      machine.succeed("mkapfs -L '000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7' /dev/vdb")
+
+    with subtest("Enable case sensitivity and normalization sensitivity"):
+      machine.succeed(
+          "mkapfs -s -z /dev/vdb",
+          # Triggers a bug, see https://github.com/linux-apfs/linux-apfs-rw/issues/15
+          # "mount -o cknodes,readwrite /dev/vdb /tmp/mnt",
+          "mount -o readwrite /dev/vdb /tmp/mnt",
+          "echo 'Hello World 1' > /tmp/mnt/test.txt",
+          "[ ! -f /tmp/mnt/TeSt.TxT ] || false", # Test case sensitivity
+          "echo 'Hello World 1' | diff - /tmp/mnt/test.txt",
+          "echo 'Hello World 2' > /tmp/mnt/\u0061\u0301.txt",
+          "echo 'Hello World 2' | diff - /tmp/mnt/\u0061\u0301.txt",
+          "[ ! -f /tmp/mnt/\u00e1.txt ] || false", # Test Unicode normalization sensitivity
+          "umount /tmp/mnt",
+          "apfsck /dev/vdb",
+      )
+    with subtest("Disable case sensitivity and normalization sensitivity"):
+      machine.succeed(
+          "mkapfs /dev/vdb",
+          "mount -o readwrite /dev/vdb /tmp/mnt",
+          "echo 'bla bla bla' > /tmp/mnt/Test.txt",
+          "echo -n 'Hello World' > /tmp/mnt/test.txt",
+          "echo ' 1' >> /tmp/mnt/TEST.TXT",
+          "umount /tmp/mnt",
+          "apfsck /dev/vdb",
+          "mount -o readwrite /dev/vdb /tmp/mnt",
+          "echo 'Hello World 1' | diff - /tmp/mnt/TeSt.TxT", # Test case insensitivity
+          "echo 'Hello World 2' > /tmp/mnt/\u0061\u0301.txt",
+          "echo 'Hello World 2' | diff - /tmp/mnt/\u0061\u0301.txt",
+          "echo 'Hello World 2' | diff - /tmp/mnt/\u00e1.txt", # Test Unicode normalization
+          "umount /tmp/mnt",
+          "apfsck /dev/vdb",
+      )
+  '';
+})
diff --git a/nixos/tests/bcachefs.nix b/nixos/tests/bcachefs.nix
index 146225e72cee..44997a746879 100644
--- a/nixos/tests/bcachefs.nix
+++ b/nixos/tests/bcachefs.nix
@@ -6,7 +6,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
     virtualisation.emptyDiskImages = [ 4096 ];
     networking.hostId = "deadbeef";
     boot.supportedFilesystems = [ "bcachefs" ];
-    environment.systemPackages = with pkgs; [ parted ];
+    environment.systemPackages = with pkgs; [ parted keyutils ];
   };
 
   testScript = ''
@@ -18,13 +18,12 @@ import ./make-test-python.nix ({ pkgs, ... }: {
         "mkdir /tmp/mnt",
         "udevadm settle",
         "parted --script /dev/vdb mklabel msdos",
-        "parted --script /dev/vdb -- mkpart primary 1024M -1s",
+        "parted --script /dev/vdb -- mkpart primary 1024M 50% mkpart primary 50% -1s",
         "udevadm settle",
-        # Due to #32279, we cannot use encryption for this test yet
-        # "echo password | bcachefs format --encrypted /dev/vdb1",
-        # "echo password | bcachefs unlock /dev/vdb1",
-        "bcachefs format /dev/vdb1",
-        "mount -t bcachefs /dev/vdb1 /tmp/mnt",
+        "keyctl link @u @s",
+        "echo password | bcachefs format --encrypted --metadata_replicas 2 --label vtest /dev/vdb1 /dev/vdb2",
+        "echo password | bcachefs unlock /dev/vdb1",
+        "mount -t bcachefs /dev/vdb1:/dev/vdb2 /tmp/mnt",
         "udevadm settle",
         "bcachefs fs usage /tmp/mnt",
         "umount /tmp/mnt",
diff --git a/nixos/tests/bird.nix b/nixos/tests/bird.nix
new file mode 100644
index 000000000000..822a7caea9ba
--- /dev/null
+++ b/nixos/tests/bird.nix
@@ -0,0 +1,129 @@
+# This test does a basic functionality check for all bird variants and demonstrates a use
+# of the preCheckConfig option.
+
+{ system ? builtins.currentSystem
+, pkgs ? import ../.. { inherit system; config = { }; }
+}:
+
+let
+  inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
+  inherit (pkgs.lib) optionalString;
+
+  makeBird2Host = hostId: { pkgs, ... }: {
+    virtualisation.vlans = [ 1 ];
+
+    environment.systemPackages = with pkgs; [ jq ];
+
+    networking = {
+      useNetworkd = true;
+      useDHCP = false;
+      firewall.enable = false;
+    };
+
+    systemd.network.networks."01-eth1" = {
+      name = "eth1";
+      networkConfig.Address = "10.0.0.${hostId}/24";
+    };
+
+    services.bird2 = {
+      enable = true;
+
+      config = ''
+        log syslog all;
+
+        debug protocols all;
+
+        router id 10.0.0.${hostId};
+
+        protocol device {
+        }
+
+        protocol kernel kernel4 {
+          ipv4 {
+            import none;
+            export all;
+          };
+        }
+
+        protocol static static4 {
+          ipv4;
+          include "static4.conf";
+        }
+
+        protocol ospf v2 ospf4 {
+          ipv4 {
+            export all;
+          };
+          area 0 {
+            interface "eth1" {
+              hello 5;
+              wait 5;
+            };
+          };
+        }
+
+        protocol kernel kernel6 {
+          ipv6 {
+            import none;
+            export all;
+          };
+        }
+
+        protocol static static6 {
+          ipv6;
+          include "static6.conf";
+        }
+
+        protocol ospf v3 ospf6 {
+          ipv6 {
+            export all;
+          };
+          area 0 {
+            interface "eth1" {
+              hello 5;
+              wait 5;
+            };
+          };
+        }
+      '';
+
+      preCheckConfig = ''
+        echo "route 1.2.3.4/32 blackhole;" > static4.conf
+        echo "route fd00::/128 blackhole;" > static6.conf
+      '';
+    };
+
+    systemd.tmpfiles.rules = [
+      "f /etc/bird/static4.conf - - - - route 10.10.0.${hostId}/32 blackhole;"
+      "f /etc/bird/static6.conf - - - - route fdff::${hostId}/128 blackhole;"
+    ];
+  };
+in
+makeTest {
+  name = "bird2";
+
+  nodes.host1 = makeBird2Host "1";
+  nodes.host2 = makeBird2Host "2";
+
+  testScript = ''
+    start_all()
+
+    host1.wait_for_unit("bird2.service")
+    host2.wait_for_unit("bird2.service")
+    host1.succeed("systemctl reload bird2.service")
+
+    with subtest("Waiting for advertised IPv4 routes"):
+      host1.wait_until_succeeds("ip --json r | jq -e 'map(select(.dst == \"10.10.0.2\")) | any'")
+      host2.wait_until_succeeds("ip --json r | jq -e 'map(select(.dst == \"10.10.0.1\")) | any'")
+    with subtest("Waiting for advertised IPv6 routes"):
+      host1.wait_until_succeeds("ip --json -6 r | jq -e 'map(select(.dst == \"fdff::2\")) | any'")
+      host2.wait_until_succeeds("ip --json -6 r | jq -e 'map(select(.dst == \"fdff::1\")) | any'")
+
+    with subtest("Check fake routes in preCheckConfig do not exists"):
+      host1.fail("ip --json r | jq -e 'map(select(.dst == \"1.2.3.4\")) | any'")
+      host2.fail("ip --json r | jq -e 'map(select(.dst == \"1.2.3.4\")) | any'")
+
+      host1.fail("ip --json -6 r | jq -e 'map(select(.dst == \"fd00::\")) | any'")
+      host2.fail("ip --json -6 r | jq -e 'map(select(.dst == \"fd00::\")) | any'")
+  '';
+}
diff --git a/nixos/tests/blocky.nix b/nixos/tests/blocky.nix
new file mode 100644
index 000000000000..18e7f45e1c73
--- /dev/null
+++ b/nixos/tests/blocky.nix
@@ -0,0 +1,34 @@
+import ./make-test-python.nix {
+  name = "blocky";
+
+  nodes = {
+    server = { pkgs, ... }: {
+      environment.systemPackages = [ pkgs.dnsutils ];
+      services.blocky = {
+        enable = true;
+
+        settings = {
+          customDNS = {
+            mapping = {
+              "printer.lan" = "192.168.178.3,2001:0db8:85a3:08d3:1319:8a2e:0370:7344";
+            };
+          };
+          upstream = {
+            default = [ "8.8.8.8" "1.1.1.1" ];
+          };
+          port = 53;
+          httpPort = 5000;
+          logLevel = "info";
+        };
+      };
+    };
+  };
+
+  testScript = ''
+    with subtest("Service test"):
+        server.wait_for_unit("blocky.service")
+        server.wait_for_open_port(53)
+        server.wait_for_open_port(5000)
+        server.succeed("dig @127.0.0.1 +short -x 192.168.178.3 | grep -qF printer.lan")
+  '';
+}
diff --git a/nixos/tests/boot-stage1.nix b/nixos/tests/boot-stage1.nix
index ce86fc5f494d..756decd2039d 100644
--- a/nixos/tests/boot-stage1.nix
+++ b/nixos/tests/boot-stage1.nix
@@ -33,6 +33,8 @@ import ./make-test-python.nix ({ pkgs, ... }: {
         #include <linux/sched/signal.h>
         #endif
 
+        MODULE_LICENSE("GPL");
+
         struct task_struct *canaryTask;
 
         static int kcanary(void *nothing)
diff --git a/nixos/tests/boot.nix b/nixos/tests/boot.nix
index 9945a1dcd62f..cf5565667131 100644
--- a/nixos/tests/boot.nix
+++ b/nixos/tests/boot.nix
@@ -12,12 +12,22 @@ let
   iso =
     (import ../lib/eval-config.nix {
       inherit system;
-      modules =
-        [ ../modules/installer/cd-dvd/installation-cd-minimal.nix
-          ../modules/testing/test-instrumentation.nix
-        ];
+      modules = [
+        ../modules/installer/cd-dvd/installation-cd-minimal.nix
+        ../modules/testing/test-instrumentation.nix
+      ];
     }).config.system.build.isoImage;
 
+  sd =
+    (import ../lib/eval-config.nix {
+      inherit system;
+      modules = [
+        ../modules/installer/sd-card/sd-image-x86_64.nix
+        ../modules/testing/test-instrumentation.nix
+        { sdImage.compressImage = false; }
+      ];
+    }).config.system.build.sdImage;
+
   pythonDict = params: "\n    {\n        ${concatStringsSep ",\n        " (mapAttrsToList (name: param: "\"${name}\": \"${param}\"") params)},\n    }\n";
 
   makeBootTest = name: extraConfig:
@@ -110,4 +120,30 @@ in {
     };
 
     biosNetboot = makeNetbootTest "bios" {};
+
+    ubootExtlinux = let
+      sdImage = "${sd}/sd-image/${sd.imageName}";
+      mutableImage = "/tmp/linked-image.qcow2";
+
+      machineConfig = pythonDict {
+        bios = "${pkgs.ubootQemuX86}/u-boot.rom";
+        qemuFlags = "-m 768 -machine type=pc,accel=tcg -drive file=${mutableImage},if=ide,format=qcow2";
+      };
+    in makeTest {
+      name = "boot-uboot-extlinux";
+      nodes = { };
+      testScript = ''
+        import os
+
+        # Create a mutable linked image backed by the read-only SD image
+        if os.system("qemu-img create -f qcow2 -F raw -b ${sdImage} ${mutableImage}") != 0:
+            raise RuntimeError("Could not create mutable linked image")
+
+        machine = create_machine(${machineConfig})
+        machine.start()
+        machine.wait_for_unit("multi-user.target")
+        machine.succeed("nix store verify -r --no-trust --option experimental-features nix-command /run/current-system")
+        machine.shutdown()
+      '';
+    };
 }
diff --git a/nixos/tests/borgbackup.nix b/nixos/tests/borgbackup.nix
index cbb28689209b..d3cd6c66bfeb 100644
--- a/nixos/tests/borgbackup.nix
+++ b/nixos/tests/borgbackup.nix
@@ -106,7 +106,7 @@ in {
       services.openssh = {
         enable = true;
         passwordAuthentication = false;
-        challengeResponseAuthentication = false;
+        kbdInteractiveAuthentication = false;
       };
 
       services.borgbackup.repos.repo1 = {
diff --git a/nixos/tests/bpf.nix b/nixos/tests/bpf.nix
index 233c7dab1ee2..e479cd057921 100644
--- a/nixos/tests/bpf.nix
+++ b/nixos/tests/bpf.nix
@@ -18,8 +18,12 @@ import ./make-test-python.nix ({ pkgs, ... }: {
     # simple BEGIN probe (user probe on bpftrace itself)
     print(machine.succeed("bpftrace -e 'BEGIN { print(\"ok\"); exit(); }'"))
     # tracepoint
-    print(machine.succeed("bpftrace -e 'tracepoint:syscalls:sys_enter_* { print(probe); exit(); }'"))
+    print(machine.succeed("bpftrace -e 'tracepoint:syscalls:sys_enter_* { print(probe); exit() }'"))
     # kprobe
     print(machine.succeed("bpftrace -e 'kprobe:schedule { print(probe); exit() }'"))
+    # BTF
+    print(machine.succeed("bpftrace -e 'kprobe:schedule { "
+        "    printf(\"tgid: %d\", ((struct task_struct*) curtask)->tgid); exit() "
+        "}'"))
   '';
 })
diff --git a/nixos/tests/breitbandmessung.nix b/nixos/tests/breitbandmessung.nix
new file mode 100644
index 000000000000..12b1a094839b
--- /dev/null
+++ b/nixos/tests/breitbandmessung.nix
@@ -0,0 +1,33 @@
+import ./make-test-python.nix ({ lib, ... }: {
+  name = "breitbandmessung";
+  meta.maintainers = with lib.maintainers; [ b4dm4n ];
+
+  machine = { pkgs, ... }: {
+    imports = [
+      ./common/user-account.nix
+      ./common/x11.nix
+    ];
+
+    # increase screen size to make the whole program visible
+    virtualisation.resolution = { x = 1280; y = 1024; };
+
+    test-support.displayManager.auto.user = "alice";
+
+    environment.systemPackages = with pkgs; [ breitbandmessung ];
+    environment.variables.XAUTHORITY = "/home/alice/.Xauthority";
+
+    # breitbandmessung is unfree
+    nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "breitbandmessung" ];
+  };
+
+  enableOCR = true;
+
+  testScript = ''
+    machine.wait_for_x()
+    machine.execute("su - alice -c breitbandmessung >&2  &")
+    machine.wait_for_window("Breitbandmessung")
+    machine.wait_for_text("Breitbandmessung")
+    machine.wait_for_text("Datenschutz")
+    machine.screenshot("breitbandmessung")
+  '';
+})
diff --git a/nixos/tests/btrbk.nix b/nixos/tests/btrbk.nix
index 2689bb66c63a..9f34f7dfbe38 100644
--- a/nixos/tests/btrbk.nix
+++ b/nixos/tests/btrbk.nix
@@ -53,7 +53,7 @@ import ./make-test-python.nix ({ pkgs, ... }:
         services.openssh = {
           enable = true;
           passwordAuthentication = false;
-          challengeResponseAuthentication = false;
+          kbdInteractiveAuthentication = false;
         };
         services.btrbk = {
           extraPackages = [ pkgs.lz4 ];
diff --git a/nixos/tests/cloud-init.nix b/nixos/tests/cloud-init.nix
index e06cbd056a32..3f191ff5616e 100644
--- a/nixos/tests/cloud-init.nix
+++ b/nixos/tests/cloud-init.nix
@@ -35,6 +35,24 @@ let
       public-keys:
         - "${snakeOilPublicKey}"
       EOF
+
+      cat << EOF > $out/iso/network-config
+      version: 1
+      config:
+          - type: physical
+            name: eth0
+            mac_address: '52:54:00:12:34:56'
+            subnets:
+            - type: static
+              address: '12.34.56.78'
+              netmask: '255.255.255.0'
+              gateway: '12.34.56.9'
+          - type: nameserver
+            address:
+            - '8.8.8.8'
+            search:
+            - 'example.com'
+      EOF
       ${pkgs.cdrkit}/bin/genisoimage -volid cidata -joliet -rock -o $out/metadata.iso $out/iso
       '';
   };
@@ -46,9 +64,13 @@ in makeTest {
   machine = { ... }:
   {
     virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ];
-    services.cloud-init.enable = true;
+    services.cloud-init = {
+      enable = true;
+      network.enable = true;
+    };
     services.openssh.enable = true;
     networking.hostName = "";
+    networking.useDHCP = false;
   };
   testScript = ''
     # To wait until cloud-init terminates its run
@@ -80,5 +102,8 @@ in makeTest {
         ).strip()
         == "test"
     )
+
+    assert "default via 12.34.56.9 dev eth0 proto static" in unnamed.succeed("ip route")
+    assert "12.34.56.0/24 dev eth0 proto kernel scope link src 12.34.56.78" in unnamed.succeed("ip route")
   '';
 }
diff --git a/nixos/tests/cntr.nix b/nixos/tests/cntr.nix
index 668470756209..e4e13545b876 100644
--- a/nixos/tests/cntr.nix
+++ b/nixos/tests/cntr.nix
@@ -28,10 +28,16 @@ let
       testScript = ''
         start_all()
         ${backend}.wait_for_unit("${backend}-nginx.service")
-        result = ${backend}.wait_until_succeeds(
-            "cntr attach -t ${backend} nginx sh -- -c 'curl localhost | grep Hello'"
+        ${backend}.wait_for_open_port(8181)
+        # For some reason, the cntr command hangs when run without the &.
+        # As such, we have to do some messy things to ensure we check the exitcode and output in a race-condition-safe manner
+        ${backend}.execute(
+            "(cntr attach -t ${backend} nginx sh -- -c 'curl localhost | grep Hello' > /tmp/result; echo $? > /tmp/exitcode; touch /tmp/done) &"
         )
-        assert "Hello" in result
+
+        ${backend}.wait_for_file("/tmp/done")
+        assert "0" == ${backend}.succeed("cat /tmp/exitcode").strip(), "non-zero exit code"
+        assert "Hello" in ${backend}.succeed("cat /tmp/result"), "no greeting in output"
       '';
     };
 
@@ -54,7 +60,13 @@ let
     testScript = ''
       machine.start()
       machine.wait_for_unit("container@test.service")
-      machine.succeed("cntr attach test sh -- -c 'ping -c5 172.16.0.1'")
+      # I haven't observed the same hanging behaviour in this version as in the OCI version which necessetates this messy invocation, but it's probably better to be safe than sorry and use it here as well
+      machine.execute(
+          "(cntr attach test sh -- -c 'ping -c5 172.16.0.1'; echo $? > /tmp/exitcode; touch /tmp/done) &"
+      )
+
+      machine.wait_for_file("/tmp/done")
+      assert "0" == machine.succeed("cat /tmp/exitcode").strip(), "non-zero exit code"
     '';
   };
 in {
diff --git a/nixos/tests/collectd.nix b/nixos/tests/collectd.nix
new file mode 100644
index 000000000000..cb196224a231
--- /dev/null
+++ b/nixos/tests/collectd.nix
@@ -0,0 +1,33 @@
+import ./make-test-python.nix ({ pkgs, ... }: {
+  name = "collectd";
+  meta = { };
+
+  machine =
+    { pkgs, ... }:
+
+    {
+      services.collectd = {
+        enable = true;
+        plugins = {
+          rrdtool = ''
+            DataDir "/var/lib/collectd/rrd"
+          '';
+          load = "";
+        };
+      };
+      environment.systemPackages = [ pkgs.rrdtool ];
+    };
+
+  testScript = ''
+    machine.wait_for_unit("collectd.service")
+    hostname = machine.succeed("hostname").strip()
+    file = f"/var/lib/collectd/rrd/{hostname}/load/load.rrd"
+    machine.wait_for_file(file);
+    machine.succeed(f"rrdinfo {file} | logger")
+    # check that this file contains a shortterm metric
+    machine.succeed(f"rrdinfo {file} | grep -F 'ds[shortterm].min = '")
+    # check that there are frequent updates
+    machine.succeed(f"cp {file} before")
+    machine.wait_until_fails(f"cmp before {file}")
+  '';
+})
diff --git a/nixos/tests/common/acme/client/default.nix b/nixos/tests/common/acme/client/default.nix
index 1e9885e375c7..9dbe345e7a01 100644
--- a/nixos/tests/common/acme/client/default.nix
+++ b/nixos/tests/common/acme/client/default.nix
@@ -5,9 +5,11 @@ let
 
 in {
   security.acme = {
-    server = "https://${caDomain}/dir";
-    email = "hostmaster@example.test";
     acceptTerms = true;
+    defaults = {
+      server = "https://${caDomain}/dir";
+      email = "hostmaster@example.test";
+    };
   };
 
   security.pki.certificateFiles = [ caCert ];
diff --git a/nixos/tests/common/acme/server/default.nix b/nixos/tests/common/acme/server/default.nix
index 1c3bfdf76b7e..450d49e60399 100644
--- a/nixos/tests/common/acme/server/default.nix
+++ b/nixos/tests/common/acme/server/default.nix
@@ -120,6 +120,11 @@ in {
         enable = true;
         description = "Pebble ACME server";
         wantedBy = [ "network.target" ];
+        environment = {
+          # We're not testing lego, we're just testing our configuration.
+          # No need to sleep.
+          PEBBLE_VA_NOSLEEP = "1";
+        };
 
         serviceConfig = {
           RuntimeDirectory = "pebble";
diff --git a/nixos/tests/containers-imperative.nix b/nixos/tests/containers-imperative.nix
index a126a5480c03..14001657bee0 100644
--- a/nixos/tests/containers-imperative.nix
+++ b/nixos/tests/containers-imperative.nix
@@ -10,8 +10,8 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
 
       # XXX: Sandbox setup fails while trying to hardlink files from the host's
       #      store file system into the prepared chroot directory.
-      nix.useSandbox = false;
-      nix.binaryCaches = []; # don't try to access cache.nixos.org
+      nix.settings.sandbox = false;
+      nix.settings.substituters = []; # don't try to access cache.nixos.org
 
       virtualisation.writableStore = true;
       # Make sure we always have all the required dependencies for creating a
diff --git a/nixos/tests/couchdb.nix b/nixos/tests/couchdb.nix
index 049532481b15..453f5dcd66e8 100644
--- a/nixos/tests/couchdb.nix
+++ b/nixos/tests/couchdb.nix
@@ -56,5 +56,8 @@ with lib;
     couchdb3.succeed(
         "${curlJqCheck testlogin "GET" "_all_dbs" ". | length" "0"}"
     )
+    couchdb3.succeed(
+        "${curlJqCheck testlogin "GET" "_node/couchdb@127.0.0.1" ".couchdb" "Welcome"}"
+    )
   '';
 })
diff --git a/nixos/tests/dnsdist.nix b/nixos/tests/dnsdist.nix
new file mode 100644
index 000000000000..cfc41c13864e
--- /dev/null
+++ b/nixos/tests/dnsdist.nix
@@ -0,0 +1,48 @@
+import ./make-test-python.nix (
+  { pkgs, ... }: {
+    name = "dnsdist";
+    meta = with pkgs.lib; {
+      maintainers = with maintainers; [ jojosch ];
+    };
+
+    machine = { pkgs, lib, ... }: {
+      services.bind = {
+        enable = true;
+        extraOptions = "empty-zones-enable no;";
+        zones = lib.singleton {
+          name = ".";
+          master = true;
+          file = pkgs.writeText "root.zone" ''
+            $TTL 3600
+            . IN SOA ns.example.org. admin.example.org. ( 1 3h 1h 1w 1d )
+            . IN NS ns.example.org.
+
+            ns.example.org. IN A    192.168.0.1
+            ns.example.org. IN AAAA abcd::1
+
+            1.0.168.192.in-addr.arpa IN PTR ns.example.org.
+          '';
+        };
+      };
+      services.dnsdist = {
+        enable = true;
+        listenPort = 5353;
+        extraConfig = ''
+          newServer({address="127.0.0.1:53", name="local-bind"})
+        '';
+      };
+
+      environment.systemPackages = with pkgs; [ dig ];
+    };
+
+    testScript = ''
+      machine.wait_for_unit("bind.service")
+      machine.wait_for_open_port(53)
+      machine.succeed("dig @127.0.0.1 +short -x 192.168.0.1 | grep -qF ns.example.org")
+
+      machine.wait_for_unit("dnsdist.service")
+      machine.wait_for_open_port(5353)
+      machine.succeed("dig @127.0.0.1 -p 5353 +short -x 192.168.0.1 | grep -qF ns.example.org")
+    '';
+  }
+)
diff --git a/nixos/tests/docker-rootless.nix b/nixos/tests/docker-rootless.nix
new file mode 100644
index 000000000000..e2a926eb3cb0
--- /dev/null
+++ b/nixos/tests/docker-rootless.nix
@@ -0,0 +1,41 @@
+# This test runs docker and checks if simple container starts
+
+import ./make-test-python.nix ({ lib, pkgs, ...} : {
+  name = "docker-rootless";
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [ abbradar ];
+  };
+
+  nodes = {
+    machine = { pkgs, ... }: {
+      virtualisation.docker.rootless.enable = true;
+
+      users.users.alice = {
+        uid = 1000;
+        isNormalUser = true;
+      };
+    };
+  };
+
+  testScript = { nodes, ... }:
+    let
+      user = nodes.machine.config.users.users.alice;
+      sudo = lib.concatStringsSep " " [
+        "XDG_RUNTIME_DIR=/run/user/${toString user.uid}"
+        "DOCKER_HOST=unix:///run/user/${toString user.uid}/docker.sock"
+        "sudo" "--preserve-env=XDG_RUNTIME_DIR,DOCKER_HOST" "-u" "alice"
+      ];
+    in ''
+      machine.wait_for_unit("multi-user.target")
+
+      machine.succeed("loginctl enable-linger alice")
+      machine.wait_until_succeeds("${sudo} systemctl --user is-active docker.service")
+
+      machine.succeed("tar cv --files-from /dev/null | ${sudo} docker import - scratchimg")
+      machine.succeed(
+          "${sudo} docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin scratchimg /bin/sleep 10"
+      )
+      machine.succeed("${sudo} docker ps | grep sleeping")
+      machine.succeed("${sudo} docker stop sleeping")
+    '';
+})
diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix
index 19ebed3ebd0b..8a240ddb17f2 100644
--- a/nixos/tests/docker-tools.nix
+++ b/nixos/tests/docker-tools.nix
@@ -215,6 +215,12 @@ import ./make-test-python.nix ({ pkgs, ... }: {
                 f"docker run --rm  ${examples.layersOrder.imageName} cat /tmp/layer{index}"
             )
 
+    with subtest("Ensure layers unpacked in correct order before runAsRoot runs"):
+        assert "abc" in docker.succeed(
+            "docker load --input='${examples.layersUnpackOrder}'",
+            "docker run --rm ${examples.layersUnpackOrder.imageName} cat /layer-order"
+        )
+
     with subtest("Ensure environment variables are correctly inherited"):
         docker.succeed(
             "docker load --input='${examples.environmentVariables}'"
diff --git a/nixos/tests/doh-proxy-rust.nix b/nixos/tests/doh-proxy-rust.nix
index 23f8616849c3..11ed87d23bbe 100644
--- a/nixos/tests/doh-proxy-rust.nix
+++ b/nixos/tests/doh-proxy-rust.nix
@@ -38,6 +38,6 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: {
     machine.wait_for_unit("doh-proxy-rust.service")
     machine.wait_for_open_port(53)
     machine.wait_for_open_port(3000)
-    machine.succeed(f"curl --fail '{url}?dns={query}' | grep -F {bin_ip}")
+    machine.succeed(f"curl --fail -H 'Accept: application/dns-message' '{url}?dns={query}' | grep -F {bin_ip}")
   '';
 })
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index ae746d7e1f03..f42be00f23b8 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -40,9 +40,8 @@ let
 
             services = {
 
-              journalbeat = let lt6 = builtins.compareVersions
-                                        elk.journalbeat.version "6" < 0; in {
-                enable = true;
+              journalbeat = {
+                enable = elk ? journalbeat;
                 package = elk.journalbeat;
                 extraConfig = pkgs.lib.mkOptionDefault (''
                   logging:
@@ -51,14 +50,29 @@ let
                     metrics.enabled: false
                   output.elasticsearch:
                     hosts: [ "127.0.0.1:9200" ]
-                    ${pkgs.lib.optionalString lt6 "template.enabled: false"}
-                '' + pkgs.lib.optionalString (!lt6) ''
                   journalbeat.inputs:
                   - paths: []
                     seek: cursor
                 '');
               };
 
+              filebeat = {
+                enable = elk ? filebeat;
+                package = elk.filebeat;
+                inputs.journald.id = "everything";
+
+                inputs.log = {
+                  enabled = true;
+                  paths = [
+                    "/var/lib/filebeat/test"
+                  ];
+                };
+
+                settings = {
+                  logging.level = "info";
+                };
+              };
+
               metricbeat = {
                 enable = true;
                 package = elk.metricbeat;
@@ -142,27 +156,43 @@ let
       };
 
     passthru.elkPackages = elk;
-    testScript = ''
+    testScript =
+      let
+        valueObject = lib.optionalString (lib.versionAtLeast elk.elasticsearch.version "7") ".value";
+      in ''
       import json
 
 
-      def total_hits(message):
+      def expect_hits(message):
+          dictionary = {"query": {"match": {"message": message}}}
+          return (
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+              + "-H 'Content-Type: application/json' "
+              + "-d '{}' ".format(json.dumps(dictionary))
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+          )
+
+
+      def expect_no_hits(message):
           dictionary = {"query": {"match": {"message": message}}}
           return (
-              "curl --silent --show-error '${esUrl}/_search' "
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + "| jq .hits.total"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} == 0 end'"
           )
 
 
       def has_metricbeat():
           dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}}
           return (
-              "curl --silent --show-error '${esUrl}/_search' "
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + "| jq '.hits.total > 0'"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
 
 
@@ -178,7 +208,8 @@ let
       # TODO: extend this test with multiple elasticsearch nodes
       #       and see if the status turns "green".
       one.wait_until_succeeds(
-          "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
+          "curl --silent --show-error --fail-with-body '${esUrl}/_cluster/health'"
+          + " | jq -es 'if . == [] then null else .[] | .status != \"red\" end'"
       )
 
       with subtest("Perform some simple logstash tests"):
@@ -189,33 +220,50 @@ let
       with subtest("Kibana is healthy"):
           one.wait_for_unit("kibana.service")
           one.wait_until_succeeds(
-              "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
+              "curl --silent --show-error --fail-with-body 'http://localhost:5601/api/status'"
+              + " | jq -es 'if . == [] then null else .[] | .status.overall.state == \"green\" end'"
           )
 
       with subtest("Metricbeat is running"):
           one.wait_for_unit("metricbeat.service")
 
       with subtest("Metricbeat metrics arrive in elasticsearch"):
-          one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'")
+          one.wait_until_succeeds(has_metricbeat())
 
       with subtest("Logstash messages arive in elasticsearch"):
-          one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
-          one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
+          one.wait_until_succeeds(expect_hits("flowers"))
+          one.wait_until_succeeds(expect_no_hits("dragons"))
 
+    '' + lib.optionalString (elk ? journalbeat) ''
       with subtest(
           "A message logged to the journal is ingested by elasticsearch via journalbeat"
       ):
           one.wait_for_unit("journalbeat.service")
           one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
           one.wait_until_succeeds(
-              total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
+              expect_hits("Supercalifragilisticexpialidocious")
           )
-
+    '' + lib.optionalString (elk ? filebeat) ''
+      with subtest(
+          "A message logged to the journal is ingested by elasticsearch via filebeat"
+      ):
+          one.wait_for_unit("filebeat.service")
+          one.execute("echo 'Superdupercalifragilisticexpialidocious' | systemd-cat")
+          one.wait_until_succeeds(
+              expect_hits("Superdupercalifragilisticexpialidocious")
+          )
+          one.execute(
+              "echo 'SuperdupercalifragilisticexpialidociousIndeed' >> /var/lib/filebeat/test"
+          )
+          one.wait_until_succeeds(
+              expect_hits("SuperdupercalifragilisticexpialidociousIndeed")
+          )
+    '' + ''
       with subtest("Elasticsearch-curator works"):
           one.systemctl("stop logstash")
           one.systemctl("start elasticsearch-curator")
           one.wait_until_succeeds(
-              '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^'
+              '! curl --silent --show-error --fail-with-body "${esUrl}/_cat/indices" | grep logstash | grep ^'
           )
     '';
   }) { inherit pkgs system; };
@@ -235,7 +283,7 @@ in {
   #   elasticsearch = pkgs.elasticsearch7-oss;
   #   logstash      = pkgs.logstash7-oss;
   #   kibana        = pkgs.kibana7-oss;
-  #   journalbeat   = pkgs.journalbeat7;
+  #   filebeat      = pkgs.filebeat7;
   #   metricbeat    = pkgs.metricbeat7;
   # };
   unfree = lib.dontRecurseIntoAttrs {
@@ -250,7 +298,7 @@ in {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
-      journalbeat   = pkgs.journalbeat7;
+      filebeat      = pkgs.filebeat7;
       metricbeat    = pkgs.metricbeat7;
     };
   };
diff --git a/nixos/tests/empty-file b/nixos/tests/empty-file
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixos/tests/empty-file
diff --git a/nixos/tests/ergochat.nix b/nixos/tests/ergochat.nix
new file mode 100644
index 000000000000..2e9dc55e648e
--- /dev/null
+++ b/nixos/tests/ergochat.nix
@@ -0,0 +1,97 @@
+let
+  clients = [
+    "ircclient1"
+    "ircclient2"
+  ];
+  server = "ergochat";
+  ircPort = 6667;
+  channel = "nixos-cat";
+  iiDir = "/tmp/irc";
+in
+
+import ./make-test-python.nix ({ pkgs, lib, ... }: {
+  name = "ergochat";
+  nodes = {
+    "${server}" = {
+      networking.firewall.allowedTCPPorts = [ ircPort ];
+      services.ergochat = {
+        enable = true;
+        settings.server.motd = pkgs.writeText "ergo.motd" ''
+          The default MOTD doesn't contain the word "nixos" in it.
+          This one does.
+        '';
+      };
+    };
+  } // lib.listToAttrs (builtins.map (client: lib.nameValuePair client {
+    imports = [
+      ./common/user-account.nix
+    ];
+
+    systemd.services.ii = {
+      requires = [ "network.target" ];
+      wantedBy = [ "default.target" ];
+
+      serviceConfig = {
+        Type = "simple";
+        ExecPreStartPre = "mkdir -p ${iiDir}";
+        ExecStart = ''
+          ${lib.getBin pkgs.ii}/bin/ii -n ${client} -s ${server} -i ${iiDir}
+        '';
+        User = "alice";
+      };
+    };
+  }) clients);
+
+  testScript =
+    let
+      msg = client: "Hello, my name is ${client}";
+      clientScript = client: [
+        ''
+          ${client}.wait_for_unit("network.target")
+          ${client}.systemctl("start ii")
+          ${client}.wait_for_unit("ii")
+          ${client}.wait_for_file("${iiDir}/${server}/out")
+        ''
+        # look for the custom text in the MOTD.
+        ''
+          ${client}.wait_until_succeeds("grep 'nixos' ${iiDir}/${server}/out")
+        ''
+        # wait until first PING from server arrives before joining,
+        # so we don't try it too early
+        ''
+          ${client}.wait_until_succeeds("grep 'PING' ${iiDir}/${server}/out")
+        ''
+        # join ${channel}
+        ''
+          ${client}.succeed("echo '/j #${channel}' > ${iiDir}/${server}/in")
+          ${client}.wait_for_file("${iiDir}/${server}/#${channel}/in")
+        ''
+        # send a greeting
+        ''
+          ${client}.succeed(
+              "echo '${msg client}' > ${iiDir}/${server}/#${channel}/in"
+          )
+        ''
+        # check that all greetings arrived on all clients
+      ] ++ builtins.map (other: ''
+        ${client}.succeed(
+            "grep '${msg other}$' ${iiDir}/${server}/#${channel}/out"
+        )
+      '') clients;
+
+      # foldl', but requires a non-empty list instead of a start value
+      reduce = f: list:
+        builtins.foldl' f (builtins.head list) (builtins.tail list);
+    in ''
+      start_all()
+      ${server}.systemctl("status ergochat")
+      ${server}.wait_for_open_port(${toString ircPort})
+
+      # run clientScript for all clients so that every list
+      # entry is executed by every client before advancing
+      # to the next one.
+    '' + lib.concatStrings
+      (reduce
+        (lib.zipListsWith (cs: c: cs + c))
+        (builtins.map clientScript clients));
+})
diff --git a/nixos/tests/frr.nix b/nixos/tests/frr.nix
new file mode 100644
index 000000000000..598d7a7d2867
--- /dev/null
+++ b/nixos/tests/frr.nix
@@ -0,0 +1,104 @@
+# This test runs FRR and checks if OSPF routing works.
+#
+# Network topology:
+#   [ client ]--net1--[ router1 ]--net2--[ router2 ]--net3--[ server ]
+#
+# All interfaces are in OSPF Area 0.
+
+import ./make-test-python.nix ({ pkgs, ... }:
+  let
+
+    ifAddr = node: iface: (pkgs.lib.head node.config.networking.interfaces.${iface}.ipv4.addresses).address;
+
+    ospfConf1 = ''
+      router ospf
+        network 192.168.0.0/16 area 0
+    '';
+
+    ospfConf2 = ''
+      interface eth2
+        ip ospf hello-interval 1
+        ip ospf dead-interval 5
+      !
+      router ospf
+        network 192.168.0.0/16 area 0
+    '';
+
+  in
+    {
+      name = "frr";
+
+      meta = with pkgs.lib.maintainers; {
+        maintainers = [ hexa ];
+      };
+
+      nodes = {
+
+        client =
+          { nodes, ... }:
+          {
+            virtualisation.vlans = [ 1 ];
+            networking.defaultGateway = ifAddr nodes.router1 "eth1";
+          };
+
+        router1 =
+          { ... }:
+          {
+            virtualisation.vlans = [ 1 2 ];
+            boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
+            networking.firewall.extraCommands = "iptables -A nixos-fw -i eth2 -p ospfigp -j ACCEPT";
+            services.frr.ospf = {
+              enable = true;
+              config = ospfConf1;
+            };
+
+            specialisation.ospf.configuration = {
+              services.frr.ospf.config = ospfConf2;
+            };
+          };
+
+        router2 =
+          { ... }:
+          {
+            virtualisation.vlans = [ 3 2 ];
+            boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
+            networking.firewall.extraCommands = "iptables -A nixos-fw -i eth2 -p ospfigp -j ACCEPT";
+            services.frr.ospf = {
+              enable = true;
+              config = ospfConf2;
+            };
+          };
+
+        server =
+          { nodes, ... }:
+          {
+            virtualisation.vlans = [ 3 ];
+            networking.defaultGateway = ifAddr nodes.router2 "eth1";
+          };
+      };
+
+      testScript =
+        { nodes, ... }:
+        ''
+          start_all()
+
+          # Wait for the networking to start on all machines
+          for machine in client, router1, router2, server:
+              machine.wait_for_unit("network.target")
+
+          with subtest("Wait for Zebra and OSPFD"):
+              for gw in router1, router2:
+                  gw.wait_for_unit("zebra")
+                  gw.wait_for_unit("ospfd")
+
+          router1.succeed("${nodes.router1.config.system.build.toplevel}/specialisation/ospf/bin/switch-to-configuration test >&2")
+
+          with subtest("Wait for OSPF to form adjacencies"):
+              for gw in router1, router2:
+                  gw.wait_until_succeeds("vtysh -c 'show ip ospf neighbor' | grep Full")
+                  gw.wait_until_succeeds("vtysh -c 'show ip route' | grep '^O>'")
+
+          with subtest("Test ICMP"):
+              client.wait_until_succeeds("ping -c 3 server >&2")
+        '';
+    })
diff --git a/nixos/tests/geth.nix b/nixos/tests/geth.nix
index 10cbd6d9038c..af8230553bbb 100644
--- a/nixos/tests/geth.nix
+++ b/nixos/tests/geth.nix
@@ -31,7 +31,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
     machine.wait_for_open_port(18545)
 
     machine.succeed(
-        'geth attach --exec "eth.chainId()" http://localhost:8545 | grep \'"0x0"\' '
+        'geth attach --exec eth.blockNumber http://localhost:8545 | grep \'^0$\' '
     )
 
     machine.succeed(
diff --git a/nixos/tests/gitolite-fcgiwrap.nix b/nixos/tests/gitolite-fcgiwrap.nix
index fc9b214b762e..38f8d5c883fd 100644
--- a/nixos/tests/gitolite-fcgiwrap.nix
+++ b/nixos/tests/gitolite-fcgiwrap.nix
@@ -42,7 +42,7 @@ import ./make-test-python.nix (
                     auth_basic_user_file /etc/gitolite/htpasswd;
 
                     # common FastCGI parameters are required
-                    include ${pkgs.nginx}/conf/fastcgi_params;
+                    include ${config.services.nginx.package}/conf/fastcgi_params;
 
                     # strip the CGI program prefix
                     fastcgi_split_path_info ^(/git)(.*)$;
diff --git a/nixos/tests/google-oslogin/default.nix b/nixos/tests/google-oslogin/default.nix
index dea660ed05a4..72c87d7153bd 100644
--- a/nixos/tests/google-oslogin/default.nix
+++ b/nixos/tests/google-oslogin/default.nix
@@ -31,10 +31,10 @@ in {
 
     # mockserver should return a non-expired ssh key for both mockuser and mockadmin
     server.succeed(
-        f'${pkgs.google-compute-engine-oslogin}/bin/google_authorized_keys {MOCKUSER} | grep -q "${snakeOilPublicKey}"'
+        f'${pkgs.google-guest-oslogin}/bin/google_authorized_keys {MOCKUSER} | grep -q "${snakeOilPublicKey}"'
     )
     server.succeed(
-        f'${pkgs.google-compute-engine-oslogin}/bin/google_authorized_keys {MOCKADMIN} | grep -q "${snakeOilPublicKey}"'
+        f'${pkgs.google-guest-oslogin}/bin/google_authorized_keys {MOCKADMIN} | grep -q "${snakeOilPublicKey}"'
     )
 
     # install snakeoil ssh key on the client, and provision .ssh/config file
diff --git a/nixos/tests/google-oslogin/server.nix b/nixos/tests/google-oslogin/server.nix
index fdb7141da317..faf5e847d7e9 100644
--- a/nixos/tests/google-oslogin/server.nix
+++ b/nixos/tests/google-oslogin/server.nix
@@ -17,13 +17,11 @@ in {
   };
 
   services.openssh.enable = true;
-  services.openssh.challengeResponseAuthentication = false;
+  services.openssh.kbdInteractiveAuthentication = false;
   services.openssh.passwordAuthentication = false;
 
   security.googleOsLogin.enable = true;
 
   # Mock google service
-  networking.extraHosts = ''
-    127.0.0.1 metadata.google.internal
-  '';
+  networking.interfaces.lo.ipv4.addresses = [ { address = "169.254.169.254"; prefixLength = 32; } ];
 }
diff --git a/nixos/tests/google-oslogin/server.py b/nixos/tests/google-oslogin/server.py
index 5ea9bbd2c96b..5ea9bbd2c96b 100644..100755
--- a/nixos/tests/google-oslogin/server.py
+++ b/nixos/tests/google-oslogin/server.py
diff --git a/nixos/tests/hadoop/default.nix b/nixos/tests/hadoop/default.nix
new file mode 100644
index 000000000000..d2a97cbeffb8
--- /dev/null
+++ b/nixos/tests/hadoop/default.nix
@@ -0,0 +1,7 @@
+{ handleTestOn, package, ... }:
+
+{
+  all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; };
+  hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; };
+  yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; };
+}
diff --git a/nixos/tests/hadoop/hadoop.nix b/nixos/tests/hadoop/hadoop.nix
index 48737debab54..b132f4fa58b0 100644
--- a/nixos/tests/hadoop/hadoop.nix
+++ b/nixos/tests/hadoop/hadoop.nix
@@ -1,121 +1,148 @@
 # This test is very comprehensive. It tests whether all hadoop services work well with each other.
 # Run this when updating the Hadoop package or making significant changes to the hadoop module.
 # For a more basic test, see hdfs.nix and yarn.nix
-import ../make-test-python.nix ({pkgs, ...}: {
-
-  nodes = let
-    package = pkgs.hadoop;
-    coreSite = {
-      "fs.defaultFS" = "hdfs://ns1";
-    };
-    hdfsSite = {
-      "dfs.namenode.rpc-bind-host" = "0.0.0.0";
-      "dfs.namenode.http-bind-host" = "0.0.0.0";
-      "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
-
-      # HA Quorum Journal Manager configuration
-      "dfs.nameservices" = "ns1";
-      "dfs.ha.namenodes.ns1" = "nn1,nn2";
-      "dfs.namenode.shared.edits.dir.ns1.nn1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
-      "dfs.namenode.shared.edits.dir.ns1.nn2" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
-      "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020";
-      "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020";
-      "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022";
-      "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022";
-      "dfs.namenode.http-address.ns1.nn1" = "nn1:9870";
-      "dfs.namenode.http-address.ns1.nn2" = "nn2:9870";
-
-      # Automatic failover configuration
-      "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
-      "dfs.ha.automatic-failover.enabled.ns1" = "true";
-      "dfs.ha.fencing.methods" = "shell(true)";
-      "ha.zookeeper.quorum" = "zk1:2181";
-    };
-    yarnSiteHA = {
-      "yarn.resourcemanager.zk-address" = "zk1:2181";
-      "yarn.resourcemanager.ha.enabled" = "true";
-      "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
-      "yarn.resourcemanager.hostname.rm1" = "rm1";
-      "yarn.resourcemanager.hostname.rm2" = "rm2";
-      "yarn.resourcemanager.ha.automatic-failover.enabled" = "true";
-      "yarn.resourcemanager.cluster-id" = "cluster1";
-      # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
-      # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
-      # that causes AM containers to fail otherwise.
-      "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
-      "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
-    };
-  in {
-    zk1 = { ... }: {
-      services.zookeeper.enable = true;
-      networking.firewall.allowedTCPPorts = [ 2181 ];
-    };
-
-    # HDFS cluster
-    nn1 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        hdfs.namenode.enable = true;
-        hdfs.zkfc.enable = true;
+import ../make-test-python.nix ({ package, ... }: {
+  name = "hadoop-combined";
+
+  nodes =
+    let
+      coreSite = {
+        "fs.defaultFS" = "hdfs://ns1";
+      };
+      hdfsSite = {
+        # HA Quorum Journal Manager configuration
+        "dfs.nameservices" = "ns1";
+        "dfs.ha.namenodes.ns1" = "nn1,nn2";
+        "dfs.namenode.shared.edits.dir.ns1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
+        "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020";
+        "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020";
+        "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022";
+        "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022";
+        "dfs.namenode.http-address.ns1.nn1" = "nn1:9870";
+        "dfs.namenode.http-address.ns1.nn2" = "nn2:9870";
+
+        # Automatic failover configuration
+        "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
+        "dfs.ha.automatic-failover.enabled.ns1" = "true";
+        "dfs.ha.fencing.methods" = "shell(true)";
+        "ha.zookeeper.quorum" = "zk1:2181";
+      };
+      yarnSite = {
+        "yarn.resourcemanager.zk-address" = "zk1:2181";
+        "yarn.resourcemanager.ha.enabled" = "true";
+        "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
+        "yarn.resourcemanager.hostname.rm1" = "rm1";
+        "yarn.resourcemanager.hostname.rm2" = "rm2";
+        "yarn.resourcemanager.ha.automatic-failover.enabled" = "true";
+        "yarn.resourcemanager.cluster-id" = "cluster1";
+        # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
+        # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
+        # that causes AM containers to fail otherwise.
+        "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
+        "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
+      };
+    in
+    {
+      zk1 = { ... }: {
+        services.zookeeper.enable = true;
+        networking.firewall.allowedTCPPorts = [ 2181 ];
       };
-    };
-    nn2 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        hdfs.namenode.enable = true;
-        hdfs.zkfc.enable = true;
+
+      # HDFS cluster
+      nn1 = { ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite;
+          hdfs.namenode = {
+            enable = true;
+            openFirewall = true;
+          };
+          hdfs.zkfc.enable = true;
+        };
+      };
+      nn2 = { ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite;
+          hdfs.namenode = {
+            enable = true;
+            openFirewall = true;
+          };
+          hdfs.zkfc.enable = true;
+        };
       };
-    };
 
-    jn1 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        hdfs.journalnode.enable = true;
+      jn1 = { ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite;
+          hdfs.journalnode = {
+            enable = true;
+            openFirewall = true;
+          };
+        };
       };
-    };
-    jn2 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        hdfs.journalnode.enable = true;
+      jn2 = { ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite;
+          hdfs.journalnode = {
+            enable = true;
+            openFirewall = true;
+          };
+        };
       };
-    };
-    jn3 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        hdfs.journalnode.enable = true;
+      jn3 = { ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite;
+          hdfs.journalnode = {
+            enable = true;
+            openFirewall = true;
+          };
+        };
+      };
+
+      dn1 = { ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite;
+          hdfs.datanode = {
+            enable = true;
+            openFirewall = true;
+          };
+        };
       };
-    };
 
-    dn1 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        hdfs.datanode.enable = true;
+      # YARN cluster
+      rm1 = { options, ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite yarnSite;
+          yarn.resourcemanager = {
+            enable = true;
+            openFirewall = true;
+          };
+        };
       };
-    };
-
-    # YARN cluster
-    rm1 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
-        yarn.resourcemanager.enable = true;
+      rm2 = { options, ... }: {
+        services.hadoop = {
+          inherit package coreSite hdfsSite yarnSite;
+          yarn.resourcemanager = {
+            enable = true;
+            openFirewall = true;
+          };
+        };
       };
-    };
-    rm2 = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
-        yarn.resourcemanager.enable = true;
+      nm1 = { options, ... }: {
+        virtualisation.memorySize = 2048;
+        services.hadoop = {
+          inherit package coreSite hdfsSite yarnSite;
+          yarn.nodemanager = {
+            enable = true;
+            openFirewall = true;
+          };
+        };
       };
-    };
-    nm1 = {pkgs, options, ...}: {
-      virtualisation.memorySize = 2048;
-      services.hadoop = {
-        inherit package coreSite hdfsSite;
-        yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
-        yarn.nodemanager.enable = true;
+      client = { options, ... }: {
+        services.hadoop = {
+          gatewayRole.enable = true;
+          inherit package coreSite hdfsSite yarnSite;
+        };
       };
-    };
   };
 
   testScript = ''
@@ -173,26 +200,26 @@ import ../make-test-python.nix ({pkgs, ...}: {
     # DN should have started by now, but confirm anyway
     dn1.wait_for_unit("hdfs-datanode")
     # Print states of namenodes
-    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
     # Wait for cluster to exit safemode
-    dn1.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
-    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+    client.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
+    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
     # test R/W
-    dn1.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
-    assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
+    client.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
+    assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
 
     # Test NN failover
     nn1.succeed("systemctl stop hdfs-namenode")
-    assert "active" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
-    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
-    assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
+    assert "active" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
+    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+    assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
 
     nn1.succeed("systemctl start hdfs-namenode")
     nn1.wait_for_open_port(9870)
     nn1.wait_for_open_port(8022)
     nn1.wait_for_open_port(8020)
-    assert "standby" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
-    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+    assert "standby" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
+    client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
 
     #### YARN tests ####
 
@@ -208,21 +235,21 @@ import ../make-test-python.nix ({pkgs, ...}: {
     nm1.wait_for_unit("yarn-nodemanager")
     nm1.wait_for_open_port(8042)
     nm1.wait_for_open_port(8040)
-    nm1.wait_until_succeeds("yarn node -list | grep Nodes:1")
-    nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
-    nm1.succeed("sudo -u yarn yarn node -list | systemd-cat")
+    client.wait_until_succeeds("yarn node -list | grep Nodes:1")
+    client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
+    client.succeed("sudo -u yarn yarn node -list | systemd-cat")
 
     # Test RM failover
     rm1.succeed("systemctl stop yarn-resourcemanager")
-    assert "standby" not in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
-    nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
+    assert "standby" not in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
+    client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
     rm1.succeed("systemctl start yarn-resourcemanager")
     rm1.wait_for_unit("yarn-resourcemanager")
     rm1.wait_for_open_port(8088)
-    assert "standby" in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
-    nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
+    assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
+    client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
 
-    assert "Estimated value of Pi is" in nm1.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
-    assert "SUCCEEDED" in nm1.succeed("yarn application -list -appStates FINISHED")
+    assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
+    assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED")
   '';
 })
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix
index b63cbf480327..9415500463de 100644
--- a/nixos/tests/hadoop/hdfs.nix
+++ b/nixos/tests/hadoop/hdfs.nix
@@ -1,32 +1,46 @@
 # Test a minimal HDFS cluster with no HA
-import ../make-test-python.nix ({...}: {
-  nodes = {
-    namenode = {pkgs, ...}: {
+import ../make-test-python.nix ({ package, lib, ... }:
+with lib;
+{
+  name = "hadoop-hdfs";
+
+  nodes = let
+    coreSite = {
+      "fs.defaultFS" = "hdfs://namenode:8020";
+      "hadoop.proxyuser.httpfs.groups" = "*";
+      "hadoop.proxyuser.httpfs.hosts" = "*";
+    };
+    in {
+    namenode = { pkgs, ... }: {
       services.hadoop = {
-        package = pkgs.hadoop;
+        inherit package;
         hdfs = {
           namenode = {
             enable = true;
+            openFirewall = true;
             formatOnInit = true;
           };
-          httpfs.enable = true;
-        };
-        coreSite = {
-          "fs.defaultFS" = "hdfs://namenode:8020";
-          "hadoop.proxyuser.httpfs.groups" = "*";
-          "hadoop.proxyuser.httpfs.hosts" = "*";
+          httpfs = {
+            # The NixOS hadoop module only support webHDFS on 3.3 and newer
+            enable = mkIf (versionAtLeast package.version "3.3") true;
+            openFirewall = true;
+          };
         };
+        inherit coreSite;
       };
     };
-    datanode = {pkgs, ...}: {
+    datanode = { pkgs, ... }: {
       services.hadoop = {
-        package = pkgs.hadoop;
-        hdfs.datanode.enable = true;
-        coreSite = {
-          "fs.defaultFS" = "hdfs://namenode:8020";
-          "hadoop.proxyuser.httpfs.groups" = "*";
-          "hadoop.proxyuser.httpfs.hosts" = "*";
+        inherit package;
+        hdfs.datanode = {
+          enable = true;
+          openFirewall = true;
+          dataDirs = [{
+            type = "DISK";
+            path = "/tmp/dn1";
+          }];
         };
+        inherit coreSite;
       };
     };
   };
@@ -37,21 +51,32 @@ import ../make-test-python.nix ({...}: {
     namenode.wait_for_unit("hdfs-namenode")
     namenode.wait_for_unit("network.target")
     namenode.wait_for_open_port(8020)
+    namenode.succeed("ss -tulpne | systemd-cat")
+    namenode.succeed("cat /etc/hadoop*/hdfs-site.xml | systemd-cat")
     namenode.wait_for_open_port(9870)
 
     datanode.wait_for_unit("hdfs-datanode")
     datanode.wait_for_unit("network.target")
+  '' + ( if versionAtLeast package.version "3" then ''
     datanode.wait_for_open_port(9864)
     datanode.wait_for_open_port(9866)
     datanode.wait_for_open_port(9867)
 
-    namenode.succeed("curl -f http://namenode:9870")
     datanode.succeed("curl -f http://datanode:9864")
+  '' else ''
+    datanode.wait_for_open_port(50075)
+    datanode.wait_for_open_port(50010)
+    datanode.wait_for_open_port(50020)
+
+    datanode.succeed("curl -f http://datanode:50075")
+  '' ) + ''
+    namenode.succeed("curl -f http://namenode:9870")
 
     datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
     datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
     assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
 
+  '' + optionalString ( versionAtLeast package.version "3.3" ) ''
     namenode.wait_for_unit("hdfs-httpfs")
     namenode.wait_for_open_port(14000)
     assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1")
diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix
index 09bdb35791c7..1bf8e3831f67 100644
--- a/nixos/tests/hadoop/yarn.nix
+++ b/nixos/tests/hadoop/yarn.nix
@@ -1,22 +1,30 @@
 # This only tests if YARN is able to start its services
-import ../make-test-python.nix ({...}: {
+import ../make-test-python.nix ({ package, ... }: {
+  name = "hadoop-yarn";
+
   nodes = {
-    resourcemanager = {pkgs, ...}: {
-      services.hadoop.package = pkgs.hadoop;
-      services.hadoop.yarn.resourcemanager.enable = true;
-      services.hadoop.yarnSite = {
-        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
+    resourcemanager = { ... }: {
+      services.hadoop = {
+        inherit package;
+        yarn.resourcemanager = {
+          enable = true;
+          openFirewall = true;
+        };
       };
     };
-    nodemanager = {pkgs, ...}: {
-      services.hadoop.package = pkgs.hadoop;
-      services.hadoop.yarn.nodemanager.enable = true;
-      services.hadoop.yarnSite = {
-        "yarn.resourcemanager.hostname" = "resourcemanager";
-        "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
+    nodemanager = { options, lib, ... }: {
+      services.hadoop = {
+        inherit package;
+        yarn.nodemanager = {
+          enable = true;
+          openFirewall = true;
+        };
+        yarnSite = options.services.hadoop.yarnSite.default // {
+          "yarn.resourcemanager.hostname" = "resourcemanager";
+          "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
+        };
       };
     };
-
   };
 
   testScript = ''
diff --git a/nixos/tests/handbrake.nix b/nixos/tests/handbrake.nix
deleted file mode 100644
index d2d41b372be1..000000000000
--- a/nixos/tests/handbrake.nix
+++ /dev/null
@@ -1,33 +0,0 @@
-import ./make-test-python.nix ({ pkgs, ... }:
-
-let
-  # Download Big Buck Bunny example, licensed under CC Attribution 3.0.
-  testMkv = pkgs.fetchurl {
-    url = "https://github.com/Matroska-Org/matroska-test-files/blob/cf0792be144ac470c4b8052cfe19bb691993e3a2/test_files/test1.mkv?raw=true";
-    sha256 = "1hfxbbgxwfkzv85pvpvx55a72qsd0hxjbm9hkl5r3590zw4s75h9";
-    name = "test1.mkv";
-  };
-
-in
-{
-  name = "handbrake";
-
-  meta = {
-    maintainers = with pkgs.lib.maintainers; [ ];
-  };
-
-  machine = { pkgs, ... }: {
-    environment.systemPackages = with pkgs; [ handbrake ];
-  };
-
-  testScript = ''
-    # Test MP4 and MKV transcoding. Since this is a short clip, transcoding typically
-    # only takes a few seconds.
-    start_all()
-
-    machine.succeed("HandBrakeCLI -i ${testMkv} -o test.mp4 -e x264 -q 20 -B 160")
-    machine.succeed("test -e test.mp4")
-    machine.succeed("HandBrakeCLI -i ${testMkv} -o test.mkv -e x264 -q 20 -B 160")
-    machine.succeed("test -e test.mkv")
-  '';
-})
diff --git a/nixos/tests/haproxy.nix b/nixos/tests/haproxy.nix
index 2c3878131b68..b6ff4102fe68 100644
--- a/nixos/tests/haproxy.nix
+++ b/nixos/tests/haproxy.nix
@@ -16,7 +16,6 @@ import ./make-test-python.nix ({ pkgs, ...}: {
           frontend http
             bind *:80
             mode http
-            option http-use-htx
             http-request use-service prometheus-exporter if { path /metrics }
             use_backend http_server
         '';
diff --git a/nixos/tests/hardened.nix b/nixos/tests/hardened.nix
index da7e0972e131..dc455f971f5c 100644
--- a/nixos/tests/hardened.nix
+++ b/nixos/tests/hardened.nix
@@ -11,7 +11,7 @@ import ./make-test-python.nix ({ pkgs, ... } : {
       users.users.sybil = { isNormalUser = true; group = "wheel"; };
       imports = [ ../modules/profiles/hardened.nix ];
       environment.memoryAllocator.provider = "graphene-hardened";
-      nix.useSandbox = false;
+      nix.settings.sandbox = false;
       virtualisation.emptyDiskImages = [ 4096 ];
       boot.initrd.postDeviceCommands = ''
         ${pkgs.dosfstools}/bin/mkfs.vfat -n EFISYS /dev/vdb
diff --git a/nixos/tests/hibernate.nix b/nixos/tests/hibernate.nix
index 4f05b99a5a11..3880f1649bd3 100644
--- a/nixos/tests/hibernate.nix
+++ b/nixos/tests/hibernate.nix
@@ -45,11 +45,11 @@ in makeTest {
         ../modules/profiles/base.nix
       ];
 
-      nix.binaryCaches = mkForce [ ];
-      nix.extraOptions = ''
-        hashed-mirrors =
-        connect-timeout = 1
-      '';
+      nix.settings = {
+        substituters = mkForce [];
+        hashed-mirrors = null;
+        connect-timeout = 1;
+      };
 
       virtualisation.diskSize = 8 * 1024;
       virtualisation.emptyDiskImages = [
diff --git a/nixos/tests/home-assistant.nix b/nixos/tests/home-assistant.nix
index 1ab5755863f7..10f9cb05c9cb 100644
--- a/nixos/tests/home-assistant.nix
+++ b/nixos/tests/home-assistant.nix
@@ -2,33 +2,48 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
 
 let
   configDir = "/var/lib/foobar";
-  mqttUsername = "homeassistant";
-  mqttPassword = "secret";
 in {
   name = "home-assistant";
   meta.maintainers = lib.teams.home-assistant.members;
 
   nodes.hass = { pkgs, ... }: {
     environment.systemPackages = with pkgs; [ mosquitto ];
-    services.mosquitto = {
+
+    services.postgresql = {
       enable = true;
-      listeners = [ {
-        users = {
-          "${mqttUsername}" = {
-            acl = [ "readwrite #" ];
-            password = mqttPassword;
-          };
+      ensureDatabases = [ "hass" ];
+      ensureUsers = [{
+        name = "hass";
+        ensurePermissions = {
+          "DATABASE hass" = "ALL PRIVILEGES";
         };
-      } ];
+      }];
     };
+
     services.home-assistant = {
-      inherit configDir;
       enable = true;
+      inherit configDir;
+
+      # tests loading components by overriding the package
       package = (pkgs.home-assistant.override {
+        extraPackages = ps: with ps; [
+          colorama
+        ];
         extraComponents = [ "zha" ];
       }).overrideAttrs (oldAttrs: {
         doInstallCheck = false;
       });
+
+      # tests loading components from the module
+      extraComponents = [
+        "wake_on_lan"
+      ];
+
+      # test extra package passing from the module
+      extraPackages = python3Packages: with python3Packages; [
+        psycopg2
+      ];
+
       config = {
         homeassistant = {
           name = "Home";
@@ -37,28 +52,39 @@ in {
           longitude = "0.0";
           elevation = 0;
         };
+
+        # configure the recorder component to use the postgresql db
+        recorder.db_url = "postgresql://@/hass";
+
+        # we can't load default_config, because the updater requires
+        # network access and would cause an error, so load frontend
+        # here explicitly.
+        # https://www.home-assistant.io/integrations/frontend/
         frontend = {};
-        mqtt = {
-          broker = "127.0.0.1";
-          username = mqttUsername;
-          password = mqttPassword;
-        };
-        binary_sensor = [{
-          platform = "mqtt";
-          state_topic = "home-assistant/test";
-          payload_on = "let_there_be_light";
-          payload_off = "off";
-        }];
-        # tests component-based capability assignment (CAP_NET_BIND_SERVICE)
+
+        # set up a wake-on-lan switch to test capset capability required
+        # for the ping suid wrapper
+        # https://www.home-assistant.io/integrations/wake_on_lan/
+        switch = [ {
+          platform = "wake_on_lan";
+          mac = "00:11:22:33:44:55";
+          host = "127.0.0.1";
+        } ];
+
+        # test component-based capability assignment (CAP_NET_BIND_SERVICE)
+        # https://www.home-assistant.io/integrations/emulated_hue/
         emulated_hue = {
           host_ip = "127.0.0.1";
           listen_port = 80;
         };
+
+        # https://www.home-assistant.io/integrations/logger/
         logger = {
           default = "info";
-          logs."homeassistant.components.mqtt" = "debug";
         };
       };
+
+      # configure the sample lovelace dashboard
       lovelaceConfig = {
         title = "My Awesome Home";
         views = [{
@@ -75,25 +101,46 @@ in {
   };
 
   testScript = ''
+    import re
+
     start_all()
+
+    # Parse the package path out of the systemd unit, as we cannot
+    # access the final package, that is overriden inside the module,
+    # by any other means.
+    pattern = re.compile(r"path=(?P<path>[\/a-z0-9-.]+)\/bin\/hass")
+    response = hass.execute("systemctl show -p ExecStart home-assistant.service")[1]
+    match = pattern.search(response)
+    package = match.group('path')
+
     hass.wait_for_unit("home-assistant.service")
+
     with subtest("Check that YAML configuration file is in place"):
         hass.succeed("test -L ${configDir}/configuration.yaml")
-    with subtest("lovelace config is copied because lovelaceConfigWritable = true"):
+
+    with subtest("Check the lovelace config is copied because lovelaceConfigWritable = true"):
         hass.succeed("test -f ${configDir}/ui-lovelace.yaml")
+
+    with subtest("Check extraComponents and extraPackages are considered from the package"):
+        hass.succeed(f"grep -q 'colorama' {package}/extra_packages")
+        hass.succeed(f"grep -q 'zha' {package}/extra_components")
+
+    with subtest("Check extraComponents and extraPackages are considered from the module"):
+        hass.succeed(f"grep -q 'psycopg2' {package}/extra_packages")
+        hass.succeed(f"grep -q 'wake_on_lan' {package}/extra_components")
+
     with subtest("Check that Home Assistant's web interface and API can be reached"):
+        hass.wait_until_succeeds("journalctl -u home-assistant.service | grep -q 'Home Assistant initialized in'")
         hass.wait_for_open_port(8123)
         hass.succeed("curl --fail http://localhost:8123/lovelace")
-    with subtest("Toggle a binary sensor using MQTT"):
-        hass.wait_for_open_port(1883)
-        hass.succeed(
-            "mosquitto_pub -V mqttv5 -t home-assistant/test -u ${mqttUsername} -P '${mqttPassword}' -m let_there_be_light"
-        )
+
     with subtest("Check that capabilities are passed for emulated_hue to bind to port 80"):
         hass.wait_for_open_port(80)
         hass.succeed("curl --fail http://localhost:80/description.xml")
+
     with subtest("Check extra components are considered in systemd unit hardening"):
         hass.succeed("systemctl show -p DeviceAllow home-assistant.service | grep -q char-ttyUSB")
+
     with subtest("Print log to ease debugging"):
         output_log = hass.succeed("cat ${configDir}/home-assistant.log")
         print("\n### home-assistant.log ###\n")
@@ -102,12 +149,8 @@ in {
     with subtest("Check that no errors were logged"):
         assert "ERROR" not in output_log
 
-    # example line: 2020-06-20 10:01:32 DEBUG (MainThread) [homeassistant.components.mqtt] Received message on home-assistant/test: b'let_there_be_light'
-    with subtest("Check we received the mosquitto message"):
-        assert "let_there_be_light" in output_log
-
     with subtest("Check systemd unit hardening"):
-        hass.log(hass.succeed("systemctl show home-assistant.service"))
+        hass.log(hass.succeed("systemctl cat home-assistant.service"))
         hass.log(hass.succeed("systemd-analyze security home-assistant.service"))
   '';
 })
diff --git a/nixos/tests/hydra/common.nix b/nixos/tests/hydra/common.nix
index 1a3a4d8fb3d4..fdf2b2c6f6dc 100644
--- a/nixos/tests/hydra/common.nix
+++ b/nixos/tests/hydra/common.nix
@@ -42,7 +42,7 @@
         hostName = "localhost";
         systems = [ system ];
       }];
-      binaryCaches = [];
+      settings.substituters = [];
     };
   };
 }
diff --git a/nixos/tests/hydra/default.nix b/nixos/tests/hydra/default.nix
index d92f032b8292..ef5e677953dc 100644
--- a/nixos/tests/hydra/default.nix
+++ b/nixos/tests/hydra/default.nix
@@ -17,7 +17,7 @@ let
   makeHydraTest = with pkgs.lib; name: package: makeTest {
     name = "hydra-${name}";
     meta = with pkgs.lib.maintainers; {
-      maintainers = [ pstn lewo ma27 ];
+      maintainers = [ lewo ma27 ];
     };
 
     machine = { pkgs, lib, ... }: {
diff --git a/nixos/tests/input-remapper.nix b/nixos/tests/input-remapper.nix
new file mode 100644
index 000000000000..f692564caa57
--- /dev/null
+++ b/nixos/tests/input-remapper.nix
@@ -0,0 +1,52 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+  {
+    name = "input-remapper";
+    meta = {
+      maintainers = with pkgs.lib.maintainers; [ LunNova ];
+    };
+
+    machine = { config, ... }:
+      let user = config.users.users.sybil; in
+      {
+        imports = [
+          ./common/user-account.nix
+          ./common/x11.nix
+        ];
+
+        services.xserver.enable = true;
+        services.input-remapper.enable = true;
+        users.users.sybil = { isNormalUser = true; group = "wheel"; };
+        test-support.displayManager.auto.user = user.name;
+        # workaround for pkexec not working in the test environment
+        # Error creating textual authentication agent:
+        #   Error opening current controlling terminal for the process (`/dev/tty'):
+        #   No such device or address
+        # passwordless pkexec with polkit module also doesn't work
+        # to allow the program to run, we replace pkexec with sudo
+        # and turn on passwordless sudo
+        # this is not correct in general but good enough for this test
+        security.sudo = { enable = true; wheelNeedsPassword = false; };
+        security.wrappers.pkexec = pkgs.lib.mkForce
+          {
+            setuid = true;
+            owner = "root";
+            group = "root";
+            source = "${pkgs.sudo}/bin/sudo";
+          };
+      };
+
+    enableOCR = true;
+
+    testScript = { nodes, ... }: ''
+      start_all()
+      machine.wait_for_x()
+
+      machine.succeed("systemctl status input-remapper.service")
+      machine.execute("su - sybil -c input-remapper-gtk >&2 &")
+
+      machine.wait_for_text("Input Remapper")
+      machine.wait_for_text("Preset")
+      machine.wait_for_text("Change Key")
+    '';
+  })
diff --git a/nixos/tests/installed-tests/appstream-qt.nix b/nixos/tests/installed-tests/appstream-qt.nix
new file mode 100644
index 000000000000..d08187bfe466
--- /dev/null
+++ b/nixos/tests/installed-tests/appstream-qt.nix
@@ -0,0 +1,9 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.libsForQt5.appstream-qt;
+
+  testConfig = {
+    appstream.enable = true;
+  };
+}
diff --git a/nixos/tests/installed-tests/appstream.nix b/nixos/tests/installed-tests/appstream.nix
new file mode 100644
index 000000000000..f71a095d4452
--- /dev/null
+++ b/nixos/tests/installed-tests/appstream.nix
@@ -0,0 +1,9 @@
+{ pkgs, makeInstalledTest, ... }:
+
+makeInstalledTest {
+  tested = pkgs.appstream;
+
+  testConfig = {
+    appstream.enable = true;
+  };
+}
diff --git a/nixos/tests/installed-tests/default.nix b/nixos/tests/installed-tests/default.nix
index 08785e5e6669..079fd54e71e5 100644
--- a/nixos/tests/installed-tests/default.nix
+++ b/nixos/tests/installed-tests/default.nix
@@ -84,6 +84,8 @@ let
 in
 
 {
+  appstream = callInstalledTest ./appstream.nix {};
+  appstream-qt = callInstalledTest ./appstream-qt.nix {};
   colord = callInstalledTest ./colord.nix {};
   flatpak = callInstalledTest ./flatpak.nix {};
   flatpak-builder = callInstalledTest ./flatpak-builder.nix {};
diff --git a/nixos/tests/installed-tests/flatpak.nix b/nixos/tests/installed-tests/flatpak.nix
index 8aeeaca90f61..c7fe9cf45882 100644
--- a/nixos/tests/installed-tests/flatpak.nix
+++ b/nixos/tests/installed-tests/flatpak.nix
@@ -6,6 +6,7 @@ makeInstalledTest {
 
   testConfig = {
     xdg.portal.enable = true;
+    xdg.portal.extraPortals = [ pkgs.xdg-desktop-portal-gtk ];
     services.flatpak.enable = true;
     environment.systemPackages = with pkgs; [ gnupg ostree python3 ];
     virtualisation.memorySize = 2047;
diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix
index bc41b6efc2e7..5525c3117b79 100644
--- a/nixos/tests/installer.nix
+++ b/nixos/tests/installer.nix
@@ -306,6 +306,9 @@ let
           # The test cannot access the network, so any packages we
           # need must be included in the VM.
           system.extraDependencies = with pkgs; [
+            brotli
+            brotli.dev
+            brotli.lib
             desktop-file-utils
             docbook5
             docbook_xsl_ns
@@ -315,6 +318,7 @@ let
             ntp
             perlPackages.ListCompare
             perlPackages.XMLLibXML
+            python3Minimal
             shared-mime-info
             sudo
             texinfo
@@ -334,11 +338,11 @@ let
             (pkgs.grub2_efi.override { inherit zfsSupport; })
           ]);
 
-          nix.binaryCaches = mkForce [ ];
-          nix.extraOptions = ''
-            hashed-mirrors =
-            connect-timeout = 1
-          '';
+          nix.settings = {
+            substituters = mkForce [];
+            hashed-mirrors = null;
+            connect-timeout = 1;
+          };
         };
 
       };
@@ -561,26 +565,16 @@ in {
           + " mkpart primary 2048M -1s"  # PV2
           + " set 2 lvm on",
           "udevadm settle",
-          "sleep 1",
           "pvcreate /dev/vda1 /dev/vda2",
-          "sleep 1",
           "vgcreate MyVolGroup /dev/vda1 /dev/vda2",
-          "sleep 1",
           "lvcreate --size 1G --name swap MyVolGroup",
-          "sleep 1",
           "lvcreate --size 3G --name nixos MyVolGroup",
-          "sleep 1",
           "mkswap -f /dev/MyVolGroup/swap -L swap",
           "swapon -L swap",
           "mkfs.xfs -L nixos /dev/MyVolGroup/nixos",
           "mount LABEL=nixos /mnt",
       )
     '';
-    postBootCommands = ''
-      assert "loaded active" in machine.succeed(
-          "systemctl list-units 'lvm2-pvscan@*' -ql --no-legend | tee /dev/stderr"
-      )
-    '';
   };
 
   # Boot off an encrypted root partition with the default LUKS header format
@@ -681,8 +675,6 @@ in {
           "modprobe bcache",
           "udevadm settle",
           "make-bcache -B /dev/vda4 -C /dev/vda3",
-          "echo /dev/vda3 > /sys/fs/bcache/register",
-          "echo /dev/vda4 > /sys/fs/bcache/register",
           "udevadm settle",
           "mkfs.ext3 -L nixos /dev/bcache0",
           "mount LABEL=nixos /mnt",
diff --git a/nixos/tests/invoiceplane.nix b/nixos/tests/invoiceplane.nix
new file mode 100644
index 000000000000..4e63f8ac21c9
--- /dev/null
+++ b/nixos/tests/invoiceplane.nix
@@ -0,0 +1,82 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+{
+  name = "invoiceplane";
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [
+      onny
+    ];
+  };
+
+  nodes = {
+    invoiceplane_caddy = { ... }: {
+      services.invoiceplane.webserver = "caddy";
+      services.invoiceplane.sites = {
+        "site1.local" = {
+          #database.name = "invoiceplane1";
+          database.createLocally = true;
+          enable = true;
+        };
+        "site2.local" = {
+          #database.name = "invoiceplane2";
+          database.createLocally = true;
+          enable = true;
+        };
+      };
+
+      networking.firewall.allowedTCPPorts = [ 80 ];
+      networking.hosts."127.0.0.1" = [ "site1.local" "site2.local" ];
+    };
+  };
+
+  testScript = ''
+    start_all()
+
+    invoiceplane_caddy.wait_for_unit("caddy")
+    invoiceplane_caddy.wait_for_open_port(80)
+    invoiceplane_caddy.wait_for_open_port(3306)
+
+    site_names = ["site1.local", "site2.local"]
+
+    for site_name in site_names:
+        machine.wait_for_unit(f"phpfpm-invoiceplane-{site_name}")
+
+        with subtest("Website returns welcome screen"):
+            assert "Please install InvoicePlane" in machine.succeed(f"curl -L {site_name}")
+
+        with subtest("Finish InvoicePlane setup"):
+          machine.succeed(
+            f"curl -sSfL --cookie-jar cjar {site_name}/index.php/setup/language"
+          )
+          csrf_token = machine.succeed(
+            "grep ip_csrf_cookie cjar | cut -f 7 | tr -d '\n'"
+          )
+          machine.succeed(
+            f"curl -sSfL --cookie cjar --cookie-jar cjar -d '_ip_csrf={csrf_token}&ip_lang=english&btn_continue=Continue' {site_name}/index.php/setup/language"
+          )
+          csrf_token = machine.succeed(
+            "grep ip_csrf_cookie cjar | cut -f 7 | tr -d '\n'"
+          )
+          machine.succeed(
+            f"curl -sSfL --cookie cjar --cookie-jar cjar -d '_ip_csrf={csrf_token}&btn_continue=Continue' {site_name}/index.php/setup/prerequisites"
+          )
+          csrf_token = machine.succeed(
+            "grep ip_csrf_cookie cjar | cut -f 7 | tr -d '\n'"
+          )
+          machine.succeed(
+            f"curl -sSfL --cookie cjar --cookie-jar cjar -d '_ip_csrf={csrf_token}&btn_continue=Continue' {site_name}/index.php/setup/configure_database"
+          )
+          csrf_token = machine.succeed(
+            "grep ip_csrf_cookie cjar | cut -f 7 | tr -d '\n'"
+          )
+          machine.succeed(
+            f"curl -sSfl --cookie cjar --cookie-jar cjar -d '_ip_csrf={csrf_token}&btn_continue=Continue' {site_name}/index.php/setup/install_tables"
+          )
+          csrf_token = machine.succeed(
+            "grep ip_csrf_cookie cjar | cut -f 7 | tr -d '\n'"
+          )
+          machine.succeed(
+            f"curl -sSfl --cookie cjar --cookie-jar cjar -d '_ip_csrf={csrf_token}&btn_continue=Continue' {site_name}/index.php/setup/upgrade_tables"
+          )
+  '';
+})
diff --git a/nixos/tests/iscsi-multipath-root.nix b/nixos/tests/iscsi-multipath-root.nix
index a26fea503b62..92ae9990c947 100644
--- a/nixos/tests/iscsi-multipath-root.nix
+++ b/nixos/tests/iscsi-multipath-root.nix
@@ -111,11 +111,11 @@ import ./make-test-python.nix (
 
         environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
 
-        nix.binaryCaches = lib.mkForce [ ];
-        nix.extraOptions = ''
-          hashed-mirrors =
-          connect-timeout = 1
-        '';
+        nix.settings = {
+          substituters = lib.mkForce [ ];
+          hashed-mirrors = null;
+          connect-timeout = 1;
+        };
       };
 
       initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
diff --git a/nixos/tests/iscsi-root.nix b/nixos/tests/iscsi-root.nix
index bda51d2c2e42..eb0719edc379 100644
--- a/nixos/tests/iscsi-root.nix
+++ b/nixos/tests/iscsi-root.nix
@@ -95,11 +95,11 @@ import ./make-test-python.nix (
 
             system.extraDependencies = [ nodes.initiatorRootDisk.config.system.build.toplevel ];
 
-            nix.binaryCaches = lib.mkForce [];
-            nix.extraOptions = ''
-              hashed-mirrors =
-              connect-timeout = 1
-            '';
+            nix.settings = {
+              substituters = lib.mkForce [];
+              hashed-mirrors = null;
+              connect-timeout = 1;
+            };
           };
 
           initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
diff --git a/nixos/tests/jibri.nix b/nixos/tests/jibri.nix
index 3dd28e6aac1a..af20e639d30e 100644
--- a/nixos/tests/jibri.nix
+++ b/nixos/tests/jibri.nix
@@ -63,7 +63,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
         """sleep 15 && curl -H "Content-Type: application/json" -X POST http://localhost:2222/jibri/api/v1.0/stopService -d '{"sessionId": "RecordTest","callParams":{"callUrlInfo":{"baseUrl": "https://machine","callName": "TestCall"}},"callLoginParams":{"domain": "recorder.machine", "username": "recorder", "password": "'"$(cat /var/lib/jitsi-meet/jibri-recorder-secret)"'" },"sinkType": "file"}'"""
     )
     machine.wait_until_succeeds(
-        "cat /var/log/jitsi/jibri/log.0.txt | grep -q 'Recording finalize script finished with exit value 0'", timeout=36
+        "cat /var/log/jitsi/jibri/log.0.txt | grep -q 'Finalize script finished with exit value 0'", timeout=36
     )
   '';
 })
diff --git a/nixos/tests/k3s-single-node-docker.nix b/nixos/tests/k3s-single-node-docker.nix
new file mode 100644
index 000000000000..7f3d15788b04
--- /dev/null
+++ b/nixos/tests/k3s-single-node-docker.nix
@@ -0,0 +1,84 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+  let
+    imageEnv = pkgs.buildEnv {
+      name = "k3s-pause-image-env";
+      paths = with pkgs; [ tini (hiPrio coreutils) busybox ];
+    };
+    pauseImage = pkgs.dockerTools.streamLayeredImage {
+      name = "test.local/pause";
+      tag = "local";
+      contents = imageEnv;
+      config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ];
+    };
+    # Don't use the default service account because there's a race where it may
+    # not be created yet; make our own instead.
+    testPodYaml = pkgs.writeText "test.yml" ''
+      apiVersion: v1
+      kind: ServiceAccount
+      metadata:
+        name: test
+      ---
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        name: test
+      spec:
+        serviceAccountName: test
+        containers:
+        - name: test
+          image: test.local/pause:local
+          imagePullPolicy: Never
+          command: ["sh", "-c", "sleep inf"]
+    '';
+  in
+  {
+    name = "k3s";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ euank ];
+    };
+
+    machine = { pkgs, ... }: {
+      environment.systemPackages = with pkgs; [ k3s gzip ];
+
+      # k3s uses enough resources the default vm fails.
+      virtualisation.memorySize = 1536;
+      virtualisation.diskSize = 4096;
+
+      services.k3s = {
+        enable = true;
+        role = "server";
+        docker = true;
+        # Slightly reduce resource usage
+        extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local";
+      };
+
+      users.users = {
+        noprivs = {
+          isNormalUser = true;
+          description = "Can't access k3s by default";
+          password = "*";
+        };
+      };
+    };
+
+    testScript = ''
+      start_all()
+
+      machine.wait_for_unit("k3s")
+      machine.succeed("k3s kubectl cluster-info")
+      machine.fail("sudo -u noprivs k3s kubectl cluster-info")
+      # FIXME: this fails with the current nixos kernel config; once it passes, we should uncomment it
+      # machine.succeed("k3s check-config")
+
+      machine.succeed(
+          "${pauseImage} | docker load"
+      )
+
+      machine.succeed("k3s kubectl apply -f ${testPodYaml}")
+      machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test")
+      machine.succeed("k3s kubectl delete -f ${testPodYaml}")
+
+      machine.shutdown()
+    '';
+  })
diff --git a/nixos/tests/k3s-single-node.nix b/nixos/tests/k3s-single-node.nix
new file mode 100644
index 000000000000..d98f20d468cb
--- /dev/null
+++ b/nixos/tests/k3s-single-node.nix
@@ -0,0 +1,82 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+  let
+    imageEnv = pkgs.buildEnv {
+      name = "k3s-pause-image-env";
+      paths = with pkgs; [ tini (hiPrio coreutils) busybox ];
+    };
+    pauseImage = pkgs.dockerTools.streamLayeredImage {
+      name = "test.local/pause";
+      tag = "local";
+      contents = imageEnv;
+      config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ];
+    };
+    # Don't use the default service account because there's a race where it may
+    # not be created yet; make our own instead.
+    testPodYaml = pkgs.writeText "test.yml" ''
+      apiVersion: v1
+      kind: ServiceAccount
+      metadata:
+        name: test
+      ---
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        name: test
+      spec:
+        serviceAccountName: test
+        containers:
+        - name: test
+          image: test.local/pause:local
+          imagePullPolicy: Never
+          command: ["sh", "-c", "sleep inf"]
+    '';
+  in
+  {
+    name = "k3s";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ euank ];
+    };
+
+    machine = { pkgs, ... }: {
+      environment.systemPackages = with pkgs; [ k3s gzip ];
+
+      # k3s uses enough resources the default vm fails.
+      virtualisation.memorySize = 1536;
+      virtualisation.diskSize = 4096;
+
+      services.k3s.enable = true;
+      services.k3s.role = "server";
+      services.k3s.package = pkgs.k3s;
+      # Slightly reduce resource usage
+      services.k3s.extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local";
+
+      users.users = {
+        noprivs = {
+          isNormalUser = true;
+          description = "Can't access k3s by default";
+          password = "*";
+        };
+      };
+    };
+
+    testScript = ''
+      start_all()
+
+      machine.wait_for_unit("k3s")
+      machine.succeed("k3s kubectl cluster-info")
+      machine.fail("sudo -u noprivs k3s kubectl cluster-info")
+      # FIXME: this fails with the current nixos kernel config; once it passes, we should uncomment it
+      # machine.succeed("k3s check-config")
+
+      machine.succeed(
+          "${pauseImage} | k3s ctr image import -"
+      )
+
+      machine.succeed("k3s kubectl apply -f ${testPodYaml}")
+      machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test")
+      machine.succeed("k3s kubectl delete -f ${testPodYaml}")
+
+      machine.shutdown()
+    '';
+  })
diff --git a/nixos/tests/k3s.nix b/nixos/tests/k3s.nix
deleted file mode 100644
index 494a3b68b59d..000000000000
--- a/nixos/tests/k3s.nix
+++ /dev/null
@@ -1,78 +0,0 @@
-import ./make-test-python.nix ({ pkgs, ... }:
-
-let
-  # A suitable k3s pause image, also used for the test pod
-  pauseImage = pkgs.dockerTools.buildImage {
-    name = "test.local/pause";
-    tag = "local";
-    contents = with pkgs; [ tini coreutils busybox ];
-    config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ];
-  };
-  testPodYaml = pkgs.writeText "test.yml" ''
-    # Don't use the default service account because there's a race where it may
-    # not be created yet; make our own instead.
-    apiVersion: v1
-    kind: ServiceAccount
-    metadata:
-      name: test
-    ---
-    apiVersion: v1
-    kind: Pod
-    metadata:
-      name: test
-    spec:
-      serviceAccountName: test
-      containers:
-      - name: test
-        image: test.local/pause:local
-        imagePullPolicy: Never
-        command: ["sh", "-c", "sleep inf"]
-  '';
-in
-{
-  name = "k3s";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ euank ];
-  };
-
-  nodes = {
-    k3s =
-      { pkgs, ... }: {
-        environment.systemPackages = [ pkgs.k3s pkgs.gzip ];
-
-        # k3s uses enough resources the default vm fails.
-        virtualisation.memorySize = pkgs.lib.mkDefault 1536;
-        virtualisation.diskSize = pkgs.lib.mkDefault 4096;
-
-        services.k3s.enable = true;
-        services.k3s.role = "server";
-        services.k3s.package = pkgs.k3s;
-        # Slightly reduce resource usage
-        services.k3s.extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local";
-
-        users.users = {
-          noprivs = {
-            isNormalUser = true;
-            description = "Can't access k3s by default";
-            password = "*";
-          };
-        };
-      };
-  };
-
-  testScript = ''
-    start_all()
-
-    k3s.wait_for_unit("k3s")
-    k3s.succeed("k3s kubectl cluster-info")
-    k3s.fail("sudo -u noprivs k3s kubectl cluster-info")
-    # k3s.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes
-
-    k3s.succeed(
-        "zcat ${pauseImage} | k3s ctr image import -"
-    )
-
-    k3s.succeed("k3s kubectl apply -f ${testPodYaml}")
-    k3s.succeed("k3s kubectl wait --for 'condition=Ready' pod/test")
-  '';
-})
diff --git a/nixos/tests/keepassxc.nix b/nixos/tests/keepassxc.nix
index 685a200b3187..924c137a9032 100644
--- a/nixos/tests/keepassxc.nix
+++ b/nixos/tests/keepassxc.nix
@@ -15,20 +15,54 @@ import ./make-test-python.nix ({ pkgs, ...} :
     ];
 
     services.xserver.enable = true;
+
+    # Regression test for https://github.com/NixOS/nixpkgs/issues/163482
+    qt5 = {
+      enable = true;
+      platformTheme = "gnome";
+      style = "adwaita-dark";
+    };
+
     test-support.displayManager.auto.user = "alice";
-    environment.systemPackages = [ pkgs.keepassxc ];
+    environment.systemPackages = with pkgs; [
+      keepassxc
+      xdotool
+    ];
   };
 
   enableOCR = true;
 
-  testScript = { nodes, ... }: ''
-    start_all()
-    machine.wait_for_x()
+  testScript = { nodes, ... }: let
+    aliceDo = cmd: ''machine.succeed("su - alice -c '${cmd}' >&2 &");'';
+    in ''
+    with subtest("Ensure X starts"):
+        start_all()
+        machine.wait_for_x()
+
+    with subtest("Can create database and entry with CLI"):
+        ${aliceDo "keepassxc-cli db-create -k foo.keyfile foo.kdbx"}
+        ${aliceDo "keepassxc-cli add --no-password -k foo.keyfile foo.kdbx bar"}
+
+    with subtest("Ensure KeePassXC starts"):
+        # start KeePassXC window
+        ${aliceDo "keepassxc >&2 &"}
 
-    # start KeePassXC window
-    machine.execute("su - alice -c keepassxc >&2 &")
+        machine.wait_for_text("KeePassXC ${pkgs.keepassxc.version}")
+        machine.screenshot("KeePassXC")
 
-    machine.wait_for_text("KeePassXC ${pkgs.keepassxc.version}")
-    machine.screenshot("KeePassXC")
+    with subtest("Can open existing database"):
+        machine.send_key("ctrl-o")
+        machine.sleep(5)
+        # Regression #163482: keepassxc did not crash
+        machine.succeed("ps -e | grep keepassxc")
+        machine.wait_for_text("foo.kdbx")
+        machine.send_key("ret")
+        machine.sleep(1)
+        # Click on "Browse" button to select keyfile
+        machine.send_key("tab")
+        machine.send_chars("/home/alice/foo.keyfile")
+        machine.send_key("ret")
+        # Passwords folder is displayed
+        machine.wait_for_text("Passwords")
   '';
 })
diff --git a/nixos/tests/keycloak.nix b/nixos/tests/keycloak.nix
index e86faa0c9a73..a4beea40279f 100644
--- a/nixos/tests/keycloak.nix
+++ b/nixos/tests/keycloak.nix
@@ -42,7 +42,7 @@ let
 
           environment.systemPackages = with pkgs; [
             xmlstarlet
-            libtidy
+            html-tidy
             jq
           ];
         };
diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix
index 1f23ca55fb23..d4410beb937e 100644
--- a/nixos/tests/kubernetes/base.nix
+++ b/nixos/tests/kubernetes/base.nix
@@ -18,7 +18,7 @@ let
         ${master.ip}  api.${domain}
         ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
       '';
-      kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
+      wrapKubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
         mkdir -p $out/bin
         makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
       '';
@@ -48,10 +48,9 @@ let
                 };
               };
               programs.bash.enableCompletion = true;
-              environment.systemPackages = [ kubectl ];
+              environment.systemPackages = [ wrapKubectl ];
               services.flannel.iface = "eth1";
               services.kubernetes = {
-                addons.dashboard.enable = true;
                 proxy.hostname = "${masterName}.${domain}";
 
                 easyCerts = true;
@@ -61,13 +60,6 @@ let
                   advertiseAddress = master.ip;
                 };
                 masterAddress = "${masterName}.${config.networking.domain}";
-                # workaround for:
-                #   https://github.com/kubernetes/kubernetes/issues/102676
-                #   (workaround from) https://github.com/kubernetes/kubernetes/issues/95488
-                kubelet.extraOpts = ''\
-                  --cgroups-per-qos=false \
-                  --enforce-node-allocatable="" \
-                '';
               };
             }
             (optionalAttrs (any (role: role == "master") machine.roles) {
diff --git a/nixos/tests/kubernetes/default.nix b/nixos/tests/kubernetes/default.nix
index 97931234fcc7..60ba482758fb 100644
--- a/nixos/tests/kubernetes/default.nix
+++ b/nixos/tests/kubernetes/default.nix
@@ -1,5 +1,5 @@
 { system ? builtins.currentSystem
-, pkgs ? import <nixpkgs> { inherit system; }
+, pkgs ? import ../../.. { inherit system; }
 }:
 let
   dns = import ./dns.nix { inherit system pkgs; };
@@ -10,8 +10,6 @@ in
 {
   dns-single-node = dns.singlenode.test;
   dns-multi-node = dns.multinode.test;
-  # timed out: https://hydra.nixos.org/build/160710933
-  #rbac-single-node = rbac.singlenode.test;
-  # timed out: https://hydra.nixos.org/build/160711286
-  #rbac-multi-node = rbac.multinode.test;
+  rbac-single-node = rbac.singlenode.test;
+  rbac-multi-node = rbac.multinode.test;
 }
diff --git a/nixos/tests/kubernetes/dns.nix b/nixos/tests/kubernetes/dns.nix
index b6cd811c5aef..3fd1dd31f746 100644
--- a/nixos/tests/kubernetes/dns.nix
+++ b/nixos/tests/kubernetes/dns.nix
@@ -1,4 +1,4 @@
-{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
+{ system ? builtins.currentSystem, pkgs ? import ../../.. { inherit system; } }:
 with import ./base.nix { inherit system; };
 let
   domain = "my.zyx";
@@ -100,7 +100,7 @@ let
       machine1.succeed("host redis.default.svc.cluster.local")
 
       # check dns inside the container
-      machine1.succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local")
+      machine1.succeed("kubectl exec probe -- /bin/host redis.default.svc.cluster.local")
     '';
   };
 
@@ -142,7 +142,7 @@ let
       machine2.succeed("host redis.default.svc.cluster.local")
 
       # check dns inside the container
-      machine1.succeed("kubectl exec -ti probe -- /bin/host redis.default.svc.cluster.local")
+      machine1.succeed("kubectl exec probe -- /bin/host redis.default.svc.cluster.local")
     '';
   };
 in {
diff --git a/nixos/tests/kubernetes/e2e.nix b/nixos/tests/kubernetes/e2e.nix
index 175d8413045e..fb29d9cc6953 100644
--- a/nixos/tests/kubernetes/e2e.nix
+++ b/nixos/tests/kubernetes/e2e.nix
@@ -1,4 +1,4 @@
-{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
+{ system ? builtins.currentSystem, pkgs ? import ../../.. { inherit system; } }:
 with import ./base.nix { inherit system; };
 let
   domain = "my.zyx";
diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix
index 3fc8ed0fbe38..9e73fbbd32a8 100644
--- a/nixos/tests/kubernetes/rbac.nix
+++ b/nixos/tests/kubernetes/rbac.nix
@@ -1,4 +1,4 @@
-{ system ? builtins.currentSystem, pkgs ? import <nixpkgs> { inherit system; } }:
+{ system ? builtins.currentSystem, pkgs ? import ../../.. { inherit system; } }:
 with import ./base.nix { inherit system; };
 let
 
@@ -76,7 +76,7 @@ let
     }];
   });
 
-  kubectl = pkgs.runCommand "copy-kubectl" { buildInputs = [ pkgs.kubernetes ]; } ''
+  copyKubectl = pkgs.runCommand "copy-kubectl" { } ''
     mkdir -p $out/bin
     cp ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl
   '';
@@ -84,7 +84,7 @@ let
   kubectlImage = pkgs.dockerTools.buildImage {
     name = "kubectl";
     tag = "latest";
-    contents = [ kubectl pkgs.busybox kubectlPod2 ];
+    contents = [ copyKubectl pkgs.busybox kubectlPod2 ];
     config.Entrypoint = ["/bin/sh"];
   };
 
@@ -115,9 +115,9 @@ let
 
       machine1.wait_until_succeeds("kubectl get pod kubectl | grep Running")
 
-      machine1.wait_until_succeeds("kubectl exec -ti kubectl -- kubectl get pods")
-      machine1.fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json")
-      machine1.fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl")
+      machine1.wait_until_succeeds("kubectl exec kubectl -- kubectl get pods")
+      machine1.fail("kubectl exec kubectl -- kubectl create -f /kubectl-pod-2.json")
+      machine1.fail("kubectl exec kubectl -- kubectl delete pods -l name=kubectl")
     '';
   };
 
@@ -152,9 +152,9 @@ let
 
       machine1.wait_until_succeeds("kubectl get pod kubectl | grep Running")
 
-      machine1.wait_until_succeeds("kubectl exec -ti kubectl -- kubectl get pods")
-      machine1.fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json")
-      machine1.fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl")
+      machine1.wait_until_succeeds("kubectl exec kubectl -- kubectl get pods")
+      machine1.fail("kubectl exec kubectl -- kubectl create -f /kubectl-pod-2.json")
+      machine1.fail("kubectl exec kubectl -- kubectl delete pods -l name=kubectl")
     '';
   };
 
diff --git a/nixos/tests/libreswan.nix b/nixos/tests/libreswan.nix
index 56ab908aed9a..ff3d2344a679 100644
--- a/nixos/tests/libreswan.nix
+++ b/nixos/tests/libreswan.nix
@@ -89,7 +89,7 @@ in
           """
           Sends a message as Alice to Bob
           """
-          bob.execute("nc -lu ::0 1234 >/tmp/msg >&2 &")
+          bob.execute("nc -lu ::0 1234 >/tmp/msg &")
           alice.sleep(1)
           alice.succeed(f"echo '{msg}' | nc -uw 0 bob 1234")
           bob.succeed(f"grep '{msg}' /tmp/msg")
@@ -100,7 +100,7 @@ in
           Starts eavesdropping on Alice and Bob
           """
           match = "src host alice and dst host bob"
-          eve.execute(f"tcpdump -i br0 -c 1 -Avv {match} >/tmp/log >&2 &")
+          eve.execute(f"tcpdump -i br0 -c 1 -Avv {match} >/tmp/log &")
 
 
       start_all()
diff --git a/nixos/tests/logrotate.nix b/nixos/tests/logrotate.nix
new file mode 100644
index 000000000000..38da8d535275
--- /dev/null
+++ b/nixos/tests/logrotate.nix
@@ -0,0 +1,37 @@
+# Test logrotate service works and is enabled by default
+
+import ./make-test-python.nix ({ pkgs, ...} : rec {
+  name = "logrotate";
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [ martinetd ];
+  };
+
+  # default machine
+  machine = { ... }: {
+  };
+
+  testScript =
+    ''
+      with subtest("whether logrotate works"):
+          machine.succeed(
+              # we must rotate once first to create logrotate stamp
+              "systemctl start logrotate.service")
+          # we need to wait for console text once here to
+          # clear console buffer up to this point for next wait
+          machine.wait_for_console_text('logrotate.service: Deactivated successfully')
+
+          machine.succeed(
+              # wtmp is present in default config.
+              "rm -f /var/log/wtmp*",
+              # we need to give it at least 1MB
+              "dd if=/dev/zero of=/var/log/wtmp bs=2M count=1",
+
+              # move into the future and check rotation.
+              "date -s 'now + 1 month + 1 day'")
+          machine.wait_for_console_text('logrotate.service: Deactivated successfully')
+          machine.succeed(
+              # check rotate worked
+              "[ -e /var/log/wtmp.1 ]",
+          )
+    '';
+})
diff --git a/nixos/tests/lorri/default.nix b/nixos/tests/lorri/default.nix
index 147ae999fdb1..c33c7503993d 100644
--- a/nixos/tests/lorri/default.nix
+++ b/nixos/tests/lorri/default.nix
@@ -14,7 +14,7 @@ import ../make-test-python.nix {
     )
 
     # Start the daemon and wait until it is ready
-    machine.execute("lorri daemon > lorri.stdout 2> lorri.stderr >&2 &")
+    machine.execute("lorri daemon > lorri.stdout 2> lorri.stderr &")
     machine.wait_until_succeeds("grep --fixed-strings 'ready' lorri.stdout")
 
     # Ping the daemon
diff --git a/nixos/tests/man.nix b/nixos/tests/man.nix
new file mode 100644
index 000000000000..1ff5af4e8059
--- /dev/null
+++ b/nixos/tests/man.nix
@@ -0,0 +1,100 @@
+
+import ./make-test-python.nix ({ pkgs, lib, ... }: let
+  manImplementations = [
+    "mandoc"
+    "man-db"
+  ];
+
+  machineNames = builtins.map machineSafe manImplementations;
+
+  makeConfig = useImpl: {
+    # Note: mandoc currently can't index symlinked section directories.
+    # So if a man section comes from one package exclusively (e. g.
+    # 1p from man-pages-posix and 2 from man-pages), it isn't searchable.
+    environment.systemPackages = [
+      pkgs.man-pages
+      pkgs.openssl
+      pkgs.libunwind
+    ];
+
+    documentation = {
+      enable = true;
+      nixos.enable = lib.mkForce true;
+      dev.enable = true;
+      man = {
+        enable = true;
+        generateCaches = true;
+      } // lib.listToAttrs (builtins.map (impl: {
+        name = impl;
+        value = {
+          enable = useImpl == impl;
+        };
+      }) manImplementations);
+    };
+  };
+
+  machineSafe = builtins.replaceStrings [ "-" ] [ "_" ];
+in {
+  name = "man";
+  meta.maintainers = [ lib.maintainers.sternenseemann ];
+
+  nodes = lib.listToAttrs (builtins.map (i: {
+    name = machineSafe i;
+    value = makeConfig i;
+  }) manImplementations);
+
+  testScript = ''
+    import re
+    start_all()
+
+    def match_man_k(page, section, haystack):
+      """
+      Check if the man page {page}({section}) occurs in
+      the output of `man -k` given as haystack. Note:
+      This is not super reliable, e. g. it can't deal
+      with man pages that are in multiple sections.
+      """
+
+      for line in haystack.split("\n"):
+        # man -k can look like this:
+        # page(3) - bla
+        # page (3) - bla
+        # pagea, pageb (3, 3P) - foo
+        # pagea, pageb, pagec(3) - bar
+        pages = line.split("(")[0]
+        sections = re.search("\\([a-zA-Z1-9, ]+\\)", line)
+        if sections is None:
+          continue
+        else:
+          sections = sections.group(0)[1:-1]
+
+        if page in pages and f'{section}' in sections:
+          return True
+
+      return False
+
+  '' + lib.concatMapStrings (machine: ''
+    with subtest("Test direct man page lookups in ${machine}"):
+      # man works
+      ${machine}.succeed("man man > /dev/null")
+      # devman works
+      ${machine}.succeed("man 3 libunwind > /dev/null")
+      # NixOS configuration man page is installed
+      ${machine}.succeed("man configuration.nix > /dev/null")
+
+    with subtest("Test generateCaches via man -k in ${machine}"):
+      expected = [
+        ("openssl", "ssl", 3),
+        ("unwind", "libunwind", 3),
+        ("user", "useradd", 8),
+        ("user", "userdel", 8),
+        ("mem", "free", 3),
+        ("mem", "free", 1),
+      ]
+
+      for (keyword, page, section) in expected:
+        matches = ${machine}.succeed(f"man -k {keyword}")
+        if not match_man_k(page, section, matches):
+          raise Exception(f"{page}({section}) missing in matches: {matches}")
+  '') machineNames;
+})
diff --git a/nixos/tests/matrix-appservice-irc.nix b/nixos/tests/matrix-appservice-irc.nix
index e1da410af060..d1c561f95dbf 100644
--- a/nixos/tests/matrix-appservice-irc.nix
+++ b/nixos/tests/matrix-appservice-irc.nix
@@ -1,6 +1,6 @@
 import ./make-test-python.nix ({ pkgs, ... }:
   let
-    homeserverUrl = "http://homeserver:8448";
+    homeserverUrl = "http://homeserver:8008";
   in
   {
     name = "matrix-appservice-irc";
@@ -14,28 +14,32 @@ import ./make-test-python.nix ({ pkgs, ... }:
         specialisation.running.configuration = {
           services.matrix-synapse = {
             enable = true;
-            database_type = "sqlite3";
-            app_service_config_files = [ "/registration.yml" ];
-
-            enable_registration = true;
-
-            listeners = [
-              # The default but tls=false
-              {
-                "bind_address" = "";
-                "port" = 8448;
-                "resources" = [
-                  { "compress" = true; "names" = [ "client" ]; }
-                  { "compress" = false; "names" = [ "federation" ]; }
+            settings = {
+              database.name = "sqlite3";
+              app_service_config_files = [ "/registration.yml" ];
+
+              enable_registration = true;
+
+              listeners = [ {
+                # The default but tls=false
+                bind_addresses = [
+                  "0.0.0.0"
                 ];
-                "tls" = false;
-                "type" = "http";
-                "x_forwarded" = false;
-              }
-            ];
+                port = 8008;
+                resources = [ {
+                  "compress" = true;
+                  "names" = [ "client" ];
+                } {
+                  "compress" = false;
+                  "names" = [ "federation" ];
+                } ];
+                tls = false;
+                type = "http";
+              } ];
+            };
           };
 
-          networking.firewall.allowedTCPPorts = [ 8448 ];
+          networking.firewall.allowedTCPPorts = [ 8008 ];
         };
       };
 
@@ -209,7 +213,7 @@ import ./make-test-python.nix ({ pkgs, ... }:
           )
 
           homeserver.wait_for_unit("matrix-synapse.service")
-          homeserver.wait_for_open_port(8448)
+          homeserver.wait_for_open_port(8008)
 
       with subtest("ensure messages can be exchanged"):
           client.succeed("do_test ${homeserverUrl} >&2")
diff --git a/nixos/tests/matrix-conduit.nix b/nixos/tests/matrix-conduit.nix
new file mode 100644
index 000000000000..d159fbaa4800
--- /dev/null
+++ b/nixos/tests/matrix-conduit.nix
@@ -0,0 +1,95 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+  let
+    name = "conduit";
+  in
+  {
+    nodes = {
+      conduit = args: {
+        services.matrix-conduit = {
+          enable = true;
+          settings.global.server_name = name;
+          settings.global.allow_registration = true;
+          extraEnvironment.RUST_BACKTRACE = "yes";
+        };
+        services.nginx = {
+          enable = true;
+          virtualHosts.${name} = {
+            enableACME = false;
+            forceSSL = false;
+            enableSSL = false;
+
+            locations."/_matrix" = {
+              proxyPass = "http://[::1]:6167";
+            };
+          };
+        };
+        networking.firewall.allowedTCPPorts = [ 80 ];
+      };
+      client = { pkgs, ... }: {
+        environment.systemPackages = [
+          (
+            pkgs.writers.writePython3Bin "do_test"
+              { libraries = [ pkgs.python3Packages.matrix-nio ]; } ''
+              import asyncio
+
+              from nio import AsyncClient
+
+
+              async def main() -> None:
+                  # Connect to conduit
+                  client = AsyncClient("http://conduit:80", "alice")
+
+                  # Register as user alice
+                  response = await client.register("alice", "my-secret-password")
+
+                  # Log in as user alice
+                  response = await client.login("my-secret-password")
+
+                  # Create a new room
+                  response = await client.room_create(federate=False)
+                  room_id = response.room_id
+
+                  # Join the room
+                  response = await client.join(room_id)
+
+                  # Send a message to the room
+                  response = await client.room_send(
+                      room_id=room_id,
+                      message_type="m.room.message",
+                      content={
+                          "msgtype": "m.text",
+                          "body": "Hello conduit!"
+                      }
+                  )
+
+                  # Sync responses
+                  response = await client.sync(timeout=30000)
+
+                  # Check the message was received by conduit
+                  last_message = response.rooms.join[room_id].timeline.events[-1].body
+                  assert last_message == "Hello conduit!"
+
+                  # Leave the room
+                  response = await client.room_leave(room_id)
+
+                  # Close the client
+                  await client.close()
+
+              asyncio.get_event_loop().run_until_complete(main())
+            ''
+          )
+        ];
+      };
+    };
+
+    testScript = ''
+      start_all()
+
+      with subtest("start conduit"):
+            conduit.wait_for_unit("conduit.service")
+            conduit.wait_for_open_port(80)
+
+      with subtest("ensure messages can be exchanged"):
+            client.succeed("do_test")
+    '';
+  })
diff --git a/nixos/tests/matrix-synapse.nix b/nixos/tests/matrix-synapse.nix
index 21e8c24e4713..1ff1e47b2840 100644
--- a/nixos/tests/matrix-synapse.nix
+++ b/nixos/tests/matrix-synapse.nix
@@ -33,6 +33,29 @@ import ./make-test-python.nix ({ pkgs, ... } : let
   testUser = "alice";
   testPassword = "alicealice";
   testEmail = "alice@example.com";
+
+  listeners = [ {
+    port = 8448;
+    bind_addresses = [
+      "127.0.0.1"
+      "::1"
+    ];
+    type = "http";
+    tls = true;
+    x_forwarded = false;
+    resources = [ {
+      names = [
+        "client"
+      ];
+      compress = true;
+    } {
+      names = [
+        "federation"
+      ];
+      compress = false;
+    } ];
+  } ];
+
 in {
 
   name = "matrix-synapse";
@@ -48,22 +71,24 @@ in {
     {
       services.matrix-synapse = {
         enable = true;
-        database_type = "psycopg2";
-        tls_certificate_path = "${cert}";
-        tls_private_key_path = "${key}";
-        database_args = {
-          password = "synapse";
+        settings = {
+          inherit listeners;
+          database = {
+            name = "psycopg2";
+            args.password = "synapse";
+          };
+          tls_certificate_path = "${cert}";
+          tls_private_key_path = "${key}";
+          registration_shared_secret = registrationSharedSecret;
+          public_baseurl = "https://example.com";
+          email = {
+            smtp_host = mailerDomain;
+            smtp_port = 25;
+            require_transport_security = true;
+            notif_from = "matrix <matrix@${mailerDomain}>";
+            app_name = "Matrix";
+          };
         };
-        registration_shared_secret = registrationSharedSecret;
-        public_baseurl = "https://example.com";
-        extraConfig = ''
-          email:
-            smtp_host: "${mailerDomain}"
-            smtp_port: 25
-            require_transport_security: true
-            notif_from: "matrix <matrix@${mailerDomain}>"
-            app_name: "Matrix"
-        '';
       };
       services.postgresql = {
         enable = true;
@@ -165,9 +190,12 @@ in {
     serversqlite = args: {
       services.matrix-synapse = {
         enable = true;
-        database_type = "sqlite3";
-        tls_certificate_path = "${cert}";
-        tls_private_key_path = "${key}";
+        settings = {
+          inherit listeners;
+          database.name = "sqlite3";
+          tls_certificate_path = "${cert}";
+          tls_private_key_path = "${key}";
+        };
       };
     };
   };
diff --git a/nixos/tests/matrix/mjolnir.nix b/nixos/tests/matrix/mjolnir.nix
index bb55f6f5440b..54094ab9d611 100644
--- a/nixos/tests/matrix/mjolnir.nix
+++ b/nixos/tests/matrix/mjolnir.nix
@@ -38,26 +38,31 @@ import ../make-test-python.nix (
       homeserver = { pkgs, ... }: {
         services.matrix-synapse = {
           enable = true;
-          database_type = "sqlite3";
-          tls_certificate_path = "${cert}";
-          tls_private_key_path = "${key}";
-          enable_registration = true;
-          registration_shared_secret = "supersecret-registration";
-
-          listeners = [
-            # The default but tls=false
-            {
-              "bind_address" = "";
-              "port" = 8448;
-              "resources" = [
-                { "compress" = true; "names" = [ "client" "webclient" ]; }
-                { "compress" = false; "names" = [ "federation" ]; }
+          settings = {
+            database.name = "sqlite3";
+            tls_certificate_path = "${cert}";
+            tls_private_key_path = "${key}";
+            enable_registration = true;
+            registration_shared_secret = "supersecret-registration";
+
+            listeners = [ {
+              # The default but tls=false
+              bind_addresses = [
+                "0.0.0.0"
               ];
-              "tls" = false;
-              "type" = "http";
-              "x_forwarded" = false;
-            }
-          ];
+              port = 8448;
+              resources = [ {
+                compress = true;
+                names = [ "client" ];
+              } {
+                compress = false;
+                names = [ "federation" ];
+              } ];
+              tls = false;
+              type = "http";
+              x_forwarded = false;
+            } ];
+          };
         };
 
         networking.firewall.allowedTCPPorts = [ 8448 ];
diff --git a/nixos/tests/matrix/pantalaimon.nix b/nixos/tests/matrix/pantalaimon.nix
index fcb9904b2138..1a9894dd2159 100644
--- a/nixos/tests/matrix/pantalaimon.nix
+++ b/nixos/tests/matrix/pantalaimon.nix
@@ -47,9 +47,32 @@ import ../make-test-python.nix (
 
       services.matrix-synapse = {
         enable = true;
-        database_type = "sqlite3";
-        tls_certificate_path = "${cert}";
-        tls_private_key_path = "${key}";
+        settings = {
+          listeners = [ {
+            port = 8448;
+            bind_addresses = [
+              "127.0.0.1"
+              "::1"
+            ];
+            type = "http";
+            tls = true;
+            x_forwarded = false;
+            resources = [ {
+              names = [
+                "client"
+              ];
+              compress = true;
+            } {
+              names = [
+                "federation"
+              ];
+              compress = false;
+            } ];
+          } ];
+          database.name = "sqlite3";
+          tls_certificate_path = "${cert}";
+          tls_private_key_path = "${key}";
+        };
       };
     };
 
diff --git a/nixos/tests/mattermost.nix b/nixos/tests/mattermost.nix
new file mode 100644
index 000000000000..49b418d9fff7
--- /dev/null
+++ b/nixos/tests/mattermost.nix
@@ -0,0 +1,124 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }:
+let
+  host = "smoke.test";
+  port = "8065";
+  url = "http://${host}:${port}";
+  siteName = "NixOS Smoke Tests, Inc.";
+
+  makeMattermost = mattermostConfig:
+    { config, ... }: {
+      environment.systemPackages = [
+        pkgs.mattermost
+        pkgs.curl
+        pkgs.jq
+      ];
+      networking.hosts = {
+        "127.0.0.1" = [ host ];
+      };
+      services.mattermost = lib.recursiveUpdate {
+        enable = true;
+        inherit siteName;
+        listenAddress = "0.0.0.0:${port}";
+        siteUrl = url;
+        extraConfig = {
+          SupportSettings.AboutLink = "https://nixos.org";
+        };
+      } mattermostConfig;
+    };
+in
+{
+  name = "mattermost";
+
+  nodes = {
+    mutable = makeMattermost {
+      mutableConfig = true;
+      extraConfig.SupportSettings.HelpLink = "https://search.nixos.org";
+    };
+    mostlyMutable = makeMattermost {
+      mutableConfig = true;
+      preferNixConfig = true;
+      plugins = let
+        mattermostDemoPlugin = pkgs.fetchurl {
+          url = "https://github.com/mattermost/mattermost-plugin-demo/releases/download/v0.9.0/com.mattermost.demo-plugin-0.9.0.tar.gz";
+          sha256 = "1h4qi34gcxcx63z8wiqcf2aaywmvv8lys5g8gvsk13kkqhlmag25";
+        };
+      in [
+        mattermostDemoPlugin
+      ];
+    };
+    immutable = makeMattermost {
+      mutableConfig = false;
+      extraConfig.SupportSettings.HelpLink = "https://search.nixos.org";
+    };
+  };
+
+  testScript = let
+    expectConfig = jqExpression: pkgs.writeShellScript "expect-config" ''
+      set -euo pipefail
+      echo "Expecting config to match: "${lib.escapeShellArg jqExpression} >&2
+      curl ${lib.escapeShellArg url} >/dev/null
+      config="$(curl ${lib.escapeShellArg "${url}/api/v4/config/client?format=old"})"
+      echo "Config: $(echo "$config" | ${pkgs.jq}/bin/jq)" >&2
+      [[ "$(echo "$config" | ${pkgs.jq}/bin/jq -r ${lib.escapeShellArg ".SiteName == $siteName and .Version == ($mattermostName / $sep)[-1] and (${jqExpression})"} --arg siteName ${lib.escapeShellArg siteName} --arg mattermostName ${lib.escapeShellArg pkgs.mattermost.name} --arg sep '-')" = "true" ]]
+    '';
+
+    setConfig = jqExpression: pkgs.writeShellScript "set-config" ''
+      set -euo pipefail
+      mattermostConfig=/var/lib/mattermost/config/config.json
+      newConfig="$(${pkgs.jq}/bin/jq -r ${lib.escapeShellArg jqExpression} $mattermostConfig)"
+      rm -f $mattermostConfig
+      echo "$newConfig" > "$mattermostConfig"
+    '';
+  in
+  ''
+    start_all()
+
+    ## Mutable node tests ##
+    mutable.wait_for_unit("mattermost.service")
+    mutable.wait_for_open_port(8065)
+
+    # Get the initial config
+    mutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"''}")
+
+    # Edit the config
+    mutable.succeed("${setConfig ''.SupportSettings.AboutLink = "https://mattermost.com"''}")
+    mutable.succeed("${setConfig ''.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"''}")
+    mutable.systemctl("restart mattermost.service")
+    mutable.wait_for_open_port(8065)
+
+    # AboutLink and HelpLink should be changed
+    mutable.succeed("${expectConfig ''.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"''}")
+
+    ## Mostly mutable node tests ##
+    mostlyMutable.wait_for_unit("mattermost.service")
+    mostlyMutable.wait_for_open_port(8065)
+
+    # Get the initial config
+    mostlyMutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org"''}")
+
+    # Edit the config
+    mostlyMutable.succeed("${setConfig ''.SupportSettings.AboutLink = "https://mattermost.com"''}")
+    mostlyMutable.succeed("${setConfig ''.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"''}")
+    mostlyMutable.systemctl("restart mattermost.service")
+    mostlyMutable.wait_for_open_port(8065)
+
+    # AboutLink should be overridden by NixOS configuration; HelpLink should be what we set above
+    mostlyMutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"''}")
+
+    ## Immutable node tests ##
+    immutable.wait_for_unit("mattermost.service")
+    immutable.wait_for_open_port(8065)
+
+    # Get the initial config
+    immutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"''}")
+
+    # Edit the config
+    immutable.succeed("${setConfig ''.SupportSettings.AboutLink = "https://mattermost.com"''}")
+    immutable.succeed("${setConfig ''.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"''}")
+    immutable.systemctl("restart mattermost.service")
+    immutable.wait_for_open_port(8065)
+
+    # Our edits should be ignored on restart
+    immutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"''}")
+  '';
+})
diff --git a/nixos/tests/minidlna.nix b/nixos/tests/minidlna.nix
index d852c7f60bc4..104b79078fd5 100644
--- a/nixos/tests/minidlna.nix
+++ b/nixos/tests/minidlna.nix
@@ -33,7 +33,9 @@ import ./make-test-python.nix ({ pkgs, ... }: {
     server.succeed("mkdir -p /tmp/stuff && chown minidlna: /tmp/stuff")
     server.wait_for_unit("minidlna")
     server.wait_for_open_port("8200")
-    server.succeed("curl --fail http://localhost:8200/")
-    client.succeed("curl --fail http://server:8200/")
+    # requests must be made *by IP* to avoid triggering minidlna's
+    # DNS-rebinding protection
+    server.succeed("curl --fail http://$(getent ahostsv4 localhost | head -n1 | cut -f 1 -d ' '):8200/")
+    client.succeed("curl --fail http://$(getent ahostsv4 server | head -n1 | cut -f 1 -d ' '):8200/")
   '';
 })
diff --git a/nixos/tests/miniflux.nix b/nixos/tests/miniflux.nix
index 1015550fa8c7..d905aea048a3 100644
--- a/nixos/tests/miniflux.nix
+++ b/nixos/tests/miniflux.nix
@@ -7,6 +7,15 @@ let
   defaultPort = 8080;
   defaultUsername = "admin";
   defaultPassword = "password";
+  adminCredentialsFile = pkgs.writeText "admin-credentials" ''
+            ADMIN_USERNAME=${defaultUsername}
+            ADMIN_PASSWORD=${defaultPassword}
+          '';
+  customAdminCredentialsFile = pkgs.writeText "admin-credentials" ''
+            ADMIN_USERNAME=${username}
+            ADMIN_PASSWORD=${password}
+          '';
+
 in
 with lib;
 {
@@ -17,13 +26,19 @@ with lib;
     default =
       { ... }:
       {
-        services.miniflux.enable = true;
+        services.miniflux = {
+          enable = true;
+          inherit adminCredentialsFile;
+        };
       };
 
     withoutSudo =
       { ... }:
       {
-        services.miniflux.enable = true;
+        services.miniflux = {
+          enable = true;
+          inherit adminCredentialsFile;
+        };
         security.sudo.enable = false;
       };
 
@@ -36,10 +51,7 @@ with lib;
             CLEANUP_FREQUENCY = "48";
             LISTEN_ADDR = "localhost:${toString port}";
           };
-          adminCredentialsFile = pkgs.writeText "admin-credentials" ''
-            ADMIN_USERNAME=${username}
-            ADMIN_PASSWORD=${password}
-          '';
+          adminCredentialsFile = customAdminCredentialsFile;
         };
       };
   };
diff --git a/nixos/tests/moinmoin.nix b/nixos/tests/moinmoin.nix
deleted file mode 100644
index ac327498eba0..000000000000
--- a/nixos/tests/moinmoin.nix
+++ /dev/null
@@ -1,28 +0,0 @@
-import ./make-test-python.nix ({ pkgs, lib, ... }: {
-  name = "moinmoin";
-  meta.maintainers = with lib.maintainers; [ mmilata ];
-
-  machine =
-    { ... }:
-    { services.moinmoin.enable = true;
-      services.moinmoin.wikis.ExampleWiki.superUsers = [ "admin" ];
-      services.moinmoin.wikis.ExampleWiki.webHost = "localhost";
-
-      services.nginx.virtualHosts.localhost.enableACME = false;
-      services.nginx.virtualHosts.localhost.forceSSL = false;
-    };
-
-  testScript = ''
-    start_all()
-
-    machine.wait_for_unit("moin-ExampleWiki.service")
-    machine.wait_for_unit("nginx.service")
-    machine.wait_for_file("/run/moin/ExampleWiki/gunicorn.sock")
-
-    assert "If you have just installed" in machine.succeed("curl -L http://localhost/")
-
-    assert "status success" in machine.succeed(
-        "moin-ExampleWiki account create --name=admin --email=admin@example.com --password=foo 2>&1"
-    )
-  '';
-})
diff --git a/nixos/tests/moosefs.nix b/nixos/tests/moosefs.nix
new file mode 100644
index 000000000000..0dc08748b828
--- /dev/null
+++ b/nixos/tests/moosefs.nix
@@ -0,0 +1,89 @@
+import ./make-test-python.nix ({ pkgs, ... } :
+
+let
+  master = { pkgs, ... } : {
+    # data base is stored in memory
+    # server crashes with default memory size
+    virtualisation.memorySize = 1024;
+
+    services.moosefs.master = {
+      enable = true;
+      openFirewall = true;
+      exports = [
+        "* / rw,alldirs,admin,maproot=0:0"
+        "* . rw"
+      ];
+    };
+  };
+
+  chunkserver = { pkgs, ... } : {
+    virtualisation.emptyDiskImages = [ 4096 ];
+    boot.initrd.postDeviceCommands = ''
+      ${pkgs.e2fsprogs}/bin/mkfs.ext4 -L data /dev/vdb
+    '';
+
+    fileSystems = pkgs.lib.mkVMOverride {
+      "/data" = {
+        device = "/dev/disk/by-label/data";
+        fsType = "ext4";
+      };
+    };
+
+    services.moosefs = {
+      masterHost = "master";
+      chunkserver = {
+        openFirewall = true;
+        enable = true;
+        hdds = [ "~/data" ];
+      };
+    };
+  };
+
+  metalogger = { pkgs, ... } : {
+    services.moosefs = {
+      masterHost = "master";
+      metalogger.enable = true;
+    };
+  };
+
+  client = { pkgs, ... } : {
+    services.moosefs.client.enable = true;
+  };
+
+in {
+  name = "moosefs";
+
+  nodes= {
+    inherit master;
+    inherit metalogger;
+    chunkserver1 = chunkserver;
+    chunkserver2 = chunkserver;
+    client1 = client;
+    client2 = client;
+  };
+
+  testScript = ''
+    # prepare master server
+    master.start()
+    master.wait_for_unit("multi-user.target")
+    master.succeed("mfsmaster-init")
+    master.succeed("systemctl restart mfs-master")
+    master.wait_for_unit("mfs-master.service")
+
+    metalogger.wait_for_unit("mfs-metalogger.service")
+
+    for chunkserver in [chunkserver1, chunkserver2]:
+        chunkserver.wait_for_unit("multi-user.target")
+        chunkserver.succeed("chown moosefs:moosefs /data")
+        chunkserver.succeed("systemctl restart mfs-chunkserver")
+        chunkserver.wait_for_unit("mfs-chunkserver.service")
+
+    for client in [client1, client2]:
+        client.wait_for_unit("multi-user.target")
+        client.succeed("mkdir /moosefs")
+        client.succeed("mount -t moosefs master:/ /moosefs")
+
+    client1.succeed("echo test > /moosefs/file")
+    client2.succeed("grep test /moosefs/file")
+  '';
+})
diff --git a/nixos/tests/mpd.nix b/nixos/tests/mpd.nix
index 5c969fc9c917..52d9c7fd33a1 100644
--- a/nixos/tests/mpd.nix
+++ b/nixos/tests/mpd.nix
@@ -96,7 +96,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
     };
 
   testScript = ''
-    mpc = "${pkgs.mpc_cli}/bin/mpc --wait"
+    mpc = "${pkgs.mpc-cli}/bin/mpc --wait"
 
     # Connects to the given server and attempts to play a tune.
     def play_some_music(server):
diff --git a/nixos/tests/mysql/common.nix b/nixos/tests/mysql/common.nix
new file mode 100644
index 000000000000..040d360b6d99
--- /dev/null
+++ b/nixos/tests/mysql/common.nix
@@ -0,0 +1,10 @@
+{ lib, pkgs }: {
+  mariadbPackages = lib.filterAttrs (n: _: lib.hasPrefix "mariadb" n) (pkgs.callPackage ../../../pkgs/servers/sql/mariadb {
+    inherit (pkgs.darwin) cctools;
+    inherit (pkgs.darwin.apple_sdk.frameworks) CoreServices;
+  });
+  mysqlPackages = {
+    inherit (pkgs) mysql57 mysql80;
+  };
+  mkTestName = pkg: "mariadb_${builtins.replaceStrings ["."] [""] (lib.versions.majorMinor pkg.version)}";
+}
diff --git a/nixos/tests/mysql/mariadb-galera-mariabackup.nix b/nixos/tests/mysql/mariadb-galera-mariabackup.nix
deleted file mode 100644
index 10682c361d1d..000000000000
--- a/nixos/tests/mysql/mariadb-galera-mariabackup.nix
+++ /dev/null
@@ -1,233 +0,0 @@
-import ./../make-test-python.nix ({ pkgs, ...} :
-
-let
-  mysqlenv-common      = pkgs.buildEnv { name = "mysql-path-env-common";      pathsToLink = [ "/bin" ]; paths = with pkgs; [ bash gawk gnutar inetutils which ]; };
-  mysqlenv-mariabackup = pkgs.buildEnv { name = "mysql-path-env-mariabackup"; pathsToLink = [ "/bin" ]; paths = with pkgs; [ gzip iproute2 netcat procps pv socat ]; };
-
-  # Common user configuration
-  users = { ... }:
-  {
-    users.users.testuser = {
-      isSystemUser = true;
-      group = "testusers";
-    };
-    users.groups.testusers = { };
-  };
-
-in {
-  name = "mariadb-galera-mariabackup";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ izorkin ];
-  };
-
-  # The test creates a Galera cluster with 3 nodes and is checking if mariabackup-based SST works. The cluster is tested by creating a DB and an empty table on one node,
-  # and checking the table's presence on the other node.
-
-  nodes = {
-    galera_01 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.1.1"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.1.1 galera_01
-          192.168.1.2 galera_02
-          192.168.1.3 galera_03
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-mariabackup ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        ensureDatabases = [ "testdb" ];
-        ensureUsers = [{
-          name = "testuser";
-          ensurePermissions = {
-            "testdb.*" = "ALL PRIVILEGES";
-          };
-        }];
-        initialScript = pkgs.writeText "mariadb-init.sql" ''
-          GRANT ALL PRIVILEGES ON *.* TO 'check_repl'@'localhost' IDENTIFIED BY 'check_pass' WITH GRANT OPTION;
-          FLUSH PRIVILEGES;
-        '';
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://";
-            wsrep_cluster_name = "galera";
-            wsrep_node_address = "192.168.1.1";
-            wsrep_node_name = "galera_01";
-            wsrep_sst_method = "mariabackup";
-            wsrep_sst_auth = "check_repl:check_pass";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_02 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.1.2"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.1.1 galera_01
-          192.168.1.2 galera_02
-          192.168.1.3 galera_03
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-mariabackup ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_01,galera_02,galera_03";
-            wsrep_cluster_name = "galera";
-            wsrep_node_address = "192.168.1.2";
-            wsrep_node_name = "galera_02";
-            wsrep_sst_method = "mariabackup";
-            wsrep_sst_auth = "check_repl:check_pass";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_03 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.1.3"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.1.1 galera_01
-          192.168.1.2 galera_02
-          192.168.1.3 galera_03
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-mariabackup ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_01,galera_02,galera_03";
-            wsrep_cluster_name = "galera";
-            wsrep_node_address = "192.168.1.3";
-            wsrep_node_name = "galera_03";
-            wsrep_sst_method = "mariabackup";
-            wsrep_sst_auth = "check_repl:check_pass";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-  };
-
-  testScript = ''
-    galera_01.start()
-    galera_01.wait_for_unit("mysql")
-    galera_01.wait_for_open_port(3306)
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (37);'"
-    )
-    galera_02.start()
-    galera_02.wait_for_unit("mysql")
-    galera_02.wait_for_open_port(3306)
-    galera_03.start()
-    galera_03.wait_for_unit("mysql")
-    galera_03.wait_for_open_port(3306)
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
-    )
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_02.succeed("systemctl stop mysql")
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (38);'"
-    )
-    galera_03.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (39);'"
-    )
-    galera_02.succeed("systemctl start mysql")
-    galera_02.wait_for_open_port(3306)
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
-    )
-    galera_03.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
-    )
-    galera_01.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 39"
-    )
-    galera_02.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 38"
-    )
-    galera_03.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
-    )
-    galera_01.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
-    galera_02.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
-    galera_03.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
-  '';
-})
diff --git a/nixos/tests/mysql/mariadb-galera-rsync.nix b/nixos/tests/mysql/mariadb-galera-rsync.nix
deleted file mode 100644
index 701e01e88718..000000000000
--- a/nixos/tests/mysql/mariadb-galera-rsync.nix
+++ /dev/null
@@ -1,226 +0,0 @@
-import ./../make-test-python.nix ({ pkgs, ...} :
-
-let
-  mysqlenv-common      = pkgs.buildEnv { name = "mysql-path-env-common";      pathsToLink = [ "/bin" ]; paths = with pkgs; [ bash gawk gnutar inetutils which ]; };
-  mysqlenv-rsync       = pkgs.buildEnv { name = "mysql-path-env-rsync";       pathsToLink = [ "/bin" ]; paths = with pkgs; [ lsof procps rsync stunnel ]; };
-
-  # Common user configuration
-  users = { ... }:
-  {
-    users.users.testuser = {
-      isSystemUser = true;
-      group = "testusers";
-    };
-    users.groups.testusers = { };
-  };
-
-in {
-  name = "mariadb-galera-rsync";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ izorkin ];
-  };
-
-  # The test creates a Galera cluster with 3 nodes and is checking if rsync-based SST works. The cluster is tested by creating a DB and an empty table on one node,
-  # and checking the table's presence on the other node.
-
-  nodes = {
-    galera_04 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.2.1"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.2.1 galera_04
-          192.168.2.2 galera_05
-          192.168.2.3 galera_06
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-rsync ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        ensureDatabases = [ "testdb" ];
-        ensureUsers = [{
-          name = "testuser";
-          ensurePermissions = {
-            "testdb.*" = "ALL PRIVILEGES";
-          };
-        }];
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://";
-            wsrep_cluster_name = "galera-rsync";
-            wsrep_node_address = "192.168.2.1";
-            wsrep_node_name = "galera_04";
-            wsrep_sst_method = "rsync";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_05 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.2.2"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.2.1 galera_04
-          192.168.2.2 galera_05
-          192.168.2.3 galera_06
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-rsync ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_04,galera_05,galera_06";
-            wsrep_cluster_name = "galera-rsync";
-            wsrep_node_address = "192.168.2.2";
-            wsrep_node_name = "galera_05";
-            wsrep_sst_method = "rsync";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-
-    galera_06 =
-      { pkgs, ... }:
-      {
-      imports = [ users ];
-      networking = {
-        interfaces.eth1 = {
-          ipv4.addresses = [
-            { address = "192.168.2.3"; prefixLength = 24; }
-          ];
-        };
-        extraHosts = ''
-          192.168.2.1 galera_04
-          192.168.2.2 galera_05
-          192.168.2.3 galera_06
-        '';
-        firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
-        firewall.allowedUDPPorts = [ 4567 ];
-      };
-      systemd.services.mysql = with pkgs; {
-        path = [ mysqlenv-common mysqlenv-rsync ];
-      };
-      services.mysql = {
-        enable = true;
-        package = pkgs.mariadb;
-        settings = {
-          mysqld = {
-            bind_address = "0.0.0.0";
-          };
-          galera = {
-            wsrep_on = "ON";
-            wsrep_debug = "NONE";
-            wsrep_retry_autocommit = "3";
-            wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-            wsrep_cluster_address = "gcomm://galera_04,galera_05,galera_06";
-            wsrep_cluster_name = "galera-rsync";
-            wsrep_node_address = "192.168.2.3";
-            wsrep_node_name = "galera_06";
-            wsrep_sst_method = "rsync";
-            binlog_format = "ROW";
-            enforce_storage_engine = "InnoDB";
-            innodb_autoinc_lock_mode = "2";
-          };
-        };
-      };
-    };
-  };
-
-  testScript = ''
-    galera_04.start()
-    galera_04.wait_for_unit("mysql")
-    galera_04.wait_for_open_port(3306)
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (41);'"
-    )
-    galera_05.start()
-    galera_05.wait_for_unit("mysql")
-    galera_05.wait_for_open_port(3306)
-    galera_06.start()
-    galera_06.wait_for_unit("mysql")
-    galera_06.wait_for_open_port(3306)
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
-    )
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_05.succeed("systemctl stop mysql")
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (42);'"
-    )
-    galera_06.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
-    )
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (43);'"
-    )
-    galera_05.succeed("systemctl start mysql")
-    galera_05.wait_for_open_port(3306)
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
-    )
-    galera_06.succeed(
-        "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
-    )
-    galera_04.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 43"
-    )
-    galera_05.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 42"
-    )
-    galera_06.succeed(
-        "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
-    )
-    galera_04.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
-    galera_05.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
-    galera_06.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
-  '';
-})
diff --git a/nixos/tests/mysql/mariadb-galera.nix b/nixos/tests/mysql/mariadb-galera.nix
new file mode 100644
index 000000000000..c9962f49c02f
--- /dev/null
+++ b/nixos/tests/mysql/mariadb-galera.nix
@@ -0,0 +1,250 @@
+{
+  system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; },
+  lib ? pkgs.lib
+}:
+
+let
+  inherit (import ./common.nix { inherit pkgs lib; }) mkTestName mariadbPackages;
+
+  makeTest = import ./../make-test-python.nix;
+
+  # Common user configuration
+  makeGaleraTest = {
+    mariadbPackage,
+    name ? mkTestName mariadbPackage,
+    galeraPackage ? pkgs.mariadb-galera
+  }: makeTest {
+    name = "${name}-galera-mariabackup";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ izorkin ajs124 das_j ];
+    };
+
+    # The test creates a Galera cluster with 3 nodes and is checking if mariabackup-based SST works. The cluster is tested by creating a DB and an empty table on one node,
+    # and checking the table's presence on the other node.
+    nodes = let
+      mkGaleraNode = {
+        id,
+        method
+      }: let
+        address = "192.168.1.${toString id}";
+        isFirstClusterNode = id == 1 || id == 4;
+      in {
+        users = {
+          users.testuser = {
+            isSystemUser = true;
+            group = "testusers";
+          };
+          groups.testusers = { };
+        };
+
+        networking = {
+          interfaces.eth1 = {
+            ipv4.addresses = [
+              { inherit address; prefixLength = 24; }
+            ];
+          };
+          extraHosts = lib.concatMapStringsSep "\n" (i: "192.168.1.${toString i} galera_0${toString i}") (lib.range 1 6);
+          firewall.allowedTCPPorts = [ 3306 4444 4567 4568 ];
+          firewall.allowedUDPPorts = [ 4567 ];
+        };
+        systemd.services.mysql = with pkgs; {
+          path = with pkgs; [
+            bash
+            gawk
+            gnutar
+            gzip
+            inetutils
+            iproute2
+            netcat
+            procps
+            pv
+            rsync
+            socat
+            stunnel
+            which
+          ];
+        };
+        services.mysql = {
+          enable = true;
+          package = mariadbPackage;
+          ensureDatabases = lib.mkIf isFirstClusterNode [ "testdb" ];
+          ensureUsers = lib.mkIf isFirstClusterNode [{
+            name = "testuser";
+            ensurePermissions = {
+              "testdb.*" = "ALL PRIVILEGES";
+            };
+          }];
+          initialScript = lib.mkIf isFirstClusterNode (pkgs.writeText "mariadb-init.sql" ''
+            GRANT ALL PRIVILEGES ON *.* TO 'check_repl'@'localhost' IDENTIFIED BY 'check_pass' WITH GRANT OPTION;
+            FLUSH PRIVILEGES;
+          '');
+          settings = {
+            mysqld = {
+              bind_address = "0.0.0.0";
+            };
+            galera = {
+              wsrep_on = "ON";
+              wsrep_debug = "NONE";
+              wsrep_retry_autocommit = "3";
+              wsrep_provider = "${galeraPackage}/lib/galera/libgalera_smm.so";
+              wsrep_cluster_address = "gcomm://"
+                + lib.optionalString (id == 2 || id == 3) "galera_01,galera_02,galera_03"
+                + lib.optionalString (id == 5 || id == 6) "galera_04,galera_05,galera_06";
+              wsrep_cluster_name = "galera";
+              wsrep_node_address = address;
+              wsrep_node_name = "galera_0${toString id}";
+              wsrep_sst_method = method;
+              wsrep_sst_auth = "check_repl:check_pass";
+              binlog_format = "ROW";
+              enforce_storage_engine = "InnoDB";
+              innodb_autoinc_lock_mode = "2";
+            };
+          };
+        };
+      };
+    in {
+      galera_01 = mkGaleraNode {
+        id = 1;
+        method = "mariabackup";
+      };
+
+      galera_02 = mkGaleraNode {
+        id = 2;
+        method = "mariabackup";
+      };
+
+      galera_03 = mkGaleraNode {
+        id = 3;
+        method = "mariabackup";
+      };
+
+      galera_04 = mkGaleraNode {
+        id = 4;
+        method = "rsync";
+      };
+
+      galera_05 = mkGaleraNode {
+        id = 5;
+        method = "rsync";
+      };
+
+      galera_06 = mkGaleraNode {
+        id = 6;
+        method = "rsync";
+      };
+
+    };
+
+    testScript = ''
+      galera_01.start()
+      galera_01.wait_for_unit("mysql")
+      galera_01.wait_for_open_port(3306)
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (37);'"
+      )
+      galera_02.start()
+      galera_02.wait_for_unit("mysql")
+      galera_02.wait_for_open_port(3306)
+      galera_03.start()
+      galera_03.wait_for_unit("mysql")
+      galera_03.wait_for_open_port(3306)
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
+      )
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_02.succeed("systemctl stop mysql")
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (38);'"
+      )
+      galera_03.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (39);'"
+      )
+      galera_02.succeed("systemctl start mysql")
+      galera_02.wait_for_open_port(3306)
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
+      )
+      galera_03.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
+      )
+      galera_01.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 39"
+      )
+      galera_02.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 38"
+      )
+      galera_03.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 37"
+      )
+      galera_01.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
+      galera_02.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
+      galera_03.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
+      galera_01.crash()
+      galera_02.crash()
+      galera_03.crash()
+
+      galera_04.start()
+      galera_04.wait_for_unit("mysql")
+      galera_04.wait_for_open_port(3306)
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db1 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db1 values (41);'"
+      )
+      galera_05.start()
+      galera_05.wait_for_unit("mysql")
+      galera_05.wait_for_open_port(3306)
+      galera_06.start()
+      galera_06.wait_for_unit("mysql")
+      galera_06.wait_for_open_port(3306)
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
+      )
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db2 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_05.succeed("systemctl stop mysql")
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db2 values (42);'"
+      )
+      galera_06.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; create table db3 (test_id INT, PRIMARY KEY (test_id)) ENGINE = InnoDB;'"
+      )
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; insert into db3 values (43);'"
+      )
+      galera_05.succeed("systemctl start mysql")
+      galera_05.wait_for_open_port(3306)
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_cluster_size.*3'"
+      )
+      galera_06.succeed(
+          "sudo -u testuser mysql -u testuser -e 'show status' -N | grep 'wsrep_local_state_comment.*Synced'"
+      )
+      galera_04.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db3;' -N | grep 43"
+      )
+      galera_05.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db2;' -N | grep 42"
+      )
+      galera_06.succeed(
+          "sudo -u testuser mysql -u testuser -e 'use testdb; select test_id from db1;' -N | grep 41"
+      )
+      galera_04.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db3;'")
+      galera_05.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db2;'")
+      galera_06.succeed("sudo -u testuser mysql -u testuser -e 'use testdb; drop table db1;'")
+    '';
+  };
+in
+  lib.mapAttrs (_: mariadbPackage: makeGaleraTest { inherit mariadbPackage; }) mariadbPackages
diff --git a/nixos/tests/mysql/mysql-autobackup.nix b/nixos/tests/mysql/mysql-autobackup.nix
index b0ec7daaf054..101122f7bdef 100644
--- a/nixos/tests/mysql/mysql-autobackup.nix
+++ b/nixos/tests/mysql/mysql-autobackup.nix
@@ -1,38 +1,53 @@
-import ./../make-test-python.nix ({ pkgs, lib, ... }:
-
 {
-  name = "automysqlbackup";
-  meta.maintainers = [ lib.maintainers.aanderse ];
-
-  machine =
-    { pkgs, ... }:
-    {
-      services.mysql.enable = true;
-      services.mysql.package = pkgs.mariadb;
-      services.mysql.initialDatabases = [ { name = "testdb"; schema = ./testdb.sql; } ];
+  system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; },
+  lib ? pkgs.lib
+}:
+
+let
+  inherit (import ./common.nix { inherit pkgs lib; }) mkTestName mariadbPackages;
+
+  makeTest = import ./../make-test-python.nix;
+
+  makeAutobackupTest = {
+    package,
+    name ? mkTestName package,
+  }: makeTest {
+    name = "${name}-automysqlbackup";
+    meta.maintainers = [ lib.maintainers.aanderse ];
+
+    machine = {
+      services.mysql = {
+        inherit package;
+        enable = true;
+        initialDatabases = [ { name = "testdb"; schema = ./testdb.sql; } ];
+      };
 
       services.automysqlbackup.enable = true;
     };
 
-  testScript = ''
-    start_all()
-
-    # Need to have mysql started so that it can be populated with data.
-    machine.wait_for_unit("mysql.service")
-
-    with subtest("Wait for testdb to be fully populated (5 rows)."):
-        machine.wait_until_succeeds(
-            "mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
-        )
-
-    with subtest("Do a backup and wait for it to start"):
-        machine.start_job("automysqlbackup.service")
-        machine.wait_for_job("automysqlbackup.service")
-
-    with subtest("wait for backup file and check that data appears in backup"):
-        machine.wait_for_file("/var/backup/mysql/daily/testdb")
-        machine.succeed(
-            "${pkgs.gzip}/bin/zcat /var/backup/mysql/daily/testdb/daily_testdb_*.sql.gz | grep hello"
-        )
-    '';
-})
+    testScript = ''
+      start_all()
+
+      # Need to have mysql started so that it can be populated with data.
+      machine.wait_for_unit("mysql.service")
+
+      with subtest("Wait for testdb to be fully populated (5 rows)."):
+          machine.wait_until_succeeds(
+              "mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
+          )
+
+      with subtest("Do a backup and wait for it to start"):
+          machine.start_job("automysqlbackup.service")
+          machine.wait_for_job("automysqlbackup.service")
+
+      with subtest("wait for backup file and check that data appears in backup"):
+          machine.wait_for_file("/var/backup/mysql/daily/testdb")
+          machine.succeed(
+              "${pkgs.gzip}/bin/zcat /var/backup/mysql/daily/testdb/daily_testdb_*.sql.gz | grep hello"
+          )
+      '';
+  };
+in
+  lib.mapAttrs (_: package: makeAutobackupTest { inherit package; }) mariadbPackages
diff --git a/nixos/tests/mysql/mysql-backup.nix b/nixos/tests/mysql/mysql-backup.nix
index 269fddc66e1d..9335b233327a 100644
--- a/nixos/tests/mysql/mysql-backup.nix
+++ b/nixos/tests/mysql/mysql-backup.nix
@@ -1,56 +1,72 @@
-# Test whether mysqlBackup option works
-import ./../make-test-python.nix ({ pkgs, ... } : {
-  name = "mysql-backup";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ rvl ];
-  };
+{
+  system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; },
+  lib ? pkgs.lib
+}:
 
-  nodes = {
-    master = { pkgs, ... }: {
-      services.mysql = {
-        enable = true;
-        initialDatabases = [ { name = "testdb"; schema = ./testdb.sql; } ];
-        package = pkgs.mariadb;
-      };
+let
+  inherit (import ./common.nix { inherit pkgs lib; }) mkTestName mariadbPackages;
+
+  makeTest = import ./../make-test-python.nix;
+
+  makeBackupTest = {
+    package,
+    name ? mkTestName package
+  }: makeTest {
+    name = "${name}-backup";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ rvl ];
+    };
+
+    nodes = {
+      master = { pkgs, ... }: {
+        services.mysql = {
+          inherit package;
+          enable = true;
+          initialDatabases = [ { name = "testdb"; schema = ./testdb.sql; } ];
+        };
 
-      services.mysqlBackup = {
-        enable = true;
-        databases = [ "doesnotexist" "testdb" ];
+        services.mysqlBackup = {
+          enable = true;
+          databases = [ "doesnotexist" "testdb" ];
+        };
       };
     };
-  };
 
-  testScript = ''
-    start_all()
+    testScript = ''
+      start_all()
 
-    # Delete backup file that may be left over from a previous test run.
-    # This is not needed on Hydra but useful for repeated local test runs.
-    master.execute("rm -f /var/backup/mysql/testdb.gz")
+      # Delete backup file that may be left over from a previous test run.
+      # This is not needed on Hydra but useful for repeated local test runs.
+      master.execute("rm -f /var/backup/mysql/testdb.gz")
 
-    # Need to have mysql started so that it can be populated with data.
-    master.wait_for_unit("mysql.service")
+      # Need to have mysql started so that it can be populated with data.
+      master.wait_for_unit("mysql.service")
 
-    # Wait for testdb to be fully populated (5 rows).
-    master.wait_until_succeeds(
-        "mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
-    )
+      # Wait for testdb to be fully populated (5 rows).
+      master.wait_until_succeeds(
+          "mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
+      )
 
-    # Do a backup and wait for it to start
-    master.start_job("mysql-backup.service")
-    master.wait_for_unit("mysql-backup.service")
+      # Do a backup and wait for it to start
+      master.start_job("mysql-backup.service")
+      master.wait_for_unit("mysql-backup.service")
 
-    # wait for backup to fail, because of database 'doesnotexist'
-    master.wait_until_fails("systemctl is-active -q mysql-backup.service")
+      # wait for backup to fail, because of database 'doesnotexist'
+      master.wait_until_fails("systemctl is-active -q mysql-backup.service")
 
-    # wait for backup file and check that data appears in backup
-    master.wait_for_file("/var/backup/mysql/testdb.gz")
-    master.succeed(
-        "${pkgs.gzip}/bin/zcat /var/backup/mysql/testdb.gz | grep hello"
-    )
+      # wait for backup file and check that data appears in backup
+      master.wait_for_file("/var/backup/mysql/testdb.gz")
+      master.succeed(
+          "${pkgs.gzip}/bin/zcat /var/backup/mysql/testdb.gz | grep hello"
+      )
 
-    # Check that a failed backup is logged
-    master.succeed(
-        "journalctl -u mysql-backup.service | grep 'fail.*doesnotexist' > /dev/null"
-    )
-  '';
-})
+      # Check that a failed backup is logged
+      master.succeed(
+          "journalctl -u mysql-backup.service | grep 'fail.*doesnotexist' > /dev/null"
+      )
+    '';
+  };
+in
+  lib.mapAttrs (_: package: makeBackupTest { inherit package; }) mariadbPackages
diff --git a/nixos/tests/mysql/mysql-replication.nix b/nixos/tests/mysql/mysql-replication.nix
index a52372ca47ce..f6014019bd53 100644
--- a/nixos/tests/mysql/mysql-replication.nix
+++ b/nixos/tests/mysql/mysql-replication.nix
@@ -1,91 +1,101 @@
-import ./../make-test-python.nix ({ pkgs, ...} :
+{
+  system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; },
+  lib ? pkgs.lib
+}:
 
 let
+  inherit (import ./common.nix { inherit pkgs lib; }) mkTestName mariadbPackages;
+
   replicateUser = "replicate";
   replicatePassword = "secret";
-in
 
-{
-  name = "mysql-replication";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ eelco shlevy ];
-  };
+  makeTest = import ./../make-test-python.nix;
 
-  nodes = {
-    master =
-      { pkgs, ... }:
+  makeReplicationTest = {
+    package,
+    name ? mkTestName package,
+  }: makeTest {
+    name = "${name}-replication";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ ajs124 das_j ];
+    };
 
-      {
-        services.mysql.enable = true;
-        services.mysql.package = pkgs.mariadb;
-        services.mysql.replication.role = "master";
-        services.mysql.replication.slaveHost = "%";
-        services.mysql.replication.masterUser = replicateUser;
-        services.mysql.replication.masterPassword = replicatePassword;
-        services.mysql.initialDatabases = [ { name = "testdb"; schema = ./testdb.sql; } ];
+    nodes = {
+      primary = {
+        services.mysql = {
+          inherit package;
+          enable = true;
+          replication.role = "master";
+          replication.slaveHost = "%";
+          replication.masterUser = replicateUser;
+          replication.masterPassword = replicatePassword;
+          initialDatabases = [ { name = "testdb"; schema = ./testdb.sql; } ];
+        };
         networking.firewall.allowedTCPPorts = [ 3306 ];
       };
 
-    slave1 =
-      { pkgs, nodes, ... }:
-
-      {
-        services.mysql.enable = true;
-        services.mysql.package = pkgs.mariadb;
-        services.mysql.replication.role = "slave";
-        services.mysql.replication.serverId = 2;
-        services.mysql.replication.masterHost = nodes.master.config.networking.hostName;
-        services.mysql.replication.masterUser = replicateUser;
-        services.mysql.replication.masterPassword = replicatePassword;
+      secondary1 = { nodes, ... }: {
+        services.mysql = {
+          inherit package;
+          enable = true;
+          replication.role = "slave";
+          replication.serverId = 2;
+          replication.masterHost = nodes.primary.config.networking.hostName;
+          replication.masterUser = replicateUser;
+          replication.masterPassword = replicatePassword;
+        };
       };
 
-    slave2 =
-      { pkgs, nodes, ... }:
-
-      {
-        services.mysql.enable = true;
-        services.mysql.package = pkgs.mariadb;
-        services.mysql.replication.role = "slave";
-        services.mysql.replication.serverId = 3;
-        services.mysql.replication.masterHost = nodes.master.config.networking.hostName;
-        services.mysql.replication.masterUser = replicateUser;
-        services.mysql.replication.masterPassword = replicatePassword;
+      secondary2 = { nodes, ... }: {
+        services.mysql = {
+          inherit package;
+          enable = true;
+          replication.role = "slave";
+          replication.serverId = 3;
+          replication.masterHost = nodes.primary.config.networking.hostName;
+          replication.masterUser = replicateUser;
+          replication.masterPassword = replicatePassword;
+        };
       };
-  };
+    };
 
-  testScript = ''
-    master.start()
-    master.wait_for_unit("mysql")
-    master.wait_for_open_port(3306)
-    # Wait for testdb to be fully populated (5 rows).
-    master.wait_until_succeeds(
-        "sudo -u mysql mysql -u mysql -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
-    )
+    testScript = ''
+      primary.start()
+      primary.wait_for_unit("mysql")
+      primary.wait_for_open_port(3306)
+      # Wait for testdb to be fully populated (5 rows).
+      primary.wait_until_succeeds(
+          "sudo -u mysql mysql -u mysql -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
+      )
 
-    slave1.start()
-    slave2.start()
-    slave1.wait_for_unit("mysql")
-    slave1.wait_for_open_port(3306)
-    slave2.wait_for_unit("mysql")
-    slave2.wait_for_open_port(3306)
+      secondary1.start()
+      secondary2.start()
+      secondary1.wait_for_unit("mysql")
+      secondary1.wait_for_open_port(3306)
+      secondary2.wait_for_unit("mysql")
+      secondary2.wait_for_open_port(3306)
 
-    # wait for replications to finish
-    slave1.wait_until_succeeds(
-        "sudo -u mysql mysql -u mysql -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
-    )
-    slave2.wait_until_succeeds(
-        "sudo -u mysql mysql -u mysql -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
-    )
+      # wait for replications to finish
+      secondary1.wait_until_succeeds(
+          "sudo -u mysql mysql -u mysql -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
+      )
+      secondary2.wait_until_succeeds(
+          "sudo -u mysql mysql -u mysql -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
+      )
 
-    slave2.succeed("systemctl stop mysql")
-    master.succeed(
-        "echo 'insert into testdb.tests values (123, 456);' | sudo -u mysql mysql -u mysql -N"
-    )
-    slave2.succeed("systemctl start mysql")
-    slave2.wait_for_unit("mysql")
-    slave2.wait_for_open_port(3306)
-    slave2.wait_until_succeeds(
-        "echo 'select * from testdb.tests where Id = 123;' | sudo -u mysql mysql -u mysql -N | grep 456"
-    )
-  '';
-})
+      secondary2.succeed("systemctl stop mysql")
+      primary.succeed(
+          "echo 'insert into testdb.tests values (123, 456);' | sudo -u mysql mysql -u mysql -N"
+      )
+      secondary2.succeed("systemctl start mysql")
+      secondary2.wait_for_unit("mysql")
+      secondary2.wait_for_open_port(3306)
+      secondary2.wait_until_succeeds(
+          "echo 'select * from testdb.tests where Id = 123;' | sudo -u mysql mysql -u mysql -N | grep 456"
+      )
+    '';
+  };
+in
+  lib.mapAttrs (_: package: makeReplicationTest { inherit package; }) mariadbPackages
diff --git a/nixos/tests/mysql/mysql.nix b/nixos/tests/mysql/mysql.nix
index 2ac2b34a18e2..197e6da80e24 100644
--- a/nixos/tests/mysql/mysql.nix
+++ b/nixos/tests/mysql/mysql.nix
@@ -1,221 +1,149 @@
-import ./../make-test-python.nix ({ pkgs, ...}:
-
+{
+  system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; },
+  lib ? pkgs.lib
+}:
 
 let
-  # Setup common users
-  users = { ... }:
-  {
-    users.groups.testusers = { };
+  inherit (import ./common.nix { inherit pkgs lib; }) mkTestName mariadbPackages mysqlPackages;
 
-    users.users.testuser = {
-      isSystemUser = true;
-      group = "testusers";
+  makeTest = import ./../make-test-python.nix;
+  # Setup common users
+  makeMySQLTest = {
+    package,
+    name ? mkTestName package,
+    useSocketAuth ? true,
+    hasMroonga ? true,
+    hasRocksDB ? true
+  }: makeTest {
+    inherit name;
+    meta = with lib.maintainers; {
+      maintainers = [ ajs124 das_j ];
     };
 
-    users.users.testuser2 = {
-      isSystemUser = true;
-      group = "testusers";
-    };
-  };
+    nodes = {
+      ${name} =
+        { pkgs, ... }: {
 
-in
+          users = {
+            groups.testusers = { };
 
-{
-  name = "mysql";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ eelco shlevy ];
-  };
-
-  nodes = {
-    mysql57 =
-      { pkgs, ... }:
+            users.testuser = {
+              isSystemUser = true;
+              group = "testusers";
+            };
 
-      {
-        imports = [ users ];
-
-        services.mysql.enable = true;
-        services.mysql.initialDatabases = [
-          { name = "testdb3"; schema = ./testdb.sql; }
-        ];
-        # note that using pkgs.writeText here is generally not a good idea,
-        # as it will store the password in world-readable /nix/store ;)
-        services.mysql.initialScript = pkgs.writeText "mysql-init.sql" ''
-          CREATE USER 'testuser3'@'localhost' IDENTIFIED BY 'secure';
-          GRANT ALL PRIVILEGES ON testdb3.* TO 'testuser3'@'localhost';
-        '';
-        services.mysql.ensureDatabases = [ "testdb" "testdb2" ];
-        services.mysql.ensureUsers = [{
-          name = "testuser";
-          ensurePermissions = {
-            "testdb.*" = "ALL PRIVILEGES";
-          };
-        } {
-          name = "testuser2";
-          ensurePermissions = {
-            "testdb2.*" = "ALL PRIVILEGES";
+            users.testuser2 = {
+              isSystemUser = true;
+              group = "testusers";
+            };
           };
-        }];
-        services.mysql.package = pkgs.mysql57;
-      };
 
-    mysql80 =
-      { pkgs, ... }:
-
-      {
-        imports = [ users ];
-
-        services.mysql.enable = true;
-        services.mysql.initialDatabases = [
-          { name = "testdb3"; schema = ./testdb.sql; }
-        ];
-        # note that using pkgs.writeText here is generally not a good idea,
-        # as it will store the password in world-readable /nix/store ;)
-        services.mysql.initialScript = pkgs.writeText "mysql-init.sql" ''
-          CREATE USER 'testuser3'@'localhost' IDENTIFIED BY 'secure';
-          GRANT ALL PRIVILEGES ON testdb3.* TO 'testuser3'@'localhost';
-        '';
-        services.mysql.ensureDatabases = [ "testdb" "testdb2" ];
-        services.mysql.ensureUsers = [{
-          name = "testuser";
-          ensurePermissions = {
-            "testdb.*" = "ALL PRIVILEGES";
-          };
-        } {
-          name = "testuser2";
-          ensurePermissions = {
-            "testdb2.*" = "ALL PRIVILEGES";
+          services.mysql = {
+            enable = true;
+            initialDatabases = [
+              { name = "testdb3"; schema = ./testdb.sql; }
+            ];
+            # note that using pkgs.writeText here is generally not a good idea,
+            # as it will store the password in world-readable /nix/store ;)
+            initialScript = pkgs.writeText "mysql-init.sql" (if (!useSocketAuth) then ''
+              CREATE USER 'testuser3'@'localhost' IDENTIFIED BY 'secure';
+              GRANT ALL PRIVILEGES ON testdb3.* TO 'testuser3'@'localhost';
+            '' else ''
+              ALTER USER root@localhost IDENTIFIED WITH unix_socket;
+              DELETE FROM mysql.user WHERE password = ''' AND plugin = ''';
+              DELETE FROM mysql.user WHERE user = ''';
+              FLUSH PRIVILEGES;
+            '');
+
+            ensureDatabases = [ "testdb" "testdb2" ];
+            ensureUsers = [{
+              name = "testuser";
+              ensurePermissions = {
+                "testdb.*" = "ALL PRIVILEGES";
+              };
+            } {
+              name = "testuser2";
+              ensurePermissions = {
+                "testdb2.*" = "ALL PRIVILEGES";
+              };
+            }];
+            package = package;
+            settings = {
+              mysqld = {
+                plugin-load-add = lib.optional hasMroonga "ha_mroonga.so"
+                  ++ lib.optional hasRocksDB "ha_rocksdb.so";
+              };
+            };
           };
-        }];
-        services.mysql.package = pkgs.mysql80;
-      };
-
-    mariadb =
-      { pkgs, ... }:
-
-      {
-        imports = [ users ];
+        };
 
-        services.mysql.enable = true;
-        services.mysql.initialScript = pkgs.writeText "mariadb-init.sql" ''
-          ALTER USER root@localhost IDENTIFIED WITH unix_socket;
-          DELETE FROM mysql.user WHERE password = ''' AND plugin = ''';
-          DELETE FROM mysql.user WHERE user = ''';
-          FLUSH PRIVILEGES;
-        '';
-        services.mysql.ensureDatabases = [ "testdb" "testdb2" ];
-        services.mysql.ensureUsers = [{
-          name = "testuser";
-          ensurePermissions = {
-            "testdb.*" = "ALL PRIVILEGES";
-          };
-        } {
-          name = "testuser2";
-          ensurePermissions = {
-            "testdb2.*" = "ALL PRIVILEGES";
-          };
-        }];
-        services.mysql.settings = {
-          mysqld = {
-            plugin-load-add = [ "ha_mroonga.so" "ha_rocksdb.so" ];
-          };
+      mariadb =        {
         };
-        services.mysql.package = pkgs.mariadb;
-      };
+    };
 
+    testScript = ''
+      start_all()
+
+      machine = ${name}
+      machine.wait_for_unit("mysql")
+      machine.succeed(
+          "echo 'use testdb; create table tests (test_id INT, PRIMARY KEY (test_id));' | sudo -u testuser mysql -u testuser"
+      )
+      machine.succeed(
+          "echo 'use testdb; insert into tests values (42);' | sudo -u testuser mysql -u testuser"
+      )
+      # Ensure testuser2 is not able to insert into testdb as mysql testuser2
+      machine.fail(
+          "echo 'use testdb; insert into tests values (23);' | sudo -u testuser2 mysql -u testuser2"
+      )
+      # Ensure testuser2 is not able to authenticate as mysql testuser
+      machine.fail(
+          "echo 'use testdb; insert into tests values (23);' | sudo -u testuser2 mysql -u testuser"
+      )
+      machine.succeed(
+          "echo 'use testdb; select test_id from tests;' | sudo -u testuser mysql -u testuser -N | grep 42"
+      )
+
+      ${lib.optionalString hasMroonga ''
+        # Check if Mroonga plugin works
+        machine.succeed(
+            "echo 'use testdb; create table mroongadb (test_id INT, PRIMARY KEY (test_id)) ENGINE = Mroonga;' | sudo -u testuser mysql -u testuser"
+        )
+        machine.succeed(
+            "echo 'use testdb; insert into mroongadb values (25);' | sudo -u testuser mysql -u testuser"
+        )
+        machine.succeed(
+            "echo 'use testdb; select test_id from mroongadb;' | sudo -u testuser mysql -u testuser -N | grep 25"
+        )
+        machine.succeed(
+            "echo 'use testdb; drop table mroongadb;' | sudo -u testuser mysql -u testuser"
+        )
+      ''}
+
+      ${lib.optionalString hasRocksDB ''
+        # Check if RocksDB plugin works
+        machine.succeed(
+            "echo 'use testdb; create table rocksdb (test_id INT, PRIMARY KEY (test_id)) ENGINE = RocksDB;' | sudo -u testuser mysql -u testuser"
+        )
+        machine.succeed(
+            "echo 'use testdb; insert into rocksdb values (28);' | sudo -u testuser mysql -u testuser"
+        )
+        machine.succeed(
+            "echo 'use testdb; select test_id from rocksdb;' | sudo -u testuser mysql -u testuser -N | grep 28"
+        )
+        machine.succeed(
+            "echo 'use testdb; drop table rocksdb;' | sudo -u testuser mysql -u testuser"
+        )
+      ''}
+    '';
   };
-
-  testScript = ''
-    start_all()
-
-    mysql57.wait_for_unit("mysql")
-    mysql57.succeed(
-        "echo 'use testdb; create table tests (test_id INT, PRIMARY KEY (test_id));' | sudo -u testuser mysql -u testuser"
-    )
-    mysql57.succeed(
-        "echo 'use testdb; insert into tests values (41);' | sudo -u testuser mysql -u testuser"
-    )
-    # Ensure testuser2 is not able to insert into testdb as mysql testuser2
-    mysql57.fail(
-        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser2"
-    )
-    # Ensure testuser2 is not able to authenticate as mysql testuser
-    mysql57.fail(
-        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser"
-    )
-    mysql57.succeed(
-        "echo 'use testdb; select test_id from tests;' | sudo -u testuser mysql -u testuser -N | grep 41"
-    )
-    mysql57.succeed(
-        "echo 'use testdb3; select * from tests;' | mysql -u testuser3 --password=secure -N | grep 4"
-    )
-
-    mysql80.wait_for_unit("mysql")
-    mysql80.succeed(
-        "echo 'use testdb; create table tests (test_id INT, PRIMARY KEY (test_id));' | sudo -u testuser mysql -u testuser"
-    )
-    mysql80.succeed(
-        "echo 'use testdb; insert into tests values (41);' | sudo -u testuser mysql -u testuser"
-    )
-    # Ensure testuser2 is not able to insert into testdb as mysql testuser2
-    mysql80.fail(
-        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser2"
-    )
-    # Ensure testuser2 is not able to authenticate as mysql testuser
-    mysql80.fail(
-        "echo 'use testdb; insert into tests values (22);' | sudo -u testuser2 mysql -u testuser"
-    )
-    mysql80.succeed(
-        "echo 'use testdb; select test_id from tests;' | sudo -u testuser mysql -u testuser -N | grep 41"
-    )
-    mysql80.succeed(
-        "echo 'use testdb3; select * from tests;' | mysql -u testuser3 --password=secure -N | grep 4"
-    )
-
-    mariadb.wait_for_unit("mysql")
-    mariadb.succeed(
-        "echo 'use testdb; create table tests (test_id INT, PRIMARY KEY (test_id));' | sudo -u testuser mysql -u testuser"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; insert into tests values (42);' | sudo -u testuser mysql -u testuser"
-    )
-    # Ensure testuser2 is not able to insert into testdb as mysql testuser2
-    mariadb.fail(
-        "echo 'use testdb; insert into tests values (23);' | sudo -u testuser2 mysql -u testuser2"
-    )
-    # Ensure testuser2 is not able to authenticate as mysql testuser
-    mariadb.fail(
-        "echo 'use testdb; insert into tests values (23);' | sudo -u testuser2 mysql -u testuser"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; select test_id from tests;' | sudo -u testuser mysql -u testuser -N | grep 42"
-    )
-
-    # Check if Mroonga plugin works
-    mariadb.succeed(
-        "echo 'use testdb; create table mroongadb (test_id INT, PRIMARY KEY (test_id)) ENGINE = Mroonga;' | sudo -u testuser mysql -u testuser"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; insert into mroongadb values (25);' | sudo -u testuser mysql -u testuser"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; select test_id from mroongadb;' | sudo -u testuser mysql -u testuser -N | grep 25"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; drop table mroongadb;' | sudo -u testuser mysql -u testuser"
-    )
-
-    # Check if RocksDB plugin works
-    mariadb.succeed(
-        "echo 'use testdb; create table rocksdb (test_id INT, PRIMARY KEY (test_id)) ENGINE = RocksDB;' | sudo -u testuser mysql -u testuser"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; insert into rocksdb values (28);' | sudo -u testuser mysql -u testuser"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; select test_id from rocksdb;' | sudo -u testuser mysql -u testuser -N | grep 28"
-    )
-    mariadb.succeed(
-        "echo 'use testdb; drop table rocksdb;' | sudo -u testuser mysql -u testuser"
-    )
-  '';
-})
+in
+  lib.mapAttrs (_: package: makeMySQLTest {
+    inherit package;
+    hasRocksDB = false; hasMroonga = false; useSocketAuth = false;
+  }) mysqlPackages
+  // (lib.mapAttrs (_: package: makeMySQLTest {
+    inherit package;
+  }) mariadbPackages)
diff --git a/nixos/tests/nano.nix b/nixos/tests/nano.nix
deleted file mode 100644
index 6585a6842e85..000000000000
--- a/nixos/tests/nano.nix
+++ /dev/null
@@ -1,44 +0,0 @@
-import ./make-test-python.nix ({ pkgs, ...} : {
-  name = "nano";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ nequissimus ];
-  };
-
-  machine = { lib, ... }: {
-    environment.systemPackages = [ pkgs.nano ];
-  };
-
-  testScript = { ... }: ''
-    start_all()
-
-    with subtest("Create user and log in"):
-        machine.wait_for_unit("multi-user.target")
-        machine.wait_until_succeeds("pgrep -f 'agetty.*tty1'")
-        machine.succeed("useradd -m alice")
-        machine.succeed("(echo foobar; echo foobar) | passwd alice")
-        machine.wait_until_tty_matches(1, "login: ")
-        machine.send_chars("alice\n")
-        machine.wait_until_tty_matches(1, "login: alice")
-        machine.wait_until_succeeds("pgrep login")
-        machine.wait_until_tty_matches(1, "Password: ")
-        machine.send_chars("foobar\n")
-        machine.wait_until_succeeds("pgrep -u alice bash")
-        machine.screenshot("prompt")
-
-    with subtest("Use nano"):
-        machine.send_chars("nano /tmp/foo")
-        machine.send_key("ret")
-        machine.sleep(2)
-        machine.send_chars("42")
-        machine.sleep(1)
-        machine.send_key("ctrl-x")
-        machine.sleep(1)
-        machine.send_key("y")
-        machine.sleep(1)
-        machine.screenshot("nano")
-        machine.sleep(1)
-        machine.send_key("ret")
-        machine.wait_for_file("/tmp/foo")
-        assert "42" in machine.succeed("cat /tmp/foo")
-  '';
-})
diff --git a/nixos/tests/nats.nix b/nixos/tests/nats.nix
index bee36f262f4c..c650904e53bf 100644
--- a/nixos/tests/nats.nix
+++ b/nixos/tests/nats.nix
@@ -45,21 +45,19 @@ in import ./make-test-python.nix ({ pkgs, lib, ... }: {
             "{}"
         ).format(" ".join(args))
 
+    def parallel(*fns):
+        from threading import Thread
+        threads = [ Thread(target=fn) for fn in fns ]
+        for t in threads: t.start()
+        for t in threads: t.join()
+
     start_all()
     server.wait_for_unit("nats.service")
 
-    client1.fail("test -f ${file}")
-
-    # Subscribe on topic on client1 and echo messages to file.
-    client1.execute("({} | tee ${file} &)".format(nats_cmd("sub", "--raw", "${topic}")))
-
-    # Give client1 some time to subscribe.
-    client1.execute("sleep 2")
-
-    # Publish message on client2.
-    client2.execute(nats_cmd("pub", "${topic}", "hello"))
-
-    # Check if message has been received.
-    client1.succeed("grep -q hello ${file}")
+    with subtest("pub sub"):
+        parallel(
+            lambda: client1.succeed(nats_cmd("sub", "--count", "1", "${topic}")),
+            lambda: client2.succeed("sleep 2 && {}".format(nats_cmd("pub", "${topic}", "hello"))),
+        )
   '';
 })
diff --git a/nixos/tests/nbd.nix b/nixos/tests/nbd.nix
new file mode 100644
index 000000000000..16255e68e8a1
--- /dev/null
+++ b/nixos/tests/nbd.nix
@@ -0,0 +1,87 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+  let
+    listenPort = 30123;
+    testString = "It works!";
+    mkCreateSmallFileService = { path, loop ? false }: {
+      script = ''
+        ${pkgs.coreutils}/bin/dd if=/dev/zero of=${path} bs=1K count=100
+        ${pkgs.lib.optionalString loop
+          "${pkgs.util-linux}/bin/losetup --find ${path}"}
+      '';
+      serviceConfig = {
+        Type = "oneshot";
+      };
+      wantedBy = [ "multi-user.target" ];
+      before = [ "nbd-server.service" ];
+    };
+  in
+  {
+    name = "nbd";
+
+    nodes = {
+      server = { config, pkgs, ... }: {
+        # Create some small files of zeros to use as the ndb disks
+        ## `vault-pub.disk` is accessible from any IP
+        systemd.services.create-pub-file =
+          mkCreateSmallFileService { path = "/vault-pub.disk"; };
+        ## `vault-priv.disk` is accessible only from localhost.
+        ## It's also a loopback device to test exporting /dev/...
+        systemd.services.create-priv-file =
+          mkCreateSmallFileService { path = "/vault-priv.disk"; loop = true; };
+
+        # Needed only for nbd-client used in the tests.
+        environment.systemPackages = [ pkgs.nbd ];
+
+        # Open the nbd port in the firewall
+        networking.firewall.allowedTCPPorts = [ listenPort ];
+
+        # Run the nbd server and expose the small file created above
+        services.nbd.server = {
+          enable = true;
+          exports = {
+            vault-pub = {
+              path = "/vault-pub.disk";
+            };
+            vault-priv = {
+              path = "/dev/loop0";
+              allowAddresses = [ "127.0.0.1" "::1" ];
+            };
+          };
+          listenAddress = "0.0.0.0";
+          listenPort = listenPort;
+        };
+      };
+
+      client = { config, pkgs, ... }: {
+        programs.nbd.enable = true;
+      };
+    };
+
+    testScript = ''
+      testString = "${testString}"
+
+      start_all()
+      server.wait_for_open_port(${toString listenPort})
+
+      # Client: Connect to the server, write a small string to the nbd disk, and cleanly disconnect
+      client.succeed("nbd-client server ${toString listenPort} /dev/nbd0 -name vault-pub -persist")
+      client.succeed(f"echo '{testString}' | dd of=/dev/nbd0 conv=notrunc")
+      client.succeed("nbd-client -d /dev/nbd0")
+
+      # Server: Check that the string written by the client is indeed in the file
+      foundString = server.succeed(f"dd status=none if=/vault-pub.disk count={len(testString)}")[:len(testString)]
+      if foundString != testString:
+         raise Exception(f"Read the wrong string from nbd disk. Expected: '{testString}'. Found: '{foundString}'")
+
+      # Client: Fail to connect to the private disk
+      client.fail("nbd-client server ${toString listenPort} /dev/nbd0 -name vault-priv -persist")
+
+      # Server: Successfully connect to the private disk
+      server.succeed("nbd-client localhost ${toString listenPort} /dev/nbd0 -name vault-priv -persist")
+      server.succeed(f"echo '{testString}' | dd of=/dev/nbd0 conv=notrunc")
+      foundString = server.succeed(f"dd status=none if=/dev/loop0 count={len(testString)}")[:len(testString)]
+      if foundString != testString:
+         raise Exception(f"Read the wrong string from nbd disk. Expected: '{testString}'. Found: '{foundString}'")
+      server.succeed("nbd-client -d /dev/nbd0")
+    '';
+  })
diff --git a/nixos/tests/networking.nix b/nixos/tests/networking.nix
index 647c8942b37d..b763cbd46657 100644
--- a/nixos/tests/networking.nix
+++ b/nixos/tests/networking.nix
@@ -246,13 +246,13 @@ let
         networking = {
           useNetworkd = networkd;
           useDHCP = false;
-          bonds.bond = {
+          bonds.bond0 = {
             interfaces = [ "eth1" "eth2" ];
-            driverOptions.mode = "balance-rr";
+            driverOptions.mode = "802.3ad";
           };
           interfaces.eth1.ipv4.addresses = mkOverride 0 [ ];
           interfaces.eth2.ipv4.addresses = mkOverride 0 [ ];
-          interfaces.bond.ipv4.addresses = mkOverride 0
+          interfaces.bond0.ipv4.addresses = mkOverride 0
             [ { inherit address; prefixLength = 30; } ];
         };
       };
@@ -274,6 +274,10 @@ let
 
               client2.wait_until_succeeds("ping -c 2 192.168.1.1")
               client2.wait_until_succeeds("ping -c 2 192.168.1.2")
+
+          with subtest("Verify bonding mode"):
+              for client in client1, client2:
+                  client.succeed('grep -q "Bonding Mode: IEEE 802.3ad Dynamic link aggregation" /proc/net/bonding/bond0')
         '';
     };
     bridge = let
@@ -489,6 +493,106 @@ let
               client2.wait_until_succeeds("ping -c 1 fc00::2")
         '';
     };
+    gre = let
+      node = { pkgs, ... }: with pkgs.lib; {
+        networking = {
+          useNetworkd = networkd;
+          useDHCP = false;
+          firewall.extraCommands = "ip6tables -A nixos-fw -p gre -j nixos-fw-accept";
+        };
+      };
+    in {
+      name = "GRE";
+      nodes.client1 = args@{ pkgs, ... }:
+        mkMerge [
+          (node args)
+          {
+            virtualisation.vlans = [ 1 2 4 ];
+            networking = {
+              greTunnels = {
+                greTunnel = {
+                  local = "192.168.2.1";
+                  remote = "192.168.2.2";
+                  dev = "eth2";
+                  type = "tap";
+                };
+                gre6Tunnel = {
+                  local = "fd00:1234:5678:4::1";
+                  remote = "fd00:1234:5678:4::2";
+                  dev = "eth3";
+                  type = "tun6";
+                };
+              };
+              bridges.bridge.interfaces = [ "greTunnel" "eth1" ];
+              interfaces.eth1.ipv4.addresses = mkOverride 0 [];
+              interfaces.bridge.ipv4.addresses = mkOverride 0 [
+                { address = "192.168.1.1"; prefixLength = 24; }
+              ];
+              interfaces.eth3.ipv6.addresses = [
+                { address = "fd00:1234:5678:4::1"; prefixLength = 64; }
+              ];
+              interfaces.gre6Tunnel.ipv6.addresses = mkOverride 0 [
+                { address = "fc00::1"; prefixLength = 64; }
+              ];
+            };
+          }
+        ];
+      nodes.client2 = args@{ pkgs, ... }:
+        mkMerge [
+          (node args)
+          {
+            virtualisation.vlans = [ 2 3 4 ];
+            networking = {
+              greTunnels = {
+                greTunnel = {
+                  local = "192.168.2.2";
+                  remote = "192.168.2.1";
+                  dev = "eth1";
+                  type = "tap";
+                };
+                gre6Tunnel = {
+                  local = "fd00:1234:5678:4::2";
+                  remote = "fd00:1234:5678:4::1";
+                  dev = "eth3";
+                  type = "tun6";
+                };
+              };
+              bridges.bridge.interfaces = [ "greTunnel" "eth2" ];
+              interfaces.eth2.ipv4.addresses = mkOverride 0 [];
+              interfaces.bridge.ipv4.addresses = mkOverride 0 [
+                { address = "192.168.1.2"; prefixLength = 24; }
+              ];
+              interfaces.eth3.ipv6.addresses = [
+                { address = "fd00:1234:5678:4::2"; prefixLength = 64; }
+              ];
+              interfaces.gre6Tunnel.ipv6.addresses = mkOverride 0 [
+                { address = "fc00::2"; prefixLength = 64; }
+              ];
+            };
+          }
+        ];
+      testScript = { ... }:
+        ''
+          start_all()
+
+          with subtest("Wait for networking to be configured"):
+              client1.wait_for_unit("network.target")
+              client2.wait_for_unit("network.target")
+
+              # Print diagnostic information
+              client1.succeed("ip addr >&2")
+              client2.succeed("ip addr >&2")
+
+          with subtest("Test GRE tunnel bridge over VLAN"):
+              client1.wait_until_succeeds("ping -c 1 192.168.1.2")
+
+              client2.wait_until_succeeds("ping -c 1 192.168.1.1")
+
+              client1.wait_until_succeeds("ping -c 1 fc00::2")
+
+              client2.wait_until_succeeds("ping -c 1 fc00::1")
+        '';
+    };
     vlan = let
       node = address: { pkgs, ... }: with pkgs.lib; {
         #virtualisation.vlans = [ 1 ];
@@ -669,6 +773,7 @@ let
     routes = {
       name = "routes";
       machine = {
+        networking.useNetworkd = networkd;
         networking.useDHCP = false;
         networking.interfaces.eth0 = {
           ipv4.addresses = [ { address = "192.168.1.2"; prefixLength = 24; } ];
@@ -678,7 +783,13 @@ let
             { address = "2001:1470:fffd:2098::"; prefixLength = 64; via = "fdfd:b3f0::1"; }
           ];
           ipv4.routes = [
-            { address = "10.0.0.0"; prefixLength = 16; options = { mtu = "1500"; }; }
+            { address = "10.0.0.0"; prefixLength = 16; options = {
+              mtu = "1500";
+              # Explicitly set scope because iproute and systemd-networkd
+              # disagree on what the scope should be
+              # if the type is the default "unicast"
+              scope = "link";
+            }; }
             { address = "192.168.2.0"; prefixLength = 24; via = "192.168.1.1"; }
           ];
         };
@@ -727,6 +838,7 @@ let
                 ipv6Table, targetIPv6Table
             )
 
+      '' + optionalString (!networkd) ''
         with subtest("test clean-up of the tables"):
             machine.succeed("systemctl stop network-addresses-eth0")
             ipv4Residue = machine.succeed("ip -4 route list dev eth0 | head -n-3").strip()
@@ -785,7 +897,7 @@ let
         print(client.succeed("ip l add name foo type dummy"))
         print(client.succeed("stat /etc/systemd/network/50-foo.link"))
         client.succeed("udevadm settle")
-        assert "mtu 1442" in client.succeed("ip l show dummy0")
+        assert "mtu 1442" in client.succeed("ip l show dev foo")
       '';
     };
     wlanInterface = let
diff --git a/nixos/tests/nextcloud/default.nix b/nixos/tests/nextcloud/default.nix
index 34d3c345354c..b7b1c5c66002 100644
--- a/nixos/tests/nextcloud/default.nix
+++ b/nixos/tests/nextcloud/default.nix
@@ -18,4 +18,4 @@ foldl
     };
   })
 { }
-  [ 21 22 23 ]
+  [ 22 23 ]
diff --git a/nixos/tests/nextcloud/with-mysql-and-memcached.nix b/nixos/tests/nextcloud/with-mysql-and-memcached.nix
index 80cb63df5dbe..891001e30b23 100644
--- a/nixos/tests/nextcloud/with-mysql-and-memcached.nix
+++ b/nixos/tests/nextcloud/with-mysql-and-memcached.nix
@@ -40,15 +40,16 @@ in {
 
       services.mysql = {
         enable = true;
-        bind = "127.0.0.1";
+        settings.mysqld = {
+          bind-address = "127.0.0.1";
+
+          # FIXME(@Ma27) Nextcloud isn't compatible with mariadb 10.6,
+          # this is a workaround.
+          # See https://help.nextcloud.com/t/update-to-next-cloud-21-0-2-has-get-an-error/117028/22
+          innodb_read_only_compressed = 0;
+        };
         package = pkgs.mariadb;
 
-        # FIXME(@Ma27) Nextcloud isn't compatible with mariadb 10.6,
-        # this is a workaround.
-        # See https://help.nextcloud.com/t/update-to-next-cloud-21-0-2-has-get-an-error/117028/22
-        extraOptions = ''
-          innodb_read_only_compressed=0
-        '';
         initialScript = pkgs.writeText "mysql-init" ''
           CREATE USER 'nextcloud'@'localhost' IDENTIFIED BY 'hunter2';
           CREATE DATABASE IF NOT EXISTS nextcloud;
diff --git a/nixos/tests/nginx-modsecurity.nix b/nixos/tests/nginx-modsecurity.nix
new file mode 100644
index 000000000000..8c53c0196d4c
--- /dev/null
+++ b/nixos/tests/nginx-modsecurity.nix
@@ -0,0 +1,39 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }: {
+  name = "nginx-modsecurity";
+
+  machine = { config, lib, pkgs, ... }: {
+    services.nginx = {
+      enable = true;
+      additionalModules = [ pkgs.nginxModules.modsecurity-nginx ];
+      virtualHosts.localhost =
+        let modsecurity_conf = pkgs.writeText "modsecurity.conf" ''
+          SecRuleEngine On
+          SecDefaultAction "phase:1,log,auditlog,deny,status:403"
+          SecDefaultAction "phase:2,log,auditlog,deny,status:403"
+          SecRule REQUEST_METHOD   "HEAD"        "id:100, phase:1, block"
+          SecRule REQUEST_FILENAME "secret.html" "id:101, phase:2, block"
+        '';
+        testroot = pkgs.runCommand "testroot" {} ''
+          mkdir -p $out
+          echo "<html><body>Hello World!</body></html>" > $out/index.html
+          echo "s3cret" > $out/secret.html
+        '';
+      in {
+        root = testroot;
+        extraConfig = ''
+          modsecurity on;
+          modsecurity_rules_file ${modsecurity_conf};
+        '';
+      };
+    };
+  };
+  testScript = ''
+    machine.wait_for_unit("nginx")
+
+    response = machine.wait_until_succeeds("curl -fvvv -s http://127.0.0.1/")
+    assert "Hello World!" in response
+
+    machine.fail("curl -fvvv -X HEAD -s http://127.0.0.1/")
+    machine.fail("curl -fvvv -s http://127.0.0.1/secret.html")
+  '';
+})
diff --git a/nixos/tests/nix-ld.nix b/nixos/tests/nix-ld.nix
new file mode 100644
index 000000000000..5c886182d969
--- /dev/null
+++ b/nixos/tests/nix-ld.nix
@@ -0,0 +1,20 @@
+import ./make-test-python.nix ({ lib, pkgs, ...} :
+{
+  name = "nix-ld";
+  nodes.machine = { pkgs, ... }: {
+    programs.nix-ld.enable = true;
+    environment.systemPackages = [
+      (pkgs.runCommand "patched-hello" {} ''
+        install -D -m755 ${pkgs.hello}/bin/hello $out/bin/hello
+        patchelf $out/bin/hello --set-interpreter ${pkgs.nix-ld.ldPath}
+      '')
+    ];
+  };
+  testScript = ''
+    start_all()
+    path = "${pkgs.stdenv.cc}/nix-support/dynamic-linker"
+    with open(path) as f:
+        real_ld = f.read().strip()
+    machine.succeed(f"NIX_LD={real_ld} hello")
+ '';
+})
diff --git a/nixos/tests/nixops/default.nix b/nixos/tests/nixops/default.nix
index ec3d028aabae..f0834c51f0b4 100644
--- a/nixos/tests/nixops/default.nix
+++ b/nixos/tests/nixops/default.nix
@@ -9,7 +9,7 @@ let
     #  - Alternatively, blocked on a NixOps 2 release
     #    https://github.com/NixOS/nixops/issues/1242
     # stable = testsLegacyNetwork { nixopsPkg = pkgs.nixops; };
-    unstable = testsForPackage { nixopsPkg = pkgs.nixopsUnstable; };
+    unstable = testsForPackage { nixopsPkg = pkgs.nixops_unstable; };
 
     # inherit testsForPackage;
   };
@@ -23,7 +23,7 @@ let
       deployer = { config, lib, nodes, pkgs, ... }: {
         imports = [ ../../modules/installer/cd-dvd/channel.nix ];
         environment.systemPackages = [ nixopsPkg ];
-        nix.binaryCaches = lib.mkForce [ ];
+        nix.settings.substituters = lib.mkForce [ ];
         users.users.person.isNormalUser = true;
         virtualisation.writableStore = true;
         virtualisation.additionalPaths = [
diff --git a/nixos/tests/nixops/legacy/base-configuration.nix b/nixos/tests/nixops/legacy/base-configuration.nix
index dba960f595c2..7f1c07a5c4a9 100644
--- a/nixos/tests/nixops/legacy/base-configuration.nix
+++ b/nixos/tests/nixops/legacy/base-configuration.nix
@@ -16,7 +16,7 @@ in
     (modulesPath + "/testing/test-instrumentation.nix")
   ];
   virtualisation.writableStore = true;
-  nix.binaryCaches = lib.mkForce [ ];
+  nix.settings.substituters = lib.mkForce [ ];
   virtualisation.graphics = false;
   documentation.enable = false;
   services.qemuGuest.enable = true;
diff --git a/nixos/tests/noto-fonts.nix b/nixos/tests/noto-fonts.nix
new file mode 100644
index 000000000000..049dc766bd3b
--- /dev/null
+++ b/nixos/tests/noto-fonts.nix
@@ -0,0 +1,44 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }: {
+  name = "noto-fonts";
+  meta = {
+    maintainers = with lib.maintainers; [ nickcao midchildan ];
+  };
+
+  machine = {
+    imports = [ ./common/x11.nix ];
+    environment.systemPackages = [ pkgs.gnome.gedit ];
+    fonts = {
+      enableDefaultFonts = false;
+      fonts = with pkgs;[
+        noto-fonts
+        noto-fonts-cjk-sans
+        noto-fonts-cjk-serif
+        noto-fonts-emoji
+      ];
+      fontconfig.defaultFonts = {
+        serif = [ "Noto Serif" "Noto Serif CJK SC" ];
+        sansSerif = [ "Noto Sans" "Noto Sans CJK SC" ];
+        monospace = [ "Noto Sans Mono" "Noto Sans Mono CJK SC" ];
+        emoji = [ "Noto Color Emoji" ];
+      };
+    };
+  };
+
+  testScript =
+    # extracted from http://www.clagnut.com/blog/2380/
+    let testText = builtins.toFile "test.txt" ''
+      the quick brown fox jumps over the lazy dog
+      視野無限廣，窗外有藍天
+      Eĥoŝanĝo ĉiuĵaŭde.
+      いろはにほへと ちりぬるを わかよたれそ つねならむ うゐのおくやま けふこえて あさきゆめみし ゑひもせす
+      다람쥐 헌 쳇바퀴에 타고파
+      中国智造，慧及全球
+    ''; in
+    ''
+      machine.wait_for_x()
+      machine.succeed("gedit ${testText} >&2 &")
+      machine.wait_for_window(".* - gedit")
+      machine.sleep(10)
+      machine.screenshot("screen")
+    '';
+})
diff --git a/nixos/tests/os-prober.nix b/nixos/tests/os-prober.nix
index c1e29b0f68b4..90375450fe1b 100644
--- a/nixos/tests/os-prober.nix
+++ b/nixos/tests/os-prober.nix
@@ -43,11 +43,11 @@ let
       # vda is a filesystem without partition table
       forceInstall = true;
     };
-    nix.binaryCaches = lib.mkForce [ ];
-    nix.extraOptions = ''
-      hashed-mirrors =
-      connect-timeout = 1
-    '';
+    nix.settings = {
+      substituters = lib.mkForce [];
+      hashed-mirrors = null;
+      connect-timeout = 1;
+    };
     # save some memory
     documentation.enable = false;
   };
diff --git a/nixos/tests/pacemaker.nix b/nixos/tests/pacemaker.nix
new file mode 100644
index 000000000000..684557614953
--- /dev/null
+++ b/nixos/tests/pacemaker.nix
@@ -0,0 +1,110 @@
+import ./make-test-python.nix  ({ pkgs, lib, ... }: rec {
+  name = "pacemaker";
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [ astro ];
+  };
+
+  nodes =
+    let
+      node = i: {
+        networking.interfaces.eth1.ipv4.addresses = [ {
+          address = "192.168.0.${toString i}";
+          prefixLength = 24;
+        } ];
+
+        services.corosync = {
+          enable = true;
+          clusterName = "zentralwerk-network";
+          nodelist = lib.imap (i: name: {
+            nodeid = i;
+            inherit name;
+            ring_addrs = [
+              (builtins.head nodes.${name}.networking.interfaces.eth1.ipv4.addresses).address
+            ];
+          }) (builtins.attrNames nodes);
+        };
+        environment.etc."corosync/authkey" = {
+          source = builtins.toFile "authkey"
+            # minimum length: 128 bytes
+            "testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest";
+          mode = "0400";
+        };
+
+        services.pacemaker.enable = true;
+
+        # used for pacemaker resource
+        systemd.services.ha-cat = {
+          description = "Highly available netcat";
+          serviceConfig.ExecStart = "${pkgs.netcat}/bin/nc -l discard";
+        };
+      };
+    in {
+      node1 = node 1;
+      node2 = node 2;
+      node3 = node 3;
+    };
+
+  # sets up pacemaker with resources configuration, then crashes a
+  # node and waits for service restart on another node
+  testScript =
+    let
+      resources = builtins.toFile "cib-resources.xml" ''
+        <resources>
+          <primitive id="cat" class="systemd" type="ha-cat">
+            <operations>
+              <op id="stop-cat" name="start" interval="0" timeout="1s"/>
+              <op id="start-cat" name="start" interval="0" timeout="1s"/>
+              <op id="monitor-cat" name="monitor" interval="1s" timeout="1s"/>
+            </operations>
+          </primitive>
+        </resources>
+      '';
+    in ''
+      import re
+      import time
+
+      start_all()
+
+      ${lib.concatMapStrings (node: ''
+        ${node}.wait_until_succeeds("corosync-quorumtool")
+        ${node}.wait_for_unit("pacemaker.service")
+      '') (builtins.attrNames nodes)}
+
+      # No STONITH device
+      node1.succeed("crm_attribute -t crm_config -n stonith-enabled -v false")
+      # Configure the cat resource
+      node1.succeed("cibadmin --replace --scope resources --xml-file ${resources}")
+
+      # wait until the service is started
+      while True:
+        output = node1.succeed("crm_resource -r cat --locate")
+        match = re.search("is running on: (.+)", output)
+        if match:
+          for machine in machines:
+            if machine.name == match.group(1):
+              current_node = machine
+          break
+        time.sleep(1)
+
+      current_node.log("Service running here!")
+      current_node.crash()
+
+      # pick another node that's still up
+      for machine in machines:
+        if machine.booted:
+          check_node = machine
+      # find where the service has been started next
+      while True:
+        output = check_node.succeed("crm_resource -r cat --locate")
+        match = re.search("is running on: (.+)", output)
+        # output will remain the old current_node until the crash is detected by pacemaker
+        if match and match.group(1) != current_node.name:
+          for machine in machines:
+            if machine.name == match.group(1):
+              next_node = machine
+          break
+        time.sleep(1)
+
+      next_node.log("Service migrated here!")
+  '';
+})
diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix
index d838d3b6a39c..50b977723e9c 100644
--- a/nixos/tests/parsedmarc/default.nix
+++ b/nixos/tests/parsedmarc/default.nix
@@ -4,6 +4,7 @@
 { pkgs, ... }@args:
 let
   inherit (import ../../lib/testing-python.nix args) makeTest;
+  inherit (pkgs) lib;
 
   dmarcTestReport = builtins.fetchurl {
     name = "dmarc-test-report";
@@ -54,7 +55,7 @@ in
   localMail = makeTest
     {
       name = "parsedmarc-local-mail";
-      meta = with pkgs.lib.maintainers; {
+      meta = with lib.maintainers; {
         maintainers = [ talyz ];
       };
 
@@ -83,7 +84,7 @@ in
             };
           };
 
-          services.elasticsearch.package = pkgs.elasticsearch7-oss;
+          services.elasticsearch.package = pkgs.elasticsearch-oss;
 
           environment.systemPackages = [
             (sendEmail "dmarc@localhost")
@@ -94,6 +95,7 @@ in
       testScript = { nodes }:
         let
           esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+          valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
         in ''
           parsedmarc.start()
           parsedmarc.wait_for_unit("postfix.service")
@@ -104,11 +106,15 @@ in
           )
 
           parsedmarc.fail(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
           parsedmarc.succeed("send-email")
           parsedmarc.wait_until_succeeds(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
         '';
     };
@@ -121,7 +127,7 @@ in
     in
       makeTest {
         name = "parsedmarc-external-mail";
-        meta = with pkgs.lib.maintainers; {
+        meta = with lib.maintainers; {
           maintainers = [ talyz ];
         };
 
@@ -153,7 +159,7 @@ in
                 };
               };
 
-              services.elasticsearch.package = pkgs.elasticsearch7-oss;
+              services.elasticsearch.package = pkgs.elasticsearch-oss;
 
               environment.systemPackages = [
                 pkgs.jq
@@ -201,6 +207,7 @@ in
         testScript = { nodes }:
           let
             esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+            valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
           in ''
             mail.start()
             mail.wait_for_unit("postfix.service")
@@ -213,11 +220,15 @@ in
             )
 
             parsedmarc.fail(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
             mail.succeed("send-email")
             parsedmarc.wait_until_succeeds(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
           '';
       };
diff --git a/nixos/tests/pgadmin4-standalone.nix b/nixos/tests/pgadmin4-standalone.nix
new file mode 100644
index 000000000000..442570c5306b
--- /dev/null
+++ b/nixos/tests/pgadmin4-standalone.nix
@@ -0,0 +1,43 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }:
+  # This is seperate from pgadmin4 since we don't want both running at once
+
+  {
+    name = "pgadmin4-standalone";
+    meta.maintainers = with lib.maintainers; [ mkg20001 ];
+
+    nodes.machine = { pkgs, ... }: {
+      environment.systemPackages = with pkgs; [
+        curl
+      ];
+
+      services.postgresql = {
+        enable = true;
+
+        authentication = ''
+          host    all             all             localhost               trust
+        '';
+
+        ensureUsers = [
+          {
+            name = "postgres";
+            ensurePermissions = {
+              "DATABASE \"postgres\"" = "ALL PRIVILEGES";
+            };
+          }
+        ];
+      };
+
+      services.pgadmin = {
+        enable = true;
+        initialEmail = "bruh@localhost.de";
+        initialPasswordFile = pkgs.writeText "pw" "bruh2012!";
+      };
+    };
+
+    testScript = ''
+      machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("pgadmin")
+
+      machine.wait_until_succeeds("curl -s localhost:5050")
+    '';
+  })
diff --git a/nixos/tests/pgadmin4.nix b/nixos/tests/pgadmin4.nix
new file mode 100644
index 000000000000..658315d3ac0c
--- /dev/null
+++ b/nixos/tests/pgadmin4.nix
@@ -0,0 +1,142 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }:
+
+  let
+    pgadmin4SrcDir = "/pgadmin";
+    pgadmin4Dir = "/var/lib/pgadmin";
+    pgadmin4LogDir = "/var/log/pgadmin";
+
+    python-with-needed-packages = pkgs.python3.withPackages (ps: with ps; [
+      selenium
+      testtools
+      testscenarios
+      flask
+      flask-babelex
+      flask-babel
+      flask-gravatar
+      flask_login
+      flask_mail
+      flask_migrate
+      flask_sqlalchemy
+      flask_wtf
+      flask-compress
+      passlib
+      pytz
+      simplejson
+      six
+      sqlparse
+      wtforms
+      flask-paranoid
+      psutil
+      psycopg2
+      python-dateutil
+      sqlalchemy
+      itsdangerous
+      flask-security-too
+      bcrypt
+      cryptography
+      sshtunnel
+      ldap3
+      gssapi
+      flask-socketio
+      eventlet
+      httpagentparser
+      user-agents
+      wheel
+      authlib
+      qrcode
+      pillow
+      pyotp
+    ]);
+  in
+  {
+    name = "pgadmin4";
+    meta.maintainers = with lib.maintainers; [ gador ];
+
+    nodes.machine = { pkgs, ... }: {
+      imports = [ ./common/x11.nix ];
+      environment.systemPackages = with pkgs; [
+        pgadmin4
+        postgresql
+        python-with-needed-packages
+        chromedriver
+        chromium
+      ];
+      services.postgresql = {
+        enable = true;
+        authentication = ''
+          host    all             all             localhost               trust
+        '';
+        ensureUsers = [
+          {
+            name = "postgres";
+            ensurePermissions = {
+              "DATABASE \"postgres\"" = "ALL PRIVILEGES";
+            };
+          }
+        ];
+      };
+    };
+
+    testScript = ''
+      machine.wait_for_unit("postgresql")
+
+      # pgadmin4 needs its data and log directories
+      machine.succeed(
+          "mkdir -p ${pgadmin4Dir} \
+          && mkdir -p ${pgadmin4LogDir} \
+          && mkdir -p ${pgadmin4SrcDir}"
+      )
+
+      machine.succeed(
+           "tar xvzf ${pkgs.pgadmin4.src} -C ${pgadmin4SrcDir}"
+      )
+
+      machine.wait_for_file("${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/README.md")
+
+      # set paths and config for tests
+      machine.succeed(
+           "cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version} \
+           && cp -v web/regression/test_config.json.in web/regression/test_config.json \
+           && sed -i 's|PostgreSQL 9.4|PostgreSQL|' web/regression/test_config.json \
+           && sed -i 's|/opt/PostgreSQL/9.4/bin/|${pkgs.postgresql}/bin|' web/regression/test_config.json \
+           && sed -i 's|\"headless_chrome\": false|\"headless_chrome\": true|' web/regression/test_config.json"
+      )
+
+      # adapt chrome config to run within a sandbox without GUI
+      # see https://stackoverflow.com/questions/50642308/webdriverexception-unknown-error-devtoolsactiveport-file-doesnt-exist-while-t#50642913
+      # add chrome binary path. use spaces to satisfy python indention (tabs throw an error)
+      # this works for selenium 3 (currently used), but will need to be updated
+      # to work with "from selenium.webdriver.chrome.service import Service" in selenium 4
+      machine.succeed(
+           "cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version} \
+           && sed -i '\|options.add_argument(\"--disable-infobars\")|a \ \ \ \ \ \ \ \ options.binary_location = \"${pkgs.chromium}/bin/chromium\"' web/regression/runtests.py \
+           && sed -i '\|options.add_argument(\"--no-sandbox\")|a \ \ \ \ \ \ \ \ options.add_argument(\"--headless\")' web/regression/runtests.py \
+           && sed -i '\|options.add_argument(\"--disable-infobars\")|a \ \ \ \ \ \ \ \ options.add_argument(\"--disable-dev-shm-usage\")' web/regression/runtests.py \
+           && sed -i 's|(chrome_options=options)|(executable_path=\"${pkgs.chromedriver}/bin/chromedriver\", chrome_options=options)|' web/regression/runtests.py \
+           && sed -i 's|driver_local.maximize_window()||' web/regression/runtests.py"
+      )
+
+      # don't bother to test LDAP authentification
+      with subtest("run browser test"):
+          machine.succeed(
+               'cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/web \
+               && ${python-with-needed-packages.interpreter} regression/runtests.py --pkg browser --exclude \
+               browser.tests.test_ldap_login.LDAPLoginTestCase,browser.tests.test_ldap_login'
+          )
+
+      # fontconfig is necessary for chromium to run
+      # https://github.com/NixOS/nixpkgs/issues/136207
+      with subtest("run feature test"):
+          machine.succeed(
+              'cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/web \
+               && export FONTCONFIG_FILE=${pkgs.makeFontsConf { fontDirectories = [];}} \
+               && ${python-with-needed-packages.interpreter} regression/runtests.py --pkg feature_tests'
+          )
+
+      with subtest("run resql test"):
+          machine.succeed(
+               'cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/web \
+               && ${python-with-needed-packages.interpreter} regression/runtests.py --pkg resql'
+          )
+    '';
+  })
diff --git a/nixos/tests/php/fpm.nix b/nixos/tests/php/fpm.nix
index 31a79bb4dbe3..718a635a6c7c 100644
--- a/nixos/tests/php/fpm.nix
+++ b/nixos/tests/php/fpm.nix
@@ -17,7 +17,7 @@ import ../make-test-python.nix ({ pkgs, lib, php, ... }: {
           locations."~ \\.php$".extraConfig = ''
             fastcgi_pass unix:${config.services.phpfpm.pools.foobar.socket};
             fastcgi_index index.php;
-            include ${pkgs.nginx}/conf/fastcgi_params;
+            include ${config.services.nginx.package}/conf/fastcgi_params;
             include ${pkgs.nginx}/conf/fastcgi.conf;
           '';
           locations."/" = {
diff --git a/nixos/tests/plausible.nix b/nixos/tests/plausible.nix
index 45e11f0270e6..58c1dd5cf4a8 100644
--- a/nixos/tests/plausible.nix
+++ b/nixos/tests/plausible.nix
@@ -8,6 +8,9 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: {
     virtualisation.memorySize = 4096;
     services.plausible = {
       enable = true;
+      releaseCookiePath = "${pkgs.runCommand "cookie" { } ''
+        ${pkgs.openssl}/bin/openssl rand -base64 64 >"$out"
+      ''}";
       adminUser = {
         email = "admin@example.org";
         passwordFile = "${pkgs.writeText "pwd" "foobar"}";
diff --git a/nixos/tests/pleroma.nix b/nixos/tests/pleroma.nix
index bf3623fce38b..90a9a2511044 100644
--- a/nixos/tests/pleroma.nix
+++ b/nixos/tests/pleroma.nix
@@ -32,8 +32,7 @@ import ./make-test-python.nix ({ pkgs, ... }:
     # system one. Overriding this pretty bad default behaviour.
     export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
-    export TOOT_LOGIN_CLI_PASSWORD="jamy-password"
-    toot login_cli -i "pleroma.nixos.test" -e "jamy@nixos.test"
+    echo "jamy-password" | toot login_cli -i "pleroma.nixos.test" -e "jamy@nixos.test"
     echo "Login OK"
 
     # Send a toot then verify it's part of the public timeline
@@ -168,21 +167,6 @@ import ./make-test-python.nix ({ pkgs, ... }:
     cp key.pem cert.pem $out
   '';
 
-  /* Toot is preventing users from feeding login_cli a password non
-     interactively. While it makes sense most of the times, it's
-     preventing us to login in this non-interactive test. This patch
-     introduce a TOOT_LOGIN_CLI_PASSWORD env variable allowing us to
-     provide a password to toot login_cli
-
-     If https://github.com/ihabunek/toot/pull/180 gets merged at some
-     point, feel free to remove this patch. */
-  custom-toot = pkgs.toot.overrideAttrs(old:{
-    patches = [ (pkgs.fetchpatch {
-      url = "https://github.com/NinjaTrappeur/toot/commit/b4a4c30f41c0cb7e336714c2c4af9bc9bfa0c9f2.patch";
-      sha256 = "sha256-0xxNwjR/fStLjjUUhwzCCfrghRVts+fc+fvVJqVcaFg=";
-    }) ];
-  });
-
   hosts = nodes: ''
     ${nodes.pleroma.config.networking.primaryIPAddress} pleroma.nixos.test
     ${nodes.client.config.networking.primaryIPAddress} client.nixos.test
@@ -194,7 +178,7 @@ import ./make-test-python.nix ({ pkgs, ... }:
       security.pki.certificateFiles = [ "${tls-cert}/cert.pem" ];
       networking.extraHosts = hosts nodes;
       environment.systemPackages = with pkgs; [
-        custom-toot
+        toot
         send-toot
       ];
     };
diff --git a/nixos/tests/podman/default.nix b/nixos/tests/podman/default.nix
index b52a7f060ad6..67c7823c5a31 100644
--- a/nixos/tests/podman/default.nix
+++ b/nixos/tests/podman/default.nix
@@ -126,7 +126,7 @@ import ../make-test-python.nix (
           podman.succeed("docker network create default")
           podman.succeed("tar cv --files-from /dev/null | podman import - scratchimg")
           podman.succeed(
-            "docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin scratchimg /bin/sleep 10"
+            "docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin localhost/scratchimg /bin/sleep 10"
           )
           podman.succeed("docker ps | grep sleeping")
           podman.succeed("podman ps | grep sleeping")
diff --git a/nixos/tests/podman/tls-ghostunnel.nix b/nixos/tests/podman/tls-ghostunnel.nix
index c0bc47cc40b1..268a55701ccf 100644
--- a/nixos/tests/podman/tls-ghostunnel.nix
+++ b/nixos/tests/podman/tls-ghostunnel.nix
@@ -129,7 +129,7 @@ import ../make-test-python.nix (
           podman.succeed("tar cv --files-from /dev/null | podman import - scratchimg")
 
           client.succeed(
-            "docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin scratchimg /bin/sleep 10"
+            "docker run -d --name=sleeping -v /nix/store:/nix/store -v /run/current-system/sw/bin:/bin localhost/scratchimg /bin/sleep 10"
           )
           client.succeed("docker ps | grep sleeping")
           podman.succeed("docker ps | grep sleeping")
diff --git a/nixos/tests/powerdns-admin.nix b/nixos/tests/powerdns-admin.nix
new file mode 100644
index 000000000000..4d763c9c6f6e
--- /dev/null
+++ b/nixos/tests/powerdns-admin.nix
@@ -0,0 +1,117 @@
+# Test powerdns-admin
+{ system ? builtins.currentSystem
+, config ? { }
+, pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
+let
+  defaultConfig = ''
+    BIND_ADDRESS = '127.0.0.1'
+    PORT = 8000
+  '';
+
+  makeAppTest = name: configs: makeTest {
+    name = "powerdns-admin-${name}";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ Flakebi zhaofengli ];
+    };
+
+    nodes.server = { pkgs, config, ... }: mkMerge ([
+      {
+        services.powerdns-admin = {
+          enable = true;
+          secretKeyFile = "/etc/powerdns-admin/secret";
+          saltFile = "/etc/powerdns-admin/salt";
+        };
+        # It's insecure to have secrets in the world-readable nix store, but this is just a test
+        environment.etc."powerdns-admin/secret".text = "secret key";
+        environment.etc."powerdns-admin/salt".text = "salt";
+        environment.systemPackages = [
+          (pkgs.writeShellScriptBin "run-test" config.system.build.testScript)
+        ];
+      }
+    ] ++ configs);
+
+    testScript = ''
+      server.wait_for_unit("powerdns-admin.service")
+      server.wait_until_succeeds("run-test", timeout=10)
+    '';
+  };
+
+  matrix = {
+    backend = {
+      mysql = {
+        services.powerdns-admin = {
+          config = ''
+            ${defaultConfig}
+            SQLALCHEMY_DATABASE_URI = 'mysql://powerdnsadmin@/powerdnsadmin?unix_socket=/run/mysqld/mysqld.sock'
+          '';
+        };
+        systemd.services.powerdns-admin = {
+          after = [ "mysql.service" ];
+          serviceConfig.BindPaths = "/run/mysqld";
+        };
+
+        services.mysql = {
+          enable = true;
+          package = pkgs.mariadb;
+          ensureDatabases = [ "powerdnsadmin" ];
+          ensureUsers = [
+            {
+              name = "powerdnsadmin";
+              ensurePermissions = {
+                "powerdnsadmin.*" = "ALL PRIVILEGES";
+              };
+            }
+          ];
+        };
+      };
+      postgresql = {
+        services.powerdns-admin = {
+          config = ''
+            ${defaultConfig}
+            SQLALCHEMY_DATABASE_URI = 'postgresql://powerdnsadmin@/powerdnsadmin?host=/run/postgresql'
+          '';
+        };
+        systemd.services.powerdns-admin = {
+          after = [ "postgresql.service" ];
+          serviceConfig.BindPaths = "/run/postgresql";
+        };
+
+        services.postgresql = {
+          enable = true;
+          ensureDatabases = [ "powerdnsadmin" ];
+          ensureUsers = [
+            {
+              name = "powerdnsadmin";
+              ensurePermissions = {
+                "DATABASE powerdnsadmin" = "ALL PRIVILEGES";
+              };
+            }
+          ];
+        };
+      };
+    };
+    listen = {
+      tcp = {
+        services.powerdns-admin.extraArgs = [ "-b" "127.0.0.1:8000" ];
+        system.build.testScript = ''
+          curl -sSf http://127.0.0.1:8000/
+        '';
+      };
+      unix = {
+        services.powerdns-admin.extraArgs = [ "-b" "unix:/run/powerdns-admin/http.sock" ];
+        system.build.testScript = ''
+          curl -sSf --unix-socket /run/powerdns-admin/http.sock http://somehost/
+        '';
+      };
+    };
+  };
+in
+with matrix; {
+  postgresql = makeAppTest "postgresql" [ backend.postgresql listen.tcp ];
+  mysql = makeAppTest "mysql" [ backend.mysql listen.tcp ];
+  unix-listener = makeAppTest "unix-listener" [ backend.postgresql listen.unix ];
+}
diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix
index 62deb3864951..ce3b3fbf3bf3 100644
--- a/nixos/tests/prometheus-exporters.nix
+++ b/nixos/tests/prometheus-exporters.nix
@@ -259,6 +259,19 @@ let
       '';
     };
 
+    fastly = {
+      exporterConfig = {
+        enable = true;
+        tokenPath = pkgs.writeText "token" "abc123";
+      };
+
+      # noop: fastly's exporter can't start without first talking to fastly
+      # see: https://github.com/peterbourgon/fastly-exporter/issues/87
+      exporterTest = ''
+        succeed("true");
+      '';
+    };
+
     fritzbox = {
       # TODO add proper test case
       exporterConfig = {
@@ -659,7 +672,7 @@ let
             basicAuth.nextcloud-exporter = "snakeoilpw";
             locations."/" = {
               root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
-              tryFiles = "/negative-space.xml =404";
+              tryFiles = "/negative-space.json =404";
             };
           };
         };
@@ -920,6 +933,27 @@ let
       '';
     };
 
+    pve = let
+      pveExporterEnvFile = pkgs.writeTextFile {
+        name = "pve.env";
+        text = ''
+          PVE_USER="test_user@pam"
+          PVE_PASSWORD="hunter3"
+          PVE_VERIFY_SSL="false"
+        '';
+      };
+    in {
+      exporterConfig = {
+        enable = true;
+        environmentFile = pveExporterEnvFile;
+      };
+      exporterTest = ''
+        wait_for_unit("prometheus-pve-exporter.service")
+        wait_for_open_port(9221)
+        wait_until_succeeds("curl localhost:9221")
+      '';
+    };
+
     py-air-control = {
       nodeName = "py_air_control";
       exporterConfig = {
@@ -939,7 +973,7 @@ let
       exporterConfig = {
         enable = true;
       };
-      metricProvider.services.redis.enable = true;
+      metricProvider.services.redis.servers."".enable = true;
       exporterTest = ''
         wait_for_unit("redis.service")
         wait_for_unit("prometheus-redis-exporter.service")
@@ -1143,6 +1177,10 @@ let
     systemd = {
       exporterConfig = {
         enable = true;
+
+        extraFlags = [
+          "--collector.enable-restart-count"
+        ];
       };
       metricProvider = { };
       exporterTest = ''
@@ -1153,6 +1191,11 @@ let
                 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
             )
         )
+        succeed(
+            "curl -sSf localhost:9558/metrics | grep '{}'".format(
+                'systemd_service_restart_total{state="prometheus-systemd-exporter.service"} 0'
+            )
+        )
       '';
     };
 
diff --git a/nixos/tests/pulseaudio.nix b/nixos/tests/pulseaudio.nix
new file mode 100644
index 000000000000..4e2ce679acd7
--- /dev/null
+++ b/nixos/tests/pulseaudio.nix
@@ -0,0 +1,71 @@
+let
+  mkTest = { systemWide ? false }:
+    import ./make-test-python.nix ({ pkgs, lib, ... }:
+      let
+        testFile = pkgs.fetchurl {
+          url =
+            "https://file-examples-com.github.io/uploads/2017/11/file_example_MP3_700KB.mp3";
+          hash = "sha256-+iggJW8s0/LfA/okfXsB550/55Q0Sq3OoIzuBrzOPJQ=";
+        };
+
+        makeTestPlay = key:
+          { sox, alsa-utils }:
+          pkgs.writeScriptBin key ''
+            set -euxo pipefail
+            ${sox}/bin/play ${testFile}
+            ${sox}/bin/sox ${testFile} -t wav - | ${alsa-utils}/bin/aplay
+            touch /tmp/${key}_success
+          '';
+
+        testers = builtins.mapAttrs makeTestPlay {
+          testPlay = { inherit (pkgs) sox alsa-utils; };
+          testPlay32 = { inherit (pkgs.pkgsi686Linux) sox alsa-utils; };
+        };
+      in {
+        name = "pulseaudio${lib.optionalString systemWide "-systemWide"}";
+        meta = with pkgs.lib.maintainers; {
+          maintainers = [ synthetica ] ++ pkgs.pulseaudio.meta.maintainers;
+        };
+
+        machine = { ... }:
+
+          {
+            imports = [ ./common/wayland-cage.nix ];
+            hardware.pulseaudio = {
+              enable = true;
+              support32Bit = true;
+              inherit systemWide;
+            };
+
+            environment.systemPackages = [ testers.testPlay pkgs.pavucontrol ]
+              ++ lib.optional pkgs.stdenv.isx86_64 testers.testPlay32;
+          } // lib.optionalAttrs systemWide {
+            users.users.alice.extraGroups = [ "audio" ];
+            systemd.services.pulseaudio.wantedBy = [ "multi-user.target" ];
+          };
+
+        enableOCR = true;
+
+        testScript = { ... }: ''
+          machine.wait_until_succeeds("pgrep xterm")
+          machine.wait_for_text("alice@machine")
+
+          machine.send_chars("testPlay \n")
+          machine.wait_for_file("/tmp/testPlay_success")
+          ${lib.optionalString pkgs.stdenv.isx86_64 ''
+            machine.send_chars("testPlay32 \n")
+            machine.wait_for_file("/tmp/testPlay32_success")
+          ''}
+          machine.screenshot("testPlay")
+
+          # Pavucontrol only loads when Pulseaudio is running. If it isn't, the
+          # text "Playback" (one of the tabs) will never show.
+          machine.send_chars("pavucontrol\n")
+          machine.wait_for_text("Playback")
+          machine.screenshot("Pavucontrol")
+        '';
+      });
+in builtins.mapAttrs (key: val: mkTest val) {
+  user = { systemWide = false; };
+  system = { systemWide = true; };
+}
diff --git a/nixos/tests/quorum.nix b/nixos/tests/quorum.nix
index 498b55ace7af..31669eb7fc38 100644
--- a/nixos/tests/quorum.nix
+++ b/nixos/tests/quorum.nix
@@ -1,4 +1,29 @@
-import ./make-test-python.nix ({ pkgs, ... }: {
+import ./make-test-python.nix ({ pkgs, ... }:
+let
+  keystore =  {
+    address = "9377bc3936de934c497e22917b81aa8774ac3bb0";
+    crypto = {
+      cipher = "aes-128-ctr";
+      ciphertext = "ad8341d8ef225650403fd366c955f41095e438dd966a3c84b3d406818c1e366c";
+      cipherparams = {
+        iv = "2a09f7a72fd6dff7c43150ff437e6ac2";
+      };
+      kdf = "scrypt";
+      kdfparams = {
+        dklen = 32;
+        n = 262144;
+        p = 1;
+        r = 8;
+        salt = "d1a153845bb80cd6274c87c5bac8ac09fdfac5ff131a6f41b5ed319667f12027";
+      };
+      mac = "a9621ad88fa1d042acca6fc2fcd711f7e05bfbadea3f30f379235570c8e270d3";
+    };
+    id = "89e847a3-1527-42f6-a321-77de0a14ce02";
+    version = 3;
+  };
+  keystore-file = pkgs.writeText "keystore-file" (builtins.toJSON keystore);
+in
+{
   name = "quorum";
   meta = with pkgs.lib.maintainers; {
     maintainers = [ mmahut ];
@@ -62,18 +87,16 @@ import ./make-test-python.nix ({ pkgs, ... }: {
 
   testScript = ''
     start_all()
-    machine.wait_until_succeeds("mkdir -p /var/lib/quorum/keystore")
-    machine.wait_until_succeeds(
-        'echo \{\\"address\\":\\"9377bc3936de934c497e22917b81aa8774ac3bb0\\",\\"crypto\\":\{\\"cipher\\":\\"aes-128-ctr\\",\\"ciphertext\\":\\"ad8341d8ef225650403fd366c955f41095e438dd966a3c84b3d406818c1e366c\\",\\"cipherparams\\":\{\\"iv\\":\\"2a09f7a72fd6dff7c43150ff437e6ac2\\"\},\\"kdf\\":\\"scrypt\\",\\"kdfparams\\":\{\\"dklen\\":32,\\"n\\":262144,\\"p\\":1,\\"r\\":8,\\"salt\\":\\"d1a153845bb80cd6274c87c5bac8ac09fdfac5ff131a6f41b5ed319667f12027\\"\},\\"mac\\":\\"a9621ad88fa1d042acca6fc2fcd711f7e05bfbadea3f30f379235570c8e270d3\\"\},\\"id\\":\\"89e847a3-1527-42f6-a321-77de0a14ce02\\",\\"version\\":3\}\\" > /var/lib/quorum/keystore/UTC--2020-03-23T11-08-34.144812212Z--9377bc3936de934c497e22917b81aa8774ac3bb0'
+    machine.succeed("mkdir -p /var/lib/quorum/keystore")
+    machine.succeed(
+        'cp ${keystore-file} /var/lib/quorum/keystore/UTC--2020-03-23T11-08-34.144812212Z--${keystore.address}'
     )
-    machine.wait_until_succeeds(
+    machine.succeed(
         "echo fe2725c4e8f7617764b845e8d939a65c664e7956eb47ed7d934573f16488efc1 > /var/lib/quorum/nodekey"
     )
-    machine.wait_until_succeeds("systemctl restart quorum")
+    machine.succeed("systemctl restart quorum")
     machine.wait_for_unit("quorum.service")
     machine.sleep(15)
-    machine.wait_until_succeeds(
-        'geth attach /var/lib/quorum/geth.ipc --exec "eth.accounts" | grep 0x9377bc3936de934c497e22917b81aa8774ac3bb0'
-    )
+    machine.succeed('geth attach /var/lib/quorum/geth.ipc --exec "eth.accounts" | grep ${keystore.address}')
   '';
 })
diff --git a/nixos/tests/redis.nix b/nixos/tests/redis.nix
index 28b6058c2c02..7b70c239ad6e 100644
--- a/nixos/tests/redis.nix
+++ b/nixos/tests/redis.nix
@@ -1,7 +1,4 @@
 import ./make-test-python.nix ({ pkgs, ... }:
-let
-  redisSocket = "/run/redis/redis.sock";
-in
 {
   name = "redis";
   meta = with pkgs.lib.maintainers; {
@@ -10,35 +7,40 @@ in
 
   nodes = {
     machine =
-      { pkgs, ... }:
+      { pkgs, lib, ... }: with lib;
 
       {
-        services.redis.enable = true;
-        services.redis.unixSocket = redisSocket;
+        services.redis.servers."".enable = true;
+        services.redis.servers."test".enable = true;
 
-        # Allow access to the unix socket for the "redis" group.
-        services.redis.unixSocketPerm = 770;
-
-        users.users."member" = {
+        users.users = listToAttrs (map (suffix: nameValuePair "member${suffix}" {
           createHome = false;
-          description = "A member of the redis group";
+          description = "A member of the redis${suffix} group";
           isNormalUser = true;
-          extraGroups = [
-            "redis"
-          ];
-        };
+          extraGroups = [ "redis${suffix}" ];
+        }) ["" "-test"]);
       };
   };
 
-  testScript = ''
+  testScript = { nodes, ... }: let
+    inherit (nodes.machine.config.services) redis;
+    in ''
     start_all()
     machine.wait_for_unit("redis")
+    machine.wait_for_unit("redis-test")
+
+    # The unnamed Redis server still opens a port for backward-compatibility
     machine.wait_for_open_port("6379")
 
+    machine.wait_for_file("${redis.servers."".unixSocket}")
+    machine.wait_for_file("${redis.servers."test".unixSocket}")
+
     # The unix socket is accessible to the redis group
     machine.succeed('su member -c "redis-cli ping | grep PONG"')
+    machine.succeed('su member-test -c "redis-cli ping | grep PONG"')
 
     machine.succeed("redis-cli ping | grep PONG")
-    machine.succeed("redis-cli -s ${redisSocket} ping | grep PONG")
+    machine.succeed("redis-cli -s ${redis.servers."".unixSocket} ping | grep PONG")
+    machine.succeed("redis-cli -s ${redis.servers."test".unixSocket} ping | grep PONG")
   '';
 })
diff --git a/nixos/tests/retroarch.nix b/nixos/tests/retroarch.nix
new file mode 100644
index 000000000000..4c96f9eabc82
--- /dev/null
+++ b/nixos/tests/retroarch.nix
@@ -0,0 +1,49 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+  {
+    name = "retroarch";
+    meta = with pkgs.lib.maintainers; { maintainers = [ j0hax ]; };
+
+    machine = { ... }:
+
+      {
+        imports = [ ./common/user-account.nix ];
+        services.xserver.enable = true;
+        services.xserver.desktopManager.retroarch = {
+          enable = true;
+          package = pkgs.retroarchFull;
+        };
+        services.xserver.displayManager = {
+          sddm.enable = true;
+          defaultSession = "RetroArch";
+          autoLogin = {
+            enable = true;
+            user = "alice";
+          };
+        };
+      };
+
+    testScript = { nodes, ... }:
+      let
+        user = nodes.machine.config.users.users.alice;
+        xdo = "${pkgs.xdotool}/bin/xdotool";
+      in ''
+        with subtest("Wait for login"):
+            start_all()
+            machine.wait_for_file("${user.home}/.Xauthority")
+            machine.succeed("xauth merge ${user.home}/.Xauthority")
+
+        with subtest("Check RetroArch started"):
+            machine.wait_until_succeeds("pgrep retroarch")
+            machine.wait_for_window("^RetroArch ")
+
+        with subtest("Check configuration created"):
+            machine.wait_for_file("${user.home}/.config/retroarch/retroarch.cfg")
+
+        with subtest("Wait to get a screenshot"):
+            machine.execute(
+                "${xdo} key Alt+F1 sleep 10"
+            )
+            machine.screenshot("screen")
+      '';
+  })
diff --git a/nixos/tests/rstudio-server.nix b/nixos/tests/rstudio-server.nix
new file mode 100644
index 000000000000..c7ac7670fbd4
--- /dev/null
+++ b/nixos/tests/rstudio-server.nix
@@ -0,0 +1,30 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+  {
+    name = "rstudio-server-test";
+    meta.maintainers = with pkgs.lib.maintainers; [ jbedo cfhammill ];
+
+    nodes.machine = { config, lib, pkgs, ... }: {
+      services.rstudio-server.enable = true;
+    };
+
+    nodes.customPackageMachine = { config, lib, pkgs, ... }: {
+      services.rstudio-server = {
+        enable = true;
+        package = pkgs.rstudioServerWrapper.override { packages = [ pkgs.rPackages.ggplot2 ]; };
+      };
+    };
+
+    users.testuser = {
+      uid = 1000;
+      group = "testgroup";
+    };
+    groups.testgroup.gid = 1000;
+
+    testScript = ''
+      machine.wait_for_unit("rstudio-server.service")
+      machine.succeed("curl -f -vvv -s http://127.0.0.1:8787")
+
+      customPackageMachine.wait_for_unit("rstudio-server.service")
+      customPackageMachine.succeed("curl -f -vvv -s http://127.0.0.1:8787")
+    '';
+  })
diff --git a/nixos/tests/samba-wsdd.nix b/nixos/tests/samba-wsdd.nix
index e7dd17c089a3..0e3185b0c684 100644
--- a/nixos/tests/samba-wsdd.nix
+++ b/nixos/tests/samba-wsdd.nix
@@ -38,7 +38,7 @@ import ./make-test-python.nix ({ pkgs, ... }:
     server_wsdd.wait_for_unit("samba-wsdd")
 
     client_wsdd.wait_until_succeeds(
-        "echo list | ${pkgs.libressl.nc}/bin/nc -U /run/wsdd/wsdd.sock | grep -i SERVER-WSDD"
+        "echo list | ${pkgs.libressl.nc}/bin/nc -N -U /run/wsdd/wsdd.sock | grep -i SERVER-WSDD"
     )
   '';
 })
diff --git a/nixos/tests/shiori.nix b/nixos/tests/shiori.nix
index 418bee43c939..6c59c394009e 100644
--- a/nixos/tests/shiori.nix
+++ b/nixos/tests/shiori.nix
@@ -12,7 +12,6 @@ import ./make-test-python.nix ({ pkgs, lib, ...}:
     authJSON = pkgs.writeText "auth.json" (builtins.toJSON {
       username = "shiori";
       password = "gopher";
-      remember = 1; # hour
       owner = true;
     });
 
diff --git a/nixos/tests/snapcast.nix b/nixos/tests/snapcast.nix
index 8d960b4cc069..30b8343e2ffe 100644
--- a/nixos/tests/snapcast.nix
+++ b/nixos/tests/snapcast.nix
@@ -40,6 +40,7 @@ in {
           };
         };
       };
+      environment.systemPackages = [ pkgs.snapcast ];
     };
     client = {
       environment.systemPackages = [ pkgs.snapcast ];
@@ -71,6 +72,13 @@ in {
             "curl --fail http://localhost:${toString httpPort}/jsonrpc -d '{json.dumps(get_rpc_version)}'"
         )
 
+    with subtest("test a ipv6 connection"):
+        server.execute("systemd-run --unit=snapcast-local-client snapclient -h ::1 -p ${toString port}")
+        server.wait_until_succeeds(
+            "journalctl -o cat -u snapserver.service | grep -q 'Hello from'"
+        )
+        server.wait_until_succeeds("journalctl -o cat -u snapcast-local-client | grep -q 'buffer: ${toString bufferSize}'")
+
     with subtest("test a connection"):
         client.execute("systemd-run --unit=snapcast-client snapclient -h server -p ${toString port}")
         server.wait_until_succeeds(
diff --git a/nixos/tests/sourcehut.nix b/nixos/tests/sourcehut.nix
index b56a14ebf85e..55757e35f9b4 100644
--- a/nixos/tests/sourcehut.nix
+++ b/nixos/tests/sourcehut.nix
@@ -1,29 +1,212 @@
-import ./make-test-python.nix ({ pkgs, ... }:
+import ./make-test-python.nix ({ pkgs, lib, ... }:
+let
+  domain = "sourcehut.localdomain";
 
+  # Note that wildcard certificates just under the TLD (eg. *.com)
+  # would be rejected by clients like curl.
+  tls-cert = pkgs.runCommand "selfSignedCerts" { buildInputs = [ pkgs.openssl ]; } ''
+    openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -nodes -days 36500 \
+      -subj '/CN=${domain}' -extensions v3_req \
+      -addext 'subjectAltName = DNS:*.${domain}'
+    install -D -t $out key.pem cert.pem
+  '';
+
+  images = {
+    nixos.unstable.x86_64 =
+      let
+        systemConfig = { pkgs, ... }: {
+          # passwordless ssh server
+          services.openssh = {
+            enable = true;
+            permitRootLogin = "yes";
+            extraConfig = "PermitEmptyPasswords yes";
+          };
+
+          users = {
+            mutableUsers = false;
+            # build user
+            extraUsers."build" = {
+              isNormalUser = true;
+              uid = 1000;
+              extraGroups = [ "wheel" ];
+              password = "";
+            };
+            users.root.password = "";
+          };
+
+          security.sudo.wheelNeedsPassword = false;
+          nix.trustedUsers = [ "root" "build" ];
+          documentation.nixos.enable = false;
+
+          # builds.sr.ht-image-specific network settings
+          networking = {
+            hostName = "build";
+            dhcpcd.enable = false;
+            defaultGateway.address = "10.0.2.2";
+            usePredictableInterfaceNames = false;
+            interfaces."eth0".ipv4.addresses = [{
+              address = "10.0.2.15";
+              prefixLength = 25;
+            }];
+            enableIPv6 = false;
+            nameservers = [
+              # OpenNIC anycast
+              "185.121.177.177"
+              "169.239.202.202"
+              # Google
+              "8.8.8.8"
+            ];
+            firewall.allowedTCPPorts = [ 22 ];
+          };
+
+          environment.systemPackages = [
+            pkgs.gitMinimal
+            #pkgs.mercurial
+            pkgs.curl
+            pkgs.gnupg
+          ];
+        };
+        qemuConfig = { pkgs, ... }: {
+          imports = [ systemConfig ];
+          fileSystems."/".device = "/dev/disk/by-label/nixos";
+          boot.initrd.availableKernelModules = [
+            "ahci"
+            "ehci_pci"
+            "sd_mod"
+            "usb_storage"
+            "usbhid"
+            "virtio_balloon"
+            "virtio_blk"
+            "virtio_pci"
+            "virtio_ring"
+            "xhci_pci"
+          ];
+          boot.loader = {
+            grub = {
+              version = 2;
+              device = "/dev/vda";
+            };
+            timeout = 0;
+          };
+        };
+        config = (import (pkgs.path + "/nixos/lib/eval-config.nix") {
+          inherit pkgs; modules = [ qemuConfig ];
+          system = "x86_64-linux";
+        }).config;
+      in
+      import (pkgs.path + "/nixos/lib/make-disk-image.nix") {
+        inherit pkgs lib config;
+        diskSize = 16000;
+        format = "qcow2-compressed";
+        contents = [
+          { source = pkgs.writeText "gitconfig" ''
+              [user]
+                name = builds.sr.ht
+                email = build@sr.ht
+            '';
+            target = "/home/build/.gitconfig";
+            user = "build";
+            group = "users";
+            mode = "644";
+          }
+        ];
+      };
+  };
+
+in
 {
   name = "sourcehut";
 
   meta.maintainers = [ pkgs.lib.maintainers.tomberek ];
 
-  machine = { config, pkgs, ... }: {
-    virtualisation.memorySize = 2048;
-    networking.firewall.allowedTCPPorts = [ 80 ];
+  machine = { config, pkgs, nodes, ... }: {
+    # buildsrht needs space
+    virtualisation.diskSize = 4 * 1024;
+    virtualisation.memorySize = 2 * 1024;
+    networking.domain = domain;
+    networking.extraHosts = ''
+      ${config.networking.primaryIPAddress} builds.${domain}
+      ${config.networking.primaryIPAddress} git.${domain}
+      ${config.networking.primaryIPAddress} meta.${domain}
+    '';
 
     services.sourcehut = {
       enable = true;
-      services = [ "meta" ];
-      originBase = "sourcehut";
-      settings."sr.ht".service-key =   "8888888888888888888888888888888888888888888888888888888888888888";
-      settings."sr.ht".network-key = "0000000000000000000000000000000000000000000=";
-      settings.webhooks.private-key = "0000000000000000000000000000000000000000000=";
+      services = [
+        "builds"
+        "git"
+        "meta"
+      ];
+      nginx.enable = true;
+      nginx.virtualHost = {
+        forceSSL = true;
+        sslCertificate = "${tls-cert}/cert.pem";
+        sslCertificateKey = "${tls-cert}/key.pem";
+      };
+      postgresql.enable = true;
+      redis.enable = true;
+
+      meta.enable = true;
+      builds = {
+        enable = true;
+        # FIXME: see why it does not seem to activate fully.
+        #enableWorker = true;
+        inherit images;
+      };
+      git.enable = true;
+
+      settings."sr.ht" = {
+        global-domain = config.networking.domain;
+        service-key = pkgs.writeText "service-key" "8b327279b77e32a3620e2fc9aabce491cc46e7d821fd6713b2a2e650ce114d01";
+        network-key = pkgs.writeText "network-key" "cEEmc30BRBGkgQZcHFksiG7hjc6_dK1XR2Oo5Jb9_nQ=";
+      };
+      settings."builds.sr.ht" = {
+        oauth-client-secret = pkgs.writeText "buildsrht-oauth-client-secret" "2260e9c4d9b8dcedcef642860e0504bc";
+        oauth-client-id = "299db9f9c2013170";
+      };
+      settings."git.sr.ht" = {
+        oauth-client-secret = pkgs.writeText "gitsrht-oauth-client-secret" "3597288dc2c716e567db5384f493b09d";
+        oauth-client-id = "d07cb713d920702e";
+      };
+      settings.webhooks.private-key = pkgs.writeText "webhook-key" "Ra3IjxgFiwG9jxgp4WALQIZw/BMYt30xWiOsqD0J7EA=";
+    };
+
+    networking.firewall.allowedTCPPorts = [ 443 ];
+    security.pki.certificateFiles = [ "${tls-cert}/cert.pem" ];
+    services.nginx = {
+      enable = true;
+      recommendedGzipSettings = true;
+      recommendedOptimisation = true;
+      recommendedTlsSettings = true;
+      recommendedProxySettings = true;
+    };
+
+    services.postgresql = {
+      enable = true;
+      enableTCPIP = false;
+      settings.unix_socket_permissions = "0770";
     };
   };
 
   testScript = ''
     start_all()
     machine.wait_for_unit("multi-user.target")
+
+    # Testing metasrht
+    machine.wait_for_unit("metasrht-api.service")
     machine.wait_for_unit("metasrht.service")
     machine.wait_for_open_port(5000)
-    machine.succeed("curl -sL http://localhost:5000 | grep meta.sourcehut")
+    machine.succeed("curl -sL http://localhost:5000 | grep meta.${domain}")
+    machine.succeed("curl -sL http://meta.${domain} | grep meta.${domain}")
+
+    # Testing buildsrht
+    machine.wait_for_unit("buildsrht.service")
+    machine.wait_for_open_port(5002)
+    machine.succeed("curl -sL http://localhost:5002 | grep builds.${domain}")
+    #machine.wait_for_unit("buildsrht-worker.service")
+
+    # Testing gitsrht
+    machine.wait_for_unit("gitsrht.service")
+    machine.succeed("curl -sL http://git.${domain} | grep git.${domain}")
   '';
 })
diff --git a/nixos/tests/ssh-keys.nix b/nixos/tests/ssh-keys.nix
index 07d422196efa..df9ff38a3b22 100644
--- a/nixos/tests/ssh-keys.nix
+++ b/nixos/tests/ssh-keys.nix
@@ -10,6 +10,6 @@ pkgs:
   snakeOilPublicKey = pkgs.lib.concatStrings [
     "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
     "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
-    "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
+    "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= snakeoil"
   ];
 }
diff --git a/nixos/tests/starship.nix b/nixos/tests/starship.nix
new file mode 100644
index 000000000000..33e9a72f7000
--- /dev/null
+++ b/nixos/tests/starship.nix
@@ -0,0 +1,42 @@
+import ./make-test-python.nix ({ pkgs, ... }: {
+  name = "starship";
+  meta.maintainers = pkgs.starship.meta.maintainers;
+
+  machine = {
+    programs = {
+      fish.enable = true;
+      zsh.enable = true;
+
+      starship = {
+        enable = true;
+        settings.format = "<starship>";
+      };
+    };
+
+    environment.systemPackages = map
+      (shell: pkgs.writeScriptBin "expect-${shell}" ''
+        #!${pkgs.expect}/bin/expect -f
+
+        spawn env TERM=xterm ${shell} -i
+
+        expect "<starship>" {
+          send "exit\n"
+        } timeout {
+          send_user "\n${shell} failed to display Starship\n"
+          exit 1
+        }
+
+        expect eof
+      '')
+      [ "bash" "fish" "zsh" ];
+  };
+
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("default.target")
+
+    machine.succeed("expect-bash")
+    machine.succeed("expect-fish")
+    machine.succeed("expect-zsh")
+  '';
+})
diff --git a/nixos/tests/sudo.nix b/nixos/tests/sudo.nix
index 4885d6e17b82..661fe9989e7a 100644
--- a/nixos/tests/sudo.nix
+++ b/nixos/tests/sudo.nix
@@ -28,6 +28,10 @@ in
           enable = true;
           wheelNeedsPassword = false;
 
+          extraConfig = ''
+            Defaults lecture="never"
+          '';
+
           extraRules = [
             # SUDOERS SYNTAX CHECK (Test whether the module produces a valid output;
             # errors being detected by the visudo checks.
@@ -73,7 +77,7 @@ in
             machine.fail('su - test1 -c "sudo -n -u root true"')
 
         with subtest("users in group 'foobar' should be able to use sudo with password"):
-            machine.succeed("sudo -u test2 echo ${password} | sudo -S -u root true")
+            machine.succeed('su - test2 -c "echo ${password} | sudo -S -u root true"')
 
         with subtest("users in group 'barfoo' should be able to use sudo without password"):
             machine.succeed("sudo -u test3 sudo -n -u root true")
diff --git a/nixos/tests/sway.nix b/nixos/tests/sway.nix
index 3476ebab3e26..1e9e146c4b6c 100644
--- a/nixos/tests/sway.nix
+++ b/nixos/tests/sway.nix
@@ -1,6 +1,4 @@
-import ./make-test-python.nix ({ pkgs, lib, ...} :
-
-{
+import ./make-test-python.nix ({ pkgs, lib, ... }: {
   name = "sway";
   meta = {
     maintainers = with lib.maintainers; [ primeos synthetica ];
@@ -13,19 +11,38 @@ import ./make-test-python.nix ({ pkgs, lib, ...} :
 
     environment = {
       # For glinfo and wayland-info:
-      systemPackages = with pkgs; [ mesa-demos wayland-utils ];
+      systemPackages = with pkgs; [ mesa-demos wayland-utils alacritty ];
       # Use a fixed SWAYSOCK path (for swaymsg):
       variables = {
         "SWAYSOCK" = "/tmp/sway-ipc.sock";
-        "WLR_RENDERER_ALLOW_SOFTWARE" = "1";
+        # TODO: Investigate if we can get hardware acceleration to work (via
+        # virtio-gpu and Virgil). We currently have to use the Pixman software
+        # renderer since the GLES2 renderer doesn't work inside the VM (even
+        # with WLR_RENDERER_ALLOW_SOFTWARE):
+        # "WLR_RENDERER_ALLOW_SOFTWARE" = "1";
+        "WLR_RENDERER" = "pixman";
       };
       # For convenience:
       shellAliases = {
-        test-x11 = "glinfo | head -n 3 | tee /tmp/test-x11.out && touch /tmp/test-x11-exit-ok";
+        test-x11 = "glinfo | tee /tmp/test-x11.out && touch /tmp/test-x11-exit-ok";
         test-wayland = "wayland-info | tee /tmp/test-wayland.out && touch /tmp/test-wayland-exit-ok";
       };
+
+      # To help with OCR:
+      etc."xdg/foot/foot.ini".text = lib.generators.toINI { } {
+        main = {
+          font = "inconsolata:size=14";
+        };
+        colors = rec {
+          foreground = "000000";
+          background = "ffffff";
+          regular2 = foreground;
+        };
+      };
     };
 
+    fonts.fonts = [ pkgs.inconsolata ];
+
     # Automatically configure and start Sway when logging in on tty1:
     programs.bash.loginShellInit = ''
       if [ "$(tty)" = "/dev/tty1" ]; then
@@ -51,6 +68,14 @@ import ./make-test-python.nix ({ pkgs, lib, ...} :
   enableOCR = true;
 
   testScript = { nodes, ... }: ''
+    import shlex
+
+    def swaymsg(command: str, succeed=True):
+        with machine.nested(f"sending swaymsg {command!r}" + " (allowed to fail)" * (not succeed)):
+          (machine.succeed if succeed else machine.execute)(
+            f"su - alice -c {shlex.quote('swaymsg -- ' + command)}"
+          )
+
     start_all()
     machine.wait_for_unit("multi-user.target")
 
@@ -61,35 +86,32 @@ import ./make-test-python.nix ({ pkgs, lib, ...} :
     machine.wait_for_file("/run/user/1000/wayland-1")
     machine.wait_for_file("/tmp/sway-ipc.sock")
 
-    # Test XWayland:
-    machine.succeed(
-        "su - alice -c 'swaymsg exec WINIT_UNIX_BACKEND=x11 WAYLAND_DISPLAY=invalid alacritty'"
-    )
+    # Test XWayland (foot does not support X):
+    swaymsg("exec WINIT_UNIX_BACKEND=x11 WAYLAND_DISPLAY=invalid alacritty")
     machine.wait_for_text("alice@machine")
     machine.send_chars("test-x11\n")
     machine.wait_for_file("/tmp/test-x11-exit-ok")
     print(machine.succeed("cat /tmp/test-x11.out"))
+    machine.copy_from_vm("/tmp/test-x11.out")
     machine.screenshot("alacritty_glinfo")
     machine.succeed("pkill alacritty")
 
-    # Start a terminal (Alacritty) on workspace 3:
+    # Start a terminal (foot) on workspace 3:
     machine.send_key("alt-3")
-    machine.succeed(
-        "su - alice -c 'swaymsg exec WINIT_UNIX_BACKEND=wayland DISPLAY=invalid alacritty'"
-    )
+    machine.sleep(3)
+    machine.send_key("alt-ret")
     machine.wait_for_text("alice@machine")
     machine.send_chars("test-wayland\n")
     machine.wait_for_file("/tmp/test-wayland-exit-ok")
     print(machine.succeed("cat /tmp/test-wayland.out"))
-    machine.screenshot("alacritty_wayland_info")
+    machine.copy_from_vm("/tmp/test-wayland.out")
+    machine.screenshot("foot_wayland_info")
     machine.send_key("alt-shift-q")
-    machine.wait_until_fails("pgrep alacritty")
+    machine.wait_until_fails("pgrep foot")
 
     # Test gpg-agent starting pinentry-gnome3 via D-Bus (tests if
     # $WAYLAND_DISPLAY is correctly imported into the D-Bus user env):
-    machine.succeed(
-        "su - alice -c 'swaymsg -- exec gpg --no-tty --yes --quick-generate-key test'"
-    )
+    swaymsg("exec gpg --no-tty --yes --quick-generate-key test")
     machine.wait_until_succeeds("pgrep --exact gpg")
     machine.wait_for_text("Passphrase")
     machine.screenshot("gpg_pinentry")
@@ -101,12 +123,16 @@ import ./make-test-python.nix ({ pkgs, lib, ...} :
     machine.wait_for_text("You pressed the exit shortcut.")
     machine.screenshot("sway_exit")
 
+    swaymsg("exec swaylock")
+    machine.wait_until_succeeds("pgrep -x swaylock")
+    machine.sleep(3)
+    machine.send_chars("${nodes.machine.config.users.users.alice.password}")
+    machine.send_key("ret")
+    machine.wait_until_fails("pgrep -x swaylock")
+
     # Exit Sway and verify process exit status 0:
-    machine.succeed("su - alice -c 'swaymsg exit || true'")
+    swaymsg("exit", succeed=False)
     machine.wait_until_fails("pgrep -x sway")
-
-    # TODO: Sway currently segfaults after "swaymsg exit" but only in this VM test:
-    # machine # [  104.090032] sway[921]: segfault at 3f800008 ip 00007f7dbdc25f10 sp 00007ffe282182f8 error 4 in libwayland-server.so.0.1.0[7f7dbdc1f000+8000]
-    # machine.wait_for_file("/tmp/sway-exit-ok")
+    machine.wait_for_file("/tmp/sway-exit-ok")
   '';
 })
diff --git a/nixos/tests/switch-test.nix b/nixos/tests/switch-test.nix
index 78adf7ffa7da..0198866b6ff8 100644
--- a/nixos/tests/switch-test.nix
+++ b/nixos/tests/switch-test.nix
@@ -1,23 +1,471 @@
 # Test configuration switching.
 
-import ./make-test-python.nix ({ pkgs, ...} : {
+import ./make-test-python.nix ({ pkgs, ...} : let
+
+  # Simple service that can either be socket-activated or that will
+  # listen on port 1234 if not socket-activated.
+  # A connection to the socket causes 'hello' to be written to the client.
+  socketTest = pkgs.writeScript "socket-test.py" /* python */ ''
+    #!${pkgs.python3}/bin/python3
+
+    from socketserver import TCPServer, StreamRequestHandler
+    import socket
+    import os
+
+
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            self.wfile.write("hello".encode("utf-8"))
+
+
+    class Server(TCPServer):
+        def __init__(self, server_address, handler_cls):
+            listenFds = os.getenv('LISTEN_FDS')
+            if listenFds is None or int(listenFds) < 1:
+                print(f'Binding to {server_address}')
+                TCPServer.__init__(
+                        self, server_address, handler_cls, bind_and_activate=True)
+            else:
+                TCPServer.__init__(
+                        self, server_address, handler_cls, bind_and_activate=False)
+                # Override socket
+                print(f'Got activated by {os.getenv("LISTEN_FDNAMES")} '
+                      f'with {listenFds} FDs')
+                self.socket = socket.fromfd(3, self.address_family,
+                                            self.socket_type)
+
+
+    if __name__ == "__main__":
+        server = Server(("localhost", 1234), Handler)
+        server.serve_forever()
+  '';
+
+in {
   name = "switch-test";
   meta = with pkgs.lib.maintainers; {
-    maintainers = [ gleber ];
+    maintainers = [ gleber das_j ];
   };
 
   nodes = {
-    machine = { ... }: {
+    machine = { pkgs, lib, ... }: {
+      environment.systemPackages = [ pkgs.socat ]; # for the socket activation stuff
       users.mutableUsers = false;
+
+      # For boot/switch testing
+      system.build.installBootLoader = lib.mkForce (pkgs.writeShellScript "install-dummy-loader" ''
+        echo "installing dummy bootloader"
+        touch /tmp/bootloader-installed
+      '');
+
+      specialisation = rec {
+        simpleService.configuration = {
+          systemd.services.test = {
+            wantedBy = [ "multi-user.target" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              ExecStart = "${pkgs.coreutils}/bin/true";
+              ExecReload = "${pkgs.coreutils}/bin/true";
+            };
+          };
+        };
+
+        simpleServiceDifferentDescription.configuration = {
+          imports = [ simpleService.configuration ];
+          systemd.services.test.description = "Test unit";
+        };
+
+        simpleServiceModified.configuration = {
+          imports = [ simpleService.configuration ];
+          systemd.services.test.serviceConfig.X-Test = true;
+        };
+
+        simpleServiceNostop.configuration = {
+          imports = [ simpleService.configuration ];
+          systemd.services.test.stopIfChanged = false;
+        };
+
+        simpleServiceReload.configuration = {
+          imports = [ simpleService.configuration ];
+          systemd.services.test = {
+            reloadIfChanged = true;
+            serviceConfig.ExecReload = "${pkgs.coreutils}/bin/true";
+          };
+        };
+
+        simpleServiceNorestart.configuration = {
+          imports = [ simpleService.configuration ];
+          systemd.services.test.restartIfChanged = false;
+        };
+
+        simpleServiceFailing.configuration = {
+          imports = [ simpleServiceModified.configuration ];
+          systemd.services.test.serviceConfig.ExecStart = lib.mkForce "${pkgs.coreutils}/bin/false";
+        };
+
+        autorestartService.configuration = {
+          # A service that immediately goes into restarting (but without failing)
+          systemd.services.autorestart = {
+            wantedBy = [ "multi-user.target" ];
+            serviceConfig = {
+              Type = "simple";
+              Restart = "always";
+              RestartSec = "20y"; # Should be long enough
+              ExecStart = "${pkgs.coreutils}/bin/true";
+            };
+          };
+        };
+
+        autorestartServiceFailing.configuration = {
+          imports = [ autorestartService.configuration ];
+          systemd.services.autorestart.serviceConfig = {
+            ExecStart = lib.mkForce "${pkgs.coreutils}/bin/false";
+          };
+        };
+
+        simpleServiceWithExtraSection.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.packages = [ (pkgs.writeTextFile {
+            name = "systemd-extra-section";
+            destination = "/etc/systemd/system/test.service";
+            text = ''
+              [X-Test]
+              X-Test-Value=a
+            '';
+          }) ];
+        };
+
+        simpleServiceWithExtraSectionOtherName.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.packages = [ (pkgs.writeTextFile {
+            name = "systemd-extra-section";
+            destination = "/etc/systemd/system/test.service";
+            text = ''
+              [X-Test2]
+              X-Test-Value=a
+            '';
+          }) ];
+        };
+
+        simpleServiceWithInstallSection.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.packages = [ (pkgs.writeTextFile {
+            name = "systemd-extra-section";
+            destination = "/etc/systemd/system/test.service";
+            text = ''
+              [Install]
+              WantedBy=multi-user.target
+            '';
+          }) ];
+        };
+
+        simpleServiceWithExtraKey.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.services.test.serviceConfig."X-Test" = "test";
+        };
+
+        simpleServiceWithExtraKeyOtherValue.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.services.test.serviceConfig."X-Test" = "test2";
+        };
+
+        simpleServiceWithExtraKeyOtherName.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.services.test.serviceConfig."X-Test2" = "test";
+        };
+
+        simpleServiceReloadTrigger.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.services.test.reloadTriggers = [ "/dev/null" ];
+        };
+
+        simpleServiceReloadTriggerModified.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.services.test.reloadTriggers = [ "/dev/zero" ];
+        };
+
+        simpleServiceReloadTriggerModifiedAndSomethingElse.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.services.test = {
+            reloadTriggers = [ "/dev/zero" ];
+            serviceConfig."X-Test" = "test";
+          };
+        };
+
+        simpleServiceReloadTriggerModifiedSomethingElse.configuration = {
+          imports = [ simpleServiceNostop.configuration ];
+          systemd.services.test.serviceConfig."X-Test" = "test";
+        };
+
+        unitWithBackslash.configuration = {
+          systemd.services."escaped\\x2ddash" = {
+            wantedBy = [ "multi-user.target" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              ExecStart = "${pkgs.coreutils}/bin/true";
+              ExecReload = "${pkgs.coreutils}/bin/true";
+            };
+          };
+        };
+
+        unitWithBackslashModified.configuration = {
+          imports = [ unitWithBackslash.configuration ];
+          systemd.services."escaped\\x2ddash".serviceConfig.X-Test = "test";
+        };
+
+        unitWithRequirement.configuration = {
+          systemd.services.required-service = {
+            wantedBy = [ "multi-user.target" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              ExecStart = "${pkgs.coreutils}/bin/true";
+              ExecReload = "${pkgs.coreutils}/bin/true";
+            };
+          };
+          systemd.services.test-service = {
+            wantedBy = [ "multi-user.target" ];
+            requires = [ "required-service.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              ExecStart = "${pkgs.coreutils}/bin/true";
+              ExecReload = "${pkgs.coreutils}/bin/true";
+            };
+          };
+        };
+
+        unitWithRequirementModified.configuration = {
+          imports = [ unitWithRequirement.configuration ];
+          systemd.services.required-service.serviceConfig.X-Test = "test";
+          systemd.services.test-service.reloadTriggers = [ "test" ];
+        };
+
+        unitWithRequirementModifiedNostart.configuration = {
+          imports = [ unitWithRequirement.configuration ];
+          systemd.services.test-service.unitConfig.RefuseManualStart = true;
+        };
+
+        restart-and-reload-by-activation-script.configuration = {
+          systemd.services = rec {
+            simple-service = {
+              # No wantedBy so we can check if the activation script restart triggers them
+              serviceConfig = {
+                Type = "oneshot";
+                RemainAfterExit = true;
+                ExecStart = "${pkgs.coreutils}/bin/true";
+                ExecReload = "${pkgs.coreutils}/bin/true";
+              };
+            };
+
+            simple-restart-service = simple-service // {
+              stopIfChanged = false;
+            };
+
+            simple-reload-service = simple-service // {
+              reloadIfChanged = true;
+            };
+
+            no-restart-service = simple-service // {
+              restartIfChanged = false;
+            };
+
+            reload-triggers = simple-service // {
+              wantedBy = [ "multi-user.target" ];
+            };
+
+            reload-triggers-and-restart-by-as = simple-service;
+
+            reload-triggers-and-restart = simple-service // {
+              stopIfChanged = false; # easier to check for this
+              wantedBy = [ "multi-user.target" ];
+            };
+          };
+
+          system.activationScripts.restart-and-reload-test = {
+            supportsDryActivation = true;
+            deps = [];
+            text = ''
+              if [ "$NIXOS_ACTION" = dry-activate ]; then
+                f=/run/nixos/dry-activation-restart-list
+                g=/run/nixos/dry-activation-reload-list
+              else
+                f=/run/nixos/activation-restart-list
+                g=/run/nixos/activation-reload-list
+              fi
+              cat <<EOF >> "$f"
+              simple-service.service
+              simple-restart-service.service
+              simple-reload-service.service
+              no-restart-service.service
+              reload-triggers-and-restart-by-as.service
+              EOF
+
+              cat <<EOF >> "$g"
+              reload-triggers.service
+              reload-triggers-and-restart-by-as.service
+              reload-triggers-and-restart.service
+              EOF
+            '';
+          };
+        };
+
+        restart-and-reload-by-activation-script-modified.configuration = {
+          imports = [ restart-and-reload-by-activation-script.configuration ];
+          systemd.services.reload-triggers-and-restart.serviceConfig.X-Modified = "test";
+        };
+
+        simple-socket.configuration = {
+          systemd.services.socket-activated = {
+            description = "A socket-activated service";
+            stopIfChanged = lib.mkDefault false;
+            serviceConfig = {
+              ExecStart = socketTest;
+              ExecReload = "${pkgs.coreutils}/bin/true";
+            };
+          };
+          systemd.sockets.socket-activated = {
+            wantedBy = [ "sockets.target" ];
+            listenStreams = [ "/run/test.sock" ];
+            socketConfig.SocketMode = lib.mkDefault "0777";
+          };
+        };
+
+        simple-socket-service-modified.configuration = {
+          imports = [ simple-socket.configuration ];
+          systemd.services.socket-activated.serviceConfig.X-Test = "test";
+        };
+
+        simple-socket-stop-if-changed.configuration = {
+          imports = [ simple-socket.configuration ];
+          systemd.services.socket-activated.stopIfChanged = true;
+        };
+
+        simple-socket-stop-if-changed-and-reloadtrigger.configuration = {
+          imports = [ simple-socket.configuration ];
+          systemd.services.socket-activated = {
+            stopIfChanged = true;
+            reloadTriggers = [ "test" ];
+          };
+        };
+
+        mount.configuration = {
+          systemd.mounts = [
+            {
+              description = "Testmount";
+              what = "tmpfs";
+              type = "tmpfs";
+              where = "/testmount";
+              options = "size=1M";
+              wantedBy = [ "local-fs.target" ];
+            }
+          ];
+        };
+
+        mountModified.configuration = {
+          systemd.mounts = [
+            {
+              description = "Testmount";
+              what = "tmpfs";
+              type = "tmpfs";
+              where = "/testmount";
+              options = "size=10M";
+              wantedBy = [ "local-fs.target" ];
+            }
+          ];
+        };
+
+        timer.configuration = {
+          systemd.timers.test-timer = {
+            wantedBy = [ "timers.target" ];
+            timerConfig.OnCalendar = "@1395716396"; # chosen by fair dice roll
+          };
+          systemd.services.test-timer = {
+            serviceConfig = {
+              Type = "oneshot";
+              ExecStart = "${pkgs.coreutils}/bin/true";
+            };
+          };
+        };
+
+        timerModified.configuration = {
+          imports = [ timer.configuration ];
+          systemd.timers.test-timer.timerConfig.OnCalendar = lib.mkForce "Fri 2012-11-23 16:00:00";
+        };
+
+        hybridSleepModified.configuration = {
+          systemd.targets.hybrid-sleep.unitConfig.X-Test = true;
+        };
+
+        target.configuration = {
+          systemd.targets.test-target.wantedBy = [ "multi-user.target" ];
+          # We use this service to figure out whether the target was modified.
+          # This is the only way because targets are filtered and therefore not
+          # printed when they are started/stopped.
+          systemd.services.test-service = {
+            bindsTo = [ "test-target.target" ];
+            serviceConfig.ExecStart = "${pkgs.coreutils}/bin/sleep infinity";
+          };
+        };
+
+        targetModified.configuration = {
+          imports = [ target.configuration ];
+          systemd.targets.test-target.unitConfig.X-Test = true;
+        };
+
+        targetModifiedStopOnReconfig.configuration = {
+          imports = [ target.configuration ];
+          systemd.targets.test-target.unitConfig.X-StopOnReconfiguration = true;
+        };
+
+        path.configuration = {
+          systemd.paths.test-watch = {
+            wantedBy = [ "paths.target" ];
+            pathConfig.PathExists = "/testpath";
+          };
+          systemd.services.test-watch = {
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              ExecStart = "${pkgs.coreutils}/bin/touch /testpath-modified";
+            };
+          };
+        };
+
+        pathModified.configuration = {
+          imports = [ path.configuration ];
+          systemd.paths.test-watch.pathConfig.PathExists = lib.mkForce "/testpath2";
+        };
+
+        slice.configuration = {
+          systemd.slices.testslice.sliceConfig.MemoryMax = "1"; # don't allow memory allocation
+          systemd.services.testservice = {
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              ExecStart = "${pkgs.coreutils}/bin/true";
+              Slice = "testslice.slice";
+            };
+          };
+        };
+
+        sliceModified.configuration = {
+          imports = [ slice.configuration ];
+          systemd.slices.testslice.sliceConfig.MemoryMax = lib.mkForce null;
+        };
+      };
     };
-    other = { ... }: {
+
+    other = {
       users.mutableUsers = true;
     };
   };
 
-  testScript = {nodes, ...}: let
+  testScript = { nodes, ... }: let
     originalSystem = nodes.machine.config.system.build.toplevel;
     otherSystem = nodes.other.config.system.build.toplevel;
+    machine = nodes.machine.config.system.build.toplevel;
 
     # Ensures failures pass through using pipefail, otherwise failing to
     # switch-to-configuration is hidden by the success of `tee`.
@@ -27,12 +475,574 @@ import ./make-test-python.nix ({ pkgs, ...} : {
       set -o pipefail
       exec env -i "$@" | tee /dev/stderr
     '';
-  in ''
+  in /* python */ ''
+    def switch_to_specialisation(system, name, action="test", fail=False):
+        if name == "":
+            stc = f"{system}/bin/switch-to-configuration"
+        else:
+            stc = f"{system}/specialisation/{name}/bin/switch-to-configuration"
+        out = machine.fail(f"{stc} {action} 2>&1") if fail \
+            else machine.succeed(f"{stc} {action} 2>&1")
+        assert_lacks(out, "switch-to-configuration line")  # Perl warnings
+        return out
+
+    def assert_contains(haystack, needle):
+        if needle not in haystack:
+            print("The haystack that will cause the following exception is:")
+            print("---")
+            print(haystack)
+            print("---")
+            raise Exception(f"Expected string '{needle}' was not found")
+
+    def assert_lacks(haystack, needle):
+        if needle in haystack:
+            print("The haystack that will cause the following exception is:")
+            print("---")
+            print(haystack, end="")
+            print("---")
+            raise Exception(f"Unexpected string '{needle}' was found")
+
+
+    machine.wait_for_unit("multi-user.target")
+
     machine.succeed(
         "${stderrRunner} ${originalSystem}/bin/switch-to-configuration test"
     )
+    # This tests whether the /etc/os-release parser works which is a fallback
+    # when /etc/NIXOS is missing. If the parser does not work, switch-to-configuration
+    # would fail.
+    machine.succeed("rm /etc/NIXOS")
     machine.succeed(
         "${stderrRunner} ${otherSystem}/bin/switch-to-configuration test"
     )
+
+
+    with subtest("actions"):
+        # boot action
+        machine.fail("test -f /tmp/bootloader-installed")
+        out = switch_to_specialisation("${machine}", "simpleService", action="boot")
+        assert_contains(out, "installing dummy bootloader")
+        assert_lacks(out, "activating the configuration...")  # good indicator of a system activation
+        machine.succeed("test -f /tmp/bootloader-installed")
+        machine.succeed("rm /tmp/bootloader-installed")
+
+        # switch action
+        machine.fail("test -f /tmp/bootloader-installed")
+        out = switch_to_specialisation("${machine}", "", action="switch")
+        assert_contains(out, "installing dummy bootloader")
+        assert_contains(out, "activating the configuration...")  # good indicator of a system activation
+        machine.succeed("test -f /tmp/bootloader-installed")
+
+        # test and dry-activate actions are tested further down below
+
+    with subtest("services"):
+        switch_to_specialisation("${machine}", "")
+        # Nothing happens when nothing is changed
+        out = switch_to_specialisation("${machine}", "")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Start a simple service
+        out = switch_to_specialisation("${machine}", "simpleService")
+        assert_lacks(out, "installing dummy bootloader")  # test does not install a bootloader
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: dbus.service\n")  # huh
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: test.service\n")
+
+        # Not changing anything doesn't do anything
+        out = switch_to_specialisation("${machine}", "simpleService")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Only changing the description does nothing
+        out = switch_to_specialisation("${machine}", "simpleServiceDifferentDescription")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Restart the simple service
+        out = switch_to_specialisation("${machine}", "simpleServiceModified")
+        assert_contains(out, "stopping the following units: test.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_contains(out, "\nstarting the following units: test.service\n")
+        assert_lacks(out, "the following new units were started:")
+
+        # Restart the service with stopIfChanged=false
+        out = switch_to_specialisation("${machine}", "simpleServiceNostop")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Reload the service with reloadIfChanged=true
+        out = switch_to_specialisation("${machine}", "simpleServiceReload")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: test.service\n")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Nothing happens when restartIfChanged=false
+        out = switch_to_specialisation("${machine}", "simpleServiceNorestart")
+        assert_lacks(out, "stopping the following units:")
+        assert_contains(out, "NOT restarting the following changed units: test.service\n")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Dry mode shows different messages
+        out = switch_to_specialisation("${machine}", "simpleService", action="dry-activate")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        assert_contains(out, "would start the following units: test.service\n")
+
+        # Ensure \ works in unit names
+        out = switch_to_specialisation("${machine}", "unitWithBackslash")
+        assert_contains(out, "stopping the following units: test.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: escaped\\x2ddash.service\n")
+
+        out = switch_to_specialisation("${machine}", "unitWithBackslashModified")
+        assert_contains(out, "stopping the following units: escaped\\x2ddash.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_contains(out, "\nstarting the following units: escaped\\x2ddash.service\n")
+        assert_lacks(out, "the following new units were started:")
+
+        # Ensure units that require changed units are properly reloaded
+        out = switch_to_specialisation("${machine}", "unitWithRequirement")
+        assert_contains(out, "stopping the following units: escaped\\x2ddash.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: required-service.service, test-service.service\n")
+
+        out = switch_to_specialisation("${machine}", "unitWithRequirementModified")
+        assert_contains(out, "stopping the following units: required-service.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_contains(out, "\nstarting the following units: required-service.service, test-service.service\n")
+        assert_lacks(out, "the following new units were started:")
+
+        # Unless the unit asks to be not restarted
+        out = switch_to_specialisation("${machine}", "unitWithRequirementModifiedNostart")
+        assert_contains(out, "stopping the following units: required-service.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_contains(out, "\nstarting the following units: required-service.service\n")
+        assert_lacks(out, "the following new units were started:")
+
+    with subtest("failing units"):
+        # Let the simple service fail
+        switch_to_specialisation("${machine}", "simpleServiceModified")
+        out = switch_to_specialisation("${machine}", "simpleServiceFailing", fail=True)
+        assert_contains(out, "stopping the following units: test.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_contains(out, "\nstarting the following units: test.service\n")
+        assert_lacks(out, "the following new units were started:")
+        assert_contains(out, "warning: the following units failed: test.service\n")
+        assert_contains(out, "Main PID:")  # output of systemctl
+
+        # A unit that gets into autorestart without failing is not treated as failed
+        out = switch_to_specialisation("${machine}", "autorestartService")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: autorestart.service\n")
+        machine.systemctl('stop autorestart.service')  # cancel the 20y timer
+
+        # Switching to the same system should do nothing (especially not treat the unit as failed)
+        out = switch_to_specialisation("${machine}", "autorestartService")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: autorestart.service\n")
+        machine.systemctl('stop autorestart.service')  # cancel the 20y timer
+
+        # If systemd thinks the unit has failed and is in autorestart, we should show it as failed
+        out = switch_to_specialisation("${machine}", "autorestartServiceFailing", fail=True)
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        assert_contains(out, "warning: the following units failed: autorestart.service\n")
+        assert_contains(out, "Main PID:")  # output of systemctl
+
+    with subtest("unit file parser"):
+        # Switch to a well-known state
+        switch_to_specialisation("${machine}", "simpleServiceNostop")
+
+        # Add a section
+        out = switch_to_specialisation("${machine}", "simpleServiceWithExtraSection")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Rename it
+        out = switch_to_specialisation("${machine}", "simpleServiceWithExtraSectionOtherName")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Remove it
+        out = switch_to_specialisation("${machine}", "simpleServiceNostop")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # [Install] section is ignored
+        out = switch_to_specialisation("${machine}", "simpleServiceWithInstallSection")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Add a key
+        out = switch_to_specialisation("${machine}", "simpleServiceWithExtraKey")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Change its value
+        out = switch_to_specialisation("${machine}", "simpleServiceWithExtraKeyOtherValue")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Rename it
+        out = switch_to_specialisation("${machine}", "simpleServiceWithExtraKeyOtherName")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Remove it
+        out = switch_to_specialisation("${machine}", "simpleServiceNostop")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Add a reload trigger
+        out = switch_to_specialisation("${machine}", "simpleServiceReloadTrigger")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: test.service\n")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Modify the reload trigger
+        out = switch_to_specialisation("${machine}", "simpleServiceReloadTriggerModified")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: test.service\n")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Modify the reload trigger and something else
+        out = switch_to_specialisation("${machine}", "simpleServiceReloadTriggerModifiedAndSomethingElse")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Remove the reload trigger
+        out = switch_to_specialisation("${machine}", "simpleServiceReloadTriggerModifiedSomethingElse")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+    with subtest("restart and reload by activation script"):
+        switch_to_specialisation("${machine}", "simpleServiceNorestart")
+        out = switch_to_specialisation("${machine}", "restart-and-reload-by-activation-script")
+        assert_contains(out, "stopping the following units: test.service\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "restarting the following units:")
+        assert_contains(out, "\nstarting the following units: no-restart-service.service, reload-triggers-and-restart-by-as.service, simple-reload-service.service, simple-restart-service.service, simple-service.service\n")
+        assert_contains(out, "the following new units were started: no-restart-service.service, reload-triggers-and-restart-by-as.service, reload-triggers-and-restart.service, reload-triggers.service, simple-reload-service.service, simple-restart-service.service, simple-service.service\n")
+        # Switch to the same system where the example services get restarted
+        # and reloaded by the activation script
+        out = switch_to_specialisation("${machine}", "restart-and-reload-by-activation-script")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: reload-triggers-and-restart.service, reload-triggers.service, simple-reload-service.service\n")
+        assert_contains(out, "restarting the following units: reload-triggers-and-restart-by-as.service, simple-restart-service.service, simple-service.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        # Switch to the same system and see if the service gets restarted when it's modified
+        # while the fact that it's supposed to be reloaded by the activation script is ignored.
+        out = switch_to_specialisation("${machine}", "restart-and-reload-by-activation-script-modified")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: reload-triggers.service, simple-reload-service.service\n")
+        assert_contains(out, "restarting the following units: reload-triggers-and-restart-by-as.service, reload-triggers-and-restart.service, simple-restart-service.service, simple-service.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        # The same, but in dry mode
+        out = switch_to_specialisation("${machine}", "restart-and-reload-by-activation-script", action="dry-activate")
+        assert_lacks(out, "would stop the following units:")
+        assert_lacks(out, "would NOT stop the following changed units:")
+        assert_contains(out, "would reload the following units: reload-triggers.service, simple-reload-service.service\n")
+        assert_contains(out, "would restart the following units: reload-triggers-and-restart-by-as.service, reload-triggers-and-restart.service, simple-restart-service.service, simple-service.service\n")
+        assert_lacks(out, "\nwould start the following units:")
+
+    with subtest("socket-activated services"):
+        # Socket-activated services don't get started, just the socket
+        machine.fail("[ -S /run/test.sock ]")
+        out = switch_to_specialisation("${machine}", "simple-socket")
+        # assert_lacks(out, "stopping the following units:") not relevant
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: socket-activated.socket\n")
+        machine.succeed("[ -S /run/test.sock ]")
+
+        # Changing a non-activated service does nothing
+        out = switch_to_specialisation("${machine}", "simple-socket-service-modified")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        machine.succeed("[ -S /run/test.sock ]")
+        # The unit is properly activated when the socket is accessed
+        if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello":
+            raise Exception("Socket was not properly activated")  # idk how that would happen tbh
+
+        # Changing an activated service with stopIfChanged=false restarts the service
+        out = switch_to_specialisation("${machine}", "simple-socket")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: socket-activated.service\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        machine.succeed("[ -S /run/test.sock ]")
+        # Socket-activation of the unit still works
+        if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello":
+            raise Exception("Socket was not properly activated after the service was restarted")
+
+        # Changing an activated service with stopIfChanged=true stops the service and
+        # socket and starts the socket
+        out = switch_to_specialisation("${machine}", "simple-socket-stop-if-changed")
+        assert_contains(out, "stopping the following units: socket-activated.service, socket-activated.socket\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_contains(out, "\nstarting the following units: socket-activated.socket\n")
+        assert_lacks(out, "the following new units were started:")
+        machine.succeed("[ -S /run/test.sock ]")
+        # Socket-activation of the unit still works
+        if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello":
+            raise Exception("Socket was not properly activated after the service was restarted")
+
+        # Changing a reload trigger of a socket-activated unit only reloads it
+        out = switch_to_specialisation("${machine}", "simple-socket-stop-if-changed-and-reloadtrigger")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: socket-activated.service\n")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units: socket-activated.socket")
+        assert_lacks(out, "the following new units were started:")
+        machine.succeed("[ -S /run/test.sock ]")
+        # Socket-activation of the unit still works
+        if machine.succeed("socat - UNIX-CONNECT:/run/test.sock") != "hello":
+            raise Exception("Socket was not properly activated after the service was restarted")
+
+    with subtest("mounts"):
+        switch_to_specialisation("${machine}", "mount")
+        out = machine.succeed("mount | grep 'on /testmount'")
+        assert_contains(out, "size=1024k")
+        out = switch_to_specialisation("${machine}", "mountModified")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_contains(out, "reloading the following units: testmount.mount\n")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        # It changed
+        out = machine.succeed("mount | grep 'on /testmount'")
+        assert_contains(out, "size=10240k")
+
+    with subtest("timers"):
+        switch_to_specialisation("${machine}", "timer")
+        out = machine.succeed("systemctl show test-timer.timer")
+        assert_contains(out, "OnCalendar=2014-03-25 02:59:56 UTC")
+        out = switch_to_specialisation("${machine}", "timerModified")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_contains(out, "\nrestarting the following units: test-timer.timer\n")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        # It changed
+        out = machine.succeed("systemctl show test-timer.timer")
+        assert_contains(out, "OnCalendar=Fri 2012-11-23 16:00:00")
+
+    with subtest("targets"):
+        # Modifying some special targets like hybrid-sleep.target does nothing
+        out = switch_to_specialisation("${machine}", "hybridSleepModified")
+        assert_contains(out, "stopping the following units: test-timer.timer\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+
+        # Adding a new target starts it
+        out = switch_to_specialisation("${machine}", "target")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: test-target.target\n")
+
+        # Changing a target doesn't print anything because the unit is filtered
+        machine.systemctl("start test-service.service")
+        out = switch_to_specialisation("${machine}", "targetModified")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        machine.succeed("systemctl is-active test-service.service")  # target was not restarted
+
+        # With X-StopOnReconfiguration, the target gets stopped and started
+        out = switch_to_specialisation("${machine}", "targetModifiedStopOnReconfig")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        machine.fail("systemctl is-active test-service.servce")  # target was restarted
+
+        # Remove the target by switching to the old specialisation
+        out = switch_to_specialisation("${machine}", "timerModified")
+        assert_contains(out, "stopping the following units: test-target.target\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: test-timer.timer\n")
+
+    with subtest("paths"):
+        out = switch_to_specialisation("${machine}", "path")
+        assert_contains(out, "stopping the following units: test-timer.timer\n")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_contains(out, "the following new units were started: test-watch.path\n")
+        machine.fail("test -f /testpath-modified")
+
+        # touch the file, unit should be triggered
+        machine.succeed("touch /testpath")
+        machine.wait_until_succeeds("test -f /testpath-modified")
+        machine.succeed("rm /testpath /testpath-modified")
+        machine.systemctl("stop test-watch.service")
+        switch_to_specialisation("${machine}", "pathModified")
+        machine.succeed("touch /testpath")
+        machine.fail("test -f /testpath-modified")
+        machine.succeed("touch /testpath2")
+        machine.wait_until_succeeds("test -f /testpath-modified")
+
+    # This test ensures that changes to slice configuration get applied.
+    # We test this by having a slice that allows no memory allocation at
+    # all and starting a service within it. If the service crashes, the slice
+    # is applied and if we modify the slice to allow memory allocation, the
+    # service should successfully start.
+    with subtest("slices"):
+        machine.succeed("echo 0 > /proc/sys/vm/panic_on_oom")  # allow OOMing
+        out = switch_to_specialisation("${machine}", "slice")
+        # assert_lacks(out, "stopping the following units:") not relevant
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        machine.fail("systemctl start testservice.service")
+
+        out = switch_to_specialisation("${machine}", "sliceModified")
+        assert_lacks(out, "stopping the following units:")
+        assert_lacks(out, "NOT restarting the following changed units:")
+        assert_lacks(out, "reloading the following units:")
+        assert_lacks(out, "\nrestarting the following units:")
+        assert_lacks(out, "\nstarting the following units:")
+        assert_lacks(out, "the following new units were started:")
+        machine.succeed("systemctl start testservice.service")
+        machine.succeed("echo 1 > /proc/sys/vm/panic_on_oom")  # disallow OOMing
   '';
 })
diff --git a/nixos/tests/systemd-binfmt.nix b/nixos/tests/systemd-binfmt.nix
index 2a676f3da98b..a3a6efac3e4d 100644
--- a/nixos/tests/systemd-binfmt.nix
+++ b/nixos/tests/systemd-binfmt.nix
@@ -1,24 +1,90 @@
 # Teach the kernel how to run armv7l and aarch64-linux binaries,
 # and run GNU Hello for these architectures.
-import ./make-test-python.nix ({ pkgs, ... }: {
-  name = "systemd-binfmt";
-  machine = {
-    boot.binfmt.emulatedSystems = [
-      "armv7l-linux"
-      "aarch64-linux"
-    ];
-  };
 
-  testScript = let
-    helloArmv7l = pkgs.pkgsCross.armv7l-hf-multiplatform.hello;
-    helloAarch64 = pkgs.pkgsCross.aarch64-multiplatform.hello;
-  in ''
-    machine.start()
-    assert "world" in machine.succeed(
-        "${helloArmv7l}/bin/hello"
-    )
-    assert "world" in machine.succeed(
-        "${helloAarch64}/bin/hello"
-    )
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+
+let
+  expectArgv0 = xpkgs: xpkgs.runCommandCC "expect-argv0" {
+    src = pkgs.writeText "expect-argv0.c" ''
+      #include <stdio.h>
+      #include <string.h>
+
+      int main(int argc, char **argv) {
+        fprintf(stderr, "Our argv[0] is %s\n", argv[0]);
+
+        if (strcmp(argv[0], argv[1])) {
+          fprintf(stderr, "ERROR: argv[0] is %s, should be %s\n", argv[0], argv[1]);
+          return 1;
+        }
+
+        return 0;
+      }
+    '';
+  } ''
+    $CC -o $out $src
   '';
-})
+in {
+  basic = makeTest {
+    name = "systemd-binfmt";
+    machine = {
+      boot.binfmt.emulatedSystems = [
+        "armv7l-linux"
+        "aarch64-linux"
+      ];
+    };
+
+    testScript = let
+      helloArmv7l = pkgs.pkgsCross.armv7l-hf-multiplatform.hello;
+      helloAarch64 = pkgs.pkgsCross.aarch64-multiplatform.hello;
+    in ''
+      machine.start()
+
+      assert "world" in machine.succeed(
+          "${helloArmv7l}/bin/hello"
+      )
+
+      assert "world" in machine.succeed(
+          "${helloAarch64}/bin/hello"
+      )
+    '';
+  };
+
+  preserveArgvZero = makeTest {
+    name = "systemd-binfmt-preserve-argv0";
+    machine = {
+      boot.binfmt.emulatedSystems = [
+        "aarch64-linux"
+      ];
+    };
+    testScript = let
+      testAarch64 = expectArgv0 pkgs.pkgsCross.aarch64-multiplatform;
+    in ''
+      machine.start()
+      machine.succeed("exec -a meow ${testAarch64} meow")
+    '';
+  };
+
+  ldPreload = makeTest {
+    name = "systemd-binfmt-ld-preload";
+    machine = {
+      boot.binfmt.emulatedSystems = [
+        "aarch64-linux"
+      ];
+    };
+    testScript = let
+      helloAarch64 = pkgs.pkgsCross.aarch64-multiplatform.hello;
+      libredirectAarch64 = pkgs.pkgsCross.aarch64-multiplatform.libredirect;
+    in ''
+      machine.start()
+
+      assert "error" not in machine.succeed(
+          "LD_PRELOAD='${libredirectAarch64}/lib/libredirect.so' ${helloAarch64}/bin/hello 2>&1"
+      ).lower()
+    '';
+  };
+}
diff --git a/nixos/tests/systemd-boot.nix b/nixos/tests/systemd-boot.nix
index c3899b58d6b3..51cfd82e6c4b 100644
--- a/nixos/tests/systemd-boot.nix
+++ b/nixos/tests/systemd-boot.nix
@@ -110,4 +110,145 @@ in
       assert "updating systemd-boot from (000.0-1-notnixos) to " in output
     '';
   };
+
+  memtest86 = makeTest {
+    name = "systemd-boot-memtest86";
+    meta.maintainers = with pkgs.lib.maintainers; [ Enzime ];
+
+    machine = { pkgs, lib, ... }: {
+      imports = [ common ];
+      boot.loader.systemd-boot.memtest86.enable = true;
+      nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [
+        "memtest86-efi"
+      ];
+    };
+
+    testScript = ''
+      machine.succeed("test -e /boot/loader/entries/memtest86.conf")
+      machine.succeed("test -e /boot/efi/memtest86/BOOTX64.efi")
+    '';
+  };
+
+  netbootxyz = makeTest {
+    name = "systemd-boot-netbootxyz";
+    meta.maintainers = with pkgs.lib.maintainers; [ Enzime ];
+
+    machine = { pkgs, lib, ... }: {
+      imports = [ common ];
+      boot.loader.systemd-boot.netbootxyz.enable = true;
+    };
+
+    testScript = ''
+      machine.succeed("test -e /boot/loader/entries/o_netbootxyz.conf")
+      machine.succeed("test -e /boot/efi/netbootxyz/netboot.xyz.efi")
+    '';
+  };
+
+  entryFilename = makeTest {
+    name = "systemd-boot-entry-filename";
+    meta.maintainers = with pkgs.lib.maintainers; [ Enzime ];
+
+    machine = { pkgs, lib, ... }: {
+      imports = [ common ];
+      boot.loader.systemd-boot.memtest86.enable = true;
+      boot.loader.systemd-boot.memtest86.entryFilename = "apple.conf";
+      nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [
+        "memtest86-efi"
+      ];
+    };
+
+    testScript = ''
+      machine.fail("test -e /boot/loader/entries/memtest86.conf")
+      machine.succeed("test -e /boot/loader/entries/apple.conf")
+      machine.succeed("test -e /boot/efi/memtest86/BOOTX64.efi")
+    '';
+  };
+
+  extraEntries = makeTest {
+    name = "systemd-boot-extra-entries";
+    meta.maintainers = with pkgs.lib.maintainers; [ Enzime ];
+
+    machine = { pkgs, lib, ... }: {
+      imports = [ common ];
+      boot.loader.systemd-boot.extraEntries = {
+        "banana.conf" = ''
+          title banana
+        '';
+      };
+    };
+
+    testScript = ''
+      machine.succeed("test -e /boot/loader/entries/banana.conf")
+      machine.succeed("test -e /boot/efi/nixos/.extra-files/loader/entries/banana.conf")
+    '';
+  };
+
+  extraFiles = makeTest {
+    name = "systemd-boot-extra-files";
+    meta.maintainers = with pkgs.lib.maintainers; [ Enzime ];
+
+    machine = { pkgs, lib, ... }: {
+      imports = [ common ];
+      boot.loader.systemd-boot.extraFiles = {
+        "efi/fruits/tomato.efi" = pkgs.netbootxyz-efi;
+      };
+    };
+
+    testScript = ''
+      machine.succeed("test -e /boot/efi/fruits/tomato.efi")
+      machine.succeed("test -e /boot/efi/nixos/.extra-files/efi/fruits/tomato.efi")
+    '';
+  };
+
+  switch-test = makeTest {
+    name = "systemd-boot-switch-test";
+    meta.maintainers = with pkgs.lib.maintainers; [ Enzime ];
+
+    nodes = {
+      inherit common;
+
+      machine = { pkgs, ... }: {
+        imports = [ common ];
+        boot.loader.systemd-boot.extraFiles = {
+          "efi/fruits/tomato.efi" = pkgs.netbootxyz-efi;
+        };
+      };
+
+      with_netbootxyz = { pkgs, ... }: {
+        imports = [ common ];
+        boot.loader.systemd-boot.netbootxyz.enable = true;
+      };
+    };
+
+    testScript = { nodes, ... }: let
+      originalSystem = nodes.machine.config.system.build.toplevel;
+      baseSystem = nodes.common.config.system.build.toplevel;
+      finalSystem = nodes.with_netbootxyz.config.system.build.toplevel;
+    in ''
+      machine.succeed("test -e /boot/efi/fruits/tomato.efi")
+      machine.succeed("test -e /boot/efi/nixos/.extra-files/efi/fruits/tomato.efi")
+
+      with subtest("remove files when no longer needed"):
+          machine.succeed("${baseSystem}/bin/switch-to-configuration boot")
+          machine.fail("test -e /boot/efi/fruits/tomato.efi")
+          machine.fail("test -d /boot/efi/fruits")
+          machine.succeed("test -d /boot/efi/nixos/.extra-files")
+          machine.fail("test -e /boot/efi/nixos/.extra-files/efi/fruits/tomato.efi")
+          machine.fail("test -d /boot/efi/nixos/.extra-files/efi/fruits")
+
+      with subtest("files are added back when needed again"):
+          machine.succeed("${originalSystem}/bin/switch-to-configuration boot")
+          machine.succeed("test -e /boot/efi/fruits/tomato.efi")
+          machine.succeed("test -e /boot/efi/nixos/.extra-files/efi/fruits/tomato.efi")
+
+      with subtest("simultaneously removing and adding files works"):
+          machine.succeed("${finalSystem}/bin/switch-to-configuration boot")
+          machine.fail("test -e /boot/efi/fruits/tomato.efi")
+          machine.fail("test -e /boot/efi/nixos/.extra-files/efi/fruits/tomato.efi")
+          machine.succeed("test -e /boot/loader/entries/o_netbootxyz.conf")
+          machine.succeed("test -e /boot/efi/netbootxyz/netboot.xyz.efi")
+          machine.succeed("test -e /boot/efi/nixos/.extra-files/loader/entries/o_netbootxyz.conf")
+          machine.succeed("test -e /boot/efi/nixos/.extra-files/efi/netbootxyz/netboot.xyz.efi")
+    '';
+  };
 }
diff --git a/nixos/tests/systemd-confinement.nix b/nixos/tests/systemd-confinement.nix
index 8fafb11e1e8c..3181af309a6e 100644
--- a/nixos/tests/systemd-confinement.nix
+++ b/nixos/tests/systemd-confinement.nix
@@ -17,15 +17,19 @@ import ./make-test-python.nix {
       exit "''${ret:-1}"
     '';
 
-    mkTestStep = num: { config ? {}, testScript }: {
-      systemd.sockets."test${toString num}" = {
+    mkTestStep = num: {
+      testScript,
+      config ? {},
+      serviceName ? "test${toString num}",
+    }: {
+      systemd.sockets.${serviceName} = {
         description = "Socket for Test Service ${toString num}";
         wantedBy = [ "sockets.target" ];
         socketConfig.ListenStream = "/run/test${toString num}.sock";
         socketConfig.Accept = true;
       };
 
-      systemd.services."test${toString num}@" = {
+      systemd.services."${serviceName}@" = {
         description = "Confined Test Service ${toString num}";
         confinement = (config.confinement or {}) // { enable = true; };
         serviceConfig = (config.serviceConfig or {}) // {
@@ -135,6 +139,16 @@ import ./make-test-python.nix {
               machine.succeed('test "$(chroot-exec \'cat "$FOOBAR"\')" = eek')
         '';
       }
+      { serviceName = "shipped-unitfile";
+        config.confinement.mode = "chroot-only";
+        testScript = ''
+          with subtest("check if shipped unit file still works"):
+              machine.succeed(
+                  'chroot-exec \'kill -9 $$ 2>&1 || :\' | '
+                  'grep -q "Too many levels of symbolic links"'
+              )
+        '';
+      }
     ];
 
     options.__testSteps = lib.mkOption {
@@ -143,6 +157,15 @@ import ./make-test-python.nix {
     };
 
     config.environment.systemPackages = lib.singleton testClient;
+    config.systemd.packages = lib.singleton (pkgs.writeTextFile {
+      name = "shipped-unitfile";
+      destination = "/etc/systemd/system/shipped-unitfile@.service";
+      text = ''
+        [Service]
+        SystemCallFilter=~kill
+        SystemCallErrorNumber=ELOOP
+      '';
+    });
 
     config.users.groups.chroot-testgroup = {};
     config.users.users.chroot-testuser = {
diff --git a/nixos/tests/systemd-escaping.nix b/nixos/tests/systemd-escaping.nix
new file mode 100644
index 000000000000..7f93eb5e4f70
--- /dev/null
+++ b/nixos/tests/systemd-escaping.nix
@@ -0,0 +1,45 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+let
+  echoAll = pkgs.writeScript "echo-all" ''
+    #! ${pkgs.runtimeShell}
+    for s in "$@"; do
+      printf '%s\n' "$s"
+    done
+  '';
+  # deliberately using a local empty file instead of pkgs.emptyFile to have
+  # a non-store path in the test
+  args = [ "a%Nything" "lang=\${LANG}" ";" "/bin/sh -c date" ./empty-file 4.2 23 ];
+in
+{
+  name = "systemd-escaping";
+
+  machine = { pkgs, lib, utils, ... }: {
+    systemd.services.echo =
+      assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ [] ])).success;
+      assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ {} ])).success;
+      assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ null ])).success;
+      assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ false ])).success;
+      assert !(builtins.tryEval (utils.escapeSystemdExecArgs [ (_:_) ])).success;
+      { description = "Echo to the journal";
+        serviceConfig.Type = "oneshot";
+        serviceConfig.ExecStart = ''
+          ${echoAll} ${utils.escapeSystemdExecArgs args}
+        '';
+      };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("multi-user.target")
+    machine.succeed("systemctl start echo.service")
+    # skip the first 'Starting <service> ...' line
+    logs = machine.succeed("journalctl -u echo.service -o cat").splitlines()[1:]
+    assert "a%Nything" == logs[0]
+    assert "lang=''${LANG}" == logs[1]
+    assert ";" == logs[2]
+    assert "/bin/sh -c date" == logs[3]
+    assert "/nix/store/ij3gw72f4n5z4dz6nnzl1731p9kmjbwr-empty-file" == logs[4]
+    assert "4.2" in logs[5] # toString produces extra fractional digits!
+    assert "23" == logs[6]
+  '';
+})
diff --git a/nixos/tests/systemd-machinectl.nix b/nixos/tests/systemd-machinectl.nix
new file mode 100644
index 000000000000..4fc5864357c0
--- /dev/null
+++ b/nixos/tests/systemd-machinectl.nix
@@ -0,0 +1,85 @@
+import ./make-test-python.nix (
+  let
+
+    container = {
+      # We re-use the NixOS container option ...
+      boot.isContainer = true;
+      # ... and revert unwanted defaults
+      networking.useHostResolvConf = false;
+
+      # use networkd to obtain systemd network setup
+      networking.useNetworkd = true;
+      networking.useDHCP = false;
+
+      # systemd-nspawn expects /sbin/init
+      boot.loader.initScript.enable = true;
+
+      imports = [ ../modules/profiles/minimal.nix ];
+    };
+
+    containerSystem = (import ../lib/eval-config.nix {
+      modules = [ container ];
+    }).config.system.build.toplevel;
+
+    containerName = "container";
+    containerRoot = "/var/lib/machines/${containerName}";
+
+  in
+  {
+    name = "systemd-machinectl";
+
+    machine = { lib, ... }: {
+      # use networkd to obtain systemd network setup
+      networking.useNetworkd = true;
+      networking.useDHCP = false;
+      services.resolved.enable = false;
+
+      # open DHCP server on interface to container
+      networking.firewall.trustedInterfaces = [ "ve-+" ];
+
+      # do not try to access cache.nixos.org
+      nix.settings.substituters = lib.mkForce [ ];
+
+      virtualisation.additionalPaths = [ containerSystem ];
+    };
+
+    testScript = ''
+      start_all()
+      machine.wait_for_unit("default.target");
+
+      # Install container
+      machine.succeed("mkdir -p ${containerRoot}");
+      # Workaround for nixos-install
+      machine.succeed("chmod o+rx /var/lib/machines");
+      machine.succeed("nixos-install --root ${containerRoot} --system ${containerSystem} --no-channel-copy --no-root-passwd");
+
+      # Allow systemd-nspawn to apply user namespace on immutable files
+      machine.succeed("chattr -i ${containerRoot}/var/empty");
+
+      # Test machinectl start
+      machine.succeed("machinectl start ${containerName}");
+      machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
+
+      # Test systemd-nspawn network configuration
+      machine.succeed("ping -n -c 1 ${containerName}");
+
+      # Test systemd-nspawn uses a user namespace
+      machine.succeed("test `stat ${containerRoot}/var/empty -c %u%g` != 00");
+
+      # Test systemd-nspawn reboot
+      machine.succeed("machinectl shell ${containerName} /run/current-system/sw/bin/reboot");
+      machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
+
+      # Test machinectl reboot
+      machine.succeed("machinectl reboot ${containerName}");
+      machine.wait_until_succeeds("systemctl -M ${containerName} is-active default.target");
+
+      # Test machinectl stop
+      machine.succeed("machinectl stop ${containerName}");
+
+      # Show to to delete the container
+      machine.succeed("chattr -i ${containerRoot}/var/empty");
+      machine.succeed("rm -rf ${containerRoot}");
+    '';
+  }
+)
diff --git a/nixos/tests/systemd-networkd-ipv6-prefix-delegation.nix b/nixos/tests/systemd-networkd-ipv6-prefix-delegation.nix
index 68836c730729..37a89fc21e44 100644
--- a/nixos/tests/systemd-networkd-ipv6-prefix-delegation.nix
+++ b/nixos/tests/systemd-networkd-ipv6-prefix-delegation.nix
@@ -36,19 +36,10 @@ import ./make-test-python.nix ({pkgs, ...}: {
       };
 
       # Since we want to program the routes that we delegate to the "customer"
-      # into our routing table we must have a way to gain the required privs.
-      # This security wrapper will do in our test setup.
-      #
-      # DO NOT COPY THIS TO PRODUCTION AS IS. Think about it at least twice.
-      # Everyone on the "isp" machine will be able to add routes to the kernel.
-      security.wrappers.add-dhcpd-lease = {
-        owner = "root";
-        group = "root";
-        source = pkgs.writeShellScript "add-dhcpd-lease" ''
-          exec ${pkgs.iproute2}/bin/ip -6 route replace "$1" via "$2"
-        '';
-        capabilities = "cap_net_admin+ep";
-      };
+      # into our routing table we must give dhcpd the required privs.
+      systemd.services.dhcpd6.serviceConfig.AmbientCapabilities =
+        [ "CAP_NET_ADMIN" ];
+
       services = {
         # Configure the DHCPv6 server
         #
@@ -80,7 +71,7 @@ import ./make-test-python.nix ({pkgs, ...}: {
               set Prefix = pick-first-value(binary-to-ascii(16, 16, ":", suffix(option dhcp6.ia-pd, 16)), "n/a");
               set PrefixLength = pick-first-value(binary-to-ascii(10, 8, ":", substring(suffix(option dhcp6.ia-pd, 17), 0, 1)), "n/a");
               log(concat(IP, " ", Prefix, " ", PrefixLength));
-              execute("/run/wrappers/bin/add-dhcpd-lease", concat(Prefix,"/",PrefixLength), IP);
+              execute("${pkgs.iproute2}/bin/ip", "-6", "route", "replace", concat(Prefix,"/",PrefixLength), "via", IP);
             }
           '';
         };
diff --git a/nixos/tests/systemd-networkd-vrf.nix b/nixos/tests/systemd-networkd-vrf.nix
index 9f09d801f77a..8a1580fc2ada 100644
--- a/nixos/tests/systemd-networkd-vrf.nix
+++ b/nixos/tests/systemd-networkd-vrf.nix
@@ -161,6 +161,7 @@ in {
 
     # NOTE: please keep in mind that the trailing whitespaces in the following strings
     # are intentional as the output is compared against the raw `iproute2`-output.
+    # editorconfig-checker-disable
     client_ipv4_table = """
     192.168.1.2 dev vrf1 proto static metric 100 
     192.168.2.3 dev vrf2 proto static metric 100
@@ -177,6 +178,7 @@ in {
     local 192.168.2.1 dev eth2 proto kernel scope host src 192.168.2.1 
     broadcast 192.168.2.255 dev eth2 proto kernel scope link src 192.168.2.1
     """.strip()
+    # editorconfig-checker-enable
 
     # Check that networkd properly configures the main routing table
     # and the routing tables for the VRF.
diff --git a/nixos/tests/systemd.nix b/nixos/tests/systemd.nix
index 6561f7efe1a5..f86daa5eea97 100644
--- a/nixos/tests/systemd.nix
+++ b/nixos/tests/systemd.nix
@@ -31,6 +31,13 @@ import ./make-test-python.nix ({ pkgs, ... }: {
       umount /tmp/shared
     '';
 
+    systemd.services.oncalendar-test = {
+      description = "calendar test";
+      # Japan does not have DST which makes the test a little bit simpler
+      startAt = "Wed 10:00 Asia/Tokyo";
+      script = "true";
+    };
+
     systemd.services.testservice1 = {
       description = "Test Service 1";
       wantedBy = [ "multi-user.target" ];
@@ -69,6 +76,11 @@ import ./make-test-python.nix ({ pkgs, ... }: {
     # wait for user services
     machine.wait_for_unit("default.target", "alice")
 
+    # Regression test for https://github.com/NixOS/nixpkgs/issues/105049
+    with subtest("systemd reads timezone database in /etc/zoneinfo"):
+        timer = machine.succeed("TZ=UTC systemctl show --property=TimersCalendar oncalendar-test.timer")
+        assert re.search("next_elapse=Wed ....-..-.. 01:00:00 UTC", timer), f"got {timer.strip()}"
+
     # Regression test for https://github.com/NixOS/nixpkgs/issues/35415
     with subtest("configuration files are recognized by systemd"):
         machine.succeed("test -e /system_conf_read")
diff --git a/nixos/tests/teeworlds.nix b/nixos/tests/teeworlds.nix
index 17e9eeb869b0..ac2c996955c8 100644
--- a/nixos/tests/teeworlds.nix
+++ b/nixos/tests/teeworlds.nix
@@ -36,12 +36,12 @@ in {
       client1.wait_for_x()
       client2.wait_for_x()
 
-      client1.execute("teeworlds 'player_name Alice;connect server'&")
+      client1.execute("teeworlds 'player_name Alice;connect server' >&2 &")
       server.wait_until_succeeds(
           'journalctl -u teeworlds -e | grep --extended-regexp -q "team_join player=\'[0-9]:Alice"'
       )
 
-      client2.execute("teeworlds 'player_name Bob;connect server'&")
+      client2.execute("teeworlds 'player_name Bob;connect server' >&2 &")
       server.wait_until_succeeds(
           'journalctl -u teeworlds -e | grep --extended-regexp -q "team_join player=\'[0-9]:Bob"'
       )
diff --git a/nixos/tests/teleport.nix b/nixos/tests/teleport.nix
new file mode 100644
index 000000000000..15b16e44409d
--- /dev/null
+++ b/nixos/tests/teleport.nix
@@ -0,0 +1,99 @@
+{ system ? builtins.currentSystem
+, config ? { }
+, pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+
+let
+  minimal = { config, ... }: {
+    services.teleport.enable = true;
+  };
+
+  client = { config, ... }: {
+    services.teleport = {
+      enable = true;
+      settings = {
+        teleport = {
+          nodename = "client";
+          advertise_ip = "192.168.1.20";
+          auth_token = "8d1957b2-2ded-40e6-8297-d48156a898a9";
+          auth_servers = [ "192.168.1.10:3025" ];
+          log.severity = "DEBUG";
+        };
+        ssh_service = {
+          enabled = true;
+          labels = {
+            role = "client";
+          };
+        };
+        proxy_service.enabled = false;
+        auth_service.enabled = false;
+      };
+    };
+    networking.interfaces.eth1.ipv4.addresses = [{
+      address = "192.168.1.20";
+      prefixLength = 24;
+    }];
+  };
+
+  server = { config, ... }: {
+    services.teleport = {
+      enable = true;
+      settings = {
+        teleport = {
+          nodename = "server";
+          advertise_ip = "192.168.1.10";
+        };
+        ssh_service.enabled = true;
+        proxy_service.enabled = true;
+        auth_service = {
+          enabled = true;
+          tokens = [ "node:8d1957b2-2ded-40e6-8297-d48156a898a9" ];
+        };
+      };
+      diag.enable = true;
+      insecure.enable = true;
+    };
+    networking = {
+      firewall.allowedTCPPorts = [ 3025 ];
+      interfaces.eth1.ipv4.addresses = [{
+        address = "192.168.1.10";
+        prefixLength = 24;
+      }];
+    };
+  };
+in
+{
+  minimal = makeTest {
+    # minimal setup should always work
+    name = "teleport-minimal-setup";
+    meta.maintainers = with pkgs.lib.maintainers; [ ymatsiuk ];
+    nodes = { inherit minimal; };
+
+    testScript = ''
+      minimal.wait_for_open_port("3025")
+      minimal.wait_for_open_port("3080")
+      minimal.wait_for_open_port("3022")
+    '';
+  };
+
+  basic = makeTest {
+    # basic server and client test
+    name = "teleport-server-client";
+    meta.maintainers = with pkgs.lib.maintainers; [ ymatsiuk ];
+    nodes = { inherit server client; };
+
+    testScript = ''
+      with subtest("teleport ready"):
+          server.wait_for_open_port("3025")
+          client.wait_for_open_port("3022")
+
+      with subtest("check applied configuration"):
+          server.wait_until_succeeds("tctl get nodes --format=json | ${pkgs.jq}/bin/jq -e '.[] | select(.spec.hostname==\"client\") | .metadata.labels.role==\"client\"'")
+          server.wait_for_open_port("3000")
+          client.succeed("journalctl -u teleport.service --grep='DEBU'")
+          server.succeed("journalctl -u teleport.service --grep='Starting teleport in insecure mode.'")
+    '';
+  };
+}
diff --git a/nixos/tests/terminal-emulators.nix b/nixos/tests/terminal-emulators.nix
new file mode 100644
index 000000000000..60161b80b965
--- /dev/null
+++ b/nixos/tests/terminal-emulators.nix
@@ -0,0 +1,207 @@
+# Terminal emulators all present a pretty similar interface.
+# That gives us an opportunity to easily test their basic functionality with a single codebase.
+#
+# There are two tests run on each terminal emulator
+# - can it successfully execute a command passed on the cmdline?
+# - can it successfully display a colour?
+# the latter is used as a proxy for "can it display text?", without going through all the intricacies of OCR.
+#
+# 256-colour terminal mode is used to display the test colour, since it has a universally-applicable palette (unlike 8- and 16- colour, where the colours are implementation-defined), and it is widely supported (unlike 24-bit colour).
+#
+# Future work:
+# - Wayland support (both for testing the existing terminals, and for testing wayland-only terminals like foot and havoc)
+# - Test keyboard input? (skipped for now, to eliminate the possibility of race conditions and focus issues)
+
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
+
+let tests = {
+      alacritty.pkg = p: p.alacritty;
+
+      contour.pkg = p: p.contour;
+      contour.cmd = "contour $command";
+
+      cool-retro-term.pkg = p: p.cool-retro-term;
+      cool-retro-term.colourTest = false; # broken by gloss effect
+
+      ctx.pkg = p: p.ctx;
+      ctx.pinkValue = "#FE0065";
+
+      darktile.pkg = p: p.darktile;
+
+      eterm.pkg = p: p.eterm;
+      eterm.executable = "Eterm";
+      eterm.pinkValue = "#D40055";
+
+      germinal.pkg = p: p.germinal;
+
+      gnome-terminal.pkg = p: p.gnome.gnome-terminal;
+
+      guake.pkg = p: p.guake;
+      guake.cmd = "SHELL=$command guake --show";
+      guake.kill = true;
+
+      hyper.pkg = p: p.hyper;
+
+      kermit.pkg = p: p.kermit-terminal;
+
+      kgx.pkg = p: p.kgx;
+      kgx.cmd = "kgx -e $command";
+      kgx.kill = true;
+
+      kitty.pkg = p: p.kitty;
+      kitty.cmd = "kitty $command";
+
+      konsole.pkg = p: p.plasma5Packages.konsole;
+
+      lxterminal.pkg = p: p.lxterminal;
+
+      mate-terminal.pkg = p: p.mate.mate-terminal;
+      mate-terminal.cmd = "SHELL=$command mate-terminal --disable-factory"; # factory mode uses dbus, and we don't have a proper dbus session set up
+
+      mlterm.pkg = p: p.mlterm;
+
+      mrxvt.pkg = p: p.mrxvt;
+
+      qterminal.pkg = p: p.lxqt.qterminal;
+      qterminal.kill = true;
+
+      roxterm.pkg = p: p.roxterm;
+      roxterm.cmd = "roxterm -e $command";
+
+      sakura.pkg = p: p.sakura;
+
+      st.pkg = p: p.st;
+      st.kill = true;
+
+      stupidterm.pkg = p: p.stupidterm;
+      stupidterm.cmd = "stupidterm -- $command";
+
+      terminator.pkg = p: p.terminator;
+      terminator.cmd = "terminator -e $command";
+
+      terminology.pkg = p: p.enlightenment.terminology;
+      terminology.cmd = "SHELL=$command terminology --no-wizard=true";
+      terminology.colourTest = false; # broken by gloss effect
+
+      termite.pkg = p: p.termite;
+
+      termonad.pkg = p: p.termonad;
+
+      tilda.pkg = p: p.tilda;
+
+      tilix.pkg = p: p.tilix;
+      tilix.cmd = "tilix -e $command";
+
+      urxvt.pkg = p: p.rxvt-unicode;
+
+      wayst.pkg = p: p.wayst;
+      wayst.pinkValue = "#FF0066";
+
+      wezterm.pkg = p: p.wezterm;
+
+      xfce4-terminal.pkg = p: p.xfce.xfce4-terminal;
+
+      xterm.pkg = p: p.xterm;
+    };
+in mapAttrs (name: { pkg, executable ? name, cmd ? "SHELL=$command ${executable}", colourTest ? true, pinkValue ? "#FF0087", kill ? false }: makeTest
+{
+  name = "terminal-emulator-${name}";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ jjjollyjim ];
+  };
+
+  machine = { pkgsInner, ... }:
+
+  {
+    imports = [ ./common/x11.nix ./common/user-account.nix ];
+
+    # Hyper (and any other electron-based terminals) won't run as root
+    test-support.displayManager.auto.user = "alice";
+
+    environment.systemPackages = [
+      (pkg pkgs)
+      (pkgs.writeShellScriptBin "report-success" ''
+        echo 1 > /tmp/term-ran-successfully
+        ${optionalString kill "pkill ${executable}"}
+      '')
+      (pkgs.writeShellScriptBin "display-colour" ''
+        # A 256-colour background colour code for pink, then spaces.
+        #
+        # Background is used rather than foreground to minimize the effect of anti-aliasing.
+        #
+        # Keep adding more in case the window is partially offscreen to the left or requires
+        # a change to correctly redraw after initialising the window (as with ctx).
+
+        while :
+        do
+            echo -ne "\e[48;5;198m                   "
+            sleep 0.5
+        done
+        sleep infinity
+      '')
+      (pkgs.writeShellScriptBin "run-in-this-term" "sudo -u alice run-in-this-term-wrapped $1")
+
+      (pkgs.writeShellScriptBin "run-in-this-term-wrapped" "command=\"$(which \"$1\")\"; ${cmd}")
+    ];
+
+    # Helpful reminder to add this test to passthru.tests
+    warnings = if !((pkg pkgs) ? "passthru" && (pkg pkgs).passthru ? "tests") then [ "The package for ${name} doesn't have a passthru.tests" ] else [ ];
+  };
+
+  # We need imagemagick, though not tesseract
+  enableOCR = true;
+
+  testScript = { nodes, ... }: let
+  in ''
+    with subtest("wait for x"):
+        start_all()
+        machine.wait_for_x()
+
+    with subtest("have the terminal run a command"):
+        # We run this command synchronously, so we can be certain the exit codes are happy
+        machine.${if kill then "execute" else "succeed"}("run-in-this-term report-success")
+        machine.wait_for_file("/tmp/term-ran-successfully")
+    ${optionalString colourTest ''
+
+    import tempfile
+    import subprocess
+
+
+    def check_for_pink(final=False) -> bool:
+        with tempfile.NamedTemporaryFile() as tmpin:
+            machine.send_monitor_command("screendump {}".format(tmpin.name))
+
+            cmd = 'convert {} -define histogram:unique-colors=true -format "%c" histogram:info:'.format(
+                tmpin.name
+            )
+            ret = subprocess.run(cmd, shell=True, capture_output=True)
+            if ret.returncode != 0:
+                raise Exception(
+                    "image analysis failed with exit code {}".format(ret.returncode)
+                )
+
+            text = ret.stdout.decode("utf-8")
+            return "${pinkValue}" in text
+
+
+    with subtest("ensuring no pink is present without the terminal"):
+        assert (
+            check_for_pink() == False
+        ), "Pink was present on the screen before we even launched a terminal!"
+
+    with subtest("have the terminal display a colour"):
+        # We run this command in the background
+        machine.shell.send(b"(run-in-this-term display-colour |& systemd-cat -t terminal) &\n")
+
+        with machine.nested("Waiting for the screen to have pink on it:"):
+            retry(check_for_pink)
+  ''}'';
+}
+
+  ) tests
diff --git a/nixos/tests/thelounge.nix b/nixos/tests/thelounge.nix
new file mode 100644
index 000000000000..e9b85685bf2d
--- /dev/null
+++ b/nixos/tests/thelounge.nix
@@ -0,0 +1,29 @@
+import ./make-test-python.nix {
+  nodes = {
+    private = { config, pkgs, ... }: {
+      services.thelounge = {
+        enable = true;
+        plugins = [ pkgs.theLoungePlugins.themes.solarized ];
+      };
+    };
+
+    public = { config, pkgs, ... }: {
+      services.thelounge = {
+        enable = true;
+        public = true;
+      };
+    };
+  };
+
+  testScript = ''
+    start_all()
+
+    for machine in machines:
+      machine.wait_for_unit("thelounge.service")
+      machine.wait_for_open_port(9000)
+
+    private.wait_until_succeeds("journalctl -u thelounge.service | grep thelounge-theme-solarized")
+    private.wait_until_succeeds("journalctl -u thelounge.service | grep 'in private mode'")
+    public.wait_until_succeeds("journalctl -u thelounge.service | grep 'in public mode'")
+  '';
+}
diff --git a/nixos/tests/tinywl.nix b/nixos/tests/tinywl.nix
new file mode 100644
index 000000000000..8fb87b533306
--- /dev/null
+++ b/nixos/tests/tinywl.nix
@@ -0,0 +1,57 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }:
+
+  {
+    name = "tinywl";
+    meta = {
+      maintainers = with lib.maintainers; [ primeos ];
+    };
+
+    machine = { config, ... }: {
+      # Automatically login on tty1 as a normal user:
+      imports = [ ./common/user-account.nix ];
+      services.getty.autologinUser = "alice";
+      security.polkit.enable = true;
+
+      environment = {
+        systemPackages = with pkgs; [ tinywl foot wayland-utils ];
+      };
+
+      # Automatically start TinyWL when logging in on tty1:
+      programs.bash.loginShellInit = ''
+        if [ "$(tty)" = "/dev/tty1" ]; then
+          set -e
+          test ! -e /tmp/tinywl.log # Only start tinywl once
+          readonly TEST_CMD="wayland-info |& tee /tmp/test-wayland.out && touch /tmp/test-wayland-exit-ok; read"
+          readonly FOOT_CMD="foot sh -c '$TEST_CMD'"
+          tinywl -s "$FOOT_CMD" |& tee /tmp/tinywl.log
+          touch /tmp/tinywl-exit-ok
+        fi
+      '';
+
+      # Switch to a different GPU driver (default: -vga std), otherwise TinyWL segfaults:
+      virtualisation.qemu.options = [ "-vga none -device virtio-gpu-pci" ];
+    };
+
+    testScript = { nodes, ... }: ''
+      start_all()
+      machine.wait_for_unit("multi-user.target")
+
+      # Wait for complete startup:
+      machine.wait_until_succeeds("pgrep tinywl")
+      machine.wait_for_file("/run/user/1000/wayland-0")
+      machine.wait_until_succeeds("pgrep foot")
+      machine.wait_for_file("/tmp/test-wayland-exit-ok")
+
+      # Make a screenshot and save the result:
+      machine.screenshot("tinywl_foot")
+      print(machine.succeed("cat /tmp/test-wayland.out"))
+      machine.copy_from_vm("/tmp/test-wayland.out")
+
+      # Terminate cleanly:
+      machine.send_key("alt-esc")
+      machine.wait_until_fails("pgrep foot")
+      machine.wait_until_fails("pgrep tinywl")
+      machine.wait_for_file("/tmp/tinywl-exit-ok")
+      machine.copy_from_vm("/tmp/tinywl.log")
+    '';
+  })
diff --git a/nixos/tests/tomcat.nix b/nixos/tests/tomcat.nix
new file mode 100644
index 000000000000..e383f224e3d1
--- /dev/null
+++ b/nixos/tests/tomcat.nix
@@ -0,0 +1,21 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+{
+  name = "tomcat";
+
+  machine = { pkgs, ... }: {
+    services.tomcat.enable = true;
+  };
+
+  testScript = ''
+    machine.wait_for_unit("tomcat.service")
+    machine.wait_for_open_port(8080)
+    machine.wait_for_file("/var/tomcat/webapps/examples");
+    machine.succeed(
+        "curl --fail http://localhost:8080/examples/servlets/servlet/HelloWorldExample | grep 'Hello World!'"
+    )
+    machine.succeed(
+        "curl --fail http://localhost:8080/examples/jsp/jsp2/simpletag/hello.jsp | grep 'Hello, world!'"
+    )
+  '';
+})
diff --git a/nixos/tests/trac.nix b/nixos/tests/trac.nix
deleted file mode 100644
index d6914c100817..000000000000
--- a/nixos/tests/trac.nix
+++ /dev/null
@@ -1,19 +0,0 @@
-import ./make-test-python.nix ({ pkgs, ... }: {
-  name = "trac";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ mmahut ];
-  };
-
-  nodes = {
-    machine = { ... }: {
-      services.trac.enable = true;
-    };
-  };
-
-  testScript = ''
-    start_all()
-    machine.wait_for_unit("trac.service")
-    machine.wait_for_open_port(8000)
-    machine.wait_until_succeeds("curl -fL http://localhost:8000/ | grep 'Trac Powered'")
-  '';
-})
diff --git a/nixos/tests/tsm-client-gui.nix b/nixos/tests/tsm-client-gui.nix
new file mode 100644
index 000000000000..e4bcd344a895
--- /dev/null
+++ b/nixos/tests/tsm-client-gui.nix
@@ -0,0 +1,57 @@
+# The tsm-client GUI first tries to connect to a server.
+# We can't simulate a server, so we just check if
+# it reports the correct connection failure error.
+# After that the test persuades the GUI
+# to show its main application window
+# and verifies some configuration information.
+
+import ./make-test-python.nix ({ lib, pkgs, ... }: {
+  name = "tsm-client";
+
+  enableOCR = true;
+
+  machine = { pkgs, ... }: {
+    imports = [ ./common/x11.nix ];
+    programs.tsmClient = {
+      enable = true;
+      package = pkgs.tsm-client-withGui;
+      defaultServername = "testserver";
+      servers.testserver = {
+        # 192.0.0.8 is a "dummy address" according to RFC 7600
+        server = "192.0.0.8";
+        node = "SOME-NODE";
+        passwdDir = "/tmp";
+      };
+    };
+  };
+
+  testScript = ''
+    machine.succeed("which dsmj")  # fail early if this is missing
+    machine.wait_for_x()
+    machine.execute("DSM_LOG=/tmp dsmj -optfile=/dev/null >&2 &")
+
+    # does it report the "TCP/IP connection failure" error code?
+    machine.wait_for_window("IBM Spectrum Protect")
+    machine.wait_for_text("ANS2610S")
+    machine.send_key("esc")
+
+    # it asks to continue to restore a local backupset now;
+    # "yes" (return) leads to the main application window
+    machine.wait_for_text("backupset")
+    machine.send_key("ret")
+
+    # main window: navigate to "Connection Information"
+    machine.wait_for_text("Welcome")
+    machine.send_key("alt-f")  # "File" menu
+    machine.send_key("c")  # "Connection Information"
+
+    # "Connection Information" dialog box
+    machine.wait_for_window("Connection Information")
+    machine.wait_for_text("SOME-NODE")
+    machine.wait_for_text("${pkgs.tsm-client.passthru.unwrapped.version}")
+
+    machine.shutdown()
+  '';
+
+  meta.maintainers = [ lib.maintainers.yarny ];
+})
diff --git a/nixos/tests/txredisapi.nix b/nixos/tests/txredisapi.nix
index bc3814a71375..7c6b36a5c47d 100644
--- a/nixos/tests/txredisapi.nix
+++ b/nixos/tests/txredisapi.nix
@@ -10,17 +10,19 @@ import ./make-test-python.nix ({ pkgs, ... }:
       { pkgs, ... }:
 
       {
-        services.redis.enable = true;
-        services.redis.unixSocket = "/run/redis/redis.sock";
+        services.redis.servers."".enable = true;
 
         environment.systemPackages = with pkgs; [ (python38.withPackages (ps: [ ps.twisted ps.txredisapi ps.mock ]))];
       };
   };
 
-  testScript = ''
+  testScript = { nodes, ... }: let
+    inherit (nodes.machine.config.services) redis;
+    in ''
     start_all()
     machine.wait_for_unit("redis")
-    machine.wait_for_open_port("6379")
+    machine.wait_for_file("${redis.servers."".unixSocket}")
+    machine.succeed("ln -s ${redis.servers."".unixSocket} /tmp/redis.sock")
 
     tests = machine.succeed("PYTHONPATH=\"${pkgs.python3Packages.txredisapi.src}\" python -m twisted.trial ${pkgs.python3Packages.txredisapi.src}/tests")
   '';
diff --git a/nixos/tests/unifi.nix b/nixos/tests/unifi.nix
new file mode 100644
index 000000000000..9dc7e5d04bd5
--- /dev/null
+++ b/nixos/tests/unifi.nix
@@ -0,0 +1,36 @@
+# Test UniFi controller
+
+{ system ? builtins.currentSystem
+, config ? { allowUnfree = true; }
+, pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
+
+let
+  makeAppTest = unifi: makeTest {
+    name = "unifi-controller-${unifi.version}";
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ patryk27 zhaofengli ];
+    };
+
+    nodes.server = {
+      services.unifi = {
+        enable = true;
+        unifiPackage = unifi;
+        openFirewall = false;
+      };
+    };
+
+    testScript = ''
+      server.wait_for_unit("unifi.service")
+      server.wait_until_succeeds("curl -Lk https://localhost:8443 >&2", timeout=300)
+    '';
+  };
+in with pkgs; {
+  unifiLTS = makeAppTest unifiLTS;
+  unifi5 = makeAppTest unifi5;
+  unifi6 = makeAppTest unifi6;
+  unifi7 = makeAppTest unifi7;
+}
diff --git a/nixos/tests/vscodium.nix b/nixos/tests/vscodium.nix
index 43a0d61c856f..688ddfe07e3e 100644
--- a/nixos/tests/vscodium.nix
+++ b/nixos/tests/vscodium.nix
@@ -3,11 +3,10 @@ let
     wayland = { pkgs, ... }: {
       imports = [ ./common/wayland-cage.nix ];
 
-      services.cage.program = ''
-        ${pkgs.vscodium}/bin/codium \
-          --enable-features=UseOzonePlatform \
-          --ozone-platform=wayland
-      '';
+      services.cage.program = "${pkgs.vscodium}/bin/codium";
+
+      environment.variables.NIXOS_OZONE_WL = "1";
+      environment.variables.DISPLAY = "do not use";
 
       fonts.fonts = with pkgs; [ dejavu_fonts ];
     };
@@ -34,36 +33,46 @@ let
       };
       enableOCR = true;
       testScript = ''
+        @polling_condition
+        def codium_running():
+            machine.succeed('pgrep -x codium')
+
+
         start_all()
 
         machine.wait_for_unit('graphical.target')
         machine.wait_until_succeeds('pgrep -x codium')
 
-        # Wait until vscodium is visible. "File" is in the menu bar.
-        machine.wait_for_text('File')
-        machine.screenshot('start_screen')
+        with codium_running:
+            # Wait until vscodium is visible. "File" is in the menu bar.
+            machine.wait_for_text('Get Started')
+            machine.screenshot('start_screen')
+
+            test_string = 'testfile'
 
-        test_string = 'testfile'
+            # Create a new file
+            machine.send_key('ctrl-n')
+            machine.wait_for_text('Untitled')
+            machine.screenshot('empty_editor')
 
-        # Create a new file
-        machine.send_key('ctrl-n')
-        machine.wait_for_text('Untitled')
-        machine.screenshot('empty_editor')
+            # Type a string
+            machine.send_chars(test_string)
+            machine.wait_for_text(test_string)
+            machine.screenshot('editor')
 
-        # Type a string
-        machine.send_chars(test_string)
-        machine.wait_for_text(test_string)
-        machine.screenshot('editor')
+            # Save the file
+            machine.send_key('ctrl-s')
+            machine.wait_for_text('Save')
+            machine.screenshot('save_window')
+            machine.send_key('ret')
 
-        # Save the file
-        machine.send_key('ctrl-s')
-        machine.wait_for_text('Save')
-        machine.screenshot('save_window')
-        machine.send_key('ret')
+            # (the default filename is the first line of the file)
+            machine.wait_for_file(f'/home/alice/{test_string}')
 
-        # (the default filename is the first line of the file)
-        machine.wait_for_file(f'/home/alice/{test_string}')
+        machine.send_key('ctrl-q')
+        machine.wait_until_fails('pgrep -x codium')
       '';
     });
 
-in builtins.mapAttrs (k: v: mkTest k v { }) tests
+in
+builtins.mapAttrs (k: v: mkTest k v { }) tests
diff --git a/nixos/tests/vsftpd.nix b/nixos/tests/vsftpd.nix
new file mode 100644
index 000000000000..4bea27f0eb10
--- /dev/null
+++ b/nixos/tests/vsftpd.nix
@@ -0,0 +1,42 @@
+import ./make-test-python.nix ({ pkgs, ... }: {
+  name = "vsftpd";
+
+  nodes = {
+    server = {
+      services.vsftpd = {
+        enable = true;
+        userlistDeny = false;
+        localUsers = true;
+        userlist = [ "ftp-test-user" ];
+        writeEnable = true;
+        localRoot = "/tmp";
+      };
+      networking.firewall.enable = false;
+
+      users = {
+        users.ftp-test-user = {
+          isSystemUser = true;
+          password = "ftp-test-password";
+          group = "ftp-test-group";
+        };
+        groups.ftp-test-group = {};
+      };
+    };
+
+    client = {};
+  };
+
+  testScript = ''
+    client.start()
+    server.wait_for_unit("vsftpd")
+    server.wait_for_open_port("21")
+
+    client.succeed("curl -u ftp-test-user:ftp-test-password ftp://server")
+    client.succeed('echo "this is a test" > /tmp/test.file.up')
+    client.succeed("curl -v -T /tmp/test.file.up -u ftp-test-user:ftp-test-password ftp://server")
+    client.succeed("curl -u ftp-test-user:ftp-test-password ftp://server/test.file.up > /tmp/test.file.down")
+    client.succeed("diff /tmp/test.file.up /tmp/test.file.down")
+    assert client.succeed("cat /tmp/test.file.up") == server.succeed("cat /tmp/test.file.up")
+    assert client.succeed("cat /tmp/test.file.down") == server.succeed("cat /tmp/test.file.up")
+  '';
+})
diff --git a/nixos/tests/web-apps/mastodon.nix b/nixos/tests/web-apps/mastodon.nix
new file mode 100644
index 000000000000..279a1c59169f
--- /dev/null
+++ b/nixos/tests/web-apps/mastodon.nix
@@ -0,0 +1,170 @@
+import ../make-test-python.nix ({pkgs, ...}:
+let
+  test-certificates = pkgs.runCommandLocal "test-certificates" { } ''
+    mkdir -p $out
+    echo insecure-root-password > $out/root-password-file
+    echo insecure-intermediate-password > $out/intermediate-password-file
+    ${pkgs.step-cli}/bin/step certificate create "Example Root CA" $out/root_ca.crt $out/root_ca.key --password-file=$out/root-password-file --profile root-ca
+    ${pkgs.step-cli}/bin/step certificate create "Example Intermediate CA 1" $out/intermediate_ca.crt $out/intermediate_ca.key --password-file=$out/intermediate-password-file --ca-password-file=$out/root-password-file --profile intermediate-ca --ca $out/root_ca.crt --ca-key $out/root_ca.key
+  '';
+
+  hosts = ''
+    192.168.2.10 ca.local
+    192.168.2.11 mastodon.local
+  '';
+
+in
+{
+  name = "mastodon";
+  meta.maintainers = with pkgs.lib.maintainers; [ erictapen izorkin ];
+
+  nodes = {
+    ca = { pkgs, ... }: {
+      networking = {
+        interfaces.eth1 = {
+          ipv4.addresses = [
+            { address = "192.168.2.10"; prefixLength = 24; }
+          ];
+        };
+        extraHosts = hosts;
+      };
+      services.step-ca = {
+        enable = true;
+        address = "0.0.0.0";
+        port = 8443;
+        openFirewall = true;
+        intermediatePasswordFile = "${test-certificates}/intermediate-password-file";
+        settings = {
+          dnsNames = [ "ca.local" ];
+          root = "${test-certificates}/root_ca.crt";
+          crt = "${test-certificates}/intermediate_ca.crt";
+          key = "${test-certificates}/intermediate_ca.key";
+          db = {
+            type = "badger";
+            dataSource = "/var/lib/step-ca/db";
+          };
+          authority = {
+            provisioners = [
+              {
+                type = "ACME";
+                name = "acme";
+              }
+            ];
+          };
+        };
+      };
+    };
+
+    server = { pkgs, ... }: {
+      networking = {
+        interfaces.eth1 = {
+          ipv4.addresses = [
+            { address = "192.168.2.11"; prefixLength = 24; }
+          ];
+        };
+        extraHosts = hosts;
+        firewall.allowedTCPPorts = [ 80 443 ];
+      };
+
+      security = {
+        acme = {
+          acceptTerms = true;
+          defaults.server = "https://ca.local:8443/acme/acme/directory";
+          defaults.email = "mastodon@mastodon.local";
+        };
+        pki.certificateFiles = [ "${test-certificates}/root_ca.crt" ];
+      };
+
+      services.redis.servers.mastodon = {
+        enable = true;
+        bind = "127.0.0.1";
+        port = 31637;
+      };
+
+      services.mastodon = {
+        enable = true;
+        configureNginx = true;
+        localDomain = "mastodon.local";
+        enableUnixSocket = false;
+        redis = {
+          createLocally = true;
+          host = "127.0.0.1";
+          port = 31637;
+        };
+        database = {
+          createLocally = true;
+          host = "/run/postgresql";
+          port = 5432;
+        };
+        smtp = {
+          createLocally = false;
+          fromAddress = "mastodon@mastodon.local";
+        };
+        extraConfig = {
+          EMAIL_DOMAIN_ALLOWLIST = "example.com";
+        };
+      };
+    };
+
+    client = { pkgs, ... }: {
+      environment.systemPackages = [ pkgs.jq ];
+      networking = {
+        interfaces.eth1 = {
+          ipv4.addresses = [
+            { address = "192.168.2.12"; prefixLength = 24; }
+          ];
+        };
+        extraHosts = hosts;
+      };
+
+      security = {
+        pki.certificateFiles = [ "${test-certificates}/root_ca.crt" ];
+      };
+    };
+  };
+
+  testScript = ''
+    start_all()
+
+    ca.wait_for_unit("step-ca.service")
+    ca.wait_for_open_port(8443)
+
+    server.wait_for_unit("nginx.service")
+    server.wait_for_unit("redis-mastodon.service")
+    server.wait_for_unit("postgresql.service")
+    server.wait_for_unit("mastodon-sidekiq.service")
+    server.wait_for_unit("mastodon-streaming.service")
+    server.wait_for_unit("mastodon-web.service")
+    server.wait_for_open_port(55000)
+    server.wait_for_open_port(55001)
+
+    # Check Mastodon version from remote client
+    client.succeed("curl --fail https://mastodon.local/api/v1/instance | jq -r '.version' | grep '${pkgs.mastodon.version}'")
+
+    # Check using admin CLI
+    # Check Mastodon version
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl version' | grep '${pkgs.mastodon.version}'")
+
+    # Manage accounts
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl email_domain_blocks add example.com'")
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl email_domain_blocks list' | grep 'example.com'")
+    server.fail("su - mastodon -s /bin/sh -c 'mastodon-env tootctl email_domain_blocks list' | grep 'mastodon.local'")
+    server.fail("su - mastodon -s /bin/sh -c 'mastodon-env tootctl accounts create alice --email=alice@example.com'")
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl email_domain_blocks remove example.com'")
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl accounts create bob --email=bob@example.com'")
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl accounts approve bob'")
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl accounts delete bob'")
+
+    # Manage IP access
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl ip_blocks add 192.168.0.0/16 --severity=no_access'")
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl ip_blocks export' | grep '192.168.0.0/16'")
+    server.fail("su - mastodon -s /bin/sh -c 'mastodon-env tootctl p_blocks export' | grep '172.16.0.0/16'")
+    client.fail("curl --fail https://mastodon.local/about")
+    server.succeed("su - mastodon -s /bin/sh -c 'mastodon-env tootctl ip_blocks remove 192.168.0.0/16'")
+    client.succeed("curl --fail https://mastodon.local/about")
+
+    ca.shutdown()
+    server.shutdown()
+    client.shutdown()
+  '';
+})
diff --git a/nixos/tests/web-apps/peertube.nix b/nixos/tests/web-apps/peertube.nix
index 38b31f6c3325..706c598338e8 100644
--- a/nixos/tests/web-apps/peertube.nix
+++ b/nixos/tests/web-apps/peertube.nix
@@ -120,6 +120,9 @@ import ../make-test-python.nix ({pkgs, ...}:
     # Check if PeerTube is running
     client.succeed("curl --fail http://peertube.local:9000/api/v1/config/about | jq -r '.instance.name' | grep 'PeerTube\ Test\ Server'")
 
+    # Check PeerTube CLI version
+    assert "${pkgs.peertube.version}" in server.succeed('su - peertube -s /bin/sh -c "peertube --version"')
+
     client.shutdown()
     server.shutdown()
     database.shutdown()
diff --git a/nixos/tests/web-servers/agate.nix b/nixos/tests/web-servers/agate.nix
new file mode 100644
index 000000000000..e364e134cfda
--- /dev/null
+++ b/nixos/tests/web-servers/agate.nix
@@ -0,0 +1,29 @@
+import ../make-test-python.nix (
+  { pkgs, lib, ... }:
+  {
+    name = "agate";
+    meta = with lib.maintainers; { maintainers = [ jk ]; };
+
+    nodes = {
+      geminiserver = { pkgs, ... }: {
+        services.agate = {
+          enable = true;
+          hostnames = [ "localhost" ];
+          contentDir = pkgs.writeTextDir "index.gmi" ''
+            # Hello NixOS!
+          '';
+        };
+      };
+    };
+
+    testScript = { nodes, ... }: ''
+      geminiserver.wait_for_unit("agate")
+      geminiserver.wait_for_open_port(1965)
+
+      with subtest("check is serving over gemini"):
+        response = geminiserver.succeed("${pkgs.gmni}/bin/gmni -j once -i -N gemini://localhost:1965")
+        print(response)
+        assert "Hello NixOS!" in response
+    '';
+  }
+)
diff --git a/nixos/tests/wine.nix b/nixos/tests/wine.nix
index c46c7d338b2e..8135cb90a591 100644
--- a/nixos/tests/wine.nix
+++ b/nixos/tests/wine.nix
@@ -3,7 +3,7 @@
 }:
 
 let
-  inherit (pkgs.lib) concatMapStrings listToAttrs;
+  inherit (pkgs.lib) concatMapStrings listToAttrs optionals optionalString;
   inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
 
   hello32 = "${pkgs.pkgsCross.mingw32.hello}/bin/hello.exe";
@@ -17,7 +17,7 @@ let
 
       machine = { pkgs, ... }: {
         environment.systemPackages = [ pkgs."${packageSet}"."${variant}" ];
-        virtualisation.diskSize = "800";
+        virtualisation.diskSize = 800;
       };
 
       testScript = ''
@@ -27,6 +27,9 @@ let
               "bash -c 'wine ${exe} 2> >(tee wine-stderr >&2)'"
           )
           assert 'Hello, world!' in greeting
+        ''
+        # only the full version contains Gecko, but the error is not printed reliably in other variants
+        + optionalString (variant == "full") ''
           machine.fail(
               "fgrep 'Could not find Wine Gecko. HTML rendering will be disabled.' wine-stderr"
           )
@@ -35,7 +38,11 @@ let
     };
   };
 
-  variants = [ "base" "full" "minimal" "staging" "unstable" ];
+  variants = [ "base" "full" "minimal" "staging" "unstable" "wayland" ];
 
-in listToAttrs (map (makeWineTest "winePackages" [ hello32 ]) variants
-  ++ map (makeWineTest "wineWowPackages" [ hello32 hello64 ]) variants)
+in
+listToAttrs (
+  map (makeWineTest "winePackages" [ hello32 ]) variants
+  ++ optionals pkgs.stdenv.is64bit
+    (map (makeWineTest "wineWowPackages" [ hello32 hello64 ]) variants)
+)
diff --git a/nixos/tests/wordpress.nix b/nixos/tests/wordpress.nix
index f7f39668c86e..416a20aa7fe8 100644
--- a/nixos/tests/wordpress.nix
+++ b/nixos/tests/wordpress.nix
@@ -15,15 +15,12 @@ import ./make-test-python.nix ({ pkgs, ... }:
       services.httpd.adminAddr = "webmaster@site.local";
       services.httpd.logPerVirtualHost = true;
 
-      services.wordpress = {
-        # Test support for old interface
+      services.wordpress.sites = {
         "site1.local" = {
           database.tablePrefix = "site1_";
         };
-        sites = {
-          "site2.local" = {
-            database.tablePrefix = "site2_";
-          };
+        "site2.local" = {
+          database.tablePrefix = "site2_";
         };
       };
 
diff --git a/nixos/tests/wpa_supplicant.nix b/nixos/tests/wpa_supplicant.nix
index 1d669d5016a7..40d934b8e1db 100644
--- a/nixos/tests/wpa_supplicant.nix
+++ b/nixos/tests/wpa_supplicant.nix
@@ -27,8 +27,19 @@ import ./make-test-python.nix ({ pkgs, lib, ...}:
       enable = lib.mkOverride 0 true;
       userControlled.enable = true;
       interfaces = [ "wlan1" ];
+      fallbackToWPA2 = true;
 
       networks = {
+        # test WPA2 fallback
+        mixed-wpa = {
+          psk = "password";
+          authProtocols = [ "WPA-PSK" "SAE" ];
+        };
+        sae-only = {
+          psk = "password";
+          authProtocols = [ "SAE" ];
+        };
+
         # test network
         nixos-test.psk = "@PSK_NIXOS_TEST@";
 
@@ -64,8 +75,12 @@ import ./make-test-python.nix ({ pkgs, lib, ...}:
           machine.succeed(f"grep -q @PSK_MISSING@ {config_file}")
           machine.succeed(f"grep -q P@ssowrdWithSome@tSymbol {config_file}")
 
-          # save file for manual inspection
-          machine.copy_from_vm(config_file)
+      with subtest("WPA2 fallbacks have been generated"):
+          assert int(machine.succeed(f"grep -c sae-only {config_file}")) == 1
+          assert int(machine.succeed(f"grep -c mixed-wpa {config_file}")) == 2
+
+      # save file for manual inspection
+      machine.copy_from_vm(config_file)
 
       with subtest("Daemon is running and accepting connections"):
           machine.wait_for_unit("wpa_supplicant-wlan1.service")
diff --git a/nixos/tests/xmonad.nix b/nixos/tests/xmonad.nix
index 078cd2118107..a2fb38e53bd1 100644
--- a/nixos/tests/xmonad.nix
+++ b/nixos/tests/xmonad.nix
@@ -1,7 +1,58 @@
-import ./make-test-python.nix ({ pkgs, ...} : {
+import ./make-test-python.nix ({ pkgs, ...}:
+
+let
+  mkConfig = name: keys: ''
+    import XMonad
+    import XMonad.Operations (restart)
+    import XMonad.Util.EZConfig
+    import XMonad.Util.SessionStart
+    import Control.Monad (when)
+    import Text.Printf (printf)
+    import System.Posix.Process (executeFile)
+    import System.Info (arch,os)
+    import System.Environment (getArgs)
+    import System.FilePath ((</>))
+
+    main = launch $ def { startupHook = startup } `additionalKeysP` myKeys
+
+    startup = isSessionStart >>= \sessInit ->
+      spawn "touch /tmp/${name}"
+        >> if sessInit then setSessionStarted else spawn "xterm"
+
+    myKeys = [${builtins.concatStringsSep ", " keys}]
+
+    compiledConfig = printf "xmonad-%s-%s" arch os
+
+    compileRestart resume =
+      whenX (recompile True) $
+        when resume writeStateToFile
+          *> catchIO
+            ( do
+                dir <- getXMonadDataDir
+                args <- getArgs
+                executeFile (dir </> compiledConfig) False args Nothing
+            )
+  '';
+
+  oldKeys =
+    [ ''("M-C-x", spawn "xterm")''
+      ''("M-q", restart "xmonad" True)''
+      ''("M-C-q", compileRestart True)''
+      ''("M-C-t", spawn "touch /tmp/somefile")'' # create somefile
+    ];
+
+  newKeys =
+    [ ''("M-C-x", spawn "xterm")''
+      ''("M-q", restart "xmonad" True)''
+      ''("M-C-q", compileRestart True)''
+      ''("M-C-r", spawn "rm /tmp/somefile")'' # delete somefile
+    ];
+
+  newConfig = pkgs.writeText "xmonad.hs" (mkConfig "newXMonad" newKeys);
+in {
   name = "xmonad";
   meta = with pkgs.lib.maintainers; {
-    maintainers = [ nequissimus ];
+    maintainers = [ nequissimus ivanbrennan ];
   };
 
   machine = { pkgs, ... }: {
@@ -10,21 +61,10 @@ import ./make-test-python.nix ({ pkgs, ...} : {
     services.xserver.displayManager.defaultSession = "none+xmonad";
     services.xserver.windowManager.xmonad = {
       enable = true;
+      enableConfiguredRecompile = true;
       enableContribAndExtras = true;
       extraPackages = with pkgs.haskellPackages; haskellPackages: [ xmobar ];
-      config = ''
-        import XMonad
-        import XMonad.Operations (restart)
-        import XMonad.Util.EZConfig
-        import XMonad.Util.SessionStart
-
-        main = launch $ def { startupHook = startup } `additionalKeysP` myKeys
-
-        startup = isSessionStart >>= \sessInit ->
-          if sessInit then setSessionStarted else spawn "xterm"
-
-        myKeys = [ ("M-C-x", spawn "xterm"), ("M-q", restart "xmonad" True) ]
-      '';
+      config = mkConfig "oldXMonad" oldKeys;
     };
   };
 
@@ -38,10 +78,37 @@ import ./make-test-python.nix ({ pkgs, ...} : {
     machine.wait_for_window("${user.name}.*machine")
     machine.sleep(1)
     machine.screenshot("terminal1")
+    machine.succeed("rm /tmp/oldXMonad")
     machine.send_key("alt-q")
-    machine.sleep(3)
+    machine.wait_for_file("/tmp/oldXMonad")
     machine.wait_for_window("${user.name}.*machine")
     machine.sleep(1)
     machine.screenshot("terminal2")
+
+    # /tmp/somefile should not exist yet
+    machine.fail("stat /tmp/somefile")
+
+    # original config has a keybinding that creates somefile
+    machine.send_key("alt-ctrl-t")
+    machine.wait_for_file("/tmp/somefile")
+
+    # set up the new config
+    machine.succeed("mkdir -p ${user.home}/.xmonad")
+    machine.copy_from_host("${newConfig}", "${user.home}/.xmonad/xmonad.hs")
+
+    # recompile xmonad using the new config
+    machine.send_key("alt-ctrl-q")
+    machine.wait_for_file("/tmp/newXMonad")
+
+    # new config has a keybinding that deletes somefile
+    machine.send_key("alt-ctrl-r")
+    machine.wait_until_fails("stat /tmp/somefile", timeout=30)
+
+    # restart with the old config, and confirm the old keybinding is back
+    machine.succeed("rm /tmp/oldXMonad")
+    machine.send_key("alt-q")
+    machine.wait_for_file("/tmp/oldXMonad")
+    machine.send_key("alt-ctrl-t")
+    machine.wait_for_file("/tmp/somefile")
   '';
 })
diff --git a/nixos/tests/xmpp/prosody-mysql.nix b/nixos/tests/xmpp/prosody-mysql.nix
index 9a00bcabf389..40f3e308a04e 100644
--- a/nixos/tests/xmpp/prosody-mysql.nix
+++ b/nixos/tests/xmpp/prosody-mysql.nix
@@ -1,50 +1,79 @@
-import ../make-test-python.nix {
-  name = "prosody-mysql";
+let
+  cert = pkgs: pkgs.runCommand "selfSignedCerts" { buildInputs = [ pkgs.openssl ]; } ''
+    openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -nodes -subj '/CN=example.com/CN=uploads.example.com/CN=conference.example.com' -days 36500
+    mkdir -p $out
+    cp key.pem cert.pem $out
+  '';
+  createUsers = pkgs: pkgs.writeScriptBin "create-prosody-users" ''
+    #!${pkgs.bash}/bin/bash
+    set -e
+
+    # Creates and set password for the 2 xmpp test users.
+    #
+    # Doing that in a bash script instead of doing that in the test
+    # script allow us to easily provision the users when running that
+    # test interactively.
+
+    prosodyctl register cthon98 example.com nothunter2
+    prosodyctl register azurediamond example.com hunter2
+  '';
+  delUsers = pkgs: pkgs.writeScriptBin "delete-prosody-users" ''
+    #!${pkgs.bash}/bin/bash
+    set -e
+
+    # Deletes the test users.
+    #
+    # Doing that in a bash script instead of doing that in the test
+    # script allow us to easily provision the users when running that
+    # test interactively.
 
+    prosodyctl deluser cthon98@example.com
+    prosodyctl deluser azurediamond@example.com
+  '';
+in import ../make-test-python.nix {
+  name = "prosody-mysql";
   nodes = {
-    client = { nodes, pkgs, ... }: {
-      environment.systemPackages = [
-        (pkgs.callPackage ./xmpp-sendmessage.nix { connectTo = nodes.server.config.networking.primaryIPAddress; })
-      ];
+    client = { nodes, pkgs, config, ... }: {
+      security.pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
+      console.keyMap = "fr-bepo";
       networking.extraHosts = ''
         ${nodes.server.config.networking.primaryIPAddress} example.com
         ${nodes.server.config.networking.primaryIPAddress} conference.example.com
         ${nodes.server.config.networking.primaryIPAddress} uploads.example.com
       '';
+      environment.systemPackages = [
+        (pkgs.callPackage ./xmpp-sendmessage.nix { connectTo = nodes.server.config.networking.primaryIPAddress; })
+      ];
     };
     server = { config, pkgs, ... }: {
       nixpkgs.overlays = [
         (self: super: {
           prosody = super.prosody.override {
-            withDBI = true;
-            withExtraLibs = [ pkgs.luaPackages.luadbi-mysql ];
+            withExtraLuaPackages = p: [ p.luadbi-mysql ];
           };
         })
       ];
+      security.pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
+      console.keyMap = "fr-bepo";
       networking.extraHosts = ''
         ${config.networking.primaryIPAddress} example.com
         ${config.networking.primaryIPAddress} conference.example.com
         ${config.networking.primaryIPAddress} uploads.example.com
       '';
       networking.firewall.enable = false;
+      environment.systemPackages = [
+        (createUsers pkgs)
+        (delUsers pkgs)
+      ];
       services.prosody = {
         enable = true;
-        # TODO: use a self-signed certificate
-        c2sRequireEncryption = false;
-        extraConfig = ''
-          storage = "sql"
-          sql = {
-            driver = "MySQL";
-            database = "prosody";
-            host = "mysql";
-            port = 3306;
-            username = "prosody";
-            password = "password123";
-          };
-        '';
-        virtualHosts.test = {
+        ssl.cert = "${cert pkgs}/cert.pem";
+        ssl.key = "${cert pkgs}/key.pem";
+        virtualHosts.example = {
           domain = "example.com";
           enabled = true;
+          ssl.cert = "${cert pkgs}/cert.pem";
+          ssl.key = "${cert pkgs}/key.pem";
         };
         muc = [
           {
@@ -54,6 +83,17 @@ import ../make-test-python.nix {
         uploadHttp = {
           domain = "uploads.example.com";
         };
+        extraConfig = ''
+          storage = "sql"
+          sql = {
+            driver = "MySQL";
+            database = "prosody";
+            host = "mysql";
+            port = 3306;
+            username = "prosody";
+            password = "password123";
+          };
+        '';
       };
     };
     mysql = { config, pkgs, ... }: {
@@ -72,21 +112,13 @@ import ../make-test-python.nix {
   };
 
   testScript = { nodes, ... }: ''
+    # Check with mysql storage
     mysql.wait_for_unit("mysql.service")
     server.wait_for_unit("prosody.service")
     server.succeed('prosodyctl status | grep "Prosody is running"')
 
-    # set password to 'nothunter2' (it's asked twice)
-    server.succeed("yes nothunter2 | prosodyctl adduser cthon98@example.com")
-    # set password to 'y'
-    server.succeed("yes | prosodyctl adduser azurediamond@example.com")
-    # correct password to 'hunter2'
-    server.succeed("yes hunter2 | prosodyctl passwd azurediamond@example.com")
-
+    server.succeed("create-prosody-users")
     client.succeed("send-message")
-
-    server.succeed("prosodyctl deluser cthon98@example.com")
-    server.succeed("prosodyctl deluser azurediamond@example.com")
+    server.succeed("delete-prosody-users")
   '';
 }
-
diff --git a/nixos/tests/xmpp/prosody.nix b/nixos/tests/xmpp/prosody.nix
index c343b580879a..14eab56fb821 100644
--- a/nixos/tests/xmpp/prosody.nix
+++ b/nixos/tests/xmpp/prosody.nix
@@ -81,6 +81,7 @@ in import ../make-test-python.nix {
   };
 
   testScript = { nodes, ... }: ''
+    # Check with sqlite storage
     server.wait_for_unit("prosody.service")
     server.succeed('prosodyctl status | grep "Prosody is running"')
 
diff --git a/nixos/tests/xxh.nix b/nixos/tests/xxh.nix
new file mode 100644
index 000000000000..3af8e53779e3
--- /dev/null
+++ b/nixos/tests/xxh.nix
@@ -0,0 +1,67 @@
+import ./make-test-python.nix ({ pkgs, lib, ... }:
+
+  let
+    inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
+    xxh-shell-zsh = pkgs.stdenv.mkDerivation {
+      pname = "xxh-shell-zsh";
+      version = "";
+      src = pkgs.fetchFromGitHub {
+        owner = "xxh";
+        repo = "xxh-shell-zsh";
+        # gets rarely updated, we can then just replace the hash
+        rev = "91e1f84f8d6e0852c3235d4813f341230cac439f";
+        sha256 = "sha256-Y1FrIRxTd0yooK+ZzKcCd6bLSy5E2fRXYAzrIsm7rIc=";
+      };
+
+      postPatch = ''
+        substituteInPlace build.sh \
+          --replace "echo Install wget or curl" "cp ${zsh-portable-binary} zsh-5.8-linux-x86_64.tar.gz" \
+          --replace "command -v curl" "command -v this-should-not-trigger"
+      '';
+
+      installPhase = ''
+        mkdir -p $out
+        mv * $out/
+      '';
+    };
+
+    zsh-portable-binary = pkgs.fetchurl {
+      # kept in sync with https://github.com/xxh/xxh-shell-zsh/tree/master/build.sh#L27
+      url = "https://github.com/romkatv/zsh-bin/releases/download/v3.0.1/zsh-5.8-linux-x86_64.tar.gz";
+      sha256 = "sha256-i8flMd2Isc0uLoeYQNDnOGb/kK3oTFVqQgIx7aOAIIo=";
+    };
+  in
+  {
+    name = "xxh";
+    meta = with lib.maintainers; {
+      maintainers = [ lom ];
+    };
+
+    nodes = {
+      server = { ... }: {
+        services.openssh.enable = true;
+        users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
+      };
+
+      client = { ... }: {
+        programs.zsh.enable = true;
+        users.users.root.shell = pkgs.zsh;
+        environment.systemPackages = with pkgs; [ xxh git ];
+      };
+    };
+
+    testScript = ''
+      start_all()
+
+      client.succeed("mkdir -m 700 /root/.ssh")
+
+      client.succeed(
+         "cat ${snakeOilPrivateKey} > /root/.ssh/id_ecdsa"
+      )
+      client.succeed("chmod 600 /root/.ssh/id_ecdsa")
+
+      server.wait_for_unit("sshd")
+
+      client.succeed("xxh server -i /root/.ssh/id_ecdsa +hc \'echo $0\' +i +s zsh +I xxh-shell-zsh+path+${xxh-shell-zsh} | grep -Fq '/root/.xxh/.xxh/shells/xxh-shell-zsh/build/zsh-bin/bin/zsh'")
+    '';
+  })
diff --git a/nixos/tests/zammad.nix b/nixos/tests/zammad.nix
new file mode 100644
index 000000000000..4e466f6e3b9b
--- /dev/null
+++ b/nixos/tests/zammad.nix
@@ -0,0 +1,60 @@
+import ./make-test-python.nix (
+  { lib, pkgs, ... }:
+
+  {
+    name = "zammad";
+
+    meta.maintainers = with lib.maintainers; [ garbas taeer ];
+
+    nodes.machine = { config, ... }: {
+      services.zammad.enable = true;
+      services.zammad.secretKeyBaseFile = pkgs.writeText "secret" ''
+        52882ef142066e09ab99ce816ba72522e789505caba224a52d750ec7dc872c2c371b2fd19f16b25dfbdd435a4dd46cb3df9f82eb63fafad715056bdfe25740d6
+      '';
+
+      systemd.services.zammad-locale-cheat =
+        let cfg = config.services.zammad; in
+        {
+          serviceConfig = {
+            Type = "simple";
+            Restart = "always";
+
+            User = "zammad";
+            Group = "zammad";
+            PrivateTmp = true;
+            StateDirectory = "zammad";
+            WorkingDirectory = cfg.dataDir;
+          };
+          wantedBy = [ "zammad-web.service" ];
+          description = "Hack in the locale files so zammad doesn't try to access the internet";
+          script = ''
+            mkdir -p ./config/translations
+            VERSION=$(cat ${cfg.package}/VERSION)
+
+            # If these files are not in place, zammad will try to access the internet.
+            # For the test, we only need to supply en-us.
+            echo '[{"locale":"en-us","alias":"en","name":"English (United States)","active":true,"dir":"ltr"}]' \
+              > ./config/locales-$VERSION.yml
+            echo '[{"locale":"en-us","format":"time","source":"date","target":"mm/dd/yyyy","target_initial":"mm/dd/yyyy"},{"locale":"en-us","format":"time","source":"timestamp","target":"mm/dd/yyyy HH:MM","target_initial":"mm/dd/yyyy HH:MM"}]' \
+              > ./config/translations/en-us-$VERSION.yml
+          '';
+        };
+    };
+
+    testScript = ''
+      start_all()
+      machine.wait_for_unit("postgresql.service")
+      machine.wait_for_unit("zammad-web.service")
+      machine.wait_for_unit("zammad-websocket.service")
+      machine.wait_for_unit("zammad-scheduler.service")
+      # wait for zammad to fully come up
+      machine.sleep(120)
+
+      # without the grep the command does not produce valid utf-8 for some reason
+      with subtest("welcome screen loads"):
+          machine.succeed(
+              "curl -sSfL http://localhost:3000/ | grep '<title>Zammad Helpdesk</title>'"
+          )
+    '';
+  }
+)