about summary refs log tree commit diff
path: root/nixos/modules
diff options
context:
space:
mode:
Diffstat (limited to 'nixos/modules')
-rw-r--r--nixos/modules/config/fonts/fontconfig.nix25
-rw-r--r--nixos/modules/installer/tools/nixos-install.sh73
-rw-r--r--nixos/modules/installer/tools/nixos-rebuild.sh10
-rw-r--r--nixos/modules/installer/tools/tools.nix2
-rw-r--r--nixos/modules/module-list.nix1
-rw-r--r--nixos/modules/security/acme.nix649
-rw-r--r--nixos/modules/security/acme.xml20
-rw-r--r--nixos/modules/services/backup/borgbackup.xml22
-rw-r--r--nixos/modules/services/development/lorri.nix13
-rw-r--r--nixos/modules/services/games/terraria.nix18
-rw-r--r--nixos/modules/services/misc/gitlab.nix17
-rw-r--r--nixos/modules/services/misc/gitlab.xml6
-rw-r--r--nixos/modules/services/monitoring/prometheus/exporters.nix4
-rw-r--r--nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix34
-rw-r--r--nixos/modules/services/monitoring/unifi-poller.nix242
-rw-r--r--nixos/modules/services/network-filesystems/cachefilesd.nix18
-rw-r--r--nixos/modules/services/networking/prosody.xml5
-rw-r--r--nixos/modules/services/networking/ssh/sshd.nix10
-rw-r--r--nixos/modules/services/networking/syncthing.nix13
-rw-r--r--nixos/modules/services/web-servers/apache-httpd/default.nix77
-rw-r--r--nixos/modules/services/web-servers/nginx/default.nix88
-rw-r--r--nixos/modules/services/web-servers/phpfpm/default.nix1
-rw-r--r--nixos/modules/services/x11/desktop-managers/cinnamon.nix205
-rw-r--r--nixos/modules/services/x11/desktop-managers/default.nix1
-rw-r--r--nixos/modules/services/x11/desktop-managers/plasma5.nix3
-rw-r--r--nixos/modules/system/boot/stage-1.nix8
-rw-r--r--nixos/modules/testing/test-instrumentation.nix9
27 files changed, 1182 insertions, 392 deletions
diff --git a/nixos/modules/config/fonts/fontconfig.nix b/nixos/modules/config/fonts/fontconfig.nix
index 1f1044bc5af8..5b681ca59464 100644
--- a/nixos/modules/config/fonts/fontconfig.nix
+++ b/nixos/modules/config/fonts/fontconfig.nix
@@ -1,6 +1,6 @@
 /*
 
-Configuration files are linked to /etc/fonts/${pkgs.fontconfig.configVersion}/conf.d/
+Configuration files are linked to /etc/fonts/conf.d/
 
 This module generates a package containing configuration files and link it in /etc/fonts.
 
@@ -35,7 +35,7 @@ let
     in
     pkgs.writeText "fc-00-nixos-cache.conf" ''
       <?xml version='1.0'?>
-      <!DOCTYPE fontconfig SYSTEM 'fonts.dtd'>
+      <!DOCTYPE fontconfig SYSTEM 'urn:fontconfig:fonts.dtd'>
       <fontconfig>
         <!-- Font directories -->
         ${concatStringsSep "\n" (map (font: "<dir>${font}</dir>") config.fonts.fonts)}
@@ -53,7 +53,7 @@ let
   # priority 10
   renderConf = pkgs.writeText "fc-10-nixos-rendering.conf" ''
     <?xml version='1.0'?>
-    <!DOCTYPE fontconfig SYSTEM 'fonts.dtd'>
+    <!DOCTYPE fontconfig SYSTEM 'urn:fontconfig:fonts.dtd'>
     <fontconfig>
 
       <!-- Default rendering settings -->
@@ -110,7 +110,7 @@ let
     in
     pkgs.writeText "fc-52-nixos-default-fonts.conf" ''
     <?xml version='1.0'?>
-    <!DOCTYPE fontconfig SYSTEM 'fonts.dtd'>
+    <!DOCTYPE fontconfig SYSTEM 'urn:fontconfig:fonts.dtd'>
     <fontconfig>
 
       <!-- Default fonts -->
@@ -129,7 +129,7 @@ let
   # priority 53
   rejectBitmaps = pkgs.writeText "fc-53-no-bitmaps.conf" ''
     <?xml version="1.0"?>
-    <!DOCTYPE fontconfig SYSTEM "fonts.dtd">
+    <!DOCTYPE fontconfig SYSTEM "urn:fontconfig:fonts.dtd">
     <fontconfig>
 
     ${optionalString (!cfg.allowBitmaps) ''
@@ -157,7 +157,7 @@ let
   # priority 53
   rejectType1 = pkgs.writeText "fc-53-nixos-reject-type1.conf" ''
     <?xml version="1.0"?>
-    <!DOCTYPE fontconfig SYSTEM "fonts.dtd">
+    <!DOCTYPE fontconfig SYSTEM "urn:fontconfig:fonts.dtd">
     <fontconfig>
 
     <!-- Reject Type 1 fonts -->
@@ -176,15 +176,16 @@ let
   confPkg = pkgs.runCommand "fontconfig-conf" {
     preferLocalBuild = true;
   } ''
-    dst=$out/etc/fonts/${pkg.configVersion}/conf.d
+    dst=$out/etc/fonts/conf.d
     mkdir -p $dst
 
     # fonts.conf
     ln -s ${pkg.out}/etc/fonts/fonts.conf \
           $dst/../fonts.conf
     # TODO: remove this legacy symlink once people stop using packages built before #95358 was merged
-    ln -s /etc/fonts/${pkg.configVersion}/fonts.conf \
-          $out/etc/fonts/fonts.conf
+    mkdir -p $out/etc/fonts/2.11
+    ln -s /etc/fonts/fonts.conf \
+          $out/etc/fonts/2.11/fonts.conf
 
     # fontconfig default config files
     ln -s ${pkg.out}/etc/fonts/conf.d/*.conf \
@@ -197,10 +198,8 @@ let
     ln -s ${renderConf}       $dst/10-nixos-rendering.conf
 
     # 50-user.conf
-    # Since latest fontconfig looks for default files inside the package,
-    # we had to move this one elsewhere to be able to exclude it here.
-    ${optionalString cfg.includeUserConf ''
-    ln -s ${pkg.out}/etc/fonts/conf.d.bak/50-user.conf $dst/50-user.conf
+    ${optionalString (!cfg.includeUserConf) ''
+    rm $dst/50-user.conf
     ''}
 
     # local.conf (indirect priority 51)
diff --git a/nixos/modules/installer/tools/nixos-install.sh b/nixos/modules/installer/tools/nixos-install.sh
index e0252befdfdc..a180d1bc4c19 100644
--- a/nixos/modules/installer/tools/nixos-install.sh
+++ b/nixos/modules/installer/tools/nixos-install.sh
@@ -10,6 +10,7 @@ umask 0022
 
 # Parse the command line for the -I flag
 extraBuildFlags=()
+flakeFlags=()
 
 mountPoint=/mnt
 channelPath=
@@ -34,6 +35,23 @@ while [ "$#" -gt 0 ]; do
         --system|--closure)
             system="$1"; shift 1
             ;;
+        --flake)
+          flake="$1"
+          flakeFlags=(--experimental-features 'nix-command flakes')
+          shift 1
+          ;;
+        --recreate-lock-file|--no-update-lock-file|--no-write-lock-file|--no-registries|--commit-lock-file)
+          lockFlags+=("$i")
+          ;;
+        --update-input)
+          j="$1"; shift 1
+          lockFlags+=("$i" "$j")
+          ;;
+        --override-input)
+          j="$1"; shift 1
+          k="$1"; shift 1
+          lockFlags+=("$i" "$j" "$k")
+          ;;
         --channel)
             channelPath="$1"; shift 1
             ;;
@@ -92,14 +110,32 @@ if [[ ${NIXOS_CONFIG:0:1} != / ]]; then
     exit 1
 fi
 
-if [[ ! -e $NIXOS_CONFIG && -z $system ]]; then
+if [[ -n $flake ]]; then
+    if [[ $flake =~ ^(.*)\#([^\#\"]*)$ ]]; then
+       flake="${BASH_REMATCH[1]}"
+       flakeAttr="${BASH_REMATCH[2]}"
+    fi
+    if [[ -z "$flakeAttr" ]]; then
+        echo "Please specify the name of the NixOS configuration to be installed, as a URI fragment in the flake-uri."
+        echo "For example, to use the output nixosConfigurations.foo from the flake.nix, append \"#foo\" to the flake-uri."
+        exit 1
+    fi
+    flakeAttr="nixosConfigurations.\"$flakeAttr\""
+fi
+
+# Resolve the flake.
+if [[ -n $flake ]]; then
+    flake=$(nix "${flakeFlags[@]}" flake info --json "${extraBuildFlags[@]}" "${lockFlags[@]}" -- "$flake" | jq -r .url)
+fi
+
+if [[ ! -e $NIXOS_CONFIG && -z $system && -z $flake ]]; then
     echo "configuration file $NIXOS_CONFIG doesn't exist"
     exit 1
 fi
 
 # A place to drop temporary stuff.
-tmpdir="$(mktemp -d -p $mountPoint)"
-trap "rm -rf $tmpdir" EXIT
+tmpdir="$(mktemp -d -p "$mountPoint")"
+trap 'rm -rf $tmpdir' EXIT
 
 # store temporary files on target filesystem by default
 export TMPDIR=${TMPDIR:-$tmpdir}
@@ -108,12 +144,19 @@ sub="auto?trusted=1"
 
 # Build the system configuration in the target filesystem.
 if [[ -z $system ]]; then
-    echo "building the configuration in $NIXOS_CONFIG..."
     outLink="$tmpdir/system"
-    nix-build --out-link "$outLink" --store "$mountPoint" "${extraBuildFlags[@]}" \
-        --extra-substituters "$sub" \
-        '<nixpkgs/nixos>' -A system -I "nixos-config=$NIXOS_CONFIG" ${verbosity[@]}
-    system=$(readlink -f $outLink)
+    if [[ -z $flake ]]; then
+        echo "building the configuration in $NIXOS_CONFIG..."
+        nix-build --out-link "$outLink" --store "$mountPoint" "${extraBuildFlags[@]}" \
+            --extra-substituters "$sub" \
+            '<nixpkgs/nixos>' -A system -I "nixos-config=$NIXOS_CONFIG" "${verbosity[@]}"
+    else
+        echo "building the flake in $flake..."
+        nix "${flakeFlags[@]}" build "$flake#$flakeAttr.config.system.build.toplevel" \
+            --extra-substituters "$sub" "${verbosity[@]}" \
+            "${extraBuildFlags[@]}" "${lockFlags[@]}" --out-link "$outLink"
+    fi
+    system=$(readlink -f "$outLink")
 fi
 
 # Set the system profile to point to the configuration. TODO: combine
@@ -121,7 +164,7 @@ fi
 # a progress bar.
 nix-env --store "$mountPoint" "${extraBuildFlags[@]}" \
         --extra-substituters "$sub" \
-        -p $mountPoint/nix/var/nix/profiles/system --set "$system" ${verbosity[@]}
+        -p "$mountPoint"/nix/var/nix/profiles/system --set "$system" "${verbosity[@]}"
 
 # Copy the NixOS/Nixpkgs sources to the target as the initial contents
 # of the NixOS channel.
@@ -131,12 +174,12 @@ if [[ -z $noChannelCopy ]]; then
     fi
     if [[ -n $channelPath ]]; then
         echo "copying channel..."
-        mkdir -p $mountPoint/nix/var/nix/profiles/per-user/root
+        mkdir -p "$mountPoint"/nix/var/nix/profiles/per-user/root
         nix-env --store "$mountPoint" "${extraBuildFlags[@]}" --extra-substituters "$sub" \
-                -p $mountPoint/nix/var/nix/profiles/per-user/root/channels --set "$channelPath" --quiet \
-                ${verbosity[@]}
-        install -m 0700 -d $mountPoint/root/.nix-defexpr
-        ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
+                -p "$mountPoint"/nix/var/nix/profiles/per-user/root/channels --set "$channelPath" --quiet \
+                "${verbosity[@]}"
+        install -m 0700 -d "$mountPoint"/root/.nix-defexpr
+        ln -sfn /nix/var/nix/profiles/per-user/root/channels "$mountPoint"/root/.nix-defexpr/channels
     fi
 fi
 
@@ -150,7 +193,7 @@ touch "$mountPoint/etc/NIXOS"
 if [[ -z $noBootLoader ]]; then
     echo "installing the boot loader..."
     # Grub needs an mtab.
-    ln -sfn /proc/mounts $mountPoint/etc/mtab
+    ln -sfn /proc/mounts "$mountPoint"/etc/mtab
     NIXOS_INSTALL_BOOTLOADER=1 nixos-enter --root "$mountPoint" -- /run/current-system/bin/switch-to-configuration boot
 fi
 
diff --git a/nixos/modules/installer/tools/nixos-rebuild.sh b/nixos/modules/installer/tools/nixos-rebuild.sh
index ed9c2509b6b6..ad40fd2811dc 100644
--- a/nixos/modules/installer/tools/nixos-rebuild.sh
+++ b/nixos/modules/installer/tools/nixos-rebuild.sh
@@ -17,6 +17,7 @@ showSyntax() {
 origArgs=("$@")
 extraBuildFlags=()
 lockFlags=()
+flakeFlags=()
 action=
 buildNix=1
 fast=
@@ -99,6 +100,7 @@ while [ "$#" -gt 0 ]; do
         ;;
       --flake)
         flake="$1"
+        flakeFlags=(--experimental-features 'nix-command flakes')
         shift 1
         ;;
       --recreate-lock-file|--no-update-lock-file|--no-write-lock-file|--no-registries|--commit-lock-file)
@@ -281,7 +283,7 @@ fi
 
 # Resolve the flake.
 if [[ -n $flake ]]; then
-    flake=$(nix flake info --json "${extraBuildFlags[@]}" "${lockFlags[@]}" -- "$flake" | jq -r .url)
+    flake=$(nix "${flakeFlags[@]}" flake info --json "${extraBuildFlags[@]}" "${lockFlags[@]}" -- "$flake" | jq -r .url)
 fi
 
 # Find configuration.nix and open editor instead of building.
@@ -293,7 +295,7 @@ if [ "$action" = edit ]; then
         fi
         exec ${EDITOR:-nano} "$NIXOS_CONFIG"
     else
-        exec nix edit "${lockFlags[@]}" -- "$flake#$flakeAttr"
+        exec nix "${flakeFlags[@]}" edit "${lockFlags[@]}" -- "$flake#$flakeAttr"
     fi
     exit 1
 fi
@@ -419,7 +421,7 @@ if [ -z "$rollback" ]; then
             pathToConfig="$(nixBuild '<nixpkgs/nixos>' --no-out-link -A system "${extraBuildFlags[@]}")"
         else
             outLink=$tmpDir/result
-            nix build "$flake#$flakeAttr.config.system.build.toplevel" \
+            nix "${flakeFlags[@]}" build "$flake#$flakeAttr.config.system.build.toplevel" \
               "${extraBuildFlags[@]}" "${lockFlags[@]}" --out-link $outLink
             pathToConfig="$(readlink -f $outLink)"
         fi
@@ -429,7 +431,7 @@ if [ -z "$rollback" ]; then
         if [[ -z $flake ]]; then
             pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A system -k "${extraBuildFlags[@]}")"
         else
-            nix build "$flake#$flakeAttr.config.system.build.toplevel" "${extraBuildFlags[@]}" "${lockFlags[@]}"
+            nix "${flakeFlags[@]}" build "$flake#$flakeAttr.config.system.build.toplevel" "${extraBuildFlags[@]}" "${lockFlags[@]}"
             pathToConfig="$(readlink -f ./result)"
         fi
     elif [ "$action" = build-vm ]; then
diff --git a/nixos/modules/installer/tools/tools.nix b/nixos/modules/installer/tools/tools.nix
index 1582f0493094..26fc8fb402e6 100644
--- a/nixos/modules/installer/tools/tools.nix
+++ b/nixos/modules/installer/tools/tools.nix
@@ -22,7 +22,7 @@ let
     src = ./nixos-install.sh;
     inherit (pkgs) runtimeShell;
     nix = config.nix.package.out;
-    path = makeBinPath [ nixos-enter ];
+    path = makeBinPath [ pkgs.nixUnstable nixos-enter ];
   };
 
   nixos-rebuild =
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index b516b1785195..c54bc6098d3e 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -554,6 +554,7 @@
   ./services/monitoring/telegraf.nix
   ./services/monitoring/thanos.nix
   ./services/monitoring/tuptime.nix
+  ./services/monitoring/unifi-poller.nix
   ./services/monitoring/ups.nix
   ./services/monitoring/uptime.nix
   ./services/monitoring/vnstat.nix
diff --git a/nixos/modules/security/acme.nix b/nixos/modules/security/acme.nix
index 29635dbe8643..8e67d4ff8716 100644
--- a/nixos/modules/security/acme.nix
+++ b/nixos/modules/security/acme.nix
@@ -1,11 +1,314 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, pkgs, options, ... }:
 with lib;
 let
-
   cfg = config.security.acme;
 
+  # Used to calculate timer accuracy for coalescing
+  numCerts = length (builtins.attrNames cfg.certs);
+  _24hSecs = 60 * 60 * 24;
+
+  # There are many services required to make cert renewals work.
+  # They all follow a common structure:
+  #   - They inherit this commonServiceConfig
+  #   - They all run as the acme user
+  #   - They all use BindPath and StateDirectory where possible
+  #     to set up a sort of build environment in /tmp
+  # The Group can vary depending on what the user has specified in
+  # security.acme.certs.<cert>.group on some of the services.
+  commonServiceConfig = {
+      Type = "oneshot";
+      User = "acme";
+      Group = mkDefault "acme";
+      UMask = 0027;
+      StateDirectoryMode = 750;
+      ProtectSystem = "full";
+      PrivateTmp = true;
+
+      WorkingDirectory = "/tmp";
+  };
+
+  # In order to avoid race conditions creating the CA for selfsigned certs,
+  # we have a separate service which will create the necessary files.
+  selfsignCAService = {
+    description = "Generate self-signed certificate authority";
+
+    path = with pkgs; [ minica ];
+
+    unitConfig = {
+      ConditionPathExists = "!/var/lib/acme/.minica/key.pem";
+    };
+
+    serviceConfig = commonServiceConfig // {
+      StateDirectory = "acme/.minica";
+      BindPaths = "/var/lib/acme/.minica:/tmp/ca";
+    };
+
+    # Working directory will be /tmp
+    script = ''
+      minica \
+        --ca-key ca/key.pem \
+        --ca-cert ca/cert.pem \
+        --domains selfsigned.local
+
+      chmod 600 ca/*
+    '';
+  };
+
+  # Previously, all certs were owned by whatever user was configured in
+  # config.security.acme.certs.<cert>.user. Now everything is owned by and
+  # run by the acme user.
+  userMigrationService = {
+    description = "Fix owner and group of all ACME certificates";
+
+    script = with builtins; concatStringsSep "\n" (mapAttrsToList (cert: data: ''
+      for fixpath in /var/lib/acme/${escapeShellArg cert} /var/lib/acme/.lego/${escapeShellArg cert}; do
+        if [ -d "$fixpath" ]; then
+          chmod -R 750 "$fixpath"
+          chown -R acme:${data.group} "$fixpath"
+        fi
+      done
+    '') certConfigs);
+
+    # We don't want this to run every time a renewal happens
+    serviceConfig.RemainAfterExit = true;
+  };
+
+  certToConfig = cert: data: let
+    acmeServer = if data.server != null then data.server else cfg.server;
+    useDns = data.dnsProvider != null;
+    destPath = "/var/lib/acme/${cert}";
+    selfsignedDeps = optionals (cfg.preliminarySelfsigned) [ "acme-selfsigned-${cert}.service" ];
+
+    # Minica and lego have a "feature" which replaces * with _. We need
+    # to make this substitution to reference the output files from both programs.
+    # End users never see this since we rename the certs.
+    keyName = builtins.replaceStrings ["*"] ["_"] data.domain;
+
+    # FIXME when mkChangedOptionModule supports submodules, change to that.
+    # This is a workaround
+    extraDomains = data.extraDomainNames ++ (
+      optionals
+      (data.extraDomains != "_mkMergedOptionModule")
+      (builtins.attrNames data.extraDomains)
+    );
+
+    # Create hashes for cert data directories based on configuration
+    # Flags are separated to avoid collisions
+    hashData = with builtins; ''
+      ${concatStringsSep " " data.extraLegoFlags} -
+      ${concatStringsSep " " data.extraLegoRunFlags} -
+      ${concatStringsSep " " data.extraLegoRenewFlags} -
+      ${toString acmeServer} ${toString data.dnsProvider}
+      ${toString data.ocspMustStaple} ${data.keyType}
+    '';
+    mkHash = with builtins; val: substring 0 20 (hashString "sha256" val);
+    certDir = mkHash hashData;
+    domainHash = mkHash "${concatStringsSep " " extraDomains} ${data.domain}";
+    othersHash = mkHash "${toString acmeServer} ${data.keyType}";
+    accountDir = "/var/lib/acme/.lego/accounts/" + othersHash;
+
+    protocolOpts = if useDns then (
+      [ "--dns" data.dnsProvider ]
+      ++ optionals (!data.dnsPropagationCheck) [ "--dns.disable-cp" ]
+    ) else (
+      [ "--http" "--http.webroot" data.webroot ]
+    );
+
+    commonOpts = [
+      "--accept-tos" # Checking the option is covered by the assertions
+      "--path" "."
+      "-d" data.domain
+      "--email" data.email
+      "--key-type" data.keyType
+    ] ++ protocolOpts
+      ++ optionals data.ocspMustStaple [ "--must-staple" ]
+      ++ optionals (acmeServer != null) [ "--server" acmeServer ]
+      ++ concatMap (name: [ "-d" name ]) extraDomains
+      ++ data.extraLegoFlags;
+
+    runOpts = escapeShellArgs (
+      commonOpts
+      ++ [ "run" ]
+      ++ data.extraLegoRunFlags
+    );
+    renewOpts = escapeShellArgs (
+      commonOpts
+      ++ [ "renew" "--reuse-key" ]
+      ++ data.extraLegoRenewFlags
+    );
+
+  in {
+    inherit accountDir selfsignedDeps;
+
+    webroot = data.webroot;
+    group = data.group;
+
+    renewTimer = {
+      description = "Renew ACME Certificate for ${cert}";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.renewInterval;
+        Unit = "acme-${cert}.service";
+        Persistent = "yes";
+
+        # Allow systemd to pick a convenient time within the day
+        # to run the check.
+        # This allows the coalescing of multiple timer jobs.
+        # We divide by the number of certificates so that if you
+        # have many certificates, the renewals are distributed over
+        # the course of the day to avoid rate limits.
+        AccuracySec = "${toString (_24hSecs / numCerts)}s";
+
+        # Skew randomly within the day, per https://letsencrypt.org/docs/integration-guide/.
+        RandomizedDelaySec = "24h";
+      };
+    };
+
+    selfsignService = {
+      description = "Generate self-signed certificate for ${cert}";
+      after = [ "acme-selfsigned-ca.service" "acme-fixperms.service" ];
+      requires = [ "acme-selfsigned-ca.service" "acme-fixperms.service" ];
+
+      path = with pkgs; [ minica ];
+
+      unitConfig = {
+        ConditionPathExists = "!/var/lib/acme/${cert}/key.pem";
+      };
+
+      serviceConfig = commonServiceConfig // {
+        Group = data.group;
+
+        StateDirectory = "acme/${cert}";
+
+        BindPaths = "/var/lib/acme/.minica:/tmp/ca /var/lib/acme/${cert}:/tmp/${keyName}";
+      };
+
+      # Working directory will be /tmp
+      # minica will output to a folder sharing the name of the first domain
+      # in the list, which will be ${data.domain}
+      script = ''
+        minica \
+          --ca-key ca/key.pem \
+          --ca-cert ca/cert.pem \
+          --domains ${escapeShellArg (builtins.concatStringsSep "," ([ data.domain ] ++ extraDomains))}
+
+        # Create files to match directory layout for real certificates
+        cd '${keyName}'
+        cp ../ca/cert.pem chain.pem
+        cat cert.pem chain.pem > fullchain.pem
+        cat key.pem fullchain.pem > full.pem
+
+        chmod 640 *
+
+        # Group might change between runs, re-apply it
+        chown 'acme:${data.group}' *
+      '';
+    };
+
+    renewService = {
+      description = "Renew ACME certificate for ${cert}";
+      after = [ "network.target" "network-online.target" "acme-fixperms.service" ] ++ selfsignedDeps;
+      wants = [ "network-online.target" "acme-fixperms.service" ] ++ selfsignedDeps;
+
+      # https://github.com/NixOS/nixpkgs/pull/81371#issuecomment-605526099
+      wantedBy = optionals (!config.boot.isContainer) [ "multi-user.target" ];
+
+      path = with pkgs; [ lego coreutils diffutils ];
+
+      serviceConfig = commonServiceConfig // {
+        Group = data.group;
+
+        # AccountDir dir will be created by tmpfiles to ensure correct permissions
+        # And to avoid deletion during systemctl clean
+        # acme/.lego/${cert} is listed so that it is deleted during systemctl clean
+        StateDirectory = "acme/${cert} acme/.lego/${cert} acme/.lego/${cert}/${certDir}";
+
+        # Needs to be space separated, but can't use a multiline string because that'll include newlines
+        BindPaths =
+          "${accountDir}:/tmp/accounts " +
+          "/var/lib/acme/${cert}:/tmp/out " +
+          "/var/lib/acme/.lego/${cert}/${certDir}:/tmp/certificates ";
+
+        # Only try loading the credentialsFile if the dns challenge is enabled
+        EnvironmentFile = mkIf useDns data.credentialsFile;
+
+        # Run as root (Prefixed with +)
+        ExecStartPost = "+" + (pkgs.writeShellScript "acme-postrun" ''
+          cd /var/lib/acme/${escapeShellArg cert}
+          if [ -e renewed ]; then
+            rm renewed
+            ${data.postRun}
+          fi
+        '');
+      };
+
+      # Working directory will be /tmp
+      script = ''
+        set -euo pipefail
+
+        echo '${domainHash}' > domainhash.txt
+
+        # Check if we can renew
+        if [ -e 'certificates/${keyName}.key' -a -e 'certificates/${keyName}.crt' ]; then
+
+          # When domains are updated, there's no need to do a full
+          # Lego run, but it's likely renew won't work if days is too low.
+          if [ -e certificates/domainhash.txt ] && cmp -s domainhash.txt certificates/domainhash.txt; then
+            lego ${renewOpts} --days ${toString cfg.validMinDays}
+          else
+            # Any number > 90 works, but this one is over 9000 ;-)
+            lego ${renewOpts} --days 9001
+          fi
+
+        # Otherwise do a full run
+        else
+          lego ${runOpts}
+        fi
+
+        mv domainhash.txt certificates/
+        chmod 640 certificates/*
+        chmod -R 700 accounts/*
+
+        # Group might change between runs, re-apply it
+        chown 'acme:${data.group}' certificates/*
+
+        # Copy all certs to the "real" certs directory
+        CERT='certificates/${keyName}.crt'
+        if [ -e "$CERT" ] && ! cmp -s "$CERT" out/fullchain.pem; then
+          touch out/renewed
+          echo Installing new certificate
+          cp -vp 'certificates/${keyName}.crt' out/fullchain.pem
+          cp -vp 'certificates/${keyName}.key' out/key.pem
+          cp -vp 'certificates/${keyName}.issuer.crt' out/chain.pem
+          ln -sf fullchain.pem out/cert.pem
+          cat out/key.pem out/fullchain.pem > out/full.pem
+        fi
+      '';
+    };
+  };
+
+  certConfigs = mapAttrs certToConfig cfg.certs;
+
   certOpts = { name, ... }: {
     options = {
+      # user option has been removed
+      user = mkOption {
+        visible = false;
+        default = "_mkRemovedOptionModule";
+      };
+
+      # allowKeysForGroup option has been removed
+      allowKeysForGroup = mkOption {
+        visible = false;
+        default = "_mkRemovedOptionModule";
+      };
+
+      # extraDomains was replaced with extraDomainNames
+      extraDomains = mkOption {
+        visible = false;
+        default = "_mkMergedOptionModule";
+      };
+
       webroot = mkOption {
         type = types.nullOr types.str;
         default = null;
@@ -41,35 +344,19 @@ let
         description = "Contact email address for the CA to be able to reach you.";
       };
 
-      user = mkOption {
-        type = types.str;
-        default = "root";
-        description = "User running the ACME client.";
-      };
-
       group = mkOption {
         type = types.str;
-        default = "root";
+        default = "acme";
         description = "Group running the ACME client.";
       };
 
-      allowKeysForGroup = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Give read permissions to the specified group
-          (<option>security.acme.cert.&lt;name&gt;.group</option>) to read SSL private certificates.
-        '';
-      };
-
       postRun = mkOption {
         type = types.lines;
         default = "";
-        example = "systemctl reload nginx.service";
+        example = "cp full.pem backup.pem";
         description = ''
-          Commands to run after new certificates go live. Typically
-          the web server and other servers using certificates need to
-          be reloaded.
+          Commands to run after new certificates go live. Note that
+          these commands run as the root user.
 
           Executed in the same directory with the new certificate.
         '';
@@ -82,18 +369,17 @@ let
         description = "Directory where certificate and other state is stored.";
       };
 
-      extraDomains = mkOption {
-        type = types.attrsOf (types.nullOr types.str);
-        default = {};
+      extraDomainNames = mkOption {
+        type = types.listOf types.str;
+        default = [];
         example = literalExample ''
-          {
-            "example.org" = null;
-            "mydomain.org" = null;
-          }
+          [
+            "example.org"
+            "mydomain.org"
+          ]
         '';
         description = ''
           A list of extra domain names, which are included in the one certificate to be issued.
-          Setting a distinct server root is deprecated and not functional in 20.03+
         '';
       };
 
@@ -176,24 +462,8 @@ let
     };
   };
 
-in
+in {
 
-{
-
-  ###### interface
-  imports = [
-    (mkRemovedOptionModule [ "security" "acme" "production" ] ''
-      Use security.acme.server to define your staging ACME server URL instead.
-
-      To use Let's Encrypt's staging server, use security.acme.server =
-      "https://acme-staging-v02.api.letsencrypt.org/directory".
-    ''
-    )
-    (mkRemovedOptionModule [ "security" "acme" "directory"] "ACME Directory is now hardcoded to /var/lib/acme and its permisisons are managed by systemd. See https://github.com/NixOS/nixpkgs/issues/53852 for more info.")
-    (mkRemovedOptionModule [ "security" "acme" "preDelay"] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
-    (mkRemovedOptionModule [ "security" "acme" "activationDelay"] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
-    (mkChangedOptionModule [ "security" "acme" "validMin"] [ "security" "acme" "validMinDays"] (config: config.security.acme.validMin / (24 * 3600)))
-  ];
   options = {
     security.acme = {
 
@@ -266,7 +536,7 @@ in
             "example.com" = {
               webroot = "/var/www/challenges/";
               email = "foo@example.com";
-              extraDomains = { "www.example.com" = null; "foo.example.com" = null; };
+              extraDomainNames = [ "www.example.com" "foo.example.com" ];
             };
             "bar.example.com" = {
               webroot = "/var/www/challenges/";
@@ -278,25 +548,40 @@ in
     };
   };
 
-  ###### implementation
+  imports = [
+    (mkRemovedOptionModule [ "security" "acme" "production" ] ''
+      Use security.acme.server to define your staging ACME server URL instead.
+
+      To use the let's encrypt staging server, use security.acme.server =
+      "https://acme-staging-v02.api.letsencrypt.org/directory".
+    ''
+    )
+    (mkRemovedOptionModule [ "security" "acme" "directory" ] "ACME Directory is now hardcoded to /var/lib/acme and its permisisons are managed by systemd. See https://github.com/NixOS/nixpkgs/issues/53852 for more info.")
+    (mkRemovedOptionModule [ "security" "acme" "preDelay" ] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
+    (mkRemovedOptionModule [ "security" "acme" "activationDelay" ] "This option has been removed. If you want to make sure that something executes before certificates are provisioned, add a RequiredBy=acme-\${cert}.service to the service you want to execute before the cert renewal")
+    (mkChangedOptionModule [ "security" "acme" "validMin" ] [ "security" "acme" "validMinDays" ] (config: config.security.acme.validMin / (24 * 3600)))
+  ];
+
   config = mkMerge [
     (mkIf (cfg.certs != { }) {
 
+      # FIXME Most of these custom warnings and filters for security.acme.certs.* are required
+      # because using mkRemovedOptionModule/mkChangedOptionModule with attrsets isn't possible.
+      warnings = filter (w: w != "") (mapAttrsToList (cert: data: if data.extraDomains != "_mkMergedOptionModule" then ''
+        The option definition `security.acme.certs.${cert}.extraDomains` has changed
+        to `security.acme.certs.${cert}.extraDomainNames` and is now a list of strings.
+        Setting a custom webroot for extra domains is not possible, instead use separate certs.
+      '' else "") cfg.certs);
+
       assertions = let
-        certs = (mapAttrsToList (k: v: v) cfg.certs);
+        certs = attrValues cfg.certs;
       in [
         {
-          assertion = all (certOpts: certOpts.dnsProvider == null || certOpts.webroot == null) certs;
-          message = ''
-            Options `security.acme.certs.<name>.dnsProvider` and
-            `security.acme.certs.<name>.webroot` are mutually exclusive.
-          '';
-        }
-        {
           assertion = cfg.email != null || all (certOpts: certOpts.email != null) certs;
           message = ''
             You must define `security.acme.certs.<name>.email` or
-            `security.acme.email` to register with the CA.
+            `security.acme.email` to register with the CA. Note that using
+            many different addresses for certs may trigger account rate limits.
           '';
         }
         {
@@ -307,184 +592,78 @@ in
             to `true`. For Let's Encrypt's ToS see https://letsencrypt.org/repository/
           '';
         }
-      ];
-
-      systemd.services = let
-          services = concatLists servicesLists;
-          servicesLists = mapAttrsToList certToServices cfg.certs;
-          certToServices = cert: data:
-              let
-                # StateDirectory must be relative, and will be created under /var/lib by systemd
-                lpath = "acme/${cert}";
-                apath = "/var/lib/${lpath}";
-                spath = "/var/lib/acme/.lego/${cert}";
-                keyName = builtins.replaceStrings ["*"] ["_"] data.domain;
-                requestedDomains = pipe ([ data.domain ] ++ (attrNames data.extraDomains)) [
-                  (domains: sort builtins.lessThan domains)
-                  (domains: concatStringsSep "," domains)
-                ];
-                fileMode = if data.allowKeysForGroup then "640" else "600";
-                globalOpts = [ "-d" data.domain "--email" data.email "--path" "." "--key-type" data.keyType ]
-                          ++ optionals (cfg.acceptTerms) [ "--accept-tos" ]
-                          ++ optionals (data.dnsProvider != null && !data.dnsPropagationCheck) [ "--dns.disable-cp" ]
-                          ++ concatLists (mapAttrsToList (name: root: [ "-d" name ]) data.extraDomains)
-                          ++ (if data.dnsProvider != null then [ "--dns" data.dnsProvider ] else [ "--http" "--http.webroot" data.webroot ])
-                          ++ optionals (cfg.server != null || data.server != null) ["--server" (if data.server == null then cfg.server else data.server)]
-                          ++ data.extraLegoFlags;
-                certOpts = optionals data.ocspMustStaple [ "--must-staple" ];
-                runOpts = escapeShellArgs (globalOpts ++ [ "run" ] ++ certOpts ++ data.extraLegoRunFlags);
-                renewOpts = escapeShellArgs (globalOpts ++
-                  [ "renew" "--days" (toString cfg.validMinDays) ] ++
-                  certOpts ++ data.extraLegoRenewFlags);
-                acmeService = {
-                  description = "Renew ACME Certificate for ${cert}";
-                  path = with pkgs; [ openssl ];
-                  after = [ "network.target" "network-online.target" ];
-                  wants = [ "network-online.target" ];
-                  wantedBy = mkIf (!config.boot.isContainer) [ "multi-user.target" ];
-                  serviceConfig = {
-                    Type = "oneshot";
-                    User = data.user;
-                    Group = data.group;
-                    PrivateTmp = true;
-                    StateDirectory = "acme/.lego/${cert} acme/.lego/accounts ${lpath}";
-                    StateDirectoryMode = if data.allowKeysForGroup then "750" else "700";
-                    WorkingDirectory = spath;
-                    # Only try loading the credentialsFile if the dns challenge is enabled
-                    EnvironmentFile = if data.dnsProvider != null then data.credentialsFile else null;
-                    ExecStart = pkgs.writeScript "acme-start" ''
-                      #!${pkgs.runtimeShell} -e
-                      test -L ${spath}/accounts -o -d ${spath}/accounts || ln -s ../accounts ${spath}/accounts
-                      LEGO_ARGS=(${runOpts})
-                      if [ -e ${spath}/certificates/${keyName}.crt ]; then
-                        REQUESTED_DOMAINS="${requestedDomains}"
-                        EXISTING_DOMAINS="$(openssl x509 -in ${spath}/certificates/${keyName}.crt -noout -ext subjectAltName | tail -n1 | sed -e 's/ *DNS://g')"
-                        if [ "''${REQUESTED_DOMAINS}" == "''${EXISTING_DOMAINS}" ]; then
-                          LEGO_ARGS=(${renewOpts})
-                        fi
-                      fi
-                      ${pkgs.lego}/bin/lego ''${LEGO_ARGS[@]}
-                    '';
-                    ExecStartPost =
-                      let
-                        script = pkgs.writeScript "acme-post-start" ''
-                          #!${pkgs.runtimeShell} -e
-                          cd ${apath}
-
-                          # Test that existing cert is older than new cert
-                          KEY=${spath}/certificates/${keyName}.key
-                          KEY_CHANGED=no
-                          if [ -e $KEY -a $KEY -nt key.pem ]; then
-                            KEY_CHANGED=yes
-                            cp -p ${spath}/certificates/${keyName}.key key.pem
-                            cp -p ${spath}/certificates/${keyName}.crt fullchain.pem
-                            cp -p ${spath}/certificates/${keyName}.issuer.crt chain.pem
-                            ln -sf fullchain.pem cert.pem
-                            cat key.pem fullchain.pem > full.pem
-                          fi
-
-                          chmod ${fileMode} *.pem
-                          chown '${data.user}:${data.group}' *.pem
-
-                          if [ "$KEY_CHANGED" = "yes" ]; then
-                            : # noop in case postRun is empty
-                            ${data.postRun}
-                          fi
-                        '';
-                      in
-                        "+${script}";
-                  };
-
-                };
-                selfsignedService = {
-                  description = "Create preliminary self-signed certificate for ${cert}";
-                  path = [ pkgs.openssl ];
-                  script =
-                    ''
-                      workdir="$(mktemp -d)"
-
-                      # Create CA
-                      openssl genrsa -des3 -passout pass:xxxx -out $workdir/ca.pass.key 2048
-                      openssl rsa -passin pass:xxxx -in $workdir/ca.pass.key -out $workdir/ca.key
-                      openssl req -new -key $workdir/ca.key -out $workdir/ca.csr \
-                        -subj "/C=UK/ST=Warwickshire/L=Leamington/O=OrgName/OU=Security Department/CN=example.com"
-                      openssl x509 -req -days 1 -in $workdir/ca.csr -signkey $workdir/ca.key -out $workdir/ca.crt
-
-                      # Create key
-                      openssl genrsa -des3 -passout pass:xxxx -out $workdir/server.pass.key 2048
-                      openssl rsa -passin pass:xxxx -in $workdir/server.pass.key -out $workdir/server.key
-                      openssl req -new -key $workdir/server.key -out $workdir/server.csr \
-                        -subj "/C=UK/ST=Warwickshire/L=Leamington/O=OrgName/OU=IT Department/CN=example.com"
-                      openssl x509 -req -days 1 -in $workdir/server.csr -CA $workdir/ca.crt \
-                        -CAkey $workdir/ca.key -CAserial $workdir/ca.srl -CAcreateserial \
-                        -out $workdir/server.crt
-
-                      # Copy key to destination
-                      cp $workdir/server.key ${apath}/key.pem
-
-                      # Create fullchain.pem (same format as "simp_le ... -f fullchain.pem" creates)
-                      cat $workdir/{server.crt,ca.crt} > "${apath}/fullchain.pem"
-
-                      # Create full.pem for e.g. lighttpd
-                      cat $workdir/{server.key,server.crt,ca.crt} > "${apath}/full.pem"
-
-                      # Give key acme permissions
-                      chown '${data.user}:${data.group}' "${apath}/"{key,fullchain,full}.pem
-                      chmod ${fileMode} "${apath}/"{key,fullchain,full}.pem
-                    '';
-                  serviceConfig = {
-                    Type = "oneshot";
-                    PrivateTmp = true;
-                    StateDirectory = lpath;
-                    User = data.user;
-                    Group = data.group;
-                  };
-                  unitConfig = {
-                    # Do not create self-signed key when key already exists
-                    ConditionPathExists = "!${apath}/key.pem";
-                  };
-                };
-              in (
-                [ { name = "acme-${cert}"; value = acmeService; } ]
-                ++ optional cfg.preliminarySelfsigned { name = "acme-selfsigned-${cert}"; value = selfsignedService; }
-              );
-          servicesAttr = listToAttrs services;
-        in
-          servicesAttr;
-
-      systemd.tmpfiles.rules =
-        map (data: "d ${data.webroot}/.well-known/acme-challenge - ${data.user} ${data.group}") (filter (data: data.webroot != null) (attrValues cfg.certs));
-
-      systemd.timers = let
-        # Allow systemd to pick a convenient time within the day
-        # to run the check.
-        # This allows the coalescing of multiple timer jobs.
-        # We divide by the number of certificates so that if you
-        # have many certificates, the renewals are distributed over
-        # the course of the day to avoid rate limits.
-        numCerts = length (attrNames cfg.certs);
-        _24hSecs = 60 * 60 * 24;
-        AccuracySec = "${toString (_24hSecs / numCerts)}s";
-      in flip mapAttrs' cfg.certs (cert: data: nameValuePair
-        ("acme-${cert}")
-        ({
-          description = "Renew ACME Certificate for ${cert}";
-          wantedBy = [ "timers.target" ];
-          timerConfig = {
-            OnCalendar = cfg.renewInterval;
-            Unit = "acme-${cert}.service";
-            Persistent = "yes";
-            inherit AccuracySec;
-            # Skew randomly within the day, per https://letsencrypt.org/docs/integration-guide/.
-            RandomizedDelaySec = "24h";
-          };
-        })
-      );
-
-      systemd.targets.acme-selfsigned-certificates = mkIf cfg.preliminarySelfsigned {};
-      systemd.targets.acme-certificates = {};
-    })
+      ] ++ (builtins.concatLists (mapAttrsToList (cert: data: [
+        {
+          assertion = data.user == "_mkRemovedOptionModule";
+          message = ''
+            The option definition `security.acme.certs.${cert}.user' no longer has any effect; Please remove it.
+            Certificate user is now hard coded to the "acme" user. If you would
+            like another user to have access, consider adding them to the
+            "acme" group or changing security.acme.certs.${cert}.group.
+          '';
+        }
+        {
+          assertion = data.allowKeysForGroup == "_mkRemovedOptionModule";
+          message = ''
+            The option definition `security.acme.certs.${cert}.allowKeysForGroup' no longer has any effect; Please remove it.
+            All certs are readable by the configured group. If this is undesired,
+            consider changing security.acme.certs.${cert}.group to an unused group.
+          '';
+        }
+        # * in the cert value breaks building of systemd services, and makes
+        # referencing them as a user quite weird too. Best practice is to use
+        # the domain option.
+        {
+          assertion = ! hasInfix "*" cert;
+          message = ''
+            The cert option path `security.acme.certs.${cert}.dnsProvider`
+            cannot contain a * character.
+            Instead, set `security.acme.certs.${cert}.domain = "${cert}";`
+            and remove the wildcard from the path.
+          '';
+        }
+        {
+          assertion = data.dnsProvider == null || data.webroot == null;
+          message = ''
+            Options `security.acme.certs.${cert}.dnsProvider` and
+            `security.acme.certs.${cert}.webroot` are mutually exclusive.
+          '';
+        }
+      ]) cfg.certs));
 
+      users.users.acme = {
+        home = "/var/lib/acme";
+        group = "acme";
+        isSystemUser = true;
+      };
+
+      users.groups.acme = {};
+
+      systemd.services = {
+        "acme-fixperms" = userMigrationService;
+      } // (mapAttrs' (cert: conf: nameValuePair "acme-${cert}" conf.renewService) certConfigs)
+        // (optionalAttrs (cfg.preliminarySelfsigned) ({
+        "acme-selfsigned-ca" = selfsignCAService;
+      } // (mapAttrs' (cert: conf: nameValuePair "acme-selfsigned-${cert}" conf.selfsignService) certConfigs)));
+
+      systemd.timers = mapAttrs' (cert: conf: nameValuePair "acme-${cert}" conf.renewTimer) certConfigs;
+
+      # .lego and .lego/accounts specified to fix any incorrect permissions
+      systemd.tmpfiles.rules = [
+        "d /var/lib/acme/.lego - acme acme"
+        "d /var/lib/acme/.lego/accounts - acme acme"
+      ] ++ (unique (concatMap (conf: [
+          "d ${conf.accountDir} - acme acme"
+        ] ++ (optional (conf.webroot != null) "d ${conf.webroot}/.well-known/acme-challenge - acme ${conf.group}")
+      ) (attrValues certConfigs)));
+
+      # Create some targets which can be depended on to be "active" after cert renewals
+      systemd.targets = mapAttrs' (cert: conf: nameValuePair "acme-finished-${cert}" {
+        wantedBy = [ "default.target" ];
+        requires = [ "acme-${cert}.service" ] ++ conf.selfsignedDeps;
+        after = [ "acme-${cert}.service" ] ++ conf.selfsignedDeps;
+      }) certConfigs;
+    })
   ];
 
   meta = {
diff --git a/nixos/modules/security/acme.xml b/nixos/modules/security/acme.xml
index f802faee9749..17e94bc12fb2 100644
--- a/nixos/modules/security/acme.xml
+++ b/nixos/modules/security/acme.xml
@@ -72,7 +72,7 @@ services.nginx = {
     "foo.example.com" = {
       <link linkend="opt-services.nginx.virtualHosts._name_.forceSSL">forceSSL</link> = true;
       <link linkend="opt-services.nginx.virtualHosts._name_.enableACME">enableACME</link> = true;
-      # All serverAliases will be added as <link linkend="opt-security.acme.certs._name_.extraDomains">extra domains</link> on the certificate.
+      # All serverAliases will be added as <link linkend="opt-security.acme.certs._name_.extraDomainNames">extra domain names</link> on the certificate.
       <link linkend="opt-services.nginx.virtualHosts._name_.serverAliases">serverAliases</link> = [ "bar.example.com" ];
       locations."/" = {
         <link linkend="opt-services.nginx.virtualHosts._name_.locations._name_.root">root</link> = "/var/www";
@@ -80,8 +80,8 @@ services.nginx = {
     };
 
     # We can also add a different vhost and reuse the same certificate
-    # but we have to append extraDomains manually.
-    <link linkend="opt-security.acme.certs._name_.extraDomains">security.acme.certs."foo.example.com".extraDomains."baz.example.com"</link> = null;
+    # but we have to append extraDomainNames manually.
+    <link linkend="opt-security.acme.certs._name_.extraDomainNames">security.acme.certs."foo.example.com".extraDomainNames</link> = [ "baz.example.com" ];
     "baz.example.com" = {
       <link linkend="opt-services.nginx.virtualHosts._name_.forceSSL">forceSSL</link> = true;
       <link linkend="opt-services.nginx.virtualHosts._name_.useACMEHost">useACMEHost</link> = "foo.example.com";
@@ -165,7 +165,7 @@ services.httpd = {
   # Since we have a wildcard vhost to handle port 80,
   # we can generate certs for anything!
   # Just make sure your DNS resolves them.
-  <link linkend="opt-security.acme.certs._name_.extraDomains">extraDomains</link> = [ "mail.example.com" ];
+  <link linkend="opt-security.acme.certs._name_.extraDomainNames">extraDomainNames</link> = [ "mail.example.com" ];
 };
 </programlisting>
 
@@ -251,4 +251,16 @@ chmod 400 /var/lib/secrets/certs.secret
    journalctl -fu acme-example.com.service</literal> and watching its log output.
   </para>
  </section>
+ <section xml:id="module-security-acme-regenerate">
+  <title>Regenerating certificates</title>
+
+  <para>
+   Should you need to regenerate a particular certificate in a hurry, such
+   as when a vulnerability is found in Let's Encrypt, there is now a convenient
+   mechanism for doing so. Running <literal>systemctl clean acme-example.com.service</literal>
+   will remove all certificate files for the given domain, allowing you to then
+   <literal>systemctl start acme-example.com.service</literal> to generate fresh
+   ones.
+  </para>
+ </section>
 </chapter>
diff --git a/nixos/modules/services/backup/borgbackup.xml b/nixos/modules/services/backup/borgbackup.xml
index bef7db608f82..a197f38ffb9d 100644
--- a/nixos/modules/services/backup/borgbackup.xml
+++ b/nixos/modules/services/backup/borgbackup.xml
@@ -197,26 +197,8 @@ sudo borg init --encryption=repokey-blake2  \
     disk failure, ransomware and theft.
   </para>
   <para>
-    It is available as a flatpak package. To enable it you must set the
-    following two configuration items.
-  </para>
-  <para>
-    <programlisting>
-services.flatpak.enable = true ;
-# next line is needed to avoid the Error
-# Error deploying: GDBus.Error:org.freedesktop.DBus.Error.ServiceUnknown:
-services.accounts-daemon.enable = true;
-    </programlisting>
-  </para>
-  <para>As a normal user you must first install, then run vorta using the
-    following commands:
-    <programlisting>
-flatpak remote-add --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
-flatpak install flathub com.borgbase.Vorta
-flatpak run --branch=stable --arch=x86_64 --command=vorta com.borgbase.Vorta
-</programlisting>
-    After running <code>flatpak install</code> you can start Vorta also via
-        the KDE application menu.
+   It can be installed in NixOS e.g. by adding <package>pkgs.vorta</package>
+   to <xref linkend="opt-environment.systemPackages" />.
   </para>
   <para>
     Details about using Vorta can be found under <link
diff --git a/nixos/modules/services/development/lorri.nix b/nixos/modules/services/development/lorri.nix
index c843aa56d133..fc576e4c18ba 100644
--- a/nixos/modules/services/development/lorri.nix
+++ b/nixos/modules/services/development/lorri.nix
@@ -15,6 +15,15 @@ in {
           issued by the `lorri` command.
         '';
       };
+      package = lib.mkOption {
+        default = pkgs.lorri;
+        type = lib.types.package;
+        description = ''
+          The lorri package to use.
+        '';
+        defaultText = lib.literalExample "pkgs.lorri";
+        example = lib.literalExample "pkgs.lorri";
+      };
     };
   };
 
@@ -34,7 +43,7 @@ in {
       after = [ "lorri.socket" ];
       path = with pkgs; [ config.nix.package git gnutar gzip ];
       serviceConfig = {
-        ExecStart = "${pkgs.lorri}/bin/lorri daemon";
+        ExecStart = "${cfg.package}/bin/lorri daemon";
         PrivateTmp = true;
         ProtectSystem = "strict";
         ProtectHome = "read-only";
@@ -42,6 +51,6 @@ in {
       };
     };
 
-    environment.systemPackages = [ pkgs.lorri ];
+    environment.systemPackages = [ cfg.package ];
   };
 }
diff --git a/nixos/modules/services/games/terraria.nix b/nixos/modules/services/games/terraria.nix
index 413660321ec3..34c8ff137d6a 100644
--- a/nixos/modules/services/games/terraria.nix
+++ b/nixos/modules/services/games/terraria.nix
@@ -25,7 +25,7 @@ let
       exit 0
     fi
 
-    ${getBin pkgs.tmux}/bin/tmux -S /var/lib/terraria/terraria.sock send-keys Enter exit Enter
+    ${getBin pkgs.tmux}/bin/tmux -S ${cfg.dataDir}/terraria.sock send-keys Enter exit Enter
     ${getBin pkgs.coreutils}/bin/tail --pid="$1" -f /dev/null
   '';
 in
@@ -36,7 +36,7 @@ in
         type        = types.bool;
         default     = false;
         description = ''
-          If enabled, starts a Terraria server. The server can be connected to via <literal>tmux -S /var/lib/terraria/terraria.sock attach</literal>
+          If enabled, starts a Terraria server. The server can be connected to via <literal>tmux -S ${cfg.dataDir}/terraria.sock attach</literal>
           for administration by users who are a part of the <literal>terraria</literal> group (use <literal>C-b d</literal> shortcut to detach again).
         '';
       };
@@ -111,13 +111,19 @@ in
         default     = false;
         description = "Disables automatic Universal Plug and Play.";
       };
+      dataDir = mkOption {
+        type        = types.str;
+        default     = "/var/lib/terraria";
+        example     = "/srv/terraria";
+        description = "Path to variable state data directory for terraria.";
+      };
     };
   };
 
   config = mkIf cfg.enable {
     users.users.terraria = {
       description = "Terraria server service user";
-      home        = "/var/lib/terraria";
+      home        = cfg.dataDir;
       createHome  = true;
       uid         = config.ids.uids.terraria;
     };
@@ -136,13 +142,13 @@ in
         User    = "terraria";
         Type = "forking";
         GuessMainPID = true;
-        ExecStart = "${getBin pkgs.tmux}/bin/tmux -S /var/lib/terraria/terraria.sock new -d ${pkgs.terraria-server}/bin/TerrariaServer ${concatStringsSep " " flags}";
+        ExecStart = "${getBin pkgs.tmux}/bin/tmux -S ${cfg.dataDir}/terraria.sock new -d ${pkgs.terraria-server}/bin/TerrariaServer ${concatStringsSep " " flags}";
         ExecStop = "${stopScript} $MAINPID";
       };
 
       postStart = ''
-        ${pkgs.coreutils}/bin/chmod 660 /var/lib/terraria/terraria.sock
-        ${pkgs.coreutils}/bin/chgrp terraria /var/lib/terraria/terraria.sock
+        ${pkgs.coreutils}/bin/chmod 660 ${cfg.dataDir}/terraria.sock
+        ${pkgs.coreutils}/bin/chgrp terraria ${cfg.dataDir}/terraria.sock
       '';
     };
   };
diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix
index 425f35f37cb6..9896b8023e44 100644
--- a/nixos/modules/services/misc/gitlab.nix
+++ b/nixos/modules/services/misc/gitlab.nix
@@ -783,6 +783,23 @@ in {
       };
     };
 
+    systemd.services.gitlab-mailroom = mkIf (gitlabConfig.production.incoming_email.enabled or false) {
+      description = "GitLab incoming mail daemon";
+      after = [ "network.target" "redis.service" "gitlab.service" ]; # gitlab.service creates configs
+      wantedBy = [ "multi-user.target" ];
+      environment = gitlabEnv;
+      serviceConfig = {
+        Type = "simple";
+        TimeoutSec = "infinity";
+        Restart = "on-failure";
+
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStart = "${cfg.packages.gitlab.rubyEnv}/bin/bundle exec mail_room -c ${cfg.packages.gitlab}/share/gitlab/config.dist/mail_room.yml";
+        WorkingDirectory = gitlabEnv.HOME;
+      };
+    };
+
     systemd.services.gitlab = {
       after = [ "gitlab-workhorse.service" "gitaly.service" "network.target" "gitlab-postgresql.service" "redis.service" ];
       requires = [ "gitlab-sidekiq.service" ];
diff --git a/nixos/modules/services/misc/gitlab.xml b/nixos/modules/services/misc/gitlab.xml
index b6171a9a194c..19a3df0a5f66 100644
--- a/nixos/modules/services/misc/gitlab.xml
+++ b/nixos/modules/services/misc/gitlab.xml
@@ -98,6 +98,12 @@ services.gitlab = {
   </para>
 
   <para>
+    When <literal>icoming_mail.enabled</literal> is set to <literal>true</literal>
+    in <link linkend="opt-services.gitlab.extraConfig">extraConfig</link> an additional
+    service called <literal>gitlab-mailroom</literal> is enabled for fetching incoming mail.
+  </para>
+
+  <para>
    Refer to <xref linkend="ch-options" /> for all available configuration
    options for the
    <link linkend="opt-services.gitlab.enable">services.gitlab</link> module.
diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix
index 59748efe0ded..cc71451bf206 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -46,6 +46,7 @@ let
     "surfboard"
     "tor"
     "unifi"
+    "unifi-poller"
     "varnish"
     "wireguard"
   ] (name:
@@ -84,7 +85,8 @@ let
     };
     firewallFilter = mkOption {
       type = types.str;
-      default = "-p tcp -m tcp --dport ${toString port}";
+      default = "-p tcp -m tcp --dport ${toString cfg.${name}.port}";
+      defaultText = "-p tcp -m tcp --dport ${toString port}";
       example = literalExample ''
         "-i eth0 -p tcp -m tcp --dport ${toString port}"
       '';
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix b/nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix
new file mode 100644
index 000000000000..394e6e201f03
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix
@@ -0,0 +1,34 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.unifi-poller;
+
+  configFile = pkgs.writeText "prometheus-unifi-poller-exporter.json" (generators.toJSON {} {
+    poller = { inherit (cfg.log) debug quiet; };
+    unifi = { inherit (cfg) controllers; };
+    influxdb.disable = true;
+    prometheus = {
+      http_listen = "${cfg.listenAddress}:${toString cfg.port}";
+      report_errors = cfg.log.prometheusErrors;
+    };
+  });
+
+in {
+  port = 9130;
+
+  extraOpts = {
+    inherit (options.services.unifi-poller.unifi) controllers;
+    log = {
+      debug = mkEnableOption "debug logging including line numbers, high resolution timestamps, per-device logs.";
+      quiet = mkEnableOption "startup and error logs only.";
+      prometheusErrors = mkEnableOption "emitting errors to prometheus.";
+    };
+  };
+
+  serviceOpts.serviceConfig = {
+    ExecStart = "${pkgs.unifi-poller}/bin/unifi-poller --config ${configFile}";
+    DynamicUser = false;
+  };
+}
diff --git a/nixos/modules/services/monitoring/unifi-poller.nix b/nixos/modules/services/monitoring/unifi-poller.nix
new file mode 100644
index 000000000000..208f5e4875b4
--- /dev/null
+++ b/nixos/modules/services/monitoring/unifi-poller.nix
@@ -0,0 +1,242 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.unifi-poller;
+
+  configFile = pkgs.writeText "unifi-poller.json" (generators.toJSON {} {
+    inherit (cfg) poller influxdb prometheus unifi;
+  });
+
+in {
+  options.services.unifi-poller = {
+    enable = mkEnableOption "unifi-poller";
+
+    poller = {
+      debug = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Turns on line numbers, microsecond logging, and a per-device log.
+          This may be noisy if you have a lot of devices. It adds one line per device.
+        '';
+      };
+      quiet = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Turns off per-interval logs. Only startup and error logs will be emitted.
+        '';
+      };
+      plugins = mkOption {
+        type = with types; listOf str;
+        default = [];
+        description = ''
+          Load additional plugins.
+        '';
+      };
+    };
+
+    prometheus = {
+      disable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to disable the prometheus ouput plugin.
+        '';
+      };
+      http_listen = mkOption {
+        type = types.str;
+        default = "[::]:9130";
+        description = ''
+          Bind the prometheus exporter to this IP or hostname.
+        '';
+      };
+      report_errors = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to report errors.
+        '';
+      };
+    };
+
+    influxdb = {
+      disable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to disable the influxdb ouput plugin.
+        '';
+      };
+      url = mkOption {
+        type = types.str;
+        default = "http://127.0.0.1:8086";
+        description = ''
+          URL of the influxdb host.
+        '';
+      };
+      user = mkOption {
+        type = types.str;
+        default = "unifipoller";
+        description = ''
+          Username for the influxdb.
+        '';
+      };
+      pass = mkOption {
+        type = types.path;
+        default = pkgs.writeText "unifi-poller-influxdb-default.password" "unifipoller";
+        defaultText = "unifi-poller-influxdb-default.password";
+        description = ''
+          Path of a file containing the password for influxdb.
+          This file needs to be readable by the unifi-poller user.
+        '';
+        apply = v: "file://${v}";
+      };
+      db = mkOption {
+        type = types.str;
+        default = "unifi";
+        description = ''
+          Database name. Database should exist.
+        '';
+      };
+      verify_ssl = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Verify the influxdb's certificate.
+        '';
+      };
+      interval = mkOption {
+        type = types.str;
+        default = "30s";
+        description = ''
+          Setting this lower than the Unifi controller's refresh
+          interval may lead to zeroes in your database.
+        '';
+      };
+    };
+
+    unifi = let
+      controllerOptions = {
+        user = mkOption {
+          type = types.str;
+          default = "unifi";
+          description = ''
+            Unifi service user name.
+          '';
+        };
+        pass = mkOption {
+          type = types.path;
+          default = pkgs.writeText "unifi-poller-unifi-default.password" "unifi";
+          defaultText = "unifi-poller-unifi-default.password";
+          description = ''
+            Path of a file containing the password for the unifi service user.
+            This file needs to be readable by the unifi-poller user.
+          '';
+          apply = v: "file://${v}";
+        };
+        url = mkOption {
+          type = types.str;
+          default = "https://unifi:8443";
+          description = ''
+            URL of the Unifi controller.
+          '';
+        };
+        sites = mkOption {
+          type = with types; either (enum [ "default" "all" ]) (listOf str);
+          default = "all";
+          description = ''
+            List of site names for which statistics should be exported.
+            Or the string "default" for the default site or the string "all" for all sites.
+          '';
+          apply = toList;
+        };
+        save_ids = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Collect and save data from the intrusion detection system to influxdb.
+          '';
+        };
+        save_dpi = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Collect and save data from deep packet inspection.
+            Adds around 150 data points and impacts performance.
+          '';
+        };
+        save_sites = mkOption {
+          type = types.bool;
+          default = true;
+          description = ''
+            Collect and save site data.
+          '';
+        };
+        hash_pii = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Hash, with md5, client names and MAC addresses. This attempts
+            to protect personally identifiable information.
+          '';
+        };
+        verify_ssl = mkOption {
+          type = types.bool;
+          default = true;
+          description = ''
+            Verify the Unifi controller's certificate.
+          '';
+        };
+      };
+
+    in {
+      dynamic = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Let prometheus select which controller to poll when scraping.
+          Use with default credentials. See unifi-poller wiki for more.
+        '';
+      };
+
+      defaults = controllerOptions;
+
+      controllers = mkOption {
+        type = with types; listOf (submodule { options = controllerOptions; });
+        default = [];
+        description = ''
+          List of Unifi controllers to poll. Use defaults if empty.
+        '';
+        apply = map (flip removeAttrs [ "_module" ]);
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.groups.unifi-poller = { };
+    users.users.unifi-poller = {
+      description = "unifi-poller Service User";
+      group = "unifi-poller";
+      isSystemUser = true;
+    };
+
+    systemd.services.unifi-poller = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.unifi-poller}/bin/unifi-poller --config ${configFile}";
+        Restart = "always";
+        PrivateTmp = true;
+        ProtectHome = true;
+        ProtectSystem = "full";
+        DevicePolicy = "closed";
+        NoNewPrivileges = true;
+        User = "unifi-poller";
+        WorkingDirectory = "/tmp";
+      };
+    };
+  };
+}
diff --git a/nixos/modules/services/network-filesystems/cachefilesd.nix b/nixos/modules/services/network-filesystems/cachefilesd.nix
index 619813408405..229c9665419f 100644
--- a/nixos/modules/services/network-filesystems/cachefilesd.nix
+++ b/nixos/modules/services/network-filesystems/cachefilesd.nix
@@ -43,17 +43,21 @@ in
 
   config = mkIf cfg.enable {
 
+    boot.kernelModules = [ "cachefiles" ];
+
     systemd.services.cachefilesd = {
       description = "Local network file caching management daemon";
       wantedBy = [ "multi-user.target" ];
-      path = [ pkgs.kmod pkgs.cachefilesd ];
-      script = ''
-        modprobe -qab cachefiles
-        mkdir -p ${cfg.cacheDir}
-        chmod 700 ${cfg.cacheDir}
-        exec cachefilesd -n -f ${cfgFile}
-      '';
+      serviceConfig = {
+        Type = "exec";
+        ExecStart = "${pkgs.cachefilesd}/bin/cachefilesd -n -f ${cfgFile}";
+        Restart = "on-failure";
+        PrivateTmp = true;
+      };
     };
 
+    systemd.tmpfiles.rules = [
+      "d ${cfg.cacheDir} 0700 root root - -"
+    ];
   };
 }
diff --git a/nixos/modules/services/networking/prosody.xml b/nixos/modules/services/networking/prosody.xml
index e3fedf8a67e1..471240cd1475 100644
--- a/nixos/modules/services/networking/prosody.xml
+++ b/nixos/modules/services/networking/prosody.xml
@@ -65,7 +65,7 @@ services.prosody = {
    you'll need a single TLS certificate covering your main endpoint,
    the MUC one as well as the HTTP Upload one. We can generate such a
    certificate by leveraging the ACME
-   <link linkend="opt-security.acme.certs._name_.extraDomains">extraDomains</link> module option.
+   <link linkend="opt-security.acme.certs._name_.extraDomainNames">extraDomainNames</link> module option.
  </para>
  <para>
    Provided the setup detailed in the previous section, you'll need the following acme configuration to generate
@@ -78,8 +78,7 @@ security.acme = {
     "example.org" = {
       <link linkend="opt-security.acme.certs._name_.webroot">webroot</link> = "/var/www/example.org";
       <link linkend="opt-security.acme.certs._name_.email">email</link> = "root@example.org";
-      <link linkend="opt-security.acme.certs._name_.extraDomains">extraDomains."conference.example.org"</link> = null;
-      <link linkend="opt-security.acme.certs._name_.extraDomains">extraDomains."upload.example.org"</link> = null;
+      <link linkend="opt-security.acme.certs._name_.extraDomainNames">extraDomainNames</link> = [ "conference.example.org" "upload.example.org" ];
     };
   };
 };</programlisting>
diff --git a/nixos/modules/services/networking/ssh/sshd.nix b/nixos/modules/services/networking/ssh/sshd.nix
index 6b27cbc56bd9..5365b8b9b107 100644
--- a/nixos/modules/services/networking/ssh/sshd.nix
+++ b/nixos/modules/services/networking/ssh/sshd.nix
@@ -232,6 +232,14 @@ in
         '';
       };
 
+      banner = mkOption {
+        type = types.nullOr types.lines;
+        default = null;
+        description = ''
+          Message to display to the remote user before authentication is allowed.
+        '';
+      };
+
       authorizedKeysFiles = mkOption {
         type = types.listOf types.str;
         default = [];
@@ -474,6 +482,8 @@ in
       ''
         UsePAM yes
 
+        Banner ${if cfg.banner == null then "none" else pkgs.writeText "ssh_banner" cfg.banner}
+
         AddressFamily ${if config.networking.enableIPv6 then "any" else "inet"}
         ${concatMapStrings (port: ''
           Port ${toString port}
diff --git a/nixos/modules/services/networking/syncthing.nix b/nixos/modules/services/networking/syncthing.nix
index e717d78feed5..28348c7893a0 100644
--- a/nixos/modules/services/networking/syncthing.nix
+++ b/nixos/modules/services/networking/syncthing.nix
@@ -18,6 +18,7 @@ let
     fsWatcherEnabled = folder.watch;
     fsWatcherDelayS = folder.watchDelay;
     ignorePerms = folder.ignorePerms;
+    ignoreDelete = folder.ignoreDelete;
     versioning = folder.versioning;
   }) (filterAttrs (
     _: folder:
@@ -284,8 +285,6 @@ in {
                 });
               };
 
-
-
               rescanInterval = mkOption {
                 type = types.int;
                 default = 3600;
@@ -327,6 +326,16 @@ in {
                 '';
               };
 
+              ignoreDelete = mkOption {
+                type = types.bool;
+                default = false;
+                description = ''
+                  Whether to delete files in destination. See <link
+                  xlink:href="https://docs.syncthing.net/advanced/folder-ignoredelete.html">
+                  upstream's docs</link>.
+                '';
+              };
+
             };
           }));
         };
diff --git a/nixos/modules/services/web-servers/apache-httpd/default.nix b/nixos/modules/services/web-servers/apache-httpd/default.nix
index fc4c2945394c..6dd1c85132c9 100644
--- a/nixos/modules/services/web-servers/apache-httpd/default.nix
+++ b/nixos/modules/services/web-servers/apache-httpd/default.nix
@@ -6,6 +6,8 @@ let
 
   cfg = config.services.httpd;
 
+  certs = config.security.acme.certs;
+
   runtimeDir = "/run/httpd";
 
   pkg = cfg.package.out;
@@ -26,6 +28,13 @@ let
 
   vhosts = attrValues cfg.virtualHosts;
 
+  # certName is used later on to determine systemd service names.
+  acmeEnabledVhosts = map (hostOpts: hostOpts // {
+    certName = if hostOpts.useACMEHost != null then hostOpts.useACMEHost else hostOpts.hostName;
+  }) (filter (hostOpts: hostOpts.enableACME || hostOpts.useACMEHost != null) vhosts);
+
+  dependentCertNames = unique (map (hostOpts: hostOpts.certName) acmeEnabledVhosts);
+
   mkListenInfo = hostOpts:
     if hostOpts.listen != [] then hostOpts.listen
     else (
@@ -125,13 +134,13 @@ let
 
       useACME = hostOpts.enableACME || hostOpts.useACMEHost != null;
       sslCertDir =
-        if hostOpts.enableACME then config.security.acme.certs.${hostOpts.hostName}.directory
-        else if hostOpts.useACMEHost != null then config.security.acme.certs.${hostOpts.useACMEHost}.directory
+        if hostOpts.enableACME then certs.${hostOpts.hostName}.directory
+        else if hostOpts.useACMEHost != null then certs.${hostOpts.useACMEHost}.directory
         else abort "This case should never happen.";
 
-      sslServerCert = if useACME then "${sslCertDir}/full.pem" else hostOpts.sslServerCert;
+      sslServerCert = if useACME then "${sslCertDir}/fullchain.pem" else hostOpts.sslServerCert;
       sslServerKey = if useACME then "${sslCertDir}/key.pem" else hostOpts.sslServerKey;
-      sslServerChain = if useACME then "${sslCertDir}/fullchain.pem" else hostOpts.sslServerChain;
+      sslServerChain = if useACME then "${sslCertDir}/chain.pem" else hostOpts.sslServerChain;
 
       acmeChallenge = optionalString useACME ''
         Alias /.well-known/acme-challenge/ "${hostOpts.acmeRoot}/.well-known/acme-challenge/"
@@ -347,7 +356,6 @@ let
       cat ${php.phpIni} > $out
       echo "$options" >> $out
     '';
-
 in
 
 
@@ -647,14 +655,17 @@ in
       wwwrun.gid = config.ids.gids.wwwrun;
     };
 
-    security.acme.certs = mapAttrs (name: hostOpts: {
-      user = cfg.user;
-      group = mkDefault cfg.group;
-      email = if hostOpts.adminAddr != null then hostOpts.adminAddr else cfg.adminAddr;
-      webroot = hostOpts.acmeRoot;
-      extraDomains = genAttrs hostOpts.serverAliases (alias: null);
-      postRun = "systemctl reload httpd.service";
-    }) (filterAttrs (name: hostOpts: hostOpts.enableACME) cfg.virtualHosts);
+    security.acme.certs = let
+      acmePairs = map (hostOpts: nameValuePair hostOpts.hostName {
+        group = mkDefault cfg.group;
+        webroot = hostOpts.acmeRoot;
+        extraDomainNames = hostOpts.serverAliases;
+        # Use the vhost-specific email address if provided, otherwise let
+        # security.acme.email or security.acme.certs.<cert>.email be used.
+        email = mkOverride 2000 (if hostOpts.adminAddr != null then hostOpts.adminAddr else cfg.adminAddr);
+      # Filter for enableACME-only vhosts. Don't want to create dud certs
+      }) (filter (hostOpts: hostOpts.useACMEHost == null) acmeEnabledVhosts);
+    in listToAttrs acmePairs;
 
     environment.systemPackages = [
       apachectl
@@ -724,16 +735,12 @@ in
           "Z '${cfg.logDir}' - ${svc.User} ${svc.Group}"
         ];
 
-    systemd.services.httpd =
-      let
-        vhostsACME = filter (hostOpts: hostOpts.enableACME) vhosts;
-      in
-      { description = "Apache HTTPD";
-
+    systemd.services.httpd = {
+        description = "Apache HTTPD";
         wantedBy = [ "multi-user.target" ];
-        wants = concatLists (map (hostOpts: [ "acme-${hostOpts.hostName}.service" "acme-selfsigned-${hostOpts.hostName}.service" ]) vhostsACME);
-        after = [ "network.target" "fs.target" ] ++ map (hostOpts: "acme-selfsigned-${hostOpts.hostName}.service") vhostsACME;
-        before = map (hostOpts: "acme-${hostOpts.hostName}.service") vhostsACME;
+        wants = concatLists (map (certName: [ "acme-finished-${certName}.target" ]) dependentCertNames);
+        after = [ "network.target" ] ++ map (certName: "acme-selfsigned-${certName}.service") dependentCertNames;
+        before = map (certName: "acme-${certName}.service") dependentCertNames;
 
         path = [ pkg pkgs.coreutils pkgs.gnugrep ];
 
@@ -767,5 +774,31 @@ in
         };
       };
 
+    # postRun hooks on cert renew can't be used to restart Apache since renewal
+    # runs as the unprivileged acme user. sslTargets are added to wantedBy + before
+    # which allows the acme-finished-$cert.target to signify the successful updating
+    # of certs end-to-end.
+    systemd.services.httpd-config-reload = let
+      sslServices = map (certName: "acme-${certName}.service") dependentCertNames;
+      sslTargets = map (certName: "acme-finished-${certName}.target") dependentCertNames;
+    in mkIf (sslServices != []) {
+      wantedBy = sslServices ++ [ "multi-user.target" ];
+      # Before the finished targets, after the renew services.
+      # This service might be needed for HTTP-01 challenges, but we only want to confirm
+      # certs are updated _after_ config has been reloaded.
+      before = sslTargets;
+      after = sslServices;
+      # Block reloading if not all certs exist yet.
+      # Happens when config changes add new vhosts/certs.
+      unitConfig.ConditionPathExists = map (certName: certs.${certName}.directory + "/fullchain.pem") dependentCertNames;
+      serviceConfig = {
+        Type = "oneshot";
+        TimeoutSec = 60;
+        ExecCondition = "/run/current-system/systemd/bin/systemctl -q is-active httpd.service";
+        ExecStartPre = "${pkg}/bin/httpd -f ${httpdConf} -t";
+        ExecStart = "/run/current-system/systemd/bin/systemctl reload httpd.service";
+      };
+    };
+
   };
 }
diff --git a/nixos/modules/services/web-servers/nginx/default.nix b/nixos/modules/services/web-servers/nginx/default.nix
index 461888c4cc4f..975b56d47822 100644
--- a/nixos/modules/services/web-servers/nginx/default.nix
+++ b/nixos/modules/services/web-servers/nginx/default.nix
@@ -6,23 +6,23 @@ let
   cfg = config.services.nginx;
   certs = config.security.acme.certs;
   vhostsConfigs = mapAttrsToList (vhostName: vhostConfig: vhostConfig) virtualHosts;
-  acmeEnabledVhosts = filter (vhostConfig: vhostConfig.enableACME && vhostConfig.useACMEHost == null) vhostsConfigs;
+  acmeEnabledVhosts = filter (vhostConfig: vhostConfig.enableACME || vhostConfig.useACMEHost != null) vhostsConfigs;
+  dependentCertNames = unique (map (hostOpts: hostOpts.certName) acmeEnabledVhosts);
   virtualHosts = mapAttrs (vhostName: vhostConfig:
     let
       serverName = if vhostConfig.serverName != null
         then vhostConfig.serverName
         else vhostName;
+      certName = if vhostConfig.useACMEHost != null
+        then vhostConfig.useACMEHost
+        else serverName;
     in
     vhostConfig // {
-      inherit serverName;
-    } // (optionalAttrs vhostConfig.enableACME {
-      sslCertificate = "${certs.${serverName}.directory}/fullchain.pem";
-      sslCertificateKey = "${certs.${serverName}.directory}/key.pem";
-      sslTrustedCertificate = "${certs.${serverName}.directory}/full.pem";
-    }) // (optionalAttrs (vhostConfig.useACMEHost != null) {
-      sslCertificate = "${certs.${vhostConfig.useACMEHost}.directory}/fullchain.pem";
-      sslCertificateKey = "${certs.${vhostConfig.useACMEHost}.directory}/key.pem";
-      sslTrustedCertificate = "${certs.${vhostConfig.useACMEHost}.directory}/fullchain.pem";
+      inherit serverName certName;
+    } // (optionalAttrs (vhostConfig.enableACME || vhostConfig.useACMEHost != null) {
+      sslCertificate = "${certs.${certName}.directory}/fullchain.pem";
+      sslCertificateKey = "${certs.${certName}.directory}/key.pem";
+      sslTrustedCertificate = "${certs.${certName}.directory}/chain.pem";
     })
   ) cfg.virtualHosts;
   enableIPv6 = config.networking.enableIPv6;
@@ -691,12 +691,12 @@ in
     systemd.services.nginx = {
       description = "Nginx Web Server";
       wantedBy = [ "multi-user.target" ];
-      wants = concatLists (map (vhostConfig: ["acme-${vhostConfig.serverName}.service" "acme-selfsigned-${vhostConfig.serverName}.service"]) acmeEnabledVhosts);
-      after = [ "network.target" ] ++ map (vhostConfig: "acme-selfsigned-${vhostConfig.serverName}.service") acmeEnabledVhosts;
+      wants = concatLists (map (certName: [ "acme-finished-${certName}.target" ]) dependentCertNames);
+      after = [ "network.target" ] ++ map (certName: "acme-selfsigned-${certName}.service") dependentCertNames;
       # Nginx needs to be started in order to be able to request certificates
       # (it's hosting the acme challenge after all)
       # This fixes https://github.com/NixOS/nixpkgs/issues/81842
-      before = map (vhostConfig: "acme-${vhostConfig.serverName}.service") acmeEnabledVhosts;
+      before = map (certName: "acme-${certName}.service") dependentCertNames;
       stopIfChanged = false;
       preStart = ''
         ${cfg.preStart}
@@ -753,37 +753,41 @@ in
       source = configFile;
     };
 
-    systemd.services.nginx-config-reload = mkIf cfg.enableReload {
-      wants = [ "nginx.service" ];
-      wantedBy = [ "multi-user.target" ];
-      restartTriggers = [ configFile ];
-      # commented, because can cause extra delays during activate for this config:
-      #      services.nginx.virtualHosts."_".locations."/".proxyPass = "http://blabla:3000";
-      # stopIfChanged = false;
-      serviceConfig.Type = "oneshot";
-      serviceConfig.TimeoutSec = 60;
-      script = ''
-        if /run/current-system/systemd/bin/systemctl -q is-active nginx.service ; then
-          /run/current-system/systemd/bin/systemctl reload nginx.service
-        fi
-      '';
-      serviceConfig.RemainAfterExit = true;
+    # postRun hooks on cert renew can't be used to restart Nginx since renewal
+    # runs as the unprivileged acme user. sslTargets are added to wantedBy + before
+    # which allows the acme-finished-$cert.target to signify the successful updating
+    # of certs end-to-end.
+    systemd.services.nginx-config-reload = let
+      sslServices = map (certName: "acme-${certName}.service") dependentCertNames;
+      sslTargets = map (certName: "acme-finished-${certName}.target") dependentCertNames;
+    in mkIf (cfg.enableReload || sslServices != []) {
+      wants = optionals (cfg.enableReload) [ "nginx.service" ];
+      wantedBy = sslServices ++ [ "multi-user.target" ];
+      # Before the finished targets, after the renew services.
+      # This service might be needed for HTTP-01 challenges, but we only want to confirm
+      # certs are updated _after_ config has been reloaded.
+      before = sslTargets;
+      after = sslServices;
+      restartTriggers = optionals (cfg.enableReload) [ configFile ];
+      # Block reloading if not all certs exist yet.
+      # Happens when config changes add new vhosts/certs.
+      unitConfig.ConditionPathExists = optionals (sslServices != []) (map (certName: certs.${certName}.directory + "/fullchain.pem") dependentCertNames);
+      serviceConfig = {
+        Type = "oneshot";
+        TimeoutSec = 60;
+        ExecCondition = "/run/current-system/systemd/bin/systemctl -q is-active nginx.service";
+        ExecStart = "/run/current-system/systemd/bin/systemctl reload nginx.service";
+      };
     };
 
-    security.acme.certs = filterAttrs (n: v: v != {}) (
-      let
-        acmePairs = map (vhostConfig: { name = vhostConfig.serverName; value = {
-            user = cfg.user;
-            group = lib.mkDefault cfg.group;
-            webroot = vhostConfig.acmeRoot;
-            extraDomains = genAttrs vhostConfig.serverAliases (alias: null);
-            postRun = ''
-              /run/current-system/systemd/bin/systemctl reload nginx
-            '';
-          }; }) acmeEnabledVhosts;
-      in
-        listToAttrs acmePairs
-    );
+    security.acme.certs = let
+      acmePairs = map (vhostConfig: nameValuePair vhostConfig.serverName {
+        group = mkDefault cfg.group;
+        webroot = vhostConfig.acmeRoot;
+        extraDomainNames = vhostConfig.serverAliases;
+      # Filter for enableACME-only vhosts. Don't want to create dud certs
+      }) (filter (vhostConfig: vhostConfig.useACMEHost == null) acmeEnabledVhosts);
+    in listToAttrs acmePairs;
 
     users.users = optionalAttrs (cfg.user == "nginx") {
       nginx = {
diff --git a/nixos/modules/services/web-servers/phpfpm/default.nix b/nixos/modules/services/web-servers/phpfpm/default.nix
index d090885a8ca5..759eebf768db 100644
--- a/nixos/modules/services/web-servers/phpfpm/default.nix
+++ b/nixos/modules/services/web-servers/phpfpm/default.nix
@@ -277,6 +277,7 @@ in {
           ExecReload = "${pkgs.coreutils}/bin/kill -USR2 $MAINPID";
           RuntimeDirectory = "phpfpm";
           RuntimeDirectoryPreserve = true; # Relevant when multiple processes are running
+          Restart = "always";
         };
       }
     ) cfg.pools;
diff --git a/nixos/modules/services/x11/desktop-managers/cinnamon.nix b/nixos/modules/services/x11/desktop-managers/cinnamon.nix
new file mode 100644
index 000000000000..a9c059125e8d
--- /dev/null
+++ b/nixos/modules/services/x11/desktop-managers/cinnamon.nix
@@ -0,0 +1,205 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.services.xserver.desktopManager.cinnamon;
+  serviceCfg = config.services.cinnamon;
+
+  nixos-gsettings-overrides = pkgs.cinnamon.cinnamon-gsettings-overrides.override {
+    extraGSettingsOverridePackages = cfg.extraGSettingsOverridePackages;
+    extraGSettingsOverrides = cfg.extraGSettingsOverrides;
+  };
+
+in
+
+{
+  options = {
+    services.cinnamon = {
+      apps.enable = mkEnableOption "Cinnamon default applications";
+    };
+
+    services.xserver.desktopManager.cinnamon = {
+      enable = mkEnableOption "the cinnamon desktop manager";
+
+      sessionPath = mkOption {
+        default = [];
+        example = literalExample "[ pkgs.gnome3.gpaste ]";
+        description = ''
+          Additional list of packages to be added to the session search path.
+          Useful for GSettings-conditional autostart.
+
+          Note that this should be a last resort; patching the package is preferred (see GPaste).
+        '';
+      };
+
+      extraGSettingsOverrides = mkOption {
+        default = "";
+        type = types.lines;
+        description = "Additional gsettings overrides.";
+      };
+
+      extraGSettingsOverridePackages = mkOption {
+        default = [];
+        type = types.listOf types.path;
+        description = "List of packages for which gsettings are overridden.";
+      };
+    };
+
+    environment.cinnamon.excludePackages = mkOption {
+      default = [];
+      example = literalExample "[ pkgs.cinnamon.blueberry ]";
+      type = types.listOf types.package;
+      description = "Which packages cinnamon should exclude from the default environment";
+    };
+
+  };
+
+  config = mkMerge [
+    (mkIf (cfg.enable && config.services.xserver.displayManager.lightdm.enable && config.services.xserver.displayManager.lightdm.greeters.gtk.enable) {
+      services.xserver.displayManager.lightdm.greeters.gtk.extraConfig = mkDefault (builtins.readFile "${pkgs.cinnamon.mint-artwork}/etc/lightdm/lightdm-gtk-greeter.conf.d/99_linuxmint.conf");
+      })
+
+    (mkIf cfg.enable {
+      services.xserver.displayManager.sessionPackages = [ pkgs.cinnamon.cinnamon-common ];
+
+      services.xserver.displayManager.sessionCommands = ''
+        if test "$XDG_CURRENT_DESKTOP" = "Cinnamon"; then
+            true
+            ${concatMapStrings (p: ''
+              if [ -d "${p}/share/gsettings-schemas/${p.name}" ]; then
+                export XDG_DATA_DIRS=$XDG_DATA_DIRS''${XDG_DATA_DIRS:+:}${p}/share/gsettings-schemas/${p.name}
+              fi
+
+              if [ -d "${p}/lib/girepository-1.0" ]; then
+                export GI_TYPELIB_PATH=$GI_TYPELIB_PATH''${GI_TYPELIB_PATH:+:}${p}/lib/girepository-1.0
+                export LD_LIBRARY_PATH=$LD_LIBRARY_PATH''${LD_LIBRARY_PATH:+:}${p}/lib
+              fi
+            '') cfg.sessionPath}
+        fi
+      '';
+
+      # Default services
+      hardware.bluetooth.enable = mkDefault true;
+      hardware.pulseaudio.enable = mkDefault true;
+      security.polkit.enable = true;
+      services.accounts-daemon.enable = true;
+      services.system-config-printer.enable = (mkIf config.services.printing.enable (mkDefault true));
+      services.dbus.packages = with pkgs.cinnamon; [
+        cinnamon-common
+        cinnamon-screensaver
+        nemo
+        xapps
+      ];
+      services.cinnamon.apps.enable = mkDefault true;
+      services.gnome3.glib-networking.enable = true;
+      services.gnome3.gnome-keyring.enable = true;
+      services.gvfs.enable = true;
+      services.udisks2.enable = true;
+      services.upower.enable = mkDefault config.powerManagement.enable;
+      services.xserver.libinput.enable = mkDefault true;
+      services.xserver.updateDbusEnvironment = true;
+      networking.networkmanager.enable = mkDefault true;
+
+      # Enable colord server
+      services.colord.enable = true;
+
+      # Enable dconf
+      programs.dconf.enable = true;
+
+      # Enable org.a11y.Bus
+      services.gnome3.at-spi2-core.enable = true;
+
+      # Fix lockscreen
+      security.pam.services = {
+        cinnamon-screensaver = {};
+      };
+
+      environment.systemPackages = with pkgs.cinnamon // pkgs; [
+        desktop-file-utils
+        nixos-artwork.wallpapers.simple-dark-gray
+        onboard
+        sound-theme-freedesktop
+
+        # common-files
+        cinnamon-common
+        cinnamon-session
+        cinnamon-desktop
+        cinnamon-menus
+
+        # utils needed by some scripts
+        killall
+
+        # session requirements
+        cinnamon-screensaver
+        # cinnamon-killer-daemon: provided by cinnamon-common
+        gnome3.networkmanagerapplet # session requirement - also nm-applet not needed
+
+        # packages
+        nemo
+        cinnamon-control-center
+        cinnamon-settings-daemon
+        gnome3.libgnomekbd
+        orca
+
+        # theme
+        gnome3.adwaita-icon-theme
+        hicolor-icon-theme
+        gnome3.gnome-themes-extra
+        gtk3.out
+        mint-artwork
+        mint-themes
+        mint-x-icons
+        mint-y-icons
+        vanilla-dmz
+
+        # other
+        glib # for gsettings
+        shared-mime-info # for update-mime-database
+        xdg-user-dirs
+      ];
+
+      # Override GSettings schemas
+      environment.sessionVariables.NIX_GSETTINGS_OVERRIDES_DIR = "${nixos-gsettings-overrides}/share/gsettings-schemas/nixos-gsettings-overrides/glib-2.0/schemas";
+
+      environment.pathsToLink = [
+        # FIXME: modules should link subdirs of `/share` rather than relying on this
+        "/share" # TODO: https://github.com/NixOS/nixpkgs/issues/47173
+      ];
+
+      # Shell integration for VTE terminals
+      programs.bash.vteIntegration = mkDefault true;
+      programs.zsh.vteIntegration = mkDefault true;
+
+      # Harmonize Qt5 applications under Pantheon
+      qt5.enable = true;
+      qt5.platformTheme = "gnome";
+      qt5.style = "adwaita";
+
+      # Default Fonts
+      fonts.fonts = with pkgs; [
+        source-code-pro # Default monospace font in 3.32
+        ubuntu_font_family # required for default theme
+      ];
+    })
+
+    (mkIf serviceCfg.apps.enable {
+      programs.geary.enable = mkDefault true;
+      programs.gnome-disks.enable = mkDefault true;
+      programs.gnome-terminal.enable = mkDefault true;
+      programs.evince.enable = mkDefault true;
+      programs.file-roller.enable = mkDefault true;
+
+      environment.systemPackages = (with pkgs // pkgs.gnome3 // pkgs.cinnamon; pkgs.gnome3.removePackagesByName [
+        # cinnamon team apps
+        # warp
+        # blueberry
+
+        # external apps shipped with linux-mint
+        hexchat
+        gnome-calculator
+      ] config.environment.cinnamon.excludePackages);
+    })
+  ];
+}
diff --git a/nixos/modules/services/x11/desktop-managers/default.nix b/nixos/modules/services/x11/desktop-managers/default.nix
index 5d3a84d71399..f5559eb53541 100644
--- a/nixos/modules/services/x11/desktop-managers/default.nix
+++ b/nixos/modules/services/x11/desktop-managers/default.nix
@@ -21,6 +21,7 @@ in
     ./none.nix ./xterm.nix ./xfce.nix ./plasma5.nix ./lumina.nix
     ./lxqt.nix ./enlightenment.nix ./gnome3.nix ./kodi.nix
     ./mate.nix ./pantheon.nix ./surf-display.nix ./cde.nix
+    ./cinnamon.nix
   ];
 
   options = {
diff --git a/nixos/modules/services/x11/desktop-managers/plasma5.nix b/nixos/modules/services/x11/desktop-managers/plasma5.nix
index 75bf55a26396..869adee5552b 100644
--- a/nixos/modules/services/x11/desktop-managers/plasma5.nix
+++ b/nixos/modules/services/x11/desktop-managers/plasma5.nix
@@ -7,7 +7,8 @@ let
   xcfg = config.services.xserver;
   cfg = xcfg.desktopManager.plasma5;
 
-  inherit (pkgs) kdeApplications plasma5 libsForQt5 qt5;
+  inherit (pkgs) kdeApplications plasma5;
+  libsForQt5 = pkgs.libsForQt514;
   inherit (pkgs) writeText;
 
   pulseaudio = config.hardware.pulseaudio;
diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix
index 1b2f22a039cb..6823e12847c2 100644
--- a/nixos/modules/system/boot/stage-1.nix
+++ b/nixos/modules/system/boot/stage-1.nix
@@ -87,9 +87,7 @@ let
   # copy what we need.  Instead of using statically linked binaries,
   # we just copy what we need from Glibc and use patchelf to make it
   # work.
-  extraUtils = let
-    # Use lvm2 without udev support, which is the same lvm2 we already have in the closure anyways
-    lvm2 = pkgs.lvm2.override { udev = null; }; in pkgs.runCommandCC "extra-utils"
+  extraUtils = pkgs.runCommandCC "extra-utils"
     { nativeBuildInputs = [pkgs.buildPackages.nukeReferences];
       allowedReferences = [ "out" ]; # prevent accidents like glibc being included in the initrd
     }
@@ -113,8 +111,8 @@ let
       copy_bin_and_libs ${pkgs.utillinux}/sbin/blkid
 
       # Copy dmsetup and lvm.
-      copy_bin_and_libs ${getBin lvm2}/bin/dmsetup
-      copy_bin_and_libs ${getBin lvm2}/bin/lvm
+      copy_bin_and_libs ${getBin pkgs.lvm2}/bin/dmsetup
+      copy_bin_and_libs ${getBin pkgs.lvm2}/bin/lvm
 
       # Add RAID mdadm tool.
       copy_bin_and_libs ${pkgs.mdadm}/sbin/mdadm
diff --git a/nixos/modules/testing/test-instrumentation.nix b/nixos/modules/testing/test-instrumentation.nix
index 30ffb12cbade..c0ec76e8a3a3 100644
--- a/nixos/modules/testing/test-instrumentation.nix
+++ b/nixos/modules/testing/test-instrumentation.nix
@@ -74,15 +74,8 @@ with import ../../lib/qemu-flags.nix { inherit pkgs; };
         # OOM killer randomly get rid of processes, since this leads
         # to failures that are hard to diagnose.
         echo 2 > /proc/sys/vm/panic_on_oom
-
-        # Coverage data is written into /tmp/coverage-data.
-        mkdir -p /tmp/xchg/coverage-data
       '';
 
-    # If the kernel has been built with coverage instrumentation, make
-    # it available under /proc/gcov.
-    boot.kernelModules = [ "gcov-proc" ];
-
     # Panic if an error occurs in stage 1 (rather than waiting for
     # user intervention).
     boot.kernelParams =
@@ -111,8 +104,6 @@ with import ../../lib/qemu-flags.nix { inherit pkgs; };
     networking.defaultGateway = mkOverride 150 "";
     networking.nameservers = mkOverride 150 [ ];
 
-    systemd.globalEnvironment.GCOV_PREFIX = "/tmp/xchg/coverage-data";
-
     system.requiredKernelConfig = with config.lib.kernelConfig; [
       (isYes "SERIAL_8250_CONSOLE")
       (isYes "SERIAL_8250")