about summary refs log tree commit diff
path: root/nixpkgs/nixos/modules/services/monitoring
diff options
context:
space:
mode:
Diffstat (limited to 'nixpkgs/nixos/modules/services/monitoring')
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/alerta.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/apcupsd.nix17
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/arbtt.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/bosun.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/cadvisor.nix10
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/cockpit.nix231
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/collectd.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/das_watchdog.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/datadog-agent.nix14
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent.nix236
-rwxr-xr-xnixpkgs/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults9
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/do-agent.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/fusion-inventory.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/grafana-agent.nix52
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/grafana-image-renderer.nix42
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/grafana-reporter.nix6
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/grafana.nix1621
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/graphite.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/hdaps.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/heapster.nix6
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/incron.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/kapacitor.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/karma.nix128
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/kthxbye.nix166
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/loki.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/mackerel-agent.nix14
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/metricbeat.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/mimir.nix11
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/monit.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/nagios.nix26
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/netdata.nix18
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/parsedmarc.md5
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/parsedmarc.nix60
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/parsedmarc.xml125
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager-irc-relay.nix107
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager.nix30
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/default.nix136
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.md180
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix46
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.xml248
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/collectd.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix18
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/graphite.nix41
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/ipmi.nix41
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/kea.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/knot.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/mail.nix12
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/node.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nut.nix50
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/openldap.nix16
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pihole.nix36
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/postgres.nix10
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pve.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rtl_433.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/shelly.nix27
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix37
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/statsd.nix19
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/tor.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix34
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unpoller.nix37
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/v2ray.nix29
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/wireguard.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/zfs.nix44
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/pushgateway.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/sachet.nix88
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/prometheus/xmpp-alerts.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/riemann-tools.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/riemann.nix9
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/smartd.nix9
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/statsd.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/sysstat.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/teamviewer.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/telegraf.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/thanos.nix132
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/tremor-rs.nix129
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/tuptime.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/unpoller.nix (renamed from nixpkgs/nixos/modules/services/monitoring/unifi-poller.nix)30
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/ups.nix6
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/uptime-kuma.nix81
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/uptime.nix4
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/vmagent.nix100
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/vmalert.nix136
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/vnstat.nix2
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/zabbix-agent.nix6
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/zabbix-proxy.nix8
-rw-r--r--nixpkgs/nixos/modules/services/monitoring/zabbix-server.nix8
90 files changed, 3230 insertions, 1661 deletions
diff --git a/nixpkgs/nixos/modules/services/monitoring/alerta.nix b/nixpkgs/nixos/modules/services/monitoring/alerta.nix
index c0caa0dc3beb..6c7ebec4191c 100644
--- a/nixpkgs/nixos/modules/services/monitoring/alerta.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/alerta.nix
@@ -21,10 +21,10 @@ let
 in
 {
   options.services.alerta = {
-    enable = mkEnableOption "alerta";
+    enable = mkEnableOption (lib.mdDoc "alerta");
 
     port = mkOption {
-      type = types.int;
+      type = types.port;
       default = 5000;
       description = lib.mdDoc "Port of Alerta";
     };
diff --git a/nixpkgs/nixos/modules/services/monitoring/apcupsd.nix b/nixpkgs/nixos/modules/services/monitoring/apcupsd.nix
index d4216b44cdc8..666479c78a84 100644
--- a/nixpkgs/nixos/modules/services/monitoring/apcupsd.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/apcupsd.nix
@@ -62,6 +62,21 @@ let
 
   );
 
+  # Ensure the CLI uses our generated configFile
+  wrappedBinaries = pkgs.runCommandLocal "apcupsd-wrapped-binaries"
+    { nativeBuildInputs = [ pkgs.makeWrapper ]; }
+    ''
+      for p in "${lib.getBin pkgs.apcupsd}/bin/"*; do
+          bname=$(basename "$p")
+          makeWrapper "$p" "$out/bin/$bname" --add-flags "-f ${configFile}"
+      done
+    '';
+
+  apcupsdWrapped = pkgs.symlinkJoin {
+    name = "apcupsd-wrapped";
+    # Put wrappers first so they "win"
+    paths = [ wrappedBinaries pkgs.apcupsd ];
+  };
 in
 
 {
@@ -138,7 +153,7 @@ in
     } ];
 
     # Give users access to the "apcaccess" tool
-    environment.systemPackages = [ pkgs.apcupsd ];
+    environment.systemPackages = [ apcupsdWrapped ];
 
     # NOTE 1: apcupsd runs as root because it needs permission to run
     # "shutdown"
diff --git a/nixpkgs/nixos/modules/services/monitoring/arbtt.nix b/nixpkgs/nixos/modules/services/monitoring/arbtt.nix
index 8bf4f78cc727..f07ecc5d5dd0 100644
--- a/nixpkgs/nixos/modules/services/monitoring/arbtt.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/arbtt.nix
@@ -7,13 +7,7 @@ let
 in {
   options = {
     services.arbtt = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = lib.mdDoc ''
-          Enable the arbtt statistics capture service.
-        '';
-      };
+      enable = mkEnableOption (lib.mdDoc "Arbtt statistics capture service");
 
       package = mkOption {
         type = types.package;
diff --git a/nixpkgs/nixos/modules/services/monitoring/bosun.nix b/nixpkgs/nixos/modules/services/monitoring/bosun.nix
index 27966e089eb9..dc75fda6ed8a 100644
--- a/nixpkgs/nixos/modules/services/monitoring/bosun.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/bosun.nix
@@ -22,13 +22,7 @@ in {
 
     services.bosun = {
 
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = lib.mdDoc ''
-          Whether to run bosun.
-        '';
-      };
+      enable = mkEnableOption (lib.mdDoc "bosun");
 
       package = mkOption {
         type = types.package;
diff --git a/nixpkgs/nixos/modules/services/monitoring/cadvisor.nix b/nixpkgs/nixos/modules/services/monitoring/cadvisor.nix
index d5e440310162..68e6e8e40b31 100644
--- a/nixpkgs/nixos/modules/services/monitoring/cadvisor.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/cadvisor.nix
@@ -8,11 +8,7 @@ let
 in {
   options = {
     services.cadvisor = {
-      enable = mkOption {
-        default = false;
-        type = types.bool;
-        description = lib.mdDoc "Whether to enable cadvisor service.";
-      };
+      enable = mkEnableOption (lib.mdDoc "Cadvisor service");
 
       listenAddress = mkOption {
         default = "127.0.0.1";
@@ -22,7 +18,7 @@ in {
 
       port = mkOption {
         default = 8080;
-        type = types.int;
+        type = types.port;
         description = lib.mdDoc "Cadvisor listening port";
       };
 
@@ -127,7 +123,7 @@ in {
             ${escapeShellArgs cfg.extraOptions} \
             ${optionalString (cfg.storageDriver != null) ''
               -storage_driver "${cfg.storageDriver}" \
-              -storage_driver_user "${cfg.storageDriverHost}" \
+              -storage_driver_host "${cfg.storageDriverHost}" \
               -storage_driver_db "${cfg.storageDriverDb}" \
               -storage_driver_user "${cfg.storageDriverUser}" \
               -storage_driver_password "$(cat "${cfg.storageDriverPasswordFile}")" \
diff --git a/nixpkgs/nixos/modules/services/monitoring/cockpit.nix b/nixpkgs/nixos/modules/services/monitoring/cockpit.nix
new file mode 100644
index 000000000000..2947b4d80120
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/cockpit.nix
@@ -0,0 +1,231 @@
+{ pkgs, config, lib, ... }:
+
+let
+  cfg = config.services.cockpit;
+  inherit (lib) types mkEnableOption mkOption mkIf mdDoc literalMD mkPackageOptionMD;
+  settingsFormat = pkgs.formats.ini {};
+in {
+  options = {
+    services.cockpit = {
+      enable = mkEnableOption (mdDoc "Cockpit");
+
+      package = mkPackageOptionMD pkgs "Cockpit" {
+        default = [ "cockpit" ];
+      };
+
+      settings = lib.mkOption {
+        type = settingsFormat.type;
+
+        default = {};
+
+        description = mdDoc ''
+          Settings for cockpit that will be saved in /etc/cockpit/cockpit.conf.
+
+          See the [documentation](https://cockpit-project.org/guide/latest/cockpit.conf.5.html), that is also available with `man cockpit.conf.5` for details.
+        '';
+      };
+
+      port = mkOption {
+        description = mdDoc "Port where cockpit will listen.";
+        type = types.port;
+        default = 9090;
+      };
+
+      openFirewall = mkOption {
+        description = mdDoc "Open port for cockpit.";
+        type = types.bool;
+        default = false;
+      };
+    };
+  };
+  config = mkIf cfg.enable {
+
+    # expose cockpit-bridge system-wide
+    environment.systemPackages = [ cfg.package ];
+
+    # allow cockpit to find its plugins
+    environment.pathsToLink = [ "/share/cockpit" ];
+
+    # generate cockpit settings
+    environment.etc."cockpit/cockpit.conf".source = settingsFormat.generate "cockpit.conf" cfg.settings;
+
+    security.pam.services.cockpit = {};
+
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ];
+
+    # units are in reverse sort order if you ls $out/lib/systemd/system
+    # all these units are basically verbatim translated from upstream
+
+    # Translation from $out/lib/systemd/system/systemd-cockpithttps.slice
+    systemd.slices.system-cockpithttps = {
+      description = "Resource limits for all cockpit-ws-https@.service instances";
+      sliceConfig = {
+        TasksMax = 200;
+        MemoryHigh = "75%";
+        MemoryMax = "90%";
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit-wsinstance-https@.socket
+    systemd.sockets."cockpit-wsinstance-https@" = {
+      unitConfig = {
+        Description = "Socket for Cockpit Web Service https instance %I";
+        BindsTo = [ "cockpit.service" "cockpit-wsinstance-https@%i.service" ];
+        # clean up the socket after the service exits, to prevent fd leak
+        # this also effectively prevents a DoS by starting arbitrarily many sockets, as
+        # the services are resource-limited by system-cockpithttps.slice
+        Documentation = "man:cockpit-ws(8)";
+      };
+      socketConfig = {
+        ListenStream = "/run/cockpit/wsinstance/https@%i.sock";
+        SocketUser = "root";
+        SocketMode = "0600";
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit-wsinstance-https@.service
+    systemd.services."cockpit-wsinstance-https@" = {
+      description = "Cockpit Web Service https instance %I";
+      bindsTo = [ "cockpit.service"];
+      path = [ cfg.package ];
+      documentation = [ "man:cockpit-ws(8)" ];
+      serviceConfig = {
+        Slice = "system-cockpithttps.slice";
+        ExecStart = "${cfg.package}/libexec/cockpit-ws --for-tls-proxy --port=0";
+        User = "root";
+        Group = "";
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit-wsinstance-http.socket
+    systemd.sockets.cockpit-wsinstance-http = {
+      unitConfig = {
+        Description = "Socket for Cockpit Web Service http instance";
+        BindsTo = "cockpit.service";
+        Documentation = "man:cockpit-ws(8)";
+      };
+      socketConfig = {
+        ListenStream = "/run/cockpit/wsinstance/http.sock";
+        SocketUser = "root";
+        SocketMode = "0600";
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit-wsinstance-https-factory.socket
+    systemd.sockets.cockpit-wsinstance-https-factory = {
+      unitConfig = {
+        Description = "Socket for Cockpit Web Service https instance factory";
+        BindsTo = "cockpit.service";
+        Documentation = "man:cockpit-ws(8)";
+      };
+      socketConfig = {
+        ListenStream = "/run/cockpit/wsinstance/https-factory.sock";
+        Accept = true;
+        SocketUser = "root";
+        SocketMode = "0600";
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit-wsinstance-https-factory@.service
+    systemd.services."cockpit-wsinstance-https-factory@" = {
+      description = "Cockpit Web Service https instance factory";
+      documentation = [ "man:cockpit-ws(8)" ];
+      path = [ cfg.package ];
+      serviceConfig = {
+        ExecStart = "${cfg.package}/libexec/cockpit-wsinstance-factory";
+        User = "root";
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit-wsinstance-http.service
+    systemd.services."cockpit-wsinstance-http" = {
+      description = "Cockpit Web Service http instance";
+      bindsTo = [ "cockpit.service" ];
+      path = [ cfg.package ];
+      documentation = [ "man:cockpit-ws(8)" ];
+      serviceConfig = {
+        ExecStart = "${cfg.package}/libexec/cockpit-ws --no-tls --port=0";
+        User = "root";
+        Group = "";
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit.socket
+    systemd.sockets."cockpit" = {
+      unitConfig = {
+        Description = "Cockpit Web Service Socket";
+        Documentation = "man:cockpit-ws(8)";
+        Wants = "cockpit-motd.service";
+      };
+      socketConfig = {
+        ListenStream = cfg.port;
+        ExecStartPost = [
+          "-${cfg.package}/share/cockpit/motd/update-motd \"\" localhost"
+          "-${pkgs.coreutils}/bin/ln -snf active.motd /run/cockpit/motd"
+        ];
+        ExecStopPost = "-${pkgs.coreutils}/bin/ln -snf inactive.motd /run/cockpit/motd";
+      };
+      wantedBy = [ "sockets.target" ];
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit.service
+    systemd.services."cockpit" = {
+      description = "Cockpit Web Service";
+      documentation = [ "man:cockpit-ws(8)" ];
+      restartIfChanged = true;
+      path = with pkgs; [ coreutils cfg.package ];
+      requires = [ "cockpit.socket" "cockpit-wsinstance-http.socket" "cockpit-wsinstance-https-factory.socket" ];
+      after = [ "cockpit-wsinstance-http.socket" "cockpit-wsinstance-https-factory.socket" ];
+      environment = {
+        G_MESSAGES_DEBUG = "cockpit-ws,cockpit-bridge";
+      };
+      serviceConfig = {
+        RuntimeDirectory="cockpit/tls";
+        ExecStartPre = [
+          # cockpit-tls runs in a more constrained environment, these + means that these commands
+          # will run with full privilege instead of inside that constrained environment
+          # See https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStart= for details
+          "+${cfg.package}/libexec/cockpit-certificate-ensure --for-cockpit-tls"
+        ];
+        ExecStart = "${cfg.package}/libexec/cockpit-tls";
+        User = "root";
+        Group = "";
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        ProtectKernelTunables = true;
+        RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
+        MemoryDenyWriteExecute = true;
+      };
+    };
+
+    # Translation from $out/lib/systemd/system/cockpit-motd.service
+    # This part basically implements a motd state machine:
+    # - If cockpit.socket is enabled then /run/cockpit/motd points to /run/cockpit/active.motd
+    # - If cockpit.socket is disabled then /run/cockpit/motd points to /run/cockpit/inactive.motd
+    # - As cockpit.socket is disabled by default, /run/cockpit/motd points to /run/cockpit/inactive.motd
+    # /run/cockpit/active.motd is generated dynamically by cockpit-motd.service
+    systemd.services."cockpit-motd" = {
+      path = with pkgs; [ nettools ];
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${cfg.package}/share/cockpit/motd/update-motd";
+      };
+      description = "Cockpit motd updater service";
+      documentation = [ "man:cockpit-ws(8)" ];
+      wants = [ "network.target" ];
+      after = [ "network.target" "cockpit.socket" ];
+    };
+
+    systemd.tmpfiles.rules = [ # From $out/lib/tmpfiles.d/cockpit-tmpfiles.conf
+      "C /run/cockpit/inactive.motd 0640 root root - ${cfg.package}/share/cockpit/motd/inactive.motd"
+      "f /run/cockpit/active.motd   0640 root root -"
+      "L+ /run/cockpit/motd - - - - inactive.motd"
+      "d /etc/cockpit/ws-certs.d 0600 root root 0"
+    ];
+  };
+
+  meta.maintainers = pkgs.cockpit.meta.maintainers;
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/collectd.nix b/nixpkgs/nixos/modules/services/monitoring/collectd.nix
index 5c62d509511d..5d525995c67a 100644
--- a/nixpkgs/nixos/modules/services/monitoring/collectd.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/collectd.nix
@@ -29,7 +29,7 @@ let
 
 in {
   options.services.collectd = with types; {
-    enable = mkEnableOption "collectd agent";
+    enable = mkEnableOption (lib.mdDoc "collectd agent");
 
     validateConfig = mkOption {
       default = true;
@@ -52,7 +52,7 @@ in {
 
     buildMinimalPackage = mkOption {
       default = false;
-      description = ''
+      description = lib.mdDoc ''
         Build a minimal collectd package with only the configured `services.collectd.plugins`
       '';
       type = bool;
diff --git a/nixpkgs/nixos/modules/services/monitoring/das_watchdog.nix b/nixpkgs/nixos/modules/services/monitoring/das_watchdog.nix
index 88ca3a9227d2..fd420b0c8a06 100644
--- a/nixpkgs/nixos/modules/services/monitoring/das_watchdog.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/das_watchdog.nix
@@ -12,7 +12,7 @@ in {
   ###### interface
 
   options = {
-    services.das_watchdog.enable = mkEnableOption "realtime watchdog";
+    services.das_watchdog.enable = mkEnableOption (lib.mdDoc "realtime watchdog");
   };
 
   ###### implementation
diff --git a/nixpkgs/nixos/modules/services/monitoring/datadog-agent.nix b/nixpkgs/nixos/modules/services/monitoring/datadog-agent.nix
index 9b984fafc12b..58a0faed962c 100644
--- a/nixpkgs/nixos/modules/services/monitoring/datadog-agent.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/datadog-agent.nix
@@ -49,18 +49,12 @@ let
   };
 in {
   options.services.datadog-agent = {
-    enable = mkOption {
-      description = lib.mdDoc ''
-        Whether to enable the datadog-agent v7 monitoring service
-      '';
-      default = false;
-      type = types.bool;
-    };
+    enable = mkEnableOption (lib.mdDoc "Datadog-agent v7 monitoring service");
 
     package = mkOption {
       default = pkgs.datadog-agent;
       defaultText = literalExpression "pkgs.datadog-agent";
-      description = ''
+      description = lib.mdDoc ''
         Which DataDog v7 agent package to use. Note that the provided
         package is expected to have an overridable `pythonPackages`-attribute
         which configures the Python environment with the Datadog
@@ -168,7 +162,7 @@ in {
     };
 
     checks = mkOption {
-      description = ''
+      description = lib.mdDoc ''
         Configuration for all Datadog checks. Keys of this attribute
         set will be used as the name of the check to create the
         appropriate configuration in `conf.d/$check.d/conf.yaml`.
@@ -241,7 +235,7 @@ in {
 
     systemd.services = let
       makeService = attrs: recursiveUpdate {
-        path = [ datadogPkg pkgs.python pkgs.sysstat pkgs.procps pkgs.iproute2 ];
+        path = [ datadogPkg pkgs.sysstat pkgs.procps pkgs.iproute2 ];
         wantedBy = [ "multi-user.target" ];
         serviceConfig = {
           User = "datadog";
diff --git a/nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix b/nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix
deleted file mode 100644
index 045128197421..000000000000
--- a/nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix
+++ /dev/null
@@ -1,8 +0,0 @@
-# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY.
-[
-  "auto_conf"
-  "agent_metrics.yaml.default"
-  "disk.yaml.default"
-  "network.yaml.default"
-  "ntp.yaml.default"
-]
diff --git a/nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent.nix b/nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
deleted file mode 100644
index 8c0070c4853a..000000000000
--- a/nixpkgs/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
+++ /dev/null
@@ -1,236 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  cfg = config.services.dd-agent;
-
-  ddConf = pkgs.writeText "datadog.conf" ''
-    [Main]
-    dd_url: https://app.datadoghq.com
-    skip_ssl_validation: no
-    api_key: ${cfg.api_key}
-    ${optionalString (cfg.hostname != null) "hostname: ${cfg.hostname}"}
-
-    collector_log_file: /var/log/datadog/collector.log
-    forwarder_log_file: /var/log/datadog/forwarder.log
-    dogstatsd_log_file: /var/log/datadog/dogstatsd.log
-    pup_log_file:       /var/log/datadog/pup.log
-
-    # proxy_host: my-proxy.com
-    # proxy_port: 3128
-    # proxy_user: user
-    # proxy_password: password
-
-    # tags: mytag0, mytag1
-    ${optionalString (cfg.tags != null ) "tags: ${concatStringsSep ", " cfg.tags }"}
-
-    # collect_ec2_tags: no
-    # recent_point_threshold: 30
-    # use_mount: no
-    # listen_port: 17123
-    # graphite_listen_port: 17124
-    # non_local_traffic: no
-    # use_curl_http_client: False
-    # bind_host: localhost
-
-    # use_pup: no
-    # pup_port: 17125
-    # pup_interface: localhost
-    # pup_url: http://localhost:17125
-
-    # dogstatsd_port : 8125
-    # dogstatsd_interval : 10
-    # dogstatsd_normalize : yes
-    # statsd_forward_host: address_of_own_statsd_server
-    # statsd_forward_port: 8125
-
-    # device_blacklist_re: .*\/dev\/mapper\/lxc-box.*
-
-    # ganglia_host: localhost
-    # ganglia_port: 8651
-  '';
-
-  diskConfig = pkgs.writeText "disk.yaml" ''
-    init_config:
-
-    instances:
-      - use_mount: no
-  '';
-
-  networkConfig = pkgs.writeText "network.yaml" ''
-    init_config:
-
-    instances:
-      # Network check only supports one configured instance
-      - collect_connection_state: false
-        excluded_interfaces:
-          - lo
-          - lo0
-  '';
-
-  postgresqlConfig = pkgs.writeText "postgres.yaml" cfg.postgresqlConfig;
-  nginxConfig = pkgs.writeText "nginx.yaml" cfg.nginxConfig;
-  mongoConfig = pkgs.writeText "mongo.yaml" cfg.mongoConfig;
-  jmxConfig = pkgs.writeText "jmx.yaml" cfg.jmxConfig;
-  processConfig = pkgs.writeText "process.yaml" cfg.processConfig;
-
-  etcfiles =
-    let
-      defaultConfd = import ./dd-agent-defaults.nix;
-    in
-      listToAttrs (map (f: {
-        name = "dd-agent/conf.d/${f}";
-        value.source = "${pkgs.dd-agent}/agent/conf.d-system/${f}";
-      }) defaultConfd) //
-      {
-        "dd-agent/datadog.conf".source = ddConf;
-        "dd-agent/conf.d/disk.yaml".source = diskConfig;
-        "dd-agent/conf.d/network.yaml".source = networkConfig;
-      } //
-      (optionalAttrs (cfg.postgresqlConfig != null)
-      {
-        "dd-agent/conf.d/postgres.yaml".source = postgresqlConfig;
-      }) //
-      (optionalAttrs (cfg.nginxConfig != null)
-      {
-        "dd-agent/conf.d/nginx.yaml".source = nginxConfig;
-      }) //
-      (optionalAttrs (cfg.mongoConfig != null)
-      {
-        "dd-agent/conf.d/mongo.yaml".source = mongoConfig;
-      }) //
-      (optionalAttrs (cfg.processConfig != null)
-      {
-        "dd-agent/conf.d/process.yaml".source = processConfig;
-      }) //
-      (optionalAttrs (cfg.jmxConfig != null)
-      {
-        "dd-agent/conf.d/jmx.yaml".source = jmxConfig;
-      });
-
-in {
-  options.services.dd-agent = {
-    enable = mkOption {
-      description = lib.mdDoc ''
-        Whether to enable the dd-agent v5 monitoring service.
-        For datadog-agent v6, see {option}`services.datadog-agent.enable`.
-      '';
-      default = false;
-      type = types.bool;
-    };
-
-    api_key = mkOption {
-      description = lib.mdDoc ''
-        The Datadog API key to associate the agent with your account.
-
-        Warning: this key is stored in cleartext within the world-readable
-        Nix store! Consider using the new v6
-        {option}`services.datadog-agent` module instead.
-      '';
-      example = "ae0aa6a8f08efa988ba0a17578f009ab";
-      type = types.str;
-    };
-
-    tags = mkOption {
-      description = lib.mdDoc "The tags to mark this Datadog agent";
-      example = [ "test" "service" ];
-      default = null;
-      type = types.nullOr (types.listOf types.str);
-    };
-
-    hostname = mkOption {
-      description = lib.mdDoc "The hostname to show in the Datadog dashboard (optional)";
-      default = null;
-      example = "mymachine.mydomain";
-      type = types.nullOr types.str;
-    };
-
-    postgresqlConfig = mkOption {
-      description = lib.mdDoc "Datadog PostgreSQL integration configuration";
-      default = null;
-      type = types.nullOr types.lines;
-    };
-
-    nginxConfig = mkOption {
-      description = lib.mdDoc "Datadog nginx integration configuration";
-      default = null;
-      type = types.nullOr types.lines;
-    };
-
-    mongoConfig = mkOption {
-      description = lib.mdDoc "MongoDB integration configuration";
-      default = null;
-      type = types.nullOr types.lines;
-    };
-
-    jmxConfig = mkOption {
-      description = lib.mdDoc "JMX integration configuration";
-      default = null;
-      type = types.nullOr types.lines;
-    };
-
-    processConfig = mkOption {
-      description = lib.mdDoc ''
-        Process integration configuration
-        See <https://docs.datadoghq.com/integrations/process/>
-      '';
-      default = null;
-      type = types.nullOr types.lines;
-    };
-
-  };
-
-  config = mkIf cfg.enable {
-    environment.systemPackages = [ pkgs.dd-agent pkgs.sysstat pkgs.procps ];
-
-    users.users.datadog = {
-      description = "Datadog Agent User";
-      uid = config.ids.uids.datadog;
-      group = "datadog";
-      home = "/var/log/datadog/";
-      createHome = true;
-    };
-
-    users.groups.datadog.gid = config.ids.gids.datadog;
-
-    systemd.services = let
-      makeService = attrs: recursiveUpdate {
-        path = [ pkgs.dd-agent pkgs.python pkgs.sysstat pkgs.procps pkgs.gohai ];
-        wantedBy = [ "multi-user.target" ];
-        serviceConfig = {
-          User = "datadog";
-          Group = "datadog";
-          Restart = "always";
-          RestartSec = 2;
-          PrivateTmp = true;
-        };
-        restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
-      } attrs;
-    in {
-      dd-agent = makeService {
-        description = "Datadog agent monitor";
-        serviceConfig.ExecStart = "${pkgs.dd-agent}/bin/dd-agent foreground";
-      };
-
-      dogstatsd = makeService {
-        description = "Datadog statsd";
-        environment.TMPDIR = "/run/dogstatsd";
-        serviceConfig = {
-          ExecStart = "${pkgs.dd-agent}/bin/dogstatsd start";
-          Type = "forking";
-          PIDFile = "/run/dogstatsd/dogstatsd.pid";
-          RuntimeDirectory = "dogstatsd";
-        };
-      };
-
-      dd-jmxfetch = lib.mkIf (cfg.jmxConfig != null) {
-        description = "Datadog JMX Fetcher";
-        path = [ pkgs.dd-agent pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
-        serviceConfig.ExecStart = "${pkgs.dd-agent}/bin/dd-jmxfetch";
-      };
-    };
-
-    environment.etc = etcfiles;
-  };
-}
diff --git a/nixpkgs/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults b/nixpkgs/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults
deleted file mode 100755
index 76724173171a..000000000000
--- a/nixpkgs/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-dd=$(nix-build --no-out-link -A dd-agent ../../../..)
-echo '# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY.' > dd-agent-defaults.nix
-echo '[' >> dd-agent-defaults.nix
-echo '  "auto_conf"' >> dd-agent-defaults.nix
-for f in $(find $dd/agent/conf.d-system -maxdepth 1 -type f | grep -v '\.example' | sort); do
-  echo "  \"$(basename $f)\"" >> dd-agent-defaults.nix
-done
-echo ']' >> dd-agent-defaults.nix
diff --git a/nixpkgs/nixos/modules/services/monitoring/do-agent.nix b/nixpkgs/nixos/modules/services/monitoring/do-agent.nix
index 4dfb6236727b..c1788c640c23 100644
--- a/nixpkgs/nixos/modules/services/monitoring/do-agent.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/do-agent.nix
@@ -8,7 +8,7 @@ let
 in
 {
   options.services.do-agent = {
-    enable = mkEnableOption "do-agent, the DigitalOcean droplet metrics agent";
+    enable = mkEnableOption (lib.mdDoc "do-agent, the DigitalOcean droplet metrics agent");
   };
 
   config = mkIf cfg.enable {
diff --git a/nixpkgs/nixos/modules/services/monitoring/fusion-inventory.nix b/nixpkgs/nixos/modules/services/monitoring/fusion-inventory.nix
index 6b440e9fa45b..7b28e8de1229 100644
--- a/nixpkgs/nixos/modules/services/monitoring/fusion-inventory.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/fusion-inventory.nix
@@ -22,7 +22,7 @@ in {
 
     services.fusionInventory = {
 
-      enable = mkEnableOption "Fusion Inventory Agent";
+      enable = mkEnableOption (lib.mdDoc "Fusion Inventory Agent");
 
       servers = mkOption {
         type = types.listOf types.str;
diff --git a/nixpkgs/nixos/modules/services/monitoring/grafana-agent.nix b/nixpkgs/nixos/modules/services/monitoring/grafana-agent.nix
index 8190f44c72f7..13604ff77c68 100644
--- a/nixpkgs/nixos/modules/services/monitoring/grafana-agent.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/grafana-agent.nix
@@ -11,14 +11,9 @@ in
   };
 
   options.services.grafana-agent = {
-    enable = mkEnableOption "grafana-agent";
+    enable = mkEnableOption (lib.mdDoc "grafana-agent");
 
-    package = mkOption {
-      type = types.package;
-      default = pkgs.grafana-agent;
-      defaultText = "pkgs.grafana-agent";
-      description = lib.mdDoc "The grafana-agent package to use.";
-    };
+    package = mkPackageOptionMD pkgs "grafana-agent" { };
 
     credentials = mkOption {
       description = lib.mdDoc ''
@@ -37,11 +32,22 @@ in
       };
     };
 
+    extraFlags = mkOption {
+      type = with types; listOf str;
+      default = [ ];
+      example = [ "-enable-features=integrations-next" "-disable-reporting" ];
+      description = lib.mdDoc ''
+        Extra command-line flags passed to {command}`grafana-agent`.
+
+        See <https://grafana.com/docs/agent/latest/static/configuration/flags/>
+      '';
+    };
+
     settings = mkOption {
-      description = ''
-        Configuration for <package>grafana-agent</package>.
+      description = lib.mdDoc ''
+        Configuration for {command}`grafana-agent`.
 
-        See https://grafana.com/docs/agent/latest/configuration/
+        See <https://grafana.com/docs/agent/latest/configuration/>
       '';
 
       type = types.submodule {
@@ -49,17 +55,18 @@ in
       };
 
       default = { };
-      defaultText = ''
-        metrics = {
-          wal_directory = "\''${STATE_DIRECTORY}";
-          global.scrape_interval = "5s";
-        };
-        integrations = {
-          agent.enabled = true;
-          agent.scrape_integration = true;
-          node_exporter.enabled = true;
-          replace_instance_label = true;
-        };
+      defaultText = lib.literalExpression ''
+        {
+          metrics = {
+            wal_directory = "\''${STATE_DIRECTORY}";
+            global.scrape_interval = "5s";
+          };
+          integrations = {
+            agent.enabled = true;
+            agent.scrape_integration = true;
+            node_exporter.enabled = true;
+          };
+        }
       '';
       example = {
         metrics.global.remote_write = [{
@@ -114,7 +121,6 @@ in
         agent.enabled = mkDefault true;
         agent.scrape_integration = mkDefault true;
         node_exporter.enabled = mkDefault true;
-        replace_instance_label = mkDefault true;
       };
     };
 
@@ -138,7 +144,7 @@ in
         # We can't use Environment=HOSTNAME=%H, as it doesn't include the domain part.
         export HOSTNAME=$(< /proc/sys/kernel/hostname)
 
-        exec ${cfg.package}/bin/agent -config.expand-env -config.file ${configFile}
+        exec ${lib.getExe cfg.package} -config.expand-env -config.file ${configFile} ${escapeShellArgs cfg.extraFlags}
       '';
       serviceConfig = {
         Restart = "always";
diff --git a/nixpkgs/nixos/modules/services/monitoring/grafana-image-renderer.nix b/nixpkgs/nixos/modules/services/monitoring/grafana-image-renderer.nix
index 4820b1946987..36258866646a 100644
--- a/nixpkgs/nixos/modules/services/monitoring/grafana-image-renderer.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/grafana-image-renderer.nix
@@ -10,7 +10,7 @@ let
   configFile = format.generate "grafana-image-renderer-config.json" cfg.settings;
 in {
   options.services.grafana-image-renderer = {
-    enable = mkEnableOption "grafana-image-renderer";
+    enable = mkEnableOption (lib.mdDoc "grafana-image-renderer");
 
     chromium = mkOption {
       type = types.package;
@@ -19,9 +19,9 @@ in {
       '';
     };
 
-    verbose = mkEnableOption "verbosity for the service";
+    verbose = mkEnableOption (lib.mdDoc "verbosity for the service");
 
-    provisionGrafana = mkEnableOption "Grafana configuration for grafana-image-renderer";
+    provisionGrafana = mkEnableOption (lib.mdDoc "Grafana configuration for grafana-image-renderer");
 
     settings = mkOption {
       type = types.submodule {
@@ -62,25 +62,23 @@ in {
             mode = mkOption {
               default = "default";
               type = types.enum [ "default" "reusable" "clustered" ];
-              description = ''
-                Rendering mode of <package>grafana-image-renderer</package>:
-                <itemizedlist>
-                <listitem><para><literal>default:</literal> Creates on browser-instance
-                  per rendering request.</para></listitem>
-                <listitem><para><literal>reusable:</literal> One browser instance
-                  will be started and reused for each rendering request.</para></listitem>
-                <listitem><para><literal>clustered:</literal> allows to precisely
+              description = lib.mdDoc ''
+                Rendering mode of `grafana-image-renderer`:
+
+                - `default:` Creates on browser-instance
+                  per rendering request.
+                - `reusable:` One browser instance
+                  will be started and reused for each rendering request.
+                - `clustered:` allows to precisely
                   configure how many browser-instances are supposed to be used. The values
-                  for that mode can be declared in <literal>rendering.clustering</literal>.
-                  </para></listitem>
-                </itemizedlist>
+                  for that mode can be declared in `rendering.clustering`.
               '';
             };
             args = mkOption {
               type = types.listOf types.str;
               default = [ "--no-sandbox" ];
-              description = ''
-                List of CLI flags passed to <package>chromium</package>.
+              description = lib.mdDoc ''
+                List of CLI flags passed to `chromium`.
               '';
             };
           };
@@ -89,10 +87,10 @@ in {
 
       default = {};
 
-      description = ''
-        Configuration attributes for <package>grafana-image-renderer</package>.
+      description = lib.mdDoc ''
+        Configuration attributes for `grafana-image-renderer`.
 
-        See <link xlink:href="https://github.com/grafana/grafana-image-renderer/blob/ce1f81438e5f69c7fd7c73ce08bab624c4c92e25/default.json"/>
+        See <https://github.com/grafana/grafana-image-renderer/blob/ce1f81438e5f69c7fd7c73ce08bab624c4c92e25/default.json>
         for supported values.
       '';
     };
@@ -108,9 +106,9 @@ in {
       }
     ];
 
-    services.grafana.extraOptions = mkIf cfg.provisionGrafana {
-      RENDERING_SERVER_URL = "http://localhost:${toString cfg.settings.service.port}/render";
-      RENDERING_CALLBACK_URL = "http://localhost:${toString config.services.grafana.port}";
+    services.grafana.settings.rendering = mkIf cfg.provisionGrafana {
+      server_url = "http://localhost:${toString cfg.settings.service.port}/render";
+      callback_url = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
     };
 
     services.grafana-image-renderer.chromium = mkDefault pkgs.chromium;
diff --git a/nixpkgs/nixos/modules/services/monitoring/grafana-reporter.nix b/nixpkgs/nixos/modules/services/monitoring/grafana-reporter.nix
index 7a27b5cbce31..eac304d63aa1 100644
--- a/nixpkgs/nixos/modules/services/monitoring/grafana-reporter.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/grafana-reporter.nix
@@ -7,7 +7,7 @@ let
 
 in {
   options.services.grafana_reporter = {
-    enable = mkEnableOption "grafana_reporter";
+    enable = mkEnableOption (lib.mdDoc "grafana_reporter");
 
     grafana = {
       protocol = mkOption {
@@ -23,7 +23,7 @@ in {
       port = mkOption {
         description = lib.mdDoc "Grafana port.";
         default = 3000;
-        type = types.int;
+        type = types.port;
       };
 
     };
@@ -36,7 +36,7 @@ in {
     port = mkOption {
       description = lib.mdDoc "Listening port.";
       default = 8686;
-      type = types.int;
+      type = types.port;
     };
 
     templateDir = mkOption {
diff --git a/nixpkgs/nixos/modules/services/monitoring/grafana.nix b/nixpkgs/nixos/modules/services/monitoring/grafana.nix
index 456fe92eea38..adffadf3fc47 100644
--- a/nixpkgs/nixos/modules/services/monitoring/grafana.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/grafana.nix
@@ -5,106 +5,93 @@ with lib;
 let
   cfg = config.services.grafana;
   opt = options.services.grafana;
+  provisioningSettingsFormat = pkgs.formats.yaml {};
   declarativePlugins = pkgs.linkFarm "grafana-plugins" (builtins.map (pkg: { name = pkg.pname; path = pkg; }) cfg.declarativePlugins);
-  useMysql = cfg.database.type == "mysql";
-  usePostgresql = cfg.database.type == "postgres";
-
-  envOptions = {
-    PATHS_DATA = cfg.dataDir;
-    PATHS_PLUGINS = if builtins.isNull cfg.declarativePlugins then "${cfg.dataDir}/plugins" else declarativePlugins;
-    PATHS_LOGS = "${cfg.dataDir}/log";
-
-    SERVER_SERVE_FROM_SUBPATH = boolToString cfg.server.serveFromSubPath;
-    SERVER_PROTOCOL = cfg.protocol;
-    SERVER_HTTP_ADDR = cfg.addr;
-    SERVER_HTTP_PORT = cfg.port;
-    SERVER_SOCKET = cfg.socket;
-    SERVER_DOMAIN = cfg.domain;
-    SERVER_ROOT_URL = cfg.rootUrl;
-    SERVER_STATIC_ROOT_PATH = cfg.staticRootPath;
-    SERVER_CERT_FILE = cfg.certFile;
-    SERVER_CERT_KEY = cfg.certKey;
-
-    DATABASE_TYPE = cfg.database.type;
-    DATABASE_HOST = cfg.database.host;
-    DATABASE_NAME = cfg.database.name;
-    DATABASE_USER = cfg.database.user;
-    DATABASE_PASSWORD = cfg.database.password;
-    DATABASE_PATH = cfg.database.path;
-    DATABASE_CONN_MAX_LIFETIME = cfg.database.connMaxLifetime;
-
-    SECURITY_ADMIN_USER = cfg.security.adminUser;
-    SECURITY_ADMIN_PASSWORD = cfg.security.adminPassword;
-    SECURITY_SECRET_KEY = cfg.security.secretKey;
-
-    USERS_ALLOW_SIGN_UP = boolToString cfg.users.allowSignUp;
-    USERS_ALLOW_ORG_CREATE = boolToString cfg.users.allowOrgCreate;
-    USERS_AUTO_ASSIGN_ORG = boolToString cfg.users.autoAssignOrg;
-    USERS_AUTO_ASSIGN_ORG_ROLE = cfg.users.autoAssignOrgRole;
-
-    AUTH_DISABLE_LOGIN_FORM = boolToString cfg.auth.disableLoginForm;
-
-    AUTH_ANONYMOUS_ENABLED = boolToString cfg.auth.anonymous.enable;
-    AUTH_ANONYMOUS_ORG_NAME = cfg.auth.anonymous.org_name;
-    AUTH_ANONYMOUS_ORG_ROLE = cfg.auth.anonymous.org_role;
-
-    AUTH_AZUREAD_NAME = "Azure AD";
-    AUTH_AZUREAD_ENABLED = boolToString cfg.auth.azuread.enable;
-    AUTH_AZUREAD_ALLOW_SIGN_UP = boolToString cfg.auth.azuread.allowSignUp;
-    AUTH_AZUREAD_CLIENT_ID = cfg.auth.azuread.clientId;
-    AUTH_AZUREAD_SCOPES = "openid email profile";
-    AUTH_AZUREAD_AUTH_URL = "https://login.microsoftonline.com/${cfg.auth.azuread.tenantId}/oauth2/v2.0/authorize";
-    AUTH_AZUREAD_TOKEN_URL = "https://login.microsoftonline.com/${cfg.auth.azuread.tenantId}/oauth2/v2.0/token";
-    AUTH_AZUREAD_ALLOWED_DOMAINS = cfg.auth.azuread.allowedDomains;
-    AUTH_AZUREAD_ALLOWED_GROUPS = cfg.auth.azuread.allowedGroups;
-    AUTH_AZUREAD_ROLE_ATTRIBUTE_STRICT = false;
-
-    AUTH_GOOGLE_ENABLED = boolToString cfg.auth.google.enable;
-    AUTH_GOOGLE_ALLOW_SIGN_UP = boolToString cfg.auth.google.allowSignUp;
-    AUTH_GOOGLE_CLIENT_ID = cfg.auth.google.clientId;
-
-    ANALYTICS_REPORTING_ENABLED = boolToString cfg.analytics.reporting.enable;
-
-    SMTP_ENABLED = boolToString cfg.smtp.enable;
-    SMTP_HOST = cfg.smtp.host;
-    SMTP_USER = cfg.smtp.user;
-    SMTP_PASSWORD = cfg.smtp.password;
-    SMTP_FROM_ADDRESS = cfg.smtp.fromAddress;
-  } // cfg.extraOptions;
-
-  datasourceConfiguration = {
-    apiVersion = 1;
-    datasources = cfg.provision.datasources;
-  };
-
-  datasourceFile = pkgs.writeText "datasource.yaml" (builtins.toJSON datasourceConfiguration);
-
-  dashboardConfiguration = {
-    apiVersion = 1;
-    providers = cfg.provision.dashboards;
-  };
-
-  dashboardFile = pkgs.writeText "dashboard.yaml" (builtins.toJSON dashboardConfiguration);
+  useMysql = cfg.settings.database.type == "mysql";
+  usePostgresql = cfg.settings.database.type == "postgres";
+
+  settingsFormatIni = pkgs.formats.ini {};
+  configFile = settingsFormatIni.generate "config.ini" cfg.settings;
+
+  mkProvisionCfg = name: attr: provisionCfg:
+    if provisionCfg.path != null
+      then provisionCfg.path
+    else
+      provisioningSettingsFormat.generate "${name}.yaml"
+        (if provisionCfg.settings != null
+          then provisionCfg.settings
+          else {
+            apiVersion = 1;
+            ${attr} = [];
+          });
+
+  datasourceFileOrDir = mkProvisionCfg "datasource" "datasources" cfg.provision.datasources;
+  dashboardFileOrDir = mkProvisionCfg "dashboard" "providers" cfg.provision.dashboards;
 
   notifierConfiguration = {
     apiVersion = 1;
     notifiers = cfg.provision.notifiers;
   };
 
-  notifierFile = pkgs.writeText "notifier.yaml" (builtins.toJSON notifierConfiguration);
-
-  provisionConfDir =  pkgs.runCommand "grafana-provisioning" { } ''
-    mkdir -p $out/{datasources,dashboards,notifiers}
-    ln -sf ${datasourceFile} $out/datasources/datasource.yaml
-    ln -sf ${dashboardFile} $out/dashboards/dashboard.yaml
-    ln -sf ${notifierFile} $out/notifiers/notifier.yaml
+  notifierFileOrDir = pkgs.writeText "notifier.yaml" (builtins.toJSON notifierConfiguration);
+
+  generateAlertingProvisioningYaml = x: if (cfg.provision.alerting."${x}".path == null)
+                                        then provisioningSettingsFormat.generate "${x}.yaml" cfg.provision.alerting."${x}".settings
+                                        else cfg.provision.alerting."${x}".path;
+  rulesFileOrDir = generateAlertingProvisioningYaml "rules";
+  contactPointsFileOrDir = generateAlertingProvisioningYaml "contactPoints";
+  policiesFileOrDir = generateAlertingProvisioningYaml "policies";
+  templatesFileOrDir = generateAlertingProvisioningYaml "templates";
+  muteTimingsFileOrDir = generateAlertingProvisioningYaml "muteTimings";
+
+  ln = { src, dir, filename }: ''
+    if [[ -d "${src}" ]]; then
+      pushd $out/${dir} &>/dev/null
+        lndir "${src}"
+      popd &>/dev/null
+    else
+      ln -sf ${src} $out/${dir}/${filename}.yaml
+    fi
+  '';
+  provisionConfDir = pkgs.runCommand "grafana-provisioning" { nativeBuildInputs = [ pkgs.xorg.lndir ]; } ''
+    mkdir -p $out/{datasources,dashboards,notifiers,alerting}
+    ${ln { src = datasourceFileOrDir;    dir = "datasources"; filename = "datasource"; }}
+    ${ln { src = dashboardFileOrDir;     dir = "dashboards";  filename = "dashboard"; }}
+    ${ln { src = notifierFileOrDir;      dir = "notifiers";   filename = "notifier"; }}
+    ${ln { src = rulesFileOrDir;         dir = "alerting";    filename = "rules"; }}
+    ${ln { src = contactPointsFileOrDir; dir = "alerting";    filename = "contactPoints"; }}
+    ${ln { src = policiesFileOrDir;      dir = "alerting";    filename = "policies"; }}
+    ${ln { src = templatesFileOrDir;     dir = "alerting";    filename = "templates"; }}
+    ${ln { src = muteTimingsFileOrDir;   dir = "alerting";    filename = "muteTimings"; }}
   '';
 
   # Get a submodule without any embedded metadata:
   _filter = x: filterAttrs (k: v: k != "_module") x;
 
+  # FIXME(@Ma27) remove before 23.05. This is just a helper-type
+  # because `mkRenamedOptionModule` doesn't work if `foo.bar` is renamed
+  # to `foo.bar.baz`.
+  submodule' = module: types.coercedTo
+    (mkOptionType {
+      name = "grafana-provision-submodule";
+      description = "Wrapper-type for backwards compat of Grafana's declarative provisioning";
+      check = x:
+        if builtins.isList x then
+          throw ''
+            Provisioning dashboards and datasources declaratively by
+            setting `dashboards` or `datasources` to a list is not supported
+            anymore. Use `services.grafana.provision.datasources.settings.datasources`
+            (or `services.grafana.provision.dashboards.settings.providers`) instead.
+          ''
+        else isAttrs x || isFunction x;
+    })
+    id
+    (types.submodule module);
+
   # http://docs.grafana.org/administration/provisioning/#datasources
   grafanaTypes.datasourceConfig = types.submodule {
+    freeformType = provisioningSettingsFormat.type;
+
     options = {
       name = mkOption {
         type = types.str;
@@ -119,11 +106,6 @@ let
         default = "proxy";
         description = lib.mdDoc "Access mode. proxy or direct (Server or Browser in the UI). Required.";
       };
-      orgId = mkOption {
-        type = types.int;
-        default = 1;
-        description = lib.mdDoc "Org id. will default to orgId 1 if not specified.";
-      };
       uid = mkOption {
         type = types.nullOr types.str;
         default = null;
@@ -131,114 +113,51 @@ let
       };
       url = mkOption {
         type = types.str;
+        default = "localhost";
         description = lib.mdDoc "Url of the datasource.";
       };
-      password = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = lib.mdDoc "Database password, if used.";
-      };
-      user = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = lib.mdDoc "Database user, if used.";
-      };
-      database = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = lib.mdDoc "Database name, if used.";
-      };
-      basicAuth = mkOption {
-        type = types.nullOr types.bool;
-        default = null;
-        description = lib.mdDoc "Enable/disable basic auth.";
-      };
-      basicAuthUser = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = lib.mdDoc "Basic auth username.";
-      };
-      basicAuthPassword = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = lib.mdDoc "Basic auth password.";
-      };
-      withCredentials = mkOption {
-        type = types.bool;
-        default = false;
-        description = lib.mdDoc "Enable/disable with credentials headers.";
-      };
-      isDefault = mkOption {
+      editable = mkOption {
         type = types.bool;
         default = false;
-        description = lib.mdDoc "Mark as default datasource. Max one per org.";
+        description = lib.mdDoc "Allow users to edit datasources from the UI.";
       };
       jsonData = mkOption {
         type = types.nullOr types.attrs;
         default = null;
-        description = lib.mdDoc "Datasource specific configuration.";
+        description = lib.mdDoc "Extra data for datasource plugins.";
       };
       secureJsonData = mkOption {
         type = types.nullOr types.attrs;
         default = null;
-        description = lib.mdDoc "Datasource specific secure configuration.";
-      };
-      version = mkOption {
-        type = types.int;
-        default = 1;
-        description = lib.mdDoc "Version.";
-      };
-      editable = mkOption {
-        type = types.bool;
-        default = false;
-        description = lib.mdDoc "Allow users to edit datasources from the UI.";
+        description = lib.mdDoc ''
+          Datasource specific secure configuration. Please note that the contents of this option
+          will end up in a world-readable Nix store. Use the file provider
+          pointing at a reasonably secured file in the local filesystem
+          to work around that. Look at the documentation for details:
+          <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
+        '';
       };
     };
   };
 
   # http://docs.grafana.org/administration/provisioning/#dashboards
   grafanaTypes.dashboardConfig = types.submodule {
+    freeformType = provisioningSettingsFormat.type;
+
     options = {
       name = mkOption {
         type = types.str;
         default = "default";
-        description = lib.mdDoc "Provider name.";
-      };
-      orgId = mkOption {
-        type = types.int;
-        default = 1;
-        description = lib.mdDoc "Organization ID.";
-      };
-      folder = mkOption {
-        type = types.str;
-        default = "";
-        description = lib.mdDoc "Add dashboards to the specified folder.";
+        description = lib.mdDoc "A unique provider name.";
       };
       type = mkOption {
         type = types.str;
         default = "file";
         description = lib.mdDoc "Dashboard provider type.";
       };
-      disableDeletion = mkOption {
-        type = types.bool;
-        default = false;
-        description = lib.mdDoc "Disable deletion when JSON file is removed.";
-      };
-      updateIntervalSeconds = mkOption {
-        type = types.int;
-        default = 10;
-        description = lib.mdDoc "How often Grafana will scan for changed dashboards.";
-      };
-      options = {
-        path = mkOption {
-          type = types.path;
-          description = lib.mdDoc "Path grafana will watch for dashboards.";
-        };
-        foldersFromFilesStructure = mkOption {
-          type = types.bool;
-          default = false;
-          description = lib.mdDoc "Use folder names from filesystem to create folders in Grafana.";
-        };
+      options.path = mkOption {
+        type = types.path;
+        description = lib.mdDoc "Path grafana will watch for dashboards. Required when using the 'file' type.";
       };
     };
   };
@@ -296,75 +215,88 @@ let
       secure_settings = mkOption {
         type = types.nullOr types.attrs;
         default = null;
-        description = lib.mdDoc "Secure settings for the notifier type.";
+        description = lib.mdDoc ''
+          Secure settings for the notifier type. Please note that the contents of this option
+          will end up in a world-readable Nix store. Use the file provider
+          pointing at a reasonably secured file in the local filesystem
+          to work around that. Look at the documentation for details:
+          <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
+        '';
       };
     };
   };
 in {
-  options.services.grafana = {
-    enable = mkEnableOption "grafana";
-
-    protocol = mkOption {
-      description = lib.mdDoc "Which protocol to listen.";
-      default = "http";
-      type = types.enum ["http" "https" "socket"];
-    };
-
-    addr = mkOption {
-      description = lib.mdDoc "Listening address.";
-      default = "127.0.0.1";
-      type = types.str;
-    };
-
-    port = mkOption {
-      description = lib.mdDoc "Listening port.";
-      default = 3000;
-      type = types.port;
-    };
-
-    socket = mkOption {
-      description = lib.mdDoc "Listening socket.";
-      default = "/run/grafana/grafana.sock";
-      type = types.str;
-    };
-
-    domain = mkOption {
-      description = lib.mdDoc "The public facing domain name used to access grafana from a browser.";
-      default = "localhost";
-      type = types.str;
-    };
-
-    rootUrl = mkOption {
-      description = lib.mdDoc "Full public facing url.";
-      default = "%(protocol)s://%(domain)s:%(http_port)s/";
-      type = types.str;
-    };
-
-    certFile = mkOption {
-      description = lib.mdDoc "Cert file for ssl.";
-      default = "";
-      type = types.str;
-    };
+  imports = [
+    (mkRenamedOptionModule [ "services" "grafana" "protocol" ] [ "services" "grafana" "settings" "server" "protocol" ])
+    (mkRenamedOptionModule [ "services" "grafana" "addr" ] [ "services" "grafana" "settings" "server" "http_addr" ])
+    (mkRenamedOptionModule [ "services" "grafana" "port" ] [ "services" "grafana" "settings" "server" "http_port" ])
+    (mkRenamedOptionModule [ "services" "grafana" "domain" ] [ "services" "grafana" "settings" "server" "domain" ])
+    (mkRenamedOptionModule [ "services" "grafana" "rootUrl" ] [ "services" "grafana" "settings" "server" "root_url" ])
+    (mkRenamedOptionModule [ "services" "grafana" "staticRootPath" ] [ "services" "grafana" "settings" "server" "static_root_path" ])
+    (mkRenamedOptionModule [ "services" "grafana" "certFile" ] [ "services" "grafana" "settings" "server" "cert_file" ])
+    (mkRenamedOptionModule [ "services" "grafana" "certKey" ] [ "services" "grafana" "settings" "server" "cert_key" ])
+    (mkRenamedOptionModule [ "services" "grafana" "socket" ] [ "services" "grafana" "settings" "server" "socket" ])
+    (mkRenamedOptionModule [ "services" "grafana" "database" "type" ] [ "services" "grafana" "settings" "database" "type" ])
+    (mkRenamedOptionModule [ "services" "grafana" "database" "host" ] [ "services" "grafana" "settings" "database" "host" ])
+    (mkRenamedOptionModule [ "services" "grafana" "database" "name" ] [ "services" "grafana" "settings" "database" "name" ])
+    (mkRenamedOptionModule [ "services" "grafana" "database" "user" ] [ "services" "grafana" "settings" "database" "user" ])
+    (mkRenamedOptionModule [ "services" "grafana" "database" "password" ] [ "services" "grafana" "settings" "database" "password" ])
+    (mkRenamedOptionModule [ "services" "grafana" "database" "path" ] [ "services" "grafana" "settings" "database" "path" ])
+    (mkRenamedOptionModule [ "services" "grafana" "database" "connMaxLifetime" ] [ "services" "grafana" "settings" "database" "conn_max_lifetime" ])
+    (mkRenamedOptionModule [ "services" "grafana" "security" "adminUser" ] [ "services" "grafana" "settings" "security" "admin_user" ])
+    (mkRenamedOptionModule [ "services" "grafana" "security" "adminPassword" ] [ "services" "grafana" "settings" "security" "admin_password" ])
+    (mkRenamedOptionModule [ "services" "grafana" "security" "secretKey" ] [ "services" "grafana" "settings" "security" "secret_key" ])
+    (mkRenamedOptionModule [ "services" "grafana" "server" "serveFromSubPath" ] [ "services" "grafana" "settings" "server" "serve_from_sub_path" ])
+    (mkRenamedOptionModule [ "services" "grafana" "smtp" "enable" ] [ "services" "grafana" "settings" "smtp" "enabled" ])
+    (mkRenamedOptionModule [ "services" "grafana" "smtp" "user" ] [ "services" "grafana" "settings" "smtp" "user" ])
+    (mkRenamedOptionModule [ "services" "grafana" "smtp" "password" ] [ "services" "grafana" "settings" "smtp" "password" ])
+    (mkRenamedOptionModule [ "services" "grafana" "smtp" "fromAddress" ] [ "services" "grafana" "settings" "smtp" "from_address" ])
+    (mkRenamedOptionModule [ "services" "grafana" "users" "allowSignUp" ] [ "services" "grafana" "settings" "users" "allow_sign_up" ])
+    (mkRenamedOptionModule [ "services" "grafana" "users" "allowOrgCreate" ] [ "services" "grafana" "settings" "users" "allow_org_create" ])
+    (mkRenamedOptionModule [ "services" "grafana" "users" "autoAssignOrg" ] [ "services" "grafana" "settings" "users" "auto_assign_org" ])
+    (mkRenamedOptionModule [ "services" "grafana" "users" "autoAssignOrgRole" ] [ "services" "grafana" "settings" "users" "auto_assign_org_role" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "disableLoginForm" ] [ "services" "grafana" "settings" "auth" "disable_login_form" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "anonymous" "enable" ] [ "services" "grafana" "settings" "auth.anonymous" "enabled" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "anonymous" "org_name" ] [ "services" "grafana" "settings" "auth.anonymous" "org_name" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "anonymous" "org_role" ] [ "services" "grafana" "settings" "auth.anonymous" "org_role" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "azuread" "enable" ] [ "services" "grafana" "settings" "auth.azuread" "enabled" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "azuread" "allowSignUp" ] [ "services" "grafana" "settings" "auth.azuread" "allow_sign_up" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "azuread" "clientId" ] [ "services" "grafana" "settings" "auth.azuread" "client_id" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "azuread" "allowedDomains" ] [ "services" "grafana" "settings" "auth.azuread" "allowed_domains" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "azuread" "allowedGroups" ] [ "services" "grafana" "settings" "auth.azuread" "allowed_groups" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "google" "enable" ] [ "services" "grafana" "settings" "auth.google" "enabled" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "google" "allowSignUp" ] [ "services" "grafana" "settings" "auth.google" "allow_sign_up" ])
+    (mkRenamedOptionModule [ "services" "grafana" "auth" "google" "clientId" ] [ "services" "grafana" "settings" "auth.google" "client_id" ])
+    (mkRenamedOptionModule [ "services" "grafana" "analytics" "reporting" "enable" ] [ "services" "grafana" "settings" "analytics" "reporting_enabled" ])
+
+    (mkRemovedOptionModule [ "services" "grafana" "database" "passwordFile" ] ''
+      This option has been removed. Use 'services.grafana.settings.database.password' with file provider instead.
+    '')
+    (mkRemovedOptionModule [ "services" "grafana" "security" "adminPasswordFile" ] ''
+      This option has been removed. Use 'services.grafana.settings.security.admin_password' with file provider instead.
+    '')
+    (mkRemovedOptionModule [ "services" "grafana" "security" "secretKeyFile" ] ''
+      This option has been removed. Use 'services.grafana.settings.security.secret_key' with file provider instead.
+    '')
+    (mkRemovedOptionModule [ "services" "grafana" "smtp" "passwordFile" ] ''
+      This option has been removed. Use 'services.grafana.settings.smtp.password' with file provider instead.
+    '')
+    (mkRemovedOptionModule [ "services" "grafana" "auth" "azuread" "clientSecretFile" ] ''
+      This option has been removed. Use 'services.grafana.settings.azuread.client_secret' with file provider instead.
+    '')
+    (mkRemovedOptionModule [ "services" "grafana" "auth" "google" "clientSecretFile" ] ''
+      This option has been removed. Use 'services.grafana.settings.google.client_secret' with file provider instead.
+    '')
+    (mkRemovedOptionModule [ "services" "grafana" "extraOptions" ] ''
+      This option has been removed. Use 'services.grafana.settings' instead. For a detailed migration guide, please
+      review the release notes of NixOS 22.11.
+    '')
+
+    (mkRemovedOptionModule [ "services" "grafana" "auth" "azuread" "tenantId" ] "This option has been deprecated upstream.")
+  ];
 
-    certKey = mkOption {
-      description = lib.mdDoc "Cert key for ssl.";
-      default = "";
-      type = types.str;
-    };
-
-    staticRootPath = mkOption {
-      description = lib.mdDoc "Root path for static assets.";
-      default = "${cfg.package}/share/grafana/public";
-      defaultText = literalExpression ''"''${package}/share/grafana/public"'';
-      type = types.str;
-    };
-
-    package = mkOption {
-      description = lib.mdDoc "Package to use.";
-      default = pkgs.grafana;
-      defaultText = literalExpression "pkgs.grafana";
-      type = types.package;
-    };
+  options.services.grafana = {
+    enable = mkEnableOption (lib.mdDoc "grafana");
 
     declarativePlugins = mkOption {
       type = with types; nullOr (listOf path);
@@ -377,348 +309,944 @@ in {
       apply = x: if isList x then lib.unique x else x;
     };
 
+    package = mkOption {
+      description = lib.mdDoc "Package to use.";
+      default = pkgs.grafana;
+      defaultText = literalExpression "pkgs.grafana";
+      type = types.package;
+    };
+
     dataDir = mkOption {
       description = lib.mdDoc "Data directory.";
       default = "/var/lib/grafana";
       type = types.path;
     };
 
-    database = {
-      type = mkOption {
-        description = lib.mdDoc "Database type.";
-        default = "sqlite3";
-        type = types.enum ["mysql" "sqlite3" "postgres"];
-      };
-
-      host = mkOption {
-        description = lib.mdDoc "Database host.";
-        default = "127.0.0.1:3306";
-        type = types.str;
-      };
-
-      name = mkOption {
-        description = lib.mdDoc "Database name.";
-        default = "grafana";
-        type = types.str;
+    settings = mkOption {
+      description = lib.mdDoc ''
+        Grafana settings. See <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/>
+        for available options. INI format is used.
+      '';
+      type = types.submodule {
+        freeformType = settingsFormatIni.type;
+
+        options = {
+          paths = {
+            plugins = mkOption {
+              description = lib.mdDoc "Directory where grafana will automatically scan and look for plugins";
+              default = if (cfg.declarativePlugins == null) then "${cfg.dataDir}/plugins" else declarativePlugins;
+              defaultText = literalExpression "if (cfg.declarativePlugins == null) then \"\${cfg.dataDir}/plugins\" else declarativePlugins";
+              type = types.path;
+            };
+
+            provisioning = mkOption {
+              description = lib.mdDoc ''
+                Folder that contains provisioning config files that grafana will apply on startup and while running.
+                Don't change the value of this option if you are planning to use `services.grafana.provision` options.
+              '';
+              default = provisionConfDir;
+              defaultText = "directory with links to files generated from services.grafana.provision";
+              type = types.path;
+            };
+          };
+
+          server = {
+            protocol = mkOption {
+              description = lib.mdDoc "Which protocol to listen.";
+              default = "http";
+              type = types.enum ["http" "https" "h2" "socket"];
+            };
+
+            http_addr = mkOption {
+              type = types.str;
+              default = "127.0.0.1";
+              description = lib.mdDoc ''
+                Listening address.
+
+                ::: {.note}
+                This setting intentionally varies from upstream's default to be a bit more secure by default.
+                :::
+              '';
+            };
+
+            http_port = mkOption {
+              description = lib.mdDoc "Listening port.";
+              default = 3000;
+              type = types.port;
+            };
+
+            domain = mkOption {
+              description = lib.mdDoc "The public facing domain name used to access grafana from a browser.";
+              default = "localhost";
+              type = types.str;
+            };
+
+            root_url = mkOption {
+              description = lib.mdDoc "Full public facing url.";
+              default = "%(protocol)s://%(domain)s:%(http_port)s/";
+              type = types.str;
+            };
+
+            static_root_path = mkOption {
+              description = lib.mdDoc "Root path for static assets.";
+              default = "${cfg.package}/share/grafana/public";
+              defaultText = literalExpression ''"''${package}/share/grafana/public"'';
+              type = types.str;
+            };
+
+            enable_gzip = mkOption {
+              description = lib.mdDoc ''
+                Set this option to true to enable HTTP compression, this can improve transfer speed and bandwidth utilization.
+                It is recommended that most users set it to true. By default it is set to false for compatibility reasons.
+              '';
+              default = false;
+              type = types.bool;
+            };
+
+            cert_file = mkOption {
+              description = lib.mdDoc "Cert file for ssl.";
+              default = "";
+              type = types.str;
+            };
+
+            cert_key = mkOption {
+              description = lib.mdDoc "Cert key for ssl.";
+              default = "";
+              type = types.str;
+            };
+
+            socket = mkOption {
+              description = lib.mdDoc "Path where the socket should be created when protocol=socket. Make sure that Grafana has appropriate permissions before you change this setting.";
+              default = "/run/grafana/grafana.sock";
+              type = types.str;
+            };
+          };
+
+          database = {
+            type = mkOption {
+              description = lib.mdDoc "Database type.";
+              default = "sqlite3";
+              type = types.enum ["mysql" "sqlite3" "postgres"];
+            };
+
+            host = mkOption {
+              description = lib.mdDoc "Database host.";
+              default = "127.0.0.1:3306";
+              type = types.str;
+            };
+
+            name = mkOption {
+              description = lib.mdDoc "Database name.";
+              default = "grafana";
+              type = types.str;
+            };
+
+            user = mkOption {
+              description = lib.mdDoc "Database user.";
+              default = "root";
+              type = types.str;
+            };
+
+            password = mkOption {
+              description = lib.mdDoc ''
+                Database password. Please note that the contents of this option
+                will end up in a world-readable Nix store. Use the file provider
+                pointing at a reasonably secured file in the local filesystem
+                to work around that. Look at the documentation for details:
+                <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
+              '';
+              default = "";
+              type = types.str;
+            };
+
+            path = mkOption {
+              description = lib.mdDoc "Only applicable to sqlite3 database. The file path where the database will be stored.";
+              default = "${cfg.dataDir}/data/grafana.db";
+              defaultText = literalExpression ''"''${config.${opt.dataDir}}/data/grafana.db"'';
+              type = types.path;
+            };
+          };
+
+          security = {
+            admin_user = mkOption {
+              description = lib.mdDoc "Default admin username.";
+              default = "admin";
+              type = types.str;
+            };
+
+            admin_password = mkOption {
+              description = lib.mdDoc ''
+                Default admin password. Please note that the contents of this option
+                will end up in a world-readable Nix store. Use the file provider
+                pointing at a reasonably secured file in the local filesystem
+                to work around that. Look at the documentation for details:
+                <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
+              '';
+              default = "admin";
+              type = types.str;
+            };
+
+            secret_key = mkOption {
+              description = lib.mdDoc ''
+                Secret key used for signing. Please note that the contents of this option
+                will end up in a world-readable Nix store. Use the file provider
+                pointing at a reasonably secured file in the local filesystem
+                to work around that. Look at the documentation for details:
+                <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
+              '';
+              default = "SW2YcwTIb9zpOOhoPsMm";
+              type = types.str;
+            };
+          };
+
+          smtp = {
+            enabled = mkOption {
+              description = lib.mdDoc "Whether to enable SMTP.";
+              default = false;
+              type = types.bool;
+            };
+            host = mkOption {
+              description = lib.mdDoc "Host to connect to.";
+              default = "localhost:25";
+              type = types.str;
+            };
+            user = mkOption {
+              description = lib.mdDoc "User used for authentication.";
+              default = "";
+              type = types.str;
+            };
+            password = mkOption {
+              description = lib.mdDoc ''
+                Password used for authentication. Please note that the contents of this option
+                will end up in a world-readable Nix store. Use the file provider
+                pointing at a reasonably secured file in the local filesystem
+                to work around that. Look at the documentation for details:
+                <https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider>
+              '';
+              default = "";
+              type = types.str;
+            };
+            from_address = mkOption {
+              description = lib.mdDoc "Email address used for sending.";
+              default = "admin@grafana.localhost";
+              type = types.str;
+            };
+          };
+
+          users = {
+            allow_sign_up = mkOption {
+              description = lib.mdDoc "Disable user signup / registration.";
+              default = false;
+              type = types.bool;
+            };
+
+            allow_org_create = mkOption {
+              description = lib.mdDoc "Whether user is allowed to create organizations.";
+              default = false;
+              type = types.bool;
+            };
+
+            auto_assign_org = mkOption {
+              description = lib.mdDoc "Whether to automatically assign new users to default org.";
+              default = true;
+              type = types.bool;
+            };
+
+            auto_assign_org_role = mkOption {
+              description = lib.mdDoc "Default role new users will be auto assigned.";
+              default = "Viewer";
+              type = types.enum ["Viewer" "Editor" "Admin"];
+            };
+          };
+
+          analytics.reporting_enabled = mkOption {
+            description = lib.mdDoc "Whether to allow anonymous usage reporting to stats.grafana.net.";
+            default = true;
+            type = types.bool;
+          };
+        };
       };
+    };
 
-      user = mkOption {
-        description = lib.mdDoc "Database user.";
-        default = "root";
-        type = types.str;
-      };
+    provision = {
+      enable = mkEnableOption (lib.mdDoc "provision");
 
-      password = mkOption {
+      datasources = mkOption {
         description = lib.mdDoc ''
-          Database password.
-          This option is mutual exclusive with the passwordFile option.
+          Declaratively provision Grafana's datasources.
         '';
-        default = "";
-        type = types.str;
-      };
+        default = {};
+        type = submodule' {
+          options.settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana datasource configuration in Nix. Can't be used with
+              [](#opt-services.grafana.provision.datasources.path) simultaneously. See
+              <https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                datasources = mkOption {
+                  description = lib.mdDoc "List of datasources to insert/update.";
+                  default = [];
+                  type = types.listOf grafanaTypes.datasourceConfig;
+                };
+
+                deleteDatasources = mkOption {
+                  description = lib.mdDoc "List of datasources that should be deleted from the database.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the datasource to delete.";
+                      type = types.str;
+                    };
+
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID of the datasource to delete.";
+                      type = types.int;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                datasources = [{
+                  name = "Graphite";
+                  type = "graphite";
+                }];
+
+                deleteDatasources = [{
+                  name = "Graphite";
+                  orgId = 1;
+                }];
+              }
+            '';
+          };
 
-      passwordFile = mkOption {
-        description = lib.mdDoc ''
-          File that containts the database password.
-          This option is mutual exclusive with the password option.
-        '';
-        default = null;
-        type = types.nullOr types.path;
+          options.path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML datasource configuration. Can't be used with
+              [](#opt-services.grafana.provision.datasources.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+        };
       };
 
-      path = mkOption {
-        description = lib.mdDoc "Database path.";
-        default = "${cfg.dataDir}/data/grafana.db";
-        defaultText = literalExpression ''"''${config.${opt.dataDir}}/data/grafana.db"'';
-        type = types.path;
-      };
 
-      connMaxLifetime = mkOption {
+      dashboards = mkOption {
         description = lib.mdDoc ''
-          Sets the maximum amount of time (in seconds) a connection may be reused.
-          For MySQL this setting should be shorter than the `wait_timeout' variable.
+          Declaratively provision Grafana's dashboards.
         '';
-        default = "unlimited";
-        example = 14400;
-        type = types.either types.int (types.enum [ "unlimited" ]);
-      };
-    };
+        default = {};
+        type = submodule' {
+          options.settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana dashboard configuration in Nix. Can't be used with
+              [](#opt-services.grafana.provision.dashboards.path) simultaneously. See
+              <https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options.apiVersion = mkOption {
+                description = lib.mdDoc "Config file version.";
+                default = 1;
+                type = types.int;
+              };
+
+              options.providers = mkOption {
+                description = lib.mdDoc "List of dashboards to insert/update.";
+                default = [];
+                type = types.listOf grafanaTypes.dashboardConfig;
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                providers = [{
+                    name = "default";
+                    options.path = "/var/lib/grafana/dashboards";
+                }];
+              }
+            '';
+          };
 
-    provision = {
-      enable = mkEnableOption "provision";
-      datasources = mkOption {
-        description = lib.mdDoc "Grafana datasources configuration.";
-        default = [];
-        type = types.listOf grafanaTypes.datasourceConfig;
-        apply = x: map _filter x;
-      };
-      dashboards = mkOption {
-        description = lib.mdDoc "Grafana dashboard configuration.";
-        default = [];
-        type = types.listOf grafanaTypes.dashboardConfig;
-        apply = x: map _filter x;
+          options.path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML dashboard configuration. Can't be used with
+              [](#opt-services.grafana.provision.dashboards.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+        };
       };
+
+
       notifiers = mkOption {
         description = lib.mdDoc "Grafana notifier configuration.";
         default = [];
         type = types.listOf grafanaTypes.notifierConfig;
         apply = x: map _filter x;
       };
-    };
-
-    security = {
-      adminUser = mkOption {
-        description = lib.mdDoc "Default admin username.";
-        default = "admin";
-        type = types.str;
-      };
-
-      adminPassword = mkOption {
-        description = lib.mdDoc ''
-          Default admin password.
-          This option is mutual exclusive with the adminPasswordFile option.
-        '';
-        default = "admin";
-        type = types.str;
-      };
-
-      adminPasswordFile = mkOption {
-        description = lib.mdDoc ''
-          Default admin password.
-          This option is mutual exclusive with the `adminPassword` option.
-        '';
-        default = null;
-        type = types.nullOr types.path;
-      };
-
-      secretKey = mkOption {
-        description = lib.mdDoc "Secret key used for signing.";
-        default = "SW2YcwTIb9zpOOhoPsMm";
-        type = types.str;
-      };
-
-      secretKeyFile = mkOption {
-        description = lib.mdDoc "Secret key used for signing.";
-        default = null;
-        type = types.nullOr types.path;
-      };
-    };
-
-    server = {
-      serveFromSubPath = mkOption {
-        description = lib.mdDoc "Serve Grafana from subpath specified in rootUrl setting";
-        default = false;
-        type = types.bool;
-      };
-    };
-
-    smtp = {
-      enable = mkEnableOption "smtp";
-      host = mkOption {
-        description = lib.mdDoc "Host to connect to.";
-        default = "localhost:25";
-        type = types.str;
-      };
-      user = mkOption {
-        description = lib.mdDoc "User used for authentication.";
-        default = "";
-        type = types.str;
-      };
-      password = mkOption {
-        description = lib.mdDoc ''
-          Password used for authentication.
-          This option is mutual exclusive with the passwordFile option.
-        '';
-        default = "";
-        type = types.str;
-      };
-      passwordFile = mkOption {
-        description = lib.mdDoc ''
-          Password used for authentication.
-          This option is mutual exclusive with the password option.
-        '';
-        default = null;
-        type = types.nullOr types.path;
-      };
-      fromAddress = mkOption {
-        description = lib.mdDoc "Email address used for sending.";
-        default = "admin@grafana.localhost";
-        type = types.str;
-      };
-    };
-
-    users = {
-      allowSignUp = mkOption {
-        description = lib.mdDoc "Disable user signup / registration.";
-        default = false;
-        type = types.bool;
-      };
-
-      allowOrgCreate = mkOption {
-        description = lib.mdDoc "Whether user is allowed to create organizations.";
-        default = false;
-        type = types.bool;
-      };
-
-      autoAssignOrg = mkOption {
-        description = lib.mdDoc "Whether to automatically assign new users to default org.";
-        default = true;
-        type = types.bool;
-      };
-
-      autoAssignOrgRole = mkOption {
-        description = lib.mdDoc "Default role new users will be auto assigned.";
-        default = "Viewer";
-        type = types.enum ["Viewer" "Editor"];
-      };
-    };
 
-    auth = {
-      disableLoginForm = mkOption {
-        description = lib.mdDoc "Set to true to disable (hide) the login form, useful if you use OAuth";
-        default = false;
-        type = types.bool;
-      };
 
-      anonymous = {
-        enable = mkOption {
-          description = lib.mdDoc "Whether to allow anonymous access.";
-          default = false;
-          type = types.bool;
-        };
-        org_name = mkOption {
-          description = lib.mdDoc "Which organization to allow anonymous access to.";
-          default = "Main Org.";
-          type = types.str;
-        };
-        org_role = mkOption {
-          description = lib.mdDoc "Which role anonymous users have in the organization.";
-          default = "Viewer";
-          type = types.str;
-        };
-      };
-      azuread = {
-        enable = mkOption {
-          description = lib.mdDoc "Whether to allow Azure AD OAuth.";
-          default = false;
-          type = types.bool;
-        };
-        allowSignUp = mkOption {
-          description = lib.mdDoc "Whether to allow sign up with Azure AD OAuth.";
-          default = false;
-          type = types.bool;
-        };
-        clientId = mkOption {
-          description = lib.mdDoc "Azure AD OAuth client ID.";
-          default = "";
-          type = types.str;
-        };
-        clientSecretFile = mkOption {
-          description = lib.mdDoc "Azure AD OAuth client secret.";
-          default = null;
-          type = types.nullOr types.path;
-        };
-        tenantId = mkOption {
-          description = lib.mdDoc ''
-            Tenant id used to create auth and token url. Default to "common"
-            , let user sign in with any tenant.
+      alerting = {
+        rules = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML rules configuration. Can't be used with
+              [](#opt-services.grafana.provision.alerting.rules.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
             '';
-          default = "common";
-          type = types.str;
-        };
-        allowedDomains = mkOption {
-          description = lib.mdDoc ''
-            To limit access to authenticated users who are members of one or more groups,
-            set allowedGroups to a comma- or space-separated list of group object IDs.
-            You can find object IDs for a specific group on the Azure portal.
-          '';
-          default = "";
-          type = types.str;
-        };
-        allowedGroups = mkOption {
-          description = lib.mdDoc ''
-            Limits access to users who belong to specific domains.
-            Separate domains with space or comma.
-          '';
-          default = "";
-          type = types.str;
-        };
-      };
-      google = {
-        enable = mkOption {
-          description = lib.mdDoc "Whether to allow Google OAuth2.";
-          default = false;
-          type = types.bool;
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana rules configuration in Nix. Can't be used with
+              [](#opt-services.grafana.provision.alerting.rules.path) simultaneously. See
+              <https://grafana.com/docs/grafana/latest/administration/provisioning/#rules>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                groups = mkOption {
+                  description = lib.mdDoc "List of rule groups to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the rule group. Required.";
+                      type = types.str;
+                    };
+
+                    options.folder = mkOption {
+                      description = lib.mdDoc "Name of the folder the rule group will be stored in. Required.";
+                      type = types.str;
+                    };
+
+                    options.interval = mkOption {
+                      description = lib.mdDoc "Interval that the rule group should be evaluated at. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteRules = mkOption {
+                  description = lib.mdDoc "List of alert rule UIDs that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.uid = mkOption {
+                      description = lib.mdDoc "Unique identifier for the rule. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                groups = [{
+                  orgId = 1;
+                  name = "my_rule_group";
+                  folder = "my_first_folder";
+                  interval = "60s";
+                  rules = [{
+                    uid = "my_id_1";
+                    title = "my_first_rule";
+                    condition = "A";
+                    data = [{
+                      refId = "A";
+                      datasourceUid = "-100";
+                      model = {
+                        conditions = [{
+                          evaluator = {
+                            params = [ 3 ];
+                            type = "git";
+                          };
+                          operator.type = "and";
+                          query.params = [ "A" ];
+                          reducer.type = "last";
+                          type = "query";
+                        }];
+                        datasource = {
+                          type = "__expr__";
+                          uid = "-100";
+                        };
+                        expression = "1==0";
+                        intervalMs = 1000;
+                        maxDataPoints = 43200;
+                        refId = "A";
+                        type = "math";
+                      };
+                    }];
+                    dashboardUid = "my_dashboard";
+                    panelId = 123;
+                    noDataState = "Alerting";
+                    for = "60s";
+                    annotations.some_key = "some_value";
+                    labels.team = "sre_team1";
+                  }];
+                }];
+
+                deleteRules = [{
+                  orgId = 1;
+                  uid = "my_id_1";
+                }];
+              }
+            '';
+          };
         };
-        allowSignUp = mkOption {
-          description = lib.mdDoc "Whether to allow sign up with Google OAuth2.";
-          default = false;
-          type = types.bool;
+
+        contactPoints = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML contact points configuration. Can't be used with
+              [](#opt-services.grafana.provision.alerting.contactPoints.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana contact points configuration in Nix. Can't be used with
+              [](#opt-services.grafana.provision.alerting.contactPoints.path) simultaneously. See
+              <https://grafana.com/docs/grafana/latest/administration/provisioning/#contact-points>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                contactPoints = mkOption {
+                  description = lib.mdDoc "List of contact points to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the contact point. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteContactPoints = mkOption {
+                  description = lib.mdDoc "List of receivers that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1.";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.uid = mkOption {
+                      description = lib.mdDoc "Unique identifier for the receiver. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                contactPoints = [{
+                  orgId = 1;
+                  name = "cp_1";
+                  receivers = [{
+                    uid = "first_uid";
+                    type = "prometheus-alertmanager";
+                    settings.url = "http://test:9000";
+                  }];
+                }];
+
+                deleteContactPoints = [{
+                  orgId = 1;
+                  uid = "first_uid";
+                }];
+              }
+            '';
+          };
         };
-        clientId = mkOption {
-          description = lib.mdDoc "Google OAuth2 client ID.";
-          default = "";
-          type = types.str;
+
+        policies = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML notification policies configuration. Can't be used with
+              [](#opt-services.grafana.provision.alerting.policies.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana notification policies configuration in Nix. Can't be used with
+              [](#opt-services.grafana.provision.alerting.policies.path) simultaneously. See
+              <https://grafana.com/docs/grafana/latest/administration/provisioning/#notification-policies>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                policies = mkOption {
+                  description = lib.mdDoc "List of contact points to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+                  });
+                };
+
+                resetPolicies = mkOption {
+                  description = lib.mdDoc "List of orgIds that should be reset to the default policy.";
+                  default = [];
+                  type = types.listOf types.int;
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                policies = [{
+                  orgId = 1;
+                  receiver = "grafana-default-email";
+                  group_by = [ "..." ];
+                  matchers = [
+                    "alertname = Watchdog"
+                    "severity =~ \"warning|critical\""
+                  ];
+                  mute_time_intervals = [
+                    "abc"
+                  ];
+                  group_wait = "30s";
+                  group_interval = "5m";
+                  repeat_interval = "4h";
+                }];
+
+                resetPolicies = [
+                  1
+                ];
+              }
+            '';
+          };
         };
-        clientSecretFile = mkOption {
-          description = lib.mdDoc "Google OAuth2 client secret.";
-          default = null;
-          type = types.nullOr types.path;
+
+        templates = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML templates configuration. Can't be used with
+              [](#opt-services.grafana.provision.alerting.templates.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana templates configuration in Nix. Can't be used with
+              [](#opt-services.grafana.provision.alerting.templates.path) simultaneously. See
+              <https://grafana.com/docs/grafana/latest/administration/provisioning/#templates>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                templates = mkOption {
+                  description = lib.mdDoc "List of templates to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the template, must be unique. Required.";
+                      type = types.str;
+                    };
+
+                    options.template = mkOption {
+                      description = lib.mdDoc "Alerting with a custom text template";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteTemplates = mkOption {
+                  description = lib.mdDoc "List of alert rule UIDs that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1.";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the template, must be unique. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                templates = [{
+                  orgId = 1;
+                  name = "my_first_template";
+                  template = "Alerting with a custom text template";
+                }];
+
+                deleteTemplates = [{
+                  orgId = 1;
+                  name = "my_first_template";
+                }];
+              }
+            '';
+          };
         };
-      };
-    };
 
-    analytics.reporting = {
-      enable = mkOption {
-        description = lib.mdDoc "Whether to allow anonymous usage reporting to stats.grafana.net.";
-        default = true;
-        type = types.bool;
+        muteTimings = {
+          path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML mute timings configuration. Can't be used with
+              [](#opt-services.grafana.provision.alerting.muteTimings.settings) simultaneously.
+              Can be either a directory or a single YAML file. Will end up in the store.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+
+          settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana mute timings configuration in Nix. Can't be used with
+              [](#opt-services.grafana.provision.alerting.muteTimings.path) simultaneously. See
+              <https://grafana.com/docs/grafana/latest/administration/provisioning/#mute-timings>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options = {
+                apiVersion = mkOption {
+                  description = lib.mdDoc "Config file version.";
+                  default = 1;
+                  type = types.int;
+                };
+
+                muteTimes = mkOption {
+                  description = lib.mdDoc "List of mute time intervals to import or update.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    freeformType = provisioningSettingsFormat.type;
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the mute time interval, must be unique. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+
+                deleteMuteTimes = mkOption {
+                  description = lib.mdDoc "List of mute time intervals that should be deleted.";
+                  default = [];
+                  type = types.listOf (types.submodule {
+                    options.orgId = mkOption {
+                      description = lib.mdDoc "Organization ID, default = 1.";
+                      default = 1;
+                      type = types.int;
+                    };
+
+                    options.name = mkOption {
+                      description = lib.mdDoc "Name of the mute time interval, must be unique. Required.";
+                      type = types.str;
+                    };
+                  });
+                };
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                muteTimes = [{
+                  orgId = 1;
+                  name = "mti_1";
+                  time_intervals = [{
+                    times = [{
+                      start_time = "06:00";
+                      end_time = "23:59";
+                    }];
+                    weekdays = [
+                      "monday:wednesday"
+                      "saturday"
+                      "sunday"
+                    ];
+                    months = [
+                      "1:3"
+                      "may:august"
+                      "december"
+                    ];
+                    years = [
+                      "2020:2022"
+                      "2030"
+                    ];
+                    days_of_month = [
+                      "1:5"
+                      "-3:-1"
+                    ];
+                  }];
+                }];
+
+                deleteMuteTimes = [{
+                  orgId = 1;
+                  name = "mti_1";
+                }];
+              }
+            '';
+          };
+        };
       };
     };
-
-    extraOptions = mkOption {
-      description = lib.mdDoc ''
-        Extra configuration options passed as env variables as specified in
-        [documentation](http://docs.grafana.org/installation/configuration/),
-        but without GF_ prefix
-      '';
-      default = {};
-      type = with types; attrsOf (either str path);
-    };
   };
 
   config = mkIf cfg.enable {
-    warnings = flatten [
-      (optional (
-        cfg.database.password != opt.database.password.default ||
-        cfg.security.adminPassword != opt.security.adminPassword.default
-      ) "Grafana passwords will be stored as plaintext in the Nix store!")
-      (optional (
-        any (x: x.password != null || x.basicAuthPassword != null || x.secureJsonData != null) cfg.provision.datasources
-      ) "Datasource passwords will be stored as plaintext in the Nix store!")
+    warnings = let
+      doesntUseFileProvider = opt: defaultValue:
+        let
+          regex = "${optionalString (defaultValue != null) "^${defaultValue}$|"}^\\$__(file|env)\\{.*}$|^\\$[^_\\$][^ ]+$";
+        in builtins.match regex opt == null;
+    in
+      # Ensure that no custom credentials are leaked into the Nix store. Unless the default value
+      # is specified, this can be achieved by using the file/env provider:
+      # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion
       (optional (
+        doesntUseFileProvider cfg.settings.database.password "" ||
+        doesntUseFileProvider cfg.settings.security.admin_password "admin"
+      ) ''
+        Grafana passwords will be stored as plaintext in the Nix store!
+        Use file provider or an env-var instead.
+      '')
+      # Warn about deprecated notifiers.
+      ++ (optional (cfg.provision.notifiers != []) ''
+        Notifiers are deprecated upstream and will be removed in Grafana 10.
+        Use `services.grafana.provision.alerting.contactPoints` instead.
+      '')
+      # Ensure that `secureJsonData` of datasources provisioned via `datasources.settings`
+      # only uses file/env providers.
+      ++ (optional (
+        let
+          datasourcesToCheck = optionals
+            (cfg.provision.datasources.settings != null)
+            cfg.provision.datasources.settings.datasources;
+          declarationUnsafe = { secureJsonData, ... }:
+            secureJsonData != null
+            && any (flip doesntUseFileProvider null) (attrValues secureJsonData);
+        in any declarationUnsafe datasourcesToCheck
+      ) ''
+        Declarations in the `secureJsonData`-block of a datasource will be leaked to the
+        Nix store unless a file-provider or an env-var is used!
+      '')
+      ++ (optional (
         any (x: x.secure_settings != null) cfg.provision.notifiers
-      ) "Notifier secure settings will be stored as plaintext in the Nix store!")
-    ];
+      ) "Notifier secure settings will be stored as plaintext in the Nix store! Use file provider instead.");
 
     environment.systemPackages = [ cfg.package ];
 
     assertions = [
       {
-        assertion = cfg.database.password != opt.database.password.default -> cfg.database.passwordFile == null;
-        message = "Cannot set both password and passwordFile";
+        assertion = cfg.provision.datasources.settings == null || cfg.provision.datasources.path == null;
+        message = "Cannot set both datasources settings and datasources path";
+      }
+      {
+        assertion = let
+          prometheusIsNotDirect = opt: all
+          ({ type, access, ... }: type == "prometheus" -> access != "direct")
+          opt;
+        in
+          cfg.provision.datasources.settings == null || prometheusIsNotDirect cfg.provision.datasources.settings.datasources;
+        message = "For datasources of type `prometheus`, the `direct` access mode is not supported anymore (since Grafana 9.2.0)";
+      }
+      {
+        assertion = cfg.provision.dashboards.settings == null || cfg.provision.dashboards.path == null;
+        message = "Cannot set both dashboards settings and dashboards path";
+      }
+      {
+        assertion = cfg.provision.alerting.rules.settings == null || cfg.provision.alerting.rules.path == null;
+        message = "Cannot set both rules settings and rules path";
+      }
+      {
+        assertion = cfg.provision.alerting.contactPoints.settings == null || cfg.provision.alerting.contactPoints.path == null;
+        message = "Cannot set both contact points settings and contact points path";
       }
       {
-        assertion = cfg.security.adminPassword != opt.security.adminPassword.default -> cfg.security.adminPasswordFile == null;
-        message = "Cannot set both adminPassword and adminPasswordFile";
+        assertion = cfg.provision.alerting.policies.settings == null || cfg.provision.alerting.policies.path == null;
+        message = "Cannot set both policies settings and policies path";
       }
       {
-        assertion = cfg.security.secretKey != opt.security.secretKey.default -> cfg.security.secretKeyFile == null;
-        message = "Cannot set both secretKey and secretKeyFile";
+        assertion = cfg.provision.alerting.templates.settings == null || cfg.provision.alerting.templates.path == null;
+        message = "Cannot set both templates settings and templates path";
       }
       {
-        assertion = cfg.smtp.password != opt.smtp.password.default -> cfg.smtp.passwordFile == null;
-        message = "Cannot set both password and passwordFile";
+        assertion = cfg.provision.alerting.muteTimings.settings == null || cfg.provision.alerting.muteTimings.path == null;
+        message = "Cannot set both mute timings settings and mute timings path";
       }
     ];
 
@@ -726,41 +1254,11 @@ in {
       description = "Grafana Service Daemon";
       wantedBy = ["multi-user.target"];
       after = ["networking.target"] ++ lib.optional usePostgresql "postgresql.service" ++ lib.optional useMysql "mysql.service";
-      environment = {
-        QT_QPA_PLATFORM = "offscreen";
-      } // mapAttrs' (n: v: nameValuePair "GF_${n}" (toString v)) envOptions;
       script = ''
         set -o errexit -o pipefail -o nounset -o errtrace
         shopt -s inherit_errexit
 
-        ${optionalString (cfg.auth.azuread.clientSecretFile != null) ''
-          GF_AUTH_AZUREAD_CLIENT_SECRET="$(<${escapeShellArg cfg.auth.azuread.clientSecretFile})"
-          export GF_AUTH_AZUREAD_CLIENT_SECRET
-        ''}
-        ${optionalString (cfg.auth.google.clientSecretFile != null) ''
-          GF_AUTH_GOOGLE_CLIENT_SECRET="$(<${escapeShellArg cfg.auth.google.clientSecretFile})"
-          export GF_AUTH_GOOGLE_CLIENT_SECRET
-        ''}
-        ${optionalString (cfg.database.passwordFile != null) ''
-          GF_DATABASE_PASSWORD="$(<${escapeShellArg cfg.database.passwordFile})"
-          export GF_DATABASE_PASSWORD
-        ''}
-        ${optionalString (cfg.security.adminPasswordFile != null) ''
-          GF_SECURITY_ADMIN_PASSWORD="$(<${escapeShellArg cfg.security.adminPasswordFile})"
-          export GF_SECURITY_ADMIN_PASSWORD
-        ''}
-        ${optionalString (cfg.security.secretKeyFile != null) ''
-          GF_SECURITY_SECRET_KEY="$(<${escapeShellArg cfg.security.secretKeyFile})"
-          export GF_SECURITY_SECRET_KEY
-        ''}
-        ${optionalString (cfg.smtp.passwordFile != null) ''
-          GF_SMTP_PASSWORD="$(<${escapeShellArg cfg.smtp.passwordFile})"
-          export GF_SMTP_PASSWORD
-        ''}
-        ${optionalString cfg.provision.enable ''
-          export GF_PATHS_PROVISIONING=${provisionConfDir};
-        ''}
-        exec ${cfg.package}/bin/grafana-server -homepath ${cfg.dataDir}
+        exec ${cfg.package}/bin/grafana-server -homepath ${cfg.dataDir} -config ${configFile}
       '';
       serviceConfig = {
         WorkingDirectory = cfg.dataDir;
@@ -768,8 +1266,8 @@ in {
         RuntimeDirectory = "grafana";
         RuntimeDirectoryMode = "0755";
         # Hardening
-        AmbientCapabilities = lib.mkIf (cfg.port < 1024) [ "CAP_NET_BIND_SERVICE" ];
-        CapabilityBoundingSet = if (cfg.port < 1024) then [ "CAP_NET_BIND_SERVICE" ] else [ "" ];
+        AmbientCapabilities = lib.mkIf (cfg.settings.server.http_port < 1024) [ "CAP_NET_BIND_SERVICE" ];
+        CapabilityBoundingSet = if (cfg.settings.server.http_port < 1024) then [ "CAP_NET_BIND_SERVICE" ] else [ "" ];
         DeviceAllow = [ "" ];
         LockPersonality = true;
         NoNewPrivileges = true;
@@ -792,7 +1290,10 @@ in {
         SystemCallArchitectures = "native";
         # Upstream grafana is not setting SystemCallFilter for compatibility
         # reasons, see https://github.com/grafana/grafana/pull/40176
-        SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ] ++ lib.optionals (cfg.settings.server.protocol == "socket") [ "@chown" ];
         UMask = "0027";
       };
       preStart = ''
diff --git a/nixpkgs/nixos/modules/services/monitoring/graphite.nix b/nixpkgs/nixos/modules/services/monitoring/graphite.nix
index 8edb2ca09974..65c91b8f79bb 100644
--- a/nixpkgs/nixos/modules/services/monitoring/graphite.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/graphite.nix
@@ -94,7 +94,7 @@ in {
       port = mkOption {
         description = lib.mdDoc "Graphite web frontend port.";
         default = 8080;
-        type = types.int;
+        type = types.port;
       };
 
       extraConfig = mkOption {
@@ -154,14 +154,14 @@ in {
       };
 
       blacklist = mkOption {
-        description = lib.mdDoc "Any metrics received which match one of the experssions will be dropped.";
+        description = lib.mdDoc "Any metrics received which match one of the expressions will be dropped.";
         default = null;
         type = types.nullOr types.str;
         example = "^some\\.noisy\\.metric\\.prefix\\..*";
       };
 
       whitelist = mkOption {
-        description = lib.mdDoc "Only metrics received which match one of the experssions will be persisted.";
+        description = lib.mdDoc "Only metrics received which match one of the expressions will be persisted.";
         default = null;
         type = types.nullOr types.str;
         example = ".*";
@@ -225,7 +225,7 @@ in {
       port = mkOption {
         description = lib.mdDoc "Seyren listening port.";
         default = 8081;
-        type = types.int;
+        type = types.port;
       };
 
       seyrenUrl = mkOption {
diff --git a/nixpkgs/nixos/modules/services/monitoring/hdaps.nix b/nixpkgs/nixos/modules/services/monitoring/hdaps.nix
index 2cad3b84d847..59b8b9b3c054 100644
--- a/nixpkgs/nixos/modules/services/monitoring/hdaps.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/hdaps.nix
@@ -9,10 +9,10 @@ in
 {
   options = {
     services.hdapsd.enable = mkEnableOption
-      ''
+      (lib.mdDoc ''
         Hard Drive Active Protection System Daemon,
         devices are detected and managed automatically by udev and systemd
-      '';
+      '');
   };
 
   config = mkIf cfg.enable {
diff --git a/nixpkgs/nixos/modules/services/monitoring/heapster.nix b/nixpkgs/nixos/modules/services/monitoring/heapster.nix
index 2f2467477ae2..fc63276b62f7 100644
--- a/nixpkgs/nixos/modules/services/monitoring/heapster.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/heapster.nix
@@ -6,11 +6,7 @@ let
   cfg = config.services.heapster;
 in {
   options.services.heapster = {
-    enable = mkOption {
-      description = lib.mdDoc "Whether to enable heapster monitoring";
-      default = false;
-      type = types.bool;
-    };
+    enable = mkEnableOption (lib.mdDoc "Heapster monitoring");
 
     source = mkOption {
       description = lib.mdDoc "Heapster metric source";
diff --git a/nixpkgs/nixos/modules/services/monitoring/incron.nix b/nixpkgs/nixos/modules/services/monitoring/incron.nix
index 53cbe1a9e26a..3766f1fa238d 100644
--- a/nixpkgs/nixos/modules/services/monitoring/incron.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/incron.nix
@@ -17,10 +17,10 @@ in
       enable = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = lib.mdDoc ''
           Whether to enable the incron daemon.
 
-          Note that commands run under incrontab only support common Nix profiles for the <envar>PATH</envar> provided variable.
+          Note that commands run under incrontab only support common Nix profiles for the {env}`PATH` provided variable.
         '';
       };
 
diff --git a/nixpkgs/nixos/modules/services/monitoring/kapacitor.nix b/nixpkgs/nixos/modules/services/monitoring/kapacitor.nix
index 9cdb0e4495a4..727b694047b4 100644
--- a/nixpkgs/nixos/modules/services/monitoring/kapacitor.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/kapacitor.nix
@@ -57,7 +57,7 @@ let
 in
 {
   options.services.kapacitor = {
-    enable = mkEnableOption "kapacitor";
+    enable = mkEnableOption (lib.mdDoc "kapacitor");
 
     dataDir = mkOption {
       type = types.path;
@@ -66,7 +66,7 @@ in
     };
 
     port = mkOption {
-      type = types.int;
+      type = types.port;
       default = 9092;
       description = lib.mdDoc "Port of Kapacitor";
     };
@@ -109,7 +109,7 @@ in
     };
 
     defaultDatabase = {
-      enable = mkEnableOption "kapacitor.defaultDatabase";
+      enable = mkEnableOption (lib.mdDoc "kapacitor.defaultDatabase");
 
       url = mkOption {
         description = lib.mdDoc "The URL to an InfluxDB server that serves as the default database";
@@ -129,7 +129,7 @@ in
     };
 
     alerta = {
-      enable = mkEnableOption "kapacitor alerta integration";
+      enable = mkEnableOption (lib.mdDoc "kapacitor alerta integration");
 
       url = mkOption {
         description = lib.mdDoc "The URL to the Alerta REST API";
diff --git a/nixpkgs/nixos/modules/services/monitoring/karma.nix b/nixpkgs/nixos/modules/services/monitoring/karma.nix
new file mode 100644
index 000000000000..85dbc81f443f
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/karma.nix
@@ -0,0 +1,128 @@
+{ config, pkgs, lib, ... }:
+with lib;
+let
+  cfg = config.services.karma;
+  yaml = pkgs.formats.yaml { };
+in
+{
+  options.services.karma = {
+    enable = mkEnableOption (mdDoc "the Karma dashboard service");
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.karma;
+      defaultText = literalExpression "pkgs.karma";
+      description = mdDoc ''
+        The Karma package that should be used.
+      '';
+    };
+
+    configFile = mkOption {
+      type = types.path;
+      default = yaml.generate "karma.yaml" cfg.settings;
+      defaultText = "A configuration file generated from the provided nix attributes settings option.";
+      description = mdDoc ''
+        A YAML config file which can be used to configure karma instead of the nix-generated file.
+      '';
+      example = "/etc/karma/karma.conf";
+    };
+
+    environment = mkOption {
+      type = with types; attrsOf str;
+      default = {};
+      description = mdDoc ''
+        Additional environment variables to provide to karma.
+      '';
+      example = {
+        ALERTMANAGER_URI = "https://alertmanager.example.com";
+        ALERTMANAGER_NAME= "single";
+      };
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Whether to open ports in the firewall needed for karma to function.
+      '';
+    };
+
+    extraOptions = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = mdDoc ''
+        Extra command line options.
+      '';
+      example = [
+        "--alertmanager.timeout 10s"
+      ];
+    };
+
+    settings = mkOption {
+      type = types.submodule {
+        freeformType = yaml.type;
+
+        options.listen = {
+          address = mkOption {
+            type = types.str;
+            default = "127.0.0.1";
+            description = mdDoc ''
+              Hostname or IP to listen on.
+            '';
+            example = "[::]";
+          };
+
+          port = mkOption {
+            type = types.port;
+            default = 8080;
+            description = mdDoc ''
+              HTTP port to listen on.
+            '';
+            example = 8182;
+          };
+        };
+      };
+      default = {
+        listen = {
+          address = "127.0.0.1";
+        };
+      };
+      description = mdDoc ''
+        Karma dashboard configuration as nix attributes.
+
+        Reference: <https://github.com/prymitive/karma/blob/main/docs/CONFIGURATION.md>
+      '';
+      example = {
+        listen = {
+          address = "192.168.1.4";
+          port = "8000";
+          prefix = "/dashboard";
+        };
+        alertmanager = {
+          interval = "15s";
+          servers = [
+            {
+              name = "prod";
+              uri = "http://alertmanager.example.com";
+            }
+          ];
+        };
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.karma = {
+      description = "Alert dashboard for Prometheus Alertmanager";
+      wantedBy = [ "multi-user.target" ];
+      environment = cfg.environment;
+      serviceConfig = {
+        Type = "simple";
+        DynamicUser = true;
+        Restart = "on-failure";
+        ExecStart = "${pkgs.karma}/bin/karma --config.file ${cfg.configFile} ${concatStringsSep " " cfg.extraOptions}";
+      };
+    };
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.settings.listen.port ];
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/kthxbye.nix b/nixpkgs/nixos/modules/services/monitoring/kthxbye.nix
new file mode 100644
index 000000000000..3f988dcb722f
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/kthxbye.nix
@@ -0,0 +1,166 @@
+{ config, pkgs, lib, ... }:
+with lib;
+
+let
+  cfg = config.services.kthxbye;
+in
+
+{
+  options.services.kthxbye = {
+    enable = mkEnableOption (mdDoc "kthxbye alert acknowledgement management daemon");
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.kthxbye;
+      defaultText = literalExpression "pkgs.kthxbye";
+      description = mdDoc ''
+        The kthxbye package that should be used.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Whether to open ports in the firewall needed for the daemon to function.
+      '';
+    };
+
+    extraOptions = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = mdDoc ''
+        Extra command line options.
+
+        Documentation can be found [here](https://github.com/prymitive/kthxbye/blob/main/README.md).
+      '';
+      example = literalExpression ''
+        [
+          "-extend-with-prefix 'ACK!'"
+        ];
+      '';
+    };
+
+    alertmanager = {
+      timeout = mkOption {
+        type = types.str;
+        default = "1m0s";
+        description = mdDoc ''
+          Alertmanager request timeout duration in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+        '';
+        example = "30s";
+      };
+      uri = mkOption {
+        type = types.str;
+        default = "http://localhost:9093";
+        description = mdDoc ''
+          Alertmanager URI to use.
+        '';
+        example = "https://alertmanager.example.com";
+      };
+    };
+
+    extendBy = mkOption {
+      type = types.str;
+      default = "15m0s";
+      description = mdDoc ''
+        Extend silences by adding DURATION seconds.
+
+        DURATION should be provided in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "6h0m0s";
+    };
+
+    extendIfExpiringIn = mkOption {
+      type = types.str;
+      default = "5m0s";
+      description = mdDoc ''
+        Extend silences that are about to expire in the next DURATION seconds.
+
+        DURATION should be provided in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "1m0s";
+    };
+
+    extendWithPrefix = mkOption {
+      type = types.str;
+      default = "ACK!";
+      description = mdDoc ''
+        Extend silences with comment starting with PREFIX string.
+      '';
+      example = "!perma-silence";
+    };
+
+    interval = mkOption {
+      type = types.str;
+      default = "45s";
+      description = mdDoc ''
+        Silence check interval duration in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "30s";
+    };
+
+    listenAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = mdDoc ''
+        The address to listen on for HTTP requests.
+      '';
+      example = "127.0.0.1";
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = 8080;
+      description = mdDoc ''
+        The port to listen on for HTTP requests.
+      '';
+    };
+
+    logJSON = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Format logged messages as JSON.
+      '';
+    };
+
+    maxDuration = mkOption {
+      type = with types; nullOr str;
+      default = null;
+      description = mdDoc ''
+        Maximum duration of a silence, it won't be extended anymore after reaching it.
+
+        Duration should be provided in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "30d";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.kthxbye = {
+      description = "kthxbye Alertmanager ack management daemon";
+      wantedBy = [ "multi-user.target" ];
+      script = ''
+        ${cfg.package}/bin/kthxbye \
+          -alertmanager.timeout ${cfg.alertmanager.timeout} \
+          -alertmanager.uri ${cfg.alertmanager.uri} \
+          -extend-by ${cfg.extendBy} \
+          -extend-if-expiring-in ${cfg.extendIfExpiringIn} \
+          -extend-with-prefix ${cfg.extendWithPrefix} \
+          -interval ${cfg.interval} \
+          -listen ${cfg.listenAddress}:${toString cfg.port} \
+          ${optionalString cfg.logJSON "-log-json"} \
+          ${optionalString (cfg.maxDuration != null) "-max-duration ${cfg.maxDuration}"} \
+          ${concatStringsSep " " cfg.extraOptions}
+      '';
+      serviceConfig = {
+        Type = "simple";
+        DynamicUser = true;
+        Restart = "on-failure";
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ];
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/loki.nix b/nixpkgs/nixos/modules/services/monitoring/loki.nix
index d73e2abb71d2..f3b97e9151ea 100644
--- a/nixpkgs/nixos/modules/services/monitoring/loki.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/loki.nix
@@ -12,7 +12,7 @@ let
 
 in {
   options.services.loki = {
-    enable = mkEnableOption "loki";
+    enable = mkEnableOption (lib.mdDoc "loki");
 
     user = mkOption {
       type = types.str;
@@ -22,6 +22,8 @@ in {
       '';
     };
 
+    package = lib.mkPackageOptionMD pkgs "grafana-loki" { };
+
     group = mkOption {
       type = types.str;
       default = "loki";
@@ -78,7 +80,7 @@ in {
       '';
     }];
 
-    environment.systemPackages = [ pkgs.grafana-loki ]; # logcli
+    environment.systemPackages = [ cfg.package ]; # logcli
 
     users.groups.${cfg.group} = { };
     users.users.${cfg.user} = {
@@ -99,7 +101,7 @@ in {
                else cfg.configFile;
       in
       {
-        ExecStart = "${pkgs.grafana-loki}/bin/loki --config.file=${conf} ${escapeShellArgs cfg.extraFlags}";
+        ExecStart = "${cfg.package}/bin/loki --config.file=${conf} ${escapeShellArgs cfg.extraFlags}";
         User = cfg.user;
         Restart = "always";
         PrivateTmp = true;
diff --git a/nixpkgs/nixos/modules/services/monitoring/mackerel-agent.nix b/nixpkgs/nixos/modules/services/monitoring/mackerel-agent.nix
index 89c6d4d6c65b..67dc1bc19edd 100644
--- a/nixpkgs/nixos/modules/services/monitoring/mackerel-agent.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/mackerel-agent.nix
@@ -7,24 +7,24 @@ let
   settingsFmt = pkgs.formats.toml {};
 in {
   options.services.mackerel-agent = {
-    enable = mkEnableOption "mackerel.io agent";
+    enable = mkEnableOption (lib.mdDoc "mackerel.io agent");
 
     # the upstream package runs as root, but doesn't seem to be strictly
     # necessary for basic functionality
-    runAsRoot = mkEnableOption "Whether to run as root.";
+    runAsRoot = mkEnableOption (lib.mdDoc "Whether to run as root");
 
-    autoRetirement = mkEnableOption ''
+    autoRetirement = mkEnableOption (lib.mdDoc ''
       Whether to automatically retire the host upon OS shutdown.
-    '';
+    '');
 
     apiKeyFile = mkOption {
       type = types.path;
       example = "/run/keys/mackerel-api-key";
-      description = ''
+      description = lib.mdDoc ''
         Path to file containing the Mackerel API key. The file should contain a
         single line of the following form:
 
-        <literallayout>apikey = "EXAMPLE_API_KEY"</literallayout>
+        `apikey = "EXAMPLE_API_KEY"`
       '';
     };
 
@@ -59,7 +59,7 @@ in {
         };
 
         options.diagnostic =
-          mkEnableOption "Collect memory usage for the agent itself";
+          mkEnableOption (lib.mdDoc "Collect memory usage for the agent itself");
       };
     };
   };
diff --git a/nixpkgs/nixos/modules/services/monitoring/metricbeat.nix b/nixpkgs/nixos/modules/services/monitoring/metricbeat.nix
index 14066da1be81..310c9d8ed509 100644
--- a/nixpkgs/nixos/modules/services/monitoring/metricbeat.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/metricbeat.nix
@@ -19,7 +19,7 @@ in
 
     services.metricbeat = {
 
-      enable = mkEnableOption "metricbeat";
+      enable = mkEnableOption (lib.mdDoc "metricbeat");
 
       package = mkOption {
         type = types.package;
@@ -99,10 +99,10 @@ in
               type = types.listOf settingsFormat.type;
               default = [];
               internal = true;
-              description = ''
-                The metric collecting modules. Use <xref linkend="opt-services.metricbeat.modules"/> instead.
+              description = lib.mdDoc ''
+                The metric collecting modules. Use [](#opt-services.metricbeat.modules) instead.
 
-                See <link xlink:href="https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html"/>.
+                See <https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html>.
               '';
             };
           };
diff --git a/nixpkgs/nixos/modules/services/monitoring/mimir.nix b/nixpkgs/nixos/modules/services/monitoring/mimir.nix
index 87f7af7855e1..edca9b7be4ff 100644
--- a/nixpkgs/nixos/modules/services/monitoring/mimir.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/mimir.nix
@@ -8,7 +8,7 @@ let
   settingsFormat = pkgs.formats.yaml {};
 in {
   options.services.mimir = {
-    enable = mkEnableOption "mimir";
+    enable = mkEnableOption (lib.mdDoc "mimir");
 
     configuration = mkOption {
       type = (pkgs.formats.json {}).type;
@@ -25,6 +25,13 @@ in {
         Specify a configuration file that Mimir should use.
       '';
     };
+
+    package = mkOption {
+      default = pkgs.mimir;
+      defaultText = lib.literalExpression "pkgs.mimir";
+      type = types.package;
+      description = lib.mdDoc ''Mimir package to use.'';
+    };
   };
 
   config = mkIf cfg.enable {
@@ -53,7 +60,7 @@ in {
                else cfg.configFile;
       in
       {
-        ExecStart = "${pkgs.mimir}/bin/mimir --config.file=${conf}";
+        ExecStart = "${cfg.package}/bin/mimir --config.file=${conf}";
         DynamicUser = true;
         Restart = "always";
         ProtectSystem = "full";
diff --git a/nixpkgs/nixos/modules/services/monitoring/monit.nix b/nixpkgs/nixos/modules/services/monitoring/monit.nix
index 6ce5b44eb279..a22bbc9046ba 100644
--- a/nixpkgs/nixos/modules/services/monitoring/monit.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/monit.nix
@@ -9,7 +9,7 @@ in
 {
   options.services.monit = {
 
-    enable = mkEnableOption "Monit";
+    enable = mkEnableOption (lib.mdDoc "Monit");
 
     config = mkOption {
       type = types.lines;
diff --git a/nixpkgs/nixos/modules/services/monitoring/nagios.nix b/nixpkgs/nixos/modules/services/monitoring/nagios.nix
index a45d0c0e070b..8feff22c1182 100644
--- a/nixpkgs/nixos/modules/services/monitoring/nagios.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/nagios.nix
@@ -88,14 +88,14 @@ in
 
   options = {
     services.nagios = {
-      enable = mkEnableOption ''<link xlink:href="http://www.nagios.org/">Nagios</link> to monitor your system or network.'';
+      enable = mkEnableOption (lib.mdDoc ''[Nagios](http://www.nagios.org/) to monitor your system or network.'');
 
       objectDefs = mkOption {
-        description = "
+        description = lib.mdDoc ''
           A list of Nagios object configuration files that must define
           the hosts, host groups, services and contacts for the
           network that you want Nagios to monitor.
-        ";
+        '';
         type = types.listOf types.path;
         example = literalExpression "[ ./objects.cfg ]";
       };
@@ -104,18 +104,18 @@ in
         type = types.listOf types.package;
         default = with pkgs; [ monitoring-plugins msmtp mailutils ];
         defaultText = literalExpression "[pkgs.monitoring-plugins pkgs.msmtp pkgs.mailutils]";
-        description = "
-          Packages to be added to the Nagios <envar>PATH</envar>.
+        description = lib.mdDoc ''
+          Packages to be added to the Nagios {env}`PATH`.
           Typically used to add plugins, but can be anything.
-        ";
+        '';
       };
 
       mainConfigFile = mkOption {
         type = types.nullOr types.package;
         default = null;
-        description = "
+        description = lib.mdDoc ''
           If non-null, overrides the main configuration file of Nagios.
-        ";
+        '';
       };
 
       extraConfig = mkOption {
@@ -139,19 +139,19 @@ in
         type = types.package;
         default = nagiosCGICfgFile;
         defaultText = literalExpression "nagiosCGICfgFile";
-        description = "
+        description = lib.mdDoc ''
           Derivation for the configuration file of Nagios CGI scripts
           that can be used in web servers for running the Nagios web interface.
-        ";
+        '';
       };
 
       enableWebInterface = mkOption {
         type = types.bool;
         default = false;
-        description = "
+        description = lib.mdDoc ''
           Whether to enable the Nagios web interface.  You should also
-          enable Apache (<option>services.httpd.enable</option>).
-        ";
+          enable Apache ({option}`services.httpd.enable`).
+        '';
       };
 
       virtualHost = mkOption {
diff --git a/nixpkgs/nixos/modules/services/monitoring/netdata.nix b/nixpkgs/nixos/modules/services/monitoring/netdata.nix
index e20eaf3b1440..bd0dea83e1a8 100644
--- a/nixpkgs/nixos/modules/services/monitoring/netdata.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/netdata.nix
@@ -49,7 +49,7 @@ let
 in {
   options = {
     services.netdata = {
-      enable = mkEnableOption "netdata";
+      enable = mkEnableOption (lib.mdDoc "netdata");
 
       package = mkOption {
         type = types.package;
@@ -169,6 +169,20 @@ in {
           See: <https://learn.netdata.cloud/docs/agent/anonymous-statistics>
         '';
       };
+
+      deadlineBeforeStopSec = mkOption {
+        type = types.int;
+        default = 120;
+        description = lib.mdDoc ''
+          In order to detect when netdata is misbehaving, we run a concurrent task pinging netdata (wait-for-netdata-up)
+          in the systemd unit.
+
+          If after a while, this task does not succeed, we stop the unit and mark it as failed.
+
+          You can control this deadline in seconds with this option, it's useful to bump it
+          if you have (1) a lot of data (2) doing upgrades (3) have low IOPS/throughput.
+        '';
+      };
     };
   };
 
@@ -205,7 +219,7 @@ in {
           while [ "$(${pkgs.netdata}/bin/netdatacli ping)" != pong ]; do sleep 0.5; done
         '';
 
-        TimeoutStopSec = 60;
+        TimeoutStopSec = cfg.deadlineBeforeStopSec;
         Restart = "on-failure";
         # User and group
         User = cfg.user;
diff --git a/nixpkgs/nixos/modules/services/monitoring/parsedmarc.md b/nixpkgs/nixos/modules/services/monitoring/parsedmarc.md
index d93134a4cc76..eac07e0cc9fe 100644
--- a/nixpkgs/nixos/modules/services/monitoring/parsedmarc.md
+++ b/nixpkgs/nixos/modules/services/monitoring/parsedmarc.md
@@ -17,7 +17,6 @@ services.parsedmarc = {
     host = "imap.example.com";
     user = "alice@example.com";
     password = "/path/to/imap_password_file";
-    watch = true;
   };
   provision.geoIp = false; # Not recommended!
 };
@@ -26,7 +25,7 @@ services.parsedmarc = {
 Note that GeoIP provisioning is disabled in the example for
 simplicity, but should be turned on for fully functional reports.
 
-## Local mail
+## Local mail {#module-services-parsedmarc-local-mail}
 Instead of watching an external inbox, a local inbox can be
 automatically provisioned. The recipient's name is by default set to
 `dmarc`, but can be configured in
@@ -50,7 +49,7 @@ services.parsedmarc = {
 };
 ```
 
-## Grafana and GeoIP
+## Grafana and GeoIP {#module-services-parsedmarc-grafana-geoip}
 The reports can be visualized and summarized with parsedmarc's
 official Grafana dashboard. For all views to work, and for the data to
 be complete, GeoIP databases are also required. The following example
diff --git a/nixpkgs/nixos/modules/services/monitoring/parsedmarc.nix b/nixpkgs/nixos/modules/services/monitoring/parsedmarc.nix
index b0858184b5fc..44fc359b6a7d 100644
--- a/nixpkgs/nixos/modules/services/monitoring/parsedmarc.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/parsedmarc.nix
@@ -20,9 +20,9 @@ in
 {
   options.services.parsedmarc = {
 
-    enable = lib.mkEnableOption ''
+    enable = lib.mkEnableOption (lib.mdDoc ''
       parsedmarc, a DMARC report monitoring service
-    '';
+    '');
 
     provision = {
       localMail = {
@@ -123,7 +123,10 @@ in
             host = "imap.example.com";
             user = "alice@example.com";
             password = { _secret = "/run/keys/imap_password" };
+          };
+          mailbox = {
             watch = true;
+            batch_size = 30;
           };
           splunk_hec = {
             url = "https://splunkhec.example.com";
@@ -170,6 +173,24 @@ in
             };
           };
 
+          mailbox = {
+            watch = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = lib.mdDoc ''
+                Use the IMAP IDLE command to process messages as they arrive.
+              '';
+            };
+
+            delete = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = lib.mdDoc ''
+                Delete messages after processing them, instead of archiving them.
+              '';
+            };
+          };
+
           imap = {
             host = lib.mkOption {
               type = lib.types.str;
@@ -216,22 +237,6 @@ in
               '';
               apply = x: if isAttrs x || x == null then x else { _secret = x; };
             };
-
-            watch = lib.mkOption {
-              type = lib.types.bool;
-              default = true;
-              description = lib.mdDoc ''
-                Use the IMAP IDLE command to process messages as they arrive.
-              '';
-            };
-
-            delete = lib.mkOption {
-              type = lib.types.bool;
-              default = false;
-              description = lib.mdDoc ''
-                Delete messages after processing them, instead of archiving them.
-              '';
-            };
           };
 
           smtp = {
@@ -360,6 +365,13 @@ in
 
   config = lib.mkIf cfg.enable {
 
+    warnings = let
+      deprecationWarning = optname: "Starting in 8.0.0, the `${optname}` option has been moved from the `services.parsedmarc.settings.imap`"
+        + "configuration section to the `services.parsedmarc.settings.mailbox` configuration section.";
+      hasImapOpt = lib.flip builtins.hasAttr cfg.settings.imap;
+      movedOptions = [ "reports_folder" "archive_folder" "watch" "delete" "test" "batch_size" ];
+    in builtins.map deprecationWarning (builtins.filter hasImapOpt movedOptions);
+
     services.elasticsearch.enable = lib.mkDefault cfg.provision.elasticsearch;
 
     services.geoipupdate = lib.mkIf cfg.provision.geoIp {
@@ -397,7 +409,7 @@ in
 
       provision = {
         enable = cfg.provision.grafana.datasource || cfg.provision.grafana.dashboard;
-        datasources =
+        datasources.settings.datasources =
           let
             esVersion = lib.getVersion config.services.elasticsearch.package;
           in
@@ -423,7 +435,7 @@ in
                 };
               }
             ];
-        dashboards = lib.mkIf cfg.provision.grafana.dashboard [{
+        dashboards.settings.providers = lib.mkIf cfg.provision.grafana.dashboard [{
           name = "parsedmarc";
           options.path = "${pkgs.python3Packages.parsedmarc.dashboard}";
         }];
@@ -444,6 +456,8 @@ in
           ssl = false;
           user = cfg.provision.localMail.recipientName;
           password = "${pkgs.writeText "imap-password" "@imap-password@"}";
+        };
+        mailbox = {
           watch = true;
         };
       })
@@ -494,7 +508,7 @@ in
             Group = "parsedmarc";
             DynamicUser = true;
             RuntimeDirectory = "parsedmarc";
-            RuntimeDirectoryMode = 0700;
+            RuntimeDirectoryMode = "0700";
             CapabilityBoundingSet = "";
             PrivateDevices = true;
             PrivateMounts = true;
@@ -525,8 +539,6 @@ in
     };
   };
 
-  # Don't edit the docbook xml directly, edit the md and generate it:
-  # `pandoc parsedmarc.md -t docbook --top-level-division=chapter --extract-media=media -f markdown+smart > parsedmarc.xml`
-  meta.doc = ./parsedmarc.xml;
+  meta.doc = ./parsedmarc.md;
   meta.maintainers = [ lib.maintainers.talyz ];
 }
diff --git a/nixpkgs/nixos/modules/services/monitoring/parsedmarc.xml b/nixpkgs/nixos/modules/services/monitoring/parsedmarc.xml
deleted file mode 100644
index 7167b52d0357..000000000000
--- a/nixpkgs/nixos/modules/services/monitoring/parsedmarc.xml
+++ /dev/null
@@ -1,125 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="module-services-parsedmarc">
-  <title>parsedmarc</title>
-  <para>
-    <link xlink:href="https://domainaware.github.io/parsedmarc/">parsedmarc</link>
-    is a service which parses incoming
-    <link xlink:href="https://dmarc.org/">DMARC</link> reports and
-    stores or sends them to a downstream service for further analysis.
-    In combination with Elasticsearch, Grafana and the included Grafana
-    dashboard, it provides a handy overview of DMARC reports over time.
-  </para>
-  <section xml:id="module-services-parsedmarc-basic-usage">
-    <title>Basic usage</title>
-    <para>
-      A very minimal setup which reads incoming reports from an external
-      email address and saves them to a local Elasticsearch instance
-      looks like this:
-    </para>
-    <programlisting language="bash">
-services.parsedmarc = {
-  enable = true;
-  settings.imap = {
-    host = &quot;imap.example.com&quot;;
-    user = &quot;alice@example.com&quot;;
-    password = &quot;/path/to/imap_password_file&quot;;
-    watch = true;
-  };
-  provision.geoIp = false; # Not recommended!
-};
-</programlisting>
-    <para>
-      Note that GeoIP provisioning is disabled in the example for
-      simplicity, but should be turned on for fully functional reports.
-    </para>
-  </section>
-  <section xml:id="local-mail">
-    <title>Local mail</title>
-    <para>
-      Instead of watching an external inbox, a local inbox can be
-      automatically provisioned. The recipient’s name is by default set
-      to <literal>dmarc</literal>, but can be configured in
-      <link xlink:href="options.html#opt-services.parsedmarc.provision.localMail.recipientName">services.parsedmarc.provision.localMail.recipientName</link>.
-      You need to add an MX record pointing to the host. More
-      concretely: for the example to work, an MX record needs to be set
-      up for <literal>monitoring.example.com</literal> and the complete
-      email address that should be configured in the domain’s dmarc
-      policy is <literal>dmarc@monitoring.example.com</literal>.
-    </para>
-    <programlisting language="bash">
-services.parsedmarc = {
-  enable = true;
-  provision = {
-    localMail = {
-      enable = true;
-      hostname = monitoring.example.com;
-    };
-    geoIp = false; # Not recommended!
-  };
-};
-</programlisting>
-  </section>
-  <section xml:id="grafana-and-geoip">
-    <title>Grafana and GeoIP</title>
-    <para>
-      The reports can be visualized and summarized with parsedmarc’s
-      official Grafana dashboard. For all views to work, and for the
-      data to be complete, GeoIP databases are also required. The
-      following example shows a basic deployment where the provisioned
-      Elasticsearch instance is automatically added as a Grafana
-      datasource, and the dashboard is added to Grafana as well.
-    </para>
-    <programlisting language="bash">
-services.parsedmarc = {
-  enable = true;
-  provision = {
-    localMail = {
-      enable = true;
-      hostname = url;
-    };
-    grafana = {
-      datasource = true;
-      dashboard = true;
-    };
-  };
-};
-
-# Not required, but recommended for full functionality
-services.geoipupdate = {
-  settings = {
-    AccountID = 000000;
-    LicenseKey = &quot;/path/to/license_key_file&quot;;
-  };
-};
-
-services.grafana = {
-  enable = true;
-  addr = &quot;0.0.0.0&quot;;
-  domain = url;
-  rootUrl = &quot;https://&quot; + url;
-  protocol = &quot;socket&quot;;
-  security = {
-    adminUser = &quot;admin&quot;;
-    adminPasswordFile = &quot;/path/to/admin_password_file&quot;;
-    secretKeyFile = &quot;/path/to/secret_key_file&quot;;
-  };
-};
-
-services.nginx = {
-  enable = true;
-  recommendedTlsSettings = true;
-  recommendedOptimisation = true;
-  recommendedGzipSettings = true;
-  recommendedProxySettings = true;
-  upstreams.grafana.servers.&quot;unix:/${config.services.grafana.socket}&quot; = {};
-  virtualHosts.${url} = {
-    root = config.services.grafana.staticRootPath;
-    enableACME = true;
-    forceSSL = true;
-    locations.&quot;/&quot;.tryFiles = &quot;$uri @grafana&quot;;
-    locations.&quot;@grafana&quot;.proxyPass = &quot;http://grafana&quot;;
-  };
-};
-users.users.nginx.extraGroups = [ &quot;grafana&quot; ];
-</programlisting>
-  </section>
-</chapter>
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager-irc-relay.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager-irc-relay.nix
new file mode 100644
index 000000000000..b81d5f6db5e0
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager-irc-relay.nix
@@ -0,0 +1,107 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.alertmanagerIrcRelay;
+
+  configFormat = pkgs.formats.yaml { };
+  configFile = configFormat.generate "alertmanager-irc-relay.yml" cfg.settings;
+in
+{
+  options.services.prometheus.alertmanagerIrcRelay = {
+    enable = mkEnableOption (mdDoc "Alertmanager IRC Relay");
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.alertmanager-irc-relay;
+      defaultText = literalExpression "pkgs.alertmanager-irc-relay";
+      description = mdDoc "Alertmanager IRC Relay package to use.";
+    };
+
+    extraFlags = mkOption {
+      type = types.listOf types.str;
+      default = [];
+      description = mdDoc "Extra command line options to pass to alertmanager-irc-relay.";
+    };
+
+    settings = mkOption {
+      type = configFormat.type;
+      example = literalExpression ''
+        {
+          http_host = "localhost";
+          http_port = 8000;
+
+          irc_host = "irc.example.com";
+          irc_port = 7000;
+          irc_nickname = "myalertbot";
+
+          irc_channels = [
+            { name = "#mychannel"; }
+          ];
+        }
+      '';
+      description = mdDoc ''
+        Configuration for Alertmanager IRC Relay as a Nix attribute set.
+        For a reference, check out the
+        [example configuration](https://github.com/google/alertmanager-irc-relay#configuring-and-running-the-bot)
+        and the
+        [source code](https://github.com/google/alertmanager-irc-relay/blob/master/config.go).
+
+        Note: The webhook's URL MUST point to the IRC channel where the message
+        should be posted. For `#mychannel` from the example, this would be
+        `http://localhost:8080/mychannel`.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.alertmanager-irc-relay = {
+      description = "Alertmanager IRC Relay";
+
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/alertmanager-irc-relay \
+          -config ${configFile} \
+          ${escapeShellArgs cfg.extraFlags}
+        '';
+
+        DynamicUser = true;
+        NoNewPrivileges = true;
+
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        ProtectHome = "tmpfs";
+
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateIPC = true;
+
+        ProtectHostname = true;
+        ProtectClock = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+
+        SystemCallFilter = [
+          "@system-service"
+          "~@cpu-emulation"
+          "~@privileged"
+          "~@reboot"
+          "~@setuid"
+          "~@swap"
+        ];
+      };
+    };
+  };
+
+  meta.maintainers = [ maintainers.oxzi ];
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager.nix
index 60e0523cc353..987f17c2c6e6 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/alertmanager.nix
@@ -6,10 +6,12 @@ let
   cfg = config.services.prometheus.alertmanager;
   mkConfigFile = pkgs.writeText "alertmanager.yml" (builtins.toJSON cfg.configuration);
 
-  checkedConfig = file: pkgs.runCommand "checked-config" { buildInputs = [ cfg.package ]; } ''
-    ln -s ${file} $out
-    amtool check-config $out
-  '';
+  checkedConfig = file:
+    if cfg.checkConfig then
+      pkgs.runCommand "checked-config" { buildInputs = [ cfg.package ]; } ''
+        ln -s ${file} $out
+        amtool check-config $out
+      '' else file;
 
   alertmanagerYml = let
     yml = if cfg.configText != null then
@@ -34,13 +36,13 @@ in {
     (mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "group" ] "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a group setting.")
     (mkRemovedOptionModule [ "services" "prometheus" "alertmanagerURL" ] ''
       Due to incompatibility, the alertmanagerURL option has been removed,
-      please use 'services.prometheus2.alertmanagers' instead.
+      please use 'services.prometheus.alertmanagers' instead.
     '')
   ];
 
   options = {
     services.prometheus.alertmanager = {
-      enable = mkEnableOption "Prometheus Alertmanager";
+      enable = mkEnableOption (lib.mdDoc "Prometheus Alertmanager");
 
       package = mkOption {
         type = types.package;
@@ -70,6 +72,20 @@ in {
         '';
       };
 
+      checkConfig = mkOption {
+        type = types.bool;
+        default = true;
+        description = lib.mdDoc ''
+          Check configuration with `amtool check-config`. The call to `amtool` is
+          subject to sandboxing by Nix.
+
+          If you use credentials stored in external files
+          (`environmentFile`, etc),
+          they will not be visible to `amtool`
+          and it will report errors, despite a correct configuration.
+        '';
+      };
+
       logFormat = mkOption {
         type = types.nullOr types.str;
         default = null;
@@ -107,7 +123,7 @@ in {
       };
 
       port = mkOption {
-        type = types.int;
+        type = types.port;
         default = 9093;
         description = lib.mdDoc ''
           Port to listen on for the web interface and API.
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/default.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/default.nix
index 3bc61fba158f..19ee3ae6f7da 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/default.nix
@@ -3,7 +3,7 @@
 with lib;
 
 let
-  json = pkgs.formats.json { };
+  yaml = pkgs.formats.yaml { };
   cfg = config.services.prometheus;
   checkConfigEnabled =
     (lib.isBool cfg.checkConfig && cfg.checkConfig)
@@ -11,8 +11,6 @@ let
 
   workingDir = "/var/lib/" + cfg.stateDir;
 
-  prometheusYmlOut = "${workingDir}/prometheus-substituted.yaml";
-
   triggerReload = pkgs.writeShellScriptBin "trigger-reload-prometheus" ''
     PATH="${makeBinPath (with pkgs; [ systemd ])}"
     if systemctl -q is-active prometheus.service; then
@@ -33,12 +31,12 @@ let
     if checkConfigEnabled then
       pkgs.runCommandLocal
         "${name}-${replaceStrings [" "] [""] what}-checked"
-        { buildInputs = [ cfg.package ]; } ''
+        { buildInputs = [ cfg.package.cli ]; } ''
         ln -s ${file} $out
         promtool ${what} $out
       '' else file;
 
-  generatedPrometheusYml = json.generate "prometheus.yml" promConfig;
+  generatedPrometheusYml = yaml.generate "prometheus.yml" promConfig;
 
   # This becomes the main config file for Prometheus
   promConfig = {
@@ -73,7 +71,8 @@ let
     "--web.listen-address=${cfg.listenAddress}:${builtins.toString cfg.port}"
     "--alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
   ] ++ optional (cfg.webExternalUrl != null) "--web.external-url=${cfg.webExternalUrl}"
-    ++ optional (cfg.retentionTime != null) "--storage.tsdb.retention.time=${cfg.retentionTime}";
+    ++ optional (cfg.retentionTime != null) "--storage.tsdb.retention.time=${cfg.retentionTime}"
+    ++ optional (cfg.webConfigFile != null) "--web.config.file=${cfg.webConfigFile}";
 
   filterValidPrometheus = filterAttrsListRecursive (n: v: !(n == "_module" || v == null));
   filterAttrsListRecursive = pred: x:
@@ -99,14 +98,14 @@ let
 
   mkDefOpt = type: defaultStr: description: mkOpt type (description + ''
 
-    Defaults to <literal>${defaultStr}</literal> in prometheus
-    when set to <literal>null</literal>.
+    Defaults to ````${defaultStr}```` in prometheus
+    when set to `null`.
   '');
 
   mkOpt = type: description: mkOption {
     type = types.nullOr type;
     default = null;
-    inherit description;
+    description = lib.mdDoc description;
   };
 
   mkSdConfigModule = extraOptions: types.submodule {
@@ -251,7 +250,7 @@ let
       authorization = mkOption {
         type = types.nullOr types.attrs;
         default = null;
-        description = ''
+        description = lib.mdDoc ''
           Sets the `Authorization` header on every scrape request with the configured credentials.
         '';
       };
@@ -288,7 +287,7 @@ let
 
         If honor_labels is set to "false", label conflicts are
         resolved by renaming conflicting labels in the scraped data
-        to "exported_&lt;original-label&gt;" (for example
+        to "exported_\<original-label\>" (for example
         "exported_instance", "exported_job") and then attaching
         server-side labels. This is useful for use cases such as
         federation, where all labels specified in the target should
@@ -299,10 +298,10 @@ let
         honor_timestamps controls whether Prometheus respects the timestamps present
         in scraped data.
 
-        If honor_timestamps is set to <literal>true</literal>, the timestamps of the metrics exposed
+        If honor_timestamps is set to `true`, the timestamps of the metrics exposed
         by the target will be used.
 
-        If honor_timestamps is set to <literal>false</literal>, the timestamps of the metrics exposed
+        If honor_timestamps is set to `false`, the timestamps of the metrics exposed
         by the target will be ignored.
       '';
 
@@ -323,13 +322,13 @@ let
       bearer_token = mkOpt types.str ''
         Sets the `Authorization` header on every scrape request with
         the configured bearer token. It is mutually exclusive with
-        <option>bearer_token_file</option>.
+        {option}`bearer_token_file`.
       '';
 
       bearer_token_file = mkOpt types.str ''
         Sets the `Authorization` header on every scrape request with
         the bearer token read from the configured file. It is mutually
-        exclusive with <option>bearer_token</option>.
+        exclusive with {option}`bearer_token`.
       '';
 
       tls_config = mkOpt promTypes.tls_config ''
@@ -379,7 +378,7 @@ let
       gce_sd_configs = mkOpt (types.listOf promTypes.gce_sd_config) ''
         List of Google Compute Engine service discovery configurations.
 
-        See <link xlink:href="https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config">the relevant Prometheus configuration docs</link>
+        See [the relevant Prometheus configuration docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
         for more detail.
       '';
 
@@ -591,7 +590,7 @@ let
 
     allow_stale = mkOpt types.bool ''
       Allow stale Consul results
-      (see <link xlink:href="https://www.consul.io/api/index.html#consistency-modes"/>).
+      (see <https://www.consul.io/api/index.html#consistency-modes>).
 
       Will reduce load on Consul.
     '';
@@ -632,16 +631,16 @@ let
         options = {
           name = mkOption {
             type = types.str;
-            description = ''
+            description = lib.mdDoc ''
               Name of the filter. The available filters are listed in the upstream documentation:
-              Services: <link xlink:href="https://docs.docker.com/engine/api/v1.40/#operation/ServiceList"/>
-              Tasks: <link xlink:href="https://docs.docker.com/engine/api/v1.40/#operation/TaskList"/>
-              Nodes: <link xlink:href="https://docs.docker.com/engine/api/v1.40/#operation/NodeList"/>
+              Services: <https://docs.docker.com/engine/api/v1.40/#operation/ServiceList>
+              Tasks: <https://docs.docker.com/engine/api/v1.40/#operation/TaskList>
+              Nodes: <https://docs.docker.com/engine/api/v1.40/#operation/NodeList>
             '';
           };
           values = mkOption {
             type = types.str;
-            description = ''
+            description = lib.mdDoc ''
               Value for the filter.
             '';
           };
@@ -664,7 +663,7 @@ let
   promTypes.dockerswarm_sd_config = mkDockerSdConfigModule {
     role = mkOption {
       type = types.enum [ "services" "tasks" "nodes" ];
-      description = ''
+      description = lib.mdDoc ''
         Role of the targets to retrieve. Must be `services`, `tasks`, or `nodes`.
       '';
     };
@@ -707,12 +706,12 @@ let
 
       access_key = mkOpt types.str ''
         The AWS API key id. If blank, the environment variable
-        <literal>AWS_ACCESS_KEY_ID</literal> is used.
+        `AWS_ACCESS_KEY_ID` is used.
       '';
 
       secret_key = mkOpt types.str ''
         The AWS API key secret. If blank, the environment variable
-         <literal>AWS_SECRET_ACCESS_KEY</literal> is used.
+         `AWS_SECRET_ACCESS_KEY` is used.
       '';
 
       profile = mkOpt types.str ''
@@ -738,8 +737,8 @@ let
           options = {
             name = mkOption {
               type = types.str;
-              description = ''
-                See <link xlink:href="https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html">this list</link>
+              description = lib.mdDoc ''
+                See [this list](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html)
                 for the available filters.
               '';
             };
@@ -747,7 +746,7 @@ let
             values = mkOption {
               type = types.listOf types.str;
               default = [ ];
-              description = ''
+              description = lib.mdDoc ''
                 Value of the filter.
               '';
             };
@@ -806,7 +805,7 @@ let
       filter = mkOpt types.str ''
         Filter can be used optionally to filter the instance list by other
         criteria Syntax of this filter string is described here in the filter
-        query parameter section: <link xlink:href="https://cloud.google.com/compute/docs/reference/latest/instances/list"/>.
+        query parameter section: <https://cloud.google.com/compute/docs/reference/latest/instances/list>.
       '';
 
       refresh_interval = mkDefOpt types.str "60s" ''
@@ -822,7 +821,7 @@ let
         The tag separator used to separate concatenated GCE instance network tags.
 
         See the GCP documentation on network tags for more information:
-        <link xlink:href="https://cloud.google.com/vpc/docs/add-remove-network-tags"/>
+        <https://cloud.google.com/vpc/docs/add-remove-network-tags>
       '';
     };
   };
@@ -917,7 +916,7 @@ let
             options = {
               role = mkOption {
                 type = types.str;
-                description = ''
+                description = lib.mdDoc ''
                   Selector role
                 '';
               };
@@ -976,11 +975,11 @@ let
       '';
 
       access_key = mkOpt types.str ''
-        The AWS API keys. If blank, the environment variable <literal>AWS_ACCESS_KEY_ID</literal> is used.
+        The AWS API keys. If blank, the environment variable `AWS_ACCESS_KEY_ID` is used.
       '';
 
       secret_key = mkOpt types.str ''
-        The AWS API keys. If blank, the environment variable <literal>AWS_SECRET_ACCESS_KEY</literal> is used.
+        The AWS API keys. If blank, the environment variable `AWS_SECRET_ACCESS_KEY` is used.
       '';
 
       profile = mkOpt types.str ''
@@ -1030,14 +1029,14 @@ let
 
     auth_token = mkOpt types.str ''
       Optional authentication information for token-based authentication:
-      <link xlink:href="https://docs.mesosphere.com/1.11/security/ent/iam-api/#passing-an-authentication-token"/>
-      It is mutually exclusive with <literal>auth_token_file</literal> and other authentication mechanisms.
+      <https://docs.mesosphere.com/1.11/security/ent/iam-api/#passing-an-authentication-token>
+      It is mutually exclusive with `auth_token_file` and other authentication mechanisms.
     '';
 
     auth_token_file = mkOpt types.str ''
       Optional authentication information for token-based authentication:
-      <link xlink:href="https://docs.mesosphere.com/1.11/security/ent/iam-api/#passing-an-authentication-token"/>
-      It is mutually exclusive with <literal>auth_token</literal> and other authentication mechanisms.
+      <https://docs.mesosphere.com/1.11/security/ent/iam-api/#passing-an-authentication-token>
+      It is mutually exclusive with `auth_token` and other authentication mechanisms.
     '';
   };
 
@@ -1222,7 +1221,7 @@ let
 
       role = mkOption {
         type = types.enum [ "instance" "baremetal" ];
-        description = ''
+        description = lib.mdDoc ''
           Role of the targets to retrieve. Must be `instance` or `baremetal`.
         '';
       };
@@ -1299,7 +1298,7 @@ let
       };
 
       groups = mkOpt (types.listOf types.str) ''
-        A list of groups for which targets are retrieved, only supported when targeting the <literal>container</literal> role.
+        A list of groups for which targets are retrieved, only supported when targeting the `container` role.
         If omitted all containers owned by the requesting account are scraped.
       '';
 
@@ -1409,7 +1408,7 @@ let
       '';
 
       action =
-        mkDefOpt (types.enum [ "replace" "keep" "drop" "hashmod" "labelmap" "labeldrop" "labelkeep" ]) "replace" ''
+        mkDefOpt (types.enum [ "replace" "lowercase" "uppercase" "keep" "drop" "hashmod" "labelmap" "labeldrop" "labelkeep" ]) "replace" ''
           Action to perform based on regex matching.
         '';
     };
@@ -1563,13 +1562,7 @@ in
 
   options.services.prometheus = {
 
-    enable = mkOption {
-      type = types.bool;
-      default = false;
-      description = lib.mdDoc ''
-        Enable the Prometheus monitoring daemon.
-      '';
-    };
+    enable = mkEnableOption (lib.mdDoc "Prometheus monitoring daemon");
 
     package = mkOption {
       type = types.package;
@@ -1621,7 +1614,7 @@ in
 
         The following property holds: switching to a configuration
         (`switch-to-configuration`) that changes the prometheus
-        configuration only finishes successully when prometheus has finished
+        configuration only finishes successfully when prometheus has finished
         loading the new configuration.
       '';
     };
@@ -1725,20 +1718,28 @@ in
       '';
     };
 
+    webConfigFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = lib.mdDoc ''
+        Specifies which file should be used as web.config.file and be passed on startup.
+        See https://prometheus.io/docs/prometheus/latest/configuration/https/ for valid options.
+      '';
+    };
+
     checkConfig = mkOption {
       type = with types; either bool (enum [ "syntax-only" ]);
       default = true;
       example = "syntax-only";
-      description = ''
-        Check configuration with <literal>promtool
-        check</literal>. The call to <literal>promtool</literal> is
+      description = lib.mdDoc ''
+        Check configuration with `promtool check`. The call to `promtool` is
         subject to sandboxing by Nix.
 
         If you use credentials stored in external files
-        (<literal>password_file</literal>, <literal>bearer_token_file</literal>, etc),
-        they will not be visible to <literal>promtool</literal>
+        (`password_file`, `bearer_token_file`, etc),
+        they will not be visible to `promtool`
         and it will report errors, despite a correct configuration.
-        To resolve this, you may set this option to <literal>"syntax-only"</literal>
+        To resolve this, you may set this option to `"syntax-only"`
         in order to only syntax check the Prometheus configuration.
       '';
     };
@@ -1797,6 +1798,33 @@ in
         WorkingDirectory = workingDir;
         StateDirectory = cfg.stateDir;
         StateDirectoryMode = "0700";
+        # Hardening
+        AmbientCapabilities = lib.mkIf (cfg.port < 1024) [ "CAP_NET_BIND_SERVICE" ];
+        CapabilityBoundingSet = if (cfg.port < 1024) then [ "CAP_NET_BIND_SERVICE" ] else [ "" ];
+        DeviceAllow = [ "/dev/null rw" ];
+        DevicePolicy = "strict";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "full";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [ "@system-service" "~@privileged" ];
       };
     };
     # prometheus-config-reload will activate after prometheus. However, what we
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.md b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.md
new file mode 100644
index 000000000000..34fadecadc74
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.md
@@ -0,0 +1,180 @@
+# Prometheus exporters {#module-services-prometheus-exporters}
+
+Prometheus exporters provide metrics for the
+[prometheus monitoring system](https://prometheus.io).
+
+## Configuration {#module-services-prometheus-exporters-configuration}
+
+One of the most common exporters is the
+[node exporter](https://github.com/prometheus/node_exporter),
+it provides hardware and OS metrics from the host it's
+running on. The exporter could be configured as follows:
+```
+  services.prometheus.exporters.node = {
+    enable = true;
+    port = 9100;
+    enabledCollectors = [
+      "logind"
+      "systemd"
+    ];
+    disabledCollectors = [
+      "textfile"
+    ];
+    openFirewall = true;
+    firewallFilter = "-i br0 -p tcp -m tcp --dport 9100";
+  };
+```
+It should now serve all metrics from the collectors that are explicitly
+enabled and the ones that are
+[enabled by default](https://github.com/prometheus/node_exporter#enabled-by-default),
+via http under `/metrics`. In this
+example the firewall should just allow incoming connections to the
+exporter's port on the bridge interface `br0` (this would
+have to be configured separately of course). For more information about
+configuration see `man configuration.nix` or search through
+the [available options](https://nixos.org/nixos/options.html#prometheus.exporters).
+
+Prometheus can now be configured to consume the metrics produced by the exporter:
+```
+    services.prometheus = {
+      # ...
+
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          static_configs = [{
+            targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+          }];
+        }
+      ];
+
+      # ...
+    }
+```
+
+## Adding a new exporter {#module-services-prometheus-exporters-new-exporter}
+
+To add a new exporter, it has to be packaged first (see
+`nixpkgs/pkgs/servers/monitoring/prometheus/` for
+examples), then a module can be added. The postfix exporter is used in this
+example:
+
+  - Some default options for all exporters are provided by
+    `nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix`:
+
+      - `enable`
+      - `port`
+      - `listenAddress`
+      - `extraFlags`
+      - `openFirewall`
+      - `firewallFilter`
+      - `user`
+      - `group`
+  - As there is already a package available, the module can now be added. This
+    is accomplished by adding a new file to the
+    `nixos/modules/services/monitoring/prometheus/exporters/`
+    directory, which will be called postfix.nix and contains all exporter
+    specific options and configuration:
+    ```
+    # nixpkgs/nixos/modules/services/prometheus/exporters/postfix.nix
+    { config, lib, pkgs, options }:
+
+    with lib;
+
+    let
+      # for convenience we define cfg here
+      cfg = config.services.prometheus.exporters.postfix;
+    in
+    {
+      port = 9154; # The postfix exporter listens on this port by default
+
+      # `extraOpts` is an attribute set which contains additional options
+      # (and optional overrides for default options).
+      # Note that this attribute is optional.
+      extraOpts = {
+        telemetryPath = mkOption {
+          type = types.str;
+          default = "/metrics";
+          description = ''
+            Path under which to expose metrics.
+          '';
+        };
+        logfilePath = mkOption {
+          type = types.path;
+          default = /var/log/postfix_exporter_input.log;
+          example = /var/log/mail.log;
+          description = ''
+            Path where Postfix writes log entries.
+            This file will be truncated by this exporter!
+          '';
+        };
+        showqPath = mkOption {
+          type = types.path;
+          default = /var/spool/postfix/public/showq;
+          example = /var/lib/postfix/queue/public/showq;
+          description = ''
+            Path at which Postfix places its showq socket.
+          '';
+        };
+      };
+
+      # `serviceOpts` is an attribute set which contains configuration
+      # for the exporter's systemd service. One of
+      # `serviceOpts.script` and `serviceOpts.serviceConfig.ExecStart`
+      # has to be specified here. This will be merged with the default
+      # service configuration.
+      # Note that by default 'DynamicUser' is 'true'.
+      serviceOpts = {
+        serviceConfig = {
+          DynamicUser = false;
+          ExecStart = ''
+            ${pkgs.prometheus-postfix-exporter}/bin/postfix_exporter \
+              --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+              --web.telemetry-path ${cfg.telemetryPath} \
+              ${concatStringsSep " \\\n  " cfg.extraFlags}
+          '';
+        };
+      };
+    }
+    ```
+  - This should already be enough for the postfix exporter. Additionally one
+    could now add assertions and conditional default values. This can be done
+    in the 'meta-module' that combines all exporter definitions and generates
+    the submodules:
+    `nixpkgs/nixos/modules/services/prometheus/exporters.nix`
+
+## Updating an exporter module {#module-services-prometheus-exporters-update-exporter-module}
+
+Should an exporter option change at some point, it is possible to add
+information about the change to the exporter definition similar to
+`nixpkgs/nixos/modules/rename.nix`:
+```
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.nginx;
+in
+{
+  port = 9113;
+  extraOpts = {
+    # additional module options
+    # ...
+  };
+  serviceOpts = {
+    # service configuration
+    # ...
+  };
+  imports = [
+    # 'services.prometheus.exporters.nginx.telemetryEndpoint' -> 'services.prometheus.exporters.nginx.telemetryPath'
+    (mkRenamedOptionModule [ "telemetryEndpoint" ] [ "telemetryPath" ])
+
+    # removed option 'services.prometheus.exporters.nginx.insecure'
+    (mkRemovedOptionModule [ "insecure" ] ''
+      This option was replaced by 'prometheus.exporters.nginx.sslVerify' which defaults to true.
+    '')
+    ({ options.warnings = options.warnings; })
+  ];
+}
+```
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix
index 47c30a0f24bb..235296168934 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -35,7 +35,9 @@ let
     "dovecot"
     "fastly"
     "fritzbox"
+    "graphite"
     "influxdb"
+    "ipmi"
     "json"
     "jitsi"
     "kea"
@@ -50,6 +52,7 @@ let
     "nginx"
     "nginxlog"
     "node"
+    "nut"
     "openldap"
     "openvpn"
     "pihole"
@@ -62,25 +65,29 @@ let
     "rspamd"
     "rtl_433"
     "script"
+    "shelly"
     "snmp"
     "smartctl"
     "smokeping"
     "sql"
+    "statsd"
     "surfboard"
     "systemd"
     "tor"
     "unbound"
     "unifi"
-    "unifi-poller"
+    "unpoller"
+    "v2ray"
     "varnish"
     "wireguard"
     "flow"
+    "zfs"
   ] (name:
     import (./. + "/exporters/${name}.nix") { inherit config lib pkgs options; }
   );
 
   mkExporterOpts = ({ name, port }: {
-    enable = mkEnableOption "the prometheus ${name} exporter";
+    enable = mkEnableOption (lib.mdDoc "the prometheus ${name} exporter");
     port = mkOption {
       type = types.port;
       default = port;
@@ -115,10 +122,10 @@ let
       example = literalExpression ''
         "-i eth0 -p tcp -m tcp --dport ${toString port}"
       '';
-      description = ''
+      description = lib.mdDoc ''
         Specify a filter for iptables to use when
-        <option>services.prometheus.exporters.${name}.openFirewall</option>
-        is true. It is used as `ip46tables -I nixos-fw <option>firewallFilter</option> -j nixos-fw-accept`.
+        {option}`services.prometheus.exporters.${name}.openFirewall`
+        is true. It is used as `ip46tables -I nixos-fw firewallFilter -j nixos-fw-accept`.
       '';
     };
     user = mkOption {
@@ -194,7 +201,7 @@ let
         serviceConfig.LockPersonality = true;
         serviceConfig.MemoryDenyWriteExecute = true;
         serviceConfig.NoNewPrivileges = true;
-        serviceConfig.PrivateDevices = true;
+        serviceConfig.PrivateDevices = mkDefault true;
         serviceConfig.ProtectClock = mkDefault true;
         serviceConfig.ProtectControlGroups = true;
         serviceConfig.ProtectHome = true;
@@ -226,6 +233,10 @@ in
   options.services.prometheus.exporters = mkOption {
     type = types.submodule {
       options = (mkSubModules);
+      imports = [
+        ../../../misc/assertions.nix
+        (lib.mkRenamedOptionModule [ "unifi-poller" ] [ "unpoller" ])
+      ];
     };
     description = lib.mdDoc "Prometheus exporter configuration";
     default = {};
@@ -242,6 +253,22 @@ in
 
   config = mkMerge ([{
     assertions = [ {
+      assertion = cfg.ipmi.enable -> (cfg.ipmi.configFile != null) -> (
+        !(lib.hasPrefix "/tmp/" cfg.ipmi.configFile)
+      );
+      message = ''
+        Config file specified in `services.prometheus.exporters.ipmi.configFile' must
+          not reside within /tmp - it won't be visible to the systemd service.
+      '';
+    } {
+      assertion = cfg.ipmi.enable -> (cfg.ipmi.webConfigFile != null) -> (
+        !(lib.hasPrefix "/tmp/" cfg.ipmi.webConfigFile)
+      );
+      message = ''
+        Config file specified in `services.prometheus.exporters.ipmi.webConfigFile' must
+          not reside within /tmp - it won't be visible to the systemd service.
+      '';
+    } {
       assertion = cfg.snmp.enable -> (
         (cfg.snmp.configurationPath == null) != (cfg.snmp.configuration == null)
       );
@@ -273,13 +300,14 @@ in
         Please specify either 'services.prometheus.exporters.sql.configuration' or
           'services.prometheus.exporters.sql.configFile'
       '';
-    } ] ++ (flip map (attrNames cfg) (exporter: {
+    } ] ++ (flip map (attrNames exporterOpts) (exporter: {
       assertion = cfg.${exporter}.firewallFilter != null -> cfg.${exporter}.openFirewall;
       message = ''
         The `firewallFilter'-option of exporter ${exporter} doesn't have any effect unless
         `openFirewall' is set to `true'!
       '';
-    }));
+    })) ++ config.services.prometheus.exporters.assertions;
+    warnings = config.services.prometheus.exporters.warnings;
   }] ++ [(mkIf config.services.minio.enable {
     services.prometheus.exporters.minio.minioAddress  = mkDefault "http://localhost:9000";
     services.prometheus.exporters.minio.minioAccessKey = mkDefault config.services.minio.accessKey;
@@ -297,7 +325,7 @@ in
   );
 
   meta = {
-    doc = ./exporters.xml;
+    doc = ./exporters.md;
     maintainers = [ maintainers.willibutz ];
   };
 }
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.xml b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.xml
deleted file mode 100644
index 1df88bb61a12..000000000000
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.xml
+++ /dev/null
@@ -1,248 +0,0 @@
-<chapter xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink"
-         xmlns:xi="http://www.w3.org/2001/XInclude"
-         version="5.0"
-         xml:id="module-services-prometheus-exporters">
- <title>Prometheus exporters</title>
- <para>
-  Prometheus exporters provide metrics for the
-  <link xlink:href="https://prometheus.io">prometheus monitoring system</link>.
- </para>
- <section xml:id="module-services-prometheus-exporters-configuration">
-  <title>Configuration</title>
-
-  <para>
-   One of the most common exporters is the
-   <link xlink:href="https://github.com/prometheus/node_exporter">node
-   exporter</link>, it provides hardware and OS metrics from the host it's
-   running on. The exporter could be configured as follows:
-<programlisting>
-  services.prometheus.exporters.node = {
-    enable = true;
-    port = 9100;
-    enabledCollectors = [
-      "logind"
-      "systemd"
-    ];
-    disabledCollectors = [
-      "textfile"
-    ];
-    openFirewall = true;
-    firewallFilter = "-i br0 -p tcp -m tcp --dport 9100";
-  };
-</programlisting>
-   It should now serve all metrics from the collectors that are explicitly
-   enabled and the ones that are
-   <link xlink:href="https://github.com/prometheus/node_exporter#enabled-by-default">enabled
-   by default</link>, via http under <literal>/metrics</literal>. In this
-   example the firewall should just allow incoming connections to the
-   exporter's port on the bridge interface <literal>br0</literal> (this would
-   have to be configured seperately of course). For more information about
-   configuration see <literal>man configuration.nix</literal> or search through
-   the
-   <link xlink:href="https://nixos.org/nixos/options.html#prometheus.exporters">available
-   options</link>.
-  </para>
-
-  <para>
-    Prometheus can now be configured to consume the metrics produced by the exporter:
-    <programlisting>
-    services.prometheus = {
-      # ...
-
-      scrapeConfigs = [
-        {
-          job_name = "node";
-          static_configs = [{
-            targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
-          }];
-        }
-      ];
-
-      # ...
-    }
-    </programlisting>
-  </para>
- </section>
- <section xml:id="module-services-prometheus-exporters-new-exporter">
-  <title>Adding a new exporter</title>
-
-  <para>
-   To add a new exporter, it has to be packaged first (see
-   <literal>nixpkgs/pkgs/servers/monitoring/prometheus/</literal> for
-   examples), then a module can be added. The postfix exporter is used in this
-   example:
-  </para>
-
-  <itemizedlist>
-   <listitem>
-    <para>
-     Some default options for all exporters are provided by
-     <literal>nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix</literal>:
-    </para>
-   </listitem>
-   <listitem override='none'>
-    <itemizedlist>
-     <listitem>
-      <para>
-       <literal>enable</literal>
-      </para>
-     </listitem>
-     <listitem>
-      <para>
-       <literal>port</literal>
-      </para>
-     </listitem>
-     <listitem>
-      <para>
-       <literal>listenAddress</literal>
-      </para>
-     </listitem>
-     <listitem>
-      <para>
-       <literal>extraFlags</literal>
-      </para>
-     </listitem>
-     <listitem>
-      <para>
-       <literal>openFirewall</literal>
-      </para>
-     </listitem>
-     <listitem>
-      <para>
-       <literal>firewallFilter</literal>
-      </para>
-     </listitem>
-     <listitem>
-      <para>
-       <literal>user</literal>
-      </para>
-     </listitem>
-     <listitem>
-      <para>
-       <literal>group</literal>
-      </para>
-     </listitem>
-    </itemizedlist>
-   </listitem>
-   <listitem>
-    <para>
-     As there is already a package available, the module can now be added. This
-     is accomplished by adding a new file to the
-     <literal>nixos/modules/services/monitoring/prometheus/exporters/</literal>
-     directory, which will be called postfix.nix and contains all exporter
-     specific options and configuration:
-<programlisting>
-# nixpgs/nixos/modules/services/prometheus/exporters/postfix.nix
-{ config, lib, pkgs, options }:
-
-with lib;
-
-let
-  # for convenience we define cfg here
-  cfg = config.services.prometheus.exporters.postfix;
-in
-{
-  port = 9154; # The postfix exporter listens on this port by default
-
-  # `extraOpts` is an attribute set which contains additional options
-  # (and optional overrides for default options).
-  # Note that this attribute is optional.
-  extraOpts = {
-    telemetryPath = mkOption {
-      type = types.str;
-      default = "/metrics";
-      description = ''
-        Path under which to expose metrics.
-      '';
-    };
-    logfilePath = mkOption {
-      type = types.path;
-      default = /var/log/postfix_exporter_input.log;
-      example = /var/log/mail.log;
-      description = ''
-        Path where Postfix writes log entries.
-        This file will be truncated by this exporter!
-      '';
-    };
-    showqPath = mkOption {
-      type = types.path;
-      default = /var/spool/postfix/public/showq;
-      example = /var/lib/postfix/queue/public/showq;
-      description = ''
-        Path at which Postfix places its showq socket.
-      '';
-    };
-  };
-
-  # `serviceOpts` is an attribute set which contains configuration
-  # for the exporter's systemd service. One of
-  # `serviceOpts.script` and `serviceOpts.serviceConfig.ExecStart`
-  # has to be specified here. This will be merged with the default
-  # service confiuration.
-  # Note that by default 'DynamicUser' is 'true'.
-  serviceOpts = {
-    serviceConfig = {
-      DynamicUser = false;
-      ExecStart = ''
-        ${pkgs.prometheus-postfix-exporter}/bin/postfix_exporter \
-          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
-          --web.telemetry-path ${cfg.telemetryPath} \
-          ${concatStringsSep " \\\n  " cfg.extraFlags}
-      '';
-    };
-  };
-}
-</programlisting>
-    </para>
-   </listitem>
-   <listitem>
-    <para>
-     This should already be enough for the postfix exporter. Additionally one
-     could now add assertions and conditional default values. This can be done
-     in the 'meta-module' that combines all exporter definitions and generates
-     the submodules:
-     <literal>nixpkgs/nixos/modules/services/prometheus/exporters.nix</literal>
-    </para>
-   </listitem>
-  </itemizedlist>
- </section>
- <section xml:id="module-services-prometheus-exporters-update-exporter-module">
-  <title>Updating an exporter module</title>
-   <para>
-     Should an exporter option change at some point, it is possible to add
-     information about the change to the exporter definition similar to
-     <literal>nixpkgs/nixos/modules/rename.nix</literal>:
-<programlisting>
-{ config, lib, pkgs, options }:
-
-with lib;
-
-let
-  cfg = config.services.prometheus.exporters.nginx;
-in
-{
-  port = 9113;
-  extraOpts = {
-    # additional module options
-    # ...
-  };
-  serviceOpts = {
-    # service configuration
-    # ...
-  };
-  imports = [
-    # 'services.prometheus.exporters.nginx.telemetryEndpoint' -> 'services.prometheus.exporters.nginx.telemetryPath'
-    (mkRenamedOptionModule [ "telemetryEndpoint" ] [ "telemetryPath" ])
-
-    # removed option 'services.prometheus.exporters.nginx.insecure'
-    (mkRemovedOptionModule [ "insecure" ] ''
-      This option was replaced by 'prometheus.exporters.nginx.sslVerify' which defaults to true.
-    '')
-    ({ options.warnings = options.warnings; })
-  ];
-}
-</programlisting>
-    </para>
-  </section>
-</chapter>
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/collectd.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/collectd.nix
index eab1f9e7b4d2..f67596f05a3a 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/collectd.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/collectd.nix
@@ -9,7 +9,7 @@ in
   port = 9103;
   extraOpts = {
     collectdBinary = {
-      enable = mkEnableOption "collectd binary protocol receiver";
+      enable = mkEnableOption (lib.mdDoc "collectd binary protocol receiver");
 
       authFile = mkOption {
         default = null;
@@ -18,7 +18,7 @@ in
       };
 
       port = mkOption {
-        type = types.int;
+        type = types.port;
         default = 25826;
         description = lib.mdDoc "Network address on which to accept collectd binary network packets.";
       };
@@ -58,10 +58,10 @@ in
     };
   };
   serviceOpts = let
-    collectSettingsArgs = if (cfg.collectdBinary.enable) then ''
+    collectSettingsArgs = optionalString (cfg.collectdBinary.enable) ''
       --collectd.listen-address ${cfg.collectdBinary.listenAddress}:${toString cfg.collectdBinary.port} \
       --collectd.security-level ${cfg.collectdBinary.securityLevel} \
-    '' else "";
+    '';
   in {
     serviceConfig = {
       ExecStart = ''
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix
index d86b750ba5f2..6fb438353a4c 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix
@@ -19,24 +19,24 @@ in
       type = types.path;
       default = "/var/run/dovecot/stats";
       example = "/var/run/dovecot2/old-stats";
-      description = ''
+      description = lib.mdDoc ''
         Path under which the stats socket is placed.
         The user/group under which the exporter runs,
         should be able to access the socket in order
         to scrape the metrics successfully.
 
         Please keep in mind that the stats module has changed in
-        <link xlink:href="https://wiki2.dovecot.org/Upgrading/2.3">Dovecot 2.3+</link> which
-        is not <link xlink:href="https://github.com/kumina/dovecot_exporter/issues/8">compatible with this exporter</link>.
+        [Dovecot 2.3+](https://wiki2.dovecot.org/Upgrading/2.3) which
+        is not [compatible with this exporter](https://github.com/kumina/dovecot_exporter/issues/8).
 
         The following extra config has to be passed to Dovecot to ensure that recent versions
         work with this exporter:
-        <programlisting>
+        ```
         {
-          <xref linkend="opt-services.prometheus.exporters.dovecot.enable"/> = true;
-          <xref linkend="opt-services.prometheus.exporters.dovecot.socketPath"/> = "/var/run/dovecot2/old-stats";
-          <xref linkend="opt-services.dovecot2.mailPlugins.globally.enable"/> = [ "old_stats" ];
-          <xref linkend="opt-services.dovecot2.extraConfig"/> = '''
+          services.prometheus.exporters.dovecot.enable = true;
+          services.prometheus.exporters.dovecot.socketPath = "/var/run/dovecot2/old-stats";
+          services.dovecot2.mailPlugins.globally.enable = [ "old_stats" ];
+          services.dovecot2.extraConfig = '''
             service old-stats {
               unix_listener old-stats {
                 user = dovecot-exporter
@@ -60,7 +60,7 @@ in
             }
           ''';
         }
-        </programlisting>
+        ```
       '';
     };
     scopes = mkOption {
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix
index 182a1131c054..36409caccf2e 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/fastly.nix
@@ -7,7 +7,7 @@ in
 {
   port = 9118;
   extraOpts = {
-    debug = mkEnableOption "Debug logging mode for fastly-exporter";
+    debug = mkEnableOption (lib.mdDoc "Debug logging mode for fastly-exporter");
 
     configFile = mkOption {
       type = types.nullOr types.path;
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/graphite.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/graphite.nix
new file mode 100644
index 000000000000..34a887104212
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/graphite.nix
@@ -0,0 +1,41 @@
+{ config, lib, pkgs, options }:
+
+let
+  cfg = config.services.prometheus.exporters.graphite;
+  format = pkgs.formats.yaml { };
+in
+{
+  port = 9108;
+  extraOpts = {
+    graphitePort = lib.mkOption {
+      type = lib.types.port;
+      default = 9109;
+      description = lib.mdDoc ''
+        Port to use for the graphite server.
+      '';
+    };
+    mappingSettings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = format.type;
+        options = { };
+      };
+      default = { };
+      description = lib.mdDoc ''
+        Mapping configuration for the exporter, see
+        <https://github.com/prometheus/graphite_exporter#yaml-config> for
+        available options.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-graphite-exporter}/bin/graphite_exporter \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --graphite.listen-address ${cfg.listenAddress}:${toString cfg.graphitePort} \
+          --graphite.mapping-config ${format.generate "mapping.yml" cfg.mappingSettings} \
+          ${lib.concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/ipmi.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/ipmi.nix
new file mode 100644
index 000000000000..55c4f4aa4826
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/ipmi.nix
@@ -0,0 +1,41 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  logPrefix = "services.prometheus.exporter.ipmi";
+  cfg = config.services.prometheus.exporters.ipmi;
+in {
+  port = 9290;
+
+  extraOpts = {
+    configFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = lib.mdDoc ''
+        Path to configuration file.
+      '';
+    };
+
+    webConfigFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = lib.mdDoc ''
+        Path to configuration file that can enable TLS or authentication.
+      '';
+    };
+  };
+
+  serviceOpts.serviceConfig = {
+    ExecStart = with cfg; concatStringsSep " " ([
+      "${pkgs.prometheus-ipmi-exporter}/bin/ipmi_exporter"
+      "--web.listen-address ${listenAddress}:${toString port}"
+    ] ++ optionals (cfg.webConfigFile != null) [
+      "--web.config.file ${escapeShellArg cfg.webConfigFile}"
+    ] ++ optionals (cfg.configFile != null) [
+      "--config.file ${escapeShellArg cfg.configFile}"
+    ] ++ extraFlags);
+
+    ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/kea.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/kea.nix
index 0682f9da4003..ed33c72f644f 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/kea.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/kea.nix
@@ -35,7 +35,7 @@ in {
         ${pkgs.prometheus-kea-exporter}/bin/kea-exporter \
           --address ${cfg.listenAddress} \
           --port ${toString cfg.port} \
-          ${concatStringsSep " \\n" cfg.controlSocketPaths}
+          ${concatStringsSep " " cfg.controlSocketPaths}
       '';
       SupplementaryGroups = [ "kea" ];
       RestrictAddressFamilies = [
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/knot.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/knot.nix
index 2df0ab93bdef..a73425b37da7 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/knot.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/knot.nix
@@ -11,8 +11,8 @@ in {
       type = types.str;
       default = "${pkgs.knot-dns.out}/lib/libknot.so";
       defaultText = literalExpression ''"''${pkgs.knot-dns.out}/lib/libknot.so"'';
-      description = ''
-        Path to the library of <package>knot-dns</package>.
+      description = lib.mdDoc ''
+        Path to the library of `knot-dns`.
       '';
     };
 
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/mail.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/mail.nix
index 205188538a21..15079f5841f4 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/mail.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/mail.nix
@@ -33,7 +33,7 @@ let
       '';
     };
     port = mkOption {
-      type = types.int;
+      type = types.port;
       example = 587;
       description = lib.mdDoc ''
         Port to use for SMTP.
@@ -112,16 +112,16 @@ let
           detectionDir = "/path/to/Maildir/new";
         } ]
       '';
-      description = ''
+      description = lib.mdDoc ''
         List of servers that should be probed.
 
-        <emphasis>Note:</emphasis> if your mailserver has <citerefentry><refentrytitle>rspamd</refentrytitle><manvolnum>8</manvolnum></citerefentry> configured,
+        *Note:* if your mailserver has {manpage}`rspamd(8)` configured,
         it can happen that emails from this exporter are marked as spam.
 
         It's possible to work around the issue with a config like this:
-        <programlisting>
+        ```
         {
-          <link linkend="opt-services.rspamd.locals._name_.text">services.rspamd.locals."multimap.conf".text</link> = '''
+          services.rspamd.locals."multimap.conf".text = '''
             ALLOWLIST_PROMETHEUS {
               filter = "email:domain:tld";
               type = "from";
@@ -130,7 +130,7 @@ let
             }
           ''';
         }
-        </programlisting>
+        ```
       '';
     };
   };
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix
index 9967d3647e00..674dc9dd4158 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nginxlog.nix
@@ -10,14 +10,14 @@ in {
     settings = mkOption {
       type = types.attrs;
       default = {};
-      description = ''
+      description = lib.mdDoc ''
         All settings of nginxlog expressed as an Nix attrset.
 
         Check the official documentation for the corresponding YAML
         settings that can all be used here: https://github.com/martin-helmich/prometheus-nginxlog-exporter
 
         The `listen` object is already generated by `port`, `listenAddress` and `metricsEndpoint` and
-        will be merged with the value of `settings` before writting it as JSON.
+        will be merged with the value of `settings` before writing it as JSON.
       '';
     };
 
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/node.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/node.nix
index ae69c29d0a51..dd8602e2c63d 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/node.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/node.nix
@@ -4,6 +4,8 @@ with lib;
 
 let
   cfg = config.services.prometheus.exporters.node;
+  collectorIsEnabled = final: any (collector: (final == collector)) cfg.enabledCollectors;
+  collectorIsDisabled = final: any (collector: (final == collector)) cfg.disabledCollectors;
 in
 {
   port = 9100;
@@ -35,15 +37,15 @@ in
           ${concatMapStringsSep " " (x: "--no-collector." + x) cfg.disabledCollectors} \
           --web.listen-address ${cfg.listenAddress}:${toString cfg.port} ${concatStringsSep " " cfg.extraFlags}
       '';
-      RestrictAddressFamilies = optionals (any (collector: (collector == "logind" || collector == "systemd")) cfg.enabledCollectors) [
+      RestrictAddressFamilies = optionals (collectorIsEnabled "logind" || collectorIsEnabled "systemd") [
         # needs access to dbus via unix sockets (logind/systemd)
         "AF_UNIX"
-      ] ++ optionals (any (collector: (collector == "network_route" || collector == "wifi")) cfg.enabledCollectors) [
+      ] ++ optionals (collectorIsEnabled "network_route" || collectorIsEnabled "wifi" || ! collectorIsDisabled "netdev") [
         # needs netlink sockets for wireless collector
         "AF_NETLINK"
       ];
       # The timex collector needs to access clock APIs
-      ProtectClock = any (collector: collector == "timex") cfg.disabledCollectors;
+      ProtectClock = collectorIsDisabled "timex";
       # Allow space monitoring under /home
       ProtectHome = true;
     };
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nut.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nut.nix
new file mode 100644
index 000000000000..1c86b48b4509
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/nut.nix
@@ -0,0 +1,50 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.nut;
+in
+{
+  port = 9199;
+  extraOpts = {
+    nutServer = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = lib.mdDoc ''
+        Hostname or address of the NUT server
+      '';
+    };
+    nutUser = mkOption {
+      type = types.str;
+      default = "";
+      example = "nut";
+      description = lib.mdDoc ''
+        The user to log in into NUT server. If set, passwordPath should
+        also be set.
+
+        Default NUT configs usually permit reading variables without
+        authentication.
+      '';
+    };
+    passwordPath = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      apply = final: if final == null then null else toString final;
+      description = lib.mdDoc ''
+        A run-time path to the nutUser password file, which should be
+        provisioned outside of Nix store.
+      '';
+    };
+  };
+  serviceOpts = {
+    script = ''
+      ${optionalString (cfg.passwordPath != null)
+      "export NUT_EXPORTER_PASSWORD=$(cat ${toString cfg.passwordPath})"}
+      ${pkgs.prometheus-nut-exporter}/bin/nut_exporter \
+        --nut.server=${cfg.nutServer} \
+        --web.listen-address="${cfg.listenAddress}:${toString cfg.port}" \
+        ${optionalString (cfg.nutUser != "") "--nut.username=${cfg.nutUser}"}
+    '';
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/openldap.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/openldap.nix
index 59fcedef9ddc..aee3ae5bb2d4 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/openldap.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/openldap.nix
@@ -10,31 +10,31 @@ in {
     ldapCredentialFile = mkOption {
       type = types.path;
       example = "/run/keys/ldap_pass";
-      description = ''
+      description = lib.mdDoc ''
         Environment file to contain the credentials to authenticate against
-        <package>openldap</package>.
+        `openldap`.
 
         The file should look like this:
-        <programlisting>
+        ```
         ---
         ldapUser: "cn=monitoring,cn=Monitor"
         ldapPass: "secret"
-        </programlisting>
+        ```
       '';
     };
     protocol = mkOption {
       default = "tcp";
       example = "udp";
       type = types.str;
-      description = ''
-        Which protocol to use to connect against <package>openldap</package>.
+      description = lib.mdDoc ''
+        Which protocol to use to connect against `openldap`.
       '';
     };
     ldapAddr = mkOption {
       default = "localhost:389";
       type = types.str;
-      description = ''
-        Address of the <package>openldap</package>-instance.
+      description = lib.mdDoc ''
+        Address of the `openldap`-instance.
       '';
     };
     metricsPath = mkOption {
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pihole.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pihole.nix
index 537d72e85c8f..6f403b3e58c8 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pihole.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pihole.nix
@@ -6,6 +6,11 @@ let
   cfg = config.services.prometheus.exporters.pihole;
 in
 {
+  imports = [
+    (mkRemovedOptionModule [ "interval"] "This option has been removed.")
+    ({ options.warnings = options.warnings; options.assertions = options.assertions; })
+  ];
+
   port = 9617;
   extraOpts = {
     apiToken = mkOption {
@@ -13,15 +18,7 @@ in
       default = "";
       example = "580a770cb40511eb85290242ac130003580a770cb40511eb85290242ac130003";
       description = lib.mdDoc ''
-        pi-hole API token which can be used instead of a password
-      '';
-    };
-    interval = mkOption {
-      type = types.str;
-      default = "10s";
-      example = "30s";
-      description = lib.mdDoc ''
-        How often to scrape new data
+        Pi-Hole API token which can be used instead of a password
       '';
     };
     password = mkOption {
@@ -29,7 +26,7 @@ in
       default = "";
       example = "password";
       description = lib.mdDoc ''
-        The password to login into pihole. An api token can be used instead.
+        The password to login into Pi-Hole. An api token can be used instead.
       '';
     };
     piholeHostname = mkOption {
@@ -37,7 +34,7 @@ in
       default = "pihole";
       example = "127.0.0.1";
       description = lib.mdDoc ''
-        Hostname or address where to find the pihole webinterface
+        Hostname or address where to find the Pi-Hole webinterface
       '';
     };
     piholePort = mkOption {
@@ -45,7 +42,7 @@ in
       default = 80;
       example = 443;
       description = lib.mdDoc ''
-        The port pihole webinterface is reachable on
+        The port Pi-Hole webinterface is reachable on
       '';
     };
     protocol = mkOption {
@@ -53,21 +50,28 @@ in
       default = "http";
       example = "https";
       description = lib.mdDoc ''
-        The protocol which is used to connect to pihole
+        The protocol which is used to connect to Pi-Hole
+      '';
+    };
+    timeout = mkOption {
+      type = types.str;
+      default = "5s";
+      description = lib.mdDoc ''
+        Controls the timeout to connect to a Pi-Hole instance
       '';
     };
   };
   serviceOpts = {
     serviceConfig = {
       ExecStart = ''
-        ${pkgs.bash}/bin/bash -c "${pkgs.prometheus-pihole-exporter}/bin/pihole-exporter \
-          -interval ${cfg.interval} \
+        ${pkgs.prometheus-pihole-exporter}/bin/pihole-exporter \
           ${optionalString (cfg.apiToken != "") "-pihole_api_token ${cfg.apiToken}"} \
           -pihole_hostname ${cfg.piholeHostname} \
           ${optionalString (cfg.password != "") "-pihole_password ${cfg.password}"} \
           -pihole_port ${toString cfg.piholePort} \
           -pihole_protocol ${cfg.protocol} \
-          -port ${toString cfg.port}"
+          -port ${toString cfg.port} \
+          -timeout ${cfg.timeout}
       '';
     };
   };
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/postgres.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/postgres.nix
index 5e8dd21af85f..755d771ecdff 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/postgres.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/postgres.nix
@@ -36,8 +36,8 @@ in
       type = types.nullOr types.path;
       default = null;
       example = "/root/prometheus-postgres-exporter.env";
-      description = ''
-        Environment file as defined in <citerefentry><refentrytitle>systemd.exec</refentrytitle><manvolnum>5</manvolnum></citerefentry>.
+      description = lib.mdDoc ''
+        Environment file as defined in {manpage}`systemd.exec(5)`.
 
         Secrets may be passed to the service without adding them to the
         world-readable Nix store, by specifying placeholder variables as
@@ -46,7 +46,7 @@ in
 
         Environment variables from this file will be interpolated into the
         config file using envsubst with this syntax:
-        <literal>$ENVIRONMENT ''${VARIABLE}</literal>
+        `$ENVIRONMENT ''${VARIABLE}`
 
         The main use is to set the DATA_SOURCE_NAME that contains the
         postgres password
@@ -54,10 +54,10 @@ in
         note that contents from this file will override dataSourceName
         if you have set it from nix.
 
-        <programlisting>
+        ```
           # Content of the environment file
           DATA_SOURCE_NAME=postgresql://username:password@localhost:5432/postgres?sslmode=disable
-        </programlisting>
+        ```
 
         Note that this file needs to be available on the host on which
         this exporter is running.
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pve.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pve.nix
index 8e2573d084bc..e02acad3ecd1 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pve.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/pve.nix
@@ -10,7 +10,7 @@ let
     text = "default:";
   };
 
-  computedConfigFile = "${if cfg.configFile == null then emptyConfigFile else cfg.configFile}";
+  computedConfigFile = if cfg.configFile == null then emptyConfigFile else cfg.configFile;
 in
 {
   port = 9221;
@@ -100,6 +100,8 @@ in
   };
   serviceOpts = {
     serviceConfig = {
+      DynamicUser = cfg.environmentFile == null;
+      LoadCredential = "configFile:${computedConfigFile}";
       ExecStart = ''
         ${cfg.package}/bin/pve_exporter \
           --${if cfg.collectors.status == true then "" else "no-"}collector.status \
@@ -108,11 +110,11 @@ in
           --${if cfg.collectors.cluster == true then "" else "no-"}collector.cluster \
           --${if cfg.collectors.resources == true then "" else "no-"}collector.resources \
           --${if cfg.collectors.config == true then "" else "no-"}collector.config \
-          ${computedConfigFile} \
+          %d/configFile \
           ${toString cfg.port} ${cfg.listenAddress}
       '';
     } // optionalAttrs (cfg.environmentFile != null) {
-          EnvironmentFile = cfg.environmentFile;
+      EnvironmentFile = cfg.environmentFile;
     };
   };
 }
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix
index 0b48827f43f7..f9dcfad07d30 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix
@@ -9,7 +9,7 @@ let
     pkgs.writeText "rspamd-exporter-config.yml" (builtins.toJSON conf);
 
   generateConfig = extraLabels: {
-    metrics = (map (path: {
+    modules.default.metrics = (map (path: {
       name = "rspamd_${replaceStrings [ "[" "." " " "]" "\\" "'" ] [ "_" "_" "_" "" "" "" ] path}";
       path = "{ .${path} }";
       labels = extraLabels;
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rtl_433.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rtl_433.nix
index 42bf1788aea9..1f7235cb7830 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rtl_433.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/rtl_433.nix
@@ -16,7 +16,7 @@ in
           };
           "${field}" = lib.mkOption {
             type = int;
-            inherit description;
+            description = lib.mdDoc description;
           };
           location = lib.mkOption {
             type = str;
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/shelly.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/shelly.nix
new file mode 100644
index 000000000000..b9cfd1b1e84a
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/shelly.nix
@@ -0,0 +1,27 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.shelly;
+in
+{
+  port = 9784;
+  extraOpts = {
+    metrics-file = mkOption {
+      type = types.path;
+      description = lib.mdDoc ''
+        Path to the JSON file with the metric definitions
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-shelly-exporter}/bin/shelly_exporter \
+          -metrics-file ${cfg.metrics-file} \
+          -listen-address ${cfg.listenAddress}:${toString cfg.port}
+      '';
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix
index 8906c25d5037..50e1321a1e9c 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/smartctl.nix
@@ -4,16 +4,12 @@ with lib;
 
 let
   cfg = config.services.prometheus.exporters.smartctl;
-  format = pkgs.formats.yaml {};
-  configFile = format.generate "smartctl-exporter.yml" {
-    smartctl_exporter = {
-      bind_to = "${cfg.listenAddress}:${toString cfg.port}";
-      url_path = "/metrics";
-      smartctl_location = "${pkgs.smartmontools}/bin/smartctl";
-      collect_not_more_than_period = cfg.maxInterval;
-      devices = cfg.devices;
-    };
-  };
+  args = lib.escapeShellArgs ([
+    "--web.listen-address=${cfg.listenAddress}:${toString cfg.port}"
+    "--smartctl.path=${pkgs.smartmontools}/bin/smartctl"
+    "--smartctl.interval=${cfg.maxInterval}"
+  ] ++ map (device: "--smartctl.device=${device}") cfg.devices
+  ++ cfg.extraFlags);
 in {
   port = 9633;
 
@@ -50,26 +46,19 @@ in {
         "CAP_SYS_ADMIN"
       ];
       DevicePolicy = "closed";
-      DeviceAllow = lib.mkOverride 100 (
-        if cfg.devices != [] then
-          cfg.devices
-        else [
-          "block-blkext rw"
-          "block-sd rw"
-          "char-nvme rw"
-        ]
-      );
+      DeviceAllow = lib.mkOverride 50 [
+        "block-blkext rw"
+        "block-sd rw"
+        "char-nvme rw"
+      ];
       ExecStart = ''
-        ${pkgs.prometheus-smartctl-exporter}/bin/smartctl_exporter -config ${configFile}
+        ${pkgs.prometheus-smartctl-exporter}/bin/smartctl_exporter ${args}
       '';
       PrivateDevices = lib.mkForce false;
       ProtectProc = "invisible";
       ProcSubset = "pid";
       SupplementaryGroups = [ "disk" ];
-      SystemCallFilter = [
-        "@system-service"
-        "~@privileged @resources"
-      ];
+      SystemCallFilter = [ "@system-service" "~@privileged" ];
     };
   };
 }
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/statsd.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/statsd.nix
new file mode 100644
index 000000000000..d9d732d8c125
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/statsd.nix
@@ -0,0 +1,19 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.statsd;
+in
+{
+  port = 9102;
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-statsd-exporter}/bin/statsd_exporter \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/tor.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/tor.nix
index edf9b57607a3..7a9167110a27 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/tor.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/tor.nix
@@ -17,7 +17,7 @@ in
     };
 
     torControlPort = mkOption {
-      type = types.int;
+      type = types.port;
       default = 9051;
       description = lib.mdDoc ''
         Tor control port.
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix
deleted file mode 100644
index 394e6e201f03..000000000000
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unifi-poller.nix
+++ /dev/null
@@ -1,34 +0,0 @@
-{ config, lib, pkgs, options }:
-
-with lib;
-
-let
-  cfg = config.services.prometheus.exporters.unifi-poller;
-
-  configFile = pkgs.writeText "prometheus-unifi-poller-exporter.json" (generators.toJSON {} {
-    poller = { inherit (cfg.log) debug quiet; };
-    unifi = { inherit (cfg) controllers; };
-    influxdb.disable = true;
-    prometheus = {
-      http_listen = "${cfg.listenAddress}:${toString cfg.port}";
-      report_errors = cfg.log.prometheusErrors;
-    };
-  });
-
-in {
-  port = 9130;
-
-  extraOpts = {
-    inherit (options.services.unifi-poller.unifi) controllers;
-    log = {
-      debug = mkEnableOption "debug logging including line numbers, high resolution timestamps, per-device logs.";
-      quiet = mkEnableOption "startup and error logs only.";
-      prometheusErrors = mkEnableOption "emitting errors to prometheus.";
-    };
-  };
-
-  serviceOpts.serviceConfig = {
-    ExecStart = "${pkgs.unifi-poller}/bin/unifi-poller --config ${configFile}";
-    DynamicUser = false;
-  };
-}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unpoller.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unpoller.nix
new file mode 100644
index 000000000000..3b7f978528cd
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/unpoller.nix
@@ -0,0 +1,37 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.unpoller;
+
+  configFile = pkgs.writeText "prometheus-unpoller-exporter.json" (generators.toJSON {} {
+    poller = { inherit (cfg.log) debug quiet; };
+    unifi = { inherit (cfg) controllers; };
+    influxdb.disable = true;
+    datadog.disable = true; # workaround for https://github.com/unpoller/unpoller/issues/442
+    prometheus = {
+      http_listen = "${cfg.listenAddress}:${toString cfg.port}";
+      report_errors = cfg.log.prometheusErrors;
+    };
+    inherit (cfg) loki;
+  });
+
+in {
+  port = 9130;
+
+  extraOpts = {
+    inherit (options.services.unpoller.unifi) controllers;
+    inherit (options.services.unpoller) loki;
+    log = {
+      debug = mkEnableOption (lib.mdDoc "debug logging including line numbers, high resolution timestamps, per-device logs");
+      quiet = mkEnableOption (lib.mdDoc "startup and error logs only");
+      prometheusErrors = mkEnableOption (lib.mdDoc "emitting errors to prometheus");
+    };
+  };
+
+  serviceOpts.serviceConfig = {
+    ExecStart = "${pkgs.unpoller}/bin/unpoller --config ${configFile}";
+    DynamicUser = false;
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/v2ray.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/v2ray.nix
new file mode 100644
index 000000000000..a019157c664b
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/v2ray.nix
@@ -0,0 +1,29 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.v2ray;
+in
+{
+  port = 9299;
+  extraOpts = {
+    v2rayEndpoint = mkOption {
+      type = types.str;
+      default = "127.0.0.1:54321";
+      description = lib.mdDoc ''
+        v2ray grpc api endpoint
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-v2ray-exporter}/bin/v2ray-exporter \
+          --v2ray-endpoint ${cfg.v2rayEndpoint} \
+          --listen ${cfg.listenAddress}:${toString cfg.port} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/wireguard.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/wireguard.nix
index 20631f2af934..c98dcd9f64bf 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/wireguard.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/wireguard.nix
@@ -11,7 +11,7 @@ in {
     ({ options.warnings = options.warnings; options.assertions = options.assertions; })
   ];
   extraOpts = {
-    verbose = mkEnableOption "Verbose logging mode for prometheus-wireguard-exporter";
+    verbose = mkEnableOption (lib.mdDoc "Verbose logging mode for prometheus-wireguard-exporter");
 
     wireguardConfig = mkOption {
       type = with types; nullOr (either path str);
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/zfs.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/zfs.nix
new file mode 100644
index 000000000000..ff12a52d49a9
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/exporters/zfs.nix
@@ -0,0 +1,44 @@
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.zfs;
+in
+{
+  port = 9134;
+
+  extraOpts = {
+    telemetryPath = mkOption {
+      type = types.str;
+      default = "/metrics";
+      description = lib.mdDoc ''
+        Path under which to expose metrics.
+      '';
+    };
+
+    pools = mkOption {
+      type = with types; nullOr (listOf str);
+      default = [ ];
+      description = lib.mdDoc ''
+        Name of the pool(s) to collect, repeat for multiple pools (default: all pools).
+      '';
+    };
+  };
+
+  serviceOpts = {
+    # needs zpool
+    path = [ config.boot.zfs.package ];
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-zfs-exporter}/bin/zfs_exporter \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --web.telemetry-path ${cfg.telemetryPath} \
+          ${concatMapStringsSep " " (x: "--pool=${x}") cfg.pools} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+      ProtectClock = false;
+      PrivateDevices = false;
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/pushgateway.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/pushgateway.nix
index ac7a2300f67c..f5c114c92752 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/pushgateway.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/pushgateway.nix
@@ -21,7 +21,7 @@ let
 in {
   options = {
     services.prometheus.pushgateway = {
-      enable = mkEnableOption "Prometheus Pushgateway";
+      enable = mkEnableOption (lib.mdDoc "Prometheus Pushgateway");
 
       package = mkOption {
         type = types.package;
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/sachet.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/sachet.nix
new file mode 100644
index 000000000000..c908d599bd4e
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/sachet.nix
@@ -0,0 +1,88 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.sachet;
+  configFile = pkgs.writeText "sachet.yml" (builtins.toJSON cfg.configuration);
+in
+{
+  options = {
+    services.prometheus.sachet = {
+      enable = mkEnableOption (lib.mdDoc "Sachet, an SMS alerting tool for the Prometheus Alertmanager");
+
+      configuration = mkOption {
+        type = types.nullOr types.attrs;
+        default = null;
+        example = literalExpression ''
+          {
+            providers = {
+              twilio = {
+                # environment variables gets expanded at runtime
+                account_sid = "$TWILIO_ACCOUNT";
+                auth_token = "$TWILIO_TOKEN";
+              };
+            };
+            templates = [ ./some-template.tmpl ];
+            receivers = [{
+              name = "pager";
+              provider = "twilio";
+              to = [ "+33123456789" ];
+              text = "{{ template \"message\" . }}";
+            }];
+          }
+        '';
+        description = lib.mdDoc ''
+          Sachet's configuration as a nix attribute set.
+        '';
+      };
+
+      address = mkOption {
+        type = types.str;
+        default = "localhost";
+        description = lib.mdDoc ''
+          The address Sachet will listen to.
+        '';
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 9876;
+        description = lib.mdDoc ''
+          The port Sachet will listen to.
+        '';
+      };
+
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = singleton {
+      assertion = cfg.configuration != null;
+      message = "Cannot enable Sachet without a configuration.";
+    };
+
+    systemd.services.sachet = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" "network-online.target" ];
+      script = ''
+        ${pkgs.envsubst}/bin/envsubst -i "${configFile}" > /tmp/sachet.yaml
+        exec ${pkgs.prometheus-sachet}/bin/sachet -config /tmp/sachet.yaml -listen-address ${cfg.address}:${builtins.toString cfg.port}
+      '';
+
+      serviceConfig = {
+        Restart = "always";
+
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectControlGroups = true;
+
+        DynamicUser = true;
+        PrivateTmp = true;
+        WorkingDirectory = "/tmp/";
+      };
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/prometheus/xmpp-alerts.nix b/nixpkgs/nixos/modules/services/monitoring/prometheus/xmpp-alerts.nix
index 1d7da7ced3fe..4545ca37d278 100644
--- a/nixpkgs/nixos/modules/services/monitoring/prometheus/xmpp-alerts.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/prometheus/xmpp-alerts.nix
@@ -15,7 +15,7 @@ in
   ];
 
   options.services.prometheus.xmpp-alerts = {
-    enable = mkEnableOption "XMPP Web hook service for Alertmanager";
+    enable = mkEnableOption (lib.mdDoc "XMPP Web hook service for Alertmanager");
 
     settings = mkOption {
       type = settingsFormat.type;
diff --git a/nixpkgs/nixos/modules/services/monitoring/riemann-tools.nix b/nixpkgs/nixos/modules/services/monitoring/riemann-tools.nix
index b5cd79c74301..28821267b4f3 100644
--- a/nixpkgs/nixos/modules/services/monitoring/riemann-tools.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/riemann-tools.nix
@@ -37,7 +37,7 @@ in {
       extraArgs = mkOption {
         type = types.listOf types.str;
         default = [];
-        description = ''
+        description = lib.mdDoc ''
           A list of commandline-switches forwarded to a riemann-tool.
           See for example `riemann-health --help` for available options.
         '';
diff --git a/nixpkgs/nixos/modules/services/monitoring/riemann.nix b/nixpkgs/nixos/modules/services/monitoring/riemann.nix
index 8d61ec2a308f..7ab8af85ed79 100644
--- a/nixpkgs/nixos/modules/services/monitoring/riemann.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/riemann.nix
@@ -27,13 +27,8 @@ in {
   options = {
 
     services.riemann = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = lib.mdDoc ''
-          Enable the Riemann network monitoring daemon.
-        '';
-      };
+      enable = mkEnableOption (lib.mdDoc "Riemann network monitoring daemon");
+
       config = mkOption {
         type = types.lines;
         description = lib.mdDoc ''
diff --git a/nixpkgs/nixos/modules/services/monitoring/smartd.nix b/nixpkgs/nixos/modules/services/monitoring/smartd.nix
index 83791631d2ca..1e654cad5dd2 100644
--- a/nixpkgs/nixos/modules/services/monitoring/smartd.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/smartd.nix
@@ -4,8 +4,7 @@ with lib;
 
 let
 
-  host = config.networking.hostName or "unknown"
-       + optionalString (config.networking.domain != null) ".${config.networking.domain}";
+  host = config.networking.fqdnOrHostName;
 
   cfg = config.services.smartd;
   opt = options.services.smartd;
@@ -95,7 +94,7 @@ in
 
     services.smartd = {
 
-      enable = mkEnableOption "smartd daemon from <literal>smartmontools</literal> package";
+      enable = mkEnableOption (lib.mdDoc "smartd daemon from `smartmontools` package");
 
       autodetect = mkOption {
         default = true;
@@ -135,9 +134,9 @@ in
             default = "root";
             example = "example@domain.tld";
             type = types.str;
-            description = ''
+            description = lib.mdDoc ''
               Sender of the notification messages.
-              Acts as the value of <literal>email</literal> in the emails' <literal>From: ... </literal> field.
+              Acts as the value of `email` in the emails' `From: ...` field.
             '';
           };
 
diff --git a/nixpkgs/nixos/modules/services/monitoring/statsd.nix b/nixpkgs/nixos/modules/services/monitoring/statsd.nix
index d109e0826204..bbc1c7146a84 100644
--- a/nixpkgs/nixos/modules/services/monitoring/statsd.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/statsd.nix
@@ -56,7 +56,7 @@ in
 
   options.services.statsd = {
 
-    enable = mkEnableOption "statsd";
+    enable = mkEnableOption (lib.mdDoc "statsd");
 
     listenAddress = mkOption {
       description = lib.mdDoc "Address that statsd listens on over UDP";
diff --git a/nixpkgs/nixos/modules/services/monitoring/sysstat.nix b/nixpkgs/nixos/modules/services/monitoring/sysstat.nix
index f8621f08bb8c..5468fc3aa454 100644
--- a/nixpkgs/nixos/modules/services/monitoring/sysstat.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/sysstat.nix
@@ -5,7 +5,7 @@ let
 in {
   options = {
     services.sysstat = {
-      enable = mkEnableOption "sar system activity collection";
+      enable = mkEnableOption (lib.mdDoc "sar system activity collection");
 
       collect-frequency = mkOption {
         type = types.str;
diff --git a/nixpkgs/nixos/modules/services/monitoring/teamviewer.nix b/nixpkgs/nixos/modules/services/monitoring/teamviewer.nix
index e2271e571c40..9b1278317943 100644
--- a/nixpkgs/nixos/modules/services/monitoring/teamviewer.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/teamviewer.nix
@@ -14,7 +14,7 @@ in
 
   options = {
 
-    services.teamviewer.enable = mkEnableOption "TeamViewer daemon";
+    services.teamviewer.enable = mkEnableOption (lib.mdDoc "TeamViewer daemon");
 
   };
 
@@ -30,7 +30,7 @@ in
       description = "TeamViewer remote control daemon";
 
       wantedBy = [ "multi-user.target" ];
-      after = [ "NetworkManager-wait-online.service" "network.target" "dbus.service" ];
+      after = [ "network-online.target" "network.target" "dbus.service" ];
       requires = [ "dbus.service" ];
       preStart = "mkdir -pv /var/lib/teamviewer /var/log/teamviewer";
 
diff --git a/nixpkgs/nixos/modules/services/monitoring/telegraf.nix b/nixpkgs/nixos/modules/services/monitoring/telegraf.nix
index d228b5cc2d0a..913e599c189a 100644
--- a/nixpkgs/nixos/modules/services/monitoring/telegraf.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/telegraf.nix
@@ -11,7 +11,7 @@ in {
   ###### interface
   options = {
     services.telegraf = {
-      enable = mkEnableOption "telegraf server";
+      enable = mkEnableOption (lib.mdDoc "telegraf server");
 
       package = mkOption {
         default = pkgs.telegraf;
diff --git a/nixpkgs/nixos/modules/services/monitoring/thanos.nix b/nixpkgs/nixos/modules/services/monitoring/thanos.nix
index c7404241fbf5..e6d8afc66624 100644
--- a/nixpkgs/nixos/modules/services/monitoring/thanos.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/thanos.nix
@@ -8,7 +8,7 @@ let
   nullOpt = type: description: mkOption {
     type = types.nullOr type;
     default = null;
-    inherit description;
+    description = lib.mdDoc description;
   };
 
   optionToArgs = opt: v  : optional (v != null)  ''--${opt}="${toString v}"'';
@@ -18,8 +18,8 @@ let
 
   mkParamDef = type: default: description: mkParam type (description + ''
 
-    Defaults to <literal>${toString default}</literal> in Thanos
-    when set to <literal>null</literal>.
+    Defaults to `${toString default}` in Thanos
+    when set to `null`.
   '');
 
   mkParam = type: description: {
@@ -32,7 +32,7 @@ let
     option = mkOption {
       type = types.bool;
       default = false;
-      inherit description;
+      description = lib.mdDoc description;
     };
   };
 
@@ -41,7 +41,7 @@ let
     option = mkOption {
       type = types.listOf types.str;
       default = [];
-      inherit description;
+      description = lib.mdDoc description;
     };
   };
 
@@ -50,7 +50,7 @@ let
     option = mkOption {
       type = types.attrsOf types.str;
       default = {};
-      inherit description;
+      description = lib.mdDoc description;
     };
   };
 
@@ -59,7 +59,7 @@ let
     option = mkOption {
       type = types.str;
       inherit default;
-      inherit description;
+      description = lib.mdDoc description;
     };
   };
 
@@ -83,8 +83,8 @@ let
   mkArgumentsOption = cmd: mkOption {
     type = types.listOf types.str;
     default = argumentsOf cmd;
-    defaultText = literalDocBook ''
-      calculated from <literal>config.services.thanos.${cmd}</literal>
+    defaultText = literalMD ''
+      calculated from `config.services.thanos.${cmd}`
     '';
     description = lib.mdDoc ''
       Arguments to the `thanos ${cmd}` command.
@@ -141,13 +141,13 @@ let
           option = nullOpt types.attrs ''
             Tracing configuration.
 
-            When not <literal>null</literal> the attribute set gets converted to
+            When not `null` the attribute set gets converted to
             a YAML file and stored in the Nix store. The option
-            <option>tracing.config-file</option> will default to its path.
+            {option}`tracing.config-file` will default to its path.
 
-            If <option>tracing.config-file</option> is set this option has no effect.
+            If {option}`tracing.config-file` is set this option has no effect.
 
-            See format details: <link xlink:href="https://thanos.io/tracing.md/#configuration"/>
+            See format details: <https://thanos.io/tracing.md/#configuration>
           '';
         };
     };
@@ -155,11 +155,11 @@ let
     common = cfg: params.log // params.tracing cfg // {
 
       http-address = mkParamDef types.str "0.0.0.0:10902" ''
-        Listen <literal>host:port</literal> for HTTP endpoints.
+        Listen `host:port` for HTTP endpoints.
       '';
 
       grpc-address = mkParamDef types.str "0.0.0.0:10901" ''
-        Listen <literal>ip:port</literal> address for gRPC endpoints (StoreAPI).
+        Listen `ip:port` address for gRPC endpoints (StoreAPI).
 
         Make sure this address is routable from other components.
       '';
@@ -206,13 +206,13 @@ let
           option = nullOpt types.attrs ''
             Object store configuration.
 
-            When not <literal>null</literal> the attribute set gets converted to
+            When not `null` the attribute set gets converted to
             a YAML file and stored in the Nix store. The option
-            <option>objstore.config-file</option> will default to its path.
+            {option}`objstore.config-file` will default to its path.
 
-            If <option>objstore.config-file</option> is set this option has no effect.
+            If {option}`objstore.config-file` is set this option has no effect.
 
-            See format details: <link xlink:href="https://thanos.io/storage.md/#configuration"/>
+            See format details: <https://thanos.io/storage.md/#configuration>
           '';
         };
     };
@@ -254,7 +254,7 @@ let
     store = params.common cfg.store // params.objstore cfg.store // {
 
       stateDir = mkStateDirParam "data-dir" "thanos-store" ''
-        Data directory relative to <literal>/var/lib</literal>
+        Data directory relative to `/var/lib`
         in which to cache remote blocks.
       '';
 
@@ -269,7 +269,7 @@ let
       store.grpc.series-sample-limit = mkParamDef types.int 0 ''
         Maximum amount of samples returned via a single Series call.
 
-        <literal>0</literal> means no limit.
+        `0` means no limit.
 
         NOTE: for efficiency we take 120 as the number of samples in chunk (it
         cannot be bigger than that), so the actual number of samples might be
@@ -300,7 +300,7 @@ let
       max-time = mkParamDef types.str "9999-12-31T23:59:59Z" ''
         End of time range limit to serve.
 
-        Thanos Store serves only blocks, which happened eariler than this
+        Thanos Store serves only blocks, which happened earlier than this
         value. Option can be a constant time in RFC3339 format or time duration
         relative to current time, such as -1d or 2h45m. Valid duration units are
         ms, s, m, h, d, w, y.
@@ -327,14 +327,14 @@ let
 
       grpc-client-server-name = mkParam types.str ''
         Server name to verify the hostname on the returned gRPC certificates.
-        See <link xlink:href="https://tools.ietf.org/html/rfc4366#section-3.1"/>
+        See <https://tools.ietf.org/html/rfc4366#section-3.1>
       '';
 
       web.route-prefix = mkParam types.str ''
         Prefix for API and UI endpoints.
 
         This allows thanos UI to be served on a sub-path. This option is
-        analogous to <option>web.route-prefix</option> of Promethus.
+        analogous to {option}`web.route-prefix` of Promethus.
       '';
 
       web.external-prefix = mkParam types.str ''
@@ -342,7 +342,7 @@ let
         interface.
 
         Actual endpoints are still served on / or the
-        <option>web.route-prefix</option>. This allows thanos UI to be served
+        {option}`web.route-prefix`. This allows thanos UI to be served
         behind a reverse proxy that strips a URL sub-path.
       '';
 
@@ -351,15 +351,15 @@ let
         redirects.
 
         This option is ignored if the option
-        <literal>web.external-prefix</literal> is set.
+        `web.external-prefix` is set.
 
         Security risk: enable this option only if a reverse proxy in front of
         thanos is resetting the header.
 
-        The setting <literal>web.prefix-header="X-Forwarded-Prefix"</literal>
+        The setting `web.prefix-header="X-Forwarded-Prefix"`
         can be useful, for example, if Thanos UI is served via Traefik reverse
-        proxy with <literal>PathPrefixStrip</literal> option enabled, which
-        sends the stripped prefix value in <literal>X-Forwarded-Prefix</literal>
+        proxy with `PathPrefixStrip` option enabled, which
+        sends the stripped prefix value in `X-Forwarded-Prefix`
         header. This allows thanos UI to be served on a sub-path.
       '';
 
@@ -376,7 +376,7 @@ let
         deduplicated.
 
         Still you will be able to query without deduplication using
-        <literal>dedup=false</literal> parameter.
+        `dedup=false` parameter.
       '';
 
       selector-labels = mkAttrsParam "selector-label" ''
@@ -386,8 +386,8 @@ let
       store.addresses = mkListParam "store" ''
         Addresses of statically configured store API servers.
 
-        The scheme may be prefixed with <literal>dns+</literal> or
-        <literal>dnssrv+</literal> to detect store API servers through
+        The scheme may be prefixed with `dns+` or
+        `dnssrv+` to detect store API servers through
         respective DNS lookups.
       '';
 
@@ -411,12 +411,12 @@ let
       query.auto-downsampling = mkFlagParam ''
         Enable automatic adjustment (step / 5) to what source of data should
         be used in store gateways if no
-        <literal>max_source_resolution</literal> param is specified.
+        `max_source_resolution` param is specified.
       '';
 
       query.partial-response = mkFlagParam ''
         Enable partial response for queries if no
-        <literal>partial_response</literal> param is specified.
+        `partial_response` param is specified.
       '';
 
       query.default-evaluation-interval = mkParamDef types.str "1m" ''
@@ -426,7 +426,7 @@ let
       store.response-timeout = mkParamDef types.str "0ms" ''
         If a Store doesn't send any data in this specified duration then a
         Store will be ignored and partial data will be returned if it's
-        enabled. <literal>0</literal> disables timeout.
+        enabled. `0` disables timeout.
       '';
     };
 
@@ -440,7 +440,7 @@ let
       '';
 
       stateDir = mkStateDirParam "data-dir" "thanos-rule" ''
-        Data directory relative to <literal>/var/lib</literal>.
+        Data directory relative to `/var/lib`.
       '';
 
       rule-files = mkListParam "rule-file" ''
@@ -464,9 +464,9 @@ let
 
         Ruler claims success if push to at least one alertmanager from
         discovered succeeds. The scheme may be prefixed with
-        <literal>dns+</literal> or <literal>dnssrv+</literal> to detect
+        `dns+` or `dnssrv+` to detect
         Alertmanager IPs through respective DNS lookups. The port defaults to
-        <literal>9093</literal> or the SRV record's value. The URL path is
+        `9093` or the SRV record's value. The URL path is
         used as a prefix for the regular Alertmanager API path.
       '';
 
@@ -491,7 +491,7 @@ let
 
         This allows thanos UI to be served on a sub-path.
 
-        This option is analogous to <literal>--web.route-prefix</literal> of Promethus.
+        This option is analogous to `--web.route-prefix` of Promethus.
       '';
 
       web.external-prefix = mkParam types.str ''
@@ -499,7 +499,7 @@ let
         interface.
 
         Actual endpoints are still served on / or the
-        <option>web.route-prefix</option>. This allows thanos UI to be served
+        {option}`web.route-prefix`. This allows thanos UI to be served
         behind a reverse proxy that strips a URL sub-path.
       '';
 
@@ -508,23 +508,23 @@ let
         redirects.
 
         This option is ignored if the option
-        <option>web.external-prefix</option> is set.
+        {option}`web.external-prefix` is set.
 
         Security risk: enable this option only if a reverse proxy in front of
         thanos is resetting the header.
 
-        The header <literal>X-Forwarded-Prefix</literal> can be useful, for
+        The header `X-Forwarded-Prefix` can be useful, for
         example, if Thanos UI is served via Traefik reverse proxy with
-        <literal>PathPrefixStrip</literal> option enabled, which sends the
-        stripped prefix value in <literal>X-Forwarded-Prefix</literal>
+        `PathPrefixStrip` option enabled, which sends the
+        stripped prefix value in `X-Forwarded-Prefix`
         header. This allows thanos UI to be served on a sub-path.
       '';
 
       query.addresses = mkListParam "query" ''
         Addresses of statically configured query API servers.
 
-        The scheme may be prefixed with <literal>dns+</literal> or
-        <literal>dnssrv+</literal> to detect query API servers through
+        The scheme may be prefixed with `dns+` or
+        `dnssrv+` to detect query API servers through
         respective DNS lookups.
       '';
 
@@ -545,11 +545,11 @@ let
     compact = params.log // params.tracing cfg.compact // params.objstore cfg.compact // {
 
       http-address = mkParamDef types.str "0.0.0.0:10902" ''
-        Listen <literal>host:port</literal> for HTTP endpoints.
+        Listen `host:port` for HTTP endpoints.
       '';
 
       stateDir = mkStateDirParam "data-dir" "thanos-compact" ''
-        Data directory relative to <literal>/var/lib</literal>
+        Data directory relative to `/var/lib`
         in which to cache blocks and process compactions.
       '';
 
@@ -562,28 +562,28 @@ let
       retention.resolution-raw = mkParamDef types.str "0d" ''
         How long to retain raw samples in bucket.
 
-        <literal>0d</literal> - disables this retention
+        `0d` - disables this retention
       '';
 
       retention.resolution-5m = mkParamDef types.str "0d" ''
         How long to retain samples of resolution 1 (5 minutes) in bucket.
 
-        <literal>0d</literal> - disables this retention
+        `0d` - disables this retention
       '';
 
       retention.resolution-1h = mkParamDef types.str "0d" ''
         How long to retain samples of resolution 2 (1 hour) in bucket.
 
-        <literal>0d</literal> - disables this retention
+        `0d` - disables this retention
       '';
 
       startAt = {
         toArgs = _opt: startAt: flagToArgs "wait" (startAt == null);
         option = nullOpt types.str ''
-          When this option is set to a <literal>systemd.time</literal>
+          When this option is set to a `systemd.time`
           specification the Thanos compactor will run at the specified period.
 
-          When this option is <literal>null</literal> the Thanos compactor service
+          When this option is `null` the Thanos compactor service
           will run continuously. So it will not exit after all compactions have
           been processed but wait for new work.
         '';
@@ -609,7 +609,7 @@ let
     downsample = params.log // params.tracing cfg.downsample // params.objstore cfg.downsample // {
 
       stateDir = mkStateDirParam "data-dir" "thanos-downsample" ''
-        Data directory relative to <literal>/var/lib</literal>
+        Data directory relative to `/var/lib`
         in which to cache blocks and process downsamplings.
       '';
 
@@ -622,7 +622,7 @@ let
       '';
 
       stateDir = mkStateDirParam "tsdb.path" "thanos-receive" ''
-        Data directory relative to <literal>/var/lib</literal> of TSDB.
+        Data directory relative to `/var/lib` of TSDB.
       '';
 
       labels = mkAttrsParam "labels" ''
@@ -635,7 +635,7 @@ let
       tsdb.retention = mkParamDef types.str "15d" ''
         How long to retain raw samples on local storage.
 
-        <literal>0d</literal> - disables this retention
+        `0d` - disables this retention
       '';
     };
 
@@ -667,46 +667,46 @@ in {
 
     sidecar = paramsToOptions params.sidecar // {
       enable = mkEnableOption
-        "the Thanos sidecar for Prometheus server";
+        (lib.mdDoc "the Thanos sidecar for Prometheus server");
       arguments = mkArgumentsOption "sidecar";
     };
 
     store = paramsToOptions params.store // {
       enable = mkEnableOption
-        "the Thanos store node giving access to blocks in a bucket provider.";
+        (lib.mdDoc "the Thanos store node giving access to blocks in a bucket provider.");
       arguments = mkArgumentsOption "store";
     };
 
     query = paramsToOptions params.query // {
       enable = mkEnableOption
-        ("the Thanos query node exposing PromQL enabled Query API " +
-         "with data retrieved from multiple store nodes");
+        (lib.mdDoc ("the Thanos query node exposing PromQL enabled Query API " +
+         "with data retrieved from multiple store nodes"));
       arguments = mkArgumentsOption "query";
     };
 
     rule = paramsToOptions params.rule // {
       enable = mkEnableOption
-        ("the Thanos ruler service which evaluates Prometheus rules against" +
-        " given Query nodes, exposing Store API and storing old blocks in bucket");
+        (lib.mdDoc ("the Thanos ruler service which evaluates Prometheus rules against" +
+        " given Query nodes, exposing Store API and storing old blocks in bucket"));
       arguments = mkArgumentsOption "rule";
     };
 
     compact = paramsToOptions params.compact // {
       enable = mkEnableOption
-        "the Thanos compactor which continuously compacts blocks in an object store bucket";
+        (lib.mdDoc "the Thanos compactor which continuously compacts blocks in an object store bucket");
       arguments = mkArgumentsOption "compact";
     };
 
     downsample = paramsToOptions params.downsample // {
       enable = mkEnableOption
-        "the Thanos downsampler which continuously downsamples blocks in an object store bucket";
+        (lib.mdDoc "the Thanos downsampler which continuously downsamples blocks in an object store bucket");
       arguments = mkArgumentsOption "downsample";
     };
 
     receive = paramsToOptions params.receive // {
       enable = mkEnableOption
-        ("the Thanos receiver which accept Prometheus remote write API requests " +
-         "and write to local tsdb (EXPERIMENTAL, this may change drastically without notice)");
+        (lib.mdDoc ("the Thanos receiver which accept Prometheus remote write API requests " +
+         "and write to local tsdb (EXPERIMENTAL, this may change drastically without notice)"));
       arguments = mkArgumentsOption "receive";
     };
   };
diff --git a/nixpkgs/nixos/modules/services/monitoring/tremor-rs.nix b/nixpkgs/nixos/modules/services/monitoring/tremor-rs.nix
new file mode 100644
index 000000000000..213e8a474868
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/tremor-rs.nix
@@ -0,0 +1,129 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+
+  cfg = config.services.tremor-rs;
+
+  loggerSettingsFormat = pkgs.formats.yaml { };
+  loggerConfigFile = loggerSettingsFormat.generate "logger.yaml" cfg.loggerSettings;
+in {
+
+  options = {
+    services.tremor-rs = {
+      enable = lib.mkEnableOption (lib.mdDoc "Tremor event- or stream-processing system");
+
+      troyFileList = mkOption {
+        type = types.listOf types.path;
+        default = [];
+        description = lib.mdDoc "List of troy files to load.";
+      };
+
+      tremorLibDir = mkOption {
+        type = types.path;
+        default = "";
+        description = lib.mdDoc "Directory where to find /lib containing tremor script files";
+      };
+
+      host = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = lib.mdDoc "The host tremor should be listening on";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 9898;
+        description = lib.mdDoc "the port tremor should be listening on";
+      };
+
+      loggerSettings = mkOption {
+        description = lib.mdDoc "Tremor logger configuration";
+        default = {};
+        type = loggerSettingsFormat.type;
+
+        example = {
+          refresh_rate = "30 seconds";
+          appenders.stdout.kind = "console";
+          root = {
+            level = "warn";
+            appenders = [ "stdout" ];
+          };
+          loggers = {
+            tremor_runtime = {
+              level = "debug";
+              appenders = [ "stdout" ];
+              additive = false;
+            };
+            tremor = {
+              level = "debug";
+              appenders = [ "stdout" ];
+              additive = false;
+            };
+          };
+        };
+
+        defaultText = literalExpression ''
+          {
+            refresh_rate = "30 seconds";
+            appenders.stdout.kind = "console";
+            root = {
+              level = "warn";
+              appenders = [ "stdout" ];
+            };
+            loggers = {
+              tremor_runtime = {
+                level = "debug";
+                appenders = [ "stdout" ];
+                additive = false;
+              };
+              tremor = {
+                level = "debug";
+                appenders = [ "stdout" ];
+                additive = false;
+              };
+            };
+          }
+        '';
+
+      };
+    };
+  };
+
+  config = mkIf (cfg.enable) {
+
+    environment.systemPackages = [ pkgs.tremor-rs ] ;
+
+    systemd.services.tremor-rs = {
+      description = "Tremor event- or stream-processing system";
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+
+      environment.TREMOR_PATH = "${pkgs.tremor-rs}/lib:${cfg.tremorLibDir}";
+
+      serviceConfig = {
+        ExecStart = "${pkgs.tremor-rs}/bin/tremor --logger-config ${loggerConfigFile} server run ${concatStringsSep " " cfg.troyFileList} --api-host ${cfg.host}:${toString cfg.port}";
+        DynamicUser = true;
+        Restart = "always";
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        SystemCallFilter = [ "@system-service" "~@privileged" ];
+      };
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/tuptime.nix b/nixpkgs/nixos/modules/services/monitoring/tuptime.nix
index 770fbee2a844..97cc37526254 100644
--- a/nixpkgs/nixos/modules/services/monitoring/tuptime.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/tuptime.nix
@@ -10,7 +10,7 @@ in {
 
   options.services.tuptime = {
 
-    enable = mkEnableOption "the total uptime service";
+    enable = mkEnableOption (lib.mdDoc "the total uptime service");
 
     timer = {
       enable = mkOption {
@@ -54,8 +54,8 @@ in {
             Type = "oneshot";
             User = "_tuptime";
             RemainAfterExit = true;
-            ExecStart = "${pkgs.tuptime}/bin/tuptime -x";
-            ExecStop = "${pkgs.tuptime}/bin/tuptime -xg";
+            ExecStart = "${pkgs.tuptime}/bin/tuptime -q";
+            ExecStop = "${pkgs.tuptime}/bin/tuptime -qg";
           };
         };
 
@@ -64,7 +64,7 @@ in {
           serviceConfig = {
             Type = "oneshot";
             User = "_tuptime";
-            ExecStart = "${pkgs.tuptime}/bin/tuptime -x";
+            ExecStart = "${pkgs.tuptime}/bin/tuptime -q";
           };
         };
       };
diff --git a/nixpkgs/nixos/modules/services/monitoring/unifi-poller.nix b/nixpkgs/nixos/modules/services/monitoring/unpoller.nix
index a955bf4907fc..557e2bff4c26 100644
--- a/nixpkgs/nixos/modules/services/monitoring/unifi-poller.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/unpoller.nix
@@ -3,15 +3,19 @@
 with lib;
 
 let
-  cfg = config.services.unifi-poller;
+  cfg = config.services.unpoller;
 
-  configFile = pkgs.writeText "unifi-poller.json" (generators.toJSON {} {
+  configFile = pkgs.writeText "unpoller.json" (generators.toJSON {} {
     inherit (cfg) poller influxdb loki prometheus unifi;
   });
 
 in {
-  options.services.unifi-poller = {
-    enable = mkEnableOption "unifi-poller";
+  imports = [
+    (lib.mkRenamedOptionModule [ "services" "unifi-poller" ] [ "services" "unpoller" ])
+  ];
+
+  options.services.unpoller = {
+    enable = mkEnableOption (lib.mdDoc "unpoller");
 
     poller = {
       debug = mkOption {
@@ -43,7 +47,7 @@ in {
         type = types.bool;
         default = false;
         description = lib.mdDoc ''
-          Whether to disable the prometheus ouput plugin.
+          Whether to disable the prometheus output plugin.
         '';
       };
       http_listen = mkOption {
@@ -67,7 +71,7 @@ in {
         type = types.bool;
         default = false;
         description = lib.mdDoc ''
-          Whether to disable the influxdb ouput plugin.
+          Whether to disable the influxdb output plugin.
         '';
       };
       url = mkOption {
@@ -86,8 +90,8 @@ in {
       };
       pass = mkOption {
         type = types.path;
-        default = pkgs.writeText "unifi-poller-influxdb-default.password" "unifipoller";
-        defaultText = literalExpression "unifi-poller-influxdb-default.password";
+        default = pkgs.writeText "unpoller-influxdb-default.password" "unifipoller";
+        defaultText = literalExpression "unpoller-influxdb-default.password";
         description = lib.mdDoc ''
           Path of a file containing the password for influxdb.
           This file needs to be readable by the unifi-poller user.
@@ -135,8 +139,8 @@ in {
       };
       pass = mkOption {
         type = types.path;
-        default = pkgs.writeText "unifi-poller-loki-default.password" "";
-        defaultText = "unifi-poller-influxdb-default.password";
+        default = pkgs.writeText "unpoller-loki-default.password" "";
+        defaultText = "unpoller-influxdb-default.password";
         description = lib.mdDoc ''
           Path of a file containing the password for Loki.
           This file needs to be readable by the unifi-poller user.
@@ -184,8 +188,8 @@ in {
         };
         pass = mkOption {
           type = types.path;
-          default = pkgs.writeText "unifi-poller-unifi-default.password" "unifi";
-          defaultText = literalExpression "unifi-poller-unifi-default.password";
+          default = pkgs.writeText "unpoller-unifi-default.password" "unifi";
+          defaultText = literalExpression "unpoller-unifi-default.password";
           description = lib.mdDoc ''
             Path of a file containing the password for the unifi service user.
             This file needs to be readable by the unifi-poller user.
@@ -303,7 +307,7 @@ in {
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
       serviceConfig = {
-        ExecStart = "${pkgs.unifi-poller}/bin/unifi-poller --config ${configFile}";
+        ExecStart = "${pkgs.unpoller}/bin/unpoller --config ${configFile}";
         Restart = "always";
         PrivateTmp = true;
         ProtectHome = true;
diff --git a/nixpkgs/nixos/modules/services/monitoring/ups.nix b/nixpkgs/nixos/modules/services/monitoring/ups.nix
index 8af2c2a1f253..bb11b6a1c1d0 100644
--- a/nixpkgs/nixos/modules/services/monitoring/ups.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/ups.nix
@@ -12,7 +12,7 @@ let
   upsOptions = {name, config, ...}:
   {
     options = {
-      # This can be infered from the UPS model by looking at
+      # This can be inferred from the UPS model by looking at
       # /nix/store/nut/share/driver.list
       driver = mkOption {
         type = types.str;
@@ -116,7 +116,7 @@ in
       mode = mkOption {
         default = "standalone";
         type = types.str;
-        description = ''
+        description = lib.mdDoc ''
           The MODE determines which part of the NUT is to be started, and
           which configuration files must be modified.
 
@@ -228,7 +228,7 @@ in
           "}
         '';
       "nut/upssched.conf".source = cfg.schedulerRules;
-      # These file are containing private informations and thus should not
+      # These file are containing private information and thus should not
       # be stored inside the Nix store.
       /*
       "nut/upsd.conf".source = "";
diff --git a/nixpkgs/nixos/modules/services/monitoring/uptime-kuma.nix b/nixpkgs/nixos/modules/services/monitoring/uptime-kuma.nix
new file mode 100644
index 000000000000..7027046b2425
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/uptime-kuma.nix
@@ -0,0 +1,81 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.uptime-kuma;
+in
+{
+
+  meta.maintainers = [ lib.maintainers.julienmalka ];
+
+  options = {
+    services.uptime-kuma = {
+      enable = mkEnableOption (mdDoc "Uptime Kuma, this assumes a reverse proxy to be set");
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.uptime-kuma;
+        defaultText = literalExpression "pkgs.uptime-kuma";
+        description = lib.mdDoc "Uptime Kuma package to use.";
+      };
+
+      appriseSupport = mkEnableOption (mdDoc "apprise support for notifications");
+
+      settings = lib.mkOption {
+        type = lib.types.submodule { freeformType = with lib.types; attrsOf str; };
+        default = { };
+        example = {
+          PORT = "4000";
+          NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/ca-certificates.crt";
+        };
+        description = lib.mdDoc ''
+          Additional configuration for Uptime Kuma, see
+          <https://github.com/louislam/uptime-kuma/wiki/Environment-Variables>
+          for supported values.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    services.uptime-kuma.settings = {
+      DATA_DIR = "/var/lib/uptime-kuma/";
+      NODE_ENV = mkDefault "production";
+      HOST = mkDefault "127.0.0.1";
+      PORT = mkDefault "3001";
+    };
+
+    systemd.services.uptime-kuma = {
+      description = "Uptime Kuma";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      environment = cfg.settings;
+      path = with pkgs; [ unixtools.ping ] ++ lib.optional cfg.appriseSupport apprise;
+      serviceConfig = {
+        Type = "simple";
+        StateDirectory = "uptime-kuma";
+        DynamicUser = true;
+        ExecStart = "${cfg.package}/bin/uptime-kuma-server";
+        Restart = "on-failure";
+        ProtectHome = true;
+        ProtectSystem = "strict";
+        PrivateTmp = true;
+        PrivateDevices = true;
+        ProtectHostname = true;
+        ProtectClock = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+        NoNewPrivileges = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        RemoveIPC = true;
+        PrivateMounts = true;
+      };
+    };
+  };
+}
+
diff --git a/nixpkgs/nixos/modules/services/monitoring/uptime.nix b/nixpkgs/nixos/modules/services/monitoring/uptime.nix
index 24ca7c3763f4..7bf9e593c95e 100644
--- a/nixpkgs/nixos/modules/services/monitoring/uptime.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/uptime.nix
@@ -51,9 +51,9 @@ in {
       type = types.bool;
     };
 
-    enableWebService = mkEnableOption "the uptime monitoring program web service";
+    enableWebService = mkEnableOption (lib.mdDoc "the uptime monitoring program web service");
 
-    enableSeparateMonitoringService = mkEnableOption "the uptime monitoring service" // {
+    enableSeparateMonitoringService = mkEnableOption (lib.mdDoc "the uptime monitoring service") // {
       default = cfg.enableWebService;
       defaultText = literalExpression "config.${opt.enableWebService}";
     };
diff --git a/nixpkgs/nixos/modules/services/monitoring/vmagent.nix b/nixpkgs/nixos/modules/services/monitoring/vmagent.nix
new file mode 100644
index 000000000000..c793bb073199
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/vmagent.nix
@@ -0,0 +1,100 @@
+{ config, pkgs, lib, ... }:
+with lib;
+let
+  cfg = config.services.vmagent;
+  settingsFormat = pkgs.formats.json { };
+in {
+  options.services.vmagent = {
+    enable = mkEnableOption (lib.mdDoc "vmagent");
+
+    user = mkOption {
+      default = "vmagent";
+      type = types.str;
+      description = lib.mdDoc ''
+        User account under which vmagent runs.
+      '';
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = "vmagent";
+      description = lib.mdDoc ''
+        Group under which vmagent runs.
+      '';
+    };
+
+    package = mkOption {
+      default = pkgs.vmagent;
+      defaultText = lib.literalMD "pkgs.vmagent";
+      type = types.package;
+      description = lib.mdDoc ''
+        vmagent package to use.
+      '';
+    };
+
+    dataDir = mkOption {
+      type = types.str;
+      default = "/var/lib/vmagent";
+      description = lib.mdDoc ''
+        The directory where vmagent stores its data files.
+      '';
+    };
+
+    remoteWriteUrl = mkOption {
+      default = "http://localhost:8428/api/v1/write";
+      type = types.str;
+      description = lib.mdDoc ''
+        The storage endpoint such as VictoriaMetrics
+      '';
+    };
+
+    prometheusConfig = mkOption {
+      type = lib.types.submodule { freeformType = settingsFormat.type; };
+      description = lib.mdDoc ''
+        Config for prometheus style metrics
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      description = lib.mdDoc ''
+        Whether to open the firewall for the default ports.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.groups = mkIf (cfg.group == "vmagent") { vmagent = { }; };
+
+    users.users = mkIf (cfg.user == "vmagent") {
+      vmagent = {
+        group = cfg.group;
+        description = "vmagent daemon user";
+        home = cfg.dataDir;
+        isSystemUser = true;
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ 8429 ];
+
+    systemd.services.vmagent = let
+      prometheusConfig = settingsFormat.generate "prometheusConfig.yaml" cfg.prometheusConfig;
+    in {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      description = "vmagent system service";
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        Type = "simple";
+        Restart = "on-failure";
+        WorkingDirectory = cfg.dataDir;
+        ExecStart = "${cfg.package}/bin/vmagent -remoteWrite.url=${cfg.remoteWriteUrl} -promscrape.config=${prometheusConfig}";
+      };
+    };
+
+    systemd.tmpfiles.rules =
+      [ "d '${cfg.dataDir}' 0755 ${cfg.user} ${cfg.group} -" ];
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/vmalert.nix b/nixpkgs/nixos/modules/services/monitoring/vmalert.nix
new file mode 100644
index 000000000000..27fb34e199b5
--- /dev/null
+++ b/nixpkgs/nixos/modules/services/monitoring/vmalert.nix
@@ -0,0 +1,136 @@
+{ config, pkgs, lib, ... }: with lib;
+let
+  cfg = config.services.vmalert;
+
+  format = pkgs.formats.yaml {};
+
+  confOpts = concatStringsSep " \\\n" (mapAttrsToList mkLine (filterAttrs (_: v: v != false) cfg.settings));
+  confType = with types;
+    let
+      valueType = oneOf [ bool int path str ];
+    in
+    attrsOf (either valueType (listOf valueType));
+
+  mkLine = key: value:
+    if value == true then "-${key}"
+    else if isList value then concatMapStringsSep " " (v: "-${key}=${escapeShellArg (toString v)}") value
+    else "-${key}=${escapeShellArg (toString value)}"
+  ;
+in
+{
+  # interface
+  options.services.vmalert = {
+    enable = mkEnableOption (mdDoc "vmalert");
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.victoriametrics;
+      defaultText = "pkgs.victoriametrics";
+      description = mdDoc ''
+        The VictoriaMetrics derivation to use.
+      '';
+    };
+
+    settings = mkOption {
+      type = types.submodule {
+        freeformType = confType;
+        options = {
+
+          "datasource.url" = mkOption {
+            type = types.nonEmptyStr;
+            example = "http://localhost:8428";
+            description = mdDoc ''
+              Datasource compatible with Prometheus HTTP API.
+            '';
+          };
+
+          "notifier.url" = mkOption {
+            type = with types; listOf nonEmptyStr;
+            default = [];
+            example = [ "http://127.0.0.1:9093" ];
+            description = mdDoc ''
+              Prometheus Alertmanager URL. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
+            '';
+          };
+
+          "rule" = mkOption {
+            type = with types; listOf path;
+            description = mdDoc ''
+              Path to the files with alerting and/or recording rules.
+
+              ::: {.note}
+              Consider using the {option}`services.vmalert.rules` option as a convenient alternative for declaring rules
+              directly in the `nix` language.
+              :::
+            '';
+          };
+
+        };
+      };
+      default = { };
+      example = {
+        "datasource.url" = "http://localhost:8428";
+        "datasource.disableKeepAlive" = true;
+        "datasource.showURL" = false;
+        "rule" = [
+          "http://<some-server-addr>/path/to/rules"
+          "dir/*.yaml"
+        ];
+      };
+      description = mdDoc ''
+        `vmalert` configuration, passed via command line flags. Refer to
+        <https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#configuration>
+        for details on supported values.
+      '';
+    };
+
+    rules = mkOption {
+      type = format.type;
+      default = {};
+      example = {
+        group = [
+          { name = "TestGroup";
+            rules = [
+              { alert = "ExampleAlertAlwaysFiring";
+                expr = ''
+                  sum by(job)
+                  (up == 1)
+                '';
+              }
+            ];
+          }
+        ];
+      };
+      description = mdDoc ''
+        A list of the given alerting or recording rules against configured `"datasource.url"` compatible with
+        Prometheus HTTP API for `vmalert` to execute. Refer to
+        <https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#rules>
+        for details on supported values.
+      '';
+    };
+  };
+
+  # implementation
+  config = mkIf cfg.enable {
+
+    environment.etc."vmalert/rules.yml".source = format.generate "rules.yml" cfg.rules;
+
+    services.vmalert.settings.rule = [
+      "/etc/vmalert/rules.yml"
+    ];
+
+    systemd.services.vmalert = {
+      description = "vmalert service";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      reloadTriggers = [ config.environment.etc."vmalert/rules.yml".source ];
+
+      serviceConfig = {
+        DynamicUser = true;
+        Restart = "on-failure";
+        ExecStart = "${cfg.package}/bin/vmalert ${confOpts}";
+        ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"'';
+      };
+    };
+  };
+}
diff --git a/nixpkgs/nixos/modules/services/monitoring/vnstat.nix b/nixpkgs/nixos/modules/services/monitoring/vnstat.nix
index 5e19c399568d..a498962ae57e 100644
--- a/nixpkgs/nixos/modules/services/monitoring/vnstat.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/vnstat.nix
@@ -6,7 +6,7 @@ let
   cfg = config.services.vnstat;
 in {
   options.services.vnstat = {
-    enable = mkEnableOption "update of network usage statistics via vnstatd";
+    enable = mkEnableOption (lib.mdDoc "update of network usage statistics via vnstatd");
   };
 
   config = mkIf cfg.enable {
diff --git a/nixpkgs/nixos/modules/services/monitoring/zabbix-agent.nix b/nixpkgs/nixos/modules/services/monitoring/zabbix-agent.nix
index f2a8adace69e..b497ecbcdb6c 100644
--- a/nixpkgs/nixos/modules/services/monitoring/zabbix-agent.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/zabbix-agent.nix
@@ -29,7 +29,7 @@ in
   options = {
 
     services.zabbixAgent = {
-      enable = mkEnableOption "the Zabbix Agent";
+      enable = mkEnableOption (lib.mdDoc "the Zabbix Agent");
 
       package = mkOption {
         type = types.package;
@@ -43,8 +43,8 @@ in
         default = with pkgs; [ nettools ];
         defaultText = literalExpression "with pkgs; [ nettools ]";
         example = literalExpression "with pkgs; [ nettools mysql ]";
-        description = ''
-          Packages to be added to the Zabbix <envar>PATH</envar>.
+        description = lib.mdDoc ''
+          Packages to be added to the Zabbix {env}`PATH`.
           Typically used to add executables for scripts, but can be anything.
         '';
       };
diff --git a/nixpkgs/nixos/modules/services/monitoring/zabbix-proxy.nix b/nixpkgs/nixos/modules/services/monitoring/zabbix-proxy.nix
index 9cfe1bdaa205..85da416ba6c3 100644
--- a/nixpkgs/nixos/modules/services/monitoring/zabbix-proxy.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/zabbix-proxy.nix
@@ -38,7 +38,7 @@ in
   options = {
 
     services.zabbixProxy = {
-      enable = mkEnableOption "the Zabbix Proxy";
+      enable = mkEnableOption (lib.mdDoc "the Zabbix Proxy");
 
       server = mkOption {
         type = types.str;
@@ -61,8 +61,8 @@ in
         type = types.listOf types.package;
         default = with pkgs; [ nettools nmap traceroute ];
         defaultText = literalExpression "[ nettools nmap traceroute ]";
-        description = ''
-          Packages to be added to the Zabbix <envar>PATH</envar>.
+        description = lib.mdDoc ''
+          Packages to be added to the Zabbix {env}`PATH`.
           Typically used to add executables for scripts, but can be anything.
         '';
       };
@@ -102,7 +102,7 @@ in
         };
 
         port = mkOption {
-          type = types.int;
+          type = types.port;
           default = if cfg.database.type == "mysql" then mysql.port else pgsql.port;
           defaultText = literalExpression ''
             if config.${opt.database.type} == "mysql"
diff --git a/nixpkgs/nixos/modules/services/monitoring/zabbix-server.nix b/nixpkgs/nixos/modules/services/monitoring/zabbix-server.nix
index 566ec4ab2f6d..2b50280e3969 100644
--- a/nixpkgs/nixos/modules/services/monitoring/zabbix-server.nix
+++ b/nixpkgs/nixos/modules/services/monitoring/zabbix-server.nix
@@ -40,7 +40,7 @@ in
   options = {
 
     services.zabbixServer = {
-      enable = mkEnableOption "the Zabbix Server";
+      enable = mkEnableOption (lib.mdDoc "the Zabbix Server");
 
       package = mkOption {
         type = types.package;
@@ -53,8 +53,8 @@ in
         type = types.listOf types.package;
         default = with pkgs; [ nettools nmap traceroute ];
         defaultText = literalExpression "[ nettools nmap traceroute ]";
-        description = ''
-          Packages to be added to the Zabbix <envar>PATH</envar>.
+        description = lib.mdDoc ''
+          Packages to be added to the Zabbix {env}`PATH`.
           Typically used to add executables for scripts, but can be anything.
         '';
       };
@@ -94,7 +94,7 @@ in
         };
 
         port = mkOption {
-          type = types.int;
+          type = types.port;
           default = if cfg.database.type == "mysql" then mysql.port else pgsql.port;
           defaultText = literalExpression ''
             if config.${opt.database.type} == "mysql"