Merge branch 'master' into staging-next

13 files changed, 105 insertions, 99 deletions

diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md
index ae196543d266..778ce16fb335 100644
--- a/nixos/doc/manual/release-notes/rl-2311.section.md
+++ b/nixos/doc/manual/release-notes/rl-2311.section.md
@@ -154,6 +154,8 @@
 
 - The latest version of `clonehero` now stores custom content in `~/.clonehero`. See the [migration instructions](https://clonehero.net/2022/11/29/v23-to-v1-migration-instructions.html). Typically, these content files would exist along side the binary, but the previous build used a wrapper script that would store them in `~/.config/unity3d/srylain Inc_/Clone Hero`.
 
+- `services.mastodon` doesn't support providing a TCP port to its `streaming` component anymore, as upstream implemented parallelization by running multiple instances instead of running multiple processes in one instance. Please create a PR if you are interested in this feature.
+
 - The `services.hostapd` module was rewritten to support `passwordFile` like options, WPA3-SAE, and management of multiple interfaces. This breaks compatibility with older configurations.
   - `hostapd` is now started with additional systemd sandbox/hardening options for better security.
   - `services.hostapd.interface` was replaced with a per-radio and per-bss configuration scheme using [services.hostapd.radios](#opt-services.hostapd.radios).
@@ -189,6 +191,8 @@
 
 - JACK tools (`jack_*` except `jack_control`) have moved from the `jack2` package to `jack-example-tools`
 
+- The `waagent` service does provisioning now
+
 - The `matrix-synapse` package & module have undergone some significant internal changes, for most setups no intervention is needed, though:
   - The option [`services.matrix-synapse.package`](#opt-services.matrix-synapse.package) is now read-only. For modifying the package, use an overlay which modifies `matrix-synapse-unwrapped` instead. More on that below.
   - The `enableSystemd` & `enableRedis` arguments have been removed and `matrix-synapse` has been renamed to `matrix-synapse-unwrapped`. Also, several optional dependencies (such as `psycopg2` or `authlib`) have been removed.
diff --git a/nixos/modules/security/acme/default.nix b/nixos/modules/security/acme/default.nix
index 932bf3e79115..7cc302969fb6 100644
--- a/nixos/modules/security/acme/default.nix
+++ b/nixos/modules/security/acme/default.nix
@@ -345,6 +345,10 @@ let
       serviceConfig = commonServiceConfig // {
         Group = data.group;
 
+        # Let's Encrypt Failed Validation Limit allows 5 retries per hour, per account, hostname and hour.
+        # This avoids eating them all up if something is misconfigured upon the first try.
+        RestartSec = 15 * 60;
+
         # Keep in mind that these directories will be deleted if the user runs
         # systemctl clean --what=state
         # acme/.lego/${cert} is listed for this reason.
diff --git a/nixos/modules/services/audio/navidrome.nix b/nixos/modules/services/audio/navidrome.nix
index e18e61eb6d44..77a0e74af9ca 100644
--- a/nixos/modules/services/audio/navidrome.nix
+++ b/nixos/modules/services/audio/navidrome.nix
@@ -28,10 +28,17 @@ in {
         '';
       };
 
+      openFirewall = mkOption {
+        type = types.bool;
+        default = false;
+        description = lib.mdDoc "Whether to open the TCP port in the firewall";
+      };
     };
   };
 
   config = mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [cfg.settings.Port];
+
     systemd.services.navidrome = {
       description = "Navidrome Media Server";
       after = [ "network.target" ];
diff --git a/nixos/modules/services/misc/autofs.nix b/nixos/modules/services/misc/autofs.nix
index 55ab15ff003d..723b67e8bb6b 100644
--- a/nixos/modules/services/misc/autofs.nix
+++ b/nixos/modules/services/misc/autofs.nix
@@ -74,7 +74,7 @@ in
 
   config = mkIf cfg.enable {
 
-    boot.kernelModules = [ "autofs4" ];
+    boot.kernelModules = [ "autofs" ];
 
     systemd.services.autofs =
       { description = "Automounts filesystems on demand";
diff --git a/nixos/modules/services/web-apps/mastodon.nix b/nixos/modules/services/web-apps/mastodon.nix
index 2aab97438b7d..ff261fdefb82 100644
--- a/nixos/modules/services/web-apps/mastodon.nix
+++ b/nixos/modules/services/web-apps/mastodon.nix
@@ -17,9 +17,6 @@ let
     WEB_CONCURRENCY = toString cfg.webProcesses;
     MAX_THREADS = toString cfg.webThreads;
 
-    # mastodon-streaming concurrency.
-    STREAMING_CLUSTER_NUM = toString cfg.streamingProcesses;
-
     DB_USER = cfg.database.user;
 
     REDIS_HOST = cfg.redis.host;
@@ -141,8 +138,44 @@ let
     })
   ) cfg.sidekiqProcesses;
 
+  streamingUnits = builtins.listToAttrs
+      (map (i: {
+        name = "mastodon-streaming-${toString i}";
+        value = {
+          after = [ "network.target" "mastodon-init-dirs.service" ]
+            ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
+            ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
+          requires = [ "mastodon-init-dirs.service" ]
+            ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
+            ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
+          wantedBy = [ "mastodon.target" "mastodon-streaming.target" ];
+          description = "Mastodon streaming ${toString i}";
+          environment = env // { SOCKET = "/run/mastodon-streaming/streaming-${toString i}.socket"; };
+          serviceConfig = {
+            ExecStart = "${cfg.package}/run-streaming.sh";
+            Restart = "always";
+            RestartSec = 20;
+            EnvironmentFile = [ "/var/lib/mastodon/.secrets_env" ] ++ cfg.extraEnvFiles;
+            WorkingDirectory = cfg.package;
+            # Runtime directory and mode
+            RuntimeDirectory = "mastodon-streaming";
+            RuntimeDirectoryMode = "0750";
+            # System Call Filtering
+            SystemCallFilter = [ ("~" + lib.concatStringsSep " " (systemCallsList ++ [ "@memlock" "@resources" ])) "pipe" "pipe2" ];
+          } // cfgService;
+        };
+      })
+      (lib.range 1 cfg.streamingProcesses));
+
 in {
 
+  imports = [
+    (lib.mkRemovedOptionModule
+      [ "services" "mastodon" "streamingPort" ]
+      "Mastodon currently doesn't support streaming via TCP ports. Please open a PR if you need this."
+    )
+  ];
+
   options = {
     services.mastodon = {
       enable = lib.mkEnableOption (lib.mdDoc "Mastodon, a federated social network server");
@@ -191,18 +224,13 @@ in {
         default = "mastodon";
       };
 
-      streamingPort = lib.mkOption {
-        description = lib.mdDoc "TCP port used by the mastodon-streaming service.";
-        type = lib.types.port;
-        default = 55000;
-      };
       streamingProcesses = lib.mkOption {
         description = lib.mdDoc ''
-          Processes used by the mastodon-streaming service.
-          Defaults to the number of CPU cores minus one.
+          Number of processes used by the mastodon-streaming service.
+          Recommended is the amount of your CPU cores minus one.
         '';
-        type = lib.types.nullOr lib.types.int;
-        default = null;
+        type = lib.types.ints.positive;
+        example = 3;
       };
 
       webPort = lib.mkOption {
@@ -603,6 +631,12 @@ in {
       after = [ "network.target" ];
     };
 
+    systemd.targets.mastodon-streaming = {
+      description = "Target for all Mastodon streaming services";
+      wantedBy = [ "multi-user.target" "mastodon.target" ];
+      after = [ "network.target" ];
+    };
+
     systemd.services.mastodon-init-dirs = {
       script = ''
         umask 077
@@ -688,33 +722,6 @@ in {
         ++ lib.optional databaseActuallyCreateLocally "postgresql.service";
     };
 
-    systemd.services.mastodon-streaming = {
-      after = [ "network.target" "mastodon-init-dirs.service" ]
-        ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
-        ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
-      requires = [ "mastodon-init-dirs.service" ]
-        ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
-        ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
-      wantedBy = [ "mastodon.target" ];
-      description = "Mastodon streaming";
-      environment = env // (if cfg.enableUnixSocket
-        then { SOCKET = "/run/mastodon-streaming/streaming.socket"; }
-        else { PORT = toString(cfg.streamingPort); }
-      );
-      serviceConfig = {
-        ExecStart = "${cfg.package}/run-streaming.sh";
-        Restart = "always";
-        RestartSec = 20;
-        EnvironmentFile = [ "/var/lib/mastodon/.secrets_env" ] ++ cfg.extraEnvFiles;
-        WorkingDirectory = cfg.package;
-        # Runtime directory and mode
-        RuntimeDirectory = "mastodon-streaming";
-        RuntimeDirectoryMode = "0750";
-        # System Call Filtering
-        SystemCallFilter = [ ("~" + lib.concatStringsSep " " (systemCallsList ++ [ "@memlock" "@resources" ])) "pipe" "pipe2" ];
-      } // cfgService;
-    };
-
     systemd.services.mastodon-web = {
       after = [ "network.target" "mastodon-init-dirs.service" ]
         ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
@@ -780,10 +787,20 @@ in {
         };
 
         locations."/api/v1/streaming/" = {
-          proxyPass = (if cfg.enableUnixSocket then "http://unix:/run/mastodon-streaming/streaming.socket" else "http://127.0.0.1:${toString(cfg.streamingPort)}/");
+          proxyPass = "http://mastodon-streaming";
           proxyWebsockets = true;
         };
       };
+      upstreams.mastodon-streaming = {
+        extraConfig = ''
+          least_conn;
+        '';
+        servers = builtins.listToAttrs
+          (map (i: {
+            name = "unix:/run/mastodon-streaming/streaming-${toString i}.socket";
+            value = { };
+          }) (lib.range 1 cfg.streamingProcesses));
+      };
     };
 
     services.postfix = lib.mkIf (cfg.smtp.createLocally && cfg.smtp.host == "127.0.0.1") {
@@ -819,7 +836,7 @@ in {
 
     users.groups.${cfg.group}.members = lib.optional cfg.configureNginx config.services.nginx.user;
   }
-  { systemd.services = sidekiqUnits; }
+  { systemd.services = lib.mkMerge [ sidekiqUnits streamingUnits ]; }
   ]);
 
   meta.maintainers = with lib.maintainers; [ happy-river erictapen ];
diff --git a/nixos/modules/system/boot/kernel.nix b/nixos/modules/system/boot/kernel.nix
index 6b07686efcba..a46331ccd431 100644
--- a/nixos/modules/system/boot/kernel.nix
+++ b/nixos/modules/system/boot/kernel.nix
@@ -96,8 +96,8 @@ in
                                         # (required, but can be null if only config changes
                                         # are needed)
 
-          extraStructuredConfig = {     # attrset of extra configuration parameters
-            FOO = lib.kernel.yes;       # (without the CONFIG_ prefix, optional)
+          extraStructuredConfig = {     # attrset of extra configuration parameters without the CONFIG_ prefix
+            FOO = lib.kernel.yes;       # (optional)
           };                            # values should generally be lib.kernel.yes,
                                         # lib.kernel.no or lib.kernel.module
 
@@ -105,8 +105,9 @@ in
             foo = true;                 # (may be checked by other NixOS modules, optional)
           };
 
-          extraConfig = "CONFIG_FOO y"; # extra configuration options in string form
-                                        # (deprecated, use extraStructuredConfig instead, optional)
+          extraConfig = "FOO y";        # extra configuration options in string form without the CONFIG_ prefix
+                                        # (optional, multiple lines allowed to specify multiple options)
+                                        # (deprecated, use extraStructuredConfig instead)
         }
         ```
 
diff --git a/nixos/modules/system/boot/systemd/initrd.nix b/nixos/modules/system/boot/systemd/initrd.nix
index e223451652b2..0e7d59b32075 100644
--- a/nixos/modules/system/boot/systemd/initrd.nix
+++ b/nixos/modules/system/boot/systemd/initrd.nix
@@ -370,7 +370,7 @@ in {
 
     boot.initrd.availableKernelModules = [
       # systemd needs this for some features
-      "autofs4"
+      "autofs"
       # systemd-cryptenroll
     ] ++ lib.optional cfg.enableTpm2 "tpm-tis"
     ++ lib.optional (cfg.enableTpm2 && !(pkgs.stdenv.hostPlatform.isRiscV64 || pkgs.stdenv.hostPlatform.isArmv7)) "tpm-crb";
diff --git a/nixos/modules/virtualisation/azure-agent.nix b/nixos/modules/virtualisation/azure-agent.nix
index a88b78bc9821..e712fac17a46 100644
--- a/nixos/modules/virtualisation/azure-agent.nix
+++ b/nixos/modules/virtualisation/azure-agent.nix
@@ -61,7 +61,7 @@ in
 
         # Which provisioning agent to use. Supported values are "auto" (default), "waagent",
         # "cloud-init", or "disabled".
-        Provisioning.Agent=disabled
+        Provisioning.Agent=auto
 
         # Password authentication for root account will be unavailable.
         Provisioning.DeleteRootPassword=n
@@ -246,7 +246,7 @@ in
         pkgs.bash
 
         # waagent's Microsoft.OSTCExtensions.VMAccessForLinux needs Python 3
-        pkgs.python3
+        pkgs.python39
 
         # waagent's Microsoft.CPlat.Core.RunCommandLinux needs lsof
         pkgs.lsof
@@ -259,5 +259,10 @@ in
       };
     };
 
+    # waagent will generate files under /etc/sudoers.d during provisioning
+    security.sudo.extraConfig = ''
+      #includedir /etc/sudoers.d
+    '';
+
   };
 }
diff --git a/nixos/modules/virtualisation/azure-image.nix b/nixos/modules/virtualisation/azure-image.nix
index 39c6cab5980a..d909680cca1f 100644
--- a/nixos/modules/virtualisation/azure-image.nix
+++ b/nixos/modules/virtualisation/azure-image.nix
@@ -37,42 +37,5 @@ in
       inherit config lib pkgs;
     };
 
-    # Azure metadata is available as a CD-ROM drive.
-    fileSystems."/metadata".device = "/dev/sr0";
-
-    systemd.services.fetch-ssh-keys = {
-      description = "Fetch host keys and authorized_keys for root user";
-
-      wantedBy = [ "sshd.service" "waagent.service" ];
-      before = [ "sshd.service" "waagent.service" ];
-
-      path  = [ pkgs.coreutils ];
-      script =
-        ''
-          eval "$(cat /metadata/CustomData.bin)"
-          if ! [ -z "$ssh_host_ecdsa_key" ]; then
-            echo "downloaded ssh_host_ecdsa_key"
-            echo "$ssh_host_ecdsa_key" > /etc/ssh/ssh_host_ed25519_key
-            chmod 600 /etc/ssh/ssh_host_ed25519_key
-          fi
-
-          if ! [ -z "$ssh_host_ecdsa_key_pub" ]; then
-            echo "downloaded ssh_host_ecdsa_key_pub"
-            echo "$ssh_host_ecdsa_key_pub" > /etc/ssh/ssh_host_ed25519_key.pub
-            chmod 644 /etc/ssh/ssh_host_ed25519_key.pub
-          fi
-
-          if ! [ -z "$ssh_root_auth_key" ]; then
-            echo "downloaded ssh_root_auth_key"
-            mkdir -m 0700 -p /root/.ssh
-            echo "$ssh_root_auth_key" > /root/.ssh/authorized_keys
-            chmod 600 /root/.ssh/authorized_keys
-          fi
-        '';
-      serviceConfig.Type = "oneshot";
-      serviceConfig.RemainAfterExit = true;
-      serviceConfig.StandardError = "journal+console";
-      serviceConfig.StandardOutput = "journal+console";
-    };
   };
 }
diff --git a/nixos/tests/web-apps/mastodon/remote-postgresql.nix b/nixos/tests/web-apps/mastodon/remote-postgresql.nix
index 715477191bfb..6548883db452 100644
--- a/nixos/tests/web-apps/mastodon/remote-postgresql.nix
+++ b/nixos/tests/web-apps/mastodon/remote-postgresql.nix
@@ -16,7 +16,7 @@ in
   meta.maintainers = with pkgs.lib.maintainers; [ erictapen izorkin ];
 
   nodes = {
-    database = {
+    database = { config, ... }: {
       networking = {
         interfaces.eth1 = {
           ipv4.addresses = [
@@ -24,11 +24,13 @@ in
           ];
         };
         extraHosts = hosts;
-        firewall.allowedTCPPorts = [ 5432 ];
+        firewall.allowedTCPPorts = [ config.services.postgresql.port ];
       };
 
       services.postgresql = {
         enable = true;
+        # TODO remove once https://github.com/NixOS/nixpkgs/pull/266270 is resolved.
+        package = pkgs.postgresql_14;
         enableTCPIP = true;
         authentication = ''
           hostnossl mastodon_local mastodon_test 192.168.2.201/32 md5
@@ -41,7 +43,7 @@ in
       };
     };
 
-    nginx = {
+    nginx = { nodes, ... }: {
       networking = {
         interfaces.eth1 = {
           ipv4.addresses = [
@@ -69,18 +71,14 @@ in
             tryFiles = "$uri @proxy";
           };
           locations."@proxy" = {
-            proxyPass = "http://192.168.2.201:55001";
-            proxyWebsockets = true;
-          };
-          locations."/api/v1/streaming/" = {
-            proxyPass = "http://192.168.2.201:55002";
+            proxyPass = "http://192.168.2.201:${toString nodes.server.services.mastodon.webPort}";
             proxyWebsockets = true;
           };
         };
       };
     };
 
-    server = { pkgs, ... }: {
+    server = { config, pkgs, ... }: {
       virtualisation.memorySize = 2048;
 
       environment = {
@@ -98,7 +96,10 @@ in
           ];
         };
         extraHosts = hosts;
-        firewall.allowedTCPPorts = [ 55001 55002 ];
+        firewall.allowedTCPPorts = [
+          config.services.mastodon.webPort
+          config.services.mastodon.sidekiqPort
+        ];
       };
 
       services.mastodon = {
@@ -106,6 +107,7 @@ in
         configureNginx = false;
         localDomain = "mastodon.local";
         enableUnixSocket = false;
+        streamingProcesses = 2;
         database = {
           createLocally = false;
           host = "192.168.2.102";
diff --git a/nixos/tests/web-apps/mastodon/script.nix b/nixos/tests/web-apps/mastodon/script.nix
index a89b4b7480e9..afb7c0e0a0eb 100644
--- a/nixos/tests/web-apps/mastodon/script.nix
+++ b/nixos/tests/web-apps/mastodon/script.nix
@@ -10,9 +10,8 @@
 
   server.wait_for_unit("redis-mastodon.service")
   server.wait_for_unit("mastodon-sidekiq-all.service")
-  server.wait_for_unit("mastodon-streaming.service")
+  server.wait_for_unit("mastodon-streaming.target")
   server.wait_for_unit("mastodon-web.service")
-  server.wait_for_open_port(55000)
   server.wait_for_open_port(55001)
 
   # Check that mastodon-media-auto-remove is scheduled
diff --git a/nixos/tests/web-apps/mastodon/standard.nix b/nixos/tests/web-apps/mastodon/standard.nix
index 14311afea3f7..e5eb30fef597 100644
--- a/nixos/tests/web-apps/mastodon/standard.nix
+++ b/nixos/tests/web-apps/mastodon/standard.nix
@@ -40,11 +40,15 @@ in
         port = 31637;
       };
 
+      # TODO remove once https://github.com/NixOS/nixpkgs/pull/266270 is resolved.
+      services.postgresql.package = pkgs.postgresql_14;
+
       services.mastodon = {
         enable = true;
         configureNginx = true;
         localDomain = "mastodon.local";
         enableUnixSocket = false;
+        streamingProcesses = 2;
         smtp = {
           createLocally = false;
           fromAddress = "mastodon@mastodon.local";
diff --git a/nixos/tests/xmpp/ejabberd.nix b/nixos/tests/xmpp/ejabberd.nix
index 7926fe80de2f..1a807b27b6f6 100644
--- a/nixos/tests/xmpp/ejabberd.nix
+++ b/nixos/tests/xmpp/ejabberd.nix
@@ -1,7 +1,7 @@
 import ../make-test-python.nix ({ pkgs, ... }: {
   name = "ejabberd";
   meta = with pkgs.lib.maintainers; {
-    maintainers = [ ajs124 ];
+    maintainers = [ ];
   };
   nodes = {
     client = { nodes, pkgs, ... }: {