Merge pull request #80447 from Ma27/bump-matrix-synapse

matrix-synapse: 1.9.1 -> 1.11.1

7 files changed, 163 insertions, 62 deletions

diff --git a/nixos/doc/manual/configuration/configuration.xml b/nixos/doc/manual/configuration/configuration.xml
index 5961209bc13a..507d28814ead 100644
--- a/nixos/doc/manual/configuration/configuration.xml
+++ b/nixos/doc/manual/configuration/configuration.xml
@@ -21,7 +21,6 @@
  <xi:include href="xfce.xml" />
  <xi:include href="networking.xml" />
  <xi:include href="linux-kernel.xml" />
- <xi:include href="matrix.xml" />
  <xi:include href="../generated/modules.xml" xpointer="xpointer(//section[@id='modules']/*)" />
  <xi:include href="profiles.xml" />
  <xi:include href="kubernetes.xml" />
diff --git a/nixos/doc/manual/release-notes/rl-2003.xml b/nixos/doc/manual/release-notes/rl-2003.xml
index b629c460f4cd..20f232c9110e 100644
--- a/nixos/doc/manual/release-notes/rl-2003.xml
+++ b/nixos/doc/manual/release-notes/rl-2003.xml
@@ -712,6 +712,55 @@ auth required pam_succeed_if.so uid >= 1000 quiet
      For further reference, please read <link xlink:href="https://github.com/NixOS/nixpkgs/pull/68953">#68953</link> or the corresponding <link xlink:href="https://discourse.nixos.org/t/predictable-network-interface-names-in-initrd/4055">discourse thread</link>.
     </para>
    </listitem>
+   <listitem>
+    <para>
+     The <package>matrix-synapse</package>-package has been updated to
+     <link xlink:href="https://github.com/matrix-org/synapse/releases/tag/v1.11.1">v1.11.1</link>.
+     Due to <link xlink:href="https://github.com/matrix-org/synapse/releases/tag/v1.10.0rc1">stricter requirements</link>
+     for database configuration when using <package>postgresql</package>, the automated database setup
+     of the module has been removed to avoid any further edge-cases.
+    </para>
+    <para>
+     <package>matrix-synapse</package> expects <literal>postgresql</literal>-databases to have the options
+     <literal>LC_COLLATE</literal> and <literal>LC_CTYPE</literal> set to
+     <link xlink:href="https://www.postgresql.org/docs/12/locale.html"><literal>'C'</literal></link> which basically
+     instructs <literal>postgresql</literal> to ignore any locale-based preferences.
+    </para>
+    <para>
+     Depending on your setup, you need to incorporate one of the following changes in your setup to
+     upgrade to 20.03:
+     <itemizedlist>
+      <listitem><para>If you use <literal>sqlite3</literal> you don't need to do anything.</para></listitem>
+      <listitem><para>If you use <literal>postgresql</literal> on a different server, you don't need
+       to change anything as well since this module was never designed to configure remote databases.
+      </para></listitem>
+      <listitem><para>If you use <literal>postgresql</literal> and configured your synapse initially on
+       <literal>19.09</literal> or older, you simply need to enable <package>postgresql</package>-support
+        explicitly:
+<programlisting>{ ... }: {
+  services.matrix-synapse = {
+    <link linkend="opt-services.matrix-synapse.enable">enable</link> = true;
+    /* and all the other config you've defined here */
+  };
+  <link linkend="opt-services.postgresql.enable">services.postgresql.enable</link> = true;
+}</programlisting>
+      </para></listitem>
+      <listitem><para>If you deploy a fresh <package>matrix-synapse</package>, you need to configure
+       the database yourself (e.g. by using the
+       <link linkend="opt-services.postgresql.initialScript">services.postgresql.initialScript</link>
+       option). An example for this can be found in the
+       <link linkend="module-services-matrix">documentation of the Matrix module</link>.
+      </para></listitem>
+      <listitem><para>If you initially deployed your <package>matrix-synapse</package> on
+       <literal>nixos-unstable</literal> <emphasis>after</emphasis> the <literal>19.09</literal>-release,
+       your database is misconfigured due to a regression in NixOS. For now, <package>matrix-synapse</package> will
+       startup with a warning, but it's recommended to reconfigure the database to set the values
+       <literal>LC_COLLATE</literal> and <literal>LC_CTYPE</literal> to
+       <link xlink:href="https://www.postgresql.org/docs/12/locale.html"><literal>'C'</literal></link>.
+      </para></listitem>
+     </itemizedlist>
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 </section>
diff --git a/nixos/modules/services/misc/matrix-synapse.nix b/nixos/modules/services/misc/matrix-synapse.nix
index 750f4a292fb4..d02fa13bb99c 100644
--- a/nixos/modules/services/misc/matrix-synapse.nix
+++ b/nixos/modules/services/misc/matrix-synapse.nix
@@ -111,6 +111,9 @@ app_service_config_files: ${builtins.toJSON cfg.app_service_config_files}
 
 ${cfg.extraConfig}
 '';
+
+  hasLocalPostgresDB = let args = cfg.database_args; in
+    usePostgresql && (!(args ? host) || (elem args.host [ "localhost" "127.0.0.1" "::1" ]));
 in {
   options = {
     services.matrix-synapse = {
@@ -354,13 +357,6 @@ in {
           The database engine name. Can be sqlite or psycopg2.
         '';
       };
-      create_local_database = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Whether to create a local database automatically.
-        '';
-      };
       database_name = mkOption {
         type = types.str;
         default = "matrix-synapse";
@@ -657,6 +653,25 @@ in {
   };
 
   config = mkIf cfg.enable {
+    assertions = [
+      { assertion = hasLocalPostgresDB -> config.services.postgresql.enable;
+        message = ''
+          Cannot deploy matrix-synapse with a configuration for a local postgresql database
+            and a missing postgresql service. Since 20.03 it's mandatory to manually configure the
+            database (please read the thread in https://github.com/NixOS/nixpkgs/pull/80447 for
+            further reference).
+
+            If you
+            - try to deploy a fresh synapse, you need to configure the database yourself. An example
+              for this can be found in <nixpkgs/nixos/tests/matrix-synapse.nix>
+            - update your existing matrix-synapse instance, you simply need to add `services.postgresql.enable = true`
+              to your configuration.
+
+          For further information about this update, please read the release-notes of 20.03 carefully.
+        '';
+      }
+    ];
+
     users.users.matrix-synapse = { 
         group = "matrix-synapse";
         home = cfg.dataDir;
@@ -669,18 +684,9 @@ in {
       gid = config.ids.gids.matrix-synapse;
     };
 
-    services.postgresql = mkIf (usePostgresql && cfg.create_local_database) {
-      enable = mkDefault true;
-      ensureDatabases = [ cfg.database_name ];
-      ensureUsers = [{
-        name = cfg.database_user;
-        ensurePermissions = { "DATABASE \"${cfg.database_name}\"" = "ALL PRIVILEGES"; };
-      }];
-    };
-
     systemd.services.matrix-synapse = {
       description = "Synapse Matrix homeserver";
-      after = [ "network.target" ] ++ lib.optional config.services.postgresql.enable "postgresql.service" ;
+      after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
       wantedBy = [ "multi-user.target" ];
       preStart = ''
         ${cfg.package}/bin/homeserver \
@@ -709,6 +715,12 @@ in {
       The `trusted_third_party_id_servers` option as been removed in `matrix-synapse` v1.4.0
       as the behavior is now obsolete.
     '')
+    (mkRemovedOptionModule [ "services" "matrix-synapse" "create_local_database" ] ''
+      Database configuration must be done manually. An exemplary setup is demonstrated in
+      <nixpkgs/nixos/tests/matrix-synapse.nix>
+    '')
   ];
 
+  meta.doc = ./matrix-synapse.xml;
+
 }
diff --git a/nixos/doc/manual/configuration/matrix.xml b/nixos/modules/services/misc/matrix-synapse.xml
index ef8d5cbda889..053a3b2a563f 100644
--- a/nixos/doc/manual/configuration/matrix.xml
+++ b/nixos/modules/services/misc/matrix-synapse.xml
@@ -40,26 +40,35 @@ let
     in join config.networking.hostName config.networking.domain;
 in {
   networking = {
-    hostName = "myhostname";
-    domain = "example.org";
+    <link linkend="opt-networking.hostName">hostName</link> = "myhostname";
+    <link linkend="opt-networking.domain">domain</link> = "example.org";
   };
-  networking.firewall.allowedTCPPorts = [ 80 443 ];
+  <link linkend="opt-networking.firewall.allowedTCPPorts">networking.firewall.allowedTCPPorts</link> = [ 80 443 ];
+
+  <link linkend="opt-services.postgresql.enable">services.postgresql.enable</link> = true;
+  <link linkend="opt-services.postgresql.initialScript">services.postgresql.initialScript</link> = ''
+    CREATE ROLE "matrix-synapse" WITH LOGIN PASSWORD 'synapse';
+    CREATE DATABASE "matrix-synapse" WITH OWNER "matrix-synapse"
+      TEMPLATE template0
+      LC_COLLATE = "C"
+      LC_CTYPE = "C";
+  '';
 
   services.nginx = {
-    enable = true;
+    <link linkend="opt-services.nginx.enable">enable</link> = true;
     # only recommendedProxySettings and recommendedGzipSettings are strictly required,
     # but the rest make sense as well
-    recommendedTlsSettings = true;
-    recommendedOptimisation = true;
-    recommendedGzipSettings = true;
-    recommendedProxySettings = true;
+    <link linkend="opt-services.nginx.recommendedTlsSettings">recommendedTlsSettings</link> = true;
+    <link linkend="opt-services.nginx.recommendedOptimisation">recommendedOptimisation</link> = true;
+    <link linkend="opt-services.nginx.recommendedGzipSettings">recommendedGzipSettings</link> = true;
+    <link linkend="opt-services.nginx.recommendedProxySettings">recommendedProxySettings</link> = true;
 
-    virtualHosts = {
+    <link linkend="opt-services.nginx.virtualHosts">virtualHosts</link> = {
       # This host section can be placed on a different host than the rest,
       # i.e. to delegate from the host being accessible as ${config.networking.domain}
       # to another host actually running the Matrix homeserver.
       "${config.networking.domain}" = {
-        locations."= /.well-known/matrix/server".extraConfig =
+        <link linkend="opt-services.nginx.virtualHosts._name_.locations._name_.extraConfig">locations."= /.well-known/matrix/server".extraConfig</link> =
           let
             # use 443 instead of the default 8448 port to unite
             # the client-server and server-server port for simplicity
@@ -68,7 +77,7 @@ in {
             add_header Content-Type application/json;
             return 200 '${builtins.toJSON server}';
           '';
-        locations."= /.well-known/matrix/client".extraConfig =
+        <link linkend="opt-services.nginx.virtualHosts._name_.locations._name_.extraConfig">locations."= /.well-known/matrix/client".extraConfig</link> =
           let
             client = {
               "m.homeserver" =  { "base_url" = "https://${fqdn}"; };
@@ -84,34 +93,37 @@ in {
 
       # Reverse proxy for Matrix client-server and server-server communication
       ${fqdn} = {
-        enableACME = true;
-        forceSSL = true;
+        <link linkend="opt-services.nginx.virtualHosts._name_.enableACME">enableACME</link> = true;
+        <link linkend="opt-services.nginx.virtualHosts._name_.forceSSL">forceSSL</link> = true;
 
         # Or do a redirect instead of the 404, or whatever is appropriate for you.
         # But do not put a Matrix Web client here! See the Riot Web section below.
-        locations."/".extraConfig = ''
+        <link linkend="opt-services.nginx.virtualHosts._name_.locations._name_.extraConfig">locations."/".extraConfig</link> = ''
           return 404;
         '';
 
         # forward all Matrix API calls to the synapse Matrix homeserver
         locations."/_matrix" = {
-          proxyPass = "http://[::1]:8008"; # without a trailing /
+          <link linkend="opt-services.nginx.virtualHosts._name_.locations._name_.proxyPass">proxyPass</link> = "http://[::1]:8008"; # without a trailing /
         };
       };
     };
   };
   services.matrix-synapse = {
-    enable = true;
-    server_name = config.networking.domain;
-    listeners = [
+    <link linkend="opt-services.matrix-synapse.enable">enable</link> = true;
+    <link linkend="opt-services.matrix-synapse.server_name">server_name</link> = config.networking.domain;
+    <link linkend="opt-services.matrix-synapse.listeners">listeners</link> = [
       {
-        port = 8008;
-        bind_address = "::1";
-        type = "http";
-        tls = false;
-        x_forwarded = true;
-        resources = [
-          { names = [ "client" "federation" ]; compress = false; }
+        <link linkend="opt-services.matrix-synapse.listeners._.port">port</link> = 8008;
+        <link linkend="opt-services.matrix-synapse.listeners._.bind_address">bind_address</link> = "::1";
+        <link linkend="opt-services.matrix-synapse.listeners._.type">type</link> = "http";
+        <link linkend="opt-services.matrix-synapse.listeners._.tls">tls</link> = false;
+        <link linkend="opt-services.matrix-synapse.listeners._.x_forwarded">x_forwarded</link> = true;
+        <link linkend="opt-services.matrix-synapse.listeners._.resources">resources</link> = [
+          {
+            <link linkend="opt-services.matrix-synapse.listeners._.resources._.names">names</link> = [ "client" "federation" ];
+            <link linkend="opt-services.matrix-synapse.listeners._.resources._.compress">compress</link> = false;
+          }
         ];
       }
     ];
@@ -135,10 +147,10 @@ in {
 
   <para>
    If you want to run a server with public registration by anybody, you can
-   then enable <option>services.matrix-synapse.enable_registration =
-   true;</option>. Otherwise, or you can generate a registration secret with
+   then enable <literal><link linkend="opt-services.matrix-synapse.enable_registration">services.matrix-synapse.enable_registration</link> =
+   true;</literal>. Otherwise, or you can generate a registration secret with
    <command>pwgen -s 64 1</command> and set it with
-   <option>services.matrix-synapse.registration_shared_secret</option>. To
+   <option><link linkend="opt-services.matrix-synapse.registration_shared_secret">services.matrix-synapse.registration_shared_secret</link></option>. To
    create a new user or admin, run the following after you have set the secret
    and have rebuilt NixOS:
 <screen>
@@ -154,8 +166,8 @@ Success!
    <literal>@your-username:example.org</literal>. Note that the registration
    secret ends up in the nix store and therefore is world-readable by any user
    on your machine, so it makes sense to only temporarily activate the
-   <option>registration_shared_secret</option> option until a better solution
-   for NixOS is in place.
+   <link linkend="opt-services.matrix-synapse.registration_shared_secret">registration_shared_secret</link>
+   option until a better solution for NixOS is in place.
   </para>
  </section>
  <section xml:id="module-services-matrix-riot-web">
@@ -177,15 +189,24 @@ Success!
    Matrix Now!</link> for a list of existing clients and their supported
    featureset.
 <programlisting>
-services.nginx.virtualHosts."riot.${fqdn}" = {
-  enableACME = true;
-  forceSSL = true;
-  serverAliases = [
-    "riot.${config.networking.domain}"
-  ];
+{
+  services.nginx.virtualHosts."riot.${fqdn}" = {
+    <link linkend="opt-services.nginx.virtualHosts._name_.enableACME">enableACME</link> = true;
+    <link linkend="opt-services.nginx.virtualHosts._name_.forceSSL">forceSSL</link> = true;
+    <link linkend="opt-services.nginx.virtualHosts._name_.serverAliases">serverAliases</link> = [
+      "riot.${config.networking.domain}"
+    ];
 
-  root = pkgs.riot-web;
-};
+    <link linkend="opt-services.nginx.virtualHosts._name_.root">root</link> = pkgs.riot-web.override {
+      conf = {
+        default_server_config."m.homeserver" = {
+          "base_url" = "${config.networking.domain}";
+          "server_name" = "${fqdn}";
+        };
+      };
+    };
+  };
+}
 </programlisting>
   </para>
 
diff --git a/nixos/tests/matrix-synapse.nix b/nixos/tests/matrix-synapse.nix
index fca53009083a..f3623aa3c094 100644
--- a/nixos/tests/matrix-synapse.nix
+++ b/nixos/tests/matrix-synapse.nix
@@ -35,12 +35,31 @@ in {
 
   nodes = {
     # Since 0.33.0, matrix-synapse doesn't allow underscores in server names
-    serverpostgres = args: {
+    serverpostgres = { pkgs, ... }: {
       services.matrix-synapse = {
         enable = true;
         database_type = "psycopg2";
         tls_certificate_path = "${cert}";
         tls_private_key_path = "${key}";
+        database_args = {
+          password = "synapse";
+        };
+      };
+      services.postgresql = {
+        enable = true;
+
+        # The database name and user are configured by the following options:
+        #   - services.matrix-synapse.database_name
+        #   - services.matrix-synapse.database_user
+        #
+        # The values used here represent the default values of the module.
+        initialScript = pkgs.writeText "synapse-init.sql" ''
+          CREATE ROLE "matrix-synapse" WITH LOGIN PASSWORD 'synapse';
+          CREATE DATABASE "matrix-synapse" WITH OWNER "matrix-synapse"
+            TEMPLATE template0
+            LC_COLLATE = "C"
+            LC_CTYPE = "C";
+        '';
       };
     };
 
diff --git a/pkgs/development/python-modules/signedjson/default.nix b/pkgs/development/python-modules/signedjson/default.nix
index 33a615fefd46..1214730dad87 100644
--- a/pkgs/development/python-modules/signedjson/default.nix
+++ b/pkgs/development/python-modules/signedjson/default.nix
@@ -4,19 +4,20 @@
 , canonicaljson
 , unpaddedbase64
 , pynacl
+, typing-extensions
 }:
 
 buildPythonPackage rec {
   pname = "signedjson";
-  version = "1.0.0";
+  version = "1.1.0";
 
   src = fetchgit {
     url = "https://github.com/matrix-org/python-signedjson.git";
     rev = "refs/tags/v${version}";
-    sha256 = "0b8xxhc3npd4567kqapfp4gs7m0h057xam3an7424az262ind82n";
+    sha256 = "18s388hm3babnvakbbgfqk0jzq25nnznvhygywd3azp9b4yzmd5c";
   };
 
-  propagatedBuildInputs = [ canonicaljson unpaddedbase64 pynacl ];
+  propagatedBuildInputs = [ canonicaljson unpaddedbase64 pynacl typing-extensions ];
 
   meta = with stdenv.lib; {
     homepage = https://pypi.org/project/signedjson/;
diff --git a/pkgs/servers/matrix-synapse/default.nix b/pkgs/servers/matrix-synapse/default.nix
index 295881b03a33..8da5d4676d68 100644
--- a/pkgs/servers/matrix-synapse/default.nix
+++ b/pkgs/servers/matrix-synapse/default.nix
@@ -23,11 +23,11 @@ let
 
 in buildPythonApplication rec {
   pname = "matrix-synapse";
-  version = "1.9.1";
+  version = "1.11.1";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "13csf18dchm75vw251a7h57diag94vw6rhg8kkkbpi35cibn0cz2";
+    sha256 = "0xd4bxsmk67r6pfj5lh0hn36r8z51mxsl39fjfrfdidvl1qqbxnk";
   };
 
   patches = [