Merge pull request #135296 from mguentner/backport_21_05_twisted_21_07_for_matrix_synapse

[Backport 21.05] override twisted 20.3.0 -> 21.7.0 for matrix-synapse

nixos/tests/matrix-synapse.nix

@@ -26,6 +26,13 @@ import ./make-test-python.nix ({ pkgs, ... } : let
       -days 365
   '';
 
+
+  mailerCerts = import ./common/acme/server/snakeoil-certs.nix;
+  mailerDomain = mailerCerts.domain;
+  registrationSharedSecret = "unsecure123";
+  testUser = "alice";
+  testPassword = "alicealice";
+  testEmail = "alice@example.com";
 in {
 
   name = "matrix-synapse";
@@ -35,7 +42,10 @@ in {
 
   nodes = {
     # Since 0.33.0, matrix-synapse doesn't allow underscores in server names
-    serverpostgres = { pkgs, ... }: {
+    serverpostgres = { pkgs, nodes, ... }: let
+      mailserverIP = nodes.mailserver.config.networking.primaryIPAddress;
+    in
+    {
       services.matrix-synapse = {
         enable = true;
         database_type = "psycopg2";
@@ -44,6 +54,16 @@ in {
         database_args = {
           password = "synapse";
         };
+        registration_shared_secret = registrationSharedSecret;
+        public_baseurl = "https://example.com";
+        extraConfig = ''
+          email:
+            smtp_host: "${mailerDomain}"
+            smtp_port: 25
+            require_transport_security: true
+            notif_from: "matrix <matrix@${mailerDomain}>"
+            app_name: "Matrix"
+        '';
       };
       services.postgresql = {
         enable = true;
@@ -61,6 +81,85 @@ in {
             LC_CTYPE = "C";
         '';
       };
+
+      networking.extraHosts = ''
+        ${mailserverIP} ${mailerDomain}
+      '';
+
+      security.pki.certificateFiles = [
+        mailerCerts.ca.cert ca_pem
+      ];
+
+      environment.systemPackages = let
+        sendTestMailStarttls = pkgs.writeScriptBin "send-testmail-starttls" ''
+          #!${pkgs.python3.interpreter}
+          import smtplib
+          import ssl
+
+          ctx = ssl.create_default_context()
+
+          with smtplib.SMTP('${mailerDomain}') as smtp:
+            smtp.ehlo()
+            smtp.starttls(context=ctx)
+            smtp.ehlo()
+            smtp.sendmail('matrix@${mailerDomain}', '${testEmail}', 'Subject: Test STARTTLS\n\nTest data.')
+            smtp.quit()
+         '';
+
+        obtainTokenAndRegisterEmail = let
+          # adding the email through the API is quite complicated as it involves more than one step and some
+          # client-side calculation
+          insertEmailForAlice = pkgs.writeText "alice-email.sql" ''
+            INSERT INTO user_threepids (user_id, medium, address, validated_at, added_at) VALUES ('${testUser}@serverpostgres', 'email', '${testEmail}', '1629149927271', '1629149927270');
+          '';
+        in
+        pkgs.writeScriptBin "obtain-token-and-register-email" ''
+          #!${pkgs.runtimeShell}
+          set -o errexit
+          set -o pipefail
+          set -o nounset
+          su postgres -c "psql -d matrix-synapse -f ${insertEmailForAlice}"
+          curl --fail -XPOST 'https://localhost:8448/_matrix/client/r0/account/password/email/requestToken' -d '{"email":"${testEmail}","client_secret":"foobar","send_attempt":1}' -v
+        '';
+        in [ sendTestMailStarttls pkgs.matrix-synapse obtainTokenAndRegisterEmail ];
+    };
+
+    # test mail delivery
+    mailserver = args: let
+    in
+    {
+      security.pki.certificateFiles = [
+        mailerCerts.ca.cert
+      ];
+
+      networking.firewall.enable = false;
+
+      services.postfix = {
+        enable = true;
+        hostname = "${mailerDomain}";
+        # open relay for subnet
+        networksStyle = "subnet";
+        enableSubmission = true;
+        tlsTrustedAuthorities = "${mailerCerts.ca.cert}";
+        sslCert = "${mailerCerts.${mailerDomain}.cert}";
+        sslKey = "${mailerCerts.${mailerDomain}.key}";
+
+        # blackhole transport
+        transport = "example.com discard:silently";
+
+        config = {
+          debug_peer_level = "10";
+          smtpd_relay_restrictions = [
+            "permit_mynetworks" "reject_unauth_destination"
+          ];
+
+          # disable obsolete protocols, something old versions of twisted are still using
+          smtpd_tls_protocols = "TLSv1.3, TLSv1.2, !TLSv1.1, !TLSv1, !SSLv2, !SSLv3";
+          smtp_tls_protocols = "TLSv1.3, TLSv1.2, !TLSv1.1, !TLSv1, !SSLv2, !SSLv3";
+          smtpd_tls_mandatory_protocols = "TLSv1.3, TLSv1.2, !TLSv1.1, !TLSv1, !SSLv2, !SSLv3";
+          smtp_tls_mandatory_protocols = "TLSv1.3, TLSv1.2, !TLSv1.1, !TLSv1, !SSLv2, !SSLv3";
+        };
+      };
     };
 
     serversqlite = args: {
@@ -75,11 +174,15 @@ in {
 
   testScript = ''
     start_all()
+    mailserver.wait_for_unit("postfix.service")
+    serverpostgres.succeed("send-testmail-starttls")
     serverpostgres.wait_for_unit("matrix-synapse.service")
     serverpostgres.wait_until_succeeds(
         "curl --fail -L --cacert ${ca_pem} https://localhost:8448/"
     )
     serverpostgres.require_unit_state("postgresql.service")
+    serverpostgres.succeed("register_new_matrix_user -u ${testUser} -p ${testPassword} -a -k ${registrationSharedSecret} ")
+    serverpostgres.succeed("obtain-token-and-register-email")
     serversqlite.wait_for_unit("matrix-synapse.service")
     serversqlite.wait_until_succeeds(
         "curl --fail -L --cacert ${ca_pem} https://localhost:8448/"

pkgs/servers/matrix-synapse/default.nix

@@ -4,13 +4,25 @@
 , callPackage
 }:
 
-with python3.pkgs;
-
 let
-  plugins = python3.pkgs.callPackage ./plugins { };
+  py = python3.override {
+    packageOverrides = self: super:  {
+      twisted = super.twisted.overridePythonAttrs (oldAttrs: rec {
+        version = "21.7.0";
+        src = oldAttrs.src.override {
+          inherit version;
+          extension = "tar.gz";
+          sha256 = "01lh225d7lfnmfx4f4kxwl3963gjc9yg8jfkn1w769v34ia55mic";
+        };
+
+        propagatedBuildInputs = with self; oldAttrs.propagatedBuildInputs ++ [ typing-extensions ];
+      });
+    };
+  };
+  plugins = py.pkgs.callPackage ./plugins { };
   tools = callPackage ./tools { };
 in
-buildPythonApplication rec {
+with py.pkgs; buildPythonApplication rec {
   pname = "matrix-synapse";
   version = "1.39.0";