From 21c9e4d054b57cbff6dcffe4ef0796a38a6084cb Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Mon, 16 Mar 2020 11:32:23 +0000 Subject: [PATCH 1/4] Fix buggy condition in account validity handler (#28) --- changelog.d/28.bugfix | 1 + synapse/handlers/account_validity.py | 6 +++++- synapse/python_dependencies.py | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 changelog.d/28.bugfix diff --git a/changelog.d/28.bugfix b/changelog.d/28.bugfix new file mode 100644 index 0000000000..38d7455971 --- /dev/null +++ b/changelog.d/28.bugfix @@ -0,0 +1 @@ +Fix a bug causing account validity renewal emails to be sent even if the feature is turned off in some cases. diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 947237d7da..51507bde61 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -45,7 +45,11 @@ def __init__(self, hs): self._show_users_in_user_directory = self.hs.config.show_users_in_user_directory self.profile_handler = self.hs.get_profile_handler() - if self._account_validity.renew_by_email_enabled and load_jinja2_templates: + if ( + self._account_validity.enabled + and self._account_validity.renew_by_email_enabled + and load_jinja2_templates + ): # Don't do email-specific configuration if renewal by email is disabled. try: app_name = self.hs.config.email_app_name diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 7dfa78dadb..437e79f27c 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -42,7 +42,9 @@ "frozendict>=1", "unpaddedbase64>=1.1.0", "canonicaljson>=1.1.3", - "signedjson>=1.0.0", + # Pin signedjson to 1.0.0 because this version of Synapse relies on a function that's + # been removed in 1.1.0. Hopefully, this will be fixed by the upcoming mainline merge. + "signedjson==1.0.0", "pynacl>=1.2.1", "idna>=2", From 29f4572db4ea5c88301fc6b7efbb3a68a1368c03 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 16 Mar 2020 19:07:29 +0000 Subject: [PATCH 2/4] Share SSL options for well-known requests (#29) --- changelog.d/29.misc | 1 + synapse/crypto/context_factory.py | 8 ++++++++ .../http/federation/matrix_federation_agent.py | 15 +++++---------- .../federation/test_matrix_federation_agent.py | 12 ++++++------ 4 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 changelog.d/29.misc diff --git a/changelog.d/29.misc b/changelog.d/29.misc new file mode 100644 index 0000000000..720e0ddcfb --- /dev/null +++ b/changelog.d/29.misc @@ -0,0 +1 @@ +Improve performance when making `.well-known` requests by sharing the SSL options between requests. diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 2bc5cc3807..3a16f5ef58 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -26,6 +26,7 @@ from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.internet.ssl import CertificateOptions, ContextFactory, platformTrust from twisted.python.failure import Failure +from twisted.web.iweb import IPolicyForHTTPS logger = logging.getLogger(__name__) @@ -59,6 +60,7 @@ def getContext(self): return self._context +@implementer(IPolicyForHTTPS) class ClientTLSOptionsFactory(object): """Factory for Twisted SSLClientConnectionCreators that are used to make connections to remote servers for federation. @@ -119,6 +121,12 @@ def _context_info_cb(ssl_connection, where, ret): f = Failure() tls_protocol.failVerification(f) + def creatorForNetloc(self, hostname, port): + """Implements the IPolicyForHTTPS interace so that this can be passed + directly to agents. + """ + return self.get_options(hostname) + @implementer(IOpenSSLClientConnectionCreator) class SSLClientConnectionCreator(object): diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index b4cbe97b41..e0c075e382 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -64,10 +64,6 @@ class MatrixFederationAgent(object): tls_client_options_factory (ClientTLSOptionsFactory|None): factory to use for fetching client tls options, or none to disable TLS. - _well_known_tls_policy (IPolicyForHTTPS|None): - TLS policy to use for fetching .well-known files. None to use a default - (browser-like) implementation. - _srv_resolver (SrvResolver|None): SRVResolver impl to use for looking up SRV records. None to use a default implementation. @@ -96,13 +92,12 @@ def __init__( self._pool.maxPersistentPerHost = 5 self._pool.cachedConnectionTimeout = 2 * 60 - agent_args = {} - if _well_known_tls_policy is not None: - # the param is called 'contextFactory', but actually passing a - # contextfactory is deprecated, and it expects an IPolicyForHTTPS. - agent_args['contextFactory'] = _well_known_tls_policy _well_known_agent = RedirectAgent( - Agent(self._reactor, pool=self._pool, **agent_args), + Agent( + self._reactor, + pool=self._pool, + contextFactory=tls_client_options_factory, + ) ) self._well_known_agent = _well_known_agent diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 6609d6b366..2ca9f8a0cd 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -73,7 +73,6 @@ def setUp(self): config_dict = default_config("test", parse=False) config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()] - # config_dict["trusted_key_servers"] = [] self._config = config = HomeServerConfig() config.parse_config_dict(config_dict) @@ -81,7 +80,6 @@ def setUp(self): self.agent = MatrixFederationAgent( reactor=self.reactor, tls_client_options_factory=ClientTLSOptionsFactory(config), - _well_known_tls_policy=TrustingTLSPolicyForHTTPS(), _srv_resolver=self.mock_resolver, _well_known_cache=self.well_known_cache, ) @@ -694,16 +692,18 @@ def test_get_well_known_unsigned_cert(self): not signed by a CA """ - # we use the same test server as the other tests, but use an agent - # with _well_known_tls_policy left to the default, which will not - # trust it (since the presented cert is signed by a test CA) + # we use the same test server as the other tests, but use an agent with + # the config left to the default, which will not trust it (since the + # presented cert is signed by a test CA) self.mock_resolver.resolve_service.side_effect = lambda _: [] self.reactor.lookups["testserv"] = "1.2.3.4" + config = default_config("test", parse=True) + agent = MatrixFederationAgent( reactor=self.reactor, - tls_client_options_factory=ClientTLSOptionsFactory(self._config), + tls_client_options_factory=ClientTLSOptionsFactory(config), _srv_resolver=self.mock_resolver, _well_known_cache=self.well_known_cache, ) From 85845e048cd8d2674ad38dd00e8a934b4b960adc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 17 Mar 2020 17:40:37 +0100 Subject: [PATCH 3/4] Share SSL contexts for non-federation requests (#30) * Share SSL contexts for non-federation requests * newsfile --- changelog.d/30.misc | 1 + synapse/crypto/context_factory.py | 60 ++++++++++++------- synapse/http/client.py | 3 - .../federation/matrix_federation_agent.py | 2 +- synapse/server.py | 6 +- .../test_matrix_federation_agent.py | 6 +- 6 files changed, 48 insertions(+), 30 deletions(-) create mode 100644 changelog.d/30.misc diff --git a/changelog.d/30.misc b/changelog.d/30.misc new file mode 100644 index 0000000000..ae68554be3 --- /dev/null +++ b/changelog.d/30.misc @@ -0,0 +1 @@ +Improve performance when making HTTP requests to sygnal, sydent, etc, by sharing the SSL context object between connections. diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 3a16f5ef58..fec197a0d8 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -61,7 +61,7 @@ def getContext(self): @implementer(IPolicyForHTTPS) -class ClientTLSOptionsFactory(object): +class FederationPolicyForHTTPS(object): """Factory for Twisted SSLClientConnectionCreators that are used to make connections to remote servers for federation. @@ -82,10 +82,10 @@ def __init__(self, config): trust_root = platformTrust() self._verify_ssl_context = CertificateOptions(trustRoot=trust_root).getContext() - self._verify_ssl_context.set_info_callback(self._context_info_cb) + self._verify_ssl_context.set_info_callback(_context_info_cb) self._no_verify_ssl_context = CertificateOptions().getContext() - self._no_verify_ssl_context.set_info_callback(self._context_info_cb) + self._no_verify_ssl_context.set_info_callback(_context_info_cb) def get_options(self, host): # Check if certificate verification has been enabled @@ -104,23 +104,6 @@ def get_options(self, host): return SSLClientConnectionCreator(host, ssl_context, should_verify) - @staticmethod - def _context_info_cb(ssl_connection, where, ret): - """The 'information callback' for our openssl context object.""" - # we assume that the app_data on the connection object has been set to - # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator) - tls_protocol = ssl_connection.get_app_data() - try: - # ... we further assume that SSLClientConnectionCreator has set the - # '_synapse_tls_verifier' attribute to a ConnectionVerifier object. - tls_protocol._synapse_tls_verifier.verify_context_info_cb( - ssl_connection, where - ) - except: # noqa: E722, taken from the twisted implementation - logger.exception("Error during info_callback") - f = Failure() - tls_protocol.failVerification(f) - def creatorForNetloc(self, hostname, port): """Implements the IPolicyForHTTPS interace so that this can be passed directly to agents. @@ -128,6 +111,43 @@ def creatorForNetloc(self, hostname, port): return self.get_options(hostname) +@implementer(IPolicyForHTTPS) +class RegularPolicyForHTTPS(object): + """Factory for Twisted SSLClientConnectionCreators that are used to make connections + to remote servers, for other than federation. + + Always uses the same OpenSSL context object, which uses the default OpenSSL CA + trust root. + """ + + def __init__(self): + trust_root = platformTrust() + self._ssl_context = CertificateOptions(trustRoot=trust_root).getContext() + self._ssl_context.set_info_callback(_context_info_cb) + + def creatorForNetloc(self, hostname, port): + return SSLClientConnectionCreator(hostname, self._ssl_context, True) + + +def _context_info_cb(ssl_connection, where, ret): + """The 'information callback' for our openssl context objects. + + Note: Once this is set as the info callback on a Context object, the Context should + only be used with the SSLClientConnectionCreator. + """ + # we assume that the app_data on the connection object has been set to + # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator) + tls_protocol = ssl_connection.get_app_data() + try: + # ... we further assume that SSLClientConnectionCreator has set the + # '_synapse_tls_verifier' attribute to a ConnectionVerifier object. + tls_protocol._synapse_tls_verifier.verify_context_info_cb(ssl_connection, where) + except: # noqa: E722, taken from the twisted implementation + logger.exception("Error during info_callback") + f = Failure() + tls_protocol.failVerification(f) + + @implementer(IOpenSSLClientConnectionCreator) class SSLClientConnectionCreator(object): """Creates openssl connection objects for client connections. diff --git a/synapse/http/client.py b/synapse/http/client.py index bf8afa703e..896e71cef3 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -246,9 +246,6 @@ def __getattr__(_self, attr): pool.maxPersistentPerHost = max((100 * CACHE_SIZE_FACTOR, 5)) pool.cachedConnectionTimeout = 2 * 60 - # The default context factory in Twisted 14.0.0 (which we require) is - # BrowserLikePolicyForHTTPS which will do regular cert validation - # 'like a browser' self.agent = ProxyAgent( self.reactor, connectTimeout=15, diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index e0c075e382..f595349a0e 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -61,7 +61,7 @@ class MatrixFederationAgent(object): Args: reactor (IReactor): twisted reactor to use for underlying requests - tls_client_options_factory (ClientTLSOptionsFactory|None): + tls_client_options_factory (FederationPolicyForHTTPS|None): factory to use for fetching client tls options, or none to disable TLS. _srv_resolver (SrvResolver|None): diff --git a/synapse/server.py b/synapse/server.py index 3f3c79498a..0b2e49cb72 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -27,7 +27,6 @@ from twisted.enterprise import adbapi from twisted.mail.smtp import sendmail -from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.api.auth import Auth from synapse.api.filtering import Filtering @@ -35,6 +34,7 @@ from synapse.appservice.api import ApplicationServiceApi from synapse.appservice.scheduler import ApplicationServiceScheduler from synapse.crypto import context_factory +from synapse.crypto.context_factory import RegularPolicyForHTTPS from synapse.crypto.keyring import Keyring from synapse.events.builder import EventBuilderFactory from synapse.events.spamcheck import SpamChecker @@ -302,7 +302,7 @@ def build_http_client_context_factory(self): return ( InsecureInterceptableContextFactory() if self.config.use_insecure_ssl_client_just_for_testing_do_not_use - else BrowserLikePolicyForHTTPS() + else RegularPolicyForHTTPS() ) def build_simple_http_client(self): @@ -412,7 +412,7 @@ def build_pusherpool(self): return PusherPool(self) def build_http_client(self): - tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + tls_client_options_factory = context_factory.FederationPolicyForHTTPS( self.config ) return MatrixFederationHttpClient(self, tls_client_options_factory) diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 2ca9f8a0cd..16a1d31a64 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -30,7 +30,7 @@ from twisted.web.iweb import IPolicyForHTTPS from synapse.config.homeserver import HomeServerConfig -from synapse.crypto.context_factory import ClientTLSOptionsFactory +from synapse.crypto.context_factory import FederationPolicyForHTTPS from synapse.http.federation.matrix_federation_agent import ( MatrixFederationAgent, _cache_period_from_headers, @@ -79,7 +79,7 @@ def setUp(self): self.agent = MatrixFederationAgent( reactor=self.reactor, - tls_client_options_factory=ClientTLSOptionsFactory(config), + tls_client_options_factory=FederationPolicyForHTTPS(config), _srv_resolver=self.mock_resolver, _well_known_cache=self.well_known_cache, ) @@ -703,7 +703,7 @@ def test_get_well_known_unsigned_cert(self): agent = MatrixFederationAgent( reactor=self.reactor, - tls_client_options_factory=ClientTLSOptionsFactory(config), + tls_client_options_factory=FederationPolicyForHTTPS(config), _srv_resolver=self.mock_resolver, _well_known_cache=self.well_known_cache, ) From e24928de9906ecbab5580dafe443b8bdea2e4322 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 19 Mar 2020 09:46:12 -0400 Subject: [PATCH 4/4] Fixes an attribute error when using the default display name during registration. (#32) --- changelog.d/32.bugfix | 1 + synapse/app/client_reader.py | 2 ++ .../slave/storage/user_directory.py | 22 +++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 changelog.d/32.bugfix create mode 100644 synapse/replication/slave/storage/user_directory.py diff --git a/changelog.d/32.bugfix b/changelog.d/32.bugfix new file mode 100644 index 0000000000..b6e7b90710 --- /dev/null +++ b/changelog.d/32.bugfix @@ -0,0 +1 @@ +Fixes a bug when using the default display name during registration. diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index a16e037f32..cd49fc5cd3 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -44,6 +44,7 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.slave.storage.room import RoomStore from synapse.replication.slave.storage.transactions import SlavedTransactionStore +from synapse.replication.slave.storage.user_directory import SlavedUserDirectoryStore from synapse.replication.tcp.client import ReplicationClientHandler from synapse.rest.client.v1.login import LoginRestServlet from synapse.rest.client.v1.push_rule import PushRuleRestServlet @@ -84,6 +85,7 @@ class ClientReaderSlavedStore( SlavedTransactionStore, SlavedProfileStore, SlavedClientIpStore, + SlavedUserDirectoryStore, BaseSlavedStore, ): pass diff --git a/synapse/replication/slave/storage/user_directory.py b/synapse/replication/slave/storage/user_directory.py new file mode 100644 index 0000000000..0d7b1a4a83 --- /dev/null +++ b/synapse/replication/slave/storage/user_directory.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.storage.user_directory import UserDirectoryStore + +from ._base import BaseSlavedStore + + +class SlavedUserDirectoryStore(UserDirectoryStore, BaseSlavedStore): + pass