From ab97b6e33c6e141d185ec203d24ea8266f924900 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:52:57 +0200 Subject: [PATCH 01/66] Fix a test docstring in frontend proxy tests Signed-off-by: Jason Robinson --- tests/app/test_frontend_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py index a83f567ebd7e..8bdbc608a953 100644 --- a/tests/app/test_frontend_proxy.py +++ b/tests/app/test_frontend_proxy.py @@ -59,7 +59,7 @@ def test_listen_http_with_presence_enabled(self): def test_listen_http_with_presence_disabled(self): """ - When presence is on, the stub servlet will register. + When presence is off, the stub servlet will register. """ # Presence is off self.hs.config.use_presence = False From 899e60be80736e3b747eb567171296829e96f716 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:55:37 +0200 Subject: [PATCH 02/66] Add parameterized Python module to test dependencies Allows running parameterized tests. BSD license. Signed-off-by: Jason Robinson --- synapse/python_dependencies.py | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 882e844eb1b8..b5555fbba9bc 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -81,7 +81,7 @@ "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], - "test": ["mock>=2.0"], + "test": ["mock>=2.0", "parameterized"], } diff --git a/tox.ini b/tox.ini index a0f548682923..9b6cb2036caa 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = python-subunit junitxml coverage + parameterized # cyptography 2.2 requires setuptools >= 18.5 # From 4f8f41c824dc326903863625ce79e5386ad0f8e1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:58:27 +0200 Subject: [PATCH 03/66] Make FederationReaderServer _http_listen use self.get_reactor() For all the homeserver classes, only the FrontendProxyServer passes its reactor when doing the http listen. Looking at previous PR's looks like this was introduced to make it possible to write a test, otherwise when you try to run a test with the test homeserver it tries to do a real bind to a port. Passing the reactor that the homeserver is instantiated with should probably be the right thing to do anyway? Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 228a297fb853..ea594f0f1af6 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -99,7 +99,8 @@ def _listen_http(self, listener_config): listener_config, root_resource, self.version_string, - ) + ), + reactor=self.get_reactor() ) logger.info("Synapse federation reader now listening on port %d", port) From 6d255990983405fe7eb1322c9d02414c859d1e96 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:59:28 +0200 Subject: [PATCH 04/66] Add tests for the openid lister for FederationReaderServer Check all possible variants of openid and federation listener on/off possibilities. Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/app/test_openid_listener.py diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py new file mode 100644 index 000000000000..cf3baad96f57 --- /dev/null +++ b/tests/app/test_openid_listener.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from mock import patch, Mock +from parameterized import parameterized + +from synapse.app.federation_reader import FederationReaderServer + +from tests.unittest import HomeserverTestCase + + +@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) +class FederationReaderOpenIDListenerTests(HomeserverTestCase): + def make_homeserver(self, reactor, clock): + hs = self.setup_test_homeserver( + http_client=None, homeserverToUse=FederationReaderServer, + ) + return hs + + @parameterized.expand([ + (["federation"], "auth_fail"), + ([], "no_resource"), + (["openid", "federation"], "auth_fail"), + (["openid"], "auth_fail"), + ]) + def test_openid_listener(self, names, expectation): + """ + Test different openid listener configurations. + + 401 is success here since it means we hit the handler and auth failed. + """ + config = { + "port": 8080, + "bind_addresses": ["0.0.0.0"], + "resources": [{"names": names}], + } + + # Listen with the config + self.hs._listen_http(config) + + # Grab the resource from the site that was told to listen + site = self.reactor.tcpServers[0][1] + try: + self.resource = ( + site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + ) + except KeyError: + if expectation == "no_resource": + return + raise + + request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + self.render(request) + + self.assertEqual(channel.code, 401) From a17bac171f301e54d6bd3d4c769f4f005c38f112 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 18 Jan 2019 10:02:08 +0200 Subject: [PATCH 05/66] Make SynapseHomeServer _http_listener use self.get_reactor() For all the homeserver classes, only the FrontendProxyServer passes its reactor when doing the http listen. Looking at previous PR's looks like this was introduced to make it possible to write a test, otherwise when you try to run a test with the test homeserver it tries to do a real bind to a port. Passing the reactor that the homeserver is instantiated with should probably be the right thing to do anyway? Signed-off-by: Jason Robinson --- synapse/app/homeserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f3ac3d19f021..5652c6201c2f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -130,6 +130,7 @@ def _listener_http(self, config, listener_config): self.version_string, ), self.tls_server_context_factory, + reactor=self.get_reactor(), ) else: @@ -142,7 +143,8 @@ def _listener_http(self, config, listener_config): listener_config, root_resource, self.version_string, - ) + ), + reactor=self.get_reactor(), ) logger.info("Synapse now listening on port %d", port) From 5336e49b39b6ad73491531c68694c92dade028be Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 18 Jan 2019 10:03:32 +0200 Subject: [PATCH 06/66] Add tests for the openid lister for SynapseHomeServer Check all possible variants of openid and federation listener on/off possibilities. Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 49 ++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index cf3baad96f57..329aadc3d393 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2018 New Vector Ltd +# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ from parameterized import parameterized from synapse.app.federation_reader import FederationReaderServer +from synapse.app.homeserver import SynapseHomeServer from tests.unittest import HomeserverTestCase @@ -64,3 +65,49 @@ def test_openid_listener(self, names, expectation): self.render(request) self.assertEqual(channel.code, 401) + + +@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) +class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): + def make_homeserver(self, reactor, clock): + hs = self.setup_test_homeserver( + http_client=None, homeserverToUse=SynapseHomeServer, + ) + return hs + + @parameterized.expand([ + (["federation"], "auth_fail"), + ([], "no_resource"), + (["openid", "federation"], "auth_fail"), + (["openid"], "auth_fail"), + ]) + def test_openid_listener(self, names, expectation): + """ + Test different openid listener configurations. + + 401 is success here since it means we hit the handler and auth failed. + """ + config = { + "port": 8080, + "bind_addresses": ["0.0.0.0"], + "resources": [{"names": names}], + } + + # Listen with the config + self.hs._listener_http(config, config) + + # Grab the resource from the site that was told to listen + site = self.reactor.tcpServers[0][1] + try: + self.resource = ( + site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + ) + except KeyError: + if expectation == "no_resource": + return + raise + + request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + self.render(request) + + self.assertEqual(channel.code, 401) From 82e13662c03a41c085e784a594b423711e0caffa Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 21 Jan 2019 01:54:43 +0200 Subject: [PATCH 07/66] Split federation OpenID userinfo endpoint out of the federation resource This allows the OpenID userinfo endpoint to be active even if the federation resource is not active. The OpenID userinfo endpoint is called by integration managers to verify user actions using the client API OpenID access token. Without this verification, the integration manager cannot know that the access token is valid. The OpenID userinfo endpoint will be loaded in the case that either "federation" or "openid" resource is defined. The new "openid" resource is defaulted to active in default configuration. Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 7 ++ synapse/app/homeserver.py | 9 ++ synapse/config/server.py | 9 +- synapse/federation/transport/server.py | 114 ++++++++++++++++--------- 4 files changed, 93 insertions(+), 46 deletions(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index ea594f0f1af6..99e8a4cf6a2c 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -87,6 +87,13 @@ def _listen_http(self, listener_config): resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), }) + if name == "openid" and "federation" not in res["names"]: + # Only load the openid resource separately if federation resource + # is not specified since federation resource includes openid + # resource. + resources.update({ + FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + }) root_resource = create_resource_tree(resources, NoResource()) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5652c6201c2f..0a924d7a8036 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -95,6 +95,10 @@ def _listener_http(self, config, listener_config): resources = {} for res in listener_config["resources"]: for name in res["names"]: + if name == "openid" and "federation" in res["names"]: + # Skip loading openid resource if federation is defined + # since federation resource will include openid + continue resources.update(self._configure_named_resource( name, res.get("compress", False), )) @@ -192,6 +196,11 @@ def _configure_named_resource(self, name, compress=False): FEDERATION_PREFIX: TransportLayerServer(self), }) + if name == "openid": + resources.update({ + FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + }) + if name in ["static", "client"]: resources.update({ STATIC_PREFIX: File( diff --git a/synapse/config/server.py b/synapse/config/server.py index fb57791098af..556f1efee5e6 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,7 @@ def read_config(self, config): "compress": gzip_responses, }, { - "names": ["federation"], + "names": ["federation", "openid"], "compress": False, } ] @@ -170,7 +170,7 @@ def read_config(self, config): "compress": gzip_responses, }, { - "names": ["federation"], + "names": ["federation", "openid"], "compress": False, } ] @@ -328,7 +328,7 @@ def default_config(self, server_name, data_dir_path, **kwargs): # that can do automatic compression. compress: true - - names: [federation] # Federation APIs + - names: [federation, openid] # Federation APIs compress: false # optional list of additional endpoints which can be loaded via @@ -350,7 +350,7 @@ def default_config(self, server_name, data_dir_path, **kwargs): resources: - names: [client] compress: true - - names: [federation] + - names: [federation, openid] compress: false # Turn on the twisted ssh manhole service on localhost on the given @@ -477,6 +477,7 @@ def _warn_if_webclient_configured(listeners): 'keys', 'media', 'metrics', + 'openid', 'replication', 'static', 'webclient', diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 4557a9e66e01..49b80beecb87 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -43,9 +43,10 @@ class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" - def __init__(self, hs): + def __init__(self, hs, servlet_groups=None): self.hs = hs self.clock = hs.get_clock() + self.servlet_groups = servlet_groups super(TransportLayerServer, self).__init__(hs, canonical_json=False) @@ -67,6 +68,7 @@ def register_servlets(self): resource=self, ratelimiter=self.ratelimiter, authenticator=self.authenticator, + servlet_groups=self.servlet_groups, ) @@ -1308,10 +1310,12 @@ def on_PUT(self, origin, content, query, group_id): FederationClientKeysClaimServlet, FederationThirdPartyInviteExchangeServlet, On3pidBindServlet, - OpenIdUserInfo, FederationVersionServlet, ) +OPENID_SERVLET_CLASSES = ( + OpenIdUserInfo, +) ROOM_LIST_CLASSES = ( PublicRoomList, @@ -1350,44 +1354,70 @@ def on_PUT(self, origin, content, query, group_id): FederationGroupsRenewAttestaionServlet, ) +DEFAULT_SERVLET_GROUPS = ( + "federation", + "room_list", + "group_server", + "group_local", + "group_attestation", + "openid", +) + -def register_servlets(hs, resource, authenticator, ratelimiter): - for servletclass in FEDERATION_SERVLET_CLASSES: - servletclass( - handler=hs.get_federation_server(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - - for servletclass in ROOM_LIST_CLASSES: - servletclass( - handler=hs.get_room_list_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - - for servletclass in GROUP_SERVER_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_server_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - - for servletclass in GROUP_LOCAL_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_local_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - - for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_attestation_renewer(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) +def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None): + if not servlet_groups: + servlet_groups = DEFAULT_SERVLET_GROUPS + + if "federation" in servlet_groups: + for servletclass in FEDERATION_SERVLET_CLASSES: + servletclass( + handler=hs.get_federation_server(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "openid" in servlet_groups: + for servletclass in OPENID_SERVLET_CLASSES: + servletclass( + handler=hs.get_federation_server(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "room_list" in servlet_groups: + for servletclass in ROOM_LIST_CLASSES: + servletclass( + handler=hs.get_room_list_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_server" in servlet_groups: + for servletclass in GROUP_SERVER_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_server_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_local" in servlet_groups: + for servletclass in GROUP_LOCAL_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_local_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_attestation" in servlet_groups: + for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_attestation_renewer(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) From 1d2c69fee897cf052cfa03f0cc6f9f419c898bb1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 21 Jan 2019 01:59:18 +0200 Subject: [PATCH 08/66] Add changelog for openid resource addition Signed-off-by: Jason Robinson --- changelog.d/4420.feature | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 changelog.d/4420.feature diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature new file mode 100644 index 000000000000..5e684d01e061 --- /dev/null +++ b/changelog.d/4420.feature @@ -0,0 +1,13 @@ +New listener resource for the federation API "openid/userinfo" endpoint + +Integration managers use the OpenID userinfo endpoint in the federation API to verify that user +OpenID access tokens are valid. If the federation resource is disabled, integration managers will not be able +to verify the access token, causing a broken experience for users. The OpenID userinfo endpoint has now been split +to a separate `openid` resource, which is enabled by default in newly generated configuration. It is also enabled +automatically if the federation resource is enabled. + +If your homeserver runs federation enabled, this change does not require any actions. + +If you run a homeserver with federation disabled, we recommend adding the `openid` resource to your homeserver +configuration in the `type: http` listener `resources` list to allow your users access to +integration manager features. From d39b7b6d38b0d9876ad305491ea05af92ea35b04 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:00:17 +0200 Subject: [PATCH 09/66] Document `servlet_groups` parameters Signed-off-by: Jason Robinson --- synapse/federation/transport/server.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 49b80beecb87..9c1c9c39e1e9 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -44,6 +44,16 @@ class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" def __init__(self, hs, servlet_groups=None): + """Initialize the TransportLayerServer + + Will by default register all servlets. For custom behaviour, pass in + a list of servlet_groups to register. + + Args: + hs (synapse.server.HomeServer): homeserver + servlet_groups (list[str], optional): List of servlet groups to register. + Defaults to ``DEFAULT_SERVLET_GROUPS``. + """ self.hs = hs self.clock = hs.get_clock() self.servlet_groups = servlet_groups @@ -1365,6 +1375,19 @@ def on_PUT(self, origin, content, query, group_id): def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None): + """Initialize and register servlet classes. + + Will by default register all servlets. For custom behaviour, pass in + a list of servlet_groups to register. + + Args: + hs (synapse.server.HomeServer): homeserver + resource (TransportLayerServer): resource class to register to + authenticator (Authenticator): authenticator to use + ratelimiter (util.ratelimitutils.FederationRateLimiter): ratelimiter to use + servlet_groups (list[str], optional): List of servlet groups to register. + Defaults to ``DEFAULT_SERVLET_GROUPS``. + """ if not servlet_groups: servlet_groups = DEFAULT_SERVLET_GROUPS From 0516dc4d85f1018beb241df6833117b8b49d08d3 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:04:10 +0200 Subject: [PATCH 10/66] Remove openid resource from default config Instead document it commented out. Signed-off-by: Jason Robinson --- synapse/config/server.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 556f1efee5e6..eebbfccafe1c 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,7 @@ def read_config(self, config): "compress": gzip_responses, }, { - "names": ["federation", "openid"], + "names": ["federation"], "compress": False, } ] @@ -170,7 +170,7 @@ def read_config(self, config): "compress": gzip_responses, }, { - "names": ["federation", "openid"], + "names": ["federation"], "compress": False, } ] @@ -328,8 +328,13 @@ def default_config(self, server_name, data_dir_path, **kwargs): # that can do automatic compression. compress: true - - names: [federation, openid] # Federation APIs + - names: [federation] # Federation APIs compress: false + + # # If federation is disabled synapse can still expose the open ID endpoint + # # to allow integrations to authenticate users + # - names: [openid] + # compress: false # optional list of additional endpoints which can be loaded via # dynamic modules @@ -350,8 +355,12 @@ def default_config(self, server_name, data_dir_path, **kwargs): resources: - names: [client] compress: true - - names: [federation, openid] + - names: [federation] compress: false + # # If federation is disabled synapse can still expose the open ID endpoint + # # to allow integrations to authenticate users + # - names: [openid] + # compress: false # Turn on the twisted ssh manhole service on localhost on the given # port. From db33634b1dc47167cffce31ff4ae44c5d3fae2af Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:05:22 +0200 Subject: [PATCH 11/66] Collapse changelog to one line Signed-off-by: Jason Robinson --- changelog.d/4420.feature | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature index 5e684d01e061..05e777c62451 100644 --- a/changelog.d/4420.feature +++ b/changelog.d/4420.feature @@ -1,13 +1 @@ -New listener resource for the federation API "openid/userinfo" endpoint - -Integration managers use the OpenID userinfo endpoint in the federation API to verify that user -OpenID access tokens are valid. If the federation resource is disabled, integration managers will not be able -to verify the access token, causing a broken experience for users. The OpenID userinfo endpoint has now been split -to a separate `openid` resource, which is enabled by default in newly generated configuration. It is also enabled -automatically if the federation resource is enabled. - -If your homeserver runs federation enabled, this change does not require any actions. - -If you run a homeserver with federation disabled, we recommend adding the `openid` resource to your homeserver -configuration in the `type: http` listener `resources` list to allow your users access to -integration manager features. +Federation OpenID listener resource can now be activated even if federation is disabled From a47fac9af6b17ae60c381675caece6e846faefff Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:07:28 +0200 Subject: [PATCH 12/66] Fix sorting of imports in tests. Remove an unnecessary mock Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 329aadc3d393..c1ebc93f525a 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -12,7 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from mock import patch, Mock +from mock import Mock, patch + from parameterized import parameterized from synapse.app.federation_reader import FederationReaderServer @@ -21,7 +22,6 @@ from tests.unittest import HomeserverTestCase -@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) class FederationReaderOpenIDListenerTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( From 1838ef1ac39d624e21786120b402d04a0c4485ba Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 23 Jan 2019 10:38:13 +0200 Subject: [PATCH 13/66] Fix openid tests after rebase Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index c1ebc93f525a..46a3b61a3cb6 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -54,7 +54,7 @@ def test_openid_listener(self, names, expectation): site = self.reactor.tcpServers[0][1] try: self.resource = ( - site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + site.resource.children[b"_matrix"].children[b"federation"] ) except KeyError: if expectation == "no_resource": @@ -100,7 +100,7 @@ def test_openid_listener(self, names, expectation): site = self.reactor.tcpServers[0][1] try: self.resource = ( - site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + site.resource.children[b"_matrix"].children[b"federation"] ) except KeyError: if expectation == "no_resource": From 6f680241bd7f53c3a3f912c257d2bfa437dfd486 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 23 Jan 2019 10:53:48 +0200 Subject: [PATCH 14/66] Fix flake8 issues Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 5 ++++- synapse/config/server.py | 2 +- tests/app/test_openid_listener.py | 10 ++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 99e8a4cf6a2c..2c99ce8c645f 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -92,7 +92,10 @@ def _listen_http(self, listener_config): # is not specified since federation resource includes openid # resource. resources.update({ - FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + FEDERATION_PREFIX: TransportLayerServer( + self, + servlet_groups=["openid"], + ), }) root_resource = create_resource_tree(resources, NoResource()) diff --git a/synapse/config/server.py b/synapse/config/server.py index eebbfccafe1c..4eefd06f4a52 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -330,7 +330,7 @@ def default_config(self, server_name, data_dir_path, **kwargs): - names: [federation] # Federation APIs compress: false - + # # If federation is disabled synapse can still expose the open ID endpoint # # to allow integrations to authenticate users # - names: [openid] diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 46a3b61a3cb6..590abc1e92ba 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -61,7 +61,10 @@ def test_openid_listener(self, names, expectation): return raise - request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + request, channel = self.make_request( + "GET", + "/_matrix/federation/v1/openid/userinfo", + ) self.render(request) self.assertEqual(channel.code, 401) @@ -107,7 +110,10 @@ def test_openid_listener(self, names, expectation): return raise - request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + request, channel = self.make_request( + "GET", + "/_matrix/federation/v1/openid/userinfo", + ) self.render(request) self.assertEqual(channel.code, 401) From ad7ac8853cab27cb7f5aedcde4f1aaae82f8d5c9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 30 Jan 2019 16:26:13 +0000 Subject: [PATCH 15/66] by default include m.room.encryption on invites (#3902) * by default include m.room.encryption on invites * fix constant * changelog --- changelog.d/3902.feature | 1 + synapse/api/constants.py | 1 + synapse/config/api.py | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 changelog.d/3902.feature diff --git a/changelog.d/3902.feature b/changelog.d/3902.feature new file mode 100644 index 000000000000..eb8d9f239370 --- /dev/null +++ b/changelog.d/3902.feature @@ -0,0 +1 @@ +Include m.room.encryption on invites by default diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0cbae9429b41..39ff4f62eb03 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -73,6 +73,7 @@ class EventTypes(object): RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" RoomAvatar = "m.room.avatar" + RoomEncryption = "m.room.encryption" GuestAccess = "m.room.guest_access" # These are used for validation diff --git a/synapse/config/api.py b/synapse/config/api.py index 403d96ba76c1..9f25bbc5cbc1 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -24,6 +24,7 @@ def read_config(self, config): EventTypes.JoinRules, EventTypes.CanonicalAlias, EventTypes.RoomAvatar, + EventTypes.RoomEncryption, EventTypes.Name, ]) @@ -36,5 +37,6 @@ def default_config(cls, **kwargs): - "{JoinRules}" - "{CanonicalAlias}" - "{RoomAvatar}" + - "{RoomEncryption}" - "{Name}" """.format(**vars(EventTypes)) From d621c5562ea6bce4fd8282da642b3123ae016d94 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 30 Jan 2019 16:33:51 +0000 Subject: [PATCH 16/66] Copy over non-federatable trait on room upgrade --- synapse/handlers/room.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13ba9291b05d..c04ba3a0c5d0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -263,6 +263,20 @@ def clone_existing_room( } } + # Check if old room was non-federatable + + # Get old room's create event + old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( + old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), + ) + old_room_create_event_dict = yield self.store.get_events(old_room_create_event_ids.values()) + old_room_create_event = list(old_room_create_event_dict.values())[0] + + # Check if the create event specified a non-federatable room + if old_room_create_event.content.get("m.federate", True) == False: + # If so, mark the new room as non-federatable as well + creation_content["m.federate"] = False + initial_state = dict() # Replicate relevant room events From cf9a2676d026c7864de44fe3440475a9cf1dc31d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 30 Jan 2019 19:04:48 +0000 Subject: [PATCH 17/66] Add changelog --- changelog.d/4530.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4530.bugfix diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix new file mode 100644 index 000000000000..d010af927e67 --- /dev/null +++ b/changelog.d/4530.bugfix @@ -0,0 +1 @@ +Copy over room federation ability on room upgrade. \ No newline at end of file From fb50934b8ff605a70b2a325eaaf51148a9651de5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 11:34:45 +0000 Subject: [PATCH 18/66] lint --- synapse/handlers/room.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c04ba3a0c5d0..a69441b96fa9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -269,11 +269,13 @@ def clone_existing_room( old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), ) - old_room_create_event_dict = yield self.store.get_events(old_room_create_event_ids.values()) + old_room_create_event_dict = yield self.store.get_events( + old_room_create_event_ids.values(), + ) old_room_create_event = list(old_room_create_event_dict.values())[0] # Check if the create event specified a non-federatable room - if old_room_create_event.content.get("m.federate", True) == False: + if not old_room_create_event.content.get("m.federate", True): # If so, mark the new room as non-federatable as well creation_content["m.federate"] = False From 563f6a832b379e2cde6b5618a7c344c2bcd793a1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 31 Jan 2019 11:44:04 +0000 Subject: [PATCH 19/66] Reject large transactions on federation (#4513) * Reject large transactions on federation * Add changelog * lint * Simplify large transaction handling --- changelog.d/4513.misc | 1 + synapse/federation/federation_server.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 changelog.d/4513.misc diff --git a/changelog.d/4513.misc b/changelog.d/4513.misc new file mode 100644 index 000000000000..1f64a96465a2 --- /dev/null +++ b/changelog.d/4513.misc @@ -0,0 +1 @@ +Reject federation transactions if they include more than 50 PDUs or 100 EDUs. \ No newline at end of file diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index aeadc9c564b2..3da86d4ba63a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -148,6 +148,22 @@ def _handle_incoming_transaction(self, origin, transaction, request_time): logger.debug("[%s] Transaction is new", transaction.transaction_id) + # Reject if PDU count > 50 and EDU count > 100 + if (len(transaction.pdus) > 50 + or (hasattr(transaction, "edus") and len(transaction.edus) > 100)): + + logger.info( + "Transaction PDU or EDU count too large. Returning 400", + ) + + response = {} + yield self.transaction_actions.set_response( + origin, + transaction, + 400, response + ) + defer.returnValue((400, response)) + received_pdus_counter.inc(len(transaction.pdus)) origin_host, _ = parse_server_name(origin) From 3ed3cb43394b41e76f4739f22760c1d8ebfed3c7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 18:21:39 +0000 Subject: [PATCH 20/66] New function for getting room's create event --- synapse/handlers/room.py | 8 +------- synapse/storage/state.py | 31 ++++++++++++++++++++----------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a69441b96fa9..5e40e9ea46b1 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -266,13 +266,7 @@ def clone_existing_room( # Check if old room was non-federatable # Get old room's create event - old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( - old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), - ) - old_room_create_event_dict = yield self.store.get_events( - old_room_create_event_ids.values(), - ) - old_room_create_event = list(old_room_create_event_dict.values())[0] + old_room_create_event = yield self.store.get_create_event_for_room(old_room_id) # Check if the create event specified a non-federatable room if not old_room_create_event.content.get("m.federate", True): diff --git a/synapse/storage/state.py b/synapse/storage/state.py index c3ab7db7ae4e..522aaee91811 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -24,7 +24,6 @@ from twisted.internet import defer from synapse.api.constants import EventTypes -from synapse.api.errors import NotFoundError from synapse.storage._base import SQLBaseStore from synapse.storage.background_updates import BackgroundUpdateStore from synapse.storage.engines import PostgresEngine @@ -428,13 +427,9 @@ def get_room_version(self, room_id): """ # for now we do this by looking at the create event. We may want to cache this # more intelligently in future. - state_ids = yield self.get_current_state_ids(room_id) - create_id = state_ids.get((EventTypes.Create, "")) - - if not create_id: - raise NotFoundError("Unknown room %s" % (room_id)) - create_event = yield self.get_event(create_id) + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) defer.returnValue(create_event.content.get("room_version", "1")) @defer.inlineCallbacks @@ -448,6 +443,22 @@ def get_room_predecessor(self, room_id): Returns: Deferred[unicode|None]: predecessor room id """ + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) + + # Return predecessor if present + defer.returnValue(create_event.content.get("predecessor", None)) + + @defer.inlineCallbacks + def get_create_event_for_room(self, room_id): + """Get the create state event for a room. + + Args: + room_id (str) + + Returns: + Deferred[EventBase|None]: The room creation event. None if can not be found + """ state_ids = yield self.get_current_state_ids(room_id) create_id = state_ids.get((EventTypes.Create, "")) @@ -455,11 +466,9 @@ def get_room_predecessor(self, room_id): if not create_id: defer.returnValue(None) - # Retrieve the room's create event + # Retrieve the room's create event and return create_event = yield self.get_event(create_id) - - # Return predecessor if present - defer.returnValue(create_event.content.get("predecessor", None)) + defer.returnValue(create_event) @cached(max_entries=100000, iterable=True) def get_current_state_ids(self, room_id): From d239f67c25a31257a11852be08e6615e99423fe7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 18:34:15 +0000 Subject: [PATCH 21/66] Raise an exception instead of returning None --- synapse/storage/state.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 522aaee91811..d14a7b25388a 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -24,6 +24,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes +from synapse.api.errors import NotFoundError from synapse.storage._base import SQLBaseStore from synapse.storage.background_updates import BackgroundUpdateStore from synapse.storage.engines import PostgresEngine @@ -442,6 +443,9 @@ def get_room_predecessor(self, room_id): Returns: Deferred[unicode|None]: predecessor room id + + Raises: + NotFoundError if the room is unknown """ # Retrieve the room's create event create_event = yield self.get_create_event_for_room(room_id) @@ -457,14 +461,17 @@ def get_create_event_for_room(self, room_id): room_id (str) Returns: - Deferred[EventBase|None]: The room creation event. None if can not be found + Deferred[EventBase]: The room creation event. + + Raises: + NotFoundError if the room is unknown """ state_ids = yield self.get_current_state_ids(room_id) create_id = state_ids.get((EventTypes.Create, "")) # If we can't find the create event, assume we've hit a dead end if not create_id: - defer.returnValue(None) + raise NotFoundError("Unknown room %s" % (room_id)) # Retrieve the room's create event and return create_event = yield self.get_event(create_id) From f0ba34f581906e3809de6313bc48b696ac4ef07c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 1 Feb 2019 12:22:57 +0000 Subject: [PATCH 22/66] Fix noisy "twisted.internet.task.TaskStopped" errors in logs Fixes #4003 --- changelog.d/4546.bugfix | 1 + synapse/http/matrixfederationclient.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4546.bugfix diff --git a/changelog.d/4546.bugfix b/changelog.d/4546.bugfix new file mode 100644 index 000000000000..056f2848edb9 --- /dev/null +++ b/changelog.d/4546.bugfix @@ -0,0 +1 @@ +Fix noisy "twisted.internet.task.TaskStopped" errors in logs diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index bb2e64ed806c..5ee4d528d2db 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -28,7 +28,7 @@ from prometheus_client import Counter from signedjson.sign import sign_json -from twisted.internet import defer, protocol +from twisted.internet import defer, protocol, task from twisted.internet.error import DNSLookupError from twisted.internet.task import _EPSILON, Cooperator from twisted.web._newclient import ResponseDone @@ -286,7 +286,7 @@ def _send_request( json, ) data = encode_canonical_json(json) - producer = FileBodyProducer( + producer = QuieterFileBodyProducer( BytesIO(data), cooperator=self._cooperator, ) @@ -839,3 +839,16 @@ def encode_query_args(args): query_bytes = urllib.parse.urlencode(encoded_args, True) return query_bytes.encode('utf8') + + +class QuieterFileBodyProducer(FileBodyProducer): + """Wrapper for FileBodyProducer that avoids CRITICAL errors when the connection drops. + + Workaround for https://github.com/matrix-org/synapse/issues/4003 / + https://twistedmatrix.com/trac/ticket/6528 + """ + def stopProducing(self): + try: + FileBodyProducer.stopProducing(self) + except task.TaskStopped: + pass From 627ecd358eda66f81afd676523f520762571728c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 5 Feb 2019 12:16:28 +0000 Subject: [PATCH 23/66] Filter user directory state query to a subset of state events (#4462) * Filter user directory state query to a subset of state events * Add changelog --- changelog.d/4462.misc | 1 + synapse/storage/user_directory.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 changelog.d/4462.misc diff --git a/changelog.d/4462.misc b/changelog.d/4462.misc new file mode 100644 index 000000000000..03a4d7ae1cbc --- /dev/null +++ b/changelog.d/4462.misc @@ -0,0 +1 @@ +Change the user directory state query to use a filtered call to the db instead of a generic one. \ No newline at end of file diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index ce4821226599..e8b574ee5e12 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -22,6 +22,7 @@ from synapse.api.constants import EventTypes, JoinRules from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.state import StateFilter from synapse.types import get_domain_from_id, get_localpart_from_id from synapse.util.caches.descriptors import cached, cachedInlineCallbacks @@ -31,12 +32,19 @@ class UserDirectoryStore(SQLBaseStore): - @cachedInlineCallbacks(cache_context=True) - def is_room_world_readable_or_publicly_joinable(self, room_id, cache_context): + @defer.inlineCallbacks + def is_room_world_readable_or_publicly_joinable(self, room_id): """Check if the room is either world_readable or publically joinable """ - current_state_ids = yield self.get_current_state_ids( - room_id, on_invalidate=cache_context.invalidate + + # Create a state filter that only queries join and history state event + types_to_filter = ( + (EventTypes.JoinRules, ""), + (EventTypes.RoomHistoryVisibility, ""), + ) + + current_state_ids = yield self.get_filtered_current_state_ids( + room_id, StateFilter.from_types(types_to_filter), ) join_rules_id = current_state_ids.get((EventTypes.JoinRules, "")) From 664c81e8b7525bfaa5c3a7620f2f831ade0754a2 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 17:47:22 -0500 Subject: [PATCH 24/66] return proper error codes for some 404s --- synapse/handlers/e2e_room_keys.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 42b040375f0c..c5d7bf0c29c9 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -267,7 +267,7 @@ def get_version_info(self, user_id, version=None): version(str): Optional; if None gives the most recent version otherwise a historical one. Raises: - StoreError: code 404 if the requested backup version doesn't exist + NotFoundError: if the requested backup version doesn't exist Returns: A deferred of a info dict that gives the info about the new version. @@ -279,7 +279,13 @@ def get_version_info(self, user_id, version=None): """ with (yield self._upload_linearizer.queue(user_id)): - res = yield self.store.get_e2e_room_keys_version_info(user_id, version) + try: + res = yield self.store.get_e2e_room_keys_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise defer.returnValue(res) @defer.inlineCallbacks @@ -290,8 +296,14 @@ def delete_version(self, user_id, version=None): user_id(str): the user whose current backup version we're deleting version(str): the version id of the backup being deleted Raises: - StoreError: code 404 if this backup version doesn't exist + NotFoundError: if this backup version doesn't exist """ with (yield self._upload_linearizer.queue(user_id)): - yield self.store.delete_e2e_room_keys_version(user_id, version) + try: + yield self.store.delete_e2e_room_keys_version(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise From 82486371738a130322f3a1829bc4b49f79c1a3e4 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 17:57:10 -0500 Subject: [PATCH 25/66] add new endpoint to update backup versions --- synapse/handlers/e2e_room_keys.py | 34 ++++++++++++++++++++++- synapse/rest/client/v2_alpha/room_keys.py | 33 ++++++++++++++++++++++ synapse/storage/e2e_room_keys.py | 21 ++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index c5d7bf0c29c9..e0e5ece7470d 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -19,7 +19,8 @@ from twisted.internet import defer -from synapse.api.errors import NotFoundError, RoomKeysVersionError, StoreError +from synapse.api.errors import Codes, NotFoundError, RoomKeysVersionError, \ + StoreError, SynapseError from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) @@ -307,3 +308,34 @@ def delete_version(self, user_id, version=None): raise NotFoundError("Unknown backup version") else: raise + + @defer.inlineCallbacks + def update_version(self, user_id, version, version_info): + """Update the info about a given version of the user's backup + + Args: + user_id(str): the user whose current backup version we're updating + version(str): the backup version we're updating + version_info(dict): the new information about the backup + Raises: + NotFoundError: if the requested backup version doesn't exist + Returns: + A deferred of an empty dict. + """ + try: + old_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise + if old_info["algorithm"] != version_info["algorithm"]: + raise SynapseError( + 400, + "Algorithm does not match", + Codes.INVALID_PARAM + ) + + yield self.store.update_e2e_room_keys_version(user_id, version, version_info) + + defer.returnValue({}) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index ab3f1bd21a2c..1c39d2af1c0b 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -380,6 +380,39 @@ def on_DELETE(self, request, version): ) defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_PUT(self, request, version): + """ + Update the information about a given version of the user's room_keys backup. + + POST /room_keys/version/12345 HTTP/1.1 + Content-Type: application/json + { + "algorithm": "m.megolm_backup.v1", + "auth_data": { + "public_key": "abcdefg", + "signatures": { + "ed25519:something": "hijklmnop" + } + } + } + + HTTP/1.1 200 OK + Content-Type: application/json + {} + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + info = parse_json_object_from_request(request) + + if version is None: + raise SynapseError(400, "No version specified to update", Codes.MISSING_PARAM) + + yield self.e2e_room_keys_handler.update_version( + user_id, version, info + ) + defer.returnValue((200, {})) + def register_servlets(hs, http_server): RoomKeysServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 45cebe61d1fa..9a3aec759e75 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -298,6 +298,27 @@ def _create_e2e_room_keys_version_txn(txn): "create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn ) + def update_e2e_room_keys_version(self, user_id, version, info): + """Update a given backup version + + Args: + user_id(str): the user whose backup version we're updating + version(str): the version ID of the backup version we're updating + info(dict): the new backup version info to store + """ + + return self._simple_update( + table="e2e_room_keys_versions", + keyvalues={ + "user_id": user_id, + "version": version, + }, + updatevalues={ + "auth_data": json.dumps(info["auth_data"]), + }, + desc="update_e2e_room_keys_version" + ) + def delete_e2e_room_keys_version(self, user_id, version=None): """Delete a given backup version of the user's room keys. Doesn't delete their actual key data. From 9ff620a518739bfe7417bac20365e4acf9c5906e Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 21:32:52 -0500 Subject: [PATCH 26/66] fix import to make isort happy --- synapse/handlers/e2e_room_keys.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index e0e5ece7470d..6c43c9db7644 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -19,8 +19,13 @@ from twisted.internet import defer -from synapse.api.errors import Codes, NotFoundError, RoomKeysVersionError, \ - StoreError, SynapseError +from synapse.api.errors import ( + Codes, + NotFoundError, + RoomKeysVersionError, + StoreError, + SynapseError +) from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) From 51b73be63bd6931018f98ea00f6b723c6196219a Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 21:39:56 -0500 Subject: [PATCH 27/66] add changelog entry --- changelog.d/4580.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4580.feature diff --git a/changelog.d/4580.feature b/changelog.d/4580.feature new file mode 100644 index 000000000000..a2a5a77dbebe --- /dev/null +++ b/changelog.d/4580.feature @@ -0,0 +1 @@ +Add ability to update backup versions \ No newline at end of file From d9e424bf649c1d4ba7b9da7fd64db84cda389e11 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 22:18:41 -0500 Subject: [PATCH 28/66] re-try to make isort happy --- synapse/handlers/e2e_room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 6c43c9db7644..546050f8e7fc 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -24,7 +24,7 @@ NotFoundError, RoomKeysVersionError, StoreError, - SynapseError + SynapseError, ) from synapse.util.async_helpers import Linearizer From acb2ac5863d34e2d01fa53598aecda274dc2fb26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Feb 2019 19:30:32 +0000 Subject: [PATCH 29/66] Update MSC1711 FAQ to be explicit about well-known (#4584) A surprising number of people are using the well-known method, and are simply copying the example configuration. This is problematic as the example includes an explicit port, which causes inbound federation requests to have the HTTP Host header include the port, upsetting some reverse proxies. Given that, we update the well-known example to be more explicit about the various ways you can set it up, and the consequence of using an explict port. --- changelog.d/4584.misc | 1 + docs/MSC1711_certificates_FAQ.md | 40 +++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 changelog.d/4584.misc diff --git a/changelog.d/4584.misc b/changelog.d/4584.misc new file mode 100644 index 000000000000..4dec2e2b5ccb --- /dev/null +++ b/changelog.d/4584.misc @@ -0,0 +1 @@ +Update MSC1711 FAQ to calrify .well-known usage diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 414af96ef301..0a781d00e381 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -107,12 +107,12 @@ hosted at a target domain of `customer.example.net`. Currently you should have an SRV record which looks like: ``` -_matrix._tcp.example.com. IN SRV 10 5 443 customer.example.net. +_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net. ``` -In this situation, you have two choices for how to proceed: +In this situation, you have three choices for how to proceed: -#### Option 1: give Synapse (or a reverse-proxy) a certificate for your matrix domain +#### Option 1: give Synapse a certificate for your matrix domain Synapse 1.0 will expect your server to present a TLS certificate for your `server_name` (`example.com` in the above example). You can achieve this by @@ -123,12 +123,16 @@ doing one of the following: and `tls_private_key_path`, or: * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the - `server_name` domain to your Synapse instance, or: + `server_name` domain to your Synapse instance. - * Set up a reverse-proxy on port 8448 on the `server_name` domain, which - forwards to Synapse. Once it is set up, you can remove the SRV record. +### Option 2: run Synapse behind a reverse proxy -#### Option 2: add a .well-known file to delegate your matrix traffic +If you have an existing reverse proxy set up with correct TLS certificates for +your domain, you can simply route all traffic through the reverse proxy by +updating the SRV record appropriately (or removing it, if the proxy listens on +8448). + +#### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to give it a certificate for the matrix domain. @@ -151,15 +155,25 @@ You can do this with a `.well-known` file as follows: `https:///.well-known/matrix/server` with contents: ```json - {"m.server": ":"} + {"m.server": ""} ``` - In the above example, `https://example.com/.well-known/matrix/server` - should have the contents: + where the target server name is resolved as usual (i.e. SRV lookup, falling + back to talking to port 8448). + + In the above example, where synapse is listening on port 8000, + `https://example.com/.well-known/matrix/server` should have `m.server` set to one of: + + 1. `customer.example.net` ─ with a SRV record on + `_matrix._tcp.customer.example.com` pointing to port 8000, or: + + 2. `customer.example.net` ─ updating synapse to listen on the default port + 8448, or: + + 3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy + on `customer.example.net:8000` it correctly handles HTTP requests with + Host header set to `customer.example.net:8000`. - ```json - {"m.server": "customer.example.net:443"} - ``` ## FAQ From afae8442b56d2e2466812916654396341038c38c Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 8 Feb 2019 01:32:45 -0500 Subject: [PATCH 30/66] make sure version is in body and wrap in linearizer queue also add tests --- synapse/handlers/e2e_room_keys.py | 37 ++++++++---- synapse/rest/client/v2_alpha/room_keys.py | 3 +- tests/handlers/test_e2e_room_keys.py | 72 +++++++++++++++++++++++ 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 546050f8e7fc..7bc174070e6c 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -327,20 +327,35 @@ def update_version(self, user_id, version, version_info): Returns: A deferred of an empty dict. """ - try: - old_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) - except StoreError as e: - if e.code == 404: - raise NotFoundError("Unknown backup version") - else: - raise - if old_info["algorithm"] != version_info["algorithm"]: + if "version" not in version_info: + raise SynapseError( + 400, + "Missing version in body", + Codes.MISSING_PARAM + ) + if version_info["version"] != version: raise SynapseError( 400, - "Algorithm does not match", + "Version in body does not match", Codes.INVALID_PARAM ) + with (yield self._upload_linearizer.queue(user_id)): + try: + old_info = yield self.store.get_e2e_room_keys_version_info( + user_id, version + ) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise + if old_info["algorithm"] != version_info["algorithm"]: + raise SynapseError( + 400, + "Algorithm does not match", + Codes.INVALID_PARAM + ) - yield self.store.update_e2e_room_keys_version(user_id, version, version_info) + yield self.store.update_e2e_room_keys_version(user_id, version, version_info) - defer.returnValue({}) + defer.returnValue({}) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 1c39d2af1c0b..220a0de30bf6 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -394,7 +394,8 @@ def on_PUT(self, request, version): "signatures": { "ed25519:something": "hijklmnop" } - } + }, + "version": "42" } HTTP/1.1 200 OK diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index c8994f416e12..1c49bbbc3c51 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -125,6 +125,78 @@ def test_create_version(self): "auth_data": "second_version_auth_data", }) + @defer.inlineCallbacks + def test_update_version(self): + """Check that we can update versions. + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": version + }) + self.assertDictEqual(res, {}) + + # check we can retrieve it as the current version + res = yield self.handler.get_version_info(self.local_user) + self.assertDictEqual(res, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": version + }) + + @defer.inlineCallbacks + def test_update_missing_version(self): + """Check that we get a 404 on updating nonexistent versions + """ + res = None + try: + yield self.handler.update_version(self.local_user, "1", { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": "1" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_update_bad_version(self): + """Check that we get a 400 if the version in the body is missing or + doesn't match + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = None + try: + yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 400) + + res = None + try: + yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": "incorrect" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 400) + @defer.inlineCallbacks def test_delete_missing_version(self): """Check that we get a 404 on deleting nonexistent versions From 9cd33d2f4bc14a165f693e2d3dfe231179e968e8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 17:25:57 +0000 Subject: [PATCH 31/66] Deduplicate some code in synapse.app (#4567) --- changelog.d/4567.misc | 1 + synapse/app/_base.py | 63 ++++++++++++++++++++++++++++++++ synapse/app/appservice.py | 7 +--- synapse/app/client_reader.py | 13 +------ synapse/app/event_creator.py | 13 +------ synapse/app/federation_reader.py | 13 +------ synapse/app/federation_sender.py | 12 +----- synapse/app/frontend_proxy.py | 13 +------ synapse/app/homeserver.py | 54 ++------------------------- synapse/app/media_repository.py | 13 +------ synapse/app/pusher.py | 3 +- synapse/app/synchrotron.py | 7 +--- synapse/app/user_dir.py | 13 +------ synapse/config/logger.py | 16 +++----- 14 files changed, 83 insertions(+), 158 deletions(-) create mode 100644 changelog.d/4567.misc diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc new file mode 100644 index 000000000000..96a2e0aefc16 --- /dev/null +++ b/changelog.d/4567.misc @@ -0,0 +1 @@ +Reduce duplication of ``synapse.app`` code. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b97a54d45a1..3cbb0030357b 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -15,7 +15,9 @@ import gc import logging +import signal import sys +import traceback import psutil from daemonize import Daemonize @@ -23,11 +25,25 @@ from twisted.internet import error, reactor from synapse.app import check_bind_error +from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit logger = logging.getLogger(__name__) +_sighup_callbacks = [] + + +def register_sighup(func): + """ + Register a function to be called when a SIGHUP occurs. + + Args: + func (function): Function to be called when sent a SIGHUP signal. + Will be called with a single argument, the homeserver. + """ + _sighup_callbacks.append(func) + def start_worker_reactor(appname, config): """ Run the reactor in the main process @@ -189,3 +205,50 @@ def listen_ssl( logger.info("Synapse now listening on port %d (TLS)", port) return r + + +def refresh_certificate(hs): + """ + Refresh the TLS certificates that Synapse is using by re-reading them from + disk and updating the TLS context factories to use them. + """ + logging.info("Loading certificate from disk...") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + hs.config + ) + logging.info("Certificate loaded.") + + +def start(hs, listeners=None): + """ + Start a Synapse server or worker. + + Args: + hs (synapse.server.HomeServer) + listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml) + """ + try: + # Set up the SIGHUP machinery. + if hasattr(signal, "SIGHUP"): + def handle_sighup(*args, **kwargs): + for i in _sighup_callbacks: + i(hs) + + signal.signal(signal.SIGHUP, handle_sighup) + + register_sighup(refresh_certificate) + + # Load the certificate from disk. + refresh_certificate(hs) + + # It is now safe to start your Synapse. + hs.start_listening(listeners) + hs.get_datastore().start_profiling() + except Exception: + traceback.print_exc(file=sys.stderr) + reactor = hs.get_reactor() + if reactor.running: + reactor.stop() + sys.exit(1) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 8559e141af47..33107f56d137 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -168,12 +168,7 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) - - def start(): - ps.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ps, config.worker_listeners) _base.start_worker_reactor("synapse-appservice", config) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index f8a417cb6070..a9d214702286 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -173,17 +172,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-client-reader", config) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 656e0edc0fbc..b8e51961528b 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -194,17 +193,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-event-creator", config) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 3de27151328c..42886dbfc149 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-federation-reader", config) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index d944e0517f2e..a461442fdc41 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation import send_queue from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -192,17 +191,8 @@ def start(config_options): ) ss.setup() + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) _base.start_worker_reactor("synapse-federation-sender", config) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d9ef6edc3c6d..d5b954361d9a 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseSite @@ -250,17 +249,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-frontend-proxy", config) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 250a17cef865..1a341568ac7f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os -import signal import sys import traceback @@ -28,7 +27,6 @@ from twisted.application import service from twisted.internet import defer, reactor -from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File @@ -49,7 +47,6 @@ from synapse.app._base import listen_ssl, listen_tcp, quit_with_error from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.additional_resource import AdditionalResource from synapse.http.server import RootRedirect @@ -241,10 +238,10 @@ def _configure_named_resource(self, name, compress=False): return resources - def start_listening(self): + def start_listening(self, listeners): config = self.get_config() - for listener in config.listeners: + for listener in listeners: if listener["type"] == "http": self._listening_services.extend( self._listener_http(config, listener) @@ -328,20 +325,11 @@ def setup(config_options): # generating config files and shouldn't try to continue. sys.exit(0) - sighup_callbacks = [] synapse.config.logger.setup_logging( config, - use_worker_options=False, - register_sighup=sighup_callbacks.append + use_worker_options=False ) - def handle_sighup(*args, **kwargs): - for i in sighup_callbacks: - i(*args, **kwargs) - - if hasattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, handle_sighup) - events.USE_FROZEN_DICTS = config.use_frozen_dicts database_engine = create_engine(config.database_config) @@ -377,31 +365,6 @@ def handle_sighup(*args, **kwargs): hs.setup() - def refresh_certificate(*args): - """ - Refresh the TLS certificates that Synapse is using by re-reading them - from disk and updating the TLS context factories to use them. - """ - logging.info("Reloading certificate from disk...") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - logging.info("Certificate reloaded.") - - logging.info("Updating context factories...") - for i in hs._listening_services: - if isinstance(i.factory, TLSMemoryBIOFactory): - i.factory = TLSMemoryBIOFactory( - hs.tls_server_context_factory, - False, - i.factory.wrappedFactory - ) - logging.info("Context factories updated.") - - sighup_callbacks.append(refresh_certificate) - @defer.inlineCallbacks def start(): try: @@ -425,18 +388,9 @@ def start(): ): yield acme.provision_certificate() - # Read the certificate from disk and build the context factories for - # TLS. - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) + _base.start(hs, config.listeners) - # It is now safe to start your Synapse. - hs.start_listening() hs.get_pusherpool().start() - hs.get_datastore().start_profiling() hs.get_datastore().start_doing_background_updates() except Exception as e: # If a DeferredList failed (like in listening on the ACME listener), diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 4ecf64031bf7..d4cc4e9443c7 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-media-repository", config) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 83b0863f00f4..cbf0d67f51fa 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -224,11 +224,10 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) def start(): + _base.start(ps, config.worker_listeners) ps.get_pusherpool().start() - ps.get_datastore().start_profiling() reactor.callWhenRunning(start) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 0354e82bf850..9163b56d86fb 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -445,12 +445,7 @@ def start(config_options): ) ss.setup() - ss.start_listening(config.worker_listeners) - - def start(): - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-synchrotron", config) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 176d55a78333..d1ab9512cd07 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -220,17 +219,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-user-dir", config) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index a795e39b1acc..4b938053fb75 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -15,7 +15,6 @@ import logging import logging.config import os -import signal import sys from string import Template @@ -24,6 +23,7 @@ from twisted.logger import STDLibLogObserver, globalLogBeginner import synapse +from synapse.app import _base as appbase from synapse.util.logcontext import LoggingContextFilter from synapse.util.versionstring import get_version_string @@ -127,7 +127,7 @@ def generate_files(self, config): ) -def setup_logging(config, use_worker_options=False, register_sighup=None): +def setup_logging(config, use_worker_options=False): """ Set up python logging Args: @@ -140,12 +140,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): register_sighup (func | None): Function to call to register a sighup handler. """ - if not register_sighup: - if getattr(signal, "SIGHUP"): - register_sighup = lambda x: signal.signal(signal.SIGHUP, x) - else: - register_sighup = lambda x: None - log_config = (config.worker_log_config if use_worker_options else config.log_config) log_file = (config.worker_log_file if use_worker_options @@ -187,7 +181,7 @@ def sighup(signum, stack): else: handler = logging.StreamHandler() - def sighup(signum, stack): + def sighup(*args): pass handler.setFormatter(formatter) @@ -200,14 +194,14 @@ def load_log_config(): with open(log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) - def sighup(signum, stack): + def sighup(*args): # it might be better to use a file watcher or something for this. load_log_config() logging.info("Reloaded log config from %s due to SIGHUP", log_config) load_log_config() - register_sighup(sighup) + appbase.register_sighup(sighup) # make sure that the first thing we log is a thing we can grep backwards # for From 56710c7df55edc782f6562d520c4ebc839dfe500 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 8 Feb 2019 18:30:46 +0000 Subject: [PATCH 32/66] Fix 'no unique or exclusion constraint' error (#4591) Add more tables to the list of tables which need a background update to complete before we can upsert into them, which fixes a race against the background updates. --- changelog.d/4591.bugfix | 1 + synapse/storage/_base.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 changelog.d/4591.bugfix diff --git a/changelog.d/4591.bugfix b/changelog.d/4591.bugfix new file mode 100644 index 000000000000..628bbb6d818f --- /dev/null +++ b/changelog.d/4591.bugfix @@ -0,0 +1 @@ +Fix 'no unique or exclusion constraint' error diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4872ff55b634..e12416184562 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -50,6 +50,21 @@ sql_txn_timer = Histogram("synapse_storage_transaction_time", "sec", ["desc"]) +# Unique indexes which have been added in background updates. Maps from table name +# to the name of the background update which added the unique index to that table. +# +# This is used by the upsert logic to figure out which tables are safe to do a proper +# UPSERT on: until the relevant background update has completed, we +# have to emulate an upsert by locking the table. +# +UNIQUE_INDEX_BACKGROUND_UPDATES = { + "user_ips": "user_ips_device_unique_index", + "device_lists_remote_extremeties": "device_lists_remote_extremeties_unique_idx", + "device_lists_remote_cache": "device_lists_remote_cache_unique_idx", + "event_search": "event_search_event_id_idx", +} + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() @@ -194,7 +209,7 @@ def __init__(self, db_conn, hs): self.database_engine = hs.database_engine # A set of tables that are not safe to use native upserts in. - self._unsafe_to_upsert_tables = {"user_ips"} + self._unsafe_to_upsert_tables = set(UNIQUE_INDEX_BACKGROUND_UPDATES.keys()) # We add the user_directory_search table to the blacklist on SQLite # because the existing search table does not have an index, making it @@ -230,12 +245,12 @@ def _check_safe_to_upsert(self): ) updates = [x["update_name"] for x in updates] - # The User IPs table in schema #53 was missing a unique index, which we - # run as a background update. - if "user_ips_device_unique_index" not in updates: - self._unsafe_to_upsert_tables.discard("user_ips") + for table, update_name in UNIQUE_INDEX_BACKGROUND_UPDATES.items(): + if update_name not in updates: + logger.debug("Now safe to upsert in %s", table) + self._unsafe_to_upsert_tables.discard(table) - # If there's any tables left to check, reschedule to run. + # If there's any updates still running, reschedule to run. if updates: self._clock.call_later( 15.0, From 2dc2b6e9f1b85163c8ebf4d9062253a54d531f1e Mon Sep 17 00:00:00 2001 From: Valentin Anger Date: Fri, 8 Feb 2019 21:09:56 +0000 Subject: [PATCH 33/66] Allow "unavailable" presence status for /sync (#4592) * Allow "unavailable" presence status for /sync Closes #3772, closes #3779 Signed-off-by: Valentin Anger * Add changelog for PR 4592 --- changelog.d/4592.feature | 2 ++ synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/client/versions.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4592.feature diff --git a/changelog.d/4592.feature b/changelog.d/4592.feature new file mode 100644 index 000000000000..112005ded4af --- /dev/null +++ b/changelog.d/4592.feature @@ -0,0 +1,2 @@ +Allow the "unavailable" presence status for /sync. +This change makes Synapse compliant with r0.4.0 of the Client-Server specification. diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 025114672286..39d157a44be8 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -75,7 +75,7 @@ class SyncRestServlet(RestServlet): """ PATTERNS = client_v2_patterns("/sync$") - ALLOWED_PRESENCE = set(["online", "offline"]) + ALLOWED_PRESENCE = set(["online", "offline", "unavailable"]) def __init__(self, hs): super(SyncRestServlet, self).__init__() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 29e62bfcddea..27e7cbf3cc00 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -38,6 +38,7 @@ def on_GET(self, request): "r0.1.0", "r0.2.0", "r0.3.0", + "r0.4.0", ], # as per MSC1497: "unstable_features": { From 4ffd10f46d62f7bc5430d148260e3240b7a0598b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 21:04:27 +1100 Subject: [PATCH 34/66] Be tolerant of blank TLS fingerprints config (#4589) --- changelog.d/4589.bugfix | 1 + synapse/config/tls.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4589.bugfix diff --git a/changelog.d/4589.bugfix b/changelog.d/4589.bugfix new file mode 100644 index 000000000000..d5783f46e876 --- /dev/null +++ b/changelog.d/4589.bugfix @@ -0,0 +1 @@ +Synapse is now tolerant of the tls_fingerprints option being None or not specified. diff --git a/synapse/config/tls.py b/synapse/config/tls.py index b5f2cfd9b7cc..81b3a659fede 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -45,7 +45,11 @@ def read_config(self, config): self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) - self._original_tls_fingerprints = config["tls_fingerprints"] + self._original_tls_fingerprints = config.get("tls_fingerprints", []) + + if self._original_tls_fingerprints is None: + self._original_tls_fingerprints = [] + self.tls_fingerprints = list(self._original_tls_fingerprints) self.no_tls = config.get("no_tls", False) From 6e2a5aa050fc132a7dee6b3e33a7a368207d7e5a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 21:36:26 +1100 Subject: [PATCH 35/66] ACME Reprovisioning (#4522) --- changelog.d/4522.feature | 1 + synapse/app/_base.py | 19 ++++++++++ synapse/app/homeserver.py | 79 +++++++++++++++++++++++++++------------ synapse/config/tls.py | 12 +++++- synapse/server.py | 3 ++ 5 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 changelog.d/4522.feature diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature new file mode 100644 index 000000000000..ef18daf60bd4 --- /dev/null +++ b/changelog.d/4522.feature @@ -0,0 +1 @@ +Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3cbb0030357b..62c633146fe6 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -23,6 +23,7 @@ from daemonize import Daemonize from twisted.internet import error, reactor +from twisted.protocols.tls import TLSMemoryBIOFactory from synapse.app import check_bind_error from synapse.crypto import context_factory @@ -220,6 +221,24 @@ def refresh_certificate(hs): ) logging.info("Certificate loaded.") + if hs._listening_services: + logging.info("Updating context factories...") + for i in hs._listening_services: + # When you listenSSL, it doesn't make an SSL port but a TCP one with + # a TLS wrapping factory around the factory you actually want to get + # requests. This factory attribute is public but missing from + # Twisted's documentation. + if isinstance(i.factory, TLSMemoryBIOFactory): + # We want to replace TLS factories with a new one, with the new + # TLS configuration. We do this by reaching in and pulling out + # the wrappedFactory, and then re-wrapping it. + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") + def start(hs, listeners=None): """ diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d1cab07bb6fc..b4476bf16e36 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -83,7 +83,6 @@ def gz_wrap(r): class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore - _listening_services = [] def _listener_http(self, config, listener_config): port = listener_config["port"] @@ -376,42 +375,73 @@ def setup(config_options): hs.setup() + @defer.inlineCallbacks + def do_acme(): + """ + Reprovision an ACME certificate, if it's required. + + Returns: + Deferred[bool]: Whether the cert has been updated. + """ + acme = hs.get_acme_handler() + + # Check how long the certificate is active for. + cert_days_remaining = hs.config.is_disk_cert_valid( + allow_self_signed=False + ) + + # We want to reprovision if cert_days_remaining is None (meaning no + # certificate exists), or the days remaining number it returns + # is less than our re-registration threshold. + provision = False + + if (cert_days_remaining is None): + provision = True + + if cert_days_remaining > hs.config.acme_reprovision_threshold: + provision = True + + if provision: + yield acme.provision_certificate() + + defer.returnValue(provision) + + @defer.inlineCallbacks + def reprovision_acme(): + """ + Provision a certificate from ACME, if required, and reload the TLS + certificate if it's renewed. + """ + reprovisioned = yield do_acme() + if reprovisioned: + _base.refresh_certificate(hs) + @defer.inlineCallbacks def start(): try: - # Check if the certificate is still valid. - cert_days_remaining = hs.config.is_disk_cert_valid() - + # Run the ACME provisioning code, if it's enabled. if hs.config.acme_enabled: - # If ACME is enabled, we might need to provision a certificate - # before starting. acme = hs.get_acme_handler() - # Start up the webservices which we will respond to ACME - # challenges with. + # challenges with, and then provision. yield acme.start_listening() + yield do_acme() - # We want to reprovision if cert_days_remaining is None (meaning no - # certificate exists), or the days remaining number it returns - # is less than our re-registration threshold. - if (cert_days_remaining is None) or ( - not cert_days_remaining > hs.config.acme_reprovision_threshold - ): - yield acme.provision_certificate() + # Check if it needs to be reprovisioned every day. + hs.get_clock().looping_call( + reprovision_acme, + 24 * 60 * 60 * 1000 + ) _base.start(hs, config.listeners) hs.get_pusherpool().start() hs.get_datastore().start_doing_background_updates() - except Exception as e: - # If a DeferredList failed (like in listening on the ACME listener), - # we need to print the subfailure explicitly. - if isinstance(e, defer.FirstError): - e.subFailure.printTraceback(sys.stderr) - sys.exit(1) - - # Something else went wrong when starting. Print it and bail out. + except Exception: + # Print the exception and bail out. traceback.print_exc(file=sys.stderr) + if reactor.running: + reactor.stop() sys.exit(1) reactor.callWhenRunning(start) @@ -420,7 +450,8 @@ def start(): class SynapseService(service.Service): - """A twisted Service class that will start synapse. Used to run synapse + """ + A twisted Service class that will start synapse. Used to run synapse via twistd and a .tac. """ def __init__(self, config): diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 81b3a659fede..9fcc79816dc0 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -64,10 +64,14 @@ def read_config(self, config): self.tls_certificate = None self.tls_private_key = None - def is_disk_cert_valid(self): + def is_disk_cert_valid(self, allow_self_signed=True): """ Is the certificate we have on disk valid, and if so, for how long? + Args: + allow_self_signed (bool): Should we allow the certificate we + read to be self signed? + Returns: int: Days remaining of certificate validity. None: No certificate exists. @@ -88,6 +92,12 @@ def is_disk_cert_valid(self): logger.exception("Failed to parse existing certificate off disk!") raise + if not allow_self_signed: + if tls_certificate.get_subject() == tls_certificate.get_issuer(): + raise ValueError( + "TLS Certificate is self signed, and this is not permitted" + ) + # YYYYMMDDhhmmssZ -- in UTC expires_on = datetime.strptime( tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ" diff --git a/synapse/server.py b/synapse/server.py index 6c521016163f..a2cf8a91cd1d 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -112,6 +112,8 @@ def build_DEPENDENCY(self) Attributes: config (synapse.config.homeserver.HomeserverConfig): + _listening_services (list[twisted.internet.tcp.Port]): TCP ports that + we are listening on to provide HTTP services. """ __metaclass__ = abc.ABCMeta @@ -196,6 +198,7 @@ def __init__(self, hostname, reactor=None, **kwargs): self._reactor = reactor self.hostname = hostname self._building = {} + self._listening_services = [] self.clock = Clock(reactor) self.distributor = Distributor() From a126f86eec53cd4807a1300bea885122f8559944 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 11 Feb 2019 11:30:37 +0000 Subject: [PATCH 36/66] Transfer Server ACLs on room upgrade --- synapse/handlers/room.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 5e40e9ea46b1..f9af1f004618 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -284,6 +284,7 @@ def clone_existing_room( (EventTypes.GuestAccess, ""), (EventTypes.RoomAvatar, ""), (EventTypes.Encryption, ""), + (EventTypes.ServerACL, ""), ) old_room_state_ids = yield self.store.get_filtered_current_state_ids( From eff204221748c7cfa0c88162f6fd1758f03a2f4a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 11 Feb 2019 11:41:57 +0000 Subject: [PATCH 37/66] Changelog --- changelog.d/4608.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4608.bugfix diff --git a/changelog.d/4608.bugfix b/changelog.d/4608.bugfix new file mode 100644 index 000000000000..e331a362c473 --- /dev/null +++ b/changelog.d/4608.bugfix @@ -0,0 +1 @@ +Transfer Server ACLs on room upgrade. \ No newline at end of file From c475275926aeee906b76621444468280d5bf569b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 11:44:28 +0000 Subject: [PATCH 38/66] Clarifications for reverse proxy docs (#4607) Factor out the reverse proxy info to a separate file, add some more info on reverse-proxying the federation port. --- INSTALL.md | 5 +- README.rst | 52 +----------------- changelog.d/4607.misc | 1 + docs/MSC1711_certificates_FAQ.md | 22 +++++--- docs/reverse_proxy.rst | 94 ++++++++++++++++++++++++++++++++ docs/workers.rst | 5 +- 6 files changed, 117 insertions(+), 62 deletions(-) create mode 100644 changelog.d/4607.misc create mode 100644 docs/reverse_proxy.rst diff --git a/INSTALL.md b/INSTALL.md index cbe4bda120c4..e496a13b21ce 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -359,8 +359,9 @@ Synapse v1.0. Instructions for having Synapse automatically provision and renew If you would like to use your own certificates, you can do so by changing `tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; -alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, -both ports are the same in the default configuration. +alternatively, you can use a reverse proxy. See +[docs/reverse_proxy.rst](docs/reverse_proxy.rst) for information on configuring +a reverse proxy. ## Registering a user diff --git a/README.rst b/README.rst index e666b3b4276f..bc7cb5f78476 100644 --- a/README.rst +++ b/README.rst @@ -263,6 +263,8 @@ So, things to check are: (it should be ``_matrix._tcp.``), and that the port and hostname it specifies are reachable from outside your network. +.. TODO: add a note about forgetting ``nocanon`` on a reverse-proxy config + Running a Demo Federation of Synapses ------------------------------------- @@ -290,7 +292,6 @@ The advantages of Postgres include: For information on how to install and use PostgreSQL, please see `docs/postgres.rst `_. - .. _reverse-proxy: Using a reverse proxy with Synapse @@ -304,54 +305,7 @@ It is recommended to put a reverse proxy such as doing so is that it means that you can expose the default https port (443) to Matrix clients without needing to run Synapse with root privileges. -The most important thing to know here is that Matrix clients and other Matrix -servers do not necessarily need to connect to your server via the same -port. Indeed, clients will use port 443 by default, whereas servers default to -port 8448. Where these are different, we refer to the 'client port' and the -'federation port'. - -All Matrix endpoints begin with ``/_matrix``, so an example nginx -configuration for forwarding client connections to Synapse might look like:: - - server { - listen 443 ssl; - listen [::]:443 ssl; - server_name matrix.example.com; - - location /_matrix { - proxy_pass http://localhost:8008; - proxy_set_header X-Forwarded-For $remote_addr; - } - } - -an example Caddy configuration might look like:: - - matrix.example.com { - proxy /_matrix http://localhost:8008 { - transparent - } - } - -and an example Apache configuration might look like:: - - - SSLEngine on - ServerName matrix.example.com; - - - ProxyPass http://127.0.0.1:8008/_matrix nocanon - ProxyPassReverse http://127.0.0.1:8008/_matrix - - - -You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` -for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are -recorded correctly. - -Having done so, you can then use ``https://matrix.example.com`` (instead of -``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to -Synapse from a client`_. - +For information on configuring one, see ``_. Identity Servers ================ diff --git a/changelog.d/4607.misc b/changelog.d/4607.misc new file mode 100644 index 000000000000..160a82437814 --- /dev/null +++ b/changelog.d/4607.misc @@ -0,0 +1 @@ +Clarifications for reverse proxy docs diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 0a781d00e381..2c52b0d517f0 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -42,9 +42,9 @@ imminent Matrix 1.0 release, you can also see our * It used to work just fine, why are you breaking everything? * Can I manage my own certificates rather than having Synapse renew certificates itself? - * Do you still recommend against using a reverse-proxy on the federation port? + * Do you still recommend against using a reverse proxy on the federation port? * Do I still need to give my TLS certificates to Synapse if I am using a - reverse-proxy? + reverse proxy? * Do I need the same certificate for the client and federation port? * How do I tell Synapse to reload my keys/certificates after I replace them? @@ -132,6 +132,9 @@ your domain, you can simply route all traffic through the reverse proxy by updating the SRV record appropriately (or removing it, if the proxy listens on 8448). +See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a +reverse proxy. + #### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to @@ -297,17 +300,20 @@ attempt to obtain certificates from Let's Encrypt if you configure it to do so.The only requirement is that there is a valid TLS cert present for federation end points. -### Do you still recommend against using a reverse-proxy on the federation port? +### Do you still recommend against using a reverse proxy on the federation port? We no longer actively recommend against using a reverse proxy. Many admins will -find it easier to direct federation traffic to a reverse-proxy and manage their +find it easier to direct federation traffic to a reverse proxy and manage their own TLS certificates, and this is a supported configuration. +See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a +reverse proxy. + ### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy? Practically speaking, this is no longer necessary. -If you are using a reverse-proxy for all of your TLS traffic, then you can set +If you are using a reverse proxy for all of your TLS traffic, then you can set `no_tls: True`. In that case, the only reason Synapse needs the certificate is to populate a legacy 'tls_fingerprints' field in the federation API. This is ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will @@ -321,9 +327,9 @@ this, you can give it any TLS certificate at all. This will be fixed soon. ### Do I need the same certificate for the client and federation port? -No. There is nothing stopping you doing so, particularly if you are using a -reverse-proxy. However, Synapse will use the same certificate on any ports -where TLS is configured. +No. There is nothing stopping you from using different certificates, +particularly if you are using a reverse proxy. However, Synapse will use the +same certificate on any ports where TLS is configured. ### How do I tell Synapse to reload my keys/certificates after I replace them? diff --git a/docs/reverse_proxy.rst b/docs/reverse_proxy.rst new file mode 100644 index 000000000000..d8aaac8a0810 --- /dev/null +++ b/docs/reverse_proxy.rst @@ -0,0 +1,94 @@ +Using a reverse proxy with Synapse +================================== + +It is recommended to put a reverse proxy such as +`nginx `_, +`Apache `_, +`Caddy `_ or +`HAProxy `_ in front of Synapse. One advantage of +doing so is that it means that you can expose the default https port (443) to +Matrix clients without needing to run Synapse with root privileges. + +**NOTE**: Your reverse proxy must not 'canonicalise' or 'normalise' the +requested URI in any way (for example, by decoding ``%xx`` escapes). Beware +that Apache *will* canonicalise URIs unless you specifify ``nocanon``. + +When setting up a reverse proxy, remember that Matrix clients and other Matrix +servers do not necessarily need to connect to your server via the same server +name or port. Indeed, clients will use port 443 by default, whereas servers +default to port 8448. Where these are different, we refer to the 'client port' +and the 'federation port'. See `Setting up federation +<../README.rst#setting-up-federation>`_ for more details of the algorithm used for +federation connections. + +Let's assume that we expect clients to connect to our server at +``https://matrix.example.com``, and other servers to connect at +``https://example.com:8448``. Here are some example configurations: + +* nginx:: + + server { + listen 443 ssl; + listen [::]:443 ssl; + server_name matrix.example.com; + + location /_matrix { + proxy_pass http://localhost:8008; + proxy_set_header X-Forwarded-For $remote_addr; + } + } + + server { + listen 8448 ssl default_server; + listen [::]:8448 ssl default_server; + server_name example.com; + + location / { + proxy_pass http://localhost:8008; + proxy_set_header X-Forwarded-For $remote_addr; + } + } + +* Caddy:: + + matrix.example.com { + proxy /_matrix http://localhost:8008 { + transparent + } + } + + example.com:8448 { + proxy / http://localhost:8008 { + transparent + } + } + +* Apache (note the ``nocanon`` options here!):: + + + SSLEngine on + ServerName matrix.example.com; + + + ProxyPass http://127.0.0.1:8008/_matrix nocanon + ProxyPassReverse http://127.0.0.1:8008/_matrix + + + + + SSLEngine on + ServerName example.com; + + + ProxyPass http://127.0.0.1:8008/_matrix nocanon + ProxyPassReverse http://127.0.0.1:8008/_matrix + + + +You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` +for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are +recorded correctly. + +Having done so, you can then use ``https://matrix.example.com`` (instead of +``https://matrix.example.com:8448``) as the "Custom server" when connecting to +Synapse from a client. diff --git a/docs/workers.rst b/docs/workers.rst index 101e95002094..dd3a84ba0db5 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -26,9 +26,8 @@ Configuration To make effective use of the workers, you will need to configure an HTTP reverse-proxy such as nginx or haproxy, which will direct incoming requests to the correct worker, or to the main synapse instance. Note that this includes -requests made to the federation port. The caveats regarding running a -reverse-proxy on the federation port still apply (see -https://github.com/matrix-org/synapse/blob/master/README.rst#reverse-proxying-the-federation-port). +requests made to the federation port. See ``_ for +information on setting up a reverse proxy. To enable workers, you need to add two replication listeners to the master synapse, e.g.:: From 24b7f3916d514f570985b6ebc3936938bb5adf45 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 12:50:30 +0000 Subject: [PATCH 39/66] Clean up default listener configuration (#4586) Rearrange the comments to try to clarify them, and expand on what some of it means. Use a sensible default 'bind_addresses' setting. For the insecure port, only bind to localhost, and enable x_forwarded, since apparently it's for use behind a load-balancer. --- changelog.d/4586.misc | 1 + synapse/config/server.py | 129 ++++++++++++++++++++++++--------------- 2 files changed, 82 insertions(+), 48 deletions(-) create mode 100644 changelog.d/4586.misc diff --git a/changelog.d/4586.misc b/changelog.d/4586.misc new file mode 100644 index 000000000000..37af371ccfe4 --- /dev/null +++ b/changelog.d/4586.misc @@ -0,0 +1 @@ +Clean up default listener configuration diff --git a/synapse/config/server.py b/synapse/config/server.py index f0a60cc712bd..ce0458195cda 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -24,6 +24,14 @@ logger = logging.Logger(__name__) +# by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes +# (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen +# on IPv6 when '::' is set. +# +# We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in +# in the list. +DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0'] + class ServerConfig(Config): @@ -124,10 +132,13 @@ def read_config(self, config): bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) + # if bind_address was specified, add it to the list of addresses if bind_address: bind_addresses.append(bind_address) - elif not bind_addresses: - bind_addresses.append('') + + # if we still have an empty list of addresses, use the default list + if not bind_addresses: + bind_addresses.extend(DEFAULT_BIND_ADDRESSES) if not self.web_client_location: _warn_if_webclient_configured(self.listeners) @@ -295,76 +306,98 @@ def default_config(self, server_name, data_dir_path, **kwargs): # List of ports that Synapse should listen on, their purpose and their # configuration. + # + # Options for each listener include: + # + # port: the TCP port to bind to + # + # bind_addresses: a list of local addresses to listen on. The default is + # 'all local interfaces'. + # + # type: the type of listener. Normally 'http', but other valid options are: + # 'manhole' (see docs/manhole.md), + # 'metrics' (see docs/metrics-howto.rst), + # 'replication' (see docs/workers.rst). + # + # tls: set to true to enable TLS for this listener. Will use the TLS + # key/cert specified in tls_private_key_path / tls_certificate_path. + # + # x_forwarded: Only valid for an 'http' listener. Set to true to use the + # X-Forwarded-For header as the client IP. Useful when Synapse is + # behind a reverse-proxy. + # + # resources: Only valid for an 'http' listener. A list of resources to host + # on this port. Options for each resource are: + # + # names: a list of names of HTTP resources. See below for a list of + # valid resource names. + # + # compress: set to true to enable HTTP comression for this resource. + # + # additional_resources: Only valid for an 'http' listener. A map of + # additional endpoints which should be loaded via dynamic modules. + # + # Valid resource names are: + # + # client: the client-server API (/_matrix/client). Also implies 'media' and + # 'static'. + # + # consent: user consent forms (/_matrix/consent). See + # docs/consent_tracking.md. + # + # federation: the server-server API (/_matrix/federation). Also implies + # 'media', 'keys', 'openid' + # + # keys: the key discovery API (/_matrix/keys). + # + # media: the media API (/_matrix/media). + # + # metrics: the metrics interface. See docs/metrics-howto.rst. + # + # openid: OpenID authentication. + # + # replication: the HTTP replication API (/_synapse/replication). See + # docs/workers.rst. + # + # static: static resources under synapse/static (/_matrix/static). (Mostly + # useful for 'fallback authentication'.) + # + # webclient: A web client. Requires web_client_location to be set. + # listeners: - # Main HTTPS listener + # Main HTTPS listener. # For when matrix traffic is sent directly to synapse. - - - # The port to listen for HTTPS requests on. - port: %(bind_port)s - - # Local addresses to listen on. - # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6 - # addresses by default. For most other OSes, this will only listen - # on IPv6. - bind_addresses: - - '::' - - '0.0.0.0' - - # This is a 'http' listener, allows us to specify 'resources'. + - port: %(bind_port)s type: http - tls: true - # Use the X-Forwarded-For (XFF) header as the client IP and not the - # actual client IP. - x_forwarded: false - # List of HTTP resources to serve on this listener. resources: - - - # List of resources to host on this listener. - names: - - client # The client-server APIs, both v1 and v2 - # - webclient # A web client. Requires web_client_location to be set. - - # Should synapse compress HTTP responses to clients that support it? - # This should be disabled if running synapse behind a load balancer - # that can do automatic compression. + - names: [client] compress: true - - - names: [federation] # Federation APIs + - names: [federation] compress: false - # # If federation is disabled synapse can still expose the open ID endpoint - # # to allow integrations to authenticate users - # - names: [openid] - # compress: false - - # optional list of additional endpoints which can be loaded via - # dynamic modules + # example addional_resources: + # # additional_resources: # "/_matrix/my/custom/endpoint": # module: my_module.CustomRequestHandler # config: {} - # Unsecure HTTP listener, - # For when matrix traffic passes through loadbalancer that unwraps TLS. + # Unsecure HTTP listener + # For when matrix traffic passes through a reverse-proxy that unwraps TLS. - port: %(unsecure_port)s tls: false - bind_addresses: ['::', '0.0.0.0'] + bind_addresses: ['::1', '127.0.0.1'] type: http - - x_forwarded: false + x_forwarded: true resources: - names: [client] compress: true - names: [federation] compress: false - # # If federation is disabled synapse can still expose the open ID endpoint - # # to allow integrations to authenticate users - # - names: [openid] - # compress: false # Turn on the twisted ssh manhole service on localhost on the given # port. From 5d27730a73d01bd3ff7eb42a21f2f024493c5b89 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 18:03:30 +0000 Subject: [PATCH 40/66] Move ClientTLSOptionsFactory init out of refresh_certificates (#4611) It's nothing to do with refreshing the certificates. No idea why it was here. --- changelog.d/4611.misc | 1 + synapse/app/_base.py | 3 --- synapse/http/matrixfederationclient.py | 4 ++-- synapse/server.py | 6 +++++- tests/http/test_fedclient.py | 4 +--- 5 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 changelog.d/4611.misc diff --git a/changelog.d/4611.misc b/changelog.d/4611.misc new file mode 100644 index 000000000000..d2e0a05daa39 --- /dev/null +++ b/changelog.d/4611.misc @@ -0,0 +1 @@ +Move ClientTLSOptionsFactory init out of refresh_certificates diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 62c633146fe6..e1fc1afd5b17 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -216,9 +216,6 @@ def refresh_certificate(hs): logging.info("Loading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - hs.config - ) logging.info("Certificate loaded.") if hs._listening_services: diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5ee4d528d2db..3c24bf3805f7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -168,7 +168,7 @@ class MatrixFederationHttpClient(object): requests. """ - def __init__(self, hs): + def __init__(self, hs, tls_client_options_factory): self.hs = hs self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname @@ -176,7 +176,7 @@ def __init__(self, hs): self.agent = MatrixFederationAgent( hs.get_reactor(), - hs.tls_client_options_factory, + tls_client_options_factory, ) self.clock = hs.get_clock() self._store = hs.get_datastore() diff --git a/synapse/server.py b/synapse/server.py index a2cf8a91cd1d..8615b67ad43e 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -31,6 +31,7 @@ from synapse.api.ratelimiting import Ratelimiter from synapse.appservice.api import ApplicationServiceApi from synapse.appservice.scheduler import ApplicationServiceScheduler +from synapse.crypto import context_factory from synapse.crypto.keyring import Keyring from synapse.events.builder import EventBuilderFactory from synapse.events.spamcheck import SpamChecker @@ -367,7 +368,10 @@ def build_pusherpool(self): return PusherPool(self) def build_http_client(self): - return MatrixFederationHttpClient(self) + tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + self.config + ) + return MatrixFederationHttpClient(self, tls_client_options_factory) def build_db_pool(self): name = self.db_config["name"] diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py index 018c77ebcdd6..b03b37affe0b 100644 --- a/tests/http/test_fedclient.py +++ b/tests/http/test_fedclient.py @@ -43,13 +43,11 @@ def check_logcontext(context): class FederationClientTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver(reactor=reactor, clock=clock) - hs.tls_client_options_factory = None return hs def prepare(self, reactor, clock, homeserver): - self.cl = MatrixFederationHttpClient(self.hs) + self.cl = MatrixFederationHttpClient(self.hs, None) self.reactor.lookups["testserv"] = "1.2.3.4" def test_client_get(self): From 086f6f27d409520e71556cad4707cb2f70476e20 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:00:41 +0000 Subject: [PATCH 41/66] Logging improvements around TLS certs Log which file we're reading keys and certs from, and refactor the code a bit in preparation for other work --- changelog.d/4615.misc | 1 + synapse/app/_base.py | 6 ++--- synapse/config/tls.py | 54 ++++++++++++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 changelog.d/4615.misc diff --git a/changelog.d/4615.misc b/changelog.d/4615.misc new file mode 100644 index 000000000000..c7266fcfc78b --- /dev/null +++ b/changelog.d/4615.misc @@ -0,0 +1 @@ +Logging improvements around TLS certs diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e1fc1afd5b17..6d72de1daad7 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -213,13 +213,11 @@ def refresh_certificate(hs): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - logging.info("Loading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) - logging.info("Certificate loaded.") if hs._listening_services: - logging.info("Updating context factories...") + logger.info("Updating context factories...") for i in hs._listening_services: # When you listenSSL, it doesn't make an SSL port but a TCP one with # a TLS wrapping factory around the factory you actually want to get @@ -234,7 +232,7 @@ def refresh_certificate(hs): False, i.factory.wrappedFactory ) - logging.info("Context factories updated.") + logger.info("Context factories updated.") def start(hs, listeners=None): diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 9fcc79816dc0..76d2add4fe43 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -25,7 +25,7 @@ from synapse.config._base import Config -logger = logging.getLogger() +logger = logging.getLogger(__name__) class TlsConfig(Config): @@ -110,20 +110,10 @@ def read_certificate_from_disk(self): """ Read the certificates from disk. """ - self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file) - - # Check if it is self-signed, and issue a warning if so. - if self.tls_certificate.get_issuer() == self.tls_certificate.get_subject(): - warnings.warn( - ( - "Self-signed TLS certificates will not be accepted by Synapse 1.0. " - "Please either provide a valid certificate, or use Synapse's ACME " - "support to provision one." - ) - ) + self.tls_certificate = self.read_tls_certificate() if not self.no_tls: - self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) + self.tls_private_key = self.read_tls_private_key() self.tls_fingerprints = list(self._original_tls_fingerprints) @@ -250,10 +240,38 @@ def default_config(self, config_dir_path, server_name, **kwargs): % locals() ) - def read_tls_certificate(self, cert_path): - cert_pem = self.read_file(cert_path, "tls_certificate") - return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + def read_tls_certificate(self): + """Reads the TLS certificate from the configured file, and returns it + + Also checks if it is self-signed, and warns if so + + Returns: + OpenSSL.crypto.X509: the certificate + """ + cert_path = self.tls_certificate_file + logger.info("Loading TLS certificate from %s", cert_path) + cert_pem = self.read_file(cert_path, "tls_certificate_path") + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + # Check if it is self-signed, and issue a warning if so. + if cert.get_issuer() == cert.get_subject(): + warnings.warn( + ( + "Self-signed TLS certificates will not be accepted by Synapse 1.0. " + "Please either provide a valid certificate, or use Synapse's ACME " + "support to provision one." + ) + ) + + return cert - def read_tls_private_key(self, private_key_path): - private_key_pem = self.read_file(private_key_path, "tls_private_key") + def read_tls_private_key(self): + """Reads the TLS private key from the configured file, and returns it + + Returns: + OpenSSL.crypto.PKey: the private key + """ + private_key_path = self.tls_private_key_file + logger.info("Loading TLS key from %s", private_key_path) + private_key_pem = self.read_file(private_key_path, "tls_private_key_path") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) From 2129dd1a023d1e221dab8753be3fbd7024963634 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:13:53 +0000 Subject: [PATCH 42/66] Fail cleanly if listener config lacks a 'port' ... otherwise we would fail with a mysterious KeyError or something later. --- changelog.d/4616.misc | 1 + synapse/config/server.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/4616.misc diff --git a/changelog.d/4616.misc b/changelog.d/4616.misc new file mode 100644 index 000000000000..ee79e208e3f2 --- /dev/null +++ b/changelog.d/4616.misc @@ -0,0 +1 @@ +Fail cleanly if listener config lacks a 'port' diff --git a/synapse/config/server.py b/synapse/config/server.py index ce0458195cda..eed9d7c81ee6 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -129,6 +129,11 @@ def read_config(self, config): self.listeners = config.get("listeners", []) for listener in self.listeners: + if not isinstance(listener.get("port", None), int): + raise ConfigError( + "Listener configuration is lacking a valid 'port' option" + ) + bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) From 9645728619828fda050fa08aaa25628f5db5d775 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:30:59 +0000 Subject: [PATCH 43/66] Don't create server contexts when TLS is disabled we aren't going to use them anyway. --- changelog.d/4617.misc | 1 + synapse/app/_base.py | 5 +++++ synapse/crypto/context_factory.py | 4 +--- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4617.misc diff --git a/changelog.d/4617.misc b/changelog.d/4617.misc new file mode 100644 index 000000000000..6d751865c959 --- /dev/null +++ b/changelog.d/4617.misc @@ -0,0 +1 @@ +Don't create server contexts when TLS is disabled diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 6d72de1daad7..6b3cb61ae9fc 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -214,6 +214,11 @@ def refresh_certificate(hs): disk and updating the TLS context factories to use them. """ hs.config.read_certificate_from_disk() + + if hs.config.no_tls: + # nothing else to do here + return + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) if hs._listening_services: diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 286ad8010015..85f2848fb183 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -43,9 +43,7 @@ def configure_context(context, config): logger.exception("Failed to enable elliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate_chain_file(config.tls_certificate_file) - - if not config.no_tls: - context.use_privatekey(config.tls_private_key) + context.use_privatekey(config.tls_private_key) # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ context.set_cipher_list( From 4fddf8fc77496d9bb3b5fa8835f0e5ba9a5a9926 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 17:57:58 +0000 Subject: [PATCH 44/66] Infer no_tls from presence of TLS listeners Rather than have to specify `no_tls` explicitly, infer whether we need to load the TLS keys etc from whether we have any TLS-enabled listeners. --- changelog.d/4613.feature | 1 + changelog.d/4615.feature | 1 + changelog.d/4615.misc | 1 - changelog.d/4617.feature | 1 + changelog.d/4617.misc | 1 - synapse/app/_base.py | 2 +- synapse/app/homeserver.py | 5 ----- synapse/config/homeserver.py | 2 +- synapse/config/server.py | 23 ++++++++++++++++++++--- synapse/config/tls.py | 10 ++-------- 10 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 changelog.d/4613.feature create mode 100644 changelog.d/4615.feature delete mode 100644 changelog.d/4615.misc create mode 100644 changelog.d/4617.feature delete mode 100644 changelog.d/4617.misc diff --git a/changelog.d/4613.feature b/changelog.d/4613.feature new file mode 100644 index 000000000000..098f906af2e8 --- /dev/null +++ b/changelog.d/4613.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4615.feature b/changelog.d/4615.feature new file mode 100644 index 000000000000..098f906af2e8 --- /dev/null +++ b/changelog.d/4615.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4615.misc b/changelog.d/4615.misc deleted file mode 100644 index c7266fcfc78b..000000000000 --- a/changelog.d/4615.misc +++ /dev/null @@ -1 +0,0 @@ -Logging improvements around TLS certs diff --git a/changelog.d/4617.feature b/changelog.d/4617.feature new file mode 100644 index 000000000000..098f906af2e8 --- /dev/null +++ b/changelog.d/4617.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4617.misc b/changelog.d/4617.misc deleted file mode 100644 index 6d751865c959..000000000000 --- a/changelog.d/4617.misc +++ /dev/null @@ -1 +0,0 @@ -Don't create server contexts when TLS is disabled diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 6b3cb61ae9fc..50fd17c0becb 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -215,7 +215,7 @@ def refresh_certificate(hs): """ hs.config.read_certificate_from_disk() - if hs.config.no_tls: + if not hs.config.has_tls_listener(): # nothing else to do here return diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b4476bf16e36..dbd98d394f97 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -90,11 +90,6 @@ def _listener_http(self, config, listener_config): tls = listener_config.get("tls", False) site_tag = listener_config.get("tag", port) - if tls and config.no_tls: - raise ConfigError( - "Listener on port %i has TLS enabled, but no_tls is set" % (port,), - ) - resources = {} for res in listener_config["resources"]: for name in res["names"]: diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5aad062c36ac..727fdc54d8b9 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -42,7 +42,7 @@ from .workers import WorkerConfig -class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, +class HomeServerConfig(ServerConfig, TlsConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, AppServiceConfig, KeyConfig, SAML2Config, CasConfig, diff --git a/synapse/config/server.py b/synapse/config/server.py index eed9d7c81ee6..767897c419f6 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -126,14 +126,22 @@ def read_config(self, config): self.public_baseurl += '/' self.start_pushers = config.get("start_pushers", True) - self.listeners = config.get("listeners", []) - - for listener in self.listeners: + self.listeners = [] + for listener in config.get("listeners", []): if not isinstance(listener.get("port", None), int): raise ConfigError( "Listener configuration is lacking a valid 'port' option" ) + if listener.setdefault("tls", False): + # no_tls is not really supported any more, but let's grandfather it in + # here. + if config.get("no_tls", False): + logger.info( + "Ignoring TLS-enabled listener on port %i due to no_tls" + ) + continue + bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) @@ -145,6 +153,8 @@ def read_config(self, config): if not bind_addresses: bind_addresses.extend(DEFAULT_BIND_ADDRESSES) + self.listeners.append(listener) + if not self.web_client_location: _warn_if_webclient_configured(self.listeners) @@ -152,6 +162,9 @@ def read_config(self, config): bind_port = config.get("bind_port") if bind_port: + if config.get("no_tls", False): + raise ConfigError("no_tls is incompatible with bind_port") + self.listeners = [] bind_host = config.get("bind_host", "") gzip_responses = config.get("gzip_responses", True) @@ -198,6 +211,7 @@ def read_config(self, config): "port": manhole, "bind_addresses": ["127.0.0.1"], "type": "manhole", + "tls": False, }) metrics_port = config.get("metrics_port") @@ -223,6 +237,9 @@ def read_config(self, config): _check_resource_config(self.listeners) + def has_tls_listener(self): + return any(l["tls"] for l in self.listeners) + def default_config(self, server_name, data_dir_path, **kwargs): _, bind_port = parse_and_validate_server_name(server_name) if bind_port is not None: diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 76d2add4fe43..e37a41eff43c 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -51,7 +51,6 @@ def read_config(self, config): self._original_tls_fingerprints = [] self.tls_fingerprints = list(self._original_tls_fingerprints) - self.no_tls = config.get("no_tls", False) # This config option applies to non-federation HTTP clients # (e.g. for talking to recaptcha, identity servers, and such) @@ -141,6 +140,8 @@ def default_config(self, config_dir_path, server_name, **kwargs): return ( """\ + ## TLS ## + # PEM-encoded X509 certificate for TLS. # This certificate, as of Synapse 1.0, will need to be a valid and verifiable # certificate, signed by a recognised Certificate Authority. @@ -201,13 +202,6 @@ def default_config(self, config_dir_path, server_name, **kwargs): # # reprovision_threshold: 30 - # If your server runs behind a reverse-proxy which terminates TLS connections - # (for both client and federation connections), it may be useful to disable - # All TLS support for incoming connections. Setting no_tls to True will - # do so (and avoid the need to give synapse a TLS private key). - # - # no_tls: True - # List of allowed TLS fingerprints for this server to publish along # with the signing keys for this server. Other matrix servers that # make HTTPS requests to this server will check that the TLS From 0ca290865350212e1834730c918973162a3067f4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 22:01:27 +0000 Subject: [PATCH 45/66] fix tests --- synapse/config/tls.py | 2 +- tests/config/test_tls.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e37a41eff43c..86e6eb80dbe3 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -111,7 +111,7 @@ def read_certificate_from_disk(self): """ self.tls_certificate = self.read_tls_certificate() - if not self.no_tls: + if self.has_tls_listener(): self.tls_private_key = self.read_tls_private_key() self.tls_fingerprints = list(self._original_tls_fingerprints) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index 4ccaf35603e0..d8fd18a9cb9f 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -20,6 +20,11 @@ from tests.unittest import TestCase +class TestConfig(TlsConfig): + def has_tls_listener(self): + return False + + class TLSConfigTests(TestCase): def test_warn_self_signed(self): @@ -55,11 +60,10 @@ def test_warn_self_signed(self): config = { "tls_certificate_path": os.path.join(config_dir, "cert.pem"), - "no_tls": True, "tls_fingerprints": [] } - t = TlsConfig() + t = TestConfig() t.read_config(config) t.read_certificate_from_disk() From 91f8cd3307fee502c6bfb064f825952b05c4617b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 22:15:08 +0000 Subject: [PATCH 46/66] Remove redundant entries from docker config * no_tls is now redundant (#4613) * we don't need a dummy cert any more (#4618) --- changelog.d/4619.misc | 1 + docker/conf/dummy.tls.crt | 17 ----------------- docker/conf/homeserver.yaml | 8 +------- 3 files changed, 2 insertions(+), 24 deletions(-) create mode 100644 changelog.d/4619.misc delete mode 100644 docker/conf/dummy.tls.crt diff --git a/changelog.d/4619.misc b/changelog.d/4619.misc new file mode 100644 index 000000000000..886fedf19801 --- /dev/null +++ b/changelog.d/4619.misc @@ -0,0 +1 @@ +Remove redundant entries from docker config diff --git a/docker/conf/dummy.tls.crt b/docker/conf/dummy.tls.crt deleted file mode 100644 index 8e3b1a9aaa2f..000000000000 --- a/docker/conf/dummy.tls.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICnTCCAYUCAgPoMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMMCWxvY2FsaG9z -dDAeFw0xOTAxMTUwMDQxNTBaFw0yOTAxMTIwMDQxNTBaMBQxEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKqm81/8j5d -R1s7VZ8ueg12gJrPVCCAOkp0UnuC/ZlXhN0HTvnhQ+B0IlSgB4CcQZyf4jnA6o4M -rwSc7VX0MPE9x/idoA0g/0WoC6tsxugOrvbzCw8Tv+fnXglm6uVc7aFPfx69wU3q -lUHGD/8jtEoHxmCG177Pt2lHAfiVLBAyMQGtETzxt/yAfkloaybe316qoljgK5WK -cokdAt9G84EEqxNeEnx5FG3Vc100bAqJS4GvQlFgtF9KFEqZKEyB1yKBpPMDfPIS -V9hIV0gswSmYI8dpyBlGf5lPElY68ZGABmOQgr0RI5qHK/h28OpFPE0q3v4AMHgZ -I36wii4NrAUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfD8kcpZ+dn08xh1qtKtp -X+/YNZaOBIeVdlCzfoZKNblSFAFD/jCfObNJYvZMUQ8NX2UtEJp1lTA6m7ltSsdY -gpC2k1VD8iN+ooXklJmL0kxc7UUqho8I0l9vn35h+lhLF0ihT6XfZVi/lDHWl+4G -rG+v9oxvCSCWrNWLearSlFPtQQ8xPtOE0nLwfXtOI/H/2kOuC38ihaIWM4jjbWXK -E/ksgUfuDv0mFiwf1YdBF5/M3/qOowqzU8HgMJ3WoT/9Po5Ya1pWc+3BcxxytUDf -XdMu0tWHKX84tZxLcR1nZHzluyvFFM8xNtLi9xV0Z7WbfT76V0C/ulEOybGInYsv -nQ== ------END CERTIFICATE----- diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index f07d5c1001f1..babd5bef9e35 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -2,13 +2,7 @@ ## TLS ## -{% if SYNAPSE_NO_TLS %} -no_tls: True - -# workaround for https://github.com/matrix-org/synapse/issues/4554 -tls_certificate_path: "/conf/dummy.tls.crt" - -{% else %} +{% if not SYNAPSE_NO_TLS %} tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" From dfc846a316a92a750133ed04f3625da70f7944d7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 12 Feb 2019 10:37:00 +0000 Subject: [PATCH 47/66] fix self-signed cert notice from generate-config fixes #4620 --- changelog.d/4625.bugfix | 1 + synapse/config/_base.py | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 changelog.d/4625.bugfix diff --git a/changelog.d/4625.bugfix b/changelog.d/4625.bugfix new file mode 100644 index 000000000000..3dc0ecf24c4a --- /dev/null +++ b/changelog.d/4625.bugfix @@ -0,0 +1 @@ +fix self-signed cert notice from generate-config. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 5858fb92b44d..5aec43b70217 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -257,7 +257,7 @@ def load_or_generate_config(cls, description, argv): "--keys-directory", metavar="DIRECTORY", help="Used with 'generate-*' options to specify where files such as" - " certs and signing keys should be stored in, unless explicitly" + " signing keys should be stored, unless explicitly" " specified in the config.", ) config_parser.add_argument( @@ -313,16 +313,11 @@ def load_or_generate_config(cls, description, argv): print( ( "A config file has been generated in %r for server name" - " %r with corresponding SSL keys and self-signed" - " certificates. Please review this file and customise it" + " %r. Please review this file and customise it" " to your needs." ) % (config_path, server_name) ) - print( - "If this server name is incorrect, you will need to" - " regenerate the SSL certificates" - ) return else: print( From 32b781bfe20034052d71558c0ed9b0df511a1f77 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:51:31 +0000 Subject: [PATCH 48/66] Fix error when loading cert if tls is disabled (#4618) If TLS is disabled, it should not be an error if no cert is given. Fixes #4554. --- changelog.d/4618.bugfix | 1 + synapse/app/_base.py | 5 ++-- synapse/config/tls.py | 57 +++++++++++++++++++++++++++++----------- tests/config/test_tls.py | 2 +- 4 files changed, 47 insertions(+), 18 deletions(-) create mode 100644 changelog.d/4618.bugfix diff --git a/changelog.d/4618.bugfix b/changelog.d/4618.bugfix new file mode 100644 index 000000000000..22115a020e96 --- /dev/null +++ b/changelog.d/4618.bugfix @@ -0,0 +1 @@ +Fix failure to start when not TLS certificate was given even if TLS was disabled. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 50fd17c0becb..5b0ca312e22a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -213,12 +213,13 @@ def refresh_certificate(hs): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - hs.config.read_certificate_from_disk() if not hs.config.has_tls_listener(): - # nothing else to do here + # attempt to reload the certs for the good of the tls_fingerprints + hs.config.read_certificate_from_disk(require_cert_and_key=False) return + hs.config.read_certificate_from_disk(require_cert_and_key=True) hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) if hs._listening_services: diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 86e6eb80dbe3..57f117a14d0e 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -23,7 +23,7 @@ from OpenSSL import crypto -from synapse.config._base import Config +from synapse.config._base import Config, ConfigError logger = logging.getLogger(__name__) @@ -45,6 +45,19 @@ def read_config(self, config): self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) + + if self.has_tls_listener(): + if not self.tls_certificate_file: + raise ConfigError( + "tls_certificate_path must be specified if TLS-enabled listeners are " + "configured." + ) + if not self.tls_private_key_file: + raise ConfigError( + "tls_certificate_path must be specified if TLS-enabled listeners are " + "configured." + ) + self._original_tls_fingerprints = config.get("tls_fingerprints", []) if self._original_tls_fingerprints is None: @@ -105,26 +118,40 @@ def is_disk_cert_valid(self, allow_self_signed=True): days_remaining = (expires_on - now).days return days_remaining - def read_certificate_from_disk(self): - """ - Read the certificates from disk. + def read_certificate_from_disk(self, require_cert_and_key): """ - self.tls_certificate = self.read_tls_certificate() + Read the certificates and private key from disk. - if self.has_tls_listener(): + Args: + require_cert_and_key (bool): set to True to throw an error if the certificate + and key file are not given + """ + if require_cert_and_key: self.tls_private_key = self.read_tls_private_key() + self.tls_certificate = self.read_tls_certificate() + elif self.tls_certificate_file: + # we only need the certificate for the tls_fingerprints. Reload it if we + # can, but it's not a fatal error if we can't. + try: + self.tls_certificate = self.read_tls_certificate() + except Exception as e: + logger.info( + "Unable to read TLS certificate (%s). Ignoring as no " + "tls listeners enabled.", e, + ) self.tls_fingerprints = list(self._original_tls_fingerprints) - # Check that our own certificate is included in the list of fingerprints - # and include it if it is not. - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, self.tls_certificate - ) - sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) - sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) - if sha256_fingerprint not in sha256_fingerprints: - self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) + if self.tls_certificate: + # Check that our own certificate is included in the list of fingerprints + # and include it if it is not. + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, self.tls_certificate + ) + sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) + sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) + if sha256_fingerprint not in sha256_fingerprints: + self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index d8fd18a9cb9f..c260d3359f36 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -65,7 +65,7 @@ def test_warn_self_signed(self): t = TestConfig() t.read_config(config) - t.read_certificate_from_disk() + t.read_certificate_from_disk(require_cert_and_key=False) warnings = self.flushWarnings() self.assertEqual(len(warnings), 1) From a4ce91396bda0c6a6e3a2392355f8297cc97071b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:52:08 +0000 Subject: [PATCH 49/66] Disable TLS by default (#4614) --- changelog.d/4614.feature | 1 + synapse/config/server.py | 48 ++++++++++++++++++++-------------------- synapse/config/tls.py | 6 ++--- 3 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 changelog.d/4614.feature diff --git a/changelog.d/4614.feature b/changelog.d/4614.feature new file mode 100644 index 000000000000..18e16dbc7bd3 --- /dev/null +++ b/changelog.d/4614.feature @@ -0,0 +1 @@ +The default configuration no longer requires TLS certificates. diff --git a/synapse/config/server.py b/synapse/config/server.py index 767897c419f6..c5c3aac8edb3 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -387,47 +387,47 @@ def default_config(self, server_name, data_dir_path, **kwargs): # webclient: A web client. Requires web_client_location to be set. # listeners: - # Main HTTPS listener. - # For when matrix traffic is sent directly to synapse. - - port: %(bind_port)s + # TLS-enabled listener: for when matrix traffic is sent directly to synapse. + # + # Disabled by default. To enable it, uncomment the following. (Note that you + # will also need to give Synapse a TLS key and certificate: see the TLS section + # below.) + # + # - port: %(bind_port)s + # type: http + # tls: true + # resources: + # - names: [client, federation] + + # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy + # that unwraps TLS. + # + # If you plan to use a reverse proxy, please see + # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst. + # + - port: %(unsecure_port)s + tls: false + bind_addresses: ['::1', '127.0.0.1'] type: http - tls: true + x_forwarded: true - # List of HTTP resources to serve on this listener. resources: - - names: [client] - compress: true - - names: [federation] + - names: [client, federation] compress: false - # example addional_resources: + # example additonal_resources: # # additional_resources: # "/_matrix/my/custom/endpoint": # module: my_module.CustomRequestHandler # config: {} - # Unsecure HTTP listener - # For when matrix traffic passes through a reverse-proxy that unwraps TLS. - - port: %(unsecure_port)s - tls: false - bind_addresses: ['::1', '127.0.0.1'] - type: http - x_forwarded: true - - resources: - - names: [client] - compress: true - - names: [federation] - compress: false - # Turn on the twisted ssh manhole service on localhost on the given # port. # - port: 9000 # bind_addresses: ['::1', '127.0.0.1'] # type: manhole - # Homeserver blocking # # How to reach the server admin, used in ResourceLimitError diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 57f117a14d0e..5fb3486db124 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -176,10 +176,10 @@ def default_config(self, config_dir_path, server_name, **kwargs): # See 'ACME support' below to enable auto-provisioning this certificate via # Let's Encrypt. # - tls_certificate_path: "%(tls_certificate_path)s" + # tls_certificate_path: "%(tls_certificate_path)s" # PEM-encoded private key for TLS - tls_private_key_path: "%(tls_private_key_path)s" + # tls_private_key_path: "%(tls_private_key_path)s" # ACME support: This will configure Synapse to request a valid TLS certificate # for your configured `server_name` via Let's Encrypt. @@ -204,7 +204,7 @@ def default_config(self, config_dir_path, server_name, **kwargs): # acme: # ACME support is disabled by default. Uncomment the following line - # to enable it. + # (and tls_certificate_path and tls_private_key_path above) to enable it. # # enabled: true From 2418b91bb7d64bfe572647565a6f1c80e82e1f5a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:53:28 +0000 Subject: [PATCH 50/66] README updates (#4621) Lots of updates to the README/INSTALL.md. Fixes #4601. --- INSTALL.md | 42 ++++++++++++------ README.rst | 100 +++++++++++++++++++++--------------------- changelog.d/4621.misc | 1 + 3 files changed, 80 insertions(+), 63 deletions(-) create mode 100644 changelog.d/4621.misc diff --git a/INSTALL.md b/INSTALL.md index e496a13b21ce..fb6a5e4e99a7 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -350,18 +350,34 @@ Once you have installed synapse as above, you will need to configure it. ## TLS certificates -The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is -configured without TLS; it should be behind a reverse proxy for TLS/SSL -termination on port 443 which in turn should be used for clients. Port 8448 -is configured to use TLS for Federation with a self-signed or verified -certificate, but please be aware that a valid certificate will be required in -Synapse v1.0. Instructions for having Synapse automatically provision and renew federation certificates through ACME can be found at [ACME.md](docs/ACME.md). - -If you would like to use your own certificates, you can do so by changing -`tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; -alternatively, you can use a reverse proxy. See -[docs/reverse_proxy.rst](docs/reverse_proxy.rst) for information on configuring -a reverse proxy. +The default configuration exposes a single HTTP port: http://localhost:8008. It +is suitable for local testing, but for any practical use, you will either need +to enable a reverse proxy, or configure Synapse to expose an HTTPS port. + +For information on using a reverse proxy, see +[docs/reverse_proxy.rst](docs/reverse_proxy.rst). + +To configure Synapse to expose an HTTPS port, you will need to edit +`homeserver.yaml`. + +First, under the `listeners` section, uncomment the configuration for the +TLS-enabled listener. (Remove the hash sign (`#`) and space at the start of +each line). The relevant lines are like this: + +``` + - port: 8448 + type: http + tls: true + resources: + - names: [client, federation] +``` + +You will also need to uncomment the `tls_certificate_path` and +`tls_private_key_path` lines under the `TLS` section. You can either point +these settings at an existing certificate and key, or you can enable Synapse's +built-in ACME (Let's Encrypt) support. Instructions for having Synapse +automatically provision and renew federation certificates through ACME can be +found at [ACME.md](docs/ACME.md). ## Registering a user @@ -375,7 +391,7 @@ users. This can be done as follows: ``` $ source ~/synapse/env/bin/activate $ synctl start # if not already running -$ register_new_matrix_user -c homeserver.yaml https://localhost:8448 +$ register_new_matrix_user -c homeserver.yaml http://localhost:8008 New user localpart: erikj Password: Confirm password: diff --git a/README.rst b/README.rst index bc7cb5f78476..9a7c04b55e32 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,6 @@ via IRC bridge at irc://irc.freenode.net/matrix. Synapse is currently in rapid development, but as of version 0.5 we believe it is sufficiently stable to be run as an internet-facing service for real usage! - About Matrix ============ @@ -88,18 +87,20 @@ Connecting to Synapse from a client =================================== The easiest way to try out your new Synapse installation is by connecting to it -from a web client. The easiest option is probably the one at -https://riot.im/app. You will need to specify a "Custom server" when you log on -or register: set this to ``https://domain.tld`` if you setup a reverse proxy -following the recommended setup, or ``https://localhost:8448`` - remember to specify the -port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity -server as the default - see `Identity servers`_.) - -If using port 8448 you will run into errors if you are using a self-signed -certificate. To overcome this, simply go to ``https://localhost:8448`` -directly with your browser and accept the presented certificate. You can then -go back in your web client and proceed further. Valid federation certificates -should not have this problem. +from a web client. + +Unless you are running a test instance of Synapse on your local machine, in +general, you will need to enable TLS support before you can successfully +connect from a client: see ``_. + +An easy way to get started is to login or register via Riot at +https://riot.im/app/#/login or https://riot.im/app/#/register respectively. +You will need to change the server you are logging into from ``matrix.org`` +and instead specify a Homeserver URL of ``https://:8448`` +(or just ``https://`` if you are using a reverse proxy). +(Leave the identity server as the default - see `Identity servers`_.) +If you prefer to use another client, refer to our +`client breakdown `_. If all goes well you should at least be able to log in, create a room, and start sending messages. @@ -174,9 +175,30 @@ Separately, Synapse may leak file handles if inbound HTTP requests get stuck during processing - e.g. blocked behind a lock or talking to a remote server etc. This is best diagnosed by matching up the 'Received request' and 'Processed request' log lines and looking for any 'Processed request' lines which take more than -a few seconds to execute. Please let us know at #matrix-dev:matrix.org if +a few seconds to execute. Please let us know at #synapse:matrix.org if you see this failure mode so we can help debug it, however. +Help!! Synapse eats all my RAM! +------------------------------- + +Synapse's architecture is quite RAM hungry currently - we deliberately +cache a lot of recent room data and metadata in RAM in order to speed up +common requests. We'll improve this in future, but for now the easiest +way to either reduce the RAM usage (at the risk of slowing things down) +is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment +variable. The default is 0.5, which can be decreased to reduce RAM usage +in memory constrained enviroments, or increased if performance starts to +degrade. + +Using `libjemalloc `_ can also yield a significant +improvement in overall amount, and especially in terms of giving back RAM +to the OS. To use it, the library must simply be put in the LD_PRELOAD +environment variable when launching Synapse. On Debian, this can be done +by installing the ``libjemalloc1`` package and adding this line to +``/etc/default/matrix-synapse``:: + + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 + Upgrading an existing Synapse ============================= @@ -196,12 +218,12 @@ Federation is the process by which users on different servers can participate in the same room. For this to work, those other servers must be able to contact yours to send messages. -The ``server_name`` in your -``homeserver.yaml`` file determines the way that other servers will reach -yours. By default, they will treat it as a hostname and try to connect to -port 8448. This is easy to set up and will work with the default configuration, -provided you set the ``server_name`` to match your machine's public DNS -hostname. +The ``server_name`` in your ``homeserver.yaml`` file determines the way that +other servers will reach yours. By default, they will treat it as a hostname +and try to connect to port 8448. This is easy to set up and will work with the +default configuration, provided you set the ``server_name`` to match your +machine's public DNS hostname, and give Synapse a TLS certificate which is +valid for your ``server_name``. For a more flexible configuration, you can set up a DNS SRV record. This allows you to run your server on a machine that might not have the same name as your @@ -243,11 +265,8 @@ largest boxes pause for thought.) Troubleshooting --------------- -You can use the federation tester to check if your homeserver is all set: -``https://matrix.org/federationtester/api/report?server_name=`` -If any of the attributes under "checks" is false, federation won't work. -There is also a nicer interface available from a community member at -``_. +You can use the `federation tester `_ to +check if your homeserver is all set. The typical failure mode with federation is that when you try to join a room, it is rejected with "401: Unauthorized". Generally this means that other @@ -263,7 +282,10 @@ So, things to check are: (it should be ``_matrix._tcp.``), and that the port and hostname it specifies are reachable from outside your network. -.. TODO: add a note about forgetting ``nocanon`` on a reverse-proxy config +Another common problem is that people on other servers can't join rooms that +you invite them to. This can be caused by an incorrectly-configured reverse +proxy: see ``_ for instructions on how to correctly +configure a reverse proxy. Running a Demo Federation of Synapses ------------------------------------- @@ -363,7 +385,7 @@ Synapse Development Before setting up a development environment for synapse, make sure you have the system dependencies (such as the python header files) installed - see -`Installing from source`_. +`Installing from source `_. To check out a synapse for development, clone the git repo into a working directory of your choice:: @@ -374,7 +396,7 @@ directory of your choice:: Synapse has a number of external dependencies, that are easiest to install using pip and a virtualenv:: - virtualenv -p python2.7 env + virtualenv -p python3 env source env/bin/activate python -m pip install -e .[all] @@ -416,25 +438,3 @@ sphinxcontrib-napoleon:: Building internal API documentation:: python setup.py build_sphinx - - -Help!! Synapse eats all my RAM! -=============================== - -Synapse's architecture is quite RAM hungry currently - we deliberately -cache a lot of recent room data and metadata in RAM in order to speed up -common requests. We'll improve this in future, but for now the easiest -way to either reduce the RAM usage (at the risk of slowing things down) -is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment -variable. The default is 0.5, which can be decreased to reduce RAM usage -in memory constrained enviroments, or increased if performance starts to -degrade. - -Using `libjemalloc `_ can also yield a significant -improvement in overall amount, and especially in terms of giving back RAM -to the OS. To use it, the library must simply be put in the LD_PRELOAD -environment variable when launching Synapse. On Debian, this can be done -by installing the ``libjemalloc1`` package and adding this line to -``/etc/default/matrix-synapse``:: - - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 diff --git a/changelog.d/4621.misc b/changelog.d/4621.misc new file mode 100644 index 000000000000..60e45cb70c29 --- /dev/null +++ b/changelog.d/4621.misc @@ -0,0 +1 @@ +README updates From 362d80b7706c7bcd57ea5ee3e8e38b31c7d1ad8b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:28:08 +0000 Subject: [PATCH 51/66] Reduce user_ips bloat during dedupe background update The background update to remove duplicate rows naively deleted and reinserted the duplicates. For large tables with a large number of duplicates this causes a lot of bloat (with postgres), as the inserted rows are appended to the table, since deleted rows will not be overwritten until a VACUUM has happened. This should hopefully also help ensure that the query in the last batch uses the correct index, as inserting a large number of new rows without analyzing will upset the query planner. --- synapse/storage/client_ips.py | 63 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 091d7116c557..a20cc8231f45 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -167,12 +167,16 @@ def remove(txn): clause = "? <= last_seen AND last_seen < ?" args = (begin_last_seen, end_last_seen) + # (Note: The DISTINCT in the inner query is important to ensure that + # the COUNT(*) is accurate, otherwise double counting may happen due + # to the join effectively being a cross product) txn.execute( """ SELECT user_id, access_token, ip, - MAX(device_id), MAX(user_agent), MAX(last_seen) + MAX(device_id), MAX(user_agent), MAX(last_seen), + COUNT(*) FROM ( - SELECT user_id, access_token, ip + SELECT DISTINCT user_id, access_token, ip FROM user_ips WHERE {} ) c @@ -186,7 +190,60 @@ def remove(txn): # We've got some duplicates for i in res: - user_id, access_token, ip, device_id, user_agent, last_seen = i + user_id, access_token, ip, device_id, user_agent, last_seen, count = i + + # We want to delete the duplicates so we end up with only a + # single row. + # + # The naive way of doing this would be just to delete all rows + # and reinsert a constructed row. However, if there are a lot of + # duplicate rows this can cause the table to grow a lot, which + # can be problematic in two ways: + # 1. If user_ips is already large then this can cause the + # table to rapidly grow, potentially filling the disk. + # 2. Reinserting a lot of rows can confuse the table + # statistics for postgres, causing it to not use the + # correct indices for the query above, resulting in a full + # table scan. This is incredibly slow for large tables and + # can kill database performance. (This seems to mainly + # happen for the last query where the clause is simply `? < + # last_seen`) + # + # So instead we want to delete all but *one* of the duplicate + # rows. That is hard to do reliably, so we cheat and do a two + # step process: + # 1. Delete all rows with a last_seen strictly less than the + # max last_seen. This hopefully results in deleting all but + # one row the majority of the time, but there may be + # duplicate last_seen + # 2. If multiple rows remain, we fall back to the naive method + # and simply delete all rows and reinsert. + # + # Note that this relies on no new duplicate rows being inserted, + # but if that is happening then this entire process is futile + # anyway. + + # Do step 1: + + txn.execute( + """ + DELETE FROM user_ips + WHERE user_id = ? AND access_token = ? AND ip = ? AND last_seen < ? + """, + (user_id, access_token, ip, last_seen) + ) + if txn.rowcount == count - 1: + # We deleted all but one of the duplicate rows, i.e. there + # is exactly one remaining and so there is nothing left to + # do. + continue + elif txn.rowcount >= count: + raise Exception( + "We deleted more duplicate rows from 'user_ips' than expected", + ) + + # The previous step didn't delete enough rows, so we fallback to + # step 2: # Drop all the duplicates txn.execute( From 218cc071c5ef027361eddc72ef1863b43b2cdd67 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:37:00 +0000 Subject: [PATCH 52/66] Newsfile --- changelog.d/4626.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4626.misc diff --git a/changelog.d/4626.misc b/changelog.d/4626.misc new file mode 100644 index 000000000000..f1a57dcf9ada --- /dev/null +++ b/changelog.d/4626.misc @@ -0,0 +1 @@ +Improve 'user_ips' table deduplication background update From 483ba85c7a1a8ee9b7eebcc5c07d522c71229c9f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:55:27 +0000 Subject: [PATCH 53/66] Analyze user_ips before running deduplication Due to the table locks taken out by the naive upsert, the table statistics may be out of date. During deduplication it is important that the correct index is used as otherwise a full table scan may be incorrectly used, which can end up thrashing the database badly. --- synapse/storage/client_ips.py | 24 +++++++++++++++++++ .../schema/delta/53/user_ips_index.sql | 10 +++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 091d7116c557..6f81406269f8 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -65,6 +65,11 @@ def __init__(self, db_conn, hs): columns=["last_seen"], ) + self.register_background_update_handler( + "user_ips_analyze", + self._analyze_user_ip, + ) + self.register_background_update_handler( "user_ips_remove_dupes", self._remove_user_ip_dupes, @@ -108,6 +113,25 @@ def f(conn): yield self._end_background_update("user_ips_drop_nonunique_index") defer.returnValue(1) + @defer.inlineCallbacks + def _analyze_user_ip(self, progress, batch_size): + # Background update to analyze user_ips table before we run the + # deduplication background update. The table may not have been analyzed + # for ages due to the table locks. + # + # This will lock out the naive upserts to user_ips while it happens, but + # the analyze should be quick (28GB table takes ~10s) + def user_ips_analyze(txn): + txn.execute("ANALYZE user_ips") + + end_last_seen = yield self.runInteraction( + "user_ips_analyze", user_ips_analyze + ) + + yield self._end_background_update("user_ips_analyze") + + defer.returnValue(1) + @defer.inlineCallbacks def _remove_user_ip_dupes(self, progress, batch_size): # This works function works by scanning the user_ips table in batches diff --git a/synapse/storage/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/delta/53/user_ips_index.sql index 4ca346c1118e..b812c5794f91 100644 --- a/synapse/storage/schema/delta/53/user_ips_index.sql +++ b/synapse/storage/schema/delta/53/user_ips_index.sql @@ -13,9 +13,13 @@ * limitations under the License. */ --- delete duplicates + -- analyze user_ips, to help ensure the correct indices are used INSERT INTO background_updates (update_name, progress_json) VALUES - ('user_ips_remove_dupes', '{}'); + ('user_ips_analyze', '{}'); + +-- delete duplicates +INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES + ('user_ips_remove_dupes', '{}', 'user_ips_analyze'); -- add a new unique index to user_ips table INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES @@ -23,4 +27,4 @@ INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES -- drop the old original index INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES - ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index'); \ No newline at end of file + ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index'); From b2327eb9cb2c03037acd58b320063ec34968ea81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:58:36 +0000 Subject: [PATCH 54/66] Newsfile --- changelog.d/4627.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4627.misc diff --git a/changelog.d/4627.misc b/changelog.d/4627.misc new file mode 100644 index 000000000000..f1a57dcf9ada --- /dev/null +++ b/changelog.d/4627.misc @@ -0,0 +1 @@ +Improve 'user_ips' table deduplication background update From 495ea92350c4a3f6bd92cded5da939326b858d9b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 12:40:42 +0000 Subject: [PATCH 55/66] Fix pep8 --- synapse/storage/client_ips.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 6f81406269f8..cc23d7cdbeaa 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -124,7 +124,7 @@ def _analyze_user_ip(self, progress, batch_size): def user_ips_analyze(txn): txn.execute("ANALYZE user_ips") - end_last_seen = yield self.runInteraction( + yield self.runInteraction( "user_ips_analyze", user_ips_analyze ) From b18cd25e42711d99c62a098203f59f4e12f3fca7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:05:31 +0000 Subject: [PATCH 56/66] Fixup changelog entries --- changelog.d/4626.bugfix | 1 + changelog.d/4626.misc | 1 - changelog.d/4627.bugfix | 1 + changelog.d/4627.misc | 1 - 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4626.bugfix delete mode 100644 changelog.d/4626.misc create mode 100644 changelog.d/4627.bugfix delete mode 100644 changelog.d/4627.misc diff --git a/changelog.d/4626.bugfix b/changelog.d/4626.bugfix new file mode 100644 index 000000000000..cc71df44cac8 --- /dev/null +++ b/changelog.d/4626.bugfix @@ -0,0 +1 @@ +Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4626.misc b/changelog.d/4626.misc deleted file mode 100644 index f1a57dcf9ada..000000000000 --- a/changelog.d/4626.misc +++ /dev/null @@ -1 +0,0 @@ -Improve 'user_ips' table deduplication background update diff --git a/changelog.d/4627.bugfix b/changelog.d/4627.bugfix new file mode 100644 index 000000000000..cc71df44cac8 --- /dev/null +++ b/changelog.d/4627.bugfix @@ -0,0 +1 @@ +Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4627.misc b/changelog.d/4627.misc deleted file mode 100644 index f1a57dcf9ada..000000000000 --- a/changelog.d/4627.misc +++ /dev/null @@ -1 +0,0 @@ -Improve 'user_ips' table deduplication background update From d2fa7b7e99a50b47b73de7e673fbe12a6cc12adc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:22:25 +0000 Subject: [PATCH 57/66] Update changelog and version --- CHANGES.md | 46 ++++++++++++++++++++++++++++++++++++++++ changelog.d/3902.feature | 1 - changelog.d/4420.feature | 1 - changelog.d/4462.misc | 1 - changelog.d/4513.misc | 1 - changelog.d/4522.feature | 1 - changelog.d/4530.bugfix | 1 - changelog.d/4546.bugfix | 1 - changelog.d/4567.misc | 1 - changelog.d/4576.misc | 1 - changelog.d/4578.misc | 1 - changelog.d/4580.feature | 1 - changelog.d/4584.misc | 1 - changelog.d/4586.misc | 1 - changelog.d/4589.bugfix | 1 - changelog.d/4591.bugfix | 1 - changelog.d/4592.feature | 2 -- changelog.d/4607.misc | 1 - changelog.d/4608.bugfix | 1 - changelog.d/4611.misc | 1 - changelog.d/4613.feature | 1 - changelog.d/4614.feature | 1 - changelog.d/4615.feature | 1 - changelog.d/4616.misc | 1 - changelog.d/4617.feature | 1 - changelog.d/4618.bugfix | 1 - changelog.d/4619.misc | 1 - changelog.d/4621.misc | 1 - changelog.d/4625.bugfix | 1 - changelog.d/4626.bugfix | 1 - changelog.d/4627.bugfix | 1 - synapse/__init__.py | 2 +- 32 files changed, 47 insertions(+), 32 deletions(-) delete mode 100644 changelog.d/3902.feature delete mode 100644 changelog.d/4420.feature delete mode 100644 changelog.d/4462.misc delete mode 100644 changelog.d/4513.misc delete mode 100644 changelog.d/4522.feature delete mode 100644 changelog.d/4530.bugfix delete mode 100644 changelog.d/4546.bugfix delete mode 100644 changelog.d/4567.misc delete mode 100644 changelog.d/4576.misc delete mode 100644 changelog.d/4578.misc delete mode 100644 changelog.d/4580.feature delete mode 100644 changelog.d/4584.misc delete mode 100644 changelog.d/4586.misc delete mode 100644 changelog.d/4589.bugfix delete mode 100644 changelog.d/4591.bugfix delete mode 100644 changelog.d/4592.feature delete mode 100644 changelog.d/4607.misc delete mode 100644 changelog.d/4608.bugfix delete mode 100644 changelog.d/4611.misc delete mode 100644 changelog.d/4613.feature delete mode 100644 changelog.d/4614.feature delete mode 100644 changelog.d/4615.feature delete mode 100644 changelog.d/4616.misc delete mode 100644 changelog.d/4617.feature delete mode 100644 changelog.d/4618.bugfix delete mode 100644 changelog.d/4619.misc delete mode 100644 changelog.d/4621.misc delete mode 100644 changelog.d/4625.bugfix delete mode 100644 changelog.d/4626.bugfix delete mode 100644 changelog.d/4627.bugfix diff --git a/CHANGES.md b/CHANGES.md index e330aea9e35f..5120631fb6fd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,49 @@ +Synapse 0.99.1rc1 (2019-02-12) +============================== + +Features +-------- + +- Include m.room.encryption on invites by default ([\#3902](https://github.com/matrix-org/synapse/issues/3902)) +- Federation OpenID listener resource can now be activated even if federation is disabled ([\#4420](https://github.com/matrix-org/synapse/issues/4420)) +- Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. ([\#4522](https://github.com/matrix-org/synapse/issues/4522)) +- Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580)) +- Allow the "unavailable" presence status for /sync. + This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592)) +- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617)) +- The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614)) + + +Bugfixes +-------- + +- Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530)) +- Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546)) +- Synapse is now tolerant of the tls_fingerprints option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) +- Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591)) +- Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608)) +- Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618)) +- fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) +- Fix performance of 'user_ips' table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) + + +Internal Changes +---------------- + +- Change the user directory state query to use a filtered call to the db instead of a generic one. ([\#4462](https://github.com/matrix-org/synapse/issues/4462)) +- Reject federation transactions if they include more than 50 PDUs or 100 EDUs. ([\#4513](https://github.com/matrix-org/synapse/issues/4513)) +- Reduce duplication of ``synapse.app`` code. ([\#4567](https://github.com/matrix-org/synapse/issues/4567)) +- Fix docker upload job to push -py2 images. ([\#4576](https://github.com/matrix-org/synapse/issues/4576)) +- Add port configuration information to ACME instructions. ([\#4578](https://github.com/matrix-org/synapse/issues/4578)) +- Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584)) +- Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586)) +- Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607)) +- Move ClientTLSOptionsFactory init out of refresh_certificates ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) +- Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616)) +- Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619)) +- README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621)) + + Synapse 0.99.0 (2019-02-05) =========================== diff --git a/changelog.d/3902.feature b/changelog.d/3902.feature deleted file mode 100644 index eb8d9f239370..000000000000 --- a/changelog.d/3902.feature +++ /dev/null @@ -1 +0,0 @@ -Include m.room.encryption on invites by default diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature deleted file mode 100644 index 05e777c62451..000000000000 --- a/changelog.d/4420.feature +++ /dev/null @@ -1 +0,0 @@ -Federation OpenID listener resource can now be activated even if federation is disabled diff --git a/changelog.d/4462.misc b/changelog.d/4462.misc deleted file mode 100644 index 03a4d7ae1cbc..000000000000 --- a/changelog.d/4462.misc +++ /dev/null @@ -1 +0,0 @@ -Change the user directory state query to use a filtered call to the db instead of a generic one. \ No newline at end of file diff --git a/changelog.d/4513.misc b/changelog.d/4513.misc deleted file mode 100644 index 1f64a96465a2..000000000000 --- a/changelog.d/4513.misc +++ /dev/null @@ -1 +0,0 @@ -Reject federation transactions if they include more than 50 PDUs or 100 EDUs. \ No newline at end of file diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature deleted file mode 100644 index ef18daf60bd4..000000000000 --- a/changelog.d/4522.feature +++ /dev/null @@ -1 +0,0 @@ -Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix deleted file mode 100644 index d010af927e67..000000000000 --- a/changelog.d/4530.bugfix +++ /dev/null @@ -1 +0,0 @@ -Copy over room federation ability on room upgrade. \ No newline at end of file diff --git a/changelog.d/4546.bugfix b/changelog.d/4546.bugfix deleted file mode 100644 index 056f2848edb9..000000000000 --- a/changelog.d/4546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix noisy "twisted.internet.task.TaskStopped" errors in logs diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc deleted file mode 100644 index 96a2e0aefc16..000000000000 --- a/changelog.d/4567.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce duplication of ``synapse.app`` code. diff --git a/changelog.d/4576.misc b/changelog.d/4576.misc deleted file mode 100644 index 94b1ade2e387..000000000000 --- a/changelog.d/4576.misc +++ /dev/null @@ -1 +0,0 @@ -Fix docker upload job to push -py2 images. diff --git a/changelog.d/4578.misc b/changelog.d/4578.misc deleted file mode 100644 index d1c006bb6b19..000000000000 --- a/changelog.d/4578.misc +++ /dev/null @@ -1 +0,0 @@ -Add port configuration information to ACME instructions. \ No newline at end of file diff --git a/changelog.d/4580.feature b/changelog.d/4580.feature deleted file mode 100644 index a2a5a77dbebe..000000000000 --- a/changelog.d/4580.feature +++ /dev/null @@ -1 +0,0 @@ -Add ability to update backup versions \ No newline at end of file diff --git a/changelog.d/4584.misc b/changelog.d/4584.misc deleted file mode 100644 index 4dec2e2b5ccb..000000000000 --- a/changelog.d/4584.misc +++ /dev/null @@ -1 +0,0 @@ -Update MSC1711 FAQ to calrify .well-known usage diff --git a/changelog.d/4586.misc b/changelog.d/4586.misc deleted file mode 100644 index 37af371ccfe4..000000000000 --- a/changelog.d/4586.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up default listener configuration diff --git a/changelog.d/4589.bugfix b/changelog.d/4589.bugfix deleted file mode 100644 index d5783f46e876..000000000000 --- a/changelog.d/4589.bugfix +++ /dev/null @@ -1 +0,0 @@ -Synapse is now tolerant of the tls_fingerprints option being None or not specified. diff --git a/changelog.d/4591.bugfix b/changelog.d/4591.bugfix deleted file mode 100644 index 628bbb6d818f..000000000000 --- a/changelog.d/4591.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 'no unique or exclusion constraint' error diff --git a/changelog.d/4592.feature b/changelog.d/4592.feature deleted file mode 100644 index 112005ded4af..000000000000 --- a/changelog.d/4592.feature +++ /dev/null @@ -1,2 +0,0 @@ -Allow the "unavailable" presence status for /sync. -This change makes Synapse compliant with r0.4.0 of the Client-Server specification. diff --git a/changelog.d/4607.misc b/changelog.d/4607.misc deleted file mode 100644 index 160a82437814..000000000000 --- a/changelog.d/4607.misc +++ /dev/null @@ -1 +0,0 @@ -Clarifications for reverse proxy docs diff --git a/changelog.d/4608.bugfix b/changelog.d/4608.bugfix deleted file mode 100644 index e331a362c473..000000000000 --- a/changelog.d/4608.bugfix +++ /dev/null @@ -1 +0,0 @@ -Transfer Server ACLs on room upgrade. \ No newline at end of file diff --git a/changelog.d/4611.misc b/changelog.d/4611.misc deleted file mode 100644 index d2e0a05daa39..000000000000 --- a/changelog.d/4611.misc +++ /dev/null @@ -1 +0,0 @@ -Move ClientTLSOptionsFactory init out of refresh_certificates diff --git a/changelog.d/4613.feature b/changelog.d/4613.feature deleted file mode 100644 index 098f906af2e8..000000000000 --- a/changelog.d/4613.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4614.feature b/changelog.d/4614.feature deleted file mode 100644 index 18e16dbc7bd3..000000000000 --- a/changelog.d/4614.feature +++ /dev/null @@ -1 +0,0 @@ -The default configuration no longer requires TLS certificates. diff --git a/changelog.d/4615.feature b/changelog.d/4615.feature deleted file mode 100644 index 098f906af2e8..000000000000 --- a/changelog.d/4615.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4616.misc b/changelog.d/4616.misc deleted file mode 100644 index ee79e208e3f2..000000000000 --- a/changelog.d/4616.misc +++ /dev/null @@ -1 +0,0 @@ -Fail cleanly if listener config lacks a 'port' diff --git a/changelog.d/4617.feature b/changelog.d/4617.feature deleted file mode 100644 index 098f906af2e8..000000000000 --- a/changelog.d/4617.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4618.bugfix b/changelog.d/4618.bugfix deleted file mode 100644 index 22115a020e96..000000000000 --- a/changelog.d/4618.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix failure to start when not TLS certificate was given even if TLS was disabled. diff --git a/changelog.d/4619.misc b/changelog.d/4619.misc deleted file mode 100644 index 886fedf19801..000000000000 --- a/changelog.d/4619.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant entries from docker config diff --git a/changelog.d/4621.misc b/changelog.d/4621.misc deleted file mode 100644 index 60e45cb70c29..000000000000 --- a/changelog.d/4621.misc +++ /dev/null @@ -1 +0,0 @@ -README updates diff --git a/changelog.d/4625.bugfix b/changelog.d/4625.bugfix deleted file mode 100644 index 3dc0ecf24c4a..000000000000 --- a/changelog.d/4625.bugfix +++ /dev/null @@ -1 +0,0 @@ -fix self-signed cert notice from generate-config. diff --git a/changelog.d/4626.bugfix b/changelog.d/4626.bugfix deleted file mode 100644 index cc71df44cac8..000000000000 --- a/changelog.d/4626.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4627.bugfix b/changelog.d/4627.bugfix deleted file mode 100644 index cc71df44cac8..000000000000 --- a/changelog.d/4627.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix performance of 'user_ips' table deduplication background update diff --git a/synapse/__init__.py b/synapse/__init__.py index 048d6e572fb3..c211cb4e6f47 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ except ImportError: pass -__version__ = "0.99.0" +__version__ = "0.99.1rc1" From 19818d66af9d67731e19020efae3a244d276b2c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:25:05 +0000 Subject: [PATCH 58/66] Fixup changelog --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5120631fb6fd..cc629978c241 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,12 +19,12 @@ Bugfixes - Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530)) - Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546)) -- Synapse is now tolerant of the tls_fingerprints option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) +- Synapse is now tolerant of the `tls_fingerprints` option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) - Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591)) - Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608)) - Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618)) -- fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) -- Fix performance of 'user_ips' table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) +- Fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) +- Fix performance of `user_ips` table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) Internal Changes @@ -38,7 +38,7 @@ Internal Changes - Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584)) - Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586)) - Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607)) -- Move ClientTLSOptionsFactory init out of refresh_certificates ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) +- Move ClientTLSOptionsFactory init out of `refresh_certificates` ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) - Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616)) - Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619)) - README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621)) From e3a0300431e6f641297cb90e936cb2d35fb51850 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 11:48:56 +0000 Subject: [PATCH 59/66] Special-case the default bind_addresses for metrics listener turns out it doesn't really support ipv6, so let's hack around that by only listening on ipv4 by default. --- synapse/app/_base.py | 5 ++--- synapse/config/server.py | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b0ca312e22a..1d2c3339ffa5 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -153,9 +153,8 @@ def listen_metrics(bind_addresses, port): from prometheus_client import start_http_server for host in bind_addresses: - reactor.callInThread(start_http_server, int(port), - addr=host, registry=RegistryProxy) - logger.info("Metrics now reporting on %s:%d", host, port) + logger.info("Starting metrics listener on %s:%d", host, port) + start_http_server(port, addr=host, registry=RegistryProxy) def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): diff --git a/synapse/config/server.py b/synapse/config/server.py index c5c3aac8edb3..93a30e4cfa26 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,11 @@ def read_config(self, config): # if we still have an empty list of addresses, use the default list if not bind_addresses: - bind_addresses.extend(DEFAULT_BIND_ADDRESSES) + if listener['type'] == 'metrics': + # the metrics listener doesn't support IPv6 + bind_addresses.append('0.0.0.0') + else: + bind_addresses.extend(DEFAULT_BIND_ADDRESSES) self.listeners.append(listener) From 2a5a15aff8f8d3146df52d5421a6c818637eb629 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 11:53:43 +0000 Subject: [PATCH 60/66] Improve logging around listening services I wanted to bring listen_tcp into line with listen_ssl in terms of returning a list of ports, and wanted to check that was a safe thing to do - hence the logging in `refresh_certificate`. Also, pull the 'Synapse now listening' message up to homeserver.py, because it was being duplicated everywhere else. --- synapse/app/_base.py | 23 ++++++++++++++--------- synapse/app/homeserver.py | 8 ++++++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 1d2c3339ffa5..c53b64493266 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -162,21 +162,23 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): Create a TCP socket for a port and several addresses Returns: - list (empty) + list of twisted.internet.tcp.Port listening for TLS connections """ + r = [] for address in bind_addresses: try: - reactor.listenTCP( - port, - factory, - backlog, - address + r.append( + reactor.listenTCP( + port, + factory, + backlog, + address + ) ) except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) - logger.info("Synapse now listening on TCP port %d", port) - return [] + return r def listen_ssl( @@ -203,7 +205,6 @@ def listen_ssl( except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) - logger.info("Synapse now listening on port %d (TLS)", port) return r @@ -229,6 +230,10 @@ def refresh_certificate(hs): # requests. This factory attribute is public but missing from # Twisted's documentation. if isinstance(i.factory, TLSMemoryBIOFactory): + addr = i.getHost() + logger.info( + "Replacing TLS context factory on [%s]:%i", addr.host, addr.port, + ) # We want to replace TLS factories with a new one, with the new # TLS configuration. We do this by reaching in and pulling out # the wrappedFactory, and then re-wrapping it. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index dbd98d394f97..2b008e3402a2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -121,7 +121,7 @@ def _listener_http(self, config, listener_config): root_resource = create_resource_tree(resources, root_resource) if tls: - return listen_ssl( + ports = listen_ssl( bind_addresses, port, SynapseSite( @@ -134,9 +134,10 @@ def _listener_http(self, config, listener_config): self.tls_server_context_factory, reactor=self.get_reactor(), ) + logger.info("Synapse now listening on TCP port %d (TLS)", port) else: - return listen_tcp( + ports = listen_tcp( bind_addresses, port, SynapseSite( @@ -148,6 +149,9 @@ def _listener_http(self, config, listener_config): ), reactor=self.get_reactor(), ) + logger.info("Synapse now listening on TCP port %d", port) + + return ports def _configure_named_resource(self, name, compress=False): """Build a resource map for a named resource From 767686af48ecbf5fe999830e0b0ce13b7fd46b1b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 11:59:04 +0000 Subject: [PATCH 61/66] Use `listen_tcp` for the replication listener Fixes the "can't listen on 0.0.0.0" error. Also makes it more consistent with what we do elsewhere. --- synapse/app/homeserver.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2b008e3402a2..dbd9a8387725 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -266,14 +266,14 @@ def start_listening(self, listeners): ) ) elif listener["type"] == "replication": - bind_addresses = listener["bind_addresses"] - for address in bind_addresses: - factory = ReplicationStreamProtocolFactory(self) - server_listener = reactor.listenTCP( - listener["port"], factory, interface=address - ) + services = listen_tcp( + listener["bind_addresses"], + listener["port"], + ReplicationStreamProtocolFactory(self), + ) + for s in services: reactor.addSystemEventTrigger( - "before", "shutdown", server_listener.stopListening, + "before", "shutdown", s.stopListening, ) elif listener["type"] == "metrics": if not self.get_config().enable_metrics: From 2d0e0a40441f299f06033c20d7ae801100eb0028 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 12:00:34 +0000 Subject: [PATCH 62/66] changelog --- changelog.d/4636.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4636.bugfix diff --git a/changelog.d/4636.bugfix b/changelog.d/4636.bugfix new file mode 100644 index 000000000000..7607aa1d537d --- /dev/null +++ b/changelog.d/4636.bugfix @@ -0,0 +1 @@ +Fix errors when using default bind_addresses with replication/metrics listeners. \ No newline at end of file From 309f3bb322ea407b29651bd31c12183ea81b05b1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Feb 2019 13:24:27 +0000 Subject: [PATCH 63/66] Update synapse/app/_base.py Co-Authored-By: richvdh <1389908+richvdh@users.noreply.github.com> --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index c53b64493266..73ca52bd8cb3 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -162,7 +162,7 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): Create a TCP socket for a port and several addresses Returns: - list of twisted.internet.tcp.Port listening for TLS connections + list[twisted.internet.tcp.Port]: listening for TCP connections """ r = [] for address in bind_addresses: From 3bc238629eb61e6a3f6c652e3ddae3261179d624 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 14:46:18 +0000 Subject: [PATCH 64/66] 0.99.1rc2 --- CHANGES.md | 9 +++++++++ changelog.d/4636.bugfix | 1 - synapse/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/4636.bugfix diff --git a/CHANGES.md b/CHANGES.md index cc629978c241..d0b3437b2d55 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 0.99.1rc2 (2019-02-13) +============================== + +Bugfixes +-------- + +- Fix errors when using default bind_addresses with replication/metrics listeners. ([\#4636](https://github.com/matrix-org/synapse/issues/4636)) + + Synapse 0.99.1rc1 (2019-02-12) ============================== diff --git a/changelog.d/4636.bugfix b/changelog.d/4636.bugfix deleted file mode 100644 index 7607aa1d537d..000000000000 --- a/changelog.d/4636.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix errors when using default bind_addresses with replication/metrics listeners. \ No newline at end of file diff --git a/synapse/__init__.py b/synapse/__init__.py index c211cb4e6f47..4629fa76ec4b 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ except ImportError: pass -__version__ = "0.99.1rc1" +__version__ = "0.99.1rc2" From a214ba93e0770f87c1a13a62375a11fd6c60bf64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B0=D0=BC=D1=98=D0=B0=D0=BD=20=D0=93=D0=B5=D0=BE?= =?UTF-8?q?=D1=80=D0=B3=D0=B8=D0=B5=D0=B2=D1=81=D0=BA=D0=B8?= Date: Thu, 14 Feb 2019 14:44:22 +0100 Subject: [PATCH 65/66] implement `reload` by sending the HUP signal (#4622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * implement `reload` by sending the HUP signal According to the 0.99 release info* synapse now uses the HUP signal to reload certificates: > Synapse will now reload TLS certificates from disk upon SIGHUP. (#4495, #4524) So the matrix-synapse.service unit file should include a reload directive. Signed-off-by: Дамјан Георгиевски --- debian/changelog | 6 ++++++ debian/matrix-synapse.service | 1 + 2 files changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 04b5d69053c7..d4b9ed8ed7f1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (0.99.1) UNRELEASED; urgency=medium + + * Added ExecReload= in service unit file to send a HUP signal + + -- Damjan Georgievski Thu, 14 Feb 2019 12:53:13 +0000 + matrix-synapse-py3 (0.99.0) stable; urgency=medium * New synapse release 0.99.0 diff --git a/debian/matrix-synapse.service b/debian/matrix-synapse.service index 2e9cd83b5fc3..942e4b83fef9 100644 --- a/debian/matrix-synapse.service +++ b/debian/matrix-synapse.service @@ -8,6 +8,7 @@ WorkingDirectory=/var/lib/matrix-synapse EnvironmentFile=/etc/default/matrix-synapse ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ +ExecReload=/bin/kill -HUP $MAINPID Restart=always RestartSec=3 From 06cd757ae718ca4ec01c8e6480547f48f9410cfa Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Feb 2019 14:24:24 +0000 Subject: [PATCH 66/66] 0.99.1 --- CHANGES.md | 15 +++------------ debian/changelog | 8 ++++++-- synapse/__init__.py | 4 ++-- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d0b3437b2d55..872ac440b53b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,5 @@ -Synapse 0.99.1rc2 (2019-02-13) -============================== - -Bugfixes --------- - -- Fix errors when using default bind_addresses with replication/metrics listeners. ([\#4636](https://github.com/matrix-org/synapse/issues/4636)) - - -Synapse 0.99.1rc1 (2019-02-12) -============================== +Synapse 0.99.1 (2019-02-14) +=========================== Features -------- @@ -19,7 +10,7 @@ Features - Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580)) - Allow the "unavailable" presence status for /sync. This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592)) -- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617)) +- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617), [\#4636](https://github.com/matrix-org/synapse/issues/4636)) - The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614)) diff --git a/debian/changelog b/debian/changelog index d4b9ed8ed7f1..bc1ba153e34e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ -matrix-synapse-py3 (0.99.1) UNRELEASED; urgency=medium +matrix-synapse-py3 (0.99.1) stable; urgency=medium + [ Damjan Georgievski ] * Added ExecReload= in service unit file to send a HUP signal - -- Damjan Georgievski Thu, 14 Feb 2019 12:53:13 +0000 + [ Synapse Packaging team ] + * New synapse release 0.99.1 + + -- Synapse Packaging team Thu, 14 Feb 2019 14:12:26 +0000 matrix-synapse-py3 (0.99.0) stable; urgency=medium diff --git a/synapse/__init__.py b/synapse/__init__.py index 4629fa76ec4b..f7bac0ea4e10 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd +# Copyright 2018-9 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,4 +27,4 @@ except ImportError: pass -__version__ = "0.99.1rc2" +__version__ = "0.99.1"