Denial of service attack via memory exhaustion

@@ -23,9 +23,15 @@
 from twisted.web._newclient import ResponseDone
 from twisted.web.http import PotentialDataLoss
 from twisted.web.iweb import UNKNOWN_LENGTH
+from twisted.web import server
+
 
 logger = logging.getLogger(__name__)
 
+# Arbitrarily limited to 512 KiB.
+MAX_REQUEST_SIZE = 512 * 1024
+
+
 class SslComponents:
     def __init__(self, sydent):
         self.sydent = sydent
@@ -61,7 +67,7 @@ def makeTrustRoot(self):
                 fp = open(caCertFilename)
                 caCert = twisted.internet.ssl.Certificate.loadPEM(fp.read())
                 fp.close()
-            except:
+            except Exception:
                 logger.warn("Failed to open CA cert file %s", caCertFilename)
                 raise
             logger.warn("Using custom CA cert file: %s", caCertFilename)
@@ -70,7 +76,6 @@ def makeTrustRoot(self):
             return twisted.internet.ssl.OpenSSLDefaultPaths()
 
 
-
 class BodyExceededMaxSize(Exception):
     """The maximum allowed size of the HTTP body was exceeded."""
 
@@ -123,7 +128,7 @@ def dataReceived(self, data) -> None:
             # discarded anyway.
             self.transport.abortConnection()
 
-    def connectionLost(self, reason = connectionDone) -> None:
+    def connectionLost(self, reason=connectionDone) -> None:
         # If the maximum size was already exceeded, there's nothing to do.
         if self.deferred.called:
             return
@@ -163,3 +168,15 @@ def read_body_with_max_size(response, max_size):
 
     response.deliverBody(_ReadBodyWithMaxSizeProtocol(d, max_size))
     return d
+
+
+class SizeLimitingRequest(server.Request):
+    def handleContentChunk(self, data):
+        if self.content.tell() + len(data) > MAX_REQUEST_SIZE:
+            logger.info(
+                "Aborting connection from %s because the request exceeds maximum size",
+                self.client.host)
+            self.transport.abortConnection()
+            return
+
+        return super().handleContentChunk(data)

@@ -29,6 +29,7 @@
 from sydent.http.servlets.authenticated_unbind_threepid_servlet import (
     AuthenticatedUnbindThreePidServlet,
 )
+from sydent.http.httpcommon import SizeLimitingRequest
 
 logger = logging.getLogger(__name__)
 
@@ -129,6 +130,7 @@ def __init__(self, sydent):
         v2.putChild(b'hash_details', self.sydent.servlets.hash_details)
 
         self.factory = Site(root)
+        self.factory.requestFactory = SizeLimitingRequest
         self.factory.displayTracebacks = False
 
     def setup(self):

@@ -69,7 +69,7 @@ def _getKeysForServer(self, server_name):
                 defer.returnValue(self.cache[server_name]['verify_keys'])
 
         client = FederationHttpClient(self.sydent)
-        result = yield client.get_json("matrix://%s/_matrix/key/v2/server/" % server_name)
+        result = yield client.get_json("matrix://%s/_matrix/key/v2/server/" % server_name, 1024 * 50)
         if 'verify_keys' not in result:
             raise SignatureVerifyException("No key found in response")
 

@@ -25,6 +25,7 @@
 from sydent.http.matrixfederationagent import MatrixFederationAgent
 
 from sydent.http.federation_tls_options import ClientTLSOptionsFactory
+from sydent.http.httpcommon import BodyExceededMaxSize, read_body_with_max_size
 
 logger = logging.getLogger(__name__)
 
@@ -34,12 +35,15 @@ class HTTPClient(object):
     requests.
     """
     @defer.inlineCallbacks
-    def get_json(self, uri):
+    def get_json(self, uri, max_size = None):
         """Make a GET request to an endpoint returning JSON and parse result
 
         :param uri: The URI to make a GET request to.
         :type uri: unicode
 
+        :param max_size: The maximum size (in bytes) to allow as a response.
+        :type max_size: int
+
         :return: A deferred containing JSON parsed into a Python object.
         :rtype: twisted.internet.defer.Deferred[dict[any, any]]
         """
@@ -49,7 +53,7 @@ def get_json(self, uri):
             b"GET",
             uri.encode("utf8"),
         )
-        body = yield readBody(response)
+        body = yield read_body_with_max_size(response, max_size)
         try:
             # json.loads doesn't allow bytes in Python 3.5
             json_body = json.loads(body.decode("UTF-8"))
@@ -94,7 +98,11 @@ def post_json_get_nothing(self, uri, post_json, opts):
         # Ensure the body object is read otherwise we'll leak HTTP connections
         # as per
         # https://twistedmatrix.com/documents/current/web/howto/client.html
-        yield readBody(response)
+        try:
+            # TODO Will this cause the server to think the request was a failure?
+            yield read_body_with_max_size(response, 0)
+        except BodyExceededMaxSize:
+            pass
 
         defer.returnValue(response)
 

@@ -51,6 +51,7 @@ def render_POST(self, request):
             "matrix://%s/_matrix/federation/v1/openid/userinfo?access_token=%s" % (
                 args['matrix_server_name'], urllib.parse.quote(args['access_token']),
             ),
+            1024 * 5,
         )
         if 'sub' not in result:
             raise Exception("Invalid response from homeserver")

@@ -15,8 +15,14 @@
 # limitations under the License.
 
 import logging
+from io import BytesIO
 
 import twisted.internet.ssl
+from twisted.internet import defer, protocol
+from twisted.internet.protocol import connectionDone
+from twisted.web._newclient import ResponseDone
+from twisted.web.http import PotentialDataLoss
+from twisted.web.iweb import UNKNOWN_LENGTH
 
 logger = logging.getLogger(__name__)
 
@@ -62,3 +68,98 @@ def makeTrustRoot(self):
             return twisted.internet._sslverify.OpenSSLCertificateAuthorities([caCert.original])
         else:
             return twisted.internet.ssl.OpenSSLDefaultPaths()
+
+
+
+class BodyExceededMaxSize(Exception):
+    """The maximum allowed size of the HTTP body was exceeded."""
+
+
+class _DiscardBodyWithMaxSizeProtocol(protocol.Protocol):
+    """A protocol which immediately errors upon receiving data."""
+
+    def __init__(self, deferred):
+        self.deferred = deferred
+
+    def _maybe_fail(self):
+        """
+        Report a max size exceed error and disconnect the first time this is called.
+        """
+        if not self.deferred.called:
+            self.deferred.errback(BodyExceededMaxSize())
+            # Close the connection (forcefully) since all the data will get
+            # discarded anyway.
+            self.transport.abortConnection()
+
+    def dataReceived(self, data) -> None:
+        self._maybe_fail()
+
+    def connectionLost(self, reason) -> None:
+        self._maybe_fail()
+
+
+class _ReadBodyWithMaxSizeProtocol(protocol.Protocol):
+    """A protocol which reads body to a stream, erroring if the body exceeds a maximum size."""
+
+    def __init__(self, deferred, max_size):
+        self.stream = BytesIO()
+        self.deferred = deferred
+        self.length = 0
+        self.max_size = max_size
+
+    def dataReceived(self, data) -> None:
+        # If the deferred was called, bail early.
+        if self.deferred.called:
+            return
+
+        self.stream.write(data)
+        self.length += len(data)
+        # The first time the maximum size is exceeded, error and cancel the
+        # connection. dataReceived might be called again if data was received
+        # in the meantime.
+        if self.max_size is not None and self.length >= self.max_size:
+            self.deferred.errback(BodyExceededMaxSize())
+            # Close the connection (forcefully) since all the data will get
+            # discarded anyway.
+            self.transport.abortConnection()
+
+    def connectionLost(self, reason = connectionDone) -> None:
+        # If the maximum size was already exceeded, there's nothing to do.
+        if self.deferred.called:
+            return
+
+        if reason.check(ResponseDone):
+            self.deferred.callback(self.stream.getvalue())
+        elif reason.check(PotentialDataLoss):
+            # stolen from https://github.com/twisted/treq/pull/49/files
+            # http://twistedmatrix.com/trac/ticket/4840
+            self.deferred.callback(self.stream.getvalue())
+        else:
+            self.deferred.errback(reason)
+
+
+def read_body_with_max_size(response, max_size):
+    """
+    Read a HTTP response body to a file-object. Optionally enforcing a maximum file size.
+
+    If the maximum file size is reached, the returned Deferred will resolve to a
+    Failure with a BodyExceededMaxSize exception.
+
+    Args:
+        response: The HTTP response to read from.
+        max_size: The maximum file size to allow.
+
+    Returns:
+        A Deferred which resolves to the read body.
+    """
+    d = defer.Deferred()
+
+    # If the Content-Length header gives a size larger than the maximum allowed
+    # size, do not bother downloading the body.
+    if max_size is not None and response.length != UNKNOWN_LENGTH:
+        if response.length > max_size:
+            response.deliverBody(_DiscardBodyWithMaxSizeProtocol(d))
+            return d
+
+    response.deliverBody(_ReadBodyWithMaxSizeProtocol(d, max_size))
+    return d

@@ -26,11 +26,12 @@
 from twisted.internet import defer
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.internet.interfaces import IStreamClientEndpoint
-from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent, readBody
+from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent
 from twisted.web.http import stringToDatetime
 from twisted.web.http_headers import Headers
 from twisted.web.iweb import IAgent
 
+from sydent.http.httpcommon import BodyExceededMaxSize, read_body_with_max_size
 from sydent.http.srvresolver import SrvResolver, pick_server_from_list
 from sydent.util.ttlcache import TTLCache
 
@@ -46,6 +47,9 @@
 # cap for .well-known cache period
 WELL_KNOWN_MAX_CACHE_PERIOD = 48 * 3600
 
+# The maximum size (in bytes) to allow a well-known file to be.
+WELL_KNOWN_MAX_SIZE = 50 * 1024  # 50 KiB
+
 logger = logging.getLogger(__name__)
 well_known_cache = TTLCache('well-known')
 
@@ -316,7 +320,7 @@ def _do_get_well_known(self, server_name):
         logger.info("Fetching %s", uri_str)
         try:
             response = yield self._well_known_agent.request(b"GET", uri)
-            body = yield readBody(response)
+            body = yield read_body_with_max_size(response, WELL_KNOWN_MAX_SIZE)
             if response.code != 200:
                 raise Exception("Non-200 response %s" % (response.code, ))
 
@@ -334,6 +338,7 @@ def _do_get_well_known(self, server_name):
             cache_period = WELL_KNOWN_INVALID_CACHE_PERIOD
             cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER)
             defer.returnValue((None, cache_period))
+            return
 
         result = parsed_body["m.server"].encode("ascii")