pyOpenSSL DTLS cookie callback buffer overflow

@@ -19,6 +19,7 @@ Deprecations:
 Changes:
 ^^^^^^^^
 
+- Properly raise an error if a DTLS cookie callback returned a cookie longer than ``DTLS1_COOKIE_LENGTH`` bytes. Previously this would result in a buffer-overflow.
 - Added ``OpenSSL.SSL.Connection.get_group_name`` to determine which group name was negotiated.
 - ``Context.set_tlsext_servername_callback`` now handles exceptions raised in the callback by calling ``sys.excepthook`` and returning a fatal TLS alert. Previously, exceptions were silently swallowed and the handshake would proceed as if the callback had succeeded.
 

@@ -717,11 +717,18 @@ class _CookieGenerateCallbackHelper(_CallbackExceptionHelper):
     def __init__(self, callback: _CookieGenerateCallback) -> None:
         _CallbackExceptionHelper.__init__(self)
 
+        max_cookie_len = getattr(_lib, "DTLS1_COOKIE_LENGTH", 255)
+
         @wraps(callback)
         def wrapper(ssl, out, outlen):  # type: ignore[no-untyped-def]
             try:
                 conn = Connection._reverse_mapping[ssl]
                 cookie = callback(conn)
+                if len(cookie) > max_cookie_len:
+                    raise ValueError(
+                        f"Cookie too long (got {len(cookie)} bytes, "
+                        f"max {max_cookie_len})"
+                    )
                 out[0 : len(cookie)] = cookie
                 outlen[0] = len(cookie)
                 return 1

@@ -5106,6 +5106,44 @@ def test_it_works_at_all(self) -> None:
     def test_it_works_with_srtp(self) -> None:
         self._test_handshake_and_data(srtp_profile=b"SRTP_AES128_CM_SHA1_80")
 
+    def test_cookie_generate_too_long(self) -> None:
+        s_ctx = Context(DTLS_METHOD)
+
+        def generate_cookie(ssl: Connection) -> bytes:
+            return b"\x00" * 256
+
+        def verify_cookie(ssl: Connection, cookie: bytes) -> bool:
+            return True
+
+        s_ctx.set_cookie_generate_callback(generate_cookie)
+        s_ctx.set_cookie_verify_callback(verify_cookie)
+        s_ctx.use_privatekey(load_privatekey(FILETYPE_PEM, server_key_pem))
+        s_ctx.use_certificate(load_certificate(FILETYPE_PEM, server_cert_pem))
+        s_ctx.set_options(OP_NO_QUERY_MTU)
+        s = Connection(s_ctx)
+        s.set_accept_state()
+
+        c_ctx = Context(DTLS_METHOD)
+        c_ctx.set_options(OP_NO_QUERY_MTU)
+        c = Connection(c_ctx)
+        c.set_connect_state()
+
+        c.set_ciphertext_mtu(1500)
+        s.set_ciphertext_mtu(1500)
+
+        # Client sends ClientHello
+        try:
+            c.do_handshake()
+        except SSL.WantReadError:
+            pass
+        chunk = c.bio_read(self.LARGE_BUFFER)
+        s.bio_write(chunk)
+
+        # Server tries DTLSv1_listen, which triggers cookie generation.
+        # The oversized cookie should raise ValueError.
+        with pytest.raises(ValueError, match="Cookie too long"):
+            s.DTLSv1_listen()
+
     def test_timeout(self, monkeypatch: pytest.MonkeyPatch) -> None:
         c_ctx = Context(DTLS_METHOD)
         c = Connection(c_ctx)