High severityNVD Advisory· Published Feb 8, 2021· Updated Aug 3, 2024
Regular Expression Denial of Service in httplib2
CVE-2021-21240
Description
httplib2 is a comprehensive HTTP client library for Python. In httplib2 before version 0.19.0, a malicious server which responds with long series of "\xa0" characters in the "www-authenticate" header may cause Denial of Service (CPU burn while parsing header) of the httplib2 client accessing said server. This is fixed in version 0.19.0 which contains a new implementation of auth headers parsing using the pyparsing library.
Affected packages
Versions sourced from the GitHub Security Advisory.
| Package | Affected versions | Patched versions |
|---|---|---|
httplib2PyPI | < 0.19.0 | 0.19.0 |
Affected products
1- Range: < 0.19.0
Patches
1bd9ee252c8f0parse auth headers using pyparsing instead of regexp
12 files changed · +579 −1032
python2/httplib2/auth.py+61 −0 added@@ -0,0 +1,61 @@ +import base64 +import re + +import pyparsing as pp + +from .error import * + +UNQUOTE_PAIRS = re.compile(r"\\(.)") +unquote = lambda s, l, t: UNQUOTE_PAIRS.sub(r"\1", t[0][1:-1]) + +# https://tools.ietf.org/html/rfc7235#section-1.2 +# https://tools.ietf.org/html/rfc7235#appendix-B +tchar = "!#$%&'*+-.^_`|~" + pp.nums + pp.alphas +token = pp.Word(tchar).setName("token") +token68 = pp.Combine(pp.Word("-._~+/" + pp.nums + pp.alphas) + pp.ZeroOrMore("=")).setName("token68") + +quoted_string = pp.dblQuotedString.copy().setName("quoted-string").setParseAction(unquote) +auth_param_name = token.copy().setName("auth-param-name").addParseAction(pp.downcaseTokens) +auth_param = auth_param_name + pp.Suppress("=") + (token ^ quoted_string) +params = pp.Dict(pp.delimitedList(pp.Group(auth_param))) + +scheme = token("scheme") +challenge = scheme + (token68("token") ^ params("params")) + +authentication_info = params.copy() +www_authenticate = pp.delimitedList(pp.Group(challenge)) + + +def _parse_authentication_info(headers, headername="authentication-info"): + """https://tools.ietf.org/html/rfc7615 + """ + header = headers.get(headername, "").strip() + if not header: + return {} + try: + parsed = authentication_info.parseString(header) + except pp.ParseException as ex: + # print(ex.explain(ex)) + raise MalformedHeader(headername) + + return parsed.asDict() + + +def _parse_www_authenticate(headers, headername="www-authenticate"): + """Returns a dictionary of dictionaries, one dict per auth_scheme.""" + header = headers.get(headername, "").strip() + if not header: + return {} + try: + parsed = www_authenticate.parseString(header) + except pp.ParseException as ex: + # print(ex.explain(ex)) + raise MalformedHeader(headername) + + retval = { + challenge["scheme"].lower(): challenge["params"].asDict() + if "params" in challenge + else {"token": challenge.get("token")} + for challenge in parsed + } + return retval
python2/httplib2/error.py+48 −0 added@@ -0,0 +1,48 @@ +# All exceptions raised here derive from HttpLib2Error +class HttpLib2Error(Exception): + pass + + +# Some exceptions can be caught and optionally +# be turned back into responses. +class HttpLib2ErrorWithResponse(HttpLib2Error): + def __init__(self, desc, response, content): + self.response = response + self.content = content + HttpLib2Error.__init__(self, desc) + + +class RedirectMissingLocation(HttpLib2ErrorWithResponse): + pass + + +class RedirectLimit(HttpLib2ErrorWithResponse): + pass + + +class FailedToDecompressContent(HttpLib2ErrorWithResponse): + pass + + +class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): + pass + + +class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): + pass + + +class MalformedHeader(HttpLib2Error): + pass + + +class RelativeURIError(HttpLib2Error): + pass + + +class ServerNotFoundError(HttpLib2Error): + pass + + +class ProxiesUnavailableError(HttpLib2Error): + pass
python2/httplib2/__init__.py+107 −386 modified@@ -61,6 +61,8 @@ import socks except (ImportError, AttributeError): socks = None +from httplib2 import auth +from httplib2.error import * # Build the appropriate socket wrapper for ssl ssl = None @@ -75,9 +77,7 @@ ssl_CertificateError = getattr(ssl, "CertificateError", None) -def _ssl_wrap_socket( - sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password -): +def _ssl_wrap_socket(sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password): if disable_validation: cert_reqs = ssl.CERT_NONE else: @@ -101,12 +101,7 @@ def _ssl_wrap_socket( if key_password: raise NotSupportedOnThisPlatform("Certificate with password is not supported.") return ssl.wrap_socket( - sock, - keyfile=key_file, - certfile=cert_file, - cert_reqs=cert_reqs, - ca_certs=ca_certs, - ssl_version=ssl_version, + sock, keyfile=key_file, certfile=cert_file, cert_reqs=cert_reqs, ca_certs=ca_certs, ssl_version=ssl_version, ) @@ -277,6 +272,7 @@ class NotRunningAppEngineEnvironment(HttpLib2Error): DEFAULT_MAX_REDIRECTS = 5 from httplib2 import certs + CA_CERTS = certs.where() # Which headers are hop-by-hop headers by default @@ -365,26 +361,17 @@ def safename(filename): def _normalize_headers(headers): - return dict( - [ - (key.lower(), NORMALIZE_SPACE.sub(value, " ").strip()) - for (key, value) in headers.iteritems() - ] - ) + return dict([(key.lower(), NORMALIZE_SPACE.sub(value, " ").strip()) for (key, value) in headers.iteritems()]) def _parse_cache_control(headers): retval = {} if "cache-control" in headers: parts = headers["cache-control"].split(",") parts_with_args = [ - tuple([x.strip().lower() for x in part.split("=", 1)]) - for part in parts - if -1 != part.find("=") - ] - parts_wo_args = [ - (name.strip().lower(), 1) for name in parts if -1 == name.find("=") + tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=") ] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] retval = dict(parts_with_args + parts_wo_args) return retval @@ -395,55 +382,6 @@ def _parse_cache_control(headers): # Set to true to turn on, usefull for testing servers. USE_WWW_AUTH_STRICT_PARSING = 0 -# In regex below: -# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP -# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space -# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: -# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x08\x0A-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"? -WWW_AUTH_STRICT = re.compile( - r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$" -) -WWW_AUTH_RELAXED = re.compile( - r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(?<!\")[^ \t\r\n,]+(?!\"))\"?)(.*)$" -) -UNQUOTE_PAIRS = re.compile(r"\\(.)") - - -def _parse_www_authenticate(headers, headername="www-authenticate"): - """Returns a dictionary of dictionaries, one dict - per auth_scheme.""" - retval = {} - if headername in headers: - try: - - authenticate = headers[headername].strip() - www_auth = ( - USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED - ) - while authenticate: - # Break off the scheme at the beginning of the line - if headername == "authentication-info": - (auth_scheme, the_rest) = ("digest", authenticate) - else: - (auth_scheme, the_rest) = authenticate.split(" ", 1) - # Now loop over all the key value pairs that come after the scheme, - # being careful not to roll into the next scheme - match = www_auth.search(the_rest) - auth_params = {} - while match: - if match and len(match.groups()) == 3: - (key, value, the_rest) = match.groups() - auth_params[key.lower()] = UNQUOTE_PAIRS.sub( - r"\1", value - ) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) - match = www_auth.search(the_rest) - retval[auth_scheme.lower()] = auth_params - authenticate = the_rest.strip() - - except ValueError: - raise MalformedHeader("WWW-Authenticate") - return retval - # TODO: add current time as _entry_disposition argument to avoid sleep in tests def _entry_disposition(response_headers, request_headers): @@ -478,10 +416,7 @@ def _entry_disposition(response_headers, request_headers): cc = _parse_cache_control(request_headers) cc_response = _parse_cache_control(response_headers) - if ( - "pragma" in request_headers - and request_headers["pragma"].lower().find("no-cache") != -1 - ): + if "pragma" in request_headers and request_headers["pragma"].lower().find("no-cache") != -1: retval = "TRANSPARENT" if "cache-control" not in request_headers: request_headers["cache-control"] = "no-cache" @@ -540,8 +475,7 @@ def _decompressContent(response, new_content): except (IOError, zlib.error): content = "" raise FailedToDecompressContent( - _("Content purported to be compressed with %s but failed to decompress.") - % response.get("content-encoding"), + _("Content purported to be compressed with %s but failed to decompress.") % response.get("content-encoding"), response, content, ) @@ -587,17 +521,12 @@ def _updateCache(request_headers, response_headers, content, cache, cachekey): def _cnonce(): - dig = _md5( - "%s:%s" - % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)]) - ).hexdigest() + dig = _md5("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).hexdigest() return dig[:16] def _wsse_username_token(cnonce, iso_now, password): - return base64.b64encode( - _sha("%s%s%s" % (cnonce, iso_now, password)).digest() - ).strip() + return base64.b64encode(_sha("%s%s%s" % (cnonce, iso_now, password)).digest()).strip() # For credentials we need two things, first @@ -610,9 +539,7 @@ def _wsse_username_token(cnonce, iso_now, password): class Authentication(object): - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): + def __init__(self, credentials, host, request_uri, headers, response, content, http): (scheme, authority, path, query, fragment) = parse_uri(request_uri) self.path = path self.host = host @@ -645,55 +572,32 @@ def response(self, response, content): class BasicAuthentication(Authentication): - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate Authorization header.""" - headers["authorization"] = ( - "Basic " + base64.b64encode("%s:%s" % self.credentials).strip() - ) + headers["authorization"] = "Basic " + base64.b64encode("%s:%s" % self.credentials).strip() class DigestAuthentication(Authentication): """Only do qop='auth' and MD5, since that is all Apache currently implements""" - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) - challenge = _parse_www_authenticate(response, "www-authenticate") - self.challenge = challenge["digest"] + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) + self.challenge = auth._parse_www_authenticate(response, "www-authenticate")["digest"] qop = self.challenge.get("qop", "auth") - self.challenge["qop"] = ( - ("auth" in [x.strip() for x in qop.split()]) and "auth" or None - ) + self.challenge["qop"] = ("auth" in [x.strip() for x in qop.split()]) and "auth" or None if self.challenge["qop"] is None: - raise UnimplementedDigestAuthOptionError( - _("Unsupported value for qop: %s." % qop) - ) + raise UnimplementedDigestAuthOptionError(_("Unsupported value for qop: %s." % qop)) self.challenge["algorithm"] = self.challenge.get("algorithm", "MD5").upper() if self.challenge["algorithm"] != "MD5": raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge["algorithm"]) ) - self.A1 = "".join( - [ - self.credentials[0], - ":", - self.challenge["realm"], - ":", - self.credentials[1], - ] - ) + self.A1 = "".join([self.credentials[0], ":", self.challenge["realm"], ":", self.credentials[1],]) self.challenge["nc"] = 1 def request(self, method, request_uri, headers, content, cnonce=None): @@ -734,17 +638,13 @@ def request(self, method, request_uri, headers, content, cnonce=None): def response(self, response, content): if "authentication-info" not in response: - challenge = _parse_www_authenticate(response, "www-authenticate").get( - "digest", {} - ) + challenge = auth._parse_www_authenticate(response, "www-authenticate").get("digest", {}) if "true" == challenge.get("stale"): self.challenge["nonce"] = challenge["nonce"] self.challenge["nc"] = 1 return True else: - updated_challenge = _parse_www_authenticate( - response, "authentication-info" - ).get("digest", {}) + updated_challenge = auth._parse_authentication_info(response, "authentication-info") if "nextnonce" in updated_challenge: self.challenge["nonce"] = updated_challenge["nextnonce"] @@ -757,13 +657,9 @@ class HmacDigestAuthentication(Authentication): __author__ = "Thomas Broyer (t.broyer@ltgt.net)" - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) - challenge = _parse_www_authenticate(response, "www-authenticate") + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) + challenge = auth._parse_www_authenticate(response, "www-authenticate") self.challenge = challenge["hmacdigest"] # TODO: self.challenge['domain'] self.challenge["reason"] = self.challenge.get("reason", "unauthorized") @@ -782,10 +678,7 @@ def __init__( self.challenge["pw-algorithm"] = self.challenge.get("pw-algorithm", "SHA-1") if self.challenge["pw-algorithm"] not in ["SHA-1", "MD5"]: raise UnimplementedHmacDigestAuthOptionError( - _( - "Unsupported value for pw-algorithm: %s." - % self.challenge["pw-algorithm"] - ) + _("Unsupported value for pw-algorithm: %s." % self.challenge["pw-algorithm"]) ) if self.challenge["algorithm"] == "HMAC-MD5": self.hashmod = _md5 @@ -799,11 +692,7 @@ def __init__( [ self.credentials[0], ":", - self.pwhashmod.new( - "".join([self.credentials[1], self.challenge["salt"]]) - ) - .hexdigest() - .lower(), + self.pwhashmod.new("".join([self.credentials[1], self.challenge["salt"]])).hexdigest().lower(), ":", self.challenge["realm"], ] @@ -817,16 +706,8 @@ def request(self, method, request_uri, headers, content): headers_val = "".join([headers[k] for k in keys]) created = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) cnonce = _cnonce() - request_digest = "%s:%s:%s:%s:%s" % ( - method, - request_uri, - cnonce, - self.challenge["snonce"], - headers_val, - ) - request_digest = ( - hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() - ) + request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge["snonce"], headers_val,) + request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() headers["authorization"] = ( 'HMACDigest username="%s", realm="%s", snonce="%s",' ' cnonce="%s", uri="%s", created="%s", ' @@ -843,9 +724,7 @@ def request(self, method, request_uri, headers, content): ) def response(self, response, content): - challenge = _parse_www_authenticate(response, "www-authenticate").get( - "hmacdigest", {} - ) + challenge = auth._parse_www_authenticate(response, "www-authenticate").get("hmacdigest", {}) if challenge.get("reason") in ["integrity", "stale"]: return True return False @@ -860,12 +739,8 @@ class WsseAuthentication(Authentication): challenge but instead requiring your client to telepathically know that their endpoint is expecting WSSE profile="UsernameToken".""" - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate @@ -874,22 +749,20 @@ def request(self, method, request_uri, headers, content): iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) cnonce = _cnonce() password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1]) - headers["X-WSSE"] = ( - 'UsernameToken Username="%s", PasswordDigest="%s", ' - 'Nonce="%s", Created="%s"' - ) % (self.credentials[0], password_digest, cnonce, iso_now) + headers["X-WSSE"] = ('UsernameToken Username="%s", PasswordDigest="%s", ' 'Nonce="%s", Created="%s"') % ( + self.credentials[0], + password_digest, + cnonce, + iso_now, + ) class GoogleLoginAuthentication(Authentication): - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): + def __init__(self, credentials, host, request_uri, headers, response, content, http): from urllib import urlencode - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) - challenge = _parse_www_authenticate(response, "www-authenticate") + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) + challenge = auth._parse_www_authenticate(response, "www-authenticate") service = challenge["googlelogin"].get("service", "xapi") # Bloggger actually returns the service in the challenge # For the rest we guess based on the URI @@ -899,12 +772,7 @@ def __init__( # elif request_uri.find("spreadsheets") > 0: # service = "wise" - auth = dict( - Email=credentials[0], - Passwd=credentials[1], - service=service, - source=headers["user-agent"], - ) + auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers["user-agent"],) resp, content = self.http.request( "https://www.google.com/accounts/ClientLogin", method="POST", @@ -941,9 +809,7 @@ class FileCache(object): be running on the same cache. """ - def __init__( - self, cache, safe=safename - ): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior + def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior self.cache = cache self.safe = safe if not os.path.exists(cache): @@ -991,6 +857,7 @@ def iter(self, domain): class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" + def add(self, key, cert, domain, password): self.credentials.append((domain.lower(), key, cert, password)) @@ -1010,14 +877,7 @@ class ProxyInfo(object): bypass_hosts = () def __init__( - self, - proxy_type, - proxy_host, - proxy_port, - proxy_rdns=True, - proxy_user=None, - proxy_pass=None, - proxy_headers=None, + self, proxy_type, proxy_host, proxy_port, proxy_rdns=True, proxy_user=None, proxy_pass=None, proxy_headers=None, ): """Args: @@ -1165,14 +1025,18 @@ def connect(self): """Connect to the host and port specified in __init__.""" # Mostly verbatim from httplib.py. if self.proxy_info and socks is None: - raise ProxiesUnavailableError( - "Proxy support missing but proxy use was requested!" - ) + raise ProxiesUnavailableError("Proxy support missing but proxy use was requested!") if self.proxy_info and self.proxy_info.isgood(): use_proxy = True - proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( - self.proxy_info.astuple() - ) + ( + proxy_type, + proxy_host, + proxy_port, + proxy_rdns, + proxy_user, + proxy_pass, + proxy_headers, + ) = self.proxy_info.astuple() host = proxy_host port = proxy_port @@ -1190,13 +1054,7 @@ def connect(self): if use_proxy: self.sock = socks.socksocket(af, socktype, proto) self.sock.setproxy( - proxy_type, - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers, ) else: self.sock = socket.socket(af, socktype, proto) @@ -1210,16 +1068,7 @@ def connect(self): if use_proxy: print( "proxy: %s ************" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) if use_proxy: self.sock.connect((self.host, self.port) + sa[2:]) @@ -1232,16 +1081,7 @@ def connect(self): if use_proxy: print( "proxy: %s" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) if self.sock: self.sock.close() @@ -1348,9 +1188,15 @@ def connect(self): if self.proxy_info and self.proxy_info.isgood(): use_proxy = True - proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( - self.proxy_info.astuple() - ) + ( + proxy_type, + proxy_host, + proxy_port, + proxy_rdns, + proxy_user, + proxy_pass, + proxy_headers, + ) = self.proxy_info.astuple() host = proxy_host port = proxy_port @@ -1369,13 +1215,7 @@ def connect(self): sock = socks.socksocket(family, socktype, proto) sock.setproxy( - proxy_type, - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers, ) else: sock = socket.socket(family, socktype, proto) @@ -1403,32 +1243,18 @@ def connect(self): if use_proxy: print( "proxy: %s" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) if not self.disable_ssl_certificate_validation: cert = self.sock.getpeercert() hostname = self.host.split(":", 0)[0] if not self._ValidateCertificateHostname(cert, hostname): raise CertificateHostnameMismatch( - "Server presented certificate that does not match " - "host %s: %s" % (hostname, cert), + "Server presented certificate that does not match " "host %s: %s" % (hostname, cert), hostname, cert, ) - except ( - ssl_SSLError, - ssl_CertificateError, - CertificateHostnameMismatch, - ) as e: + except (ssl_SSLError, ssl_CertificateError, CertificateHostnameMismatch,) as e: if sock: sock.close() if self.sock: @@ -1451,16 +1277,7 @@ def connect(self): if use_proxy: print( "proxy: %s" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) if self.sock: self.sock.close() @@ -1478,15 +1295,8 @@ def connect(self): def _new_fixed_fetch(validate_certificate): - def fixed_fetch( - url, - payload=None, - method="GET", - headers={}, - allow_truncated=False, - follow_redirects=True, - deadline=None, + url, payload=None, method="GET", headers={}, allow_truncated=False, follow_redirects=True, deadline=None, ): return fetch( url, @@ -1523,9 +1333,7 @@ def __init__( disable_ssl_certificate_validation=False, ssl_version=None, ): - httplib.HTTPConnection.__init__( - self, host, port=port, strict=strict, timeout=timeout - ) + httplib.HTTPConnection.__init__(self, host, port=port, strict=strict, timeout=timeout) class AppEngineHttpsConnection(httplib.HTTPSConnection): @@ -1552,23 +1360,19 @@ def __init__( if key_password: raise NotSupportedOnThisPlatform("Certificate with password is not supported.") httplib.HTTPSConnection.__init__( - self, - host, - port=port, - key_file=key_file, - cert_file=cert_file, - strict=strict, - timeout=timeout, + self, host, port=port, key_file=key_file, cert_file=cert_file, strict=strict, timeout=timeout, ) self._fetch = _new_fixed_fetch(not disable_ssl_certificate_validation) # Use a different connection object for Google App Engine Standard Environment. def is_gae_instance(): - server_software = os.environ.get('SERVER_SOFTWARE', '') - if (server_software.startswith('Google App Engine/') or - server_software.startswith('Development/') or - server_software.startswith('testutil/')): + server_software = os.environ.get("SERVER_SOFTWARE", "") + if ( + server_software.startswith("Google App Engine/") + or server_software.startswith("Development/") + or server_software.startswith("testutil/") + ): return True return False @@ -1578,6 +1382,7 @@ def is_gae_instance(): raise NotRunningAppEngineEnvironment() from google.appengine.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub("urlfetch") is None: raise ImportError @@ -1716,13 +1521,11 @@ def _auth_from_challenge(self, host, request_uri, headers, response, content): """A generator that creates Authorization objects that can be applied to requests. """ - challenges = _parse_www_authenticate(response, "www-authenticate") + challenges = auth._parse_www_authenticate(response, "www-authenticate") for cred in self.credentials.iter(host): for scheme in AUTH_SCHEME_ORDER: if scheme in challenges: - yield AUTH_SCHEME_CLASSES[scheme]( - cred, host, request_uri, headers, response, content, self - ) + yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) def add_credentials(self, name, password, domain=""): """Add a name and password that will be used @@ -1818,79 +1621,48 @@ def _conn_request(self, conn, request_uri, method, body, headers): return (response, content) def _request( - self, - conn, - host, - absolute_uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, + self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey, ): """Do the actual request using the connection object and also follow one level of redirects if necessary""" - auths = [ - (auth.depth(request_uri), auth) - for auth in self.authorizations - if auth.inscope(host, request_uri) - ] + auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] auth = auths and sorted(auths)[0][1] or None if auth: auth.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) + (response, content) = self._conn_request(conn, request_uri, method, body, headers) if auth: if auth.response(response, body): auth.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) + (response, content) = self._conn_request(conn, request_uri, method, body, headers) response._stale_digest = 1 if response.status == 401: - for authorization in self._auth_from_challenge( - host, request_uri, headers, response, content - ): + for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): authorization.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) + (response, content) = self._conn_request(conn, request_uri, method, body, headers) if response.status != 401: self.authorizations.append(authorization) authorization.response(response, body) break - if ( - self.follow_all_redirects - or method in self.safe_methods - or response.status in (303, 308) - ): + if self.follow_all_redirects or method in self.safe_methods or response.status in (303, 308): if self.follow_redirects and response.status in self.redirect_codes: # Pick out the location header and basically start from the beginning # remembering first to strip the ETag header and decrement our 'depth' if redirections: if "location" not in response and response.status != 300: raise RedirectMissingLocation( - _( - "Redirected but the response is missing a Location: header." - ), - response, - content, + _("Redirected but the response is missing a Location: header."), response, content, ) # Fix-up relative redirects (which violate an RFC 2616 MUST) if "location" in response: location = response["location"] (scheme, authority, path, query, fragment) = parse_uri(location) if authority == None: - response["location"] = urlparse.urljoin( - absolute_uri, location - ) + response["location"] = urlparse.urljoin(absolute_uri, location) if response.status == 308 or (response.status == 301 and method in self.safe_methods): response["-x-permanent-redirect-url"] = response["location"] if "content-location" not in response: @@ -1900,10 +1672,7 @@ def _request( del headers["if-none-match"] if "if-modified-since" in headers: del headers["if-modified-since"] - if ( - "authorization" in headers - and not self.forward_authorization_headers - ): + if "authorization" in headers and not self.forward_authorization_headers: del headers["authorization"] if "location" in response: location = response["location"] @@ -1915,18 +1684,12 @@ def _request( redirect_method = "GET" body = None (response, content) = self.request( - location, - method=redirect_method, - body=body, - headers=headers, - redirections=redirections - 1, + location, method=redirect_method, body=body, headers=headers, redirections=redirections - 1, ) response.previous = old_response else: raise RedirectLimit( - "Redirected more times than rediection_limit allows.", - response, - content, + "Redirected more times than rediection_limit allows.", response, content, ) elif response.status in [200, 203] and method in self.safe_methods: # Don't cache 206's since we aren't going to handle byte range requests @@ -1944,13 +1707,7 @@ def _normalize_headers(self, headers): # including all socket.* and httplib.* exceptions. def request( - self, - uri, - method="GET", - body=None, - headers=None, - redirections=DEFAULT_MAX_REDIRECTS, - connection_type=None, + self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, ): """ Performs a single HTTP request. @@ -1973,7 +1730,7 @@ def request( being and instance of the 'Response' class, the second being a string that contains the response entity body. """ - conn_key = '' + conn_key = "" try: if headers is None: @@ -2094,9 +1851,7 @@ def request( # Should cached permanent redirects be counted in our redirection count? For now, yes. if redirections <= 0: raise RedirectLimit( - "Redirected more times than rediection_limit allows.", - {}, - "", + "Redirected more times than rediection_limit allows.", {}, "", ) (response, new_content) = self.request( info["-x-permanent-redirect-url"], @@ -2127,27 +1882,15 @@ def request( return (response, content) if entry_disposition == "STALE": - if ( - "etag" in info - and not self.ignore_etag - and not "if-none-match" in headers - ): + if "etag" in info and not self.ignore_etag and not "if-none-match" in headers: headers["if-none-match"] = info["etag"] if "last-modified" in info and not "last-modified" in headers: headers["if-modified-since"] = info["last-modified"] elif entry_disposition == "TRANSPARENT": pass (response, new_content) = self._request( - conn, - authority, - uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, + conn, authority, uri, request_uri, method, body, headers, redirections, cachekey, ) if response.status == 304 and method == "GET": @@ -2161,9 +1904,7 @@ def request( merged_response = Response(info) if hasattr(response, "_stale_digest"): merged_response._stale_digest = response._stale_digest - _updateCache( - headers, merged_response, content, self.cache, cachekey - ) + _updateCache(headers, merged_response, content, self.cache, cachekey) response = merged_response response.status = 200 response.fromcache = True @@ -2181,15 +1922,7 @@ def request( content = "" else: (response, content) = self._request( - conn, - authority, - uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, + conn, authority, uri, request_uri, method, body, headers, redirections, cachekey, ) except Exception as e: is_timeout = isinstance(e, socket.timeout) @@ -2206,23 +1939,11 @@ def request( response.reason = str(e) elif is_timeout: content = "Request Timeout" - response = Response( - { - "content-type": "text/plain", - "status": "408", - "content-length": len(content), - } - ) + response = Response({"content-type": "text/plain", "status": "408", "content-length": len(content),}) response.reason = "Request Timeout" else: content = str(e) - response = Response( - { - "content-type": "text/plain", - "status": "400", - "content-length": len(content), - } - ) + response = Response({"content-type": "text/plain", "status": "400", "content-length": len(content),}) response.reason = "Bad Request" else: raise
python3/httplib2/auth.py+61 −0 added@@ -0,0 +1,61 @@ +import base64 +import re + +import pyparsing as pp + +from .error import * + +UNQUOTE_PAIRS = re.compile(r"\\(.)") +unquote = lambda s, l, t: UNQUOTE_PAIRS.sub(r"\1", t[0][1:-1]) + +# https://tools.ietf.org/html/rfc7235#section-1.2 +# https://tools.ietf.org/html/rfc7235#appendix-B +tchar = "!#$%&'*+-.^_`|~" + pp.nums + pp.alphas +token = pp.Word(tchar).setName("token") +token68 = pp.Combine(pp.Word("-._~+/" + pp.nums + pp.alphas) + pp.ZeroOrMore("=")).setName("token68") + +quoted_string = pp.dblQuotedString.copy().setName("quoted-string").setParseAction(unquote) +auth_param_name = token.copy().setName("auth-param-name").addParseAction(pp.downcaseTokens) +auth_param = auth_param_name + pp.Suppress("=") + (token ^ quoted_string) +params = pp.Dict(pp.delimitedList(pp.Group(auth_param))) + +scheme = token("scheme") +challenge = scheme + (token68("token") ^ params("params")) + +authentication_info = params.copy() +www_authenticate = pp.delimitedList(pp.Group(challenge)) + + +def _parse_authentication_info(headers, headername="authentication-info"): + """https://tools.ietf.org/html/rfc7615 + """ + header = headers.get(headername, "").strip() + if not header: + return {} + try: + parsed = authentication_info.parseString(header) + except pp.ParseException as ex: + # print(ex.explain(ex)) + raise MalformedHeader(headername) + + return parsed.asDict() + + +def _parse_www_authenticate(headers, headername="www-authenticate"): + """Returns a dictionary of dictionaries, one dict per auth_scheme.""" + header = headers.get(headername, "").strip() + if not header: + return {} + try: + parsed = www_authenticate.parseString(header) + except pp.ParseException as ex: + # print(ex.explain(ex)) + raise MalformedHeader(headername) + + retval = { + challenge["scheme"].lower(): challenge["params"].asDict() + if "params" in challenge + else {"token": challenge.get("token")} + for challenge in parsed + } + return retval
python3/httplib2/error.py+48 −0 added@@ -0,0 +1,48 @@ +# All exceptions raised here derive from HttpLib2Error +class HttpLib2Error(Exception): + pass + + +# Some exceptions can be caught and optionally +# be turned back into responses. +class HttpLib2ErrorWithResponse(HttpLib2Error): + def __init__(self, desc, response, content): + self.response = response + self.content = content + HttpLib2Error.__init__(self, desc) + + +class RedirectMissingLocation(HttpLib2ErrorWithResponse): + pass + + +class RedirectLimit(HttpLib2ErrorWithResponse): + pass + + +class FailedToDecompressContent(HttpLib2ErrorWithResponse): + pass + + +class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): + pass + + +class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): + pass + + +class MalformedHeader(HttpLib2Error): + pass + + +class RelativeURIError(HttpLib2Error): + pass + + +class ServerNotFoundError(HttpLib2Error): + pass + + +class ProxiesUnavailableError(HttpLib2Error): + pass
python3/httplib2/__init__.py+126 −423 modified@@ -49,6 +49,8 @@ # TODO: remove this fallback and copypasted socksipy module upon py2/3 merge, # idea is to have soft-dependency on any compatible module called socks from . import socks +from . import auth +from .error import * from .iri2uri import iri2uri @@ -79,56 +81,6 @@ def has_timeout(timeout): RETRIES = 2 -# All exceptions raised here derive from HttpLib2Error -class HttpLib2Error(Exception): - pass - - -# Some exceptions can be caught and optionally -# be turned back into responses. -class HttpLib2ErrorWithResponse(HttpLib2Error): - def __init__(self, desc, response, content): - self.response = response - self.content = content - HttpLib2Error.__init__(self, desc) - - -class RedirectMissingLocation(HttpLib2ErrorWithResponse): - pass - - -class RedirectLimit(HttpLib2ErrorWithResponse): - pass - - -class FailedToDecompressContent(HttpLib2ErrorWithResponse): - pass - - -class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): - pass - - -class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): - pass - - -class MalformedHeader(HttpLib2Error): - pass - - -class RelativeURIError(HttpLib2Error): - pass - - -class ServerNotFoundError(HttpLib2Error): - pass - - -class ProxiesUnavailableError(HttpLib2Error): - pass - - # Open Items: # ----------- @@ -169,28 +121,31 @@ class ProxiesUnavailableError(HttpLib2Error): from httplib2 import certs + CA_CERTS = certs.where() # PROTOCOL_TLS is python 3.5.3+. PROTOCOL_SSLv23 is deprecated. # Both PROTOCOL_TLS and PROTOCOL_SSLv23 are equivalent and means: # > Selects the highest protocol version that both the client and server support. # > Despite the name, this option can select “TLS” protocols as well as “SSL”. # source: https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLS -DEFAULT_TLS_VERSION = getattr(ssl, "PROTOCOL_TLS", None) or getattr( - ssl, "PROTOCOL_SSLv23" -) +DEFAULT_TLS_VERSION = getattr(ssl, "PROTOCOL_TLS", None) or getattr(ssl, "PROTOCOL_SSLv23") + def _build_ssl_context( - disable_ssl_certificate_validation, ca_certs, cert_file=None, key_file=None, - maximum_version=None, minimum_version=None, key_password=None, + disable_ssl_certificate_validation, + ca_certs, + cert_file=None, + key_file=None, + maximum_version=None, + minimum_version=None, + key_password=None, ): if not hasattr(ssl, "SSLContext"): raise RuntimeError("httplib2 requires Python 3.2+ for ssl.SSLContext") context = ssl.SSLContext(DEFAULT_TLS_VERSION) - context.verify_mode = ( - ssl.CERT_NONE if disable_ssl_certificate_validation else ssl.CERT_REQUIRED - ) + context.verify_mode = ssl.CERT_NONE if disable_ssl_certificate_validation else ssl.CERT_REQUIRED # SSLContext.maximum_version and SSLContext.minimum_version are python 3.7+. # source: https://docs.python.org/3/library/ssl.html#ssl.SSLContext.maximum_version @@ -288,10 +243,7 @@ def safename(filename): def _normalize_headers(headers): return dict( [ - ( - _convert_byte_str(key).lower(), - NORMALIZE_SPACE.sub(_convert_byte_str(value), " ").strip(), - ) + (_convert_byte_str(key).lower(), NORMALIZE_SPACE.sub(_convert_byte_str(value), " ").strip(),) for (key, value) in headers.items() ] ) @@ -308,13 +260,9 @@ def _parse_cache_control(headers): if "cache-control" in headers: parts = headers["cache-control"].split(",") parts_with_args = [ - tuple([x.strip().lower() for x in part.split("=", 1)]) - for part in parts - if -1 != part.find("=") - ] - parts_wo_args = [ - (name.strip().lower(), 1) for name in parts if -1 == name.find("=") + tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=") ] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] retval = dict(parts_with_args + parts_wo_args) return retval @@ -325,53 +273,6 @@ def _parse_cache_control(headers): # Set to true to turn on, useful for testing servers. USE_WWW_AUTH_STRICT_PARSING = 0 -# In regex below: -# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP -# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space -# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: -# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x08\x0A-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"? -WWW_AUTH_STRICT = re.compile( - r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$" -) -WWW_AUTH_RELAXED = re.compile( - r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(?<!\")[^ \t\r\n,]+(?!\"))\"?)(.*)$" -) -UNQUOTE_PAIRS = re.compile(r"\\(.)") - - -def _parse_www_authenticate(headers, headername="www-authenticate"): - """Returns a dictionary of dictionaries, one dict - per auth_scheme.""" - retval = {} - if headername in headers: - try: - authenticate = headers[headername].strip() - www_auth = ( - USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED - ) - while authenticate: - # Break off the scheme at the beginning of the line - if headername == "authentication-info": - (auth_scheme, the_rest) = ("digest", authenticate) - else: - (auth_scheme, the_rest) = authenticate.split(" ", 1) - # Now loop over all the key value pairs that come after the scheme, - # being careful not to roll into the next scheme - match = www_auth.search(the_rest) - auth_params = {} - while match: - if match and len(match.groups()) == 3: - (key, value, the_rest) = match.groups() - auth_params[key.lower()] = UNQUOTE_PAIRS.sub( - r"\1", value - ) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) - match = www_auth.search(the_rest) - retval[auth_scheme.lower()] = auth_params - authenticate = the_rest.strip() - except ValueError: - raise MalformedHeader("WWW-Authenticate") - return retval - def _entry_disposition(response_headers, request_headers): """Determine freshness from the Date, Expires and Cache-Control headers. @@ -405,10 +306,7 @@ def _entry_disposition(response_headers, request_headers): cc = _parse_cache_control(request_headers) cc_response = _parse_cache_control(response_headers) - if ( - "pragma" in request_headers - and request_headers["pragma"].lower().find("no-cache") != -1 - ): + if "pragma" in request_headers and request_headers["pragma"].lower().find("no-cache") != -1: retval = "TRANSPARENT" if "cache-control" not in request_headers: request_headers["cache-control"] = "no-cache" @@ -467,8 +365,7 @@ def _decompressContent(response, new_content): except (IOError, zlib.error): content = "" raise FailedToDecompressContent( - _("Content purported to be compressed with %s but failed to decompress.") - % response.get("content-encoding"), + _("Content purported to be compressed with %s but failed to decompress.") % response.get("content-encoding"), response, content, ) @@ -484,9 +381,7 @@ def _write_headers(self): print(v.encode(maxlinelen=self._maxheaderlen), file=self._fp) else: # email.Header got lots of smarts, so use it. - headers = header.Header( - v, maxlinelen=self._maxheaderlen, charset="utf-8", header_name=h - ) + headers = header.Header(v, maxlinelen=self._maxheaderlen, charset="utf-8", header_name=h) print(headers.encode(), file=self._fp) # A blank line always separates headers from body. print(file=self._fp) @@ -531,27 +426,22 @@ def _updateCache(request_headers, response_headers, content, cache, cachekey): header_str = info.as_string() header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str) - text = b"".join( - [status_header.encode("utf-8"), header_str.encode("utf-8"), content] - ) + text = b"".join([status_header.encode("utf-8"), header_str.encode("utf-8"), content]) cache.set(cachekey, text) def _cnonce(): dig = _md5( - ( - "%s:%s" - % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)]) - ).encode("utf-8") + ("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).encode("utf-8") ).hexdigest() return dig[:16] def _wsse_username_token(cnonce, iso_now, password): - return base64.b64encode( - _sha(("%s%s%s" % (cnonce, iso_now, password)).encode("utf-8")).digest() - ).strip().decode("utf-8") + return ( + base64.b64encode(_sha(("%s%s%s" % (cnonce, iso_now, password)).encode("utf-8")).digest()).strip().decode("utf-8") + ) # For credentials we need two things, first @@ -564,9 +454,7 @@ def _wsse_username_token(cnonce, iso_now, password): class Authentication(object): - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): + def __init__(self, credentials, host, request_uri, headers, response, content, http): (scheme, authority, path, query, fragment) = parse_uri(request_uri) self.path = path self.host = host @@ -620,12 +508,8 @@ def __bool__(self): class BasicAuthentication(Authentication): - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate @@ -639,36 +523,19 @@ class DigestAuthentication(Authentication): """Only do qop='auth' and MD5, since that is all Apache currently implements""" - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) - challenge = _parse_www_authenticate(response, "www-authenticate") - self.challenge = challenge["digest"] + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) + self.challenge = auth._parse_www_authenticate(response, "www-authenticate")["digest"] qop = self.challenge.get("qop", "auth") - self.challenge["qop"] = ( - ("auth" in [x.strip() for x in qop.split()]) and "auth" or None - ) + self.challenge["qop"] = ("auth" in [x.strip() for x in qop.split()]) and "auth" or None if self.challenge["qop"] is None: - raise UnimplementedDigestAuthOptionError( - _("Unsupported value for qop: %s." % qop) - ) + raise UnimplementedDigestAuthOptionError(_("Unsupported value for qop: %s." % qop)) self.challenge["algorithm"] = self.challenge.get("algorithm", "MD5").upper() if self.challenge["algorithm"] != "MD5": raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge["algorithm"]) ) - self.A1 = "".join( - [ - self.credentials[0], - ":", - self.challenge["realm"], - ":", - self.credentials[1], - ] - ) + self.A1 = "".join([self.credentials[0], ":", self.challenge["realm"], ":", self.credentials[1],]) self.challenge["nc"] = 1 def request(self, method, request_uri, headers, content, cnonce=None): @@ -709,17 +576,13 @@ def request(self, method, request_uri, headers, content, cnonce=None): def response(self, response, content): if "authentication-info" not in response: - challenge = _parse_www_authenticate(response, "www-authenticate").get( - "digest", {} - ) + challenge = auth._parse_www_authenticate(response, "www-authenticate").get("digest", {}) if "true" == challenge.get("stale"): self.challenge["nonce"] = challenge["nonce"] self.challenge["nc"] = 1 return True else: - updated_challenge = _parse_www_authenticate( - response, "authentication-info" - ).get("digest", {}) + updated_challenge = auth._parse_authentication_info(response, "authentication-info") if "nextnonce" in updated_challenge: self.challenge["nonce"] = updated_challenge["nextnonce"] @@ -732,13 +595,9 @@ class HmacDigestAuthentication(Authentication): __author__ = "Thomas Broyer (t.broyer@ltgt.net)" - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) - challenge = _parse_www_authenticate(response, "www-authenticate") + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) + challenge = auth._parse_www_authenticate(response, "www-authenticate") self.challenge = challenge["hmacdigest"] # TODO: self.challenge['domain'] self.challenge["reason"] = self.challenge.get("reason", "unauthorized") @@ -757,10 +616,7 @@ def __init__( self.challenge["pw-algorithm"] = self.challenge.get("pw-algorithm", "SHA-1") if self.challenge["pw-algorithm"] not in ["SHA-1", "MD5"]: raise UnimplementedHmacDigestAuthOptionError( - _( - "Unsupported value for pw-algorithm: %s." - % self.challenge["pw-algorithm"] - ) + _("Unsupported value for pw-algorithm: %s." % self.challenge["pw-algorithm"]) ) if self.challenge["algorithm"] == "HMAC-MD5": self.hashmod = _md5 @@ -774,11 +630,7 @@ def __init__( [ self.credentials[0], ":", - self.pwhashmod.new( - "".join([self.credentials[1], self.challenge["salt"]]) - ) - .hexdigest() - .lower(), + self.pwhashmod.new("".join([self.credentials[1], self.challenge["salt"]])).hexdigest().lower(), ":", self.challenge["realm"], ] @@ -792,16 +644,8 @@ def request(self, method, request_uri, headers, content): headers_val = "".join([headers[k] for k in keys]) created = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) cnonce = _cnonce() - request_digest = "%s:%s:%s:%s:%s" % ( - method, - request_uri, - cnonce, - self.challenge["snonce"], - headers_val, - ) - request_digest = ( - hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() - ) + request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge["snonce"], headers_val,) + request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() headers["authorization"] = ( 'HMACDigest username="%s", realm="%s", snonce="%s",' ' cnonce="%s", uri="%s", created="%s", ' @@ -818,9 +662,7 @@ def request(self, method, request_uri, headers, content): ) def response(self, response, content): - challenge = _parse_www_authenticate(response, "www-authenticate").get( - "hmacdigest", {} - ) + challenge = auth._parse_www_authenticate(response, "www-authenticate").get("hmacdigest", {}) if challenge.get("reason") in ["integrity", "stale"]: return True return False @@ -835,12 +677,8 @@ class WsseAuthentication(Authentication): challenge but instead requiring your client to telepathically know that their endpoint is expecting WSSE profile="UsernameToken".""" - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) + def __init__(self, credentials, host, request_uri, headers, response, content, http): + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate @@ -849,22 +687,20 @@ def request(self, method, request_uri, headers, content): iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) cnonce = _cnonce() password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1]) - headers["X-WSSE"] = ( - 'UsernameToken Username="%s", PasswordDigest="%s", ' - 'Nonce="%s", Created="%s"' - ) % (self.credentials[0], password_digest, cnonce, iso_now) + headers["X-WSSE"] = ('UsernameToken Username="%s", PasswordDigest="%s", ' 'Nonce="%s", Created="%s"') % ( + self.credentials[0], + password_digest, + cnonce, + iso_now, + ) class GoogleLoginAuthentication(Authentication): - def __init__( - self, credentials, host, request_uri, headers, response, content, http - ): + def __init__(self, credentials, host, request_uri, headers, response, content, http): from urllib.parse import urlencode - Authentication.__init__( - self, credentials, host, request_uri, headers, response, content, http - ) - challenge = _parse_www_authenticate(response, "www-authenticate") + Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) + challenge = auth._parse_www_authenticate(response, "www-authenticate") service = challenge["googlelogin"].get("service", "xapi") # Bloggger actually returns the service in the challenge # For the rest we guess based on the URI @@ -874,12 +710,7 @@ def __init__( # elif request_uri.find("spreadsheets") > 0: # service = "wise" - auth = dict( - Email=credentials[0], - Passwd=credentials[1], - service=service, - source=headers["user-agent"], - ) + auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers["user-agent"],) resp, content = self.http.request( "https://www.google.com/accounts/ClientLogin", method="POST", @@ -916,9 +747,7 @@ class FileCache(object): be running on the same cache. """ - def __init__( - self, cache, safe=safename - ): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior + def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior self.cache = cache self.safe = safe if not os.path.exists(cache): @@ -966,6 +795,7 @@ def iter(self, domain): class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" + def add(self, key, cert, domain, password): self.credentials.append((domain.lower(), key, cert, password)) @@ -985,14 +815,7 @@ class ProxyInfo(object): bypass_hosts = () def __init__( - self, - proxy_type, - proxy_host, - proxy_port, - proxy_rdns=True, - proxy_user=None, - proxy_pass=None, - proxy_headers=None, + self, proxy_type, proxy_host, proxy_port, proxy_rdns=True, proxy_user=None, proxy_pass=None, proxy_headers=None, ): """Args: @@ -1015,7 +838,15 @@ def __init__( proxy_user = proxy_user.decode() if isinstance(proxy_pass, bytes): proxy_pass = proxy_pass.decode() - self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass, self.proxy_headers = ( + ( + self.proxy_type, + self.proxy_host, + self.proxy_port, + self.proxy_rdns, + self.proxy_user, + self.proxy_pass, + self.proxy_headers, + ) = ( proxy_type, proxy_host, proxy_port, @@ -1149,14 +980,18 @@ def __init__(self, host, port=None, timeout=None, proxy_info=None): def connect(self): """Connect to the host and port specified in __init__.""" if self.proxy_info and socks is None: - raise ProxiesUnavailableError( - "Proxy support missing but proxy use was requested!" - ) + raise ProxiesUnavailableError("Proxy support missing but proxy use was requested!") if self.proxy_info and self.proxy_info.isgood() and self.proxy_info.applies_to(self.host): use_proxy = True - proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( - self.proxy_info.astuple() - ) + ( + proxy_type, + proxy_host, + proxy_port, + proxy_rdns, + proxy_user, + proxy_pass, + proxy_headers, + ) = self.proxy_info.astuple() host = proxy_host port = proxy_port @@ -1175,35 +1010,19 @@ def connect(self): if use_proxy: self.sock = socks.socksocket(af, socktype, proto) self.sock.setproxy( - proxy_type, - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, ) else: self.sock = socket.socket(af, socktype, proto) self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) if has_timeout(self.timeout): self.sock.settimeout(self.timeout) if self.debuglevel > 0: - print( - "connect: ({0}, {1}) ************".format(self.host, self.port) - ) + print("connect: ({0}, {1}) ************".format(self.host, self.port)) if use_proxy: print( "proxy: {0} ************".format( - str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) ) @@ -1215,16 +1034,7 @@ def connect(self): if use_proxy: print( "proxy: {0}".format( - str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) ) if self.sock: @@ -1268,15 +1078,16 @@ def __init__( self.proxy_info = proxy_info("https") context = _build_ssl_context( - self.disable_ssl_certificate_validation, self.ca_certs, cert_file, key_file, - maximum_version=tls_maximum_version, minimum_version=tls_minimum_version, + self.disable_ssl_certificate_validation, + self.ca_certs, + cert_file, + key_file, + maximum_version=tls_maximum_version, + minimum_version=tls_minimum_version, key_password=key_password, ) super(HTTPSConnectionWithTimeout, self).__init__( - host, - port=port, - timeout=timeout, - context=context, + host, port=port, timeout=timeout, context=context, ) self.key_file = key_file self.cert_file = cert_file @@ -1286,9 +1097,15 @@ def connect(self): """Connect to a host on a given (SSL) port.""" if self.proxy_info and self.proxy_info.isgood() and self.proxy_info.applies_to(self.host): use_proxy = True - proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( - self.proxy_info.astuple() - ) + ( + proxy_type, + proxy_host, + proxy_port, + proxy_rdns, + proxy_user, + proxy_pass, + proxy_headers, + ) = self.proxy_info.astuple() host = proxy_host port = proxy_port @@ -1309,12 +1126,7 @@ def connect(self): sock = socks.socksocket(family, socktype, proto) sock.setproxy( - proxy_type, - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, ) else: sock = socket.socket(family, socktype, proto) @@ -1326,10 +1138,7 @@ def connect(self): self.sock = self._context.wrap_socket(sock, server_hostname=self.host) # Python 3.3 compatibility: emulate the check_hostname behavior - if ( - not hasattr(self._context, "check_hostname") - and not self.disable_ssl_certificate_validation - ): + if not hasattr(self._context, "check_hostname") and not self.disable_ssl_certificate_validation: try: ssl.match_hostname(self.sock.getpeercert(), self.host) except Exception: @@ -1342,16 +1151,7 @@ def connect(self): if use_proxy: print( "proxy: {0}".format( - str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) ) except (ssl.SSLError, ssl.CertificateError) as e: @@ -1370,16 +1170,7 @@ def connect(self): if use_proxy: print( "proxy: {0}".format( - str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) + str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers,)) ) ) if self.sock: @@ -1523,13 +1314,11 @@ def _auth_from_challenge(self, host, request_uri, headers, response, content): """A generator that creates Authorization objects that can be applied to requests. """ - challenges = _parse_www_authenticate(response, "www-authenticate") + challenges = auth._parse_www_authenticate(response, "www-authenticate") for cred in self.credentials.iter(host): for scheme in AUTH_SCHEME_ORDER: if scheme in challenges: - yield AUTH_SCHEME_CLASSES[scheme]( - cred, host, request_uri, headers, response, content, self - ) + yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) def add_credentials(self, name, password, domain=""): """Add a name and password that will be used @@ -1563,9 +1352,7 @@ def _conn_request(self, conn, request_uri, method, body, headers): conn.close() raise ServerNotFoundError("Unable to find the server at %s" % conn.host) except socket.error as e: - errno_ = ( - e.args[0].errno if isinstance(e.args[0], socket.error) else e.errno - ) + errno_ = e.args[0].errno if isinstance(e.args[0], socket.error) else e.errno if errno_ in (errno.ENETUNREACH, errno.EADDRNOTAVAIL) and i < RETRIES: continue # retry on potentially transient errors raise @@ -1624,79 +1411,48 @@ def _conn_request(self, conn, request_uri, method, body, headers): return (response, content) def _request( - self, - conn, - host, - absolute_uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, + self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey, ): """Do the actual request using the connection object and also follow one level of redirects if necessary""" - auths = [ - (auth.depth(request_uri), auth) - for auth in self.authorizations - if auth.inscope(host, request_uri) - ] + auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] auth = auths and sorted(auths)[0][1] or None if auth: auth.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) + (response, content) = self._conn_request(conn, request_uri, method, body, headers) if auth: if auth.response(response, body): auth.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) + (response, content) = self._conn_request(conn, request_uri, method, body, headers) response._stale_digest = 1 if response.status == 401: - for authorization in self._auth_from_challenge( - host, request_uri, headers, response, content - ): + for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): authorization.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) + (response, content) = self._conn_request(conn, request_uri, method, body, headers) if response.status != 401: self.authorizations.append(authorization) authorization.response(response, body) break - if ( - self.follow_all_redirects - or method in self.safe_methods - or response.status in (303, 308) - ): + if self.follow_all_redirects or method in self.safe_methods or response.status in (303, 308): if self.follow_redirects and response.status in self.redirect_codes: # Pick out the location header and basically start from the beginning # remembering first to strip the ETag header and decrement our 'depth' if redirections: if "location" not in response and response.status != 300: raise RedirectMissingLocation( - _( - "Redirected but the response is missing a Location: header." - ), - response, - content, + _("Redirected but the response is missing a Location: header."), response, content, ) # Fix-up relative redirects (which violate an RFC 2616 MUST) if "location" in response: location = response["location"] (scheme, authority, path, query, fragment) = parse_uri(location) if authority == None: - response["location"] = urllib.parse.urljoin( - absolute_uri, location - ) + response["location"] = urllib.parse.urljoin(absolute_uri, location) if response.status == 308 or (response.status == 301 and (method in self.safe_methods)): response["-x-permanent-redirect-url"] = response["location"] if "content-location" not in response: @@ -1706,10 +1462,7 @@ def _request( del headers["if-none-match"] if "if-modified-since" in headers: del headers["if-modified-since"] - if ( - "authorization" in headers - and not self.forward_authorization_headers - ): + if "authorization" in headers and not self.forward_authorization_headers: del headers["authorization"] if "location" in response: location = response["location"] @@ -1721,18 +1474,12 @@ def _request( redirect_method = "GET" body = None (response, content) = self.request( - location, - method=redirect_method, - body=body, - headers=headers, - redirections=redirections - 1, + location, method=redirect_method, body=body, headers=headers, redirections=redirections - 1, ) response.previous = old_response else: raise RedirectLimit( - "Redirected more times than redirection_limit allows.", - response, - content, + "Redirected more times than redirection_limit allows.", response, content, ) elif response.status in [200, 203] and method in self.safe_methods: # Don't cache 206's since we aren't going to handle byte range requests @@ -1750,13 +1497,7 @@ def _normalize_headers(self, headers): # including all socket.* and httplib.* exceptions. def request( - self, - uri, - method="GET", - body=None, - headers=None, - redirections=DEFAULT_MAX_REDIRECTS, - connection_type=None, + self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, ): """ Performs a single HTTP request. The 'uri' is the URI of the HTTP resource and can begin @@ -1778,7 +1519,7 @@ def request( being and instance of the 'Response' class, the second being a string that contains the response entity body. """ - conn_key = '' + conn_key = "" try: if headers is None: @@ -1847,9 +1588,7 @@ def request( info = email.message_from_bytes(info) for k, v in info.items(): if v.startswith("=?") and v.endswith("?="): - info.replace_header( - k, str(*email.header.decode_header(v)[0]) - ) + info.replace_header(k, str(*email.header.decode_header(v)[0])) except (IndexError, ValueError): self.cache.delete(cachekey) cachekey = None @@ -1896,9 +1635,7 @@ def request( # Should cached permanent redirects be counted in our redirection count? For now, yes. if redirections <= 0: raise RedirectLimit( - "Redirected more times than redirection_limit allows.", - {}, - "", + "Redirected more times than redirection_limit allows.", {}, "", ) (response, new_content) = self.request( info["-x-permanent-redirect-url"], @@ -1929,27 +1666,15 @@ def request( return (response, content) if entry_disposition == "STALE": - if ( - "etag" in info - and not self.ignore_etag - and not "if-none-match" in headers - ): + if "etag" in info and not self.ignore_etag and not "if-none-match" in headers: headers["if-none-match"] = info["etag"] if "last-modified" in info and not "last-modified" in headers: headers["if-modified-since"] = info["last-modified"] elif entry_disposition == "TRANSPARENT": pass (response, new_content) = self._request( - conn, - authority, - uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, + conn, authority, uri, request_uri, method, body, headers, redirections, cachekey, ) if response.status == 304 and method == "GET": @@ -1963,9 +1688,7 @@ def request( merged_response = Response(info) if hasattr(response, "_stale_digest"): merged_response._stale_digest = response._stale_digest - _updateCache( - headers, merged_response, content, self.cache, cachekey - ) + _updateCache(headers, merged_response, content, self.cache, cachekey) response = merged_response response.status = 200 response.fromcache = True @@ -1983,15 +1706,7 @@ def request( content = b"" else: (response, content) = self._request( - conn, - authority, - uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, + conn, authority, uri, request_uri, method, body, headers, redirections, cachekey, ) except Exception as e: is_timeout = isinstance(e, socket.timeout) @@ -2008,23 +1723,11 @@ def request( response.reason = str(e) elif isinstance(e, socket.timeout): content = b"Request Timeout" - response = Response( - { - "content-type": "text/plain", - "status": "408", - "content-length": len(content), - } - ) + response = Response({"content-type": "text/plain", "status": "408", "content-length": len(content),}) response.reason = "Request Timeout" else: content = str(e).encode("utf-8") - response = Response( - { - "content-type": "text/plain", - "status": "400", - "content-length": len(content), - } - ) + response = Response({"content-type": "text/plain", "status": "400", "content-length": len(content),}) response.reason = "Bad Request" else: raise
requirements.txt+1 −0 added@@ -0,0 +1 @@ +pyparsing>=2.4.2,<3 # TODO include v3 after dropping Python2 support
script/release+4 −4 modified@@ -210,11 +210,11 @@ assert_tree_clean() { version_check() { local need=$1 - local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '\"" |cut -d\= -f2) - local version_py2=$(cd python2 ; python2 -Es -c 'import httplib2;print(httplib2.__version__)') - local version_py3=$(cd python3 ; python3 -Es -c 'import httplib2;print(httplib2.__version__)') + local version_setup=$(python setup.py --version) + local version_py2=$(python -Es -c "$(egrep '^__version__' python2/httplib2/__init__.py);print(__version__)") + local version_py3=$(python -Es -c "$(egrep '^__version__' python3/httplib2/__init__.py);print(__version__)") if [[ "$version_setup" != "$need" ]] ; then - echo "error: setup.py VERSION=$version_setup expected=$need" >&1 + echo "error: setup.py version=$version_setup expected=$need" >&1 exit 1 fi if [[ "$version_py2" != "$need" ]] ; then
script/test+2 −2 modified@@ -65,8 +65,8 @@ main() { install_check_version() { local pip="$1" $pip install dist/httplib2* - version_source=$(cd python3 ; python3 -Es -c 'import httplib2;print(httplib2.__version__)') - version_installed=$($pip show httplib2 |fgrep Version |cut -d' ' -f2) + version_source=$(python setup.py --version) + version_installed=$($pip show httplib2 |fgrep Version: |cut -d' ' -f2) if [[ "$version_source" != "$version_installed" ]] ; then echo "error: installed package version=$version_installed does not match source=$version_source" >&2 exit 1
setup.py+7 −5 modified@@ -14,19 +14,20 @@ class TestCommand(setuptools.command.test.test): def run_tests(self): # pytest may be not installed yet import pytest - args = ['--forked', '--fulltrace', '--no-cov', 'tests/'] + + args = ["--forked", "--fulltrace", "--no-cov", "tests/"] if self.test_suite: - args += ['-k', self.test_suite] - sys.stderr.write('setup.py:test run pytest {}\n'.format(' '.join(args))) + args += ["-k", self.test_suite] + sys.stderr.write("setup.py:test run pytest {}\n".format(" ".join(args))) errno = pytest.main(args) sys.exit(errno) def read_requirements(name): project_root = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(project_root, name), 'rb') as f: + with open(os.path.join(project_root, name), "rb") as f: # remove whitespace and comments - g = (line.decode('utf-8').lstrip().split('#', 1)[0].rstrip() for line in f) + g = (line.decode("utf-8").lstrip().split("#", 1)[0].rstrip() for line in f) return [l for l in g if l] @@ -85,6 +86,7 @@ def read_requirements(name): package_dir=pkgdir, packages=["httplib2"], package_data={"httplib2": ["*.txt"]}, + install_requires=read_requirements("requirements.txt"), tests_require=read_requirements("requirements-test.txt"), cmdclass={"test": TestCommand}, classifiers=[
tests/__init__.py+31 −83 modified@@ -119,9 +119,9 @@ def parse_http_message(kind, buf): msg = kind() msg.raw = start_line if kind is HttpRequest: - assert re.match( - br".+ HTTP/\d\.\d\r\n$", start_line - ), "Start line does not look like HTTP request: " + repr(start_line) + assert re.match(br".+ HTTP/\d\.\d\r\n$", start_line), "Start line does not look like HTTP request: " + repr( + start_line + ) msg.method, msg.uri, msg.proto = start_line.rstrip().decode().split(" ", 2) assert msg.proto.startswith("HTTP/"), repr(start_line) elif kind is HttpResponse: @@ -201,14 +201,7 @@ class MockHTTPConnection(object): """ def __init__( - self, - host, - port=None, - key_file=None, - cert_file=None, - strict=None, - timeout=None, - proxy_info=None, + self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=None, proxy_info=None, ): self.host = host self.port = port @@ -240,14 +233,7 @@ class MockHTTPBadStatusConnection(object): num_calls = 0 def __init__( - self, - host, - port=None, - key_file=None, - cert_file=None, - strict=None, - timeout=None, - proxy_info=None, + self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=None, proxy_info=None, ): self.host = host self.port = port @@ -328,11 +314,7 @@ def server_socket_thread(srv): # at least in other/connection_close test # should not be a problem since socket would close upon garbage collection if gcounter[0] > request_count: - gresult[0] = Exception( - "Request count expected={0} actual={1}".format( - request_count, gcounter[0] - ) - ) + gresult[0] = Exception("Request count expected={0} actual={1}".format(request_count, gcounter[0])) except Exception as e: # traceback.print_exc caused IOError: concurrent operation on sys.stderr.close() under setup.py test print(traceback.format_exc(), file=sys.stderr) @@ -458,21 +440,12 @@ def http_response_bytes( if add_etag: headers.setdefault("etag", '"{0}"'.format(hashlib.md5(body).hexdigest())) header_string = "".join("{0}: {1}\r\n".format(k, v) for k, v in headers.items()) - if ( - not undefined_body_length - and proto != "HTTP/1.0" - and "content-length" not in headers - ): - raise Exception( - "httplib2.tests.http_response_bytes: client could not figure response body length" - ) + if not undefined_body_length and proto != "HTTP/1.0" and "content-length" not in headers: + raise Exception("httplib2.tests.http_response_bytes: client could not figure response body length") if str(status).isdigit(): status = "{} {}".format(status, http_client.responses[status]) response = ( - "{proto} {status}\r\n{headers}\r\n".format( - proto=proto, status=status, headers=header_string - ).encode() - + body + "{proto} {status}\r\n{headers}\r\n".format(proto=proto, status=status, headers=header_string).encode() + body ) return response @@ -526,21 +499,6 @@ def server_reflect(**kwargs): return server_request(http_handler, **kwargs) -def http_parse_auth(s): - """https://tools.ietf.org/html/rfc7235#section-2.1 - """ - scheme, rest = s.split(" ", 1) - result = {} - while True: - m = httplib2.WWW_AUTH_RELAXED.search(rest) - if not m: - break - if len(m.groups()) == 3: - key, value, rest = m.groups() - result[key.lower()] = httplib2.UNQUOTE_PAIRS.sub(r"\1", value) - return result - - def store_request_response(out): def wrapper(fun): @functools.wraps(fun) @@ -609,14 +567,19 @@ def http_reflect_with_auth_handler(request): auth_header = request.headers.get("authorization", "") if not auth_header: return deny() - if " " not in auth_header: + try: + auth_parsed = httplib2.auth._parse_www_authenticate(request.headers, "authorization") + print("debug: auth_parsed", auth_parsed) + except httplib2.error.MalformedHeader: + print("debug: auth header error") return http_response_bytes(status=400, body=b"authorization header syntax error") - scheme, data = auth_header.split(" ", 1) - scheme = scheme.lower() + scheme = auth_header.split(" ", 1)[0].lower() + print("debug: first auth scheme='{}'".format(scheme)) if scheme != allow_scheme: return deny(body=b"must use different auth scheme") + auth_info = auth_parsed[scheme] if scheme == "basic": - decoded = base64.b64decode(data).decode() + decoded = base64.b64decode(auth_info["token"]).decode() username, password = decoded.split(":", 1) if (username, password) in allow_credentials: return make_http_reflect()(request) @@ -630,7 +593,6 @@ def http_reflect_with_auth_handler(request): gserver_nonce[0] = nextnonce gnextnonce[0] = None server_nonce_current = gserver_nonce[0] - auth_info = http_parse_auth(data) client_cnonce = auth_info.get("cnonce", "") client_nc = auth_info.get("nc", "") client_nonce = auth_info.get("nonce", "") @@ -651,45 +613,30 @@ def http_reflect_with_auth_handler(request): return deny(body=b"auth-info nc missing") if client_opaque != server_opaque: return deny( - body="auth-info opaque mismatch expected={} actual={}".format( - server_opaque, client_opaque - ).encode() + body="auth-info opaque mismatch expected={} actual={}".format(server_opaque, client_opaque).encode() ) for allow_username, allow_password in allow_credentials: - ha1 = hasher( - ":".join((allow_username, realm, allow_password)).encode() - ).hexdigest() + ha1 = hasher(":".join((allow_username, realm, allow_password)).encode()).hexdigest() allow_response = hasher( - ":".join( - (ha1, client_nonce, client_nc, client_cnonce, client_qop, ha2) - ).encode() + ":".join((ha1, client_nonce, client_nc, client_cnonce, client_qop, ha2)).encode() ).hexdigest() rspauth_ha2 = hasher(":{}".format(request.uri).encode()).hexdigest() rspauth = hasher( - ":".join( - ( - ha1, - client_nonce, - client_nc, - client_cnonce, - client_qop, - rspauth_ha2, - ) - ).encode() + ":".join((ha1, client_nonce, client_nc, client_cnonce, client_qop, rspauth_ha2,)).encode() ).hexdigest() if auth_info.get("response", "") == allow_response: # TODO: fix or remove doubtful comment # do we need to save nc only on success? glastnc[0] = client_nc allow_headers = { - "authentication-info": " ".join( + "authentication-info": ", ".join(filter(None, ( 'nextnonce="{}"'.format(nextnonce) if nextnonce else "", "qop={}".format(client_qop), 'rspauth="{}"'.format(rspauth), 'cnonce="{}"'.format(client_cnonce), "nc={}".format(client_nc), - ) + )) ).strip() } return make_http_reflect(headers=allow_headers)(request) @@ -698,11 +645,12 @@ def http_reflect_with_auth_handler(request): x_wsse = request.headers.get("x-wsse", "") if x_wsse.count(",") != 3: return http_response_bytes(status=400, body=b"x-wsse header syntax error") - auth_info = http_parse_auth(x_wsse) - client_username = auth_info.get("username", "") - client_nonce = auth_info.get("nonce", "") - client_created = auth_info.get("created", "") - client_digest = auth_info.get("passworddigest", "") + wsse_params = httplib2.auth._parse_www_authenticate(request.headers, "x-wsse").get("usernametoken", {}) + print("debug: wsse_params", wsse_params) + client_username = wsse_params.get("username", "") + client_nonce = wsse_params.get("nonce", "") + client_created = wsse_params.get("created", "") + client_digest = wsse_params.get("passworddigest", "") allow_password = None for allow_username, allow_password in allow_credentials: if client_username == allow_username: @@ -712,7 +660,7 @@ def http_reflect_with_auth_handler(request): digest = hashlib.sha1("".join((client_nonce, client_created, allow_password)).encode("utf-8")).digest() digest_b64 = base64.b64encode(digest).decode() - print("$$$ check client={} == real={}".format(client_digest, digest_b64)) + print("debug: check client={} == real={}".format(client_digest, digest_b64)) if client_digest == digest_b64: return make_http_reflect()(request)
tests/test_auth.py+83 −129 modified@@ -1,3 +1,5 @@ +import time + import httplib2 import pytest import tests @@ -26,9 +28,7 @@ def test_basic(): # Test Basic Authentication http = httplib2.Http() password = tests.gen_password() - handler = tests.http_reflect_with_auth( - allow_scheme="basic", allow_credentials=(("joe", password),) - ) + handler = tests.http_reflect_with_auth(allow_scheme="basic", allow_credentials=(("joe", password),)) with tests.server_request(handler, request_count=3) as uri: response, content = http.request(uri, "GET") assert response.status == 401 @@ -41,9 +41,7 @@ def test_basic_for_domain(): # Test Basic Authentication http = httplib2.Http() password = tests.gen_password() - handler = tests.http_reflect_with_auth( - allow_scheme="basic", allow_credentials=(("joe", password),) - ) + handler = tests.http_reflect_with_auth(allow_scheme="basic", allow_credentials=(("joe", password),)) with tests.server_request(handler, request_count=4) as uri: response, content = http.request(uri, "GET") assert response.status == 401 @@ -62,9 +60,7 @@ def test_basic_two_credentials(): password1 = tests.gen_password() password2 = tests.gen_password() allowed = [("joe", password1)] # exploit shared mutable list - handler = tests.http_reflect_with_auth( - allow_scheme="basic", allow_credentials=allowed - ) + handler = tests.http_reflect_with_auth(allow_scheme="basic", allow_credentials=allowed) with tests.server_request(handler, request_count=7) as uri: http.add_credentials("fred", password2) response, content = http.request(uri, "GET") @@ -81,9 +77,7 @@ def test_digest(): # Test that we support Digest Authentication http = httplib2.Http() password = tests.gen_password() - handler = tests.http_reflect_with_auth( - allow_scheme="digest", allow_credentials=(("joe", password),) - ) + handler = tests.http_reflect_with_auth(allow_scheme="digest", allow_credentials=(("joe", password),)) with tests.server_request(handler, request_count=3) as uri: response, content = http.request(uri, "GET") assert response.status == 401 @@ -99,25 +93,24 @@ def test_digest_next_nonce_nc(): password = tests.gen_password() grenew_nonce = [None] handler = tests.http_reflect_with_auth( - allow_scheme="digest", - allow_credentials=(("joe", password),), - out_renew_nonce=grenew_nonce, + allow_scheme="digest", allow_credentials=(("joe", password),), out_renew_nonce=grenew_nonce, ) with tests.server_request(handler, request_count=5) as uri: http.add_credentials("joe", password) response1, _ = http.request(uri, "GET") - info = httplib2._parse_www_authenticate(response1, "authentication-info") + info = httplib2.auth._parse_authentication_info(response1) + print("debug: response1 authentication-info: {}\nparsed: {}".format(response1.get("authentication-info"), info)) assert response1.status == 200 - assert info.get("digest", {}).get("nc") == "00000001", info + assert info.get("nc") == "00000001", info assert not info.get("digest", {}).get("nextnonce"), info response2, _ = http.request(uri, "GET") - info2 = httplib2._parse_www_authenticate(response2, "authentication-info") - assert info2.get("digest", {}).get("nc") == "00000002", info2 + info2 = httplib2.auth._parse_authentication_info(response2) + assert info2.get("nc") == "00000002", info2 grenew_nonce[0]() response3, content = http.request(uri, "GET") - info3 = httplib2._parse_www_authenticate(response3, "authentication-info") + info3 = httplib2.auth._parse_authentication_info(response3) assert response3.status == 200 - assert info3.get("digest", {}).get("nc") == "00000001", info3 + assert info3.get("nc") == "00000001", info3 def test_digest_auth_stale(): @@ -136,17 +129,13 @@ def test_digest_auth_stale(): http.add_credentials("joe", password) response, _ = http.request(uri, "GET") assert response.status == 200 - info = httplib2._parse_www_authenticate( - requests[0][1].headers, "www-authenticate" - ) + info = httplib2.auth._parse_www_authenticate(requests[0][1].headers, "www-authenticate") grenew_nonce[0]() response, _ = http.request(uri, "GET") assert response.status == 200 assert not response.fromcache assert getattr(response, "_stale_digest", False) - info2 = httplib2._parse_www_authenticate( - requests[2][1].headers, "www-authenticate" - ) + info2 = httplib2.auth._parse_www_authenticate(requests[2][1].headers, "www-authenticate") nonce1 = info.get("digest", {}).get("nonce", "") nonce2 = info2.get("digest", {}).get("nonce", "") assert nonce1 != "" @@ -160,73 +149,33 @@ def test_digest_auth_stale(): ({}, {}), ({"www-authenticate": ""}, {}), ( - { - "www-authenticate": 'Test realm="test realm" , foo=foo ,bar="bar", baz=baz,qux=qux' - }, - { - "test": { - "realm": "test realm", - "foo": "foo", - "bar": "bar", - "baz": "baz", - "qux": "qux", - } - }, + {"www-authenticate": 'Test realm="test realm" , foo=foo ,bar="bar", baz=baz,qux=qux'}, + {"test": {"realm": "test realm", "foo": "foo", "bar": "bar", "baz": "baz", "qux": "qux"}}, ), ( {"www-authenticate": 'T*!%#st realm=to*!%#en, to*!%#en="quoted string"'}, {"t*!%#st": {"realm": "to*!%#en", "to*!%#en": "quoted string"}}, ), - ( - {"www-authenticate": 'Test realm="a \\"test\\" realm"'}, - {"test": {"realm": 'a "test" realm'}}, - ), + ({"www-authenticate": 'Test realm="a \\"test\\" realm"'}, {"test": {"realm": 'a "test" realm'}},), ({"www-authenticate": 'Basic realm="me"'}, {"basic": {"realm": "me"}}), - ( - {"www-authenticate": 'Basic realm="me", algorithm="MD5"'}, - {"basic": {"realm": "me", "algorithm": "MD5"}}, - ), - ( - {"www-authenticate": 'Basic realm="me", algorithm=MD5'}, - {"basic": {"realm": "me", "algorithm": "MD5"}}, - ), - ( - {"www-authenticate": 'Basic realm="me",other="fred" '}, - {"basic": {"realm": "me", "other": "fred"}}, - ), + ({"www-authenticate": 'Basic realm="me", algorithm="MD5"'}, {"basic": {"realm": "me", "algorithm": "MD5"}},), + ({"www-authenticate": 'Basic realm="me", algorithm=MD5'}, {"basic": {"realm": "me", "algorithm": "MD5"}},), + ({"www-authenticate": 'Basic realm="me",other="fred" '}, {"basic": {"realm": "me", "other": "fred"}},), ({"www-authenticate": 'Basic REAlm="me" '}, {"basic": {"realm": "me"}}), ( - { - "www-authenticate": 'Digest realm="digest1", qop="auth,auth-int", nonce="7102dd2", opaque="e9517f"' - }, - { - "digest": { - "realm": "digest1", - "qop": "auth,auth-int", - "nonce": "7102dd2", - "opaque": "e9517f", - } - }, + {"www-authenticate": 'Digest realm="digest1", qop="auth,auth-int", nonce="7102dd2", opaque="e9517f"'}, + {"digest": {"realm": "digest1", "qop": "auth,auth-int", "nonce": "7102dd2", "opaque": "e9517f"}}, ), - # multiple schema choice + # comma between schemas (glue for multiple headers with same name) ( - { - "www-authenticate": 'Digest realm="multi-d", nonce="8b11d0f6", opaque="cc069c" Basic realm="multi-b" ' - }, - { - "digest": {"realm": "multi-d", "nonce": "8b11d0f6", "opaque": "cc069c"}, - "basic": {"realm": "multi-b"}, - }, + {"www-authenticate": 'Digest realm="2-comma-d", qop="auth-int", nonce="c0c8ff1", Basic realm="2-comma-b"'}, + {"digest": {"realm": "2-comma-d", "qop": "auth-int", "nonce": "c0c8ff1"}, "basic": {"realm": "2-comma-b"}}, ), - # FIXME - # comma between schemas (glue for multiple headers with same name) - # ({'www-authenticate': 'Digest realm="2-comma-d", qop="auth-int", nonce="c0c8ff1", Basic realm="2-comma-b"'}, - # {'digest': {'realm': '2-comma-d', 'qop': 'auth-int', 'nonce': 'c0c8ff1'}, - # 'basic': {'realm': '2-comma-b'}}), - # FIXME # comma between schemas + WSSE (glue for multiple headers with same name) - # ({'www-authenticate': 'Digest realm="com3d", Basic realm="com3b", WSSE realm="com3w", profile="token"'}, - # {'digest': {'realm': 'com3d'}, 'basic': {'realm': 'com3b'}, 'wsse': {'realm': 'com3w', profile': 'token'}}), + ( + {"www-authenticate": 'Digest realm="com3d", Basic realm="com3b", WSSE realm="com3w", profile="token"'}, + {"digest": {"realm": "com3d"}, "basic": {"realm": "com3b"}, "wsse": {"realm": "com3w", "profile": "token"}}, + ), # FIXME # multiple syntax figures # ({'www-authenticate': @@ -237,19 +186,10 @@ def test_digest_auth_stale(): # 'wsse': {'realm': 'very', 'profile': 'UsernameToken'}}), # more quote combos ( - { - "www-authenticate": 'Digest realm="myrealm", nonce="KBAA=3", algorithm=MD5, qop="auth", stale=true' - }, - { - "digest": { - "realm": "myrealm", - "nonce": "KBAA=3", - "algorithm": "MD5", - "qop": "auth", - "stale": "true", - } - }, + {"www-authenticate": 'Digest realm="myrealm", nonce="KBAA=3", algorithm=MD5, qop="auth", stale=true'}, + {"digest": {"realm": "myrealm", "nonce": "KBAA=3", "algorithm": "MD5", "qop": "auth", "stale": "true"}}, ), + ({"www-authenticate": "Basic param='single quote'"}, {"basic": {"param": "'single"}}), ), ids=lambda data: str(data[0]), ) @@ -259,40 +199,67 @@ def test_parse_www_authenticate_correct(data, strict): # FIXME: move strict to parse argument httplib2.USE_WWW_AUTH_STRICT_PARSING = strict try: - assert httplib2._parse_www_authenticate(headers) == info + assert httplib2.auth._parse_www_authenticate(headers) == info finally: httplib2.USE_WWW_AUTH_STRICT_PARSING = 0 -def test_parse_www_authenticate_malformed(): +@pytest.mark.parametrize( + "data", + (({"www-authenticate": 'OAuth "Facebook Platform" "invalid_token" "Invalid OAuth access token."'}, None),), + ids=lambda data: str(data[0]), +) +def test_parse_www_authenticate_malformed(data): # TODO: test (and fix) header value 'barbqwnbm-bb...:asd' leads to dead loop - with tests.assert_raises(httplib2.MalformedHeader): - httplib2._parse_www_authenticate( - { - "www-authenticate": 'OAuth "Facebook Platform" "invalid_token" "Invalid OAuth access token."' - } - ) + headers, info = data + try: + result = httplib2.auth._parse_www_authenticate(headers) + except httplib2.error.MalformedHeader: + assert info is None, "unexpected MalformedHeader" + else: + assert result == info + assert info is not None, "expected parsing error" + + +def test_parse_www_authenticate_complexity(): + # TODO just use time.process_time() after python2 support is removed + process_time = getattr(time, "process_time", time.time) + + def check(size): + header = {"www-authenticate": 'scheme {0}key=value,{0}quoted="foo=bar"'.format(" \t" * size)} + tbegin = process_time() + result = httplib2.auth._parse_www_authenticate(header) + tend = process_time() + assert result == {"scheme": {"key": "value", "quoted": "foo=bar"}} + elapsed_us = round((tend * 1e6) - (tbegin * 1e6), 0) + return elapsed_us + + n1, n2, repeat = 50, 100, 7 + time1 = min(check(n1) for _ in range(repeat)) + time2 = min(check(n2) for _ in range(repeat)) + speed1 = round(time1 / n1, 1) + speed2 = round(time2 / n2, 1) + expect2 = round(speed1 * (float(n2) / n1), 1) + error = round(speed2 / expect2, 1) + print("x{}: time={}us speed={} us/op".format(n1, time1, speed1)) + print("x{}: time={}us speed={} us/op expected={} us/op error={}".format(n2, time2, speed2, expect2, error)) + assert error < 2, "_parse_www_authenticate scales too fast" def test_digest_object(): credentials = ("joe", "password") host = None request_uri = "/test/digest/" headers = {} - response = { - "www-authenticate": 'Digest realm="myrealm", nonce="KBAA=35", algorithm=MD5, qop="auth"' - } + response = {"www-authenticate": 'Digest realm="myrealm", nonce="KBAA=35", algorithm=MD5, qop="auth"'} content = b"" - d = httplib2.DigestAuthentication( - credentials, host, request_uri, headers, response, content, None - ) + d = httplib2.DigestAuthentication(credentials, host, request_uri, headers, response, content, None) d.request("GET", request_uri, headers, content, cnonce="33033375ec278a46") our_request = "authorization: " + headers["authorization"] working_request = ( 'authorization: Digest username="joe", realm="myrealm", ' - 'nonce="KBAA=35", uri="/test/digest/"' - + ', algorithm=MD5, response="de6d4a123b80801d0e94550411b6283f", ' + 'nonce="KBAA=35", uri="/test/digest/"' + ', algorithm=MD5, response="de6d4a123b80801d0e94550411b6283f", ' 'qop=auth, nc=00000001, cnonce="33033375ec278a46"' ) assert our_request == working_request @@ -304,14 +271,11 @@ def test_digest_object_with_opaque(): request_uri = "/digest/opaque/" headers = {} response = { - "www-authenticate": 'Digest realm="myrealm", nonce="30352fd", algorithm=MD5, ' - 'qop="auth", opaque="atestopaque"' + "www-authenticate": 'Digest realm="myrealm", nonce="30352fd", algorithm=MD5, ' 'qop="auth", opaque="atestopaque"' } content = "" - d = httplib2.DigestAuthentication( - credentials, host, request_uri, headers, response, content, None - ) + d = httplib2.DigestAuthentication(credentials, host, request_uri, headers, response, content, None) d.request("GET", request_uri, headers, content, cnonce="5ec2") our_request = "authorization: " + headers["authorization"] working_request = ( @@ -329,15 +293,10 @@ def test_digest_object_stale(): request_uri = "/digest/stale/" headers = {} response = httplib2.Response({}) - response["www-authenticate"] = ( - 'Digest realm="myrealm", nonce="bd669f", ' - 'algorithm=MD5, qop="auth", stale=true' - ) + response["www-authenticate"] = 'Digest realm="myrealm", nonce="bd669f", ' 'algorithm=MD5, qop="auth", stale=true' response.status = 401 content = b"" - d = httplib2.DigestAuthentication( - credentials, host, request_uri, headers, response, content, None - ) + d = httplib2.DigestAuthentication(credentials, host, request_uri, headers, response, content, None) # Returns true to force a retry assert d.response(response, content) @@ -348,15 +307,10 @@ def test_digest_object_auth_info(): request_uri = "/digest/nextnonce/" headers = {} response = httplib2.Response({}) - response["www-authenticate"] = ( - 'Digest realm="myrealm", nonce="barney", ' - 'algorithm=MD5, qop="auth", stale=true' - ) + response["www-authenticate"] = 'Digest realm="myrealm", nonce="barney", ' 'algorithm=MD5, qop="auth", stale=true' response["authentication-info"] = 'nextnonce="fred"' content = b"" - d = httplib2.DigestAuthentication( - credentials, host, request_uri, headers, response, content, None - ) + d = httplib2.DigestAuthentication(credentials, host, request_uri, headers, response, content, None) # Returns true to force a retry assert not d.response(response, content) assert d.challenge["nonce"] == "fred"
Vulnerability mechanics
Generated by null/stub on May 9, 2026. Inputs: CWE entries + fix-commit diffs from this CVE's patches. Citations validated against bundle.
References
7- github.com/advisories/GHSA-93xj-8mrv-444mghsaADVISORY
- nvd.nist.gov/vuln/detail/CVE-2021-21240ghsaADVISORY
- github.com/httplib2/httplib2/commit/bd9ee252c8f099608019709e22c0d705e98d26bcghsax_refsource_MISCWEB
- github.com/httplib2/httplib2/pull/182ghsax_refsource_MISCWEB
- github.com/httplib2/httplib2/security/advisories/GHSA-93xj-8mrv-444mghsax_refsource_CONFIRMWEB
- github.com/pypa/advisory-database/tree/main/vulns/httplib2/PYSEC-2021-16.yamlghsaWEB
- pypi.org/project/httplib2ghsax_refsource_MISCWEB
News mentions
0No linked articles in our index yet.