CVE-2007-5712

@@ -237,7 +237,7 @@
 
 # The User-Agent string to use when checking for URL validity through the
 # isExistingURL validator.
-URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
+URL_VALIDATOR_USER_AGENT = "Django/0.96.1 (http://www.djangoproject.com)"
 
 ##############
 # MIDDLEWARE #

@@ -1 +1 @@
-VERSION = (0, 96, None)
+VERSION = (0, 96.1, None)

@@ -1,6 +1,9 @@
 "Translation helper functions"
 
-import os, re, sys
+import locale
+import os
+import re
+import sys
 import gettext as gettext_module
 from cStringIO import StringIO
 from django.utils.functional import lazy
@@ -25,15 +28,25 @@ def currentThread():
 # The default translation is based on the settings file.
 _default = None
 
-# This is a cache for accept-header to translation object mappings to prevent
-# the accept parser to run multiple times for one user.
+# This is a cache for normalised accept-header languages to prevent multiple
+# file lookups when checking the same locale on repeated requests.
 _accepted = {}
 
-def to_locale(language):
+# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9.
+accept_language_re = re.compile(r'''
+        ([A-Za-z]{1,8}(?:-[A-Za-z]{1,8})*|\*)   # "en", "en-au", "x-y-z", "*"
+        (?:;q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"
+        (?:\s*,\s*|$)                            # Multiple accepts per header.
+        ''', re.VERBOSE)
+
+def to_locale(language, to_lower=False):
     "Turns a language name (en-us) into a locale name (en_US)."
     p = language.find('-')
     if p >= 0:
-        return language[:p].lower()+'_'+language[p+1:].upper()
+        if to_lower:
+            return language[:p].lower()+'_'+language[p+1:].lower()
+        else:
+            return language[:p].lower()+'_'+language[p+1:].upper()
     else:
         return language.lower()
 
@@ -309,46 +322,40 @@ def get_language_from_request(request):
         if lang_code in supported and lang_code is not None and check_for_language(lang_code):
             return lang_code
 
-    lang_code = request.COOKIES.get('django_language', None)
-    if lang_code in supported and lang_code is not None and check_for_language(lang_code):
+    lang_code = request.COOKIES.get('django_language')
+    if lang_code and lang_code in supported and check_for_language(lang_code):
         return lang_code
 
-    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', None)
-    if accept is not None:
-
-        t = _accepted.get(accept, None)
-        if t is not None:
-            return t
-
-        def _parsed(el):
-            p = el.find(';q=')
-            if p >= 0:
-                lang = el[:p].strip()
-                order = int(float(el[p+3:].strip())*100)
-            else:
-                lang = el
-                order = 100
-            p = lang.find('-')
-            if p >= 0:
-                mainlang = lang[:p]
-            else:
-                mainlang = lang
-            return (lang, mainlang, order)
-
-        langs = [_parsed(el) for el in accept.split(',')]
-        langs.sort(lambda a,b: -1*cmp(a[2], b[2]))
-
-        for lang, mainlang, order in langs:
-            if lang in supported or mainlang in supported:
-                langfile = gettext_module.find('django', globalpath, [to_locale(lang)])
-                if langfile:
-                    # reconstruct the actual language from the language
-                    # filename, because otherwise we might incorrectly
-                    # report de_DE if we only have de available, but
-                    # did find de_DE because of language normalization
-                    lang = langfile[len(globalpath):].split(os.path.sep)[1]
-                    _accepted[accept] = lang
-                    return lang
+    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
+    for lang, unused in parse_accept_lang_header(accept):
+        if lang == '*':
+            break
+
+        # We have a very restricted form for our language files (no encoding
+        # specifier, since they all must be UTF-8 and only one possible
+        # language each time. So we avoid the overhead of gettext.find() and
+        # look up the MO file manually.
+
+        normalized = locale.locale_alias.get(to_locale(lang, True))
+        if not normalized:
+            continue
+
+        # Remove the default encoding from locale_alias
+        normalized = normalized.split('.')[0]
+
+        if normalized in _accepted:
+            # We've seen this locale before and have an MO file for it, so no
+            # need to check again.
+            return _accepted[normalized]
+
+        for lang in (normalized, normalized.split('_')[0]):
+            if lang not in supported:
+                continue
+            langfile = os.path.join(globalpath, lang, 'LC_MESSAGES',
+                    'django.mo')
+            if os.path.exists(langfile):
+                _accepted[normalized] = lang
+            return lang
 
     return settings.LANGUAGE_CODE
 
@@ -494,3 +501,24 @@ def string_concat(*strings):
     return ''.join([str(el) for el in strings])
 
 string_concat = lazy(string_concat, str)
+
+def parse_accept_lang_header(lang_string):
+    """
+    Parses the lang_string, which is the body of an HTTP Accept-Language
+    header, and returns a list of (lang, q-value), ordered by 'q' values.
+
+    Any format errors in lang_string results in an empty list being returned.
+    """
+    result = []
+    pieces = accept_language_re.split(lang_string)
+    if pieces[-1]:
+        return []
+    for i in range(0, len(pieces) - 1, 3):
+        first, lang, priority = pieces[i : i + 3]
+        if first:
+            return []
+        priority = priority and float(priority) or 1.0
+        result.append((lang, priority))
+    result.sort(lambda x, y: -cmp(x[1], y[1]))
+    return result
+

@@ -1,12 +1,12 @@
-=================================
-Django version 0.96 release notes
-=================================
+===================================
+Django version 0.96.1 release notes
+===================================
 
-Welcome to Django 0.96!
+Welcome to Django 0.96.1!
 
 The primary goal for 0.96 is a cleanup and stabilization of the features
 introduced in 0.95. There have been a few small `backwards-incompatible
-changes`_ since 0.95, but the upgrade process should be fairly simple
+changes since 0.95`_, but the upgrade process should be fairly simple
 and should not require major changes to existing applications.
 
 However, we're also releasing 0.96 now because we have a set of
@@ -17,9 +17,21 @@ next official release; then you'll be able to upgrade in one step
 instead of needing to make incremental changes to keep up with the
 development version of Django.
 
-Backwards-incompatible changes
+Changes since the 0.96 release
 ==============================
 
+This release contains fixes for a security vulnerability discovered after the
+initial release of Django 0.96. A bug in the i18n framework could allow an
+attacker to send extremely large strings in the Accept-Language header and
+cause a denial of service by filling available memory.
+
+Because this problems wasn't discovered and fixed until after the 0.96
+release, it's recommended that you use this release rather than the original
+0.96.
+
+Backwards-incompatible changes since 0.95
+=========================================
+
 The following changes may require you to update your code when you switch from
 0.95 to 0.96:
 

@@ -32,12 +32,9 @@
     for file_info in data_files:
         file_info[0] = '/PURELIB/%s' % file_info[0]
 
-# Dynamically calculate the version based on django.VERSION.
-version = "%d.%d-%s" % (__import__('django').VERSION)
-
 setup(
     name = "Django",
-    version = version,
+    version = "0.96.1",
     url = 'http://www.djangoproject.com/',
     author = 'Lawrence Journal-World',
     author_email = 'holovaty@gmail.com',

@@ -1 +1 @@
-VERSION = (0, 95.1, None)
+VERSION = (0, 95.2, None)

@@ -1,6 +1,9 @@
 "Translation helper functions"
 
-import os, re, sys
+import locale
+import os
+import re
+import sys
 import gettext as gettext_module
 from cStringIO import StringIO
 from django.utils.functional import lazy
@@ -25,15 +28,25 @@ def currentThread():
 # The default translation is based on the settings file.
 _default = None
 
-# This is a cache for accept-header to translation object mappings to prevent
-# the accept parser to run multiple times for one user.
+# This is a cache for normalised accept-header languages to prevent multiple
+# file lookups when checking the same locale on repeated requests.
 _accepted = {}
 
-def to_locale(language):
+# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9.
+accept_language_re = re.compile(r'''
+        ([A-Za-z]{1,8}(?:-[A-Za-z]{1,8})*|\*)   # "en", "en-au", "x-y-z", "*"
+        (?:;q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"
+        (?:\s*,\s*|$)                            # Multiple accepts per header.
+        ''', re.VERBOSE)
+
+def to_locale(language, to_lower=False):
     "Turns a language name (en-us) into a locale name (en_US)."
     p = language.find('-')
     if p >= 0:
-        return language[:p].lower()+'_'+language[p+1:].upper()
+        if to_lower:
+            return language[:p].lower()+'_'+language[p+1:].lower()
+        else:
+            return language[:p].lower()+'_'+language[p+1:].upper()
     else:
         return language.lower()
 
@@ -309,46 +322,40 @@ def get_language_from_request(request):
         if lang_code in supported and lang_code is not None and check_for_language(lang_code):
             return lang_code
 
-    lang_code = request.COOKIES.get('django_language', None)
-    if lang_code in supported and lang_code is not None and check_for_language(lang_code):
+    lang_code = request.COOKIES.get('django_language')
+    if lang_code and lang_code in supported and check_for_language(lang_code):
         return lang_code
 
-    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', None)
-    if accept is not None:
-
-        t = _accepted.get(accept, None)
-        if t is not None:
-            return t
-
-        def _parsed(el):
-            p = el.find(';q=')
-            if p >= 0:
-                lang = el[:p].strip()
-                order = int(float(el[p+3:].strip())*100)
-            else:
-                lang = el
-                order = 100
-            p = lang.find('-')
-            if p >= 0:
-                mainlang = lang[:p]
-            else:
-                mainlang = lang
-            return (lang, mainlang, order)
-
-        langs = [_parsed(el) for el in accept.split(',')]
-        langs.sort(lambda a,b: -1*cmp(a[2], b[2]))
-
-        for lang, mainlang, order in langs:
-            if lang in supported or mainlang in supported:
-                langfile = gettext_module.find('django', globalpath, [to_locale(lang)])
-                if langfile:
-                    # reconstruct the actual language from the language
-                    # filename, because otherwise we might incorrectly
-                    # report de_DE if we only have de available, but
-                    # did find de_DE because of language normalization
-                    lang = langfile[len(globalpath):].split(os.path.sep)[1]
-                    _accepted[accept] = lang
-                    return lang
+    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
+    for lang, unused in parse_accept_lang_header(accept):
+        if lang == '*':
+            break
+
+        # We have a very restricted form for our language files (no encoding
+        # specifier, since they all must be UTF-8 and only one possible
+        # language each time. So we avoid the overhead of gettext.find() and
+        # look up the MO file manually.
+
+        normalized = locale.locale_alias.get(to_locale(lang, True))
+        if not normalized:
+            continue
+
+        # Remove the default encoding from locale_alias
+        normalized = normalized.split('.')[0]
+
+        if normalized in _accepted:
+            # We've seen this locale before and have an MO file for it, so no
+            # need to check again.
+            return _accepted[normalized]
+
+        for lang in (normalized, normalized.split('_')[0]):
+            if lang not in supported:
+                continue
+            langfile = os.path.join(globalpath, lang, 'LC_MESSAGES',
+                    'django.mo')
+            if os.path.exists(langfile):
+                _accepted[normalized] = lang
+            return lang
 
     return settings.LANGUAGE_CODE
 
@@ -494,3 +501,24 @@ def string_concat(*strings):
     return ''.join([str(el) for el in strings])
 
 string_concat = lazy(string_concat, str)
+
+def parse_accept_lang_header(lang_string):
+    """
+    Parses the lang_string, which is the body of an HTTP Accept-Language
+    header, and returns a list of (lang, q-value), ordered by 'q' values.
+
+    Any format errors in lang_string results in an empty list being returned.
+    """
+    result = []
+    pieces = accept_language_re.split(lang_string)
+    if pieces[-1]:
+        return []
+    for i in range(0, len(pieces) - 1, 3):
+        first, lang, priority = pieces[i : i + 3]
+        if first:
+            return []
+        priority = priority and float(priority) or 1.0
+        result.append((lang, priority))
+    result.sort(lambda x, y: -cmp(x[1], y[1]))
+    return result
+

@@ -1,9 +1,8 @@
 ===================================
-Django version 0.95.1 release notes
+Django version 0.95.2 release notes
 ===================================
 
-
-Welcome to the Django 0.95.1 release.
+Welcome to the Django 0.95.2 release.
 
 This represents a significant advance in Django development since the 0.91
 release in January 2006. The details of every change in this release would be
@@ -107,6 +106,10 @@ initial release of Django 0.95; these include:
     * A patch which disables debugging mode in the flup FastCGI
       package Django uses to launch its FastCGI server, which prevents
       tracebacks from bubbling up during production use.
+      
+    * A security fix to the i18n framework which could allow an 
+      attacker to send extremely large strings in the Accept-Language 
+      header and cause a denial of service by filling available memory.
 
 Because these problems weren't discovered and fixed until after the
 0.95 release, it's recommended that you use this release rather than

@@ -5,7 +5,7 @@
 
 setup(
     name = "Django",
-    version = "0.95.1",
+    version = "0.95.2",
     url = 'http://www.djangoproject.com/',
     author = 'Lawrence Journal-World',
     author_email = 'holovaty@gmail.com',

@@ -1,6 +1,9 @@
 "translation helper functions"
 
-import os, re, sys
+import locale
+import os
+import re
+import sys
 import gettext as gettext_module
 from cStringIO import StringIO
 from django.utils.functional import lazy
@@ -25,15 +28,25 @@ def currentThread():
 # The default translation is based on the settings file.
 _default = None
 
-# This is a cache for accept-header to translation object mappings to prevent
-# the accept parser to run multiple times for one user.
+# This is a cache for normalised accept-header languages to prevent multiple
+# file lookups when checking the same locale on repeated requests.
 _accepted = {}
 
-def to_locale(language):
+# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9.
+accept_language_re = re.compile(r'''
+        ([A-Za-z]{1,8}(?:-[A-Za-z]{1,8})*|\*)   # "en", "en-au", "x-y-z", "*"
+        (?:;q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"
+        (?:\s*,\s*|$)                            # Multiple accepts per header.
+        ''', re.VERBOSE)
+
+def to_locale(language, to_lower=False):
     "Turns a language name (en-us) into a locale name (en_US)."
     p = language.find('-')
     if p >= 0:
-        return language[:p].lower()+'_'+language[p+1:].upper()
+        if to_lower:
+            return language[:p].lower()+'_'+language[p+1:].lower()
+        else:
+            return language[:p].lower()+'_'+language[p+1:].upper()
     else:
         return language.lower()
 
@@ -297,46 +310,40 @@ def get_language_from_request(request):
         if lang_code in supported and lang_code is not None and check_for_language(lang_code):
             return lang_code
 
-    lang_code = request.COOKIES.get('django_language', None)
-    if lang_code in supported and lang_code is not None and check_for_language(lang_code):
+    lang_code = request.COOKIES.get('django_language')
+    if lang_code and lang_code in supported and check_for_language(lang_code):
         return lang_code
 
-    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', None)
-    if accept is not None:
-
-        t = _accepted.get(accept, None)
-        if t is not None:
-            return t
-
-        def _parsed(el):
-            p = el.find(';q=')
-            if p >= 0:
-                lang = el[:p].strip()
-                order = int(float(el[p+3:].strip())*100)
-            else:
-                lang = el
-                order = 100
-            p = lang.find('-')
-            if p >= 0:
-                mainlang = lang[:p]
-            else:
-                mainlang = lang
-            return (lang, mainlang, order)
-
-        langs = [_parsed(el) for el in accept.split(',')]
-        langs.sort(lambda a,b: -1*cmp(a[2], b[2]))
-
-        for lang, mainlang, order in langs:
-            if lang in supported or mainlang in supported:
-                langfile = gettext_module.find('django', globalpath, [to_locale(lang)])
-                if langfile:
-                    # reconstruct the actual language from the language
-                    # filename, because otherwise we might incorrectly
-                    # report de_DE if we only have de available, but
-                    # did find de_DE because of language normalization
-                    lang = langfile[len(globalpath):].split(os.path.sep)[1]
-                    _accepted[accept] = lang
-                    return lang
+    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
+    for lang, unused in parse_accept_lang_header(accept):
+        if lang == '*':
+            break
+
+        # We have a very restricted form for our language files (no encoding
+        # specifier, since they all must be UTF-8 and only one possible
+        # language each time. So we avoid the overhead of gettext.find() and
+        # look up the MO file manually.
+
+        normalized = locale.locale_alias.get(to_locale(lang, True))
+        if not normalized:
+            continue
+
+        # Remove the default encoding from locale_alias
+        normalized = normalized.split('.')[0]
+
+        if normalized in _accepted:
+            # We've seen this locale before and have an MO file for it, so no
+            # need to check again.
+            return _accepted[normalized]
+
+        for lang in (normalized, normalized.split('_')[0]):
+            if lang not in supported:
+                continue
+            langfile = os.path.join(globalpath, lang, 'LC_MESSAGES',
+                    'django.mo')
+            if os.path.exists(langfile):
+                _accepted[normalized] = lang
+            return lang
 
     return settings.LANGUAGE_CODE
 
@@ -457,3 +464,23 @@ def templatize(src):
             else:
                 out.write(blankout(t.contents, 'X'))
     return out.getvalue()
+
+def parse_accept_lang_header(lang_string):
+    """
+    Parses the lang_string, which is the body of an HTTP Accept-Language
+    header, and returns a list of (lang, q-value), ordered by 'q' values.
+
+    Any format errors in lang_string results in an empty list being returned.
+    """
+    result = []
+    pieces = accept_language_re.split(lang_string)
+    if pieces[-1]:
+        return []
+    for i in range(0, len(pieces) - 1, 3):
+        first, lang, priority = pieces[i : i + 3]
+        if first:
+            return []
+        priority = priority and float(priority) or 1.0
+        result.append((lang, priority))
+    result.sort(lambda x, y: -cmp(x[1], y[1]))
+    return result

@@ -5,7 +5,7 @@
 
 setup(
     name = "Django",
-    version = "0.91",
+    version = "0.91.1",
     url = 'http://www.djangoproject.com/',
     author = 'Lawrence Journal-World',
     author_email = 'holovaty@gmail.com',