Added html_entities_to_unicode function and template tag.

requirements.txt

@@ -1,2 +1,4 @@
-Django<2
+django<2
 python-dateutil
+beautifulsoup
+translitcodec

utils/templatetags/text_tags.py

@@ -9,6 +9,8 @@ from django.template.defaultfilters import stringfilter
 from django.utils.html import conditional_escape
 from django.utils.safestring import mark_safe
 
+from .. import text as text_utils
+
 
 register = template.Library()
 
@@ -39,3 +41,9 @@ def nbsp(text, autoescape=True):
     else:
         esc = lambda x: x
     return mark_safe(WHITESPACE.sub(' ', esc(text.strip())))
+
+
+@register.filter(needs_autoescape=False)
+@stringfilter
+def html_entities_to_unicode(text):
+    return mark_safe(text_utils.html_entities_to_unicode(text))

utils/text.py

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
-# Erik Stein <code@classlibrary.net>, 2015
+# Erik Stein <code@classlibrary.net>, 2015-2017
 
 from django.utils.text import slugify
 from django.utils import six
-from django.utils.encoding import force_text
+from django.utils.encoding import force_text, smart_text
 from django.utils.functional import allow_lazy
 from django.utils.safestring import SafeText
 
-# import unicodedata
+from BeautifulSoup import BeautifulStoneSoup
-import translitcodec
+import translitcodec  # provides 'translit/long', used by codecs.encode()
 import codecs
 
 
@@ -27,21 +27,11 @@ def slugify_long(value):
 slugify_long = allow_lazy(slugify_long, six.text_type, SafeText)
 
 
-def slugify_german(value):
+# Spreading umlauts is included in the translit/long codec.
-    """
+slugify_german = slugify_long
-    Transliterates Umlaute before calling django's slugify function.
-    """
-    umlaute = {
-        'Ä': 'Ae',
-        'Ö': 'Oe',
-        'Ü': 'Ue',
-        'ä': 'ae',
-        'ö': 'oe',
-        'ü': 'ue',
-        'ß': 'ss',
-    }
 
-    value = force_text(value)
+
-    umap = {ord(key): unicode(val) for key, val in umlaute.items()}
+def html_entities_to_unicode(html):
-    return slugify(value.translate(umap))
+    text = smart_text(BeautifulStoneSoup(html, convertEntities=BeautifulStoneSoup.ALL_ENTITIES))
-slugify_german = allow_lazy(slugify_german, six.text_type, SafeText)
+    return text
+html_entities_to_unicode = allow_lazy(html_entities_to_unicode, six.text_type, SafeText)

utils/translation.py

@@ -18,6 +18,9 @@ from django.views.generic import TemplateView
 from django.views.i18n import LANGUAGE_QUERY_PARAMETER
 
 
+FALLBACK_LANGUAGE_CODE = getattr(settings, 'FALLBACK_LANGUAGE_CODE', 'en')
+
+
 def _normalize_language_code(language_code):
     """
     Makes sure language code is not an empty string.
@@ -26,7 +29,7 @@ def _normalize_language_code(language_code):
         language_code or
         translation.get_language() or
         settings.LANGUAGE_CODE or
-        'en'
+        FALLBACK_LANGUAGE_CODE
     )