diff --git a/requirements.txt b/requirements.txt index bb38adb..fcad15e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ -Django<2 +django<2 python-dateutil +beautifulsoup +translitcodec diff --git a/utils/templatetags/text_tags.py b/utils/templatetags/text_tags.py index ef0c0b5..1d82338 100644 --- a/utils/templatetags/text_tags.py +++ b/utils/templatetags/text_tags.py @@ -9,6 +9,8 @@ from django.template.defaultfilters import stringfilter from django.utils.html import conditional_escape from django.utils.safestring import mark_safe +from .. import text as text_utils + register = template.Library() @@ -39,3 +41,9 @@ def nbsp(text, autoescape=True): else: esc = lambda x: x return mark_safe(WHITESPACE.sub(' ', esc(text.strip()))) + + +@register.filter(needs_autoescape=False) +@stringfilter +def html_entities_to_unicode(text): + return mark_safe(text_utils.html_entities_to_unicode(text)) diff --git a/utils/text.py b/utils/text.py index 1c88a85..d6d1f38 100644 --- a/utils/text.py +++ b/utils/text.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -# Erik Stein , 2015 +# Erik Stein , 2015-2017 from django.utils.text import slugify from django.utils import six -from django.utils.encoding import force_text +from django.utils.encoding import force_text, smart_text from django.utils.functional import allow_lazy from django.utils.safestring import SafeText -# import unicodedata -import translitcodec +from BeautifulSoup import BeautifulStoneSoup +import translitcodec # provides 'translit/long', used by codecs.encode() import codecs @@ -27,21 +27,11 @@ def slugify_long(value): slugify_long = allow_lazy(slugify_long, six.text_type, SafeText) -def slugify_german(value): - """ - Transliterates Umlaute before calling django's slugify function. - """ - umlaute = { - 'Ä': 'Ae', - 'Ö': 'Oe', - 'Ü': 'Ue', - 'ä': 'ae', - 'ö': 'oe', - 'ü': 'ue', - 'ß': 'ss', - } +# Spreading umlauts is included in the translit/long codec. +slugify_german = slugify_long - value = force_text(value) - umap = {ord(key): unicode(val) for key, val in umlaute.items()} - return slugify(value.translate(umap)) -slugify_german = allow_lazy(slugify_german, six.text_type, SafeText) + +def html_entities_to_unicode(html): + text = smart_text(BeautifulStoneSoup(html, convertEntities=BeautifulStoneSoup.ALL_ENTITIES)) + return text +html_entities_to_unicode = allow_lazy(html_entities_to_unicode, six.text_type, SafeText) diff --git a/utils/translation.py b/utils/translation.py index f71467c..58b4631 100644 --- a/utils/translation.py +++ b/utils/translation.py @@ -18,6 +18,9 @@ from django.views.generic import TemplateView from django.views.i18n import LANGUAGE_QUERY_PARAMETER +FALLBACK_LANGUAGE_CODE = getattr(settings, 'FALLBACK_LANGUAGE_CODE', 'en') + + def _normalize_language_code(language_code): """ Makes sure language code is not an empty string. @@ -26,7 +29,7 @@ def _normalize_language_code(language_code): language_code or translation.get_language() or settings.LANGUAGE_CODE or - 'en' + FALLBACK_LANGUAGE_CODE )