Commit 2c837233 authored by Baptiste Mispelon's avatar Baptiste Mispelon
Browse files

Fixed #21574 -- Handle bytes consistently in utils.text.normalize_newlines. 

All input is now coerced to text before being normalized.
This changes nothing under Python 2 but it allows bytes
to be passed to the function without a TypeError under Python3
(bytes are assumed to be utf-8 encoded text).

Thanks to trac user vajrasky for the report.
parent b9c7234e

django/utils/text.py

+3 −1
Original line number Diff line number Diff line
@@ -250,7 +250,9 @@ get_text_list = allow_lazy(get_text_list, six.text_type) 




def normalize_newlines(text):

    return force_text(re_newlines.sub('\n', text))

    """Normalizes CRLF and CR newlines to just LF."""

    text = force_text(text)

    return re_newlines.sub('\n', text)

normalize_newlines = allow_lazy(normalize_newlines, six.text_type)

tests/utils_tests/test_text.py

+7 −1
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
from __future__ import unicode_literals



from django.test import SimpleTestCase

from django.utils import text

from django.utils import six, text





class TestUtilsText(SimpleTestCase):
@@ -114,6 +114,12 @@ class TestUtilsText(SimpleTestCase): 
        self.assertEqual(text.normalize_newlines("abcdefghi"), "abcdefghi")

        self.assertEqual(text.normalize_newlines(""), "")



    def test_normalize_newlines_bytes(self):

        """normalize_newlines should be able to handle bytes too"""

        normalized = text.normalize_newlines(b"abc\ndef\rghi\r\n")

        self.assertEqual(normalized, "abc\ndef\nghi\n")

        self.assertIsInstance(normalized, six.text_type)



    def test_slugify(self):

        items = (

            ('Hello, World!', 'hello-world'),