Commit ff7556c4 authored by Ramiro Morales's avatar Ramiro Morales
Browse files

Fixed #11240 -- Made makemessages i18n command escape % symbols in literals... 

Fixed #11240 -- Made makemessages i18n command escape % symbols in literals passed to the trans tag.

This avoids problems with unintended automatic detection, marking and
validation of Python string formatting specifiers performed by
xgettext(1)/msgfmt(1) that stem from the fact that under the hood makemessages
converts templates to Python code before passing them to xgettext.

This also makes it consistent with its behavior on literals passed to the
blocktrans tag.

Thanks Jannis and claude for reviewing and feedback.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@17190 bcc190cf-cafb-0310-a4f2-bffc1f526a37
parent 073d987a

django/utils/translation/trans_real.py

+2 −0
Original line number Diff line number Diff line
@@ -439,6 +439,7 @@ block_re = re.compile(r"""^\s*blocktrans(\s+.*context\s+(?:"[^"]*?")|(?:'[^']*?' 
endblock_re = re.compile(r"""^\s*endblocktrans$""")

plural_re = re.compile(r"""^\s*plural$""")

constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?'))\)""")

one_percent_re = re.compile(r"""(?<!%)%(?!%)""")





def templatize(src, origin=None):
@@ -529,6 +530,7 @@ def templatize(src, origin=None): 
                        g = g.strip('"')

                    elif g[0] == "'":

                        g = g.strip("'")

                    g = one_percent_re.sub('%%', g)

                    if imatch.group(2):

                        # A context is provided

                        context_match = context_re.match(imatch.group(2))

tests/regressiontests/i18n/commands/compilation.py

+55 −8
Original line number Diff line number Diff line
@@ -7,17 +7,15 @@ except ImportError: 
from django.core.management import CommandError

from django.core.management.commands.compilemessages import compile_messages

from django.test import TestCase

from django.test.utils import override_settings

from django.utils import translation





LOCALE='es_AR'

test_dir = os.path.abspath(os.path.dirname(__file__))



class MessageCompilationTests(TestCase):



    MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE



    def setUp(self):

        self._cwd = os.getcwd()

        self.test_dir = os.path.abspath(os.path.dirname(__file__))



    def tearDown(self):

        os.chdir(self._cwd)
@@ -25,11 +23,60 @@ class MessageCompilationTests(TestCase): 


class PoFileTests(MessageCompilationTests):



    LOCALE='es_AR'

    MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE



    def test_bom_rejection(self):

        os.chdir(self.test_dir)

        # We don't use the django.core.management intrastructure (call_command()

        os.chdir(test_dir)

        # We don't use the django.core.management infrastructure (call_command()

        # et al) because CommandError's cause exit(1) there. We test the

        # underlying compile_messages function instead

        out = StringIO()

        self.assertRaises(CommandError, compile_messages, out, locale=LOCALE)

        self.assertRaises(CommandError, compile_messages, out, locale=self.LOCALE)

        self.assertFalse(os.path.exists(self.MO_FILE))





class PoFileContentsTests(MessageCompilationTests):

    # Ticket #11240



    LOCALE='fr'

    MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE



    def setUp(self):

        super(PoFileContentsTests, self).setUp()

        self.addCleanup(os.unlink, os.path.join(test_dir, self.MO_FILE))



    def test_percent_symbol_in_po_file(self):

        os.chdir(test_dir)

        # We don't use the django.core.management infrastructure (call_command()

        # et al) because CommandError's cause exit(1) there. We test the

        # underlying compile_messages function instead

        out = StringIO()

        compile_messages(out, locale=self.LOCALE)

        self.assertTrue(os.path.exists(self.MO_FILE))





class PercentRenderingTests(MessageCompilationTests):

    # Ticket #11240 -- Testing rendering doesn't belong here but we are trying

    # to keep tests for all the stack together



    LOCALE='it'

    MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE



    @override_settings(LOCALE_PATHS=(os.path.join(test_dir, 'locale'),))

    def test_percent_symbol_escaping(self):

        from django.template import Template, Context

        os.chdir(test_dir)

        # We don't use the django.core.management infrastructure (call_command()

        # et al) because CommandError's cause exit(1) there. We test the

        # underlying compile_messages function instead

        out = StringIO()

        compile_messages(out, locale=self.LOCALE)

        with translation.override(self.LOCALE):

            t = Template('{% load i18n %}{% trans "Looks like a str fmt spec %% o but shouldn\'t be interpreted as such" %}')

            rendered = t.render(Context({}))

            self.assertEqual(rendered, 'IT translation contains %% for the above string')



            t = Template('{% load i18n %}{% trans "Completed 50%% of all the tasks" %}')

            rendered = t.render(Context({}))

            self.assertEqual(rendered, 'IT translation of Completed 50%% of all the tasks')

tests/regressiontests/i18n/commands/extraction.py

+114 −71
Original line number Diff line number Diff line
@@ -31,10 +31,13 @@ class ExtractorTests(TestCase): 
        os.chdir(self._cwd)



    def assertMsgId(self, msgid, s, use_quotes=True):

        q = '"'

        if use_quotes:

            msgid = '"%s"' % msgid

            q = "'"

        needle = 'msgid %s' % msgid

        msgid = re.escape(msgid)

        return self.assertTrue(re.search('^msgid %s' % msgid, s, re.MULTILINE))

        return self.assertTrue(re.search('^msgid %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})



    def assertNotMsgId(self, msgid, s, use_quotes=True):

        if use_quotes:
@@ -49,7 +52,9 @@ class BasicExtractorTests(ExtractorTests): 
        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertTrue('#. Translators: This comment should be extracted' in po_contents)

            self.assertTrue('This comment should not be extracted' not in po_contents)

            # Comments in templates
@@ -71,11 +76,31 @@ class BasicExtractorTests(ExtractorTests): 
            self.assertTrue('#. Translators: One-line translator comment #5 -- with non ASCII characters: áéíóúö' in po_contents)

            self.assertTrue('#. Translators: Two-line translator comment #5 -- with non ASCII characters: áéíóúö\n#. continued here.' in po_contents)



    def test_templatize(self):

    def test_templatize_trans_tag(self):

        # ticket #11240

        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertMsgId('Literal with a percent symbol at the end %%', po_contents)

            self.assertMsgId('Literal with a percent %% symbol in the middle', po_contents)

            self.assertMsgId('Completed 50%% of all the tasks', po_contents)

            self.assertMsgId('Completed 99%% of all the tasks', po_contents)

            self.assertMsgId("Shouldn't double escape this sequence: %% (two percent signs)", po_contents)

            self.assertMsgId("Shouldn't double escape this sequence %% either", po_contents)

            self.assertMsgId("Looks like a str fmt spec %%s but shouldn't be interpreted as such", po_contents)

            self.assertMsgId("Looks like a str fmt spec %% o but shouldn't be interpreted as such", po_contents)



    def test_templatize_blocktrans_tag(self):

        # ticket #11966

        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0)

        self.assertTrue(os.path.exists(self.PO_FILE))

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertMsgId('I think that 100%% is more that 50%% of anything.', po_contents)

            self.assertMsgId('I think that 100%% is more that 50%% of %(obj)s.', po_contents)
@@ -103,7 +128,9 @@ class BasicExtractorTests(ExtractorTests): 
        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            # {% trans %}

            self.assertTrue('msgctxt "Special trans context #1"' in po_contents)

            self.assertTrue("Translatable literal #7a" in po_contents)
@@ -132,7 +159,9 @@ class JavascriptExtractorTests(ExtractorTests): 
        os.chdir(self.test_dir)

        management.call_command('makemessages', domain='djangojs', locale=LOCALE, verbosity=0)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertMsgId('This literal should be included.', po_contents)

            self.assertMsgId('This one as well.', po_contents)

            self.assertMsgId(r'He said, \"hello\".', po_contents)
@@ -154,7 +183,9 @@ class IgnoredExtractorTests(ExtractorTests): 
        pattern1 = os.path.join('ignore_dir', '*')

        management.call_command('makemessages', locale=LOCALE, verbosity=0, ignore_patterns=[pattern1])

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertMsgId('This literal should be included.', po_contents)

            self.assertNotMsgId('This should be ignored.', po_contents)
@@ -184,7 +215,9 @@ class SymlinkExtractorTests(ExtractorTests): 
            os.chdir(self.test_dir)

            management.call_command('makemessages', locale=LOCALE, verbosity=0, symlinks=True)

            self.assertTrue(os.path.exists(self.PO_FILE))

            po_contents = open(self.PO_FILE, 'r').read()

            #po_contents = open(self.PO_FILE, 'r').read()

            with open(self.PO_FILE, 'r') as fp:

                po_contents = fp.read()

                self.assertMsgId('This literal should be included.', po_contents)

                self.assertTrue('templates_symlinked/test.html' in po_contents)
@@ -195,7 +228,9 @@ class CopyPluralFormsExtractorTests(ExtractorTests): 
        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertTrue('Plural-Forms: nplurals=2; plural=(n != 1)' in po_contents)
@@ -205,14 +240,18 @@ class NoWrapExtractorTests(ExtractorTests): 
        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0, no_wrap=True)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertMsgId('This literal should also be included wrapped or not wrapped depending on the use of the --no-wrap option.', po_contents)



    def test_no_wrap_disabled(self):

        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0, no_wrap=False)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertMsgId('""\n"This literal should also be included wrapped or not wrapped depending on the "\n"use of the --no-wrap option."', po_contents, use_quotes=False)
@@ -222,12 +261,16 @@ class NoLocationExtractorTests(ExtractorTests): 
        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0, no_location=True)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertFalse('#: templates/test.html:55' in po_contents)



    def test_no_location_disabled(self):

        os.chdir(self.test_dir)

        management.call_command('makemessages', locale=LOCALE, verbosity=0, no_location=False)

        self.assertTrue(os.path.exists(self.PO_FILE))

        po_contents = open(self.PO_FILE, 'r').read()

        #po_contents = open(self.PO_FILE, 'r').read()

        with open(self.PO_FILE, 'r') as fp:

            po_contents = fp.read()

            self.assertTrue('#: templates/test.html:55' in po_contents)

tests/regressiontests/i18n/commands/locale/fr/LC_MESSAGES/django.po

0 → 100644
+71 −0
Original line number Diff line number Diff line 
# SOME DESCRIPTIVE TITLE.

# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER

# This file is distributed under the same license as the PACKAGE package.

# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.

#

msgid ""

msgstr ""

"Project-Id-Version: PACKAGE VERSION\n"

"Report-Msgid-Bugs-To: \n"

"POT-Creation-Date: 2011-12-04 04:59-0600\n"

"PO-Revision-Date: 2011-12-10 19:12-0300\n"

"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"

"Language-Team: LANGUAGE <LL@li.org>\n"

"Language: fr\n"

"MIME-Version: 1.0\n"

"Content-Type: text/plain; charset=UTF-8\n"

"Content-Transfer-Encoding: 8bit\n"

"Plural-Forms: nplurals=2; plural=(n != 1)\n"



#. Translators: Django template comment for translators

#: templates/test.html:9

#, python-format

msgid "I think that 100%% is more that 50%% of anything."

msgstr ""



#: templates/test.html:10

#, python-format

msgid "I think that 100%% is more that 50%% of %(obj)s."

msgstr ""



#: templates/test.html:70

#, python-format

msgid "Literal with a percent symbol at the end %%"

msgstr ""



#: templates/test.html:71

#, python-format

msgid "Literal with a percent %% symbol in the middle"

msgstr ""



#: templates/test.html:72

#, python-format

msgid "Completed 50%% of all the tasks"

msgstr ""



#: templates/test.html:73

#, python-format

msgctxt "ctx0"

msgid "Completed 99%% of all the tasks"

msgstr ""



#: templates/test.html:74

#, python-format

msgid "Shouldn't double escape this sequence: %% (two percent signs)"

msgstr ""



#: templates/test.html:75

#, python-format

msgctxt "ctx1"

msgid "Shouldn't double escape this sequence %% either"

msgstr ""



#: templates/test.html:76

#, python-format

msgid "Looks like a str fmt spec %%s but shouldn't be interpreted as such"

msgstr "Translation of the above string"



#: templates/test.html:77

#, python-format

msgid "Looks like a str fmt spec %% o but shouldn't be interpreted as such"

msgstr "Translation contains %% for the above string"

tests/regressiontests/i18n/commands/templates/test.html

+10 −1
Original line number Diff line number Diff line
@@ -66,3 +66,12 @@ continued here.{% endcomment %} 
{% blocktrans count 2 context "Special blocktrans context #2" %}Translatable literal #8b-singular{% plural %}Translatable literal #8b-plural{% endblocktrans %}

{% blocktrans context "Special blocktrans context #3" count 2 %}Translatable literal #8c-singular{% plural %}Translatable literal #8c-plural{% endblocktrans %}

{% blocktrans with a=1 context "Special blocktrans context #4" %}Translatable literal #8d {{ a }}{% endblocktrans %}



{% trans "Literal with a percent symbol at the end %" %}

{% trans "Literal with a percent % symbol in the middle" %}

{% trans "Completed 50% of all the tasks" %}

{% trans "Completed 99% of all the tasks" context "ctx0" %}

{% trans "Shouldn't double escape this sequence: %% (two percent signs)" %}

{% trans "Shouldn't double escape this sequence %% either" context "ctx1" %}

{% trans "Looks like a str fmt spec %s but shouldn't be interpreted as such" %}

{% trans "Looks like a str fmt spec % o but shouldn't be interpreted as such" %}