Commit 47ddd6a4 authored by Ramiro Morales's avatar Ramiro Morales
Browse files

Fixed #19552 -- Enhanced makemessages handling of ``{# #}``-style template comments. 

They are simply ignored now. This allows for a more correct behavior when
they are placed before translatable constructs on the same line.

Previously, the latter were wrongly ignored because the former were
preserved when converting template code to the internal Python-syntax
form later fed to xgettext but Python has no ``/* ... */``-style
comments.

Also, special comments directed to translators are now only taken in
account when they are located at the end of a line. e.g.::

  {# Translators: ignored #}{% trans "Literal A" %}{# Translators: valid, associated with "Literal B" below #}
  {% trans "Literal B" %}

Behavior of ``{% comment %}...{% endcomment %}``tags remains unchanged.

Thanks juneih at redpill-linpro dot com for the report and Claude for
his work on the issue.
parent eb9430fc

django/utils/translation/__init__.py

+5 −0
Original line number Diff line number Diff line
@@ -21,6 +21,11 @@ __all__ = [ 
    'npgettext', 'npgettext_lazy',

]





class TranslatorCommentWarning(SyntaxWarning):

    pass





# Here be dragons, so a short explanation of the logic won't hurt:

# We are trying to solve two problems: (1) access settings, in particular

# settings.USE_I18N, as late as possible, so that modules can be imported

django/utils/translation/trans_real.py

+30 −1
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ import re 
import sys

import gettext as gettext_module

from threading import local

import warnings



from django.utils.importlib import import_module

from django.utils.encoding import force_str, force_text
@@ -14,6 +15,7 @@ from django.utils._os import upath 
from django.utils.safestring import mark_safe, SafeData

from django.utils import six

from django.utils.six import StringIO

from django.utils.translation import TranslatorCommentWarning





# Translations are cached in a dictionary for every language+app tuple.
@@ -41,6 +43,7 @@ accept_language_re = re.compile(r''' 


language_code_prefix_re = re.compile(r'^/([\w-]+)(/|$)')





def to_locale(language, to_lower=False):

    """

    Turns a language name (en-us) into a locale name (en_US). If 'to_lower' is
@@ -468,6 +471,9 @@ def templatize(src, origin=None): 
    plural = []

    incomment = False

    comment = []

    lineno_comment_map = {}

    comment_lineno_cache = None



    for t in Lexer(src, origin).tokenize():

        if incomment:

            if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment':
@@ -529,7 +535,27 @@ def templatize(src, origin=None): 
                    plural.append(contents)

                else:

                    singular.append(contents)



        else:

            # Handle comment tokens (`{# ... #}`) plus other constructs on

            # the same line:

            if comment_lineno_cache is not None:

                cur_lineno = t.lineno + t.contents.count('\n')

                if comment_lineno_cache == cur_lineno:

                    if t.token_type != TOKEN_COMMENT:

                        for c in lineno_comment_map[comment_lineno_cache]:

                            filemsg = ''

                            if origin:

                                filemsg = 'file %s, ' % origin

                            warn_msg = ("The translator-targeted comment '%s' "

                                "(%sline %d) was ignored, because it wasn't the last item "

                                "on the line.") % (c, filemsg, comment_lineno_cache)

                            warnings.warn(warn_msg, TranslatorCommentWarning)

                        lineno_comment_map[comment_lineno_cache] = []

                else:

                    out.write('# %s' % ' | '.join(lineno_comment_map[comment_lineno_cache]))

                comment_lineno_cache = None



            if t.token_type == TOKEN_BLOCK:

                imatch = inline_re.match(t.contents)

                bmatch = block_re.match(t.contents)
@@ -586,7 +612,10 @@ def templatize(src, origin=None): 
                    else:

                        out.write(blankout(p, 'F'))

            elif t.token_type == TOKEN_COMMENT:

                out.write(' # %s' % t.contents)

                if t.contents.lstrip().startswith(TRANSLATOR_COMMENT_MARK):

                    lineno_comment_map.setdefault(t.lineno,

                                                  []).append(t.contents)

                    comment_lineno_cache = t.lineno

            else:

                out.write(blankout(t.contents, 'X'))

    return force_str(out.getvalue())

docs/releases/1.6.txt

+20 −0
Original line number Diff line number Diff line
@@ -47,6 +47,26 @@ Backwards incompatible changes in 1.6 
  should review it as ``type='text'`` widgets might be now output as

  ``type='email'`` or ``type='url'`` depending on their corresponding field type.



* Extraction of translatable literals from templates with the

  :djadmin:`makemessages` command now correctly detects i18n constructs when

  they are located after a ``{#`` / ``#}``-type comment on the same line. E.g.:



  .. code-block:: html+django



    {# A comment #}{% trans "This literal was incorrectly ignored. Not anymore" %}



* (Related to the above item.) Validation of the placement of

  :ref:`translator-comments-in-templates` specified using ``{#`` / ``#}`` is now

  stricter. All translator comments not located at the end of their respective

  lines in a template are ignored and a warning is generated by

  :djadmin:`makemessages` when it finds them. E.g.:



  .. code-block:: html+django



    {# Translators: This is ignored #}{% trans "Translate me" %}

    {{ title }}{# Translators: Extracted and associated with 'Welcome' below #}

    <h1>{% trans "Welcome" %}</h1>



.. warning::



    In addition to the changes outlined in this section, be sure to review the

docs/topics/i18n/translation.txt

+70 −5
Original line number Diff line number Diff line
@@ -142,14 +142,22 @@ preceding the string, e.g.:: 
        # Translators: This message appears on the home page only

        output = ugettext("Welcome to my site.")



This also works in templates with the :ttag:`comment` tag:

The comment will then appear in the resulting ``.po`` file associated with the

translatable contruct located below it and should also be displayed by most

translation tools.



.. code-block:: html+django

.. note:: Just for completeness, this is the corresponding fragment of the

    resulting ``.po`` file:



    .. code-block:: po



    {% comment %}Translators: This is a text of the base template {% endcomment %}

        #. Translators: This message appears on the home page only

        # path/to/python/file.py:123

        msgid "Welcome to my site."

        msgstr ""



The comment will then appear in the resulting ``.po`` file and should also be

displayed by most translation tools.

This also works in templates. See :ref:`translator-comments-in-templates` for

more details.



Marking strings as no-op

------------------------
@@ -620,6 +628,63 @@ markers<contextual-markers>` using the ``context`` keyword: 


    {% blocktrans with name=user.username context "greeting" %}Hi {{ name }}{% endblocktrans %}



.. _translator-comments-in-templates:



Comments for translators in templates

-------------------------------------



Just like with :ref:`Python code <translator-comments>`, these notes for

translators can be specified using comments, either with the :ttag:`comment`

tag:



.. code-block:: html+django



    {% comment %}Translators: View verb{% endcomment %}

    {% trans "View" %}



    {% comment %}Translators: Short intro blurb{% endcomment %}

    <p>{% blocktrans %}A multiline translatable

    literal.{% endblocktrans %}</p>



or with the ``{#`` ... ``#}`` :ref:`one-line comment constructs <template-comments>`:



.. code-block:: html+django



    {# Translators: Label of a button that triggers search{% endcomment #}

    <button type="submit">{% trans "Go" %}</button>



    {# Translators: This is a text of the base template #}

    {% blocktrans %}Ambiguous translatable block of text{% endtransblock %}



.. note:: Just for completeness, these are the corresponding fragments of the

    resulting ``.po`` file:



    .. code-block:: po



        #. Translators: View verb

        # path/to/template/file.html:10

        msgid "View"

        msgstr ""



        #. Translators: Short intro blurb

        # path/to/template/file.html:13

        msgid ""

        "A multiline translatable"

        "literal."

        msgstr ""



        # ...



        #. Translators: Label of a button that triggers search

        # path/to/template/file.html:100

        msgid "Go"

        msgstr ""



        #. Translators:

        # path/to/template/file.html:103

        msgid "Ambiguous translatable block of text"

        msgstr ""



.. _template-translation-vars:



Other tags

docs/topics/templates.txt

+2 −0
Original line number Diff line number Diff line
@@ -250,6 +250,8 @@ You can also create your own custom template tags; see 
    tags and filters available for a given site. See

    :doc:`/ref/contrib/admin/admindocs`.



.. _template-comments:



Comments

========