Fixed #24372 - Replaced TokenParser usage with traditional parsing. (35885078) · Commits · Dom Sekotill / django

django/template/base.py

+0 −116

Original line number	Diff line number	Diff line
		@@ -431,122 +431,6 @@ class Parser(object):
		raise TemplateSyntaxError("Invalid filter: '%s'" % filter_name)


		class TokenParser(object):
		"""
		Subclass this and implement the top() method to parse a template line.
		When instantiating the parser, pass in the line from the Django template
		parser.

		The parser's "tagname" instance-variable stores the name of the tag that
		the filter was called with.
		"""
		def __init__(self, subject):
		self.subject = subject
		self.pointer = 0
		self.backout = []
		self.tagname = self.tag()

		def top(self):
		"""
		Overload this method to do the actual parsing and return the result.
		"""
		raise NotImplementedError('subclasses of Tokenparser must provide a top() method')

		def more(self):
		"""
		Returns True if there is more stuff in the tag.
		"""
		return self.pointer < len(self.subject)

		def back(self):
		"""
		Undoes the last microparser. Use this for lookahead and backtracking.
		"""
		if not len(self.backout):
		raise TemplateSyntaxError("back called without some previous "
		"parsing")
		self.pointer = self.backout.pop()

		def tag(self):
		"""
		A microparser that just returns the next tag from the line.
		"""
		subject = self.subject
		i = self.pointer
		if i >= len(subject):
		raise TemplateSyntaxError("expected another tag, found "
		"end of string: %s" % subject)
		p = i
		while i < len(subject) and subject[i] not in (' ', '\t'):
		i += 1
		s = subject[p:i]
		while i < len(subject) and subject[i] in (' ', '\t'):
		i += 1
		self.backout.append(self.pointer)
		self.pointer = i
		return s

		def value(self):
		"""
		A microparser that parses for a value: some string constant or
		variable name.
		"""
		subject = self.subject
		i = self.pointer

		def next_space_index(subject, i):
		"""
		Increment pointer until a real space (i.e. a space not within
		quotes) is encountered
		"""
		while i < len(subject) and subject[i] not in (' ', '\t'):
		if subject[i] in ('"', "'"):
		c = subject[i]
		i += 1
		while i < len(subject) and subject[i] != c:
		i += 1
		if i >= len(subject):
		raise TemplateSyntaxError("Searching for value. "
		"Unexpected end of string in column %d: %s" %
		(i, subject))
		i += 1
		return i

		if i >= len(subject):
		raise TemplateSyntaxError("Searching for value. Expected another "
		"value but found end of string: %s" %
		subject)
		if subject[i] in ('"', "'"):
		p = i
		i += 1
		while i < len(subject) and subject[i] != subject[p]:
		i += 1
		if i >= len(subject):
		raise TemplateSyntaxError("Searching for value. Unexpected "
		"end of string in column %d: %s" %
		(i, subject))
		i += 1

		# Continue parsing until next "real" space,
		# so that filters are also included
		i = next_space_index(subject, i)

		res = subject[p:i]
		while i < len(subject) and subject[i] in (' ', '\t'):
		i += 1
		self.backout.append(self.pointer)
		self.pointer = i
		return res
		else:
		p = i
		i = next_space_index(subject, i)
		s = subject[p:i]
		while i < len(subject) and subject[i] in (' ', '\t'):
		i += 1
		self.backout.append(self.pointer)
		self.pointer = i
		return s

		# This only matches constant strings (things in quotes or marked for
		# translation). Numbers are treated as variables for implementation reasons
		# (so that they retain their type when passed to filters).

django/templatetags/i18n.py

+49 −40

Original line number	Diff line number	Diff line
		from __future__ import unicode_literals

		import re
		import sys

		from django.conf import settings
		from django.template import Library, Node, TemplateSyntaxError, Variable
		from django.template.base import (
		TOKEN_TEXT, TOKEN_VAR, TokenParser, render_value_in_context,
		)
		from django.template.base import TOKEN_TEXT, TOKEN_VAR, render_value_in_context
		from django.template.defaulttags import token_kwargs
		from django.utils import six, translation

		@@ -348,42 +345,54 @@ def do_translate(parser, token):

		This is equivalent to calling pgettext instead of (u)gettext.
		"""
		class TranslateParser(TokenParser):
		def top(self):
		value = self.value()

		# Backwards Compatibility fix:
		# FilterExpression does not support single-quoted strings,
		# so we make a cheap localized fix in order to maintain
		# backwards compatibility with existing uses of ``trans``
		# where single quote use is supported.
		if value[0] == "'":
		m = re.match("^'([^']+)'(\\|.*$)", value)
		if m:
		value = '"%s"%s' % (m.group(1).replace('"', '\\"'), m.group(2))
		elif value[-1] == "'":
		value = '"%s"' % value[1:-1].replace('"', '\\"')
		bits = token.split_contents()
		if len(bits) < 2:
		raise TemplateSyntaxError("'%s' takes at least one argument" % bits[0])
		message_string = parser.compile_filter(bits[1])
		remaining = bits[2:]

		noop = False
		asvar = None
		message_context = None
		seen = set()
		invalid_context = {'as', 'noop'}

		while self.more():
		tag = self.tag()
		if tag == 'noop':
		while remaining:
		option = remaining.pop(0)
		if option in seen:
		raise TemplateSyntaxError(
		"The '%s' option was specified more than once." % option,
		)
		elif option == 'noop':
		noop = True
		elif tag == 'context':
		message_context = parser.compile_filter(self.value())
		elif tag == 'as':
		asvar = self.tag()
		elif option == 'context':
		try:
		value = remaining.pop(0)
		except IndexError:
		msg = "No argument provided to the '%s' tag for the context option." % bits[0]
		six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
		if value in invalid_context:
		raise TemplateSyntaxError(
		"Invalid argument '%s' provided to the '%s' tag for the context option" % (value, bits[0]),
		)
		message_context = parser.compile_filter(value)
		elif option == 'as':
		try:
		value = remaining.pop(0)
		except IndexError:
		msg = "No argument provided to the '%s' tag for the as option." % bits[0]
		six.reraise(TemplateSyntaxError, TemplateSyntaxError(msg), sys.exc_info()[2])
		asvar = value
		else:
		raise TemplateSyntaxError(
		"Only options for 'trans' are 'noop', "
		"'context \"xxx\"', and 'as VAR'.")
		return value, noop, asvar, message_context
		value, noop, asvar, message_context = TranslateParser(token.contents).top()
		return TranslateNode(parser.compile_filter(value), noop, asvar,
		message_context)
		"Unknown argument for '%s' tag: '%s'. The only options "
		"available are 'noop', 'context' \"xxx\", and 'as VAR'." % (
		bits[0], option,
		)
		)
		seen.add(option)

		return TranslateNode(message_string, noop, asvar, message_context)


		@register.tag("blocktrans")

tests/i18n/tests.py

+0 −4

Original line number	Diff line number	Diff line
		@@ -261,10 +261,6 @@ class TranslationTests(TestCase):
		rendered = t.render(Context())
		self.assertEqual(rendered, 'Value: Kann')

		# Mis-uses
		self.assertRaises(TemplateSyntaxError, Template, '{% load i18n %}{% trans "May" context as var %}{{ var }}')
		self.assertRaises(TemplateSyntaxError, Template, '{% load i18n %}{% trans "May" as var context %}{{ var }}')

		# {% blocktrans %} ------------------------------

		# Inexisting context...

tests/template_tests/syntax_tests/test_i18n.py

+43 −0

Original line number	Diff line number	Diff line
		# coding: utf-8
		from __future__ import unicode_literals

		from django.template import TemplateSyntaxError
		from django.test import SimpleTestCase
		from django.utils import translation
		from django.utils.safestring import mark_safe
		@@ -412,3 +413,45 @@ class I18nTagTests(SimpleTestCase):
		def test_i18n38_2(self):
		output = self.engine.render_to_string('i18n38_2', {'langcodes': ['it', 'no']})
		self.assertEqual(output, 'it: Italian/italiano bidi=False; no: Norwegian/norsk bidi=False; ')

		@setup({'template': '{% load i18n %}{% trans %}A}'})
		def test_syntax_error_no_arguments(self):
		msg = "'trans' takes at least one argument"
		with self.assertRaisesMessage(TemplateSyntaxError, msg):
		self.engine.render_to_string('template')

		@setup({'template': '{% load i18n %}{% trans "Yes" badoption %}'})
		def test_syntax_error_bad_option(self):
		msg = "Unknown argument for 'trans' tag: 'badoption'"
		with self.assertRaisesMessage(TemplateSyntaxError, msg):
		self.engine.render_to_string('template')

		@setup({'template': '{% load i18n %}{% trans "Yes" as %}'})
		def test_syntax_error_missing_assignment(self):
		msg = "No argument provided to the 'trans' tag for the as option."
		with self.assertRaisesMessage(TemplateSyntaxError, msg):
		self.engine.render_to_string('template')

		@setup({'template': '{% load i18n %}{% trans "Yes" as var context %}'})
		def test_syntax_error_missing_context(self):
		msg = "No argument provided to the 'trans' tag for the context option."
		with self.assertRaisesMessage(TemplateSyntaxError, msg):
		self.engine.render_to_string('template')

		@setup({'template': '{% load i18n %}{% trans "Yes" context as var %}'})
		def test_syntax_error_context_as(self):
		msg = "Invalid argument 'as' provided to the 'trans' tag for the context option"
		with self.assertRaisesMessage(TemplateSyntaxError, msg):
		self.engine.render_to_string('template')

		@setup({'template': '{% load i18n %}{% trans "Yes" context noop %}'})
		def test_syntax_error_context_noop(self):
		msg = "Invalid argument 'noop' provided to the 'trans' tag for the context option"
		with self.assertRaisesMessage(TemplateSyntaxError, msg):
		self.engine.render_to_string('template')

		@setup({'template': '{% load i18n %}{% trans "Yes" noop noop %}'})
		def test_syntax_error_duplicate_option(self):
		msg = "The 'noop' option was specified more than once."
		with self.assertRaisesMessage(TemplateSyntaxError, msg):
		self.engine.render_to_string('template')

tests/template_tests/test_parser.py

+1 −26

Original line number	Diff line number	Diff line
		@@ -7,7 +7,7 @@ from unittest import TestCase

		from django.template import Library, Template, TemplateSyntaxError
		from django.template.base import (
		TOKEN_BLOCK, FilterExpression, Parser, Token, TokenParser, Variable,
		TOKEN_BLOCK, FilterExpression, Parser, Token, Variable,
		)
		from django.test import override_settings
		from django.utils import six
		@@ -23,31 +23,6 @@ class ParserTests(TestCase):
		split = token.split_contents()
		self.assertEqual(split, ["sometag", '_("Page not found")', 'value\|yesno:_("yes,no")'])

		def test_token_parsing(self):
		# Tests for TokenParser behavior in the face of quoted strings with
		# spaces.

		p = TokenParser("tag thevar\|filter sometag")
		self.assertEqual(p.tagname, "tag")
		self.assertEqual(p.value(), "thevar\|filter")
		self.assertTrue(p.more())
		self.assertEqual(p.tag(), "sometag")
		self.assertFalse(p.more())

		p = TokenParser('tag "a value"\|filter sometag')
		self.assertEqual(p.tagname, "tag")
		self.assertEqual(p.value(), '"a value"\|filter')
		self.assertTrue(p.more())
		self.assertEqual(p.tag(), "sometag")
		self.assertFalse(p.more())

		p = TokenParser("tag 'a value'\|filter sometag")
		self.assertEqual(p.tagname, "tag")
		self.assertEqual(p.value(), "'a value'\|filter")
		self.assertTrue(p.more())
		self.assertEqual(p.tag(), "sometag")
		self.assertFalse(p.more())

		def test_filter_parsing(self):
		c = {"article": {"section": "News"}}
		p = Parser("")