[1.3.X] Altered the behavior of URLField to avoid a potential DOS vector, and... (1a76dbef) · Commits · Dom Sekotill / django

django/core/validators.py

+32 −18

Original line number	Diff line number	Diff line
		import platform
		import re
		import urllib2
		import urlparse
		@@ -39,10 +40,6 @@ class RegexValidator(object):
		if not self.regex.search(smart_unicode(value)):
		raise ValidationError(self.message, code=self.code)

		class HeadRequest(urllib2.Request):
		def get_method(self):
		return "HEAD"

		class URLValidator(RegexValidator):
		regex = re.compile(
		r'^(?:http\|ftp)s?://' # http:// or https://
		@@ -52,7 +49,8 @@ class URLValidator(RegexValidator):
		r'(?::\d+)?' # optional port
		r'(?:/?\|[/?]\S+)$', re.IGNORECASE)

		def __init__(self, verify_exists=False, validator_user_agent=URL_VALIDATOR_USER_AGENT):
		def __init__(self, verify_exists=False,
		validator_user_agent=URL_VALIDATOR_USER_AGENT):
		super(URLValidator, self).__init__()
		self.verify_exists = verify_exists
		self.user_agent = validator_user_agent
		@@ -76,6 +74,7 @@ class URLValidator(RegexValidator):
		else:
		url = value

		#This is deprecated and will be removed in a future release.
		if self.verify_exists:
		headers = {
		"Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,/;q=0.5",
		@@ -87,22 +86,37 @@ class URLValidator(RegexValidator):
		url = url.encode('utf-8')
		broken_error = ValidationError(
		_(u'This URL appears to be a broken link.'), code='invalid_link')
		try:
		req = HeadRequest(url, None, headers)
		u = urllib2.urlopen(req)
		except ValueError:
		raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
		except urllib2.HTTPError, e:
		if e.code in (405, 501):
		# Try a GET request (HEAD refused)
		# See also: http://www.w3.org/Protocols/rfc2616/rfc2616.html
		try:
		req = urllib2.Request(url, None, headers)
		u = urllib2.urlopen(req)
		req.get_method = lambda: 'HEAD'
		#Create an opener that does not support local file access
		opener = urllib2.OpenerDirector()

		#Don't follow redirects, but don't treat them as errors either
		error_nop = lambda args, *kwargs: True
		http_error_processor = urllib2.HTTPErrorProcessor()
		http_error_processor.http_error_301 = error_nop
		http_error_processor.http_error_302 = error_nop
		http_error_processor.http_error_307 = error_nop

		handlers = [urllib2.UnknownHandler(),
		urllib2.HTTPHandler(),
		urllib2.HTTPDefaultErrorHandler(),
		urllib2.FTPHandler(),
		http_error_processor]
		try:
		import ssl
		handlers.append(urllib2.HTTPSHandler())
		except:
		raise broken_error
		#Python isn't compiled with SSL support
		pass
		map(opener.add_handler, handlers)
		if platform.python_version_tuple() >= (2, 6):
		opener.open(req, timeout=10)
		else:
		raise broken_error
		opener.open(req)
		except ValueError:
		raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
		except: # urllib2.URLError, httplib.InvalidURL, etc.
		raise broken_error

django/db/models/fields/init.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -1119,7 +1119,7 @@ class TimeField(Field):
		class URLField(CharField):
		description = _("URL")

		def __init__(self, verbose_name=None, name=None, verify_exists=True, **kwargs):
		def __init__(self, verbose_name=None, name=None, verify_exists=False, **kwargs):
		kwargs['max_length'] = kwargs.get('max_length', 200)
		CharField.__init__(self, verbose_name, name, **kwargs)
		self.validators.append(validators.URLValidator(verify_exists=verify_exists))

docs/internals/deprecation.txt

+6 −0

Original line number	Diff line number	Diff line
		@@ -108,6 +108,12 @@ their deprecation, as per the :ref:`Django deprecation policy
		beyond that of a simple ``TextField`` since the removal of oldforms.
		All uses of ``XMLField`` can be replaced with ``TextField``.

		* ``django.db.models.fields.URLField.verify_exists`` has been
		deprecated due to intractable security and performance
		issues. Validation behavior has been removed in 1.4, and the
		argument will be removed in 1.5.


		* 1.5
		* The ``mod_python`` request handler has been deprecated since the 1.3
		release. The ``mod_wsgi`` handler should be used instead.

docs/ref/forms/fields.txt

+5 −0

Original line number	Diff line number	Diff line
		@@ -756,6 +756,11 @@ Takes the following optional arguments:
		If ``True``, the validator will attempt to load the given URL, raising
		``ValidationError`` if the page gives a 404. Defaults to ``False``.

		.. deprecated:: 1.3.1

		``verify_exists`` was deprecated for security reasons and will be
		removed in 1.4. This deprecation also removes ``validator_user_agent``.

		.. attribute:: URLField.validator_user_agent

		String used as the user-agent used when checking for a URL's existence.

docs/ref/models/fields.txt

+10 −3

Original line number	Diff line number	Diff line
		@@ -831,14 +831,21 @@ shortcuts.
		``URLField``
		------------

		.. class:: URLField([verify_exists=True, max_length=200, **options])
		.. class:: URLField([verify_exists=False, max_length=200, **options])

		A :class:`CharField` for a URL. Has one extra optional argument:

		.. deprecated:: 1.3.1

		``verify_exists`` is deprecated for security reasons as of 1.3.1
		and will be removed in 1.4. Prior to 1.3.1, the default value was
		``True``.

		.. attribute:: URLField.verify_exists

		If ``True`` (the default), the URL given will be checked for existence
		(i.e., the URL actually loads and doesn't give a 404 response).
		If ``True``, the URL given will be checked for existence (i.e.,
		the URL actually loads and doesn't give a 404 response) using a
		``HEAD`` request. Redirects are allowed, but will not be followed.

		Note that when you're using the single-threaded development server,
		validating a URL being served by the same server will hang. This should not