Commit cfcca7cc authored by Aymeric Augustin's avatar Aymeric Augustin
Browse files

Fixed #3711, #6734, #12581 -- Bounded connection.queries. 

Prevented unlimited memory consumption when running background tasks
with DEBUG=True.

Thanks Rob, Alex, Baptiste, and others.
parent e2112edd

django/db/__init__.py

+1 −1
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ connection = DefaultConnectionProxy() 
# Register an event to reset saved queries when a Django request is started.

def reset_queries(**kwargs):

    for conn in connections.all():

        conn.queries = []

        conn.queries_log.clear()

signals.request_started.connect(reset_queries)

django/db/backends/__init__.py

+15 −2
Original line number Diff line number Diff line 
from collections import deque

import datetime

import time

import warnings
@@ -30,17 +31,21 @@ class BaseDatabaseWrapper(object): 
    ops = None

    vendor = 'unknown'



    queries_limit = 9000



    def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS,

                 allow_thread_sharing=False):

        # Connection related attributes.

        # The underlying database connection.

        self.connection = None

        self.queries = []

        # `settings_dict` should be a dictionary containing keys such as

        # NAME, USER, etc. It's called `settings_dict` instead of `settings`

        # to disambiguate it from Django settings modules.

        self.settings_dict = settings_dict

        self.alias = alias

        # Query logging in debug mode.

        self.use_debug_cursor = None

        self.queries_log = deque(maxlen=self.queries_limit)



        # Transaction related attributes.

        # Tracks if the connection is in autocommit mode. Per PEP 249, by
@@ -79,6 +84,14 @@ class BaseDatabaseWrapper(object): 
    def __hash__(self):

        return hash(self.alias)



    @property

    def queries(self):

        if len(self.queries_log) == self.queries_log.maxlen:

            warnings.warn(

                "Limit for query logging exceeded, only the last {} queries "

                "will be returned.".format(self.queries_log.maxlen))

        return list(self.queries_log)



    ##### Backend-specific methods for creating connections and cursors #####



    def get_connection_params(self):
@@ -429,7 +442,7 @@ class BaseDatabaseWrapper(object): 


    def make_debug_cursor(self, cursor):

        """

        Creates a cursor that logs all queries in self.queries.

        Creates a cursor that logs all queries in self.queries_log.

        """

        return utils.CursorDebugWrapper(cursor, self)

django/db/backends/utils.py

+2 −2
Original line number Diff line number Diff line
@@ -80,7 +80,7 @@ class CursorDebugWrapper(CursorWrapper): 
            stop = time()

            duration = stop - start

            sql = self.db.ops.last_executed_query(self.cursor, sql, params)

            self.db.queries.append({

            self.db.queries_log.append({

                'sql': sql,

                'time': "%.3f" % duration,

            })
@@ -99,7 +99,7 @@ class CursorDebugWrapper(CursorWrapper): 
                times = len(param_list)

            except TypeError:           # param_list could be an iterator

                times = '?'

            self.db.queries.append({

            self.db.queries_log.append({

                'sql': '%s times: %s' % (times, sql),

                'time': "%.3f" % duration,

            })

django/test/utils.py

+2 −2
Original line number Diff line number Diff line
@@ -509,7 +509,7 @@ class CaptureQueriesContext(object): 
    def __enter__(self):

        self.use_debug_cursor = self.connection.use_debug_cursor

        self.connection.use_debug_cursor = True

        self.initial_queries = len(self.connection.queries)

        self.initial_queries = len(self.connection.queries_log)

        self.final_queries = None

        request_started.disconnect(reset_queries)

        return self
@@ -519,7 +519,7 @@ class CaptureQueriesContext(object): 
        request_started.connect(reset_queries)

        if exc_type is not None:

            return

        self.final_queries = len(self.connection.queries)

        self.final_queries = len(self.connection.queries_log)





class IgnoreDeprecationWarningsMixin(object):

docs/faq/models.txt

+6 −19
Original line number Diff line number Diff line
@@ -32,6 +32,12 @@ same interface on each member of the ``connections`` dictionary:: 
    >>> from django.db import connections

    >>> connections['my_db_alias'].queries



If you need to clear the query list manually at any point in your functions,

just call ``reset_queries()``, like this::



    from django.db import reset_queries

    reset_queries()



Can I use Django with a pre-existing database?

----------------------------------------------
@@ -76,22 +82,3 @@ type, create an initial data file and put something like this in it:: 
As explained in the :ref:`SQL initial data file <initial-sql>` documentation,

this SQL file can contain arbitrary SQL, so you can make any sorts of changes

you need to make.
 


Why is Django leaking memory?

-----------------------------



Django isn't known to leak memory. If you find your Django processes are

allocating more and more memory, with no sign of releasing it, check to make

sure your :setting:`DEBUG` setting is set to ``False``. If :setting:`DEBUG`

is ``True``, then Django saves a copy of every SQL statement it has executed.



(The queries are saved in ``django.db.connection.queries``. See

:ref:`faq-see-raw-sql-queries`.)



To fix the problem, set :setting:`DEBUG` to ``False``.



If you need to clear the query list manually at any point in your functions,

just call ``reset_queries()``, like this::



    from django import db

    db.reset_queries()