Commit e22c64df authored by wrwrwr's avatar wrwrwr Committed by Tim Graham
Browse files

Fixed #23742 -- Added an option to reverse tests order.

This is useful for debugging side effects affecting tests that
are usually executed before a given test. Full suite and pair
tests sort cases more or less deterministically, thus some test
cross-dependencies are easier to reveal by reversing the order.

Thanks Preston Timmons for the review.
parent ca801b8c
Loading
Loading
Loading
Loading
+17 −7
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ class DiscoverRunner(object):
    reorder_by = (TestCase, SimpleTestCase)

    def __init__(self, pattern=None, top_level=None,
                 verbosity=1, interactive=True, failfast=False, keepdb=False,
                 verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False,
                 **kwargs):

        self.pattern = pattern
@@ -30,6 +30,7 @@ class DiscoverRunner(object):
        self.interactive = interactive
        self.failfast = failfast
        self.keepdb = keepdb
        self.reverse = reverse

    @classmethod
    def add_arguments(cls, parser):
@@ -41,7 +42,10 @@ class DiscoverRunner(object):
            help='The test matching pattern. Defaults to test*.py.')
        parser.add_argument('-k', '--keepdb', action='store_true', dest='keepdb',
            default=False,
            help='Preserve the test DB between runs. Defaults to False')
            help='Preserves the test DB between runs.')
        parser.add_argument('-r', '--reverse', action='store_true', dest='reverse',
            default=False,
            help='Reverses test cases order.')

    def setup_test_environment(self, **kwargs):
        setup_test_environment()
@@ -107,7 +111,7 @@ class DiscoverRunner(object):
        for test in extra_tests:
            suite.addTest(test)

        return reorder_suite(suite, self.reorder_by)
        return reorder_suite(suite, self.reorder_by, self.reverse)

    def setup_databases(self, **kwargs):
        return setup_databases(self.verbosity, self.interactive, self.keepdb, **kwargs)
@@ -213,7 +217,7 @@ def dependency_ordered(test_databases, dependencies):
    return ordered_test_databases


def reorder_suite(suite, classes):
def reorder_suite(suite, classes, reverse=False):
    """
    Reorders a test suite by test type.

@@ -221,30 +225,36 @@ def reorder_suite(suite, classes):

    All tests of type classes[0] are placed first, then tests of type
    classes[1], etc. Tests with no match in classes are placed last.

    If `reverse` is True, tests within classes are sorted in opposite order,
    but test classes are not reversed.
    """
    class_count = len(classes)
    suite_class = type(suite)
    bins = [suite_class() for i in range(class_count + 1)]
    partition_suite(suite, classes, bins)
    partition_suite(suite, classes, bins, reverse=reverse)
    for i in range(class_count):
        bins[0].addTests(bins[i + 1])
    return bins[0]


def partition_suite(suite, classes, bins):
def partition_suite(suite, classes, bins, reverse=False):
    """
    Partitions a test suite by test type. Also prevents duplicated tests.

    classes is a sequence of types
    bins is a sequence of TestSuites, one more than classes
    reverse changes the ordering of tests within bins

    Tests of type classes[i] are added to bins[i],
    tests with no match found in classes are place in bins[-1]
    """
    suite_class = type(suite)
    if reverse:
        suite = reversed(tuple(suite))
    for test in suite:
        if isinstance(test, suite_class):
            partition_suite(test, classes, bins)
            partition_suite(test, classes, bins, reverse=reverse)
        else:
            for i in range(len(classes)):
                if isinstance(test, classes[i]):
+12 −0
Original line number Diff line number Diff line
@@ -307,3 +307,15 @@ the first one:
.. code-block:: bash

   $ ./runtests.py --pair basic.tests.ModelTest.test_eq queries transactions

You can also try running any set of tests in reverse using the ``--reverse``
option in order to verify that executing tests in a different order does not
cause any trouble:

.. code-block:: bash

   $ ./runtests.py basic --reverse

.. versionadded:: 1.8

    The ``--reverse`` option was added.
+9 −0
Original line number Diff line number Diff line
@@ -1411,6 +1411,15 @@ test suite. If the test database does not exist, it will be created on the first
run and then preserved for each subsequent run. Any unapplied migrations will also
be applied to the test database before running the test suite.

.. django-admin-option:: --reverse

.. versionadded:: 1.8

The ``--reverse`` option can be used to sort test cases in the opposite order.
This may help in debugging tests that aren't properly isolated and have side
effects. :ref:`Grouping by test class <order-of-tests>` is preserved when using
this option.

testserver <fixture fixture ...>
--------------------------------

+3 −2
Original line number Diff line number Diff line
@@ -471,8 +471,9 @@ Tests
* The new :meth:`~django.test.SimpleTestCase.assertJSONNotEqual` assertion
  allows you to test that two JSON fragments are not equal.

* Added the ability to preserve the test database by adding the
  :djadminopt:`--keepdb` flag.
* Added options to the :djadmin:`test` command to preserve the test database
  (:djadminopt:`--keepdb`) and to run the test cases in reverse order
  (:djadminopt:`--reverse`).

* Added the :attr:`~django.test.Response.resolver_match` attribute to test
  client responses.
+7 −2
Original line number Diff line number Diff line
@@ -355,7 +355,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a
selection of other methods that are used to by ``run_tests()`` to set up,
execute and tear down the test suite.

.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=True, keepdb=False **kwargs)
.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=True, keepdb=False, reverse=False, **kwargs)

    ``DiscoverRunner`` will search for tests in any file matching ``pattern``.

@@ -381,6 +381,11 @@ execute and tear down the test suite.
    or create one if necessary. If ``False``, a new database will be created,
    prompting the user to remove the existing one, if present.

    If ``reverse`` is ``True``, test cases will be executed in the opposite
    order. This could be useful to debug tests that aren't properly isolated
    and have side effects. :ref:`Grouping by test class <order-of-tests>` is
    preserved when using this option.

    Django may, from time to time, extend the capabilities of the test runner
    by adding new arguments. The ``**kwargs`` declaration allows for this
    expansion. If you subclass ``DiscoverRunner`` or write your own test
@@ -397,7 +402,7 @@ execute and tear down the test suite.
        subclassed test runner to add options to the list of command-line
        options that the :djadmin:`test` command could use.

        The ``keepdb`` argument was added.
        The ``keepdb`` and the ``reverse`` arguments were added.

Attributes
~~~~~~~~~~
Loading