Commit 5aa55038 authored by Markus Holtermann's avatar Markus Holtermann
Browse files

Fixed #24743, #24745 -- Optimized migration plan handling 

The change partly goes back to the old behavior for forwards migrations
which should reduce the amount of memory consumption (#24745). However,
by the way the current state computation is done (there is no
`state_backwards` on a migration class) this change cannot be applied to
backwards migrations. Hence rolling back migrations still requires the
precomputation and storage of the intermediate migration states.

This improvement also implies that Django does not handle mixed
migration plans anymore. Mixed plans consist of a list of migrations
where some are being applied and others are being unapplied.

Thanks Andrew Godwin, Josh Smeaton and Tim Graham for the review as well
as everybody involved on the ticket that kept me looking into the issue.
parent 186eb21d

django/db/migrations/exceptions.py

+4 −0
Original line number Diff line number Diff line
@@ -58,3 +58,7 @@ class NodeNotFoundError(LookupError): 


class MigrationSchemaMissing(DatabaseError):

    pass





class InvalidMigrationPlan(ValueError):

    pass

django/db/migrations/executor.py

+77 −27
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ from __future__ import unicode_literals 
from django.apps.registry import apps as global_apps

from django.db import migrations



from .exceptions import InvalidMigrationPlan

from .loader import MigrationLoader

from .recorder import MigrationRecorder

from .state import ProjectState
@@ -71,46 +72,95 @@ class MigrationExecutor(object): 
        """

        if plan is None:

            plan = self.migration_plan(targets)

        migrations_to_run = {m[0] for m in plan}

        # Create the forwards plan Django would follow on an empty database

        full_plan = self.migration_plan(self.loader.graph.leaf_nodes(), clean_start=True)

        # Holds all states right before a migration is applied

        # if the migration is being run.



        all_forwards = all(not backwards for mig, backwards in plan)

        all_backwards = all(backwards for mig, backwards in plan)



        if not plan:

            pass  # Nothing to do for an empty plan

        elif all_forwards == all_backwards:

            # This should only happen if there's a mixed plan

            raise InvalidMigrationPlan(

                "Migration plans with both forwards and backwards migrations "

                "are not supported. Please split your migration process into "

                "separate plans of only forwards OR backwards migrations.",

                plan

            )

        elif all_forwards:

            self._migrate_all_forwards(plan, full_plan, fake=fake, fake_initial=fake_initial)

        else:

            # No need to check for `elif all_backwards` here, as that condition

            # would always evaluate to true.

            self._migrate_all_backwards(plan, full_plan, fake=fake)



        self.check_replacements()



    def _migrate_all_forwards(self, plan, full_plan, fake, fake_initial):

        """

        Take a list of 2-tuples of the form (migration instance, False) and

        apply them in the order they occur in the full_plan.

        """

        migrations_to_run = {m[0] for m in plan}

        state = ProjectState(real_apps=list(self.loader.unmigrated_apps))

        for migration, _ in full_plan:

            if not migrations_to_run:

                # We remove every migration that we applied from this set so

                # that we can bail out once the last migration has been applied

                # and don't always run until the very end of the migration

                # process.

                break

            if migration in migrations_to_run:

                if 'apps' not in state.__dict__:

                    if self.progress_callback:

                        self.progress_callback("render_start")

                    state.apps  # Render all -- performance critical

                    if self.progress_callback:

                        self.progress_callback("render_success")

                state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)

                migrations_to_run.remove(migration)

            else:

                migration.mutate_state(state, preserve=False)



    def _migrate_all_backwards(self, plan, full_plan, fake):

        """

        Take a list of 2-tuples of the form (migration instance, True) and

        unapply them in reverse order they occur in the full_plan.



        Since unapplying a migration requires the project state prior to that

        migration, Django will compute the migration states before each of them

        in a first run over the plan and then unapply them in a second run over

        the plan.

        """

        migrations_to_run = {m[0] for m in plan}

        # Holds all migration states prior to the migrations being unapplied

        states = {}

        state = ProjectState(real_apps=list(self.loader.unmigrated_apps))

        if self.progress_callback:

            self.progress_callback("render_start")

        # Phase 1 -- Store all project states of migrations right before they

        # are applied. The first migration that will be applied in phase 2 will

        # trigger the rendering of the initial project state. From this time on

        # models will be recursively reloaded as explained in

        # `django.db.migrations.state.get_related_models_recursive()`.

        for migration, _ in full_plan:

            if not migrations_to_run:

                # We remove every migration whose state was already computed

                # from the set below (`migrations_to_run.remove(migration)`).

                # If no states for migrations must be computed, we can exit

                # this loop. Migrations that occur after the latest migration

                # that is about to be applied would only trigger unneeded

                # mutate_state() calls.

                # We remove every migration that we applied from this set so

                # that we can bail out once the last migration has been applied

                # and don't always run until the very end of the migration

                # process.

                break

            do_run = migration in migrations_to_run

            if do_run:

            if migration in migrations_to_run:

                if 'apps' not in state.__dict__:

                    state.apps  # Render all real_apps -- performance critical

                states[migration] = state.clone()

                    state.apps  # Render all -- performance critical

                # The state before this migration

                states[migration] = state

                # The old state keeps as-is, we continue with the new state

                state = migration.mutate_state(state, preserve=True)

                migrations_to_run.remove(migration)

            # Only preserve the state if the migration is being run later

            state = migration.mutate_state(state, preserve=do_run)

            else:

                migration.mutate_state(state, preserve=False)

        if self.progress_callback:

            self.progress_callback("render_success")

        # Phase 2 -- Run the migrations

        for migration, backwards in plan:

            if not backwards:

                self.apply_migration(states[migration], migration, fake=fake, fake_initial=fake_initial)

            else:



        for migration, _ in plan:

            self.unapply_migration(states[migration], migration, fake=fake)

        self.check_replacements()



    def collect_sql(self, plan):

        """

docs/releases/1.9.txt

+19 −0
Original line number Diff line number Diff line
@@ -458,6 +458,21 @@ Migrations 
* When supplying ``None`` as a value in :setting:`MIGRATION_MODULES`, Django

  will consider the app an app without migrations.



* When applying migrations, the "Rendering model states" step that's displayed

  when running migrate with verbosity 2 or higher now computes only the states

  for the migrations that have already been applied. The model states for

  migrations being applied are generated on demand, drastically reducing the

  amount of required memory.



  However, this improvement is not available when unapplying migrations and

  therefore still requires the precomputation and storage of the intermediate

  migration states.



  This improvement also requires that Django no longer supports mixed migration

  plans. Mixed plans consist of a list of migrations where some are being

  applied and others are being unapplied. This was never officially supported

  and never had a public API that supports this behavior.



Models

^^^^^^
@@ -1094,6 +1109,10 @@ Miscellaneous 
* The system checks for :class:`~django.contrib.admin.ModelAdmin` now check

  instances rather than classes.



* The private API to apply mixed migration plans has been dropped for

  performance reasons. Mixed plans consist of a list of migrations where some

  are being applied and others are being unapplied.



.. _deprecated-features-1.9:



Features deprecated in 1.9

docs/spelling_wordlist

+1 −0
Original line number Diff line number Diff line
@@ -580,6 +580,7 @@ postgresql 
pq

pre

precisions

precomputation

preconfigured

prefetch

prefetched

tests/migrations/test_executor.py

+59 −0
Original line number Diff line number Diff line 
from django.apps.registry import apps as global_apps

from django.db import connection

from django.db.migrations.exceptions import InvalidMigrationPlan

from django.db.migrations.executor import MigrationExecutor

from django.db.migrations.graph import MigrationGraph

from django.db.migrations.recorder import MigrationRecorder
@@ -145,6 +146,64 @@ class ExecutorTests(MigrationTestBase): 
        executor.recorder.record_unapplied("migrations", "0002_second")

        executor.recorder.record_unapplied("migrations", "0001_initial")



    @override_settings(MIGRATION_MODULES={

        "migrations": "migrations.test_migrations",

        "migrations2": "migrations2.test_migrations_2_no_deps",

    })

    def test_mixed_plan_not_supported(self):

        """

        Although the MigrationExecutor interfaces allows for mixed migration

        plans (combined forwards and backwards migrations) this is not

        supported.

        """

        # Prepare for mixed plan

        executor = MigrationExecutor(connection)

        plan = executor.migration_plan([("migrations", "0002_second")])

        self.assertEqual(

            plan,

            [

                (executor.loader.graph.nodes["migrations", "0001_initial"], False),

                (executor.loader.graph.nodes["migrations", "0002_second"], False),

            ],

        )

        executor.migrate(None, plan)

        # Rebuild the graph to reflect the new DB state

        executor.loader.build_graph()

        self.assertIn(('migrations', '0001_initial'), executor.loader.applied_migrations)

        self.assertIn(('migrations', '0002_second'), executor.loader.applied_migrations)

        self.assertNotIn(('migrations2', '0001_initial'), executor.loader.applied_migrations)



        # Generate mixed plan

        plan = executor.migration_plan([

            ("migrations", None),

            ("migrations2", "0001_initial"),

        ])

        msg = (

            'Migration plans with both forwards and backwards migrations are '

            'not supported. Please split your migration process into separate '

            'plans of only forwards OR backwards migrations.'

        )

        with self.assertRaisesMessage(InvalidMigrationPlan, msg) as cm:

            executor.migrate(None, plan)

        self.assertEqual(

            cm.exception.args[1],

            [

                (executor.loader.graph.nodes["migrations", "0002_second"], True),

                (executor.loader.graph.nodes["migrations", "0001_initial"], True),

                (executor.loader.graph.nodes["migrations2", "0001_initial"], False),

            ],

        )

        # Rebuild the graph to reflect the new DB state

        executor.loader.build_graph()

        executor.migrate([

            ("migrations", None),

            ("migrations2", None),

        ])

        # Are the tables gone?

        self.assertTableNotExists("migrations_author")

        self.assertTableNotExists("migrations_book")

        self.assertTableNotExists("migrations2_otherauthor")



    @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"})

    def test_soft_apply(self):

        """