Commit c296e55d authored by Claude Paroz's avatar Claude Paroz
Browse files

Fixed #22258 -- Added progress status for dumpdata when outputting to file

Thanks Gwildor Sok for the report and Tim Graham for the review.
parent 03aec35a
Loading
Loading
Loading
Loading
+4 −1
Original line number Diff line number Diff line
@@ -89,7 +89,7 @@ class OutputWrapper(object):

    @style_func.setter
    def style_func(self, style_func):
        if style_func and hasattr(self._out, 'isatty') and self._out.isatty():
        if style_func and self.isatty():
            self._style_func = style_func
        else:
            self._style_func = lambda x: x
@@ -102,6 +102,9 @@ class OutputWrapper(object):
    def __getattr__(self, name):
        return getattr(self._out, name)

    def isatty(self):
        return hasattr(self._out, 'isatty') and self._out.isatty()

    def write(self, msg, style_func=None, ending=None):
        ending = self.ending if ending is None else ending
        if ending and not msg.endswith(ending):
+18 −5
Original line number Diff line number Diff line
@@ -127,8 +127,11 @@ class Command(BaseCommand):

            raise CommandError("Unknown serialization format: %s" % format)

        def get_objects():
            # Collate the objects to be serialized.
        def get_objects(count_only=False):
            """
            Collate the objects to be serialized. If count_only is True, just
            count the number of objects to be serialized.
            """
            for model in serializers.sort_dependencies(app_list.items()):
                if model in excluded_models:
                    continue
@@ -141,17 +144,27 @@ class Command(BaseCommand):
                    queryset = objects.using(using).order_by(model._meta.pk.name)
                    if primary_keys:
                        queryset = queryset.filter(pk__in=primary_keys)
                    if count_only:
                        yield queryset.order_by().count()
                    else:
                        for obj in queryset.iterator():
                            yield obj

        try:
            self.stdout.ending = None
            progress_output = None
            object_count = 0
            # If dumpdata is outputting to stdout, there is no way to display progress
            if (output and self.stdout.isatty() and options['verbosity'] > 0):
                progress_output = self.stdout
                object_count = sum(get_objects(count_only=True))
            stream = open(output, 'w') if output else None
            try:
                serializers.serialize(format, get_objects(), indent=indent,
                        use_natural_foreign_keys=use_natural_foreign_keys,
                        use_natural_primary_keys=use_natural_primary_keys,
                        stream=stream or self.stdout)
                        stream=stream or self.stdout, progress_output=progress_output,
                        object_count=object_count)
            finally:
                if stream:
                    stream.close()
+29 −1
Original line number Diff line number Diff line
@@ -27,6 +27,29 @@ class DeserializationError(Exception):
        return cls("%s: (%s:pk=%s) field_value was '%s'" % (original_exc, model, fk, field_value))


class ProgressBar(object):
    progress_width = 75

    def __init__(self, output, total_count):
        self.output = output
        self.total_count = total_count
        self.prev_done = 0

    def update(self, count):
        if not self.output:
            return
        perc = count * 100 // self.total_count
        done = perc * self.progress_width // 100
        if self.prev_done >= done:
            return
        self.prev_done = done
        cr = '' if self.total_count == 1 else '\r'
        self.output.write(cr + '[' + '.' * done + ' ' * (self.progress_width - done) + ']')
        if done == self.progress_width:
            self.output.write('\n')
        self.output.flush()


class Serializer(object):
    """
    Abstract serializer base class.
@@ -35,6 +58,7 @@ class Serializer(object):
    # Indicates if the implemented serializer is only available for
    # internal Django use.
    internal_use_only = False
    progress_class = ProgressBar

    def serialize(self, queryset, **options):
        """
@@ -46,10 +70,13 @@ class Serializer(object):
        self.selected_fields = options.pop("fields", None)
        self.use_natural_foreign_keys = options.pop('use_natural_foreign_keys', False)
        self.use_natural_primary_keys = options.pop('use_natural_primary_keys', False)
        progress_bar = self.progress_class(
            options.pop('progress_output', None), options.pop('object_count', 0)
        )

        self.start_serialization()
        self.first = True
        for obj in queryset:
        for count, obj in enumerate(queryset, start=1):
            self.start_object(obj)
            # Use the concrete parent class' _meta instead of the object's _meta
            # This is to avoid local_fields problems for proxy models. Refs #17717.
@@ -67,6 +94,7 @@ class Serializer(object):
                    if self.selected_fields is None or field.attname in self.selected_fields:
                        self.handle_m2m_field(obj, field)
            self.end_object(obj)
            progress_bar.update(count)
            if self.first:
                self.first = False
        self.end_serialization()
+6 −0
Original line number Diff line number Diff line
@@ -309,6 +309,12 @@ one model.

By default ``dumpdata`` will output all the serialized data to standard output.
This option allows you to specify the file to which the data is to be written.
When this option is set and the verbosity is greater than 0 (the default), a
progress bar is shown in the terminal.

.. versionchanged:: 1.9

    The progress bar in the terminal was added.

flush
-----
+2 −0
Original line number Diff line number Diff line
@@ -364,6 +364,8 @@ Management Commands
  preceded by the operation's description.

* The :djadmin:`dumpdata` command output is now deterministically ordered.
  Moreover, when the ``--ouput`` option is specified, it also shows a progress
  bar in the terminal.

* The :djadmin:`createcachetable` command now has a ``--dry-run`` flag to
  print out the SQL rather than execute it.
Loading