Commit 60416260 authored by Marc Tamlyn's avatar Marc Tamlyn
Browse files

Added array field support for PostgreSQL. 

The first part of django.contrib.postgres, including model and two form
fields for arrays of other data types.

This commit is formed of the following work:

    Add shell of postgres app and test handling.

    First draft of array fields.

    Use recursive deconstruction.

    Stop creating classes at lookup time.

    Add validation and size parameter.

    Add contained_by lookup.

    Add SimpleArrayField for forms.

    Add SplitArrayField (mainly for admin).

    Fix prepare_value for SimpleArrayField.

    Stop using MultiValueField and MultiWidget.

    They don't play nice with flexible sizes.

    Add basics of admin integration.

    Missing:
    - Tests
    - Fully working js

    Add reference document for django.contrib.postgres.fields.ArrayField.

    Various performance and style tweaks.

    Fix internal docs link, formalise code snippets.

    Remove the admin code for now.

    It needs a better way of handing JS widgets in the admin as a whole
    before it is easy to write. In particular there are serious issues
    involving DateTimePicker when used in an array.

    Add a test for nested array fields with different delimiters.

    This will be a documented pattern so having a test for it is useful.

    Add docs for SimpleArrayField.

    Add docs for SplitArrayField.

    Remove admin related code for now.

    definition -> description

    Fix typo.

    Py3 errors.

    Avoid using regexes where they're not needed.

    Allow passing tuples by the programmer.

    Add some more tests for multidimensional arrays.

    Also fix slicing as much as it can be fixed.

    Simplify SplitArrayWidget's data loading.

    If we aren't including the variable size one, we don't need to search
    like this.
parent d9d92425

django/contrib/postgres/__init__.py

0 → 100644
+0 −0

Empty file added.

django/contrib/postgres/fields/__init__.py

0 → 100644
+1 −0
Original line number Diff line number Diff line 
from .array import *  # NOQA

django/contrib/postgres/fields/array.py

0 → 100644
+254 −0
Original line number Diff line number Diff line 
import json



from django.contrib.postgres.forms import SimpleArrayField

from django.contrib.postgres.validators import ArrayMaxLengthValidator

from django.core import checks, exceptions

from django.db.models import Field, Lookup, Transform, IntegerField

from django.utils import six

from django.utils.translation import string_concat, ugettext_lazy as _





__all__ = ['ArrayField']





class AttributeSetter(object):

    def __init__(self, name, value):

        setattr(self, name, value)





class ArrayField(Field):

    empty_strings_allowed = False

    default_error_messages = {

        'item_invalid': _('Item %(nth)s in the array did not validate: '),

        'nested_array_mismatch': _('Nested arrays must have the same length.'),

    }



    def __init__(self, base_field, size=None, **kwargs):

        self.base_field = base_field

        self.size = size

        if self.size:

            self.default_validators = self.default_validators[:]

            self.default_validators.append(ArrayMaxLengthValidator(self.size))

        super(ArrayField, self).__init__(**kwargs)



    def check(self, **kwargs):

        errors = super(ArrayField, self).check(**kwargs)

        if self.base_field.rel:

            errors.append(

                checks.Error(

                    'Base field for array cannot be a related field.',

                    hint=None,

                    obj=self,

                    id='postgres.E002'

                )

            )

        else:

            # Remove the field name checks as they are not needed here.

            base_errors = self.base_field.check()

            if base_errors:

                messages = '\n    '.join('%s (%s)' % (error.msg, error.id) for error in base_errors)

                errors.append(

                    checks.Error(

                        'Base field for array has errors:\n    %s' % messages,

                        hint=None,

                        obj=self,

                        id='postgres.E001'

                    )

                )

        return errors



    def set_attributes_from_name(self, name):

        super(ArrayField, self).set_attributes_from_name(name)

        self.base_field.set_attributes_from_name(name)



    @property

    def description(self):

        return 'Array of %s' % self.base_field.description



    def db_type(self, connection):

        size = self.size or ''

        return '%s[%s]' % (self.base_field.db_type(connection), size)



    def get_prep_value(self, value):

        if isinstance(value, list) or isinstance(value, tuple):

            return [self.base_field.get_prep_value(i) for i in value]

        return value



    def get_db_prep_lookup(self, lookup_type, value, connection, prepared=False):

        if lookup_type == 'contains':

            return [self.get_prep_value(value)]

        return super(ArrayField, self).get_db_prep_lookup(lookup_type, value,

                connection, prepared=False)



    def deconstruct(self):

        name, path, args, kwargs = super(ArrayField, self).deconstruct()

        path = 'django.contrib.postgres.fields.ArrayField'

        args.insert(0, self.base_field)

        kwargs['size'] = self.size

        return name, path, args, kwargs



    def to_python(self, value):

        if isinstance(value, six.string_types):

            # Assume we're deserializing

            vals = json.loads(value)

            value = [self.base_field.to_python(val) for val in vals]

        return value



    def value_to_string(self, obj):

        values = []

        vals = self._get_val_from_obj(obj)

        base_field = self.base_field



        for val in vals:

            obj = AttributeSetter(base_field.attname, val)

            values.append(base_field.value_to_string(obj))

        return json.dumps(values)



    def get_transform(self, name):

        transform = super(ArrayField, self).get_transform(name)

        if transform:

            return transform

        try:

            index = int(name)

        except ValueError:

            pass

        else:

            index += 1  # postgres uses 1-indexing

            return IndexTransformFactory(index, self.base_field)

        try:

            start, end = name.split('_')

            start = int(start) + 1

            end = int(end)  # don't add one here because postgres slices are weird

        except ValueError:

            pass

        else:

            return SliceTransformFactory(start, end)



    def validate(self, value, model_instance):

        super(ArrayField, self).validate(value, model_instance)

        for i, part in enumerate(value):

            try:

                self.base_field.validate(part, model_instance)

            except exceptions.ValidationError as e:

                raise exceptions.ValidationError(

                    string_concat(self.error_messages['item_invalid'], e.message),

                    code='item_invalid',

                    params={'nth': i},

                )

        if isinstance(self.base_field, ArrayField):

            if len({len(i) for i in value}) > 1:

                raise exceptions.ValidationError(

                    self.error_messages['nested_array_mismatch'],

                    code='nested_array_mismatch',

                )



    def formfield(self, **kwargs):

        defaults = {

            'form_class': SimpleArrayField,

            'base_field': self.base_field.formfield(),

            'max_length': self.size,

        }

        defaults.update(kwargs)

        return super(ArrayField, self).formfield(**defaults)





class ArrayContainsLookup(Lookup):

    lookup_name = 'contains'



    def as_sql(self, qn, connection):

        lhs, lhs_params = self.process_lhs(qn, connection)

        rhs, rhs_params = self.process_rhs(qn, connection)

        params = lhs_params + rhs_params

        return '%s @> %s' % (lhs, rhs), params





ArrayField.register_lookup(ArrayContainsLookup)





class ArrayContainedByLookup(Lookup):

    lookup_name = 'contained_by'



    def as_sql(self, qn, connection):

        lhs, lhs_params = self.process_lhs(qn, connection)

        rhs, rhs_params = self.process_rhs(qn, connection)

        params = lhs_params + rhs_params

        return '%s <@ %s' % (lhs, rhs), params





ArrayField.register_lookup(ArrayContainedByLookup)





class ArrayOverlapLookup(Lookup):

    lookup_name = 'overlap'



    def as_sql(self, qn, connection):

        lhs, lhs_params = self.process_lhs(qn, connection)

        rhs, rhs_params = self.process_rhs(qn, connection)

        params = lhs_params + rhs_params

        return '%s && %s' % (lhs, rhs), params





ArrayField.register_lookup(ArrayOverlapLookup)





class ArrayLenTransform(Transform):

    lookup_name = 'len'



    @property

    def output_type(self):

        return IntegerField()



    def as_sql(self, qn, connection):

        lhs, params = qn.compile(self.lhs)

        return 'array_length(%s, 1)' % lhs, params





ArrayField.register_lookup(ArrayLenTransform)





class IndexTransform(Transform):



    def __init__(self, index, base_field, *args, **kwargs):

        super(IndexTransform, self).__init__(*args, **kwargs)

        self.index = index

        self.base_field = base_field



    def as_sql(self, qn, connection):

        lhs, params = qn.compile(self.lhs)

        return '%s[%s]' % (lhs, self.index), params



    @property

    def output_type(self):

        return self.base_field





class IndexTransformFactory(object):



    def __init__(self, index, base_field):

        self.index = index

        self.base_field = base_field



    def __call__(self, *args, **kwargs):

        return IndexTransform(self.index, self.base_field, *args, **kwargs)





class SliceTransform(Transform):



    def __init__(self, start, end, *args, **kwargs):

        super(SliceTransform, self).__init__(*args, **kwargs)

        self.start = start

        self.end = end



    def as_sql(self, qn, connection):

        lhs, params = qn.compile(self.lhs)

        return '%s[%s:%s]' % (lhs, self.start, self.end), params





class SliceTransformFactory(object):



    def __init__(self, start, end):

        self.start = start

        self.end = end



    def __call__(self, *args, **kwargs):

        return SliceTransform(self.start, self.end, *args, **kwargs)

django/contrib/postgres/forms/__init__.py

0 → 100644
+1 −0
Original line number Diff line number Diff line 
from .array import *  # NOQA

django/contrib/postgres/forms/array.py

0 → 100644
+185 −0
Original line number Diff line number Diff line 
import copy



from django.contrib.postgres.validators import ArrayMinLengthValidator, ArrayMaxLengthValidator

from django.core.exceptions import ValidationError

from django import forms

from django.utils.safestring import mark_safe

from django.utils import six

from django.utils.translation import string_concat, ugettext_lazy as _





class SimpleArrayField(forms.CharField):

    default_error_messages = {

        'item_invalid': _('Item %(nth)s in the array did not validate: '),

    }



    def __init__(self, base_field, delimiter=',', max_length=None, min_length=None, *args, **kwargs):

        self.base_field = base_field

        self.delimiter = delimiter

        super(SimpleArrayField, self).__init__(*args, **kwargs)

        if min_length is not None:

            self.min_length = min_length

            self.validators.append(ArrayMinLengthValidator(int(min_length)))

        if max_length is not None:

            self.max_length = max_length

            self.validators.append(ArrayMaxLengthValidator(int(max_length)))



    def prepare_value(self, value):

        if isinstance(value, list):

            return self.delimiter.join([six.text_type(self.base_field.prepare_value(v)) for v in value])

        return value



    def to_python(self, value):

        if value:

            items = value.split(self.delimiter)

        else:

            items = []

        errors = []

        values = []

        for i, item in enumerate(items):

            try:

                values.append(self.base_field.to_python(item))

            except ValidationError as e:

                for error in e.error_list:

                    errors.append(ValidationError(

                        string_concat(self.error_messages['item_invalid'], error.message),

                        code='item_invalid',

                        params={'nth': i},

                    ))

        if errors:

            raise ValidationError(errors)

        return values



    def validate(self, value):

        super(SimpleArrayField, self).validate(value)

        errors = []

        for i, item in enumerate(value):

            try:

                self.base_field.validate(item)

            except ValidationError as e:

                for error in e.error_list:

                    errors.append(ValidationError(

                        string_concat(self.error_messages['item_invalid'], error.message),

                        code='item_invalid',

                        params={'nth': i},

                    ))

        if errors:

            raise ValidationError(errors)



    def run_validators(self, value):

        super(SimpleArrayField, self).run_validators(value)

        errors = []

        for i, item in enumerate(value):

            try:

                self.base_field.run_validators(item)

            except ValidationError as e:

                for error in e.error_list:

                    errors.append(ValidationError(

                        string_concat(self.error_messages['item_invalid'], error.message),

                        code='item_invalid',

                        params={'nth': i},

                    ))

        if errors:

            raise ValidationError(errors)





class SplitArrayWidget(forms.Widget):



    def __init__(self, widget, size, **kwargs):

        self.widget = widget() if isinstance(widget, type) else widget

        self.size = size

        super(SplitArrayWidget, self).__init__(**kwargs)



    @property

    def is_hidden(self):

        return self.widget.is_hidden



    def value_from_datadict(self, data, files, name):

        return [self.widget.value_from_datadict(data, files, '%s_%s' % (name, index))

                for index in range(self.size)]



    def id_for_label(self, id_):

        # See the comment for RadioSelect.id_for_label()

        if id_:

            id_ += '_0'

        return id_



    def render(self, name, value, attrs=None):

        if self.is_localized:

            self.widget.is_localized = self.is_localized

        value = value or []

        output = []

        final_attrs = self.build_attrs(attrs)

        id_ = final_attrs.get('id', None)

        for i in range(max(len(value), self.size)):

            try:

                widget_value = value[i]

            except IndexError:

                widget_value = None

            if id_:

                final_attrs = dict(final_attrs, id='%s_%s' % (id_, i))

            output.append(self.widget.render(name + '_%s' % i, widget_value, final_attrs))

        return mark_safe(self.format_output(output))



    def format_output(self, rendered_widgets):

        return ''.join(rendered_widgets)



    @property

    def media(self):

        return self.widget.media



    def __deepcopy__(self, memo):

        obj = super(SplitArrayWidget, self).__deepcopy__(memo)

        obj.widget = copy.deepcopy(self.widget)

        return obj



    @property

    def needs_multipart_form(self):

        return self.widget.needs_multipart_form





class SplitArrayField(forms.Field):

    default_error_messages = {

        'item_invalid': _('Item %(nth)s in the array did not validate: '),

    }



    def __init__(self, base_field, size, remove_trailing_nulls=False, **kwargs):

        self.base_field = base_field

        self.size = size

        self.remove_trailing_nulls = remove_trailing_nulls

        widget = SplitArrayWidget(widget=base_field.widget, size=size)

        kwargs.setdefault('widget', widget)

        super(SplitArrayField, self).__init__(**kwargs)



    def clean(self, value):

        cleaned_data = []

        errors = []

        if not any(value) and self.required:

            raise ValidationError(self.error_messages['required'])

        max_size = max(self.size, len(value))

        for i in range(max_size):

            item = value[i]

            try:

                cleaned_data.append(self.base_field.clean(item))

                errors.append(None)

            except ValidationError as error:

                errors.append(ValidationError(

                    string_concat(self.error_messages['item_invalid'], error.message),

                    code='item_invalid',

                    params={'nth': i},

                ))

                cleaned_data.append(None)

        if self.remove_trailing_nulls:

            null_index = None

            for i, value in reversed(list(enumerate(cleaned_data))):

                if value in self.base_field.empty_values:

                    null_index = i

                else:

                    break

            if null_index:

                cleaned_data = cleaned_data[:null_index]

                errors = errors[:null_index]

        errors = list(filter(None, errors))

        if errors:

            raise ValidationError(errors)

        return cleaned_data