From dbad7bd40dfdb63801095bdaf94509cd51c6c5cf Mon Sep 17 00:00:00 2001 From: "un.def" Date: Wed, 19 Apr 2017 12:56:04 +0300 Subject: [PATCH 1/3] Migrate from unused in Django 1.11 qs.iterator() to custom qs._iterable_class --- polymorphic/query.py | 96 +++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/polymorphic/query.py b/polymorphic/query.py index c842414..2743c20 100644 --- a/polymorphic/query.py +++ b/polymorphic/query.py @@ -8,7 +8,7 @@ import copy from collections import defaultdict import django -from django.db.models.query import QuerySet, Q +from django.db.models.query import ModelIterable, QuerySet, Q from django.contrib.contenttypes.models import ContentType from django.utils import six @@ -16,7 +16,7 @@ from .query_translate import translate_polymorphic_filter_definitions_in_kwargs, from .query_translate import translate_polymorphic_field_path, translate_polymorphic_Q_object # chunk-size: maximum number of objects requested per db-request -# by the polymorphic queryset.iterator() implementation; we use the same chunk size as Django +# by the PolymorphicModelIterable; we use the same chunk size as Django try: from django.db.models.query import CHUNK_SIZE # this is 100 for Django 1.1/1.2 except ImportError: @@ -25,6 +25,52 @@ except ImportError: Polymorphic_QuerySet_objects_per_request = CHUNK_SIZE +class PolymorphicModelIterable(ModelIterable): + """ + ModelIterable for PolymorphicModel + + Yields real instances if qs.polymorphic_disabled is False, + otherwise acts like a regular ModelIterable. + """ + + def __iter__(self): + base_iter = super(PolymorphicModelIterable, self).__iter__() + if self.queryset.polymorphic_disabled: + return base_iter + return self._polymorhic_iterator(base_iter) + + def _polymorhic_iterator(self, base_iter): + """ + Here we do the same as:: + + base_result_objects = list(super(PolymorphicModelIterable, self).__iter__()) + real_results = self.queryset._get_real_instances(base_result_objects) + for o in real_results: yield o + + but it requests the objects in chunks from the database, + with Polymorphic_QuerySet_objects_per_request per chunk + """ + while True: + base_result_objects = [] + reached_end = False + + for i in range(Polymorphic_QuerySet_objects_per_request): + try: + o = next(base_iter) + base_result_objects.append(o) + except StopIteration: + reached_end = True + break + + real_results = self.queryset._get_real_instances(base_result_objects) + + for o in real_results: + yield o + + if reached_end: + return + + def transmogrify(cls, obj): """ Upcast a class to a different type without asking questions. @@ -63,6 +109,8 @@ class PolymorphicQuerySet(QuerySet): """ def __init__(self, *args, **kwargs): + super(PolymorphicQuerySet, self).__init__(*args, **kwargs) + self._iterable_class = PolymorphicModelIterable # init our queryset object member variables self.polymorphic_disabled = False # A parallel structure to django.db.models.query.Query.deferred_loading, @@ -71,7 +119,6 @@ class PolymorphicQuerySet(QuerySet): # retrieving the real instance (so that the deferred fields apply # to that queryset as well). self.polymorphic_deferred_loading = (set([]), True) - super(PolymorphicQuerySet, self).__init__(*args, **kwargs) def _clone(self, *args, **kwargs): # Django's _clone only copies its own variables, so we need to copy ours here @@ -407,49 +454,6 @@ class PolymorphicQuerySet(QuerySet): return resultlist - def iterator(self): - """ - This function is used by Django for all object retrieval. - By overriding it, we modify the objects that this queryset returns - when it is evaluated (or its get method or other object-returning methods are called). - - Here we do the same as:: - - base_result_objects = list(super(PolymorphicQuerySet, self).iterator()) - real_results = self._get_real_instances(base_result_objects) - for o in real_results: yield o - - but it requests the objects in chunks from the database, - with Polymorphic_QuerySet_objects_per_request per chunk - """ - base_iter = super(PolymorphicQuerySet, self).iterator() - - # disabled => work just like a normal queryset - if self.polymorphic_disabled: - for o in base_iter: - yield o - return - - while True: - base_result_objects = [] - reached_end = False - - for i in range(Polymorphic_QuerySet_objects_per_request): - try: - o = next(base_iter) - base_result_objects.append(o) - except StopIteration: - reached_end = True - break - - real_results = self._get_real_instances(base_result_objects) - - for o in real_results: - yield o - - if reached_end: - return - def __repr__(self, *args, **kwargs): if self.model.polymorphic_query_multiline_output: result = [repr(o) for o in self.all()] From 8d539143444e99aa7daf3923e1d17f2f26cfb7a5 Mon Sep 17 00:00:00 2001 From: "un.def" Date: Wed, 19 Apr 2017 13:03:43 +0300 Subject: [PATCH 2/3] Disallow Travis failures for Django 1.11 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index be2fa59..175a66b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,7 +90,6 @@ matrix: - python: "2.6" env: DJANGO="https://github.com/django/django/tarball/master" allow_failures: - - env: DJANGO="Django>=1.11,<1.12" - env: DJANGO="https://github.com/django/django/tarball/master" before_install: From 78d3cd4945fbf0a35c5629d77536568c7f70e33b Mon Sep 17 00:00:00 2001 From: "un.def" Date: Wed, 19 Apr 2017 14:43:51 +0300 Subject: [PATCH 3/3] Fix regression with Django < 1.9 --- polymorphic/query.py | 93 ++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/polymorphic/query.py b/polymorphic/query.py index 2743c20..0d716ff 100644 --- a/polymorphic/query.py +++ b/polymorphic/query.py @@ -8,7 +8,7 @@ import copy from collections import defaultdict import django -from django.db.models.query import ModelIterable, QuerySet, Q +from django.db.models.query import QuerySet, Q from django.contrib.contenttypes.models import ContentType from django.utils import six @@ -25,50 +25,54 @@ except ImportError: Polymorphic_QuerySet_objects_per_request = CHUNK_SIZE -class PolymorphicModelIterable(ModelIterable): +def _polymorhic_iterator(queryset, base_iter): """ - ModelIterable for PolymorphicModel + Here we do the same as:: - Yields real instances if qs.polymorphic_disabled is False, - otherwise acts like a regular ModelIterable. + real_results = queryset._get_real_instances(list(base_iter)) + for o in real_results: yield o + + but it requests the objects in chunks from the database, + with Polymorphic_QuerySet_objects_per_request per chunk """ + while True: + base_result_objects = [] + reached_end = False - def __iter__(self): - base_iter = super(PolymorphicModelIterable, self).__iter__() - if self.queryset.polymorphic_disabled: - return base_iter - return self._polymorhic_iterator(base_iter) + for i in range(Polymorphic_QuerySet_objects_per_request): + try: + o = next(base_iter) + base_result_objects.append(o) + except StopIteration: + reached_end = True + break - def _polymorhic_iterator(self, base_iter): + real_results = queryset._get_real_instances(base_result_objects) + + for o in real_results: + yield o + + if reached_end: + return + + +if django.VERSION >= (1, 9): + + from django.db.models.query import ModelIterable + + class PolymorphicModelIterable(ModelIterable): """ - Here we do the same as:: + ModelIterable for PolymorphicModel - base_result_objects = list(super(PolymorphicModelIterable, self).__iter__()) - real_results = self.queryset._get_real_instances(base_result_objects) - for o in real_results: yield o - - but it requests the objects in chunks from the database, - with Polymorphic_QuerySet_objects_per_request per chunk + Yields real instances if qs.polymorphic_disabled is False, + otherwise acts like a regular ModelIterable. """ - while True: - base_result_objects = [] - reached_end = False - for i in range(Polymorphic_QuerySet_objects_per_request): - try: - o = next(base_iter) - base_result_objects.append(o) - except StopIteration: - reached_end = True - break - - real_results = self.queryset._get_real_instances(base_result_objects) - - for o in real_results: - yield o - - if reached_end: - return + def __iter__(self): + base_iter = super(PolymorphicModelIterable, self).__iter__() + if self.queryset.polymorphic_disabled: + return base_iter + return _polymorhic_iterator(self.queryset, base_iter) def transmogrify(cls, obj): @@ -110,7 +114,8 @@ class PolymorphicQuerySet(QuerySet): def __init__(self, *args, **kwargs): super(PolymorphicQuerySet, self).__init__(*args, **kwargs) - self._iterable_class = PolymorphicModelIterable + if django.VERSION >= (1, 9): + self._iterable_class = PolymorphicModelIterable # init our queryset object member variables self.polymorphic_disabled = False # A parallel structure to django.db.models.query.Query.deferred_loading, @@ -454,6 +459,20 @@ class PolymorphicQuerySet(QuerySet): return resultlist + if django.VERSION < (1, 9): + # Before Django 1.9 ModelIterable functionality was implemented in `iterator` method + def iterator(self): + base_iter = super(PolymorphicQuerySet, self).iterator() + + # disabled => work just like a normal queryset + if self.polymorphic_disabled: + for o in base_iter: + yield o + return + + for o in _polymorhic_iterator(self, base_iter): + yield o + def __repr__(self, *args, **kwargs): if self.model.polymorphic_query_multiline_output: result = [repr(o) for o in self.all()]