From 343aa41ec199c41b6bfa94ad29fbf35bc59c9abd Mon Sep 17 00:00:00 2001 From: Austin Matsick Date: Fri, 27 May 2016 20:35:39 -0500 Subject: [PATCH 1/3] #216 Allow ContentType queries to be performed on non-default databases. --- polymorphic/models.py | 24 +++++++++++++----------- polymorphic/query.py | 7 ++++--- polymorphic/query_translate.py | 18 ++++++++++-------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/polymorphic/models.py b/polymorphic/models.py index eebcc02..316267c 100644 --- a/polymorphic/models.py +++ b/polymorphic/models.py @@ -16,6 +16,7 @@ Please see LICENSE and AUTHORS for more information. from __future__ import absolute_import from django.db import models +from django.db.utils import DEFAULT_DB_ALIAS from django.contrib.contenttypes.models import ContentType from django.utils import six @@ -71,7 +72,7 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): def translate_polymorphic_Q_object(self_class, q): return translate_polymorphic_Q_object(self_class, q) - def pre_save_polymorphic(self): + def pre_save_polymorphic(self, using=DEFAULT_DB_ALIAS): """Normally not needed. This function may be called manually in special use-cases. When the object is saved for the first time, we store its real class in polymorphic_ctype. @@ -80,17 +81,18 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): (used by PolymorphicQuerySet._get_real_instances) """ if not self.polymorphic_ctype_id: - self.polymorphic_ctype = ContentType.objects.get_for_model(self, for_concrete_model=False) + self.polymorphic_ctype = ContentType.objects.db_manager(using).get_for_model(self, for_concrete_model=False) pre_save_polymorphic.alters_data = True def save(self, *args, **kwargs): """Overridden model save function which supports the polymorphism functionality (through pre_save_polymorphic).""" - self.pre_save_polymorphic() + using = kwargs.get('using', DEFAULT_DB_ALIAS) + self.pre_save_polymorphic(using=using) return super(PolymorphicModel, self).save(*args, **kwargs) save.alters_data = True - def get_real_instance_class(self): + def get_real_instance_class(self, using=DEFAULT_DB_ALIAS): """ Normally not needed. If a non-polymorphic manager (like base_objects) has been used to @@ -103,7 +105,7 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): # Note that model_class() can return None for stale content types; # when the content type record still exists but no longer refers to an existing model. try: - model = ContentType.objects.get_for_id(self.polymorphic_ctype_id).model_class() + model = ContentType.objects.db_manager(using).get_for_id(self.polymorphic_ctype_id).model_class() except AttributeError: # Django <1.6 workaround return None @@ -118,19 +120,19 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): )) return model - def get_real_concrete_instance_class_id(self): + def get_real_concrete_instance_class_id(self, using=DEFAULT_DB_ALIAS): model_class = self.get_real_instance_class() if model_class is None: return None - return ContentType.objects.get_for_model(model_class, for_concrete_model=True).pk + return ContentType.objects.db_manager(using).get_for_model(model_class, for_concrete_model=True).pk - def get_real_concrete_instance_class(self): + def get_real_concrete_instance_class(self, using=DEFAULT_DB_ALIAS): model_class = self.get_real_instance_class() if model_class is None: return None - return ContentType.objects.get_for_model(model_class, for_concrete_model=True).model_class() + return ContentType.objects.db_manager(using).get_for_model(model_class, for_concrete_model=True).model_class() - def get_real_instance(self): + def get_real_instance(self, using=DEFAULT_DB_ALIAS): """Normally not needed. If a non-polymorphic manager (like base_objects) has been used to retrieve objects, then the complete object with it's real class/type @@ -139,7 +141,7 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): real_model = self.get_real_instance_class() if real_model == self.__class__: return self - return real_model.objects.get(pk=self.pk) + return real_model.objects.db_manager(using).get(pk=self.pk) def __init__(self, * args, ** kwargs): """Replace Django's inheritance accessor member functions for our model diff --git a/polymorphic/query.py b/polymorphic/query.py index b868764..a990e2c 100644 --- a/polymorphic/query.py +++ b/polymorphic/query.py @@ -309,8 +309,9 @@ class PolymorphicQuerySet(QuerySet): # - also record the correct result order in "ordered_id_list" # - store objects that already have the correct class into "results" base_result_objects_by_id = {} - self_model_class_id = ContentType.objects.get_for_model(self.model, for_concrete_model=False).pk - self_concrete_model_class_id = ContentType.objects.get_for_model(self.model, for_concrete_model=True).pk + content_type_manager = ContentType.objects.db_manager(self._db) + self_model_class_id = content_type_manager.get_for_model(self.model, for_concrete_model=False).pk + self_concrete_model_class_id = content_type_manager.get_for_model(self.model, for_concrete_model=True).pk for base_object in base_result_objects: ordered_id_list.append(base_object.pk) @@ -335,7 +336,7 @@ class PolymorphicQuerySet(QuerySet): # upcast it and put it in the results results[base_object.pk] = transmogrify(real_concrete_class, base_object) else: - real_concrete_class = ContentType.objects.get_for_id(real_concrete_class_id).model_class() + real_concrete_class = content_type_manager.get_for_id(real_concrete_class_id).model_class() idlist_per_model[real_concrete_class].append(getattr(base_object, pk_name)) # For each model in "idlist_per_model" request its objects (the real model) diff --git a/polymorphic/query_translate.py b/polymorphic/query_translate.py index 3424eaf..2408e5c 100644 --- a/polymorphic/query_translate.py +++ b/polymorphic/query_translate.py @@ -8,6 +8,7 @@ import django from django.db import models from django.contrib.contenttypes.models import ContentType from django.db.models import Q, FieldDoesNotExist +from django.db.utils import DEFAULT_DB_ALIAS from django.db.models.fields.related import RelatedField if django.VERSION < (1, 6): @@ -50,9 +51,10 @@ def translate_polymorphic_filter_definitions_in_kwargs(queryset_model, kwargs): Returns: a list of non-keyword-arguments (Q objects) to be added to the filter() query. """ additional_args = [] + using = kwargs.get('using', DEFAULT_DB_ALIAS) for field_path, val in kwargs.copy().items(): # Python 3 needs copy - new_expr = _translate_polymorphic_filter_definition(queryset_model, field_path, val) + new_expr = _translate_polymorphic_filter_definition(queryset_model, field_path, val, using=using) if type(new_expr) == tuple: # replace kwargs element @@ -66,7 +68,7 @@ def translate_polymorphic_filter_definitions_in_kwargs(queryset_model, kwargs): return additional_args -def translate_polymorphic_Q_object(queryset_model, potential_q_object): +def translate_polymorphic_Q_object(queryset_model, potential_q_object, using=DEFAULT_DB_ALIAS): def tree_node_correct_field_specs(my_model, node): " process all children of this Q node " for i in range(len(node.children)): @@ -75,7 +77,7 @@ def translate_polymorphic_Q_object(queryset_model, potential_q_object): if type(child) == tuple: # this Q object child is a tuple => a kwarg like Q( instance_of=ModelB ) key, val = child - new_expr = _translate_polymorphic_filter_definition(my_model, key, val) + new_expr = _translate_polymorphic_filter_definition(my_model, key, val, using=using) if new_expr: node.children[i] = new_expr else: @@ -105,7 +107,7 @@ def translate_polymorphic_filter_definitions_in_args(queryset_model, args): translate_polymorphic_Q_object(queryset_model, q) -def _translate_polymorphic_filter_definition(queryset_model, field_path, field_val): +def _translate_polymorphic_filter_definition(queryset_model, field_path, field_val, using=DEFAULT_DB_ALIAS): """ Translate a keyword argument (field_path=field_val), as used for PolymorphicQuerySet.filter()-like functions (and Q objects). @@ -120,9 +122,9 @@ def _translate_polymorphic_filter_definition(queryset_model, field_path, field_v # handle instance_of expressions or alternatively, # if this is a normal Django filter expression, return None if field_path == 'instance_of': - return _create_model_filter_Q(field_val) + return _create_model_filter_Q(field_val, using=using) elif field_path == 'not_instance_of': - return _create_model_filter_Q(field_val, not_instance_of=True) + return _create_model_filter_Q(field_val, not_instance_of=True, using=using) elif not '___' in field_path: return None # no change @@ -229,7 +231,7 @@ def translate_polymorphic_field_path(queryset_model, field_path): return newpath -def _create_model_filter_Q(modellist, not_instance_of=False): +def _create_model_filter_Q(modellist, not_instance_of=False, using=DEFAULT_DB_ALIAS): """ Helper function for instance_of / not_instance_of Creates and returns a Q object that filters for the models in modellist, @@ -254,7 +256,7 @@ def _create_model_filter_Q(modellist, not_instance_of=False): assert False, 'PolymorphicModel: instance_of expects a list of (polymorphic) models or a single (polymorphic) model' def q_class_with_subclasses(model): - q = Q(polymorphic_ctype=ContentType.objects.get_for_model(model, for_concrete_model=False)) + q = Q(polymorphic_ctype=ContentType.objects.db_manager(using).get_for_model(model, for_concrete_model=False)) for subclass in model.__subclasses__(): q = q | q_class_with_subclasses(subclass) return q From 2f11cb6ffd875fc7b7684da04d62428343261552 Mon Sep 17 00:00:00 2001 From: Austin Matsick Date: Sun, 29 May 2016 14:05:45 -0500 Subject: [PATCH 2/3] #216 Specify `using` kwarg in get_real_instance method calls. --- polymorphic/models.py | 6 +++--- polymorphic/query.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/polymorphic/models.py b/polymorphic/models.py index 316267c..4674128 100644 --- a/polymorphic/models.py +++ b/polymorphic/models.py @@ -121,13 +121,13 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): return model def get_real_concrete_instance_class_id(self, using=DEFAULT_DB_ALIAS): - model_class = self.get_real_instance_class() + model_class = self.get_real_instance_class(using=using) if model_class is None: return None return ContentType.objects.db_manager(using).get_for_model(model_class, for_concrete_model=True).pk def get_real_concrete_instance_class(self, using=DEFAULT_DB_ALIAS): - model_class = self.get_real_instance_class() + model_class = self.get_real_instance_class(using=using) if model_class is None: return None return ContentType.objects.db_manager(using).get_for_model(model_class, for_concrete_model=True).model_class() @@ -138,7 +138,7 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): retrieve objects, then the complete object with it's real class/type and all fields may be retrieved with this method. Each method call executes one db query (if necessary).""" - real_model = self.get_real_instance_class() + real_model = self.get_real_instance_class(using=using) if real_model == self.__class__: return self return real_model.objects.db_manager(using).get(pk=self.pk) diff --git a/polymorphic/query.py b/polymorphic/query.py index a990e2c..1465e76 100644 --- a/polymorphic/query.py +++ b/polymorphic/query.py @@ -325,8 +325,8 @@ class PolymorphicQuerySet(QuerySet): results[base_object.pk] = base_object else: - real_concrete_class = base_object.get_real_instance_class() - real_concrete_class_id = base_object.get_real_concrete_instance_class_id() + real_concrete_class = base_object.get_real_instance_class(using=self._db) + real_concrete_class_id = base_object.get_real_concrete_instance_class_id(using=self._db) if real_concrete_class_id is None: # Dealing with a stale content type @@ -372,7 +372,7 @@ class PolymorphicQuerySet(QuerySet): for real_object in real_objects: o_pk = getattr(real_object, pk_name) - real_class = real_object.get_real_instance_class() + real_class = real_object.get_real_instance_class(using=self._db) # If the real class is a proxy, upcast it if real_class != real_concrete_class: From 4aece2b5d3c45e705824ec15163501f83c6bc54d Mon Sep 17 00:00:00 2001 From: Austin Matsick Date: Mon, 30 May 2016 18:55:03 -0500 Subject: [PATCH 3/3] #216 Use self._state.db instead of `using` kwarg in model methods. Thanks to @vdboor for the suggestion. Also: - Add missing `using` kwargs in query_translate functions. - Add a couple unit tests for non-default database functionality. --- polymorphic/models.py | 24 ++++++++++---------- polymorphic/query.py | 12 +++++----- polymorphic/query_translate.py | 7 +++--- polymorphic/tests.py | 41 ++++++++++++++++++++++++++++++++++ runtests.py | 4 ++++ 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/polymorphic/models.py b/polymorphic/models.py index 4674128..b7f91ff 100644 --- a/polymorphic/models.py +++ b/polymorphic/models.py @@ -87,12 +87,12 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): def save(self, *args, **kwargs): """Overridden model save function which supports the polymorphism functionality (through pre_save_polymorphic).""" - using = kwargs.get('using', DEFAULT_DB_ALIAS) + using = kwargs.get('using', self._state.db or DEFAULT_DB_ALIAS) self.pre_save_polymorphic(using=using) return super(PolymorphicModel, self).save(*args, **kwargs) save.alters_data = True - def get_real_instance_class(self, using=DEFAULT_DB_ALIAS): + def get_real_instance_class(self): """ Normally not needed. If a non-polymorphic manager (like base_objects) has been used to @@ -105,7 +105,7 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): # Note that model_class() can return None for stale content types; # when the content type record still exists but no longer refers to an existing model. try: - model = ContentType.objects.db_manager(using).get_for_id(self.polymorphic_ctype_id).model_class() + model = ContentType.objects.db_manager(self._state.db).get_for_id(self.polymorphic_ctype_id).model_class() except AttributeError: # Django <1.6 workaround return None @@ -120,28 +120,28 @@ class PolymorphicModel(six.with_metaclass(PolymorphicModelBase, models.Model)): )) return model - def get_real_concrete_instance_class_id(self, using=DEFAULT_DB_ALIAS): - model_class = self.get_real_instance_class(using=using) + def get_real_concrete_instance_class_id(self): + model_class = self.get_real_instance_class() if model_class is None: return None - return ContentType.objects.db_manager(using).get_for_model(model_class, for_concrete_model=True).pk + return ContentType.objects.db_manager(self._state.db).get_for_model(model_class, for_concrete_model=True).pk - def get_real_concrete_instance_class(self, using=DEFAULT_DB_ALIAS): - model_class = self.get_real_instance_class(using=using) + def get_real_concrete_instance_class(self): + model_class = self.get_real_instance_class() if model_class is None: return None - return ContentType.objects.db_manager(using).get_for_model(model_class, for_concrete_model=True).model_class() + return ContentType.objects.db_manager(self._state.db).get_for_model(model_class, for_concrete_model=True).model_class() - def get_real_instance(self, using=DEFAULT_DB_ALIAS): + def get_real_instance(self): """Normally not needed. If a non-polymorphic manager (like base_objects) has been used to retrieve objects, then the complete object with it's real class/type and all fields may be retrieved with this method. Each method call executes one db query (if necessary).""" - real_model = self.get_real_instance_class(using=using) + real_model = self.get_real_instance_class() if real_model == self.__class__: return self - return real_model.objects.db_manager(using).get(pk=self.pk) + return real_model.objects.db_manager(self._state.db).get(pk=self.pk) def __init__(self, * args, ** kwargs): """Replace Django's inheritance accessor member functions for our model diff --git a/polymorphic/query.py b/polymorphic/query.py index 1465e76..ead4fca 100644 --- a/polymorphic/query.py +++ b/polymorphic/query.py @@ -112,8 +112,8 @@ class PolymorphicQuerySet(QuerySet): def _filter_or_exclude(self, negate, *args, **kwargs): "We override this internal Django functon as it is used for all filter member functions." - translate_polymorphic_filter_definitions_in_args(self.model, args) # the Q objects - additional_args = translate_polymorphic_filter_definitions_in_kwargs(self.model, kwargs) # filter_field='data' + translate_polymorphic_filter_definitions_in_args(self.model, args, using=self._db) # the Q objects + additional_args = translate_polymorphic_filter_definitions_in_kwargs(self.model, kwargs, using=self._db) # filter_field='data' return super(PolymorphicQuerySet, self)._filter_or_exclude(negate, *(list(args) + additional_args), **kwargs) def order_by(self, *args, **kwargs): @@ -325,8 +325,8 @@ class PolymorphicQuerySet(QuerySet): results[base_object.pk] = base_object else: - real_concrete_class = base_object.get_real_instance_class(using=self._db) - real_concrete_class_id = base_object.get_real_concrete_instance_class_id(using=self._db) + real_concrete_class = base_object.get_real_instance_class() + real_concrete_class_id = base_object.get_real_concrete_instance_class_id() if real_concrete_class_id is None: # Dealing with a stale content type @@ -345,7 +345,7 @@ class PolymorphicQuerySet(QuerySet): # Then we copy the extra() select fields from the base objects to the real objects. # TODO: defer(), only(): support for these would be around here for real_concrete_class, idlist in idlist_per_model.items(): - real_objects = real_concrete_class.base_objects.filter(**{ + real_objects = real_concrete_class.base_objects.db_manager(self._db).filter(**{ ('%s__in' % pk_name): idlist, }) real_objects.query.select_related = self.query.select_related # copy select related configuration to new qs @@ -372,7 +372,7 @@ class PolymorphicQuerySet(QuerySet): for real_object in real_objects: o_pk = getattr(real_object, pk_name) - real_class = real_object.get_real_instance_class(using=self._db) + real_class = real_object.get_real_instance_class() # If the real class is a proxy, upcast it if real_class != real_concrete_class: diff --git a/polymorphic/query_translate.py b/polymorphic/query_translate.py index 2408e5c..b7a53e9 100644 --- a/polymorphic/query_translate.py +++ b/polymorphic/query_translate.py @@ -36,7 +36,7 @@ from functools import reduce # functionality to filters and Q objects. # Probably a more general queryset enhancement class could be made out of them. -def translate_polymorphic_filter_definitions_in_kwargs(queryset_model, kwargs): +def translate_polymorphic_filter_definitions_in_kwargs(queryset_model, kwargs, using=DEFAULT_DB_ALIAS): """ Translate the keyword argument list for PolymorphicQuerySet.filter() @@ -51,7 +51,6 @@ def translate_polymorphic_filter_definitions_in_kwargs(queryset_model, kwargs): Returns: a list of non-keyword-arguments (Q objects) to be added to the filter() query. """ additional_args = [] - using = kwargs.get('using', DEFAULT_DB_ALIAS) for field_path, val in kwargs.copy().items(): # Python 3 needs copy new_expr = _translate_polymorphic_filter_definition(queryset_model, field_path, val, using=using) @@ -90,7 +89,7 @@ def translate_polymorphic_Q_object(queryset_model, potential_q_object, using=DEF return potential_q_object -def translate_polymorphic_filter_definitions_in_args(queryset_model, args): +def translate_polymorphic_filter_definitions_in_args(queryset_model, args, using=DEFAULT_DB_ALIAS): """ Translate the non-keyword argument list for PolymorphicQuerySet.filter() @@ -104,7 +103,7 @@ def translate_polymorphic_filter_definitions_in_args(queryset_model, args): """ for q in args: - translate_polymorphic_Q_object(queryset_model, q) + translate_polymorphic_Q_object(queryset_model, q, using=using) def _translate_polymorphic_filter_definition(queryset_model, field_path, field_val, using=DEFAULT_DB_ALIAS): diff --git a/polymorphic/tests.py b/polymorphic/tests.py index 60c60cf..ca11733 100644 --- a/polymorphic/tests.py +++ b/polymorphic/tests.py @@ -430,6 +430,8 @@ class PolymorphicTests(TestCase): The test suite """ + multi_db = True + def test_annotate_aggregate_order(self): # create a blog of type BlogA # create two blog entries in BlogA @@ -1162,6 +1164,45 @@ class PolymorphicTests(TestCase): result = DateModel.objects.annotate(val=DateTime('date', 'day', utc)) self.assertEqual(list(result), []) + def test_save_to_non_default_database(self): + Model2A.objects.db_manager('secondary').create(field1='A1') + Model2C(field1='C1', field2='C2', field3='C3').save(using='secondary') + Model2B.objects.create(field1='B1', field2='B2') + Model2D(field1='D1', field2='D2', field3='D3', field4='D4').save('secondary') + + default_objects = list(Model2A.objects.order_by('id')) + self.assertEqual(len(default_objects), 2) + self.assertEqual(repr(default_objects[0]), '') + self.assertEqual(repr(default_objects[1]), '') + + secondary_objects = list(Model2A.objects.db_manager('secondary').order_by('id')) + self.assertEqual(len(secondary_objects), 2) + self.assertEqual(repr(secondary_objects[0]), '') + self.assertEqual(repr(secondary_objects[1]), '') + + def test_instance_of_filter_on_non_default_database(self): + Base.objects.db_manager('secondary').create(field_b='B1') + ModelX.objects.db_manager('secondary').create(field_b='B', field_x='X') + ModelY.objects.db_manager('secondary').create(field_b='Y', field_y='Y') + + objects = Base.objects.db_manager('secondary').filter(instance_of=Base) + self.assertEqual(len(objects), 3) + self.assertEqual(repr(objects[0]), '') + self.assertEqual(repr(objects[1]), '') + self.assertEqual(repr(objects[2]), '') + + objects = Base.objects.db_manager('secondary').filter(instance_of=ModelX) + self.assertEqual(len(objects), 1) + self.assertEqual(repr(objects[0]), '') + + objects = Base.objects.db_manager('secondary').filter(instance_of=ModelY) + self.assertEqual(len(objects), 1) + self.assertEqual(repr(objects[0]), '') + + objects = Base.objects.db_manager('secondary').filter(Q(instance_of=ModelX) | Q(instance_of=ModelY)) + self.assertEqual(len(objects), 2) + self.assertEqual(repr(objects[0]), '') + self.assertEqual(repr(objects[1]), '') class RegressionTests(TestCase): diff --git a/runtests.py b/runtests.py index f9517d2..26b2665 100755 --- a/runtests.py +++ b/runtests.py @@ -25,6 +25,10 @@ if not settings.configured: 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:' + }, + 'secondary': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:' } }, TEMPLATE_LOADERS=(