From dd84e911d08d4a638937df860c1ca9f1e4c36446 Mon Sep 17 00:00:00 2001 From: Bert Constantin Date: Thu, 28 Jan 2010 23:55:16 +0100 Subject: [PATCH] fix "manage.py dumpdata", by adding polymorphic_dumpdata command (github issue 4) --- manage.py | 4 +- .../commands/polymorphic_dumpdata.py | 11 ++ .../commands/polymorphic_dumpdata_11.py | 92 ++++++++++ .../commands/polymorphic_dumpdata_12.py | 173 ++++++++++++++++++ 4 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 poly/management/commands/polymorphic_dumpdata.py create mode 100644 poly/management/commands/polymorphic_dumpdata_11.py create mode 100644 poly/management/commands/polymorphic_dumpdata_12.py diff --git a/manage.py b/manage.py index e530c4e..0d73405 100755 --- a/manage.py +++ b/manage.py @@ -8,7 +8,9 @@ project_path = os.path.dirname(os.path.abspath(__file__)) libs_local_path = os.path.join(project_path, 'libraries-local') if libs_local_path not in sys.path: sys.path.insert(1, libs_local_path) import django -print 'using Django version: %s, from %s' % (django.get_version(), os.path.dirname(os.path.abspath(django.__file__))) +sys.stderr.write( 'using Django version: %s, from %s\n' % ( + django.get_version(), + os.path.dirname(os.path.abspath(django.__file__))) ) # vanilla Django manage.py from here on: diff --git a/poly/management/commands/polymorphic_dumpdata.py b/poly/management/commands/polymorphic_dumpdata.py new file mode 100644 index 0000000..4316530 --- /dev/null +++ b/poly/management/commands/polymorphic_dumpdata.py @@ -0,0 +1,11 @@ + +import django + +if django.VERSION[:2]==(1,1): + from polymorphic_dumpdata_11 import Command + +elif django.VERSION[:2]==(1,2): + from polymorphic_dumpdata_12 import Command + +else: + assert False, 'Django version not supported' diff --git a/poly/management/commands/polymorphic_dumpdata_11.py b/poly/management/commands/polymorphic_dumpdata_11.py new file mode 100644 index 0000000..4d8897c --- /dev/null +++ b/poly/management/commands/polymorphic_dumpdata_11.py @@ -0,0 +1,92 @@ + +from django.core.exceptions import ImproperlyConfigured +from django.core.management.base import BaseCommand, CommandError +from django.core import serializers +from django.utils.datastructures import SortedDict + +from optparse import make_option + +class Command(BaseCommand): + option_list = BaseCommand.option_list + ( + make_option('--format', default='json', dest='format', + help='Specifies the output serialization format for fixtures.'), + make_option('--indent', default=None, dest='indent', type='int', + help='Specifies the indent level to use when pretty-printing output'), + make_option('-e', '--exclude', dest='exclude',action='append', default=[], + help='App to exclude (use multiple --exclude to exclude multiple apps).'), + ) + help = 'Output the contents of the database as a fixture of the given format.' + args = '[appname ...]' + + def handle(self, *app_labels, **options): + from django.db.models import get_app, get_apps, get_models, get_model + + format = options.get('format','json') + indent = options.get('indent',None) + exclude = options.get('exclude',[]) + show_traceback = options.get('traceback', False) + + excluded_apps = [get_app(app_label) for app_label in exclude] + + if len(app_labels) == 0: + app_list = SortedDict([(app, None) for app in get_apps() if app not in excluded_apps]) + else: + app_list = SortedDict() + for label in app_labels: + try: + app_label, model_label = label.split('.') + try: + app = get_app(app_label) + except ImproperlyConfigured: + raise CommandError("Unknown application: %s" % app_label) + + model = get_model(app_label, model_label) + if model is None: + raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) + + if app in app_list.keys(): + if app_list[app] and model not in app_list[app]: + app_list[app].append(model) + else: + app_list[app] = [model] + except ValueError: + # This is just an app - no model qualifier + app_label = label + try: + app = get_app(app_label) + except ImproperlyConfigured: + raise CommandError("Unknown application: %s" % app_label) + app_list[app] = None + + # Check that the serialization format exists; this is a shortcut to + # avoid collating all the objects and _then_ failing. + if format not in serializers.get_public_serializer_formats(): + raise CommandError("Unknown serialization format: %s" % format) + + try: + serializers.get_serializer(format) + except KeyError: + raise CommandError("Unknown serialization format: %s" % format) + + objects = [] + for app, model_list in app_list.items(): + if model_list is None: + model_list = get_models(app) + + for model in model_list: + if not model._meta.proxy: + # modified for django_polymorphic compatibility: + # do not use polymorphic queryset for serialisation + # (as the dumpdata/serializer implementation depends + # on non-polymorphic behavious) + base_manager=model._default_manager + if getattr(model,'polymorphic_model_marker',None) != None: + base_manager=getattr(model,'base_objects',None) + objects.extend(base_manager.all()) + + try: + return serializers.serialize(format, objects, indent=indent) + except Exception, e: + if show_traceback: + raise + raise CommandError("Unable to serialize database: %s" % e) diff --git a/poly/management/commands/polymorphic_dumpdata_12.py b/poly/management/commands/polymorphic_dumpdata_12.py new file mode 100644 index 0000000..ad3a38f --- /dev/null +++ b/poly/management/commands/polymorphic_dumpdata_12.py @@ -0,0 +1,173 @@ +from django.core.exceptions import ImproperlyConfigured +from django.core.management.base import BaseCommand, CommandError +from django.core import serializers +from django.db import connections, DEFAULT_DB_ALIAS +from django.utils.datastructures import SortedDict + +from optparse import make_option + +class Command(BaseCommand): + option_list = BaseCommand.option_list + ( + make_option('--format', default='json', dest='format', + help='Specifies the output serialization format for fixtures.'), + make_option('--indent', default=None, dest='indent', type='int', + help='Specifies the indent level to use when pretty-printing output'), + make_option('--database', action='store', dest='database', + default=DEFAULT_DB_ALIAS, help='Nominates a specific database to load ' + 'fixtures into. Defaults to the "default" database.'), + make_option('-e', '--exclude', dest='exclude',action='append', default=[], + help='App to exclude (use multiple --exclude to exclude multiple apps).'), + make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False, + help='Use natural keys if they are available.'), + ) + help = 'Output the contents of the database as a fixture of the given format.' + args = '[appname ...]' + + def handle(self, *app_labels, **options): + from django.db.models import get_app, get_apps, get_models, get_model + + format = options.get('format','json') + indent = options.get('indent',None) + using = options.get('database', DEFAULT_DB_ALIAS) + connection = connections[using] + exclude = options.get('exclude',[]) + show_traceback = options.get('traceback', False) + use_natural_keys = options.get('use_natural_keys', False) + + excluded_apps = set(get_app(app_label) for app_label in exclude) + + if len(app_labels) == 0: + app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps) + else: + app_list = SortedDict() + for label in app_labels: + try: + app_label, model_label = label.split('.') + try: + app = get_app(app_label) + except ImproperlyConfigured: + raise CommandError("Unknown application: %s" % app_label) + + model = get_model(app_label, model_label) + if model is None: + raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) + + if app in app_list.keys(): + if app_list[app] and model not in app_list[app]: + app_list[app].append(model) + else: + app_list[app] = [model] + except ValueError: + # This is just an app - no model qualifier + app_label = label + try: + app = get_app(app_label) + except ImproperlyConfigured: + raise CommandError("Unknown application: %s" % app_label) + app_list[app] = None + + # Check that the serialization format exists; this is a shortcut to + # avoid collating all the objects and _then_ failing. + if format not in serializers.get_public_serializer_formats(): + raise CommandError("Unknown serialization format: %s" % format) + + try: + serializers.get_serializer(format) + except KeyError: + raise CommandError("Unknown serialization format: %s" % format) + + # Now collate the objects to be serialized. + objects = [] + for model in sort_dependencies(app_list.items()): + if not model._meta.proxy: + # modified for django_polymorphic compatibility: + # do not use polymorphic queryset for serialisation + # (as the dumpdata/serializer implementation depends + # on non-polymorphic behavious) + base_manager=model._default_manager + if getattr(model,'polymorphic_model_marker',None) != None: + base_manager=getattr(model,'base_objects',None) + objects.extend(base_manager.using(using).all()) + + try: + return serializers.serialize(format, objects, indent=indent, + use_natural_keys=use_natural_keys) + except Exception, e: + if show_traceback: + raise + raise CommandError("Unable to serialize database: %s" % e) + +def sort_dependencies(app_list): + """Sort a list of app,modellist pairs into a single list of models. + + The single list of models is sorted so that any model with a natural key + is serialized before a normal model, and any model with a natural key + dependency has it's dependencies serialized first. + """ + from django.db.models import get_model, get_models + # Process the list of models, and get the list of dependencies + model_dependencies = [] + models = set() + for app, model_list in app_list: + if model_list is None: + model_list = get_models(app) + + for model in model_list: + models.add(model) + # Add any explicitly defined dependencies + if hasattr(model, 'natural_key'): + deps = getattr(model.natural_key, 'dependencies', []) + if deps: + deps = [get_model(*d.split('.')) for d in deps] + else: + deps = [] + + # Now add a dependency for any FK or M2M relation with + # a model that defines a natural key + for field in model._meta.fields: + if hasattr(field.rel, 'to'): + rel_model = field.rel.to + if hasattr(rel_model, 'natural_key'): + deps.append(rel_model) + for field in model._meta.many_to_many: + rel_model = field.rel.to + if hasattr(rel_model, 'natural_key'): + deps.append(rel_model) + model_dependencies.append((model, deps)) + + model_dependencies.reverse() + # Now sort the models to ensure that dependencies are met. This + # is done by repeatedly iterating over the input list of models. + # If all the dependencies of a given model are in the final list, + # that model is promoted to the end of the final list. This process + # continues until the input list is empty, or we do a full iteration + # over the input models without promoting a model to the final list. + # If we do a full iteration without a promotion, that means there are + # circular dependencies in the list. + model_list = [] + while model_dependencies: + skipped = [] + changed = False + while model_dependencies: + model, deps = model_dependencies.pop() + + # If all of the models in the dependency list are either already + # on the final model list, or not on the original serialization list, + # then we've found another model with all it's dependencies satisfied. + found = True + for candidate in ((d not in models or d in model_list) for d in deps): + if not candidate: + found = False + if found: + model_list.append(model) + changed = True + else: + skipped.append((model, deps)) + if not changed: + raise CommandError("Can't resolve dependencies for %s in serialized app list." % + ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name) + for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__)) + ) + model_dependencies = skipped + + return model_list \ No newline at end of file