Skip to content

Instantly share code, notes, and snippets.

@sveetch
Last active August 29, 2015 14:10
Show Gist options
  • Save sveetch/658956e0ca9a25ec825e to your computer and use it in GitHub Desktop.
Save sveetch/658956e0ca9a25ec825e to your computer and use it in GitHub Desktop.
Django datas dump script writer
[
["sites", {
"comments": "django.contrib.sites: standalone",
"use_natural_key": true,
"models": "sites"
}],
["auth", {
"comments": "django.contrib.auth [user and groups, no perms]: standalone",
"use_natural_key": true,
"models": ["auth.group","auth.user"]
}],
["emencia.django.countries", {
"comments": "emencia.django.countries: standalone",
"use_natural_key": true,
"models": "emencia.django.countries"
}],
["tagging", {
"comments": "django-tagging: standalone, using natural keys seems to broke the dump",
"use_natural_key": false,
"models": "tagging"
}],
["taggit", {
"comments": "django-taggit [alternative to tagging]: standalone, using natural keys seems to broke the dump",
"use_natural_key": false,
"models": "taggit"
}],
["contact_form_app", {
"comments": "contact_form: depends of emencia.django.countries",
"use_natural_key": true,
"models": "project.contact_form",
"dependancies": "emencia.django.countries"
}],
["googletools", {
"comments": "googletools: depends of sites",
"use_natural_key": true,
"models": "googletools",
"dependancies": "sites"
}],
["zinnia", {
"comments": "zinnia: depends of sites, auth and tagging",
"use_natural_key": true,
"models": "zinnia",
"dependancies": ["sites","auth","tagging"]
}],
["south", {
"comments": "south: standalone",
"use_natural_key": true,
"models": "south"
}],
["easy_thumbnails", {
"comments": "easy-thumbnails: standalone",
"use_natural_key": true,
"models": "easy_thumbnails"
}],
["filer", {
"comments": "django-filer: standalone",
"use_natural_key": true,
"models": "filer"
}],
["porticus", {
"comments": "porticus: depends of tagging",
"use_natural_key": true,
"models": "porticus",
"dependancies": "tagging"
}],
["djangocms-placeholder", {
"comments": "djangocms placeholders: standalone",
"use_natural_key": true,
"models": "cms.placeholder"
}],
["djangocms", {
"comments": "djangocms base: depends of sites, auth and cms placeholders",
"use_natural_key": true,
"models": ["cms", "-e cms.usersettings"],
"dependancies": ["sites","auth","djangocms-placeholder","cmsplugin-ckeditor", "cmsplugin-base"]
}],
["cmsplugin-base", {
"comments": "djangocms plugins: depends of cms",
"use_natural_key": true,
"models": ["picture", "link", "file", "flash", "video", "googlemap", "twitter", "teaser"],
"dependancies": "djangocms"
}],
["cmsplugin-ckeditor", {
"comments": "djangocms_text_ckeditor: required for cms but depends of cms [circular dependancy]",
"use_natural_key": true,
"models": "djangocms_text_ckeditor",
"dependancies": "djangocms"
}],
["cmsplugin-snippet", {
"comments": "djangocms common plugins: depends of cms",
"use_natural_key": true,
"models": "snippet",
"dependancies": "djangocms"
}],
["cmsplugin-filer", {
"comments": "django-filer's cms plugins: depends of filer and djangocms",
"use_natural_key": true,
"models": ["cmsplugin_filer_file","cmsplugin_filer_folder","cmsplugin_filer_image"],
"dependancies": ["filer","djangocms"]
}],
["cmsplugin-zinnia", {
"comments": "zinnia's cms plugins: depends of zinnia and djangocms",
"use_natural_key": true,
"models": "cmsplugin_zinnia",
"dependancies": ["zinnia","djangocms"]
}],
["cmsplugin-porticus", {
"comments": "porticus's cms plugins: depends of porticus and djangocms",
"use_natural_key": true,
"models": "cmsplugin_porticus",
"dependancies": ["porticus","djangocms"]
}],
["slideshows", {
"comments": "slideshows and its cms plugin: depends of djangocms",
"use_natural_key": true,
"models": "slideshows",
"dependancies": "djangocms"
}],
["socialaggregator", {
"comments": "socialaggregator and its cms plugin: depends of djangocms",
"use_natural_key": true,
"models": "socialaggregator",
"dependancies": ["taggit", "djangocms"]
}]
]
[
["sites", {
"comments": "django.contrib.sites: standalone",
"use_natural_key": true,
"models": "sites"
}],
["auth", {
"comments": "django.contrib.auth [user and groups, no perms]: standalone",
"use_natural_key": true,
"models": ["auth.group","auth.user"]
}],
["emencia.django.countries", {
"comments": "emencia.django.countries: standalone",
"use_natural_key": true,
"models": "emencia.django.countries"
}],
["tagging", {
"comments": "django-tagging: standalone, using natural keys seems to broke the dump",
"use_natural_key": false,
"models": "tagging"
}],
["taggit", {
"comments": "django-taggit [alternative to tagging]: standalone, using natural keys seems to broke the dump",
"use_natural_key": false,
"models": "taggit"
}],
["contact_form_app", {
"comments": "contact_form: depends of emencia.django.countries",
"use_natural_key": true,
"models": "project.contact_form",
"dependancies": "emencia.django.countries"
}],
["googletools", {
"comments": "googletools: depends of sites",
"use_natural_key": true,
"models": "googletools",
"dependancies": "sites"
}],
["zinnia", {
"comments": "zinnia: depends of sites, auth and tagging",
"use_natural_key": true,
"models": "zinnia",
"dependancies": ["sites","auth","tagging"]
}],
["south", {
"comments": "south: standalone",
"use_natural_key": true,
"models": "south"
}],
["easy_thumbnails", {
"comments": "easy-thumbnails: standalone",
"use_natural_key": true,
"models": "easy_thumbnails"
}],
["filer", {
"comments": "django-filer: standalone",
"use_natural_key": true,
"models": "filer"
}],
["porticus", {
"comments": "porticus: depends of tagging",
"use_natural_key": true,
"models": "porticus",
"dependancies": "tagging"
}],
["djangocms-placeholder", {
"comments": "djangocms placeholders: standalone",
"use_natural_key": true,
"models": "cms.placeholder"
}],
["djangocms", {
"comments": "djangocms base: depends of sites, auth and cms placeholders",
"use_natural_key": true,
"models": ["cms", "-e cms.usersettings"],
"dependancies": ["sites","auth","djangocms-placeholder","cmsplugin-ckeditor"]
}],
["cmsplugin-ckeditor", {
"comments": "djangocms_text_ckeditor: required for cms but depends of cms [circular dependancy]",
"use_natural_key": true,
"models": "djangocms_text_ckeditor",
"dependancies": "djangocms"
}],
["cmsplugin-snippet", {
"comments": "djangocms common plugins: depends of cms",
"use_natural_key": true,
"models": "djangocms_snippet",
"dependancies": "djangocms"
}],
["cmsplugin-various", {
"comments": "djangocms common plugins: depends of cms",
"use_natural_key": true,
"models": ["djangocms_file", "djangocms_flash", "djangocms_picture"],
"dependancies": "djangocms"
}],
["cmsplugin-filer", {
"comments": "django-filer's cms plugins: depends of filer and djangocms",
"use_natural_key": true,
"models": ["cmsplugin_filer_file","cmsplugin_filer_folder","cmsplugin_filer_image"],
"dependancies": ["filer","djangocms"]
}],
["cmsplugin-zinnia", {
"comments": "zinnia's cms plugins: depends of zinnia and djangocms",
"use_natural_key": true,
"models": "cmsplugin_zinnia",
"dependancies": ["zinnia","djangocms"]
}],
["cmsplugin-porticus", {
"comments": "porticus's cms plugins: depends of porticus and djangocms",
"use_natural_key": true,
"models": "cmsplugin_porticus",
"dependancies": ["porticus","djangocms"]
}],
["slideshows", {
"comments": "slideshows and its cms plugin: depends of djangocms",
"use_natural_key": true,
"models": "slideshows",
"dependancies": "djangocms"
}],
["socialaggregator", {
"comments": "socialaggregator and its cms plugin: depends of djangocms",
"use_natural_key": true,
"models": "socialaggregator",
"dependancies": ["taggit", "djangocms"]
}]
]
"""
Django datas dump script writer
Produce command line script usable within a Makefile or as a simple shell script to dump or load data with Django from the many app name you give it.
It need a dependancies map to know what is required to be dumped, actually this is only for "emencia-paste-djangocms-2" or "emencia-paste-djangocms-3" projects so it only knows about:
* Django contrib auth;
* Django sites;
* emencia.django.countries;
* contact_form;
* DjangoCMS and its common plugins;
* Zinnia;
* Porticus;
* South;
* django-tagging;
* django-taggit;
* easy-thumbnails;
* django-filer;
* django-google-tools;
* emencia-django-socialaggregator;
* emencia-django-slideshows;
Note : Many app depends on Django's content types but we don't matter because it is automatically filled by Django and we should never try to dump/load it.
"""
import StringIO
from collections import OrderedDict
class DataDumpScriptMakefile(OrderedDict):
"""
Object to store a catalog of available dumps with some methods to get a
clean dump map with their required dependancies.
* Dumps order does matter to respect module's dependancies;
* model or dependancy names can be string or either a list of names, take care that string is splitted on white spaces, if you use excude flag like '-e' with your model names, allways use a list;
* Circular dependancies is possible;
[
('DUMP_NAME_KEY', {
'use_natural_key': USE NATURAL KEY? BOOLEAN,
'models': MODEL NAME STRING OR MODEL NAMES LIST,
'dependancies': DEPENDANCY DUMP NAME STRING OR DEPENDANCY DUMP NAMES LIST,
}),
]
"""
deps_index = {}
makefile_silencer = '@' # Make it empty to avoid silencer like without a Makefile
django_instance_path = 'bin/django-instance'
dumps_import_path = '$(IMPORTFROM_DATA_DUMP_DIR)/' # Must end with a slash
dumps_export_path = '$(EXPORTTO_DATA_DUMP_DIR)/' # Must end with a slash
def __setitem__(self, key, value):
"""
Perform string to list translation and dependancies indexing when setting an item
"""
if isinstance(value['models'], basestring):
value['models'] = value['models'].split()
if 'dependancies' in value:
if isinstance(value['dependancies'], basestring):
value['dependancies'] = value['dependancies'].split()
for k in value['dependancies']:
if k not in self.deps_index:
self.deps_index[k] = set([])
self.deps_index[k].add(key)
OrderedDict.__setitem__(self, key, value)
def get_dump_names(self, names, dumps=None):
"""
Find and return all dump names required (by dependancies) for a given
dump names list
Beware, the returned name list does not respect order, you should only
use it when walking throught the "original" dict builded by OrderedDict
"""
# Default value for dumps argument is an empty set (setting directly
# as a python argument would result as a shared value between
# instances)
if dumps is None:
dumps = set([])
#print "* Wanted:", names
# Add name to the dumps and find its dependancies
for item in names:
dumps.add(item)
if item not in self:
raise KeyError("Dump name '{0}' is unknowed".format(item))
# Add dependancies names to the dumps
deps = self.__getitem__(item).get('dependancies', [])
dumps.update(deps)
# Avoid maximum recursion when we allready find all dependancies
if names == dumps:
return dumps
# Seem we don't have finded other dependancies yet, recurse to do it
return self.get_dump_names(dumps.copy(), dumps)
def get_dump_order(self, names):
"""
Return ordered dump names required for a given dump names list
"""
finded_names = self.get_dump_names(names)
return [item for item in self if item in finded_names]
def build_template(self, names, renderer):
"""
Return a string contained all dumpdata lines for the given dump names
"""
fp = StringIO.StringIO()
for i, item in enumerate(self.get_dump_order(names), start=1):
fp = renderer(fp, i, item, self.__getitem__(item))
content = fp.getvalue()
fp.close()
return content
def _get_dump_item_context(self, index, name, opts):
"""
Return a formated dict context
"""
c = {
'item_no': index,
'label': name,
'name': name,
'models': ' '.join(opts['models']),
'natural_key': '',
'silencer': self.makefile_silencer,
'django_instance': self.django_instance_path,
'import_path': self.dumps_import_path,
'export_path': self.dumps_export_path,
}
if opts.get('use_natural_key', False):
c['natural_key'] = ' -n'
return c
def build_dumpdata(self, names):
"""
Build dumpdata commands
"""
return self.build_template(names, self._dumpdata_template)
def _dumpdata_template(self, stringbuffer, index, name, opts):
"""
StringIO "templates" to build a command line for 'dumpdata'
"""
context = self._get_dump_item_context(index, name, opts)
stringbuffer.write('{silencer}echo "* {label}: dump.{item_no}.{name}.json"'.format(**context))
stringbuffer.write('\n')
stringbuffer.write('{silencer}{django_instance} dumpdata {natural_key}--indent=2 {models} > {export_path}dump.{item_no}.{name}.json'.format(**context))
stringbuffer.write('\n\n')
return stringbuffer
def build_loaddata(self, names):
"""
Build loaddata commands
"""
return self.build_template(names, self._loaddata_template)
def _loaddata_template(self, stringbuffer, index, name, opts):
"""
StringIO "templates" to build a command line for 'loaddata'
"""
context = self._get_dump_item_context(index, name, opts)
stringbuffer.write('{silencer}echo "* Importing: dump.{item_no}.{name}.json"'.format(**context))
stringbuffer.write('\n')
stringbuffer.write('{silencer}{django_instance} loaddata {import_path}dump.{item_no}.{name}.json'.format(**context))
stringbuffer.write('\n\n')
return stringbuffer
class DataDumpScriptCLI(DataDumpScriptMakefile):
"""
Same as DataDumpScriptMakefile but for a simple shell script, so disable the silencer
"""
makefile_silencer = ''
"""
Sample
"""
if __name__ == "__main__":
import json
AVAILABLE_DUMPS = json.load(open("data_dependancies_djangocms-3.json", "r"))
dump_manager = DataDumpScriptMakefile(AVAILABLE_DUMPS)
print "=== Dump map ==="
print dump_manager.build_dumpdata(['djangocms','porticus',])
print
print
#print dump_manager.build_loaddata(['deep-foo','socialaggregator',])
#print
@sveetch
Copy link
Author

sveetch commented Dec 4, 2014

Ok so i added a JSON file for dependancies with "emencia-paste-djangocms-2" that is pretty almost the same than "emencia-paste-djangocms-3" but the "cmsplugin-various" that become "cmsplugin-base" because in cms2 they were shipped in the cms and almost allways enabled, so the cms depend on them.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment