Skip to content

Instantly share code, notes, and snippets.

@rakanalh
Created January 8, 2014 10:01
Show Gist options
  • Save rakanalh/8314423 to your computer and use it in GitHub Desktop.
Save rakanalh/8314423 to your computer and use it in GitHub Desktop.
Using Django-Dynamic-Scraper to extract multiple values to ManyToMany field
"""
What you have to do to use this is extending BaseItem rather than extending DjangoItem because DjangoItem looks for _meta.fields and ignores _meta.many_to_many.
This will handle the ManyToMany fields in your model. So you have to scrape the data using DDS and return an Item that contains the ManyToMany field but as a list [].
The BaseItem class will basically call a function in your Item class named save_%s_values for each ManyToMany field. So if your ManyToMany field is called
photos, you'll have to have a method called save_photos_values which accepts the model, the model_field and the list of values. Example:
def save_photos_values(self, model, model_field, values):
for value in values:
image_filename = settings.IMAGES_STORE + value['path']
image = create_and_save_image_from_file(model.title, image_filename)
model_field.add(image)
"""
class BaseItemMeta(DjangoItemMeta):
def __new__(mcs, class_name, bases, attrs):
cls = super(BaseItemMeta, mcs).__new__(mcs, class_name, bases, attrs)
if cls.django_model:
cls._model_meta = cls.django_model._meta
for model_field in cls._model_meta.many_to_many:
if not model_field.auto_created:
if model_field.name not in cls.fields:
cls.fields[model_field.name] = Field()
cls._model_fields.append(model_field.name)
return cls
class BaseItem(DjangoItem):
__metaclass__ = BaseItemMeta
def save(self, commit=True):
modelargs = dict((k, self.get(k)) for k in self._values if k in self._model_fields)
"""
Remove many to many fields to continue to save
"""
many_to_many_fields = {}
for model_field in self.__class__._model_meta.many_to_many:
many_to_many_fields[model_field.name] = modelargs[model_field.name]
self._values.__delitem__(model_field.name)
model = super(BaseItem, self).save(commit=commit)
for key, values in many_to_many_fields.iteritems():
model_field = getattr(model, key)
func = getattr(self, 'save_%s_values' % key, None)
if callable(func):
func(model, model_field, values)
model.save()
return model
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment