Skip to content

Instantly share code, notes, and snippets.

@zachwill
Last active September 18, 2023 09:28
Show Gist options
  • Star 8 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save zachwill/00056d4304c58a035c87cdf5ff1e5e3e to your computer and use it in GitHub Desktop.
Save zachwill/00056d4304c58a035c87cdf5ff1e5e3e to your computer and use it in GitHub Desktop.
An easier way to meld Peewee models and Scrapy items.
import copy
from scrapy import Item
class ModelItem(Item):
"""
Make Peewee models easily turn into Scrapy Items.
>>> from models import Player
>>> item = ModelItem(Player())
"""
def __init__(self, model, **kwds):
super(self.__class__, self).__init__()
self._model = model
for key in model._meta.fields.keys():
self.fields[key] = Field()
if kwds is not None:
for key, processor in kwds.iteritems():
self.fields[key] = Field(input_processor=MapCompose(
strip_whitespace, processor
))
def __setitem__(self, key, value):
if key not in self.fields:
self.fields[key] = Field()
self._values[key] = value
def copy(self):
return copy.deepcopy(self)
@property
def model(self):
return self._model
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment