Skip to content

Instantly share code, notes, and snippets.

@metamatik
Created September 22, 2011 18:18
Show Gist options
  • Save metamatik/1235556 to your computer and use it in GitHub Desktop.
Save metamatik/1235556 to your computer and use it in GitHub Desktop.
zipping the pigeon
### pigeon_push.py
# (...)
class Command(BaseCommand):
"""Push items in the ItemToPush queue."""
help = __doc__
def handle(self, *args, **options):
rules = {}
# First, ident/init all the configurations involved in this push
for row in item_to_push_queue():
rule_name = row.rule_name
rule = REGISTRY[rule_name]
if rule_name not in rules.keys():
rule.initialize_push()
rules['rule_name'] = rule
# Then, iterate through the items to push
for row in item_to_push_queue():
logger.debug(u'processing row id=%s, rule_name=%s' %
(row.pk, row.rule_name))
row.status = ItemToPush.STATUS.IN_PROGRESS
row.save()
rule_name = row.rule_name
rule = rules[rule_name]
rule.push_one(row)
# Then, finalize the push for each relevant rule
for rule in rules:
rule.finalize_push()
#########################################################################
### configuration.py
class DefaultConfiguration:
# (...)
def push_one(self, row):
rule_name = self.name
instance = row.content_object
output_files = []
# Build output
try:
output = self.output(instance)
except Exception, e:
message = u"Exception during output generation. "
message += u'Exception ``%s`` raised: %s ' % (
e.__class__.__name__, e.message)
row.status = ItemToPush.STATUS.OUTPUT_GENERATION_ERROR
row.message = message
row.save()
logger.error(message, exc_info=True)
raise
# Validate output
validation = True
for validator in self.validators:
try:
validator(output)
logger.debug('validation ``%s`` passed successfully'
% validator.__name__)
except Exception, e:
validation = False
message = u"Validation ``%s`` failed ! " % validator.__name__
message += u'Catched exception %s : %s' % (
e.__class__.__name__, e.message)
row.status = ItemToPush.STATUS.VALIDATION_ERROR
if row.message != None:
row.message += '\n' + message
else:
row.message = message
logger.error(message, exc_info=True)
row.save()
if not validation: # if one validator did not pass we
# do no want to send the file
logger.debug('the output was not validated')
raise
output_filename = self.get_output_filename(instance)
# Get target directory
target_directory = None
try:
target_directory = self.get_directory(instance)
except Exception, e:
message = u"Error during ``get_directory``. "
message += u"%s: %s" % (
e.__class__.__name__, e.message)
row = ItemToPush(rule_name=rule_name,
content_object=instance)
row.status = ItemToPush.STATUS.GET_DIRECTORY_ERROR
row.message = message
row.save()
logger.error(message, exc_info=True)
raise
# Build output file path for archiving
output_directory = settings.CARRIER_PIGEON_OUTPUT_DIRECTORY
output_directory += '/%s/' % rule_name
output_directory += '/%s/' % target_directory
# Create output directory if necessary
if not os.path.exists(output_directory):
os.makedirs(output_directory)
output_path = '%s/%s' % (output_directory, output_filename)
# Write output file
f = open(output_path, 'w')
f.write(output)
f.close()
output_files.append(output_path)
return output_files
# (...)
class ArchivePusherConfiguration(DefaultConfiguration):
"""
Configurations inheriting this class will first archive all their files
into a ZIP archive and _then_ send this archive onto the destination server.
"""
rows = []
def get_items_to_push(self):
now = datetime.now()
yesterday = now - datetime.timedelta(days=1)
articles = Article.objects.filter(publication_date__gt=yesterday)
for article in articles:
add_item_to_push(article, self.name)
def initialize_push(self):
# Nothing here at the moment
pass
def push_one(self, row):
files = super(FilesPusherConfiguration, self).push_one(self, row)
self.rows.append(row)
def finalize_push(self):
# Build the ZIP archive
directory_name = self.get_export_root_directory()
archive_name = self.get_archive_name()
zipdir(directory_name, archive_name)
# Send it
sent = False
max_ = settings.CARRIER_PIGEON_MAX_PUSH_ATTEMPTS
for push_attempt_num in xrange(max_):
logger.debug('push attempt %s' % push_attempt_num)
if send(archive_name, target_url, output_path):
# Mark all the rows as sent
logger.debug('succeeded')
sent = True
for row in self.rows:
row.status = ItemToPush.STATUS.PUSHED
row.save()
continue
if sent:
# Cleanup...
else:
logger.error(u'Send failed for "%s" after %d attempts' % (
archive_name, max_))
def get_export_root_directory(self):
""" Build the name of the directory to zip. Override me! """
return '%s/%s' % (
settings.CARRIER_PIGEON_OUTPUT_DIRECTORY,
self.name(),
)
def get_archive_name(self):
""" Build the filename of the zip archive to create. Override me! """
now = datetime.datetime.now()
return '%s/%s_%s.zip' % (
settings.CARRIER_PIGEON_OUTPUT_DIRECTORY,
self.name(),
now.strftime(settings.CARRIER_PIGEON_TIMESTAMP_FORMAT)
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment