gistfile1.py

""" Announcer Extension

This extension has the objective of log useful stuff at the beginning.

Usage:

EXTENSIONS = {
        'toolbox.extensions.announcer.AnnouncerExtension': 1,
        }

ANNOUNCER_ENABLED = True
ANNOUNCER_SETTINGS_PREFIXES = ['HODOR_']
"""
import pprint

from scrapy import log
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.settings import overridden_settings


class AnnouncerExtension(object):

    def __init__(self, settings):
        self._done = False
        self.settings = settings

    @classmethod
    def from_crawler(cls, crawler):
        if not crawler.settings.getbool('ANNOUNCER_ENABLED'):
            raise NotConfigured

        obj = cls(crawler.settings)
        crawler.signals.connect(obj.announce, signal=signals.spider_opened)
        crawler.signals.connect(obj.announce, signal=signals.engine_started)
        return obj

    def log(self, msg, dct):
        log.msg('{}\n{}'.format(msg, pprint.pformat(dct)), level=log.INFO)

    def announce(self):
        if self._done:
            return
        self._done = True

        overridden = dict(overridden_settings(self.settings))
        self.log('Overridden default settings:', overridden)

        custom_settings = {}
        prefixes = self.settings.getlist('ANNOUNCER_SETTINGS_PREFIXES')
        for key, value in self.settings.attributes.iteritems():
            if any(key.startswith(prefix) for prefix in prefixes):
                custom_settings[key] = value.value
        self.log('Custom settings:', custom_settings)