Spider which helps to send HackerOne reports to Pushbullet.

scrapping_h1.py

"""
Spider which helps to send HackerOne reports to Pushbullet.
"""
import datetime
from scrapy import signals, Spider
from scrapy.xlib.pydispatch import dispatcher
from pushbullet import Pushbullet

from settings_prod import PUSHBULLET_KEY


class H1Spider(Spider):
    """
    Scrapping unnofficial H1 disclosed reports
    """
    name = 'H1Spider'
    start_urls = ['http://h1.nobbd.de/']

    def __init__(self):
        self.h1_datas = []
        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def parse(self, response):
        for report in response.css('div.report-wrapper'):
            date_report = report.css('div.report-wrapper').css('div.date::text')[0].extract().strip()
            if date_report == (datetime.datetime.today() - datetime.timedelta(days=1)).strftime('%d %b %Y'):
                h1_data = {
                    'company': report.css('div.report a.company::text').extract_first(),
                    'submitter': report.css('div.report a.submitter::text').extract_first(),
                    'report_description': report.css('div.report a.title::text').extract_first(),
                    'report_url': report.css('div.report a.title::attr(href)').extract_first(),
                    'date': date_report
                }
                self.h1_datas.append(h1_data)
                yield h1_data

    def spider_closed(self, spider):
        """
        Function is executed after parse data
        """
        print 'Sending notifications'
        pb = Pushbullet(PUSHBULLET_KEY)
        for h1 in self.h1_datas:
            pb.push_note(
                "{} - {}...".format(h1['company'], h1['report_description'][:50]),
                "{}\n{}\n{}".format(
                    h1['report_description'], h1['report_url'], h1['date'])
            )