Skip to content

Instantly share code, notes, and snippets.

@StasDeep
Created June 10, 2020 07:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save StasDeep/c7f0b775eb4897b1012f2c76ce6296f5 to your computer and use it in GitHub Desktop.
Save StasDeep/c7f0b775eb4897b1012f2c76ce6296f5 to your computer and use it in GitHub Desktop.
Missing storage.store call
from logging import getLogger
import scrapy
from scrapy.extensions.feedexport import BlockingFeedStorage
logger = getLogger(__name__)
class GsFeedStorage(BlockingFeedStorage):
def __init__(self, uri):
self.uri = uri
def _store_in_thread(self, file):
logger.info(f'Storing in thread: {self.uri}')
class QuotesSpider(scrapy.Spider):
name = 'quotes'
start_urls = ['http://quotes.toscrape.com/']
custom_settings = {
'FEEDS': {
'gs://bucket/output.json': {'format': 'json'},
'gs://bucket/output.csv': {'format': 'csv'}
},
'FEED_STORAGES': {'gs': 'test.GsFeedStorage'},
'FEED_STORE_EMPTY': False,
}
def parse(self, response):
pass
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment