scrapy_debug.py

import datetime
import time

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from my_project.spiders.regular import SpiderClassName


def run_spider(spider):
    start_time = time.time()
    timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d')
    settings = get_project_settings()
    settings['FEEDS'] = {
        f'{spider.name}_{timestamp}.csv': {
            'format': 'csv'
        },
    }
    process = CrawlerProcess(settings)
    process.crawl(spider)
    process.start()
    print("\n\n{:.2f} Seconds".format(time.time() - start_time))


if __name__ == '__main__':
    run_spider(SpiderClassName)
    print('Done!')