crawler_stats.txt

{'downloader/exception_count': 1801,
 'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 4,
 'downloader/exception_type_count/twisted.internet.error.TimeoutError': 5,
 'downloader/exception_type_count/twisted.web._newclient.ResponseNeverReceived': 1792,
 'downloader/request_bytes': 8780679,
 'downloader/request_count': 10100,
 'downloader/request_method_count/GET': 2,
 'downloader/request_method_count/POST': 10098,
 'downloader/response_bytes': 12366649,
 'downloader/response_count': 8299,
 'downloader/response_status_count/200': 2730,
 'downloader/response_status_count/404': 1,
 'downloader/response_status_count/500': 15,
 'downloader/response_status_count/502': 5194,
 'downloader/response_status_count/503': 359,
 'finish_reason': 'finished',
 'finish_time': datetime.datetime(2016, 1, 27, 16, 40, 24, 700020),
 'item_scraped_count': 23520,
 'log_count/DEBUG': 33623,
 'log_count/INFO': 137,
 'request_depth_max': 4,
 'response_received_count': 2731,
 'scheduler/dequeued': 10099,
 'scheduler/dequeued/memory': 10099,
 'scheduler/enqueued': 10099,
 'scheduler/enqueued/memory': 10099,
 'start_time': datetime.datetime(2016, 1, 27, 14, 30, 58, 231642)}