Created
December 19, 2021 09:42
-
-
Save finloop/f23ac73206f34eba3167e5c0b2a8bac4 to your computer and use it in GitHub Desktop.
Logi
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.805264, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 10, 427176), | |
| 'httpcompression/response_bytes': 62728, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 9, 621912)} | |
| 2021-12-19 10:29:10 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:10 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:10 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:10 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:10 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:10 [scrapy.extensions.telnet] INFO: Telnet Password: 4732dab72967ef3e | |
| 2021-12-19 10:29:10 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:10 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:10 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:10 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:10 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:10 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:10 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:10 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:10 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:10 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:11 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/kroscienko-wyzne> (referer: None) | |
| 2021-12-19 10:29:11 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:11 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11773, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.768299, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 11, 348882), | |
| 'httpcompression/response_bytes': 65644, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 10, 580583)} | |
| 2021-12-19 10:29:11 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:11 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:11 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:11 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:11 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:11 [scrapy.extensions.telnet] INFO: Telnet Password: 3fb7876d1ce55a0d | |
| 2021-12-19 10:29:11 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:11 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:11 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:11 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:11 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:11 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:11 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:11 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:11 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:11 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:12 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/miejsce-piastowe> (referer: None) | |
| 2021-12-19 10:29:12 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:12 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11611, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.780619, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 12, 272160), | |
| 'httpcompression/response_bytes': 64478, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 11, 491541)} | |
| 2021-12-19 10:29:12 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:12 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:12 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:12 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:12 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:12 [scrapy.extensions.telnet] INFO: Telnet Password: eeb5c57075547447 | |
| 2021-12-19 10:29:12 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:12 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:12 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:12 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:12 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:12 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:12 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:12 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:12 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/rymanow> (referer: None) | |
| 2021-12-19 10:29:13 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:13 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 425, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11660, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.692104, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 13, 115058), | |
| 'httpcompression/response_bytes': 64773, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 12, 422954)} | |
| 2021-12-19 10:29:13 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:13 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:13 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:13 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:13 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:13 [scrapy.extensions.telnet] INFO: Telnet Password: 46c141328b854b0d | |
| 2021-12-19 10:29:13 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:13 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:13 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:13 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:13 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:13 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:13 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:13 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:13 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/wojaszowka> (referer: None) | |
| 2021-12-19 10:29:14 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:14 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 428, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11455, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.73561, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 14, 9959), | |
| 'httpcompression/response_bytes': 63746, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 13, 274349)} | |
| 2021-12-19 10:29:14 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:14 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:14 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:14 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:14 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:14 [scrapy.extensions.telnet] INFO: Telnet Password: 5e4d61d9fef9b1dd | |
| 2021-12-19 10:29:14 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:14 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:14 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:14 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:14 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:14 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:14 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:14 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:14 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:14 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:14 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/chorkowka> (referer: None) | |
| 2021-12-19 10:29:14 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:14 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 427, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11498, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.779025, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 14, 931829), | |
| 'httpcompression/response_bytes': 63414, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 89063424, | |
| 'memusage/startup': 89063424, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 14, 152804)} | |
| 2021-12-19 10:29:14 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:15 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:15 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:15 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:15 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:15 [scrapy.extensions.telnet] INFO: Telnet Password: 300ad6f393f0d82f | |
| 2021-12-19 10:29:15 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:15 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:15 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:15 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:15 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:15 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:15 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:15 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:15 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/robots.txt> from <GET http://krosno24.pl/robots.txt> | |
| 2021-12-19 10:29:15 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:15 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/archiwum/> from <GET http://krosno24.pl/archiwum/> | |
| 2021-12-19 10:29:15 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/archiwum/> (referer: None) | |
| 2021-12-19 10:29:15 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:15 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 826, | |
| 'downloader/request_count': 4, | |
| 'downloader/request_method_count/GET': 4, | |
| 'downloader/response_bytes': 11435, | |
| 'downloader/response_count': 4, | |
| 'downloader/response_status_count/200': 2, | |
| 'downloader/response_status_count/301': 2, | |
| 'elapsed_time_seconds': 0.764205, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 15, 904666), | |
| 'httpcompression/response_bytes': 55846, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 4, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 2, | |
| 'scheduler/dequeued/memory': 2, | |
| 'scheduler/enqueued': 2, | |
| 'scheduler/enqueued/memory': 2, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 15, 140461)} | |
| 2021-12-19 10:29:15 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:15 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:15 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:15 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:15 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:15 [scrapy.extensions.telnet] INFO: Telnet Password: d0d07d3d599b8d03 | |
| 2021-12-19 10:29:16 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:16 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:16 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:16 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:16 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:16 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:16 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:16 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:16 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://krosno112.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:16 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:16 [protego] DEBUG: Rule at line 2 without any user agent to enforce it on. | |
| 2021-12-19 10:29:16 [protego] DEBUG: Rule at line 24 without any user agent to enforce it on. | |
| 2021-12-19 10:29:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno112.pl/aktualnosci.html> (referer: None) | |
| 2021-12-19 10:29:17 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:17 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 422, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 20290, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 1, | |
| 'downloader/response_status_count/404': 1, | |
| 'elapsed_time_seconds': 1.304875, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 17, 373853), | |
| 'httpcompression/response_bytes': 101162, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 5, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/404': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 16, 68978)} | |
| 2021-12-19 10:29:17 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:17 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:17 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:17 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:17 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:17 [scrapy.extensions.telnet] INFO: Telnet Password: 1d3923ad0e9c03ec | |
| 2021-12-19 10:29:17 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:17 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:17 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:17 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:17 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:17 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:17 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:17 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:17 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/chorkowka> (referer: None) | |
| 2021-12-19 10:29:18 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:18 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 427, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11490, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.781878, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 18, 316420), | |
| 'httpcompression/response_bytes': 63414, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 17, 534542)} | |
| 2021-12-19 10:29:18 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:18 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:18 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:18 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:18 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:18 [scrapy.extensions.telnet] INFO: Telnet Password: fafa9ba62ca22e7b | |
| 2021-12-19 10:29:18 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:18 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:18 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:18 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:18 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:18 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:18 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:18 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:18 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/krosno> (referer: None) | |
| 2021-12-19 10:29:19 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:19 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 424, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11283, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.760878, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 19, 235858), | |
| 'httpcompression/response_bytes': 61270, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 18, 474980)} | |
| 2021-12-19 10:29:19 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:19 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:19 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:19 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:19 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:19 [scrapy.extensions.telnet] INFO: Telnet Password: 5fdf2f428775fb64 | |
| 2021-12-19 10:29:19 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:19 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:19 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:19 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:19 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:19 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:19 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:19 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:19 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/dukla> (referer: None) | |
| 2021-12-19 10:29:20 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:20 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 423, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11610, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.6711, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 20, 85978), | |
| 'httpcompression/response_bytes': 63442, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 19, 414878)} | |
| 2021-12-19 10:29:20 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:20 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:20 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:20 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:20 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:20 [scrapy.extensions.telnet] INFO: Telnet Password: c5407743e61b0e5c | |
| 2021-12-19 10:29:20 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:20 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:20 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:20 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:20 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:20 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:20 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:20 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:20 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:20 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:20 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/iwonicz-zdroj> (referer: None) | |
| 2021-12-19 10:29:20 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:20 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 431, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11503, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.724223, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 20, 973611), | |
| 'httpcompression/response_bytes': 63759, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 20, 249388)} | |
| 2021-12-19 10:29:20 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:21 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:21 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:21 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:21 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:21 [scrapy.extensions.telnet] INFO: Telnet Password: 0cc9be2430bd83d0 | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:21 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:21 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:21 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:21 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:21 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jasliska> (referer: None) | |
| 2021-12-19 10:29:21 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:21 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11496, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.674674, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 21, 797257), | |
| 'httpcompression/response_bytes': 62892, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 21, 122583)} | |
| 2021-12-19 10:29:21 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:21 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:21 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:21 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:21 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:21 [scrapy.extensions.telnet] INFO: Telnet Password: 0b7ecfd7bd6d3901 | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:21 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:21 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:21 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:21 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:21 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:22 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:22 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:22 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jedlicze> (referer: None) | |
| 2021-12-19 10:29:22 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:22 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11692, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.665354, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 22, 611313), | |
| 'httpcompression/response_bytes': 64087, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 21, 945959)} | |
| 2021-12-19 10:29:22 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:22 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:22 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:22 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:22 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:22 [scrapy.extensions.telnet] INFO: Telnet Password: 36a02ee79a52a727 | |
| 2021-12-19 10:29:22 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:22 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:22 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:22 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:22 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:22 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:22 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:22 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:22 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:22 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:23 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/korczyna> (referer: None) | |
| 2021-12-19 10:29:23 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:23 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11457, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.673016, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 23, 430344), | |
| 'httpcompression/response_bytes': 62728, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 22, 757328)} | |
| 2021-12-19 10:29:23 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:23 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:23 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:23 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:23 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:23 [scrapy.extensions.telnet] INFO: Telnet Password: 835f32a921685e40 | |
| 2021-12-19 10:29:23 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:23 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:23 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:23 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:23 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:23 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:23 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:23 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:23 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:23 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/kroscienko-wyzne> (referer: None) | |
| 2021-12-19 10:29:24 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:24 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11782, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.681343, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 24, 252030), | |
| 'httpcompression/response_bytes': 65644, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 23, 570687)} | |
| 2021-12-19 10:29:24 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:24 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:24 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:24 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:24 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:24 [scrapy.extensions.telnet] INFO: Telnet Password: 640e5f44fde3d7b5 | |
| 2021-12-19 10:29:24 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:24 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:24 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:24 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:24 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:24 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:24 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:24 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:24 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/miejsce-piastowe> (referer: None) | |
| 2021-12-19 10:29:25 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:25 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11599, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.674466, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 25, 70654), | |
| 'httpcompression/response_bytes': 64478, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 24, 396188)} | |
| 2021-12-19 10:29:25 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:25 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:25 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:25 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:25 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:25 [scrapy.extensions.telnet] INFO: Telnet Password: d3ce134ba7bdc117 | |
| 2021-12-19 10:29:25 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:25 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:25 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:25 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:25 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:25 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:25 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:25 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:25 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:25 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:25 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/rymanow> (referer: None) | |
| 2021-12-19 10:29:25 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:25 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 425, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11656, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.767459, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 25, 991731), | |
| 'httpcompression/response_bytes': 64773, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 25, 224272)} | |
| 2021-12-19 10:29:25 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:26 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:26 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:26 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:26 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:26 [scrapy.extensions.telnet] INFO: Telnet Password: cc1a398c8628d2bc | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:26 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:26 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:26 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:26 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:26 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:26 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:26 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/wojaszowka> (referer: None) | |
| 2021-12-19 10:29:26 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:26 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 428, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11456, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.667377, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 26, 809226), | |
| 'httpcompression/response_bytes': 63746, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 26, 141849)} | |
| 2021-12-19 10:29:26 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:26 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:26 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:26 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:26 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:26 [scrapy.extensions.telnet] INFO: Telnet Password: 648215aafc7071f1 | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:26 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:26 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:26 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:26 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:26 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:27 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/krosno> (referer: None) | |
| 2021-12-19 10:29:27 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:27 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 424, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11278, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.885183, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 27, 833932), | |
| 'httpcompression/response_bytes': 61270, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 89063424, | |
| 'memusage/startup': 89063424, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 26, 948749)} | |
| 2021-12-19 10:29:27 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:27 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:27 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:27 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:27 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:27 [scrapy.extensions.telnet] INFO: Telnet Password: 772c4c634b4aa180 | |
| 2021-12-19 10:29:28 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:28 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:28 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:28 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:28 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:28 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:28 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:28 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:28 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/robots.txt> from <GET http://krosno24.pl/robots.txt> | |
| 2021-12-19 10:29:28 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:28 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/archiwum/> from <GET http://krosno24.pl/archiwum/> | |
| 2021-12-19 10:29:28 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/archiwum/> (referer: None) | |
| 2021-12-19 10:29:28 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:28 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 826, | |
| 'downloader/request_count': 4, | |
| 'downloader/request_method_count/GET': 4, | |
| 'downloader/response_bytes': 11449, | |
| 'downloader/response_count': 4, | |
| 'downloader/response_status_count/200': 2, | |
| 'downloader/response_status_count/301': 2, | |
| 'elapsed_time_seconds': 0.801037, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 28, 844293), | |
| 'httpcompression/response_bytes': 55847, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 4, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 2, | |
| 'scheduler/dequeued/memory': 2, | |
| 'scheduler/enqueued': 2, | |
| 'scheduler/enqueued/memory': 2, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 28, 43256)} | |
| 2021-12-19 10:29:28 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:28 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:28 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:28 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:28 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:28 [scrapy.extensions.telnet] INFO: Telnet Password: 9391ed305123b28f | |
| 2021-12-19 10:29:28 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:29 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:29 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:29 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:29 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:29 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:29 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:29 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://krosno112.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:29 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:29 [protego] DEBUG: Rule at line 2 without any user agent to enforce it on. | |
| 2021-12-19 10:29:29 [protego] DEBUG: Rule at line 24 without any user agent to enforce it on. | |
| 2021-12-19 10:29:30 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno112.pl/aktualnosci.html> (referer: None) | |
| 2021-12-19 10:29:30 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:30 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 422, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 20355, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 1, | |
| 'downloader/response_status_count/404': 1, | |
| 'elapsed_time_seconds': 1.386435, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 30, 395937), | |
| 'httpcompression/response_bytes': 101175, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 5, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/404': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 29, 9502)} | |
| 2021-12-19 10:29:30 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:30 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:30 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:30 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:30 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:30 [scrapy.extensions.telnet] INFO: Telnet Password: 4e75862601207d5f | |
| 2021-12-19 10:29:30 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:30 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:30 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:30 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:30 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:30 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:30 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:30 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:30 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:30 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:31 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/chorkowka> (referer: None) | |
| 2021-12-19 10:29:31 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:31 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 427, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11499, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.079987, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 31, 622253), | |
| 'httpcompression/response_bytes': 63414, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 30, 542266)} | |
| 2021-12-19 10:29:31 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:31 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:31 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:31 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:31 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:31 [scrapy.extensions.telnet] INFO: Telnet Password: 3d2914020740316a | |
| 2021-12-19 10:29:31 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:31 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:31 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:31 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:31 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:31 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:31 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:31 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:31 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:31 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:32 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/krosno> (referer: None) | |
| 2021-12-19 10:29:32 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:32 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 424, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11269, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.071881, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 32, 850577), | |
| 'httpcompression/response_bytes': 61270, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 31, 778696)} | |
| 2021-12-19 10:29:32 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:32 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:32 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:32 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:32 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:32 [scrapy.extensions.telnet] INFO: Telnet Password: f06170659c96b6c6 | |
| 2021-12-19 10:29:32 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:32 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:32 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:32 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:32 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:33 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:33 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:33 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:33 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:33 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:34 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/dukla> (referer: None) | |
| 2021-12-19 10:29:34 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:34 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 423, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11621, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.484816, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 34, 488104), | |
| 'httpcompression/response_bytes': 63442, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 33, 3288)} | |
| 2021-12-19 10:29:34 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:34 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:34 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:34 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:34 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:34 [scrapy.extensions.telnet] INFO: Telnet Password: b06425a77758422b | |
| 2021-12-19 10:29:34 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:34 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:34 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:34 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:34 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:34 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:34 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:34 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:34 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:34 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/iwonicz-zdroj> (referer: None) | |
| 2021-12-19 10:29:35 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:35 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 431, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11504, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.280214, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 35, 921712), | |
| 'httpcompression/response_bytes': 63759, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 34, 641498)} | |
| 2021-12-19 10:29:35 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:35 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:35 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:35 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:35 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:36 [scrapy.extensions.telnet] INFO: Telnet Password: 173329ff179dddde | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:36 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:36 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:36 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:36 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:36 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jasliska> (referer: None) | |
| 2021-12-19 10:29:36 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:36 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11491, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.769571, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 36, 844025), | |
| 'httpcompression/response_bytes': 62892, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 36, 74454)} | |
| 2021-12-19 10:29:36 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:36 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:36 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:36 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:36 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:36 [scrapy.extensions.telnet] INFO: Telnet Password: 4295d5b0b7b5a917 | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:36 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:36 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:37 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:37 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:37 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:37 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:37 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:38 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jedlicze> (referer: None) | |
| 2021-12-19 10:29:38 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:38 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11687, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.173116, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 38, 175443), | |
| 'httpcompression/response_bytes': 64087, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 37, 2327)} | |
| 2021-12-19 10:29:38 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:38 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:38 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:38 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:38 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:38 [scrapy.extensions.telnet] INFO: Telnet Password: febdcf41ed6ed0a1 | |
| 2021-12-19 10:29:38 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:38 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:38 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:38 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:38 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:38 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:38 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:38 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:38 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:38 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:39 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/korczyna> (referer: None) | |
| 2021-12-19 10:29:39 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:39 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11448, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.384802, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 39, 712270), | |
| 'httpcompression/response_bytes': 62728, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 38, 327468)} | |
| 2021-12-19 10:29:39 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:39 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:39 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:39 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:39 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:39 [scrapy.extensions.telnet] INFO: Telnet Password: 9db5b897dcd48e43 | |
| 2021-12-19 10:29:39 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:39 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:39 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:39 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:39 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:39 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:39 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:39 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:39 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:39 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:40 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/kroscienko-wyzne> (referer: None) | |
| 2021-12-19 10:29:41 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:41 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11772, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.201806, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 41, 62177), | |
| 'httpcompression/response_bytes': 65644, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 39, 860371)} | |
| 2021-12-19 10:29:41 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:41 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:41 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:41 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:41 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:41 [scrapy.extensions.telnet] INFO: Telnet Password: b28007d3b7274ca2 | |
| 2021-12-19 10:29:41 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:41 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:41 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:41 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:41 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:41 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:41 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:41 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:41 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:41 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:42 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/miejsce-piastowe> (referer: None) | |
| 2021-12-19 10:29:42 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:42 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11607, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.367264, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 42, 583792), | |
| 'httpcompression/response_bytes': 64478, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 41, 216528)} | |
| 2021-12-19 10:29:42 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:42 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:42 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:42 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:42 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:42 [scrapy.extensions.telnet] INFO: Telnet Password: e59cb42096bbf836 | |
| 2021-12-19 10:29:42 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:42 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:42 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:42 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:42 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:42 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:42 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:42 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:42 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:42 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:44 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/rymanow> (referer: None) | |
| 2021-12-19 10:29:44 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:44 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 425, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11657, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.381565, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 44, 118206), | |
| 'httpcompression/response_bytes': 64773, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 42, 736641)} | |
| 2021-12-19 10:29:44 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:44 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:44 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:44 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:44 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:44 [scrapy.extensions.telnet] INFO: Telnet Password: 9a502f33c65dc3e0 | |
| 2021-12-19 10:29:44 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:44 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:44 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:44 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:44 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:44 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:44 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:44 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:44 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:44 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:45 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/wojaszowka> (referer: None) | |
| 2021-12-19 10:29:45 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:45 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 428, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11461, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.28214, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 45, 547779), | |
| 'httpcompression/response_bytes': 63746, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 44, 265639)} | |
| 2021-12-19 10:29:45 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:45 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:45 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:45 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:45 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:45 [scrapy.extensions.telnet] INFO: Telnet Password: 3187c32b05035738 | |
| 2021-12-19 10:29:45 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:45 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:45 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:45 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:45 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:45 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:45 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:45 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:45 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:45 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:46 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/dukla> (referer: None) | |
| 2021-12-19 10:29:47 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:47 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 423, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11620, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.396224, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 47, 85533), | |
| 'httpcompression/response_bytes': 63442, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 89063424, | |
| 'memusage/startup': 89063424, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 45, 689309)} | |
| 2021-12-19 10:29:47 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:47 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:47 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:47 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:47 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:47 [scrapy.extensions.telnet] INFO: Telnet Password: 5ef358338573d960 | |
| 2021-12-19 10:29:47 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:47 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:47 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:47 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:47 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:47 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:47 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:47 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:47 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/robots.txt> from <GET http://krosno24.pl/robots.txt> | |
| 2021-12-19 10:29:47 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:47 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/archiwum/> from <GET http://krosno24.pl/archiwum/> | |
| 2021-12-19 10:29:47 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/archiwum/> (referer: None) | |
| 2021-12-19 10:29:48 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:48 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 826, | |
| 'downloader/request_count': 4, | |
| 'downloader/request_method_count/GET': 4, | |
| 'downloader/response_bytes': 11458, | |
| 'downloader/response_count': 4, | |
| 'downloader/response_status_count/200': 2, | |
| 'downloader/response_status_count/301': 2, | |
| 'elapsed_time_seconds': 0.789704, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 48, 89041), | |
| 'httpcompression/response_bytes': 55953, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 4, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 2, | |
| 'scheduler/dequeued/memory': 2, | |
| 'scheduler/enqueued': 2, | |
| 'scheduler/enqueued/memory': 2, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 47, 299337)} | |
| 2021-12-19 10:29:48 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:48 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:48 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:48 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:48 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:48 [scrapy.extensions.telnet] INFO: Telnet Password: 11661b54c8f3f501 | |
| 2021-12-19 10:29:48 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:48 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:48 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:48 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:48 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:48 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:48 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:48 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:48 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://krosno112.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:48 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:48 [protego] DEBUG: Rule at line 2 without any user agent to enforce it on. | |
| 2021-12-19 10:29:48 [protego] DEBUG: Rule at line 24 without any user agent to enforce it on. | |
| 2021-12-19 10:29:49 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno112.pl/aktualnosci.html> (referer: None) | |
| 2021-12-19 10:29:49 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:49 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 422, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 20311, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 1, | |
| 'downloader/response_status_count/404': 1, | |
| 'elapsed_time_seconds': 1.286888, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 49, 543244), | |
| 'httpcompression/response_bytes': 101175, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 5, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/404': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 48, 256356)} | |
| 2021-12-19 10:29:49 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:49 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:49 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:49 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:49 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:49 [scrapy.extensions.telnet] INFO: Telnet Password: ea2d2a432fc29e38 | |
| 2021-12-19 10:29:49 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:49 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:49 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:49 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:49 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:49 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:49 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:49 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:49 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:51 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/chorkowka> (referer: None) | |
| 2021-12-19 10:29:51 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:51 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 427, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11486, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.496244, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 51, 184077), | |
| 'httpcompression/response_bytes': 63414, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 49, 687833)} | |
| 2021-12-19 10:29:51 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:51 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:51 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:51 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:51 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:51 [scrapy.extensions.telnet] INFO: Telnet Password: 7f814f22d110b2a3 | |
| 2021-12-19 10:29:51 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:51 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:51 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:51 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:51 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:51 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:51 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:51 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:51 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:51 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:52 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/krosno> (referer: None) | |
| 2021-12-19 10:29:52 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:52 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 424, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11264, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.282564, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 52, 616039), | |
| 'httpcompression/response_bytes': 61270, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 51, 333475)} | |
| 2021-12-19 10:29:52 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:52 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:52 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:52 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:52 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:52 [scrapy.extensions.telnet] INFO: Telnet Password: e4ce444dbd6a243b | |
| 2021-12-19 10:29:52 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:52 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:52 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:52 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:52 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:52 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:52 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:52 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:52 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:52 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:54 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/dukla> (referer: None) | |
| 2021-12-19 10:29:54 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:54 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 423, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11612, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.487602, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 54, 254532), | |
| 'httpcompression/response_bytes': 63442, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 52, 766930)} | |
| 2021-12-19 10:29:54 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:54 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:54 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:54 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:54 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:54 [scrapy.extensions.telnet] INFO: Telnet Password: bb072c4ee707aef3 | |
| 2021-12-19 10:29:54 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:54 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:54 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:54 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:54 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:54 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:54 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:54 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:54 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:54 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:55 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/iwonicz-zdroj> (referer: None) | |
| 2021-12-19 10:29:55 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:55 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 431, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11502, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.384942, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 55, 790131), | |
| 'httpcompression/response_bytes': 63759, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 54, 405189)} | |
| 2021-12-19 10:29:55 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:55 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:55 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:55 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:55 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:55 [scrapy.extensions.telnet] INFO: Telnet Password: 2e06dac33383ab4b | |
| 2021-12-19 10:29:55 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:55 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:55 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:55 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:55 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:55 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:55 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:55 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:56 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:56 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:57 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jasliska> (referer: None) | |
| 2021-12-19 10:29:57 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:57 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11489, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.478787, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 57, 426020), | |
| 'httpcompression/response_bytes': 62892, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 55, 947233)} | |
| 2021-12-19 10:29:57 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:57 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:57 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:57 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:57 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:57 [scrapy.extensions.telnet] INFO: Telnet Password: 4806d099217fb91a | |
| 2021-12-19 10:29:57 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:57 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:57 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:57 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:57 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:57 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:57 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:57 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:57 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:57 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:29:59 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jedlicze> (referer: None) | |
| 2021-12-19 10:29:59 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:29:59 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11685, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.895847, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 29, 59, 476072), | |
| 'httpcompression/response_bytes': 64087, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 57, 580225)} | |
| 2021-12-19 10:29:59 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:29:59 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:29:59 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:29:59 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:29:59 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:29:59 [scrapy.extensions.telnet] INFO: Telnet Password: 3d59ccd8ebd5e989 | |
| 2021-12-19 10:29:59 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:29:59 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:29:59 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:29:59 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:29:59 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:29:59 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:29:59 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:29:59 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:29:59 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:29:59 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:00 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/korczyna> (referer: None) | |
| 2021-12-19 10:30:01 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:01 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11451, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.384017, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 1, 19289), | |
| 'httpcompression/response_bytes': 62728, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 29, 59, 635272)} | |
| 2021-12-19 10:30:01 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:01 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:01 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:01 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:01 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:01 [scrapy.extensions.telnet] INFO: Telnet Password: 2c31c606166b49a6 | |
| 2021-12-19 10:30:01 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:01 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:01 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:01 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:01 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:01 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:01 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:01 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:01 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:01 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:02 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/kroscienko-wyzne> (referer: None) | |
| 2021-12-19 10:30:02 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:02 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11766, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.656145, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 2, 833250), | |
| 'httpcompression/response_bytes': 65644, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 1, 177105)} | |
| 2021-12-19 10:30:02 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:02 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:02 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:02 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:02 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:02 [scrapy.extensions.telnet] INFO: Telnet Password: 8a253f9e35601495 | |
| 2021-12-19 10:30:02 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:02 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:02 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:02 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:02 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:02 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:02 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:02 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:03 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:03 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:04 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/miejsce-piastowe> (referer: None) | |
| 2021-12-19 10:30:04 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:04 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11600, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.365484, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 4, 360684), | |
| 'httpcompression/response_bytes': 64478, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 2, 995200)} | |
| 2021-12-19 10:30:04 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:04 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:04 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:04 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:04 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:04 [scrapy.extensions.telnet] INFO: Telnet Password: 6ca6e5c1d7652a61 | |
| 2021-12-19 10:30:04 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:04 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:04 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:04 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:04 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:04 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:04 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:04 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:04 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:04 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/rymanow> (referer: None) | |
| 2021-12-19 10:30:05 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:05 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 425, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11656, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.199148, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 5, 722707), | |
| 'httpcompression/response_bytes': 64773, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 4, 523559)} | |
| 2021-12-19 10:30:05 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:05 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:05 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:05 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:05 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:05 [scrapy.extensions.telnet] INFO: Telnet Password: be8099b04164cd9e | |
| 2021-12-19 10:30:05 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:05 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:05 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:05 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:05 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:05 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:05 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:06 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:06 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:06 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/wojaszowka> (referer: None) | |
| 2021-12-19 10:30:06 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:06 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 428, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11459, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.658968, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 6, 541281), | |
| 'httpcompression/response_bytes': 63746, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 5, 882313)} | |
| 2021-12-19 10:30:06 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:06 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:06 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:06 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:06 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:06 [scrapy.extensions.telnet] INFO: Telnet Password: f8d3f0b35723d8c7 | |
| 2021-12-19 10:30:06 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:06 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:06 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:06 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:06 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:06 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:06 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:06 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:06 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:06 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:07 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/iwonicz-zdroj> (referer: None) | |
| 2021-12-19 10:30:07 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:07 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 431, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11513, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.888588, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 7, 574549), | |
| 'httpcompression/response_bytes': 63759, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 89063424, | |
| 'memusage/startup': 89063424, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 6, 685961)} | |
| 2021-12-19 10:30:07 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:07 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:07 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:07 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:07 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:07 [scrapy.extensions.telnet] INFO: Telnet Password: d2d2152a94f942c6 | |
| 2021-12-19 10:30:07 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:07 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:07 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:07 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:07 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:07 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:07 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:07 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:07 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/robots.txt> from <GET http://krosno24.pl/robots.txt> | |
| 2021-12-19 10:30:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:08 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/archiwum/> from <GET http://krosno24.pl/archiwum/> | |
| 2021-12-19 10:30:08 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/archiwum/> (referer: None) | |
| 2021-12-19 10:30:08 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:08 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 826, | |
| 'downloader/request_count': 4, | |
| 'downloader/request_method_count/GET': 4, | |
| 'downloader/response_bytes': 11435, | |
| 'downloader/response_count': 4, | |
| 'downloader/response_status_count/200': 2, | |
| 'downloader/response_status_count/301': 2, | |
| 'elapsed_time_seconds': 0.788089, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 8, 574679), | |
| 'httpcompression/response_bytes': 55846, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 4, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 2, | |
| 'scheduler/dequeued/memory': 2, | |
| 'scheduler/enqueued': 2, | |
| 'scheduler/enqueued/memory': 2, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 7, 786590)} | |
| 2021-12-19 10:30:08 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:08 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:08 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:08 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:08 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:08 [scrapy.extensions.telnet] INFO: Telnet Password: c4dee3ed27b7a1de | |
| 2021-12-19 10:30:08 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:08 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:08 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:08 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:08 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:08 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:08 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:08 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:09 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://krosno112.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:09 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:09 [protego] DEBUG: Rule at line 2 without any user agent to enforce it on. | |
| 2021-12-19 10:30:09 [protego] DEBUG: Rule at line 24 without any user agent to enforce it on. | |
| 2021-12-19 10:30:09 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno112.pl/aktualnosci.html> (referer: None) | |
| 2021-12-19 10:30:09 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:09 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 422, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 20314, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 1, | |
| 'downloader/response_status_count/404': 1, | |
| 'elapsed_time_seconds': 1.176409, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 9, 922551), | |
| 'httpcompression/response_bytes': 101106, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 5, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/404': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 8, 746142)} | |
| 2021-12-19 10:30:09 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:09 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:09 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:09 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:09 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:10 [scrapy.extensions.telnet] INFO: Telnet Password: 4eabcaddde659696 | |
| 2021-12-19 10:30:10 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:10 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:10 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:10 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:10 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:10 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:10 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:10 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:10 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:10 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:11 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/chorkowka> (referer: None) | |
| 2021-12-19 10:30:11 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:11 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 427, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11493, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.384451, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 11, 457872), | |
| 'httpcompression/response_bytes': 63414, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 10, 73421)} | |
| 2021-12-19 10:30:11 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:11 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:11 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:11 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:11 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:11 [scrapy.extensions.telnet] INFO: Telnet Password: 9a3277653f07679a | |
| 2021-12-19 10:30:11 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:11 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:11 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:11 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:11 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:11 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:11 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:11 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:11 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:11 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:12 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/krosno> (referer: None) | |
| 2021-12-19 10:30:12 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:12 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 424, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11276, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.372264, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 12, 996931), | |
| 'httpcompression/response_bytes': 61270, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 11, 624667)} | |
| 2021-12-19 10:30:12 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:13 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:13 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:13 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:13 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:13 [scrapy.extensions.telnet] INFO: Telnet Password: b7848952ab15964e | |
| 2021-12-19 10:30:13 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:13 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:13 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:13 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:13 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:13 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:13 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:13 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:13 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:14 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/dukla> (referer: None) | |
| 2021-12-19 10:30:14 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:14 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 423, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11618, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.29182, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 14, 435333), | |
| 'httpcompression/response_bytes': 63442, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 13, 143513)} | |
| 2021-12-19 10:30:14 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:14 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:14 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:14 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:14 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:14 [scrapy.extensions.telnet] INFO: Telnet Password: 879c252188e60808 | |
| 2021-12-19 10:30:14 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:14 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:14 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:14 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:14 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:14 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:14 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:14 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:14 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:14 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:15 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/iwonicz-zdroj> (referer: None) | |
| 2021-12-19 10:30:16 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:16 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 431, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11504, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.476012, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 16, 77822), | |
| 'httpcompression/response_bytes': 63759, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 14, 601810)} | |
| 2021-12-19 10:30:16 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:16 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:16 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:16 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:16 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:16 [scrapy.extensions.telnet] INFO: Telnet Password: f8ed95b9c16261ed | |
| 2021-12-19 10:30:16 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:16 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:16 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:16 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:16 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:16 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:16 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:16 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:16 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:16 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jasliska> (referer: None) | |
| 2021-12-19 10:30:17 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:17 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11492, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.475712, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 17, 702591), | |
| 'httpcompression/response_bytes': 62892, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 16, 226879)} | |
| 2021-12-19 10:30:17 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:17 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:17 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:17 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:17 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:17 [scrapy.extensions.telnet] INFO: Telnet Password: b42742756a0df5c3 | |
| 2021-12-19 10:30:17 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:17 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:17 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:17 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:17 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:17 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:17 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:17 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:18 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jedlicze> (referer: None) | |
| 2021-12-19 10:30:19 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:19 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11682, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.588963, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 19, 443927), | |
| 'httpcompression/response_bytes': 64087, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 17, 854964)} | |
| 2021-12-19 10:30:19 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:19 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:19 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:19 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:19 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:19 [scrapy.extensions.telnet] INFO: Telnet Password: 3fd5e574ae65f941 | |
| 2021-12-19 10:30:19 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:19 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:19 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:19 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:19 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:19 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:19 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:19 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:19 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/korczyna> (referer: None) | |
| 2021-12-19 10:30:21 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:21 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11455, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.584998, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 21, 185971), | |
| 'httpcompression/response_bytes': 62728, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 19, 600973)} | |
| 2021-12-19 10:30:21 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:21 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:21 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:21 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:21 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:21 [scrapy.extensions.telnet] INFO: Telnet Password: 2e21a2e8547b1dc6 | |
| 2021-12-19 10:30:21 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:21 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:21 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:21 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:21 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:21 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:21 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:21 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:21 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:21 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:22 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/kroscienko-wyzne> (referer: None) | |
| 2021-12-19 10:30:22 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:22 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11765, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.382084, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 22, 720879), | |
| 'httpcompression/response_bytes': 65644, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 21, 338795)} | |
| 2021-12-19 10:30:22 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:22 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:22 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:22 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:22 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:22 [scrapy.extensions.telnet] INFO: Telnet Password: 59f7387937a8a478 | |
| 2021-12-19 10:30:22 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:22 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:22 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:22 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:22 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:22 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:22 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:22 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:23 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:23 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/miejsce-piastowe> (referer: None) | |
| 2021-12-19 10:30:24 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:24 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 434, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11600, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.495834, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 24, 361167), | |
| 'httpcompression/response_bytes': 64478, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 22, 865333)} | |
| 2021-12-19 10:30:24 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:24 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:24 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:24 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:24 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:24 [scrapy.extensions.telnet] INFO: Telnet Password: b4dd3dae9ba8689a | |
| 2021-12-19 10:30:24 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:24 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:24 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:24 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:24 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:24 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:24 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:24 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:24 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:24 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:25 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/rymanow> (referer: None) | |
| 2021-12-19 10:30:25 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:25 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 425, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11656, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.393213, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 25, 900297), | |
| 'httpcompression/response_bytes': 64773, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 24, 507084)} | |
| 2021-12-19 10:30:25 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:25 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:25 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:25 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:25 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:25 [scrapy.extensions.telnet] INFO: Telnet Password: f345dc0b559ce8ee | |
| 2021-12-19 10:30:26 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:26 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:26 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:26 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:26 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:26 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:26 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:26 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:26 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:26 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/wojaszowka> (referer: None) | |
| 2021-12-19 10:30:27 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:27 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 428, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11457, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.278045, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 27, 328425), | |
| 'httpcompression/response_bytes': 63746, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 26, 50380)} | |
| 2021-12-19 10:30:27 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:27 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:27 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:27 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:27 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:27 [scrapy.extensions.telnet] INFO: Telnet Password: b76b67a97dca758e | |
| 2021-12-19 10:30:27 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:27 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:27 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:27 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:27 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:27 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:27 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:27 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:27 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:28 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/jasliska> (referer: None) | |
| 2021-12-19 10:30:28 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:28 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 426, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11491, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.095041, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 28, 569087), | |
| 'httpcompression/response_bytes': 62892, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 89063424, | |
| 'memusage/startup': 89063424, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 27, 474046)} | |
| 2021-12-19 10:30:28 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:28 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:28 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:28 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:28 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:28 [scrapy.extensions.telnet] INFO: Telnet Password: c9d3ed0de5bd54d8 | |
| 2021-12-19 10:30:28 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:28 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:28 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:28 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:28 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:28 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:28 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:28 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:28 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/robots.txt> from <GET http://krosno24.pl/robots.txt> | |
| 2021-12-19 10:30:29 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:29 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://krosno24.pl/archiwum/> from <GET http://krosno24.pl/archiwum/> | |
| 2021-12-19 10:30:29 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno24.pl/archiwum/> (referer: None) | |
| 2021-12-19 10:30:29 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:29 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 826, | |
| 'downloader/request_count': 4, | |
| 'downloader/request_method_count/GET': 4, | |
| 'downloader/response_bytes': 11457, | |
| 'downloader/response_count': 4, | |
| 'downloader/response_status_count/200': 2, | |
| 'downloader/response_status_count/301': 2, | |
| 'elapsed_time_seconds': 0.737834, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 29, 521050), | |
| 'httpcompression/response_bytes': 55953, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 4, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 2, | |
| 'scheduler/dequeued/memory': 2, | |
| 'scheduler/enqueued': 2, | |
| 'scheduler/enqueued/memory': 2, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 28, 783216)} | |
| 2021-12-19 10:30:29 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:29 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:29 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:29 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:29 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:29 [scrapy.extensions.telnet] INFO: Telnet Password: 5fa25909beb6ce3c | |
| 2021-12-19 10:30:29 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:29 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:29 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:29 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:29 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:29 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:29 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:29 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:30 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://krosno112.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:30 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:30 [protego] DEBUG: Rule at line 2 without any user agent to enforce it on. | |
| 2021-12-19 10:30:30 [protego] DEBUG: Rule at line 24 without any user agent to enforce it on. | |
| 2021-12-19 10:30:30 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://krosno112.pl/aktualnosci.html> (referer: None) | |
| 2021-12-19 10:30:30 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:30 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 422, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 20349, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 1, | |
| 'downloader/response_status_count/404': 1, | |
| 'elapsed_time_seconds': 1.245182, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 30, 930410), | |
| 'httpcompression/response_bytes': 101175, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 5, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/404': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 29, 685228)} | |
| 2021-12-19 10:30:30 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:30 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:30 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:30 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:30 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:31 [scrapy.extensions.telnet] INFO: Telnet Password: 3cd41de4615c0721 | |
| 2021-12-19 10:30:31 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:31 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:31 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:31 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:31 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:31 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:31 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:31 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:31 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:31 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:32 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/chorkowka> (referer: None) | |
| 2021-12-19 10:30:32 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:32 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 427, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11486, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 1.065981, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 32, 143302), | |
| 'httpcompression/response_bytes': 63414, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 31, 77321)} | |
| 2021-12-19 10:30:32 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:32 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:32 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:32 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:32 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:32 [scrapy.extensions.telnet] INFO: Telnet Password: 4d9fbc3f4d23ef3d | |
| 2021-12-19 10:30:32 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:32 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:32 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:32 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:32 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:32 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:32 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:32 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:32 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:32 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:32 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/krosno> (referer: None) | |
| 2021-12-19 10:30:33 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:33 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 424, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11260, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.778696, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 33, 66576), | |
| 'httpcompression/response_bytes': 61270, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 32, 287880)} | |
| 2021-12-19 10:30:33 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:33 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:33 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:33 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:33 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:33 [scrapy.extensions.telnet] INFO: Telnet Password: 45b9d8de49d648ae | |
| 2021-12-19 10:30:33 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:33 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:33 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:33 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:33 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:33 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:33 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:33 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:33 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:33 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| 2021-12-19 10:30:33 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/dukla> (referer: None) | |
| 2021-12-19 10:30:33 [scrapy.core.engine] INFO: Closing spider (finished) | |
| 2021-12-19 10:30:33 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 423, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11619, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.763381, | |
| 'finish_reason': 'finished', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 33, 985910), | |
| 'httpcompression/response_bytes': 63442, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 11, | |
| 'memusage/max': 82173952, | |
| 'memusage/startup': 82173952, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 33, 222529)} | |
| 2021-12-19 10:30:33 [scrapy.core.engine] INFO: Spider closed (finished) | |
| 2021-12-19 10:30:34 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: Scrapers) | |
| 2021-12-19 10:30:34 [scrapy.utils.log] INFO: Versions: lxml 4.6.3.0, libxml2 2.9.10, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.10 (default, Feb 16 2021, 02:11:57) - [GCC 10.3.0], pyOpenSSL 20.0.1 (OpenSSL 1.1.1k 25 Mar 2021), cryptography 3.4.7, Platform Linux-5.15.7-x86_64-with | |
| 2021-12-19 10:30:34 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor | |
| 2021-12-19 10:30:34 [scrapy.crawler] INFO: Overridden settings: | |
| {'BOT_NAME': 'Scrapers', | |
| 'COOKIES_ENABLED': False, | |
| 'DOWNLOAD_DELAY': 0.1, | |
| 'FEED_EXPORT_ENCODING': 'utf-8', | |
| 'NEWSPIDER_MODULE': 'Scrapers.spiders', | |
| 'ROBOTSTXT_OBEY': True, | |
| 'SPIDER_MODULES': ['Scrapers.spiders']} | |
| 2021-12-19 10:30:34 [scrapy.extensions.telnet] INFO: Telnet Password: 914728e44f2ec4e1 | |
| 2021-12-19 10:30:34 [scrapy.middleware] INFO: Enabled extensions: | |
| ['scrapy.extensions.corestats.CoreStats', | |
| 'scrapy.extensions.telnet.TelnetConsole', | |
| 'scrapy.extensions.memusage.MemoryUsage', | |
| 'scrapy.extensions.logstats.LogStats'] | |
| 2021-12-19 10:30:34 [scrapy.middleware] INFO: Enabled downloader middlewares: | |
| ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', | |
| 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', | |
| 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', | |
| 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', | |
| 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', | |
| 'Scrapers.middlewares.ScrapersDownloaderMiddleware', | |
| 'scrapy.downloadermiddlewares.retry.RetryMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', | |
| 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', | |
| 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', | |
| 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', | |
| 'scrapy.downloadermiddlewares.stats.DownloaderStats'] | |
| 2021-12-19 10:30:34 [scrapy.middleware] INFO: Enabled spider middlewares: | |
| ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', | |
| 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', | |
| 'scrapy.spidermiddlewares.referer.RefererMiddleware', | |
| 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', | |
| 'scrapy.spidermiddlewares.depth.DepthMiddleware'] | |
| 2021-12-19 10:30:34 [scrapy.middleware] INFO: Enabled item pipelines: | |
| [] | |
| 2021-12-19 10:30:34 [scrapy.core.engine] INFO: Spider opened | |
| 2021-12-19 10:30:34 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) | |
| 2021-12-19 10:30:34 [Uniwersalscraper] INFO: Spider opened: Uniwersalscraper | |
| 2021-12-19 10:30:34 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 | |
| 2021-12-19 10:30:34 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/robots.txt> (referer: None) | |
| 2021-12-19 10:30:34 [protego] DEBUG: Rule at line 1 without any user agent to enforce it on. | |
| ^CTraceback (most recent call last): | |
| File "Uniwersal.py", line 129, in <module> | |
| 2021-12-19 10:30:34 [scrapy.crawler] INFO: Received SIGINT, shutting down gracefully. Send again to force | |
| run_spider('Uniwersalscraper', Strings_and_XPaths) | |
| File "Uniwersal.py", line 75, in run_spider | |
| result = q.get() | |
| File "/nix/store/kk28zrhy9dzcv98r9h6m8zziqb1pkk61-python3-3.7.10/lib/python3.7/multiprocessing/queues.py", line 94, in get | |
| res = self._recv_bytes() | |
| File "/nix/store/kk28zrhy9dzcv98r9h6m8zziqb1pkk61-python3-3.7.10/lib/python3.7/multiprocessing/connection.py", line 216, in recv_bytes | |
| 2021-12-19 10:30:34 [scrapy.core.engine] INFO: Closing spider (shutdown) | |
| buf = self._recv_bytes(maxlength) | |
| File "/nix/store/kk28zrhy9dzcv98r9h6m8zziqb1pkk61-python3-3.7.10/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes | |
| buf = self._recv(4) | |
| File "/nix/store/kk28zrhy9dzcv98r9h6m8zziqb1pkk61-python3-3.7.10/lib/python3.7/multiprocessing/connection.py", line 379, in _recv | |
| chunk = read(handle, remaining) | |
| KeyboardInterrupt | |
| 2021-12-19 10:30:34 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.terazkrosno.pl/iwonicz-zdroj> (referer: None) | |
| 2021-12-19 10:30:35 [scrapy.statscollectors] INFO: Dumping Scrapy stats: | |
| {'downloader/request_bytes': 431, | |
| 'downloader/request_count': 2, | |
| 'downloader/request_method_count/GET': 2, | |
| 'downloader/response_bytes': 11505, | |
| 'downloader/response_count': 2, | |
| 'downloader/response_status_count/200': 2, | |
| 'elapsed_time_seconds': 0.875292, | |
| 'finish_reason': 'shutdown', | |
| 'finish_time': datetime.datetime(2021, 12, 19, 9, 30, 35, 9449), | |
| 'httpcompression/response_bytes': 63759, | |
| 'httpcompression/response_count': 2, | |
| 'log_count/DEBUG': 3, | |
| 'log_count/INFO': 12, | |
| 'memusage/max': 82157568, | |
| 'memusage/startup': 82157568, | |
| 'response_received_count': 2, | |
| 'robotstxt/request_count': 1, | |
| 'robotstxt/response_count': 1, | |
| 'robotstxt/response_status_count/200': 1, | |
| 'scheduler/dequeued': 1, | |
| 'scheduler/dequeued/memory': 1, | |
| 'scheduler/enqueued': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'scheduler/enqueued/memory': 1, | |
| 'start_time': datetime.datetime(2021, 12, 19, 9, 30, 34, 134157)} | |
| 2021-12-19 10:30:35 [scrapy.core.engine] INFO: Spider closed (shutdown) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment