Proxy For Scrapy

get_proxy.py

import scrapy
import datetime
import scraper.items as ProxyItems
from scraper.settings import PROXY_DATA
import logging


class ProxySpider(scrapy.Spider):
    name = 'proxy'
    allowed_domains = ['sslproxies.org']
    start_urls = ['http://sslproxies.org/']
    custom_settings = {
        'FEED_URI': PROXY_DATA,
        'FEED_FORMAT': 'csv',
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy_splash.SplashCookiesMiddleware': 723,
            'scrapy_splash.SplashMiddleware': 725,
            'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
        },
        'ITEM_PIPELINES':{
            'DeHaze.pipelines.ProxyPipeline': 100
        }
    }

    def parse(self, response):
        logging.log(logging.INFO, "Scraping Page {}".format(response.request.url))
        ips = response.css('#proxylisttable td:nth-child(1)::text').extract()
        ports = response.css('#proxylisttable td:nth-child(2)::text').extract()
        time = datetime.datetime.utcnow().timestamp()
        total_ips = len(ips)
        logging.log(logging.INFO, "Found {} Total IP".format(total_ips))
        for i in range(total_ips):
            item = ProxyItems.ProxyItem(ip=ips[i], port=ports[i], timestamp=time)
            logging.log(logging.INFO, "Scraped Following Proxy {}".format(itm))
            yield item