saeed.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import Selector
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import datetime


class StackDataSpider(scrapy.Spider):
    name = 'stack_data'

    responses = []

    def start_requests(self):

        yield SeleniumRequest(
            url='https://seffaflik.epias.com.tr/transparency/piyasalar/gop/arz-talep.xhtml',
            wait_time=3,
            screenshot=True,
            callback=self.parse
        )

    def parse(self, response):
        driver = response.meta['driver']
        start_date = datetime.datetime(day=18, month=4, year=2020)
        list_dates = list_dates = [
            start_date + datetime.timedelta(days=i) for i in range(0, 2)]

        # Here you should fill in the input and then click the submit button
        self.responses.append(driver.page_source)

        # Handling pagination
        exists = True
        while exists:
            next_page = driver.find_element_by_xpath(
                '//a[@aria-label="Next Page" and @tabindex="0"]')
            if next_page:
                driver.execute_script(
                    'document.querySelector(".ui-paginator-next").click()')
                try:
                    WebDriverWait(driver, 10).until(
                        EC.element_to_be_clickable(
                            (By.CLASS_NAME, "ui-paginator-next"))
                    )
                except Exception:
                    pass

                self.responses.append(driver.page_source)
            else:
                exists = False

        print(self.responses)

        #loop through all responses and parse them
        for resp in self.responses:
            r = Selector(text=resp)