scrapy_infinite_scroll.py

import scrapy
from scrapy.http import FormRequest


class Spider1(scrapy.spiders.Spider):
    name = 'scroll5'
    api_url = 'https://tournaments.hjgt.org/tournament/TournamentResultSearch'
    start_urls = ['https://tournaments.hjgt.org/Tournament/Results/']
    
    def parse(self, response):
        token = response.xpath('//*[@name="__RequestVerificationToken"]/@value').extract_first()
        
        params = {
            '__RequestVerificationToken': token,
            'PageIndex': '1',
            'PageSize': '10',
            'SearchForm.UpcomingPast': '',
            'SearchForm.SearchString': '',
            'SearchForm.StartDate': '',
            'SearchForm.Distance': '',
            'SearchForm.ZipCode': '',
            'SearchForm.SeasonSelected': '',
        }

        yield FormRequest(self.api_url, method="POST", formdata = params, callback=self.finished)

    def finished(self, response):
        print('\n\n headers:', response.text)