semetary_crawler.py

import requests
from bs4 import BeautifulSoup
from urllib.parse import quote
import datetime
from pathlib import Path

# make sure this directory exists
dl_root = '/tmp/cenaze/'

target_url = 'https://www.turkiye.gov.tr/inegol-belediyesi-vefat-sorgulama'
start_date = '01/12/2019'
end_date = '07/04/2020'

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Accept-Encoding': 'gzip, deflate, br',
    'DNT': '1',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'Pragma': 'no-cache',
    'Cache-Control': 'no-cache'
}

s = requests.Session()

first_res = s.get(target_url, headers=headers)
first_page = BeautifulSoup(first_res.text, 'lxml')

token = first_page.find('input', attrs={'name': 'token'})['value']

start = datetime.datetime.strptime(start_date, "%d/%m/%Y")
end = datetime.datetime.strptime(end_date, "%d/%m/%Y")
date_generated = [start + datetime.timedelta(days=x) for x in range(0, (end - start).days)]

for d in date_generated:
    tarih = d.strftime('%d/%m/%Y')
    nicedate = d.strftime('%Y%m%d')
    file_path = Path(dl_root + nicedate + '.html')

    if not file_path.exists():
        postData = {
            'tarih': tarih,
            'token': token,
            'btn': 'Sorgula'
        }

        resPost = s.post(target_url + '?submit', data=postData, headers=headers)

        resFinal = s.get(target_url + '?asama=1', headers=headers)
        fpage = BeautifulSoup(resFinal.text, 'lxml')

        tbl = fpage.find('table', id='table', class_='resultTable')
        if tbl:
            print('%s: %s adet cenaze' % (nicedate, len(tbl.find_all('tr'))))
            with file_path.open('w') as f:
                f.write(tbl.prettify())