Fetch Data from Metryki

metryki_scrape.py

import requests
import time


list_of_last_names = ['Nowak','Smith']

def get_all_hits(last_name):
  url = f"https://geneszukacz.genealodzy.pl/index.php?search_lastname={last_name}&from_date=&to_date=&rpp1=&bdm=&url1=&w=&lang=pol&op=se"

  payload={}
  headers = {
    'authority': 'geneszukacz.genealodzy.pl',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
    'accept-language': 'en;q=0.8',
    'cache-control': 'no-cache',
    'pragma': 'no-cache',
    'referer': 'https://geneszukacz.genealodzy.pl/index.php?search_lastname=Kraszewski&from_date=&to_date=&rpp1=&bdm=&url1=&w=&lang=pol&op=se',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-user': '?1',
    'sec-gpc': '1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36'
  }

  response = requests.request("GET", url, headers=headers, data=payload)
  if response.status_code == 200:
    return response.content


DESTINATION = '/content/people'
for name in all_names:
  time.sleep(4)
  print(f'Fetching data for: {name}')
  result = get_all_hits(name)
  if result:
    print(f'{name} captured! Size: {len(result)}')
    # save content to HTML file
    with open(f'{DESTINATION}/{name.lower()}.html','wb') as f:
      f.write(result)
  else:
    raise ValueError(f'No data collected for: {name}')
    
#download results as zip
!zip -r /content/people.zip /content/people/