collect.py

#!/usr/bin/env python

import requests
import hashlib
import json

operations_log = {}

headers = {
    "Cache-Control": "no-cache",
    "Content-Type": "application/json",
    "User-agent": "Mozilla/5.0 (compatible; promisebot/1.0 +https://www.kbr.be/en/promise-project)"
}

with open('./list_urls.txt', 'r') as input:
    for url in input:
        url = url.strip('\n')
        try:
            r = requests.post('http://localhost:3000/screenshot', headers=headers, json={
                "url": url,
                "viewport": {
                    "width": '1920',
                    "height": '1080',
                    "isMobile": False
                },
                "options": {
                    "fullPage": True,
                    "type": "jpeg",
                    "quality": 100
                }
            })

            output_filename = hashlib.sha1(bytes(url, encoding='utf-8')).hexdigest() + '.jpeg'

            operations_log[url] = {
                'output_filename': output_filename,
                'status_code': r.status_code
            }

            with open(output_filename, 'wb') as f:
                f.write(r.content)
        except:
            raise

with open('originals_collect.json', 'w') as f:
    json.dump(operations_log, f)