vdp_crawler.py

import requests
import os
import shutil
import pendulum


base_url = "https://vidadeprogramador.com.br/uploads/"
comic_id = 2074
exit = False
year = "2020"

dates = {}

while int(year) >= 2010:
    control_month = 13
    while control_month > 1:
        control_month -= 1
        dt = pendulum.parse(f"{year}-{str(control_month).zfill(2)}-01 00:00:00")
        if pendulum.now() <= dt:
            continue

        year = dt.year
        month = str(dt.month).zfill(2)
        if year not in dates.keys():
            dates[year] = {}

        if month not in dates[year].keys():
            dates[year][month] = []

        path = f"{year}/{month}"

        not_found = False
        while not_found is False:
            filepath = f"{path}/tirinha{comic_id}.png"
            url = f"{base_url}{filepath}"
            if os.path.isfile(f"./images/{filepath}"):
                comic_id -= 1
                continue

            resp = requests.get(url, stream=True)
            if resp.status_code == 200:
                comic_id -= 1
                if not os.path.isdir(f"./images/{path}"):
                    os.system(f"mkdir -p ./images/{path}")
                print(f"Salvando em ./images/{filepath}")
                with open(f"./images/{filepath}", "wb") as f:
                    resp.raw.decode_content = True
                    shutil.copyfileobj(resp.raw, f)
                    continue
            not_found = resp.status_code == 404
    year -= 1