Skip to content

Instantly share code, notes, and snippets.

@phineas-pta
Forked from yanqd0/dl_requests_tqdm.py
Last active April 18, 2024 16:10
Show Gist options
  • Save phineas-pta/d73f9a035b05f8e923af8c01df057175 to your computer and use it in GitHub Desktop.
Save phineas-pta/d73f9a035b05f8e923af8c01df057175 to your computer and use it in GitHub Desktop.
Python requests download file with a tqdm progress bar
import os
import requests
import werkzeug
from tqdm import tqdm
def download_ckpt(url:str, filename:str="", overwrite:bool=False, chunk_size:int=1) -> None:
"""
try auto detect file name if left empty
option to overwrite if file already existed
chunk_size in MBytes
"""
HEADERS = {"User-Agent": "my gist code"} # some sites block if u dont have at least user agent
with requests.get(url, headers=HEADERS, stream=True) as resp:
# get file name
if filename == "":
MISSING_FILENAME = "missing_name"
if content_disposition := resp.headers.get("Content-Disposition"):
param, options = werkzeug.http.parse_options_header(content_disposition)
if param == "attachment":
filename = options.get("filename", MISSING_FILENAME)
else:
filename = MISSING_FILENAME
else:
filename = os.path.basename(url)
fileext = os.path.splitext(filename)[-1]
if fileext == "":
filename = MISSING_FILENAME
# download file
if overwrite or not os.path.exists(filename):
TOTAL_SIZE = int(resp.headers.get("Content-Length", 0))
CHUNK_SIZE = chunk_size * 10**6
with (
open(filename, mode="wb") as file,
tqdm(total=TOTAL_SIZE, desc=f"download {filename}", unit="B", unit_scale=True) as bar
):
for data in resp.iter_content(chunk_size=CHUNK_SIZE):
size = file.write(data)
bar.update(size)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment