-
-
Save phineas-pta/d73f9a035b05f8e923af8c01df057175 to your computer and use it in GitHub Desktop.
Python requests download file with a tqdm progress bar
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import requests | |
import werkzeug | |
from tqdm import tqdm | |
def download_ckpt(url:str, filename:str="", overwrite:bool=False, chunk_size:int=1) -> None: | |
""" | |
try auto detect file name if left empty | |
option to overwrite if file already existed | |
chunk_size in MBytes | |
""" | |
HEADERS = {"User-Agent": "my gist code"} # some sites block if u dont have at least user agent | |
with requests.get(url, headers=HEADERS, stream=True) as resp: | |
# get file name | |
if filename == "": | |
MISSING_FILENAME = "missing_name" | |
if content_disposition := resp.headers.get("Content-Disposition"): | |
param, options = werkzeug.http.parse_options_header(content_disposition) | |
if param == "attachment": | |
filename = options.get("filename", MISSING_FILENAME) | |
else: | |
filename = MISSING_FILENAME | |
else: | |
filename = os.path.basename(url) | |
fileext = os.path.splitext(filename)[-1] | |
if fileext == "": | |
filename = MISSING_FILENAME | |
# download file | |
if overwrite or not os.path.exists(filename): | |
TOTAL_SIZE = int(resp.headers.get("Content-Length", 0)) | |
CHUNK_SIZE = chunk_size * 10**6 | |
with ( | |
open(filename, mode="wb") as file, | |
tqdm(total=TOTAL_SIZE, desc=f"download {filename}", unit="B", unit_scale=True) as bar | |
): | |
for data in resp.iter_content(chunk_size=CHUNK_SIZE): | |
size = file.write(data) | |
bar.update(size) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment