Skip to content

Instantly share code, notes, and snippets.

@tgsmith61591 tgsmith61591/download.py

Last active Jun 4, 2018
Embed
What would you like to do?
Download a file from the web using requests and a pretty progress bar
# -*- coding: utf-8 -*-
#
# Download a file from the web using requests with a nice progress bar.
from __future__ import print_function
from tqdm import tqdm
import requests
import warnings
import argparse
import math
def download(url, target, headers, block_size):
# Stream so we can iterate the response.
r = requests.get(url, stream=True, headers=headers)
# Total size in bytes.
total_size = int(r.headers.get('content-length', 0))
bytes_written = 0
with open(target, 'wb') as f:
for data in tqdm(r.iter_content(block_size),
total=math.ceil(total_size // block_size),
unit='KB', unit_scale=True):
bytes_written += len(data)
f.write(data)
if total_size != 0 and bytes_written != total_size:
warnings.warn("Only read %i/%i bytes!" % (bytes_written, total_size))
elif total_size == 0:
warnings.warn("Downloaded 0 bytes!")
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Download a file over HTTP "
"with a nice progress bar.")
parser.add_argument("--url", dest="url", type=str,
help="The URL from which to download.")
parser.add_argument("--output", dest="output", type=str,
help="The absolute path to the file to which to save "
"the web content.")
parser.add_argument("--headers", dest="headers", type=str, default=None,
help="Optional headers.")
parser.add_argument("--block-size", dest="block_size", type=int,
default=1024, help="The (optional) block size.")
parser.set_defaults(url=None, output=None, headers=None, block_size=1024)
args = parser.parse_args()
# validate args
if not all(arg for arg in (args.url, args.output)):
raise ValueError("URL and output required!")
download(url=args.url, target=args.output, headers=args.headers,
block_size=args.block_size)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.