Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
#requirements: requests, pillow, imthread
#usage: python -i urls.txt -o images --threads 1000
import requests
from requests import exceptions
from PIL import Image
import os
import time
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--input", required=True,
help="text file containing URLs")
ap.add_argument("-o", "--output", required=True,
help="image download directory")
ap.add_argument("-ht", "--height", required=False,
default=600, help=">height constraint")
ap.add_argument("-wd", "--width", required=False,
default=600, help=">width constraint")
ap.add_argument("-s", "--status", required=False,
default="True", help="show live download status")
ap.add_argument("-t", "--threads", required=False,
default=1, help="number of downloader threads")
args = vars(ap.parse_args())
input_file = args["input"]
output_dir = args["output"]
max_threads = int(args["threads"])
height = int(args["height"])
width = int(args["width"])
show_status = True if args["status"] in ["yes", "true", "True", "1", "t", "y", "a", "+"] else False
assert max_threads>0, "ValueError: --threads {}".format(max_threads)
assert os.path.exists(input_file), "no such file {}".format(input_file)
if not os.path.exists(output_dir):
URLS = [line.rstrip("\n") for line in open(input_file)]
EXCEPTIONS = set([IOError, FileNotFoundError,
exceptions.RequestException, exceptions.HTTPError,
exceptions.ConnectionError, exceptions.Timeout])
def download_images(*args):
idx = int(args[0]["counter"] if type(args[0]) == dict else args[0])
link = URLS[idx]
r = requests.get(link, stream=True, allow_redirects=True, timeout=30)
r.raw.decode_content = True
file_name = os.path.join(output_dir, str(idx)+".jpg")
with as img:
if img.mode in ("RGBA", "P"):
img = img.convert("RGB")
w, h = img.size
if h >= height or w >= width:
if show_status: print("downloaded: {}".format(file_name))
except Exception as e:
if not type(e) in EXCEPTIONS:
print("downloading images")
if max_threads == 1:
st = time.time()
for i,link in enumerate(URLS):
print("elapsed: {} sec".format(round((time.time()-st),3)))
import imthread
st = time.time()
imthread.start(download_images, repeat=len(URLS), max_threads=max_threads)
print("elapsed: {} min".format(round(((time.time()-st)/60),3)))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment