A script that downloads multiple files in parallel with support for s3:// | http:// | https:// protocols
Usage: python downloader.py url1 url2 url3
import time | |
from concurrent.futures import ThreadPoolExecutor | |
from functools import wraps | |
import requests | |
from tqdm import tqdm | |
def timeit(method): | |
@wraps(method) |
#!/usr/bin/env python | |
__author__ = "github.com/ruxi" | |
__license__ = "MIT" | |
import requests | |
import tqdm # progress bar | |
import os.path | |
def download_file(url, filename=False, verbose = False): | |
""" | |
Download file with progressbar | |
// based on https://gist.github.com/szhernovoy/276e69eb90a0de84dd90 | |
// and my performance measurements | |
public func randomString(of length: Int) -> String { | |
let letters = Array("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") | |
var s = "" | |
for _ in 0 ..< length { | |
s.append(letters.randomElement()!) | |
} | |
return s | |
} |
// based on gist https://gist.github.com/kristopherjohnson/4201fbe86473f6edb207 | |
// and discussion | |
// https://forums.swift.org/t/recommended-way-to-measure-time-in-swift/33326/4 | |
// https://github.com/apple/swift-corelibs-xctest/pull/109 | |
// Ensures Monotonic time | |
public func executionTimeInterval(block: () -> ()) -> TimeInterval { | |
let start = Foundation.ProcessInfo.processInfo.systemUptime | |
block(); | |
let end = Foundation.ProcessInfo.processInfo.systemUptime |