Skip to content

Instantly share code, notes, and snippets.

@flying-sheep
Created August 10, 2017 11:44
Show Gist options
  • Save flying-sheep/13035b29c9f8a0de26371aa7db4766d0 to your computer and use it in GitHub Desktop.
Save flying-sheep/13035b29c9f8a0de26371aa7db4766d0 to your computer and use it in GitHub Desktop.
Download and cache file
#!/usr/bin/env python3
import sys
from pathlib import Path
from urllib.request import urlretrieve
import requests
from appdirs import user_cache_dir
from tqdm import tqdm
cache_dir = Path(user_cache_dir('programm-name', 'dein-name'))
filename = 'bigfile.bin'
url = 'http://example.com/' + filename
file_path = cache_dir / filename
if not file_path.is_file():
print('downloading', repr(url), 'to cache', repr(file_path), file=sys.stderr)
r = requests.get(url, stream=True)
total_size = int(r.headers.get('content-length', 0))
with file_path.open('wb') as f:
progressbar = tqdm(r.iter_content(32*1024), total=total_size, unit='B', unit_scale=True)
for data in progressbar:
f.write(data)
with file_path.open('r'):
...
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment