Skip to content

Instantly share code, notes, and snippets.

Created July 10, 2018 12:30
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save fqx/c7f36a9d7f7a79ce936b62ebc351cb12 to your computer and use it in GitHub Desktop.
Save fqx/c7f36a9d7f7a79ce936b62ebc351cb12 to your computer and use it in GitHub Desktop.
modified for Google Colab
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import shutil
import tqdm
import requests
import time
from functools import wraps
import traceback
tqdm.monitor_interval = 0 # workaround for
def retry(ExceptionToCheck, tries=4, delay=3, backoff=2):
def deco_retry(f):
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
while mtries > 1:
return f(*args, **kwargs)
except KeyboardInterrupt as e:
raise e
except ExceptionToCheck as e:
print("%s, retrying in %d seconds..." % (str(e), mdelay))
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
return f_retry # true decorator
return deco_retry
def download_file(url, file_path):
r = requests.get(url, stream=True)
total_size = int(r.headers.get('content-length'))
bar = tqdm.tqdm_notebook(total=total_size, unit='B', unit_scale=True)
incomplete_download = False
with open(file_path, 'wb', buffering=16 * 1024 * 1024) as f:
for chunk in r.iter_content(1 * 1024 * 1024):
except Exception as e:
raise e
if os.path.exists(file_path) and os.path.getsize(file_path) != total_size:
incomplete_download = True
if incomplete_download:
raise Exception("Incomplete download")
def download_from_github(version, fn, target_dir):
url = "{0}/{1}".format(version, fn)
file_path = os.path.join(target_dir, fn)
download_file(url, file_path)
def sequential_downloader(version, fns, target_dir):
os.makedirs(target_dir, exist_ok=True)
for fn in fns:
download_from_github(version, fn, target_dir)
def link_all_files_from_dir(src_dir, dst_dir):
os.makedirs(dst_dir, exist_ok=True)
if not os.path.exists(src_dir):
# Coursera "readonly/readonly" bug workaround
src_dir = src_dir.replace("readonly", "readonly/readonly")
for fn in os.listdir(src_dir):
src_file = os.path.join(src_dir, fn)
dst_file = os.path.join(dst_dir, fn)
if == "nt":
shutil.copyfile(src_file, dst_file)
if os.path.islink(dst_file):
os.symlink(os.path.abspath(src_file), dst_file)
def link_all_keras_resources():
link_all_files_from_dir("readonly/keras/datasets/", os.path.expanduser("~/.keras/datasets"))
link_all_files_from_dir("readonly/keras/models/", os.path.expanduser("~/.keras/models"))
def link_week_3_resources():
link_all_files_from_dir("readonly/week3/", ".")
def link_week_4_resources():
link_all_files_from_dir("readonly/week4/", ".")
def link_week_6_resources():
link_all_files_from_dir("readonly/week6/", ".")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment