Skip to content

Instantly share code, notes, and snippets.

@satoshihirose
Created December 11, 2016 08:34
Show Gist options
  • Save satoshihirose/226405c02bdf2febce0ccd7551d0c0ca to your computer and use it in GitHub Desktop.
Save satoshihirose/226405c02bdf2febce0ccd7551d0c0ca to your computer and use it in GitHub Desktop.
import urllib.request, urllib.parse, urllib.error
from datetime import datetime as dt
import csv
import time
import os.path
date = dt.now()
datestr = date.strftime('%Y%m%d')
print("downloading stock list")
stock_list_url = 'http://k-db.com/stocks/?download=csv'
stock_list_file = "data/stock_list_%s.csv" % datestr
urllib.request.urlretrieve(stock_list_url, stock_list_file)
print("downloading stocks")
with open(stock_list_file, 'r', encoding = "shift-jis") as input:
stock_list = csv.reader(input, delimiter=',')
next(stock_list, None)
for stock in stock_list:
code = stock[0]
url = 'http://k-db.com/stocks/%s?download=csv' % code
path = "data/%s_%s.csv" % (code, datestr)
if os.path.isfile(path):
print(path, 'is already exist.')
else:
print('download from ', url, ' to ', path)
while True:
try:
time.sleep(5)
urllib.request.urlretrieve(url, path)
except urllib.error.HTTPError:
continue
break
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment