Created
August 6, 2019 18:12
-
-
Save mamedshahmaliyev/bca9242b7ea6a13b3f76dee7a5aa111a to your computer and use it in GitHub Desktop.
Download forex historical data monthly in csv format from http://www.truefx.com (pepperstone) using python and selenium
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import datetime, time, os | |
from dateutil.relativedelta import relativedelta | |
from selenium import webdriver | |
tmp_dir = os.path.join(os.getcwd(), 'forex') | |
if not os.path.isdir(tmp_dir): os.makedirs(tmp_dir) | |
options = webdriver.ChromeOptions(); | |
options.add_argument("--window-size=1300,900") | |
options.add_experimental_option("prefs", { | |
"download.default_directory": tmp_dir, | |
"download.prompt_for_download": False, | |
"download.directory_upgrade": True, | |
"safebrowsing.enabled": False, | |
"safebrowsing.disable_download_protection": True | |
}) | |
options.add_argument("--disable-gpu") | |
options.add_argument("--disable-extensions") | |
options.add_argument('--disable-logging') | |
options.add_argument('--ignore-certificate-errors') | |
options.add_argument('--ignore-certificate-errors-spki-list') | |
options.add_argument('--no-sandbox') | |
browser = webdriver.Chrome(options=options) | |
pairs = ['AUDJPY', 'AUDNZD', 'AUDUSD', 'CADJPY', 'CHFJPY', 'EURCHF', 'EURGBP', 'EURJPY', 'EURUSD', 'GBPJPY', 'GBPUSD', 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY'] | |
for pair in pairs: | |
curr_date = datetime.datetime(2015, 1, 1) | |
while curr_date + relativedelta(months=1) < datetime.datetime.now(): | |
file_name = '{p}-{ym2}.zip'.format(p=pair, ym2=curr_date.strftime('%Y-%m')) | |
url = 'http://www.truefx.com/dev/data/{y}/{ym1}/{p}-{ym2}.zip'.format( | |
y=curr_date.strftime('%Y'), | |
ym1=curr_date.strftime('%B').upper()+'-'+curr_date.strftime('%Y') if curr_date <= datetime.datetime(2017, 3, 1) else curr_date.strftime('%Y-%m'), | |
p=pair, | |
ym2=curr_date.strftime('%Y-%m') | |
) | |
file_found = False | |
for root, dirs, files in os.walk(tmp_dir): | |
for file in files: | |
if file_name in file: file_found = True | |
if not file_found: | |
time.sleep(5) | |
browser.get (url) | |
file_downloaded = False | |
while not file_downloaded: | |
time.sleep(1) | |
for root, dirs, files in os.walk(tmp_dir): | |
for file in files: | |
if file_name in file and not '.crdownload' in file: file_downloaded = True | |
print(file_name, 'downloaded from', url) | |
curr_date = curr_date + relativedelta(months=1) | |
print('completed') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Great work but it seems it requires a login now