Skip to content

Instantly share code, notes, and snippets.

@mamedshahmaliyev
Created August 6, 2019 18:12
Show Gist options
  • Save mamedshahmaliyev/bca9242b7ea6a13b3f76dee7a5aa111a to your computer and use it in GitHub Desktop.
Save mamedshahmaliyev/bca9242b7ea6a13b3f76dee7a5aa111a to your computer and use it in GitHub Desktop.
Download forex historical data monthly in csv format from http://www.truefx.com (pepperstone) using python and selenium
import datetime, time, os
from dateutil.relativedelta import relativedelta
from selenium import webdriver
tmp_dir = os.path.join(os.getcwd(), 'forex')
if not os.path.isdir(tmp_dir): os.makedirs(tmp_dir)
options = webdriver.ChromeOptions();
options.add_argument("--window-size=1300,900")
options.add_experimental_option("prefs", {
"download.default_directory": tmp_dir,
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": False,
"safebrowsing.disable_download_protection": True
})
options.add_argument("--disable-gpu")
options.add_argument("--disable-extensions")
options.add_argument('--disable-logging')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--ignore-certificate-errors-spki-list')
options.add_argument('--no-sandbox')
browser = webdriver.Chrome(options=options)
pairs = ['AUDJPY', 'AUDNZD', 'AUDUSD', 'CADJPY', 'CHFJPY', 'EURCHF', 'EURGBP', 'EURJPY', 'EURUSD', 'GBPJPY', 'GBPUSD', 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY']
for pair in pairs:
curr_date = datetime.datetime(2015, 1, 1)
while curr_date + relativedelta(months=1) < datetime.datetime.now():
file_name = '{p}-{ym2}.zip'.format(p=pair, ym2=curr_date.strftime('%Y-%m'))
url = 'http://www.truefx.com/dev/data/{y}/{ym1}/{p}-{ym2}.zip'.format(
y=curr_date.strftime('%Y'),
ym1=curr_date.strftime('%B').upper()+'-'+curr_date.strftime('%Y') if curr_date <= datetime.datetime(2017, 3, 1) else curr_date.strftime('%Y-%m'),
p=pair,
ym2=curr_date.strftime('%Y-%m')
)
file_found = False
for root, dirs, files in os.walk(tmp_dir):
for file in files:
if file_name in file: file_found = True
if not file_found:
time.sleep(5)
browser.get (url)
file_downloaded = False
while not file_downloaded:
time.sleep(1)
for root, dirs, files in os.walk(tmp_dir):
for file in files:
if file_name in file and not '.crdownload' in file: file_downloaded = True
print(file_name, 'downloaded from', url)
curr_date = curr_date + relativedelta(months=1)
print('completed')
@Karlheinzniebuhr
Copy link

Great work but it seems it requires a login now
image

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment