OFX Currency Crawler as a Python Script

fetchCurrencyData.py

#!/usr/bin/env python

##############################################################################
# fetchCurrencyData.py                                                       #
#                                                                            #
# Requirements                                                               #
#                                                                            #
# - Python 3                                                                 #
# - Selenium package for Python                                              #
# - The web driver for Chrome                                                #
#                                                                            #
# 2018, Michael Chen; Apache 2.0                                             #                    
##############################################################################

import csv
import os
import sys
import time

from selenium import webdriver
from selenium.webdriver.chrome.options import Options


baseCurrency = None
targetCurrency = None
targetDuration = "Last 1 year"
downloadPath = os.path.dirname(os.path.abspath(__file__))

if len(sys.argv) < 3:
    sys.stderr.write("No valid currency\n")
    sys.exit(1)

baseCurrency = sys.argv[1]
targetCurrency = sys.argv[2]

# Use headless mode
options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')

# Create a new instance of the Chrome driver
driver = webdriver.Chrome(chrome_options=options)

# Go to OFX Historical Data page
driver.get("https://www.ofx.com/en-us/forex-news/historical-exchange-rates/")

# Wait the page to fresh.
time.sleep(10)

# Select base arrow.
baseArrow = driver.find_element_by_css_selector(
    ".historical-rates--camparison--base .select2 .selection span .select2-selection__arrow")
baseArrow.click()

time.sleep(2)  # Simulate idling.

# Click on base currency.
baseOptions = driver.find_elements_by_css_selector(
    ".historical-rates--camparison--base select optgroup option")
for option in baseOptions:
    if baseCurrency in option.text:
        option.click()
        break

time.sleep(1)  # Simulate idling.

baseArrow.click()

time.sleep(1)  # Simulate idling.

# Select target arrow.
targetArrow = driver.find_element_by_css_selector(
    ".historical-rates--camparison--target .select2 .selection span .select2-selection__arrow")
targetArrow.click()

time.sleep(2)  # Simulate idling.

# Click on target currency.
targetOptions = driver.find_elements_by_css_selector(
    ".historical-rates--camparison--target select optgroup option")
for option in targetOptions:
    if targetCurrency in option.text:
        option.click()
        break

time.sleep(1)  # Simulate idling.

targetArrow.click()

time.sleep(1)  # Simulate idling.

# Select target period.
periodArrow = driver.find_element_by_css_selector(
    ".historical-rates--period ~ .select2 .selection span .select2-selection__arrow")
periodArrow.click()

time.sleep(2)  # Simulate idling.

# Click on target currency.
periodOptions = driver.find_elements_by_css_selector(
    ".historical-rates--period option")
for option in periodOptions:
    if targetDuration in option.text:
        option.click()
        break

time.sleep(1)  # Simulate idling.

periodArrow.click()

time.sleep(1)  # Simulate idling.

# Submit for the result.
submitButton = driver.find_element_by_css_selector(".historical-rates--submit")
submitButton.click()

# Wait the page to fresh.
time.sleep(5)

sys.stderr.write("Write data to csv file...\n")
avgRate = float(driver.find_element_by_css_selector(".historical-rates--table--average--value").text)
ratio = 15
upLimit = avgRate * ratio
downLimit = avgRate / ratio

with open("%sto%s.csv" % (baseCurrency, targetCurrency), 'w', newline='') as csvfile:
    csvwriter = csv.writer(csvfile)

    csvwriter.writerow(["Date", "Rate"])

    records = driver.find_elements_by_css_selector(".historical-rates--table--results tr")
    for record in records:
        d = record.find_element_by_css_selector(".historical-rates--table--date").text
        r = float(record.find_element_by_css_selector(".historical-rates--table--rate").text)

        if r >= upLimit or r <= downLimit:
            continue
        
        csvwriter.writerow([d, r])

# Close the browser.
driver.quit()