三菱UFJ銀行公表の対顧客外国為替相場のダウンロード

mufg_fx_download.py

"""
三菱UFJ銀行公表の対顧客外国為替相場を http://www.murc-kawasesouba.jp/fx/ からダウンロードします。
conda install lxml が必要です。
"""

from datetime import date
from time import sleep
from typing import Optional

import pandas as pd
import requests
from lxml import html


def safe_strip(s):
    """エラーにならないように strip()"""
    if type(s) == str:
        return s.strip()
    else:
        return s


def download_1day(download_date: date) -> Optional[pd.DataFrame]:
    """1日分のデータをダウンロード"""
    download_day = "{:02}{:02}{:02}".format(download_date.year % 100, download_date.month, download_date.day)
    page = requests.get("http://www.murc-kawasesouba.jp/fx/past/index.php?id={}".format(download_day))
    tree = html.fromstring(page.content)
    ths = [safe_strip(th.text) for th in tree.xpath("//table[@class='data-table7'][1]/tr/th")]
    if len(ths) == 0:
        return None
    tds = [safe_strip(td.text) for td in tree.xpath("//table[@class='data-table7'][1]/tr/td")]
    assert len(tds) % len(ths) == 0
    df = pd.DataFrame(dict([(ths[i], tds[i::len(ths)]) for i in range(len(ths))]))
    df.insert(0, "date", [download_date] * len(df))
    return df


def download_1year(year: int) -> pd.DataFrame:
    """1年分のデータをダウンロード"""
    dfs = []
    for month in range(1, 12 + 1):
        for day in range(1, 31 + 1):
            try:
                download_date = date(year, month, day)
                if download_date.weekday() <= 4:
                    print("download_date = {}".format(download_date))
                    df = download_1day(download_date)
                    if df is not None:
                        dfs.append(df)
            except ValueError:
                pass
            sleep(0.1)
    df = pd.concat(dfs)
    df = df.set_index([df.columns[0], df.columns[1]])
    return df


def write_one_currency_csv(df: pd.DataFrame, currency: str, path: str):
    """特定の為替だけをCSVに保存"""
    df.xs(currency, level=1, drop_level=False).to_csv(path)


if __name__ == '__main__':
    # print(download_1day(date(2019, 2, 19)))

    df_1year = download_1year(2018)
    write_one_currency_csv(df_1year, "US Dollar", "/tmp/usd.csv")