JsfDataReader.py

#!/usr/local/bin python
# -*- coding: UTF-8 -*-
from pandas_datareader.base import _DailyBaseReader
from datetime import datetime
import pandas as pd
import urllib.request
import numpy as np
from time import sleep

_SLEEP_TIME = 0.5


class _JsfDataReader(_DailyBaseReader):
    @property
    def url(self):
        return 'http://www.taisyaku.jp/search_admin/comp/'

    def _get_params(self):
        params = dict(
            start=self.start,
            end=self.end
        )
        return params

    def _read_one_data(self, url, params):
        results = []
        for date in self._date_strings(self.start, self.end):
            params['date'] = date
            csv = urllib.request.urlopen(url.format(**params))

            try:
                dtypes = {'コード': np.object}
                table = pd.read_csv(csv, encoding='Shift-JIS', skiprows=4, dtype=dtypes)
            except ValueError:
                table = pd.DataFrame()

            results.append(table)
            sleep(_SLEEP_TIME)

        result = pd.concat(results, ignore_index=True)
        return result

    @staticmethod
    def _date_strings(start, end):
        dates = [datetime.strftime(date, '%Y%m%d') for date in pd.date_range(start, end, freq='D')]
        return sorted(dates)


class JsfShinaDataReader(_JsfDataReader):
    _url_base = 'pcsl/shina{date}.csv'

    def read(self):
        # Use _DailyBaseReader's definition
        result = self._read_one_data(self.url + self._url_base, params=self._get_params())

        if not result.empty:
            for col in ('貸借申込日', '決済日', '決算等'):
                result[col] = pd.to_datetime(result[col], format='%Y%m%d')
        return result


class JsfZandakaDataReader(_JsfDataReader):
    _url_base = 'balance/zandaka{date}.csv'

    def read(self):
        # Use _DailyBaseReader's definition
        result = self._read_one_data(self.url + self._url_base, params=self._get_params())
        if not result.empty:
            result['申込日'] = pd.to_datetime(result['申込日'], format='%Y/%m/%d')
        return result