Insert RTC Cocoa Beach weather data provided by Manajit Sengupta (NREL) into Solar Forecast Arbiter reference database

cocoa_beach_insert.py

"""Insert RTC Cocoa Beach weather data provided by Manajit Sengupta (NREL)."""

import sys

from pathlib import Path
import os

import numpy as np
import pandas as pd

from solarforecastarbiter.io.api import APISession, request_cli_access_token
from solarforecastarbiter.io.reference_observations.common import (
    _prepare_data_to_post
)

print(sys.path)

path = Path('fsec_rtc')

files = [f for f in path.iterdir()]
print(files)

rtc_to_sfa = {
    'Direct_Wm2_Avg': 'dni',
    'Global_Wm2_Avg': 'ghi',
    'Diffuse_Wm2_Avg': 'dhi',
    'Temp_C_Avg': 'air_temperature',
    'WS_ms_Mean': 'wind_speed',
    'RH_pct_Avg': 'relative_humidity'
}

username = "reference@solarforecastarbiter.org"
# assumes username/password is stored in default os x keychain
cmd = f"security find-internet-password -a '{username}' -w"
with os.popen(cmd) as p:
    password = p.read().rstrip('\n')

token = request_cli_access_token(username, password)

session = APISession(token)

sites = session.list_sites()

cocoa = list(filter(
    lambda x: 'DOE RTC Cocoa FL' == x.name and 'Reference' == x.provider,
    sites))[0]

obs = session.list_observations()

cocoa_obs = list(filter(lambda x: x.site == cocoa, obs))
print(*cocoa_obs, sep='\n\n')


def read_data_file(file):
    df = pd.read_csv(file, index_col='Time', parse_dates=True)
    df = df.tz_localize('Etc/GMT+5').rename(columns=rtc_to_sfa)
    df = fix_st_to_dst(df)
    return df


def fix_st_to_dst(df):
    # assume duplicated only occurs at ST to DST transition
    duplicated = df.index.duplicated()
    num_duplicates = duplicated.sum()
    first_duplicate_index = np.argmax(duplicated)
    first_shift_point = first_duplicate_index - num_duplicates
    new_hour = \
        df.index[first_shift_point:first_duplicate_index] - pd.Timedelta('1h')
    new_hour_df = df.iloc[first_shift_point:first_duplicate_index].copy()
    new_hour_df.index = new_hour
    df_fixed = pd.concat([
        df.iloc[:first_shift_point],
        new_hour_df,
        df.iloc[first_duplicate_index:]
    ])
    return df_fixed


def post_data(session, data):
    for obs in cocoa_obs:
        try:
            observation_df = _prepare_data_to_post(
                data, obs.variable, obs, data.index[0], data.index[-1])
        except KeyError:
            print(f'no {obs.variable} in data. continuing')
            continue

        print(f'posting {obs.variable}')
        for split in np.array_split(observation_df, 12):
            session.post_observation_values(obs.observation_id, split)


for file in files:
    df = read_data_file(file)
    post_data(session, df)