Process HIS from W@ves21 RAW into time indexed Pandas DataFrames

buoy_his_to_df_sorted.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:\\\n"
     ]
    }
   ],
   "source": [
    "cd D:/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bragar_his_files = !ls B*.his\n",
    "siadar_his_files = !ls S*.his\n",
    "roag_his_files = !ls R*.his\n",
    "buoys_his_files_dict = {'Bragar':bragar_his_files, 'Roag':roag_his_files, \n",
    "                        'Siadar':siadar_his_files} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "\n",
    "columns = ['date_time', 'tp', 'dirp', 'sprp', 'tz', 'hm0', 'ti', 't1', \n",
    "               'tc', 'tdw2', 'tdw1', 'tpc', 'nu','eps','qp','ss','tref','tsea',\n",
    "               'bat']\n",
    "\n",
    "df = pd.read_csv(\"D:/Roag_Wavegen$}2012-02.his\", names = columns)\n",
    "\n",
    "def date_index_df(df):  \n",
    "    date_times = []\n",
    "    for date_time_string in df['date_time'].values:\n",
    "        if date_time_string != 'nan':\n",
    "            date_time = datetime.strptime(date_time_string[:-5],\"%Y-%m-%dT%H:%M:%S\")\n",
    "            date_times.append(date_time)\n",
    "        else:\n",
    "            date_times.append(datetime(1970,1,1))\n",
    "    df.index = pd.DatetimeIndex(date_times)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "buoys_df_his_dict = {}\n",
    "\n",
    "for key, value in buoys_his_files_dict.items():\n",
    "    his_df = []\n",
    "    for his_file in value:\n",
    "        df = pd.read_csv(his_file, names = columns)\n",
    "        his_df.append(date_index_df(df))\n",
    "    buoy_his = pd.concat(his_df)\n",
    "    buoy_his = buoy_his.sort_index()\n",
    "    buoy_his.to_pickle(key + '_his_df')\n",
    "    buoys_df_his_dict[key] = buoy_his"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}