Cumulus log files to Pandas Dataframe

log_to_dataframe.py

# Python modules for taking Cumulus ( sandaysoft.com ) logs into a Pandas
# ( pandas.pydata.org ) DataFrame, and using the date and time columns to
# create a DateTimeIndex for the DataFrame and sort upon it.

import pandas as pd
import os
import glob
from datetime import datetime

log_path = 'D:\\log_files_folder'

os.chdir(log_path)
log_files = glob.glob('*log.txt')

fields = ['date', 'time', 'current_temp', 'current_humidity', 
          'current_dewpoint', 'current_wind_speed', 'current_wind_gust', 
          'current_wind_bearing', 'current_rainfall_rate', 
          'total_rainfall_today', 'current_sea_level_pressure', 
          'current_rainfall_counter', 'inside_temperature', 'inside_humidity',
          'latest_gust', 'wind_chill', 'heat_index', 'uv_index', 
          'solar_radiation', 'evapotranspiration', 'annual_evapotranspiration',
          'apparent_temperature', 'current_theoretical_max_solar_radiation', 
          'hours_of_sunshine_so_far_today', 'sortcurrent_wind_bearing_2', 
          ]

large_log_dataframe = pd.DataFrame(columns = fields)
          
for file_path in log_files:
    log_dataframe = pd.io.parsers.read_csv(log_path + os.sep + file_path, names = fields)
    large_log_dataframe = pd.concat([large_log_dataframe, log_dataframe])
    
date_time_strings = large_log_dataframe['date'] + ' ' + large_log_dataframe['time']
date_time_array = []

for date_time_string in date_time_strings.values:
    date_time_array.append(datetime.strptime(date_time_string, "%d/%m/%y %H:%M"))

date_time_index = pd.DatetimeIndex(date_time_array)
large_log_dataframe.index = date_time_index
sorted_large_log_dataframe = large_log_dataframe.sort_index()
sorted_large_log_dataframe.save('date_time_indexed_sorted_dataframe')