34401A-listen.py

#!/usr/bin/env python

import serial
import sys
import time

ser = serial.Serial(
    port = sys.argv[1],
    baudrate = 9600,
    bytesize=serial.EIGHTBITS,
    stopbits = serial.STOPBITS_TWO,
    parity = serial.PARITY_NONE,
    timeout = 10
)

sys.stdout.write("timestamp,%s\n" % sys.argv[2]); sys.stdout.flush()

last_sample_timestamp = time.time()

while True:
    line = ser.readline()
    now = time.time()

    # The FTDI chip seems to buffer up a certain amount of outgoing bytes if
    # there is no listener to recieve them.  Thus, when we first run this
    # script, we will get an initial flood of queued readings.  Throw those
    # away, because we don't have a timestamp for them.
    if now - last_sample_timestamp < 0.3:
        continue
    last_sample_timestamp = now

    microvolts = float(line.rstrip()) * 1000000
    sys.stdout.write("%0.3f,%0.1f\n" % (now, microvolts)); sys.stdout.flush()

README.md

Example of starting logs:

./start-logs.sh svrt

Example of stopping logs:

./stop-logs.sh 1500270000-svrt/pids

Example of plotting:

./make-plots.sh 1500270000-svrt/

csv-cull.py

#!/usr/bin/env python

# csv-cull.py: strip columns out of a CSV file.
# specify the column indexes you want to keep.
# $ cat input.csv | ./csv-cull.py 0,1,3 > output.csv

import csv
import sys

# adapted from https://stackoverflow.com/a/7589615

cols = [int(col) for col in sys.argv[1].split(',')]

reader = csv.reader(sys.stdin)
writer = csv.writer(sys.stdout)
for r in reader:
    values = [r[i] for i in cols]
    writer.writerow(values)

make-plots.sh

#!/bin/bash

set -e -o pipefail
set -x

./split-env-csv.sh "${1}"

nice ./plot.py -y 2 "${1}"/temp_c.csv
nice ./plot.py -y 2 "${1}"/humidity.csv

nice ./plot.py "${1}"/data.csv
mv "${1}"/data.png "${1}/data-a0.png"

for i in 10 100 1000
do
    nice ./plot.py -a "${i}" "${1}"/data.csv
    mv "${1}"/data.png "${1}/data-a${i}.png"
done

plot.py

#!/usr/bin/env python

"""
Copyright (c) 2017 Muxr, http://www.eevblog.com/forum/profile/?u=105823

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE
"""

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import time as std_time
from scipy.interpolate import spline
import matplotlib.ticker as plticker
from matplotlib.ticker import FormatStrFormatter
import matplotlib.dates as mdates
import argparse
import sys
import locale

# By default, matplotlib will try to plot all of the data points in
# one pass.  If you are plotting more than a few days worth of data
# (e.g. ~1 million data points), you'll likely hit this error:
#
#   "OverflowError: Allocated too many blocks"
#
# To avoid this, we instruct matplotlib to break up the plot into
# chunks.  Thanks to https://stackoverflow.com/a/23361090
import matplotlib
matplotlib.rcParams['agg.path.chunksize'] = 100000

COLORS = ["#6e3c82", "#3498db", "#95a5a6", "#e74c3c", "#34495e", "#2ecc71"]


def format_timestamps(ts):
    return [format_timestamp(t) for t in ts]


def format_timestamp(t):
    return std_time.strftime("%H:%M.%S", std_time.localtime(np.asscalar(np.int32(t))))


def get_date_range(df, timestamp_colkey):
    max_time = df[timestamp_colkey].max()
    min_time = df[timestamp_colkey].min()
    t_to = std_time.strftime("%d-%b-%Y", std_time.localtime(np.asscalar(np.int32(max_time))))
    t_from = std_time.strftime("%d-%b-%Y", std_time.localtime(np.asscalar(np.int32(min_time))))
    if t_to == t_from:
        return t_to
    return "{} - {}".format(t_from, t_to)


def time_delta(df, timestamp_colkey):
    start = df[timestamp_colkey].min()
    stop = df[timestamp_colkey].max()
    d = divmod(stop-start, 86400)  # days
    h = divmod(d[1], 3600)         # hours
    m = divmod(h[1], 60)           # minutes
    s = m[1]                       # seconds

    return '{:.0f}d {:02.0f}:{:02.0f}.{:02.0f}'.format(d[0], h[0], m[0], int(s))


def y_fmt(value, _):
    # thanks to https://stackoverflow.com/a/38553110
    global y_fmt_str
    return y_fmt_str.format(value)


def x_fmt(value, _):
    return format_timestamp(value)


# Auto-detect which delimiter is being used in the csv file.
def detect_delimiter(options):
    delims = [';', ',', '|']
    with open(options.infile, 'r') as f:
        first_line = f.readline()
        for delim in delims:
            if delim in first_line:
                return delim
        raise Exception("Could not detect delimiter used in CSV file!")


def stderr_log(msg):
    sys.stderr.write(msg)
    sys.stderr.flush()


def plot(options):

    sns.set(style="darkgrid")
    sns.set_palette(COLORS)

    stderr_log("reading csv file...")
    df = pd.read_csv(options.infile, delimiter=detect_delimiter(options))
    stderr_log(" done.\n")

    if options.skip is not None:
        df = df[options.skip:]

    if options.drop is not None:
        df = df[:0-options.drop]

    # Apply a rolling average filter if requested via cmdline options.
    if options.avg_window is not None:
        window_len = options.avg_window
        avg_df = df.rolling(window=window_len).mean()
        # Until the window fills up, the output will be a bunch of NaN values,
        # which we remove here:
        avg_df = avg_df[window_len-1:]
        df = avg_df

    plt.locator_params(axis='y', nticks=20)

    # Assume the timestamps are in the first column ('colkey' being short for "column key").
    timestamp_colkey = df.columns[0]

    stderr_log("plotting...")
    plot = df.set_index(timestamp_colkey).plot(figsize=(21, 9), linewidth=0.3)
    stderr_log(" done.\n")

    # set labels for X and Y axis
    # n = len(plot.xaxis.get_ticklabels())
    # evened_out_ts = np.linspace(df[timestamp_colkey].min(), df[timestamp_colkey].max(), n)
    # plot.set_xticklabels(format_timestamps(evened_out_ts), rotation=-15)
    plot.xaxis.set_major_formatter(plticker.FuncFormatter(x_fmt))

    # The below plot accessories are based off of the first (non-timestamp) column of values.
    values_colkey = df.columns[1]

    value_max = ("max: " + y_fmt_str).format(df[values_colkey].max())
    value_min = ("min: " + y_fmt_str).format(df[values_colkey].min())

    # if ydigits isn't specified, try to figure it out automatically
    if options.ydigits is None:
        options.ydigits = max(7 - len(str(value_max)), 0)

    # Create Y-axis tick labels.
    ny = len(plot.yaxis.get_ticklabels())
    plot.set_yticklabels(np.linspace(df[values_colkey].min(), df[values_colkey].max(), ny))
    plot.yaxis.set_major_formatter(plticker.FuncFormatter(y_fmt))

    # TODO add minor ticks
    # plot.yaxis.set_tick_params(which='minor', right='off')

    fig = plot.get_figure()

    #
    # Plot the trend line of the first data column
    data_colkey = df.columns[1]
    z = np.polyfit(df[timestamp_colkey], df[data_colkey], 1)
    p = np.poly1d(z)
    plt.plot(df[timestamp_colkey], p(df[timestamp_colkey]), "r--", color=COLORS[0], linewidth=0.8)

    #
    # add some captions
    title = '{} ({})'.format(options.title, get_date_range(df, timestamp_colkey))
    if options.avg_window is not None:
        title = title + " (%s-point rolling average)" % options.avg_window
    fig.text(0.40, 0.90, title, fontsize=13, fontweight='bold', color=COLORS[0])
    stderr_log('\n')
    print "title:", title

    height = 0.265
    spacing = 0.025
    
    fig.text(0.905, height, value_max, fontsize=12, color=COLORS[0])
    height -= spacing
    print value_max

    fig.text(0.905, height, value_min, fontsize=12, color=COLORS[0])
    height -= spacing
    print value_min

    value_p2p = ("p-p: " + y_fmt_str).format(float(df[values_colkey].max() - df[values_colkey].min()))
    fig.text(0.905, height, value_p2p, fontsize=12, color=COLORS[0])
    height -= spacing
    print value_p2p

    std_dev_fmt_str = "{:0,.%if}" % (int(options.ydigits) + 1)
    value_std_dev = ("std-dev: " + std_dev_fmt_str).format(round(df[values_colkey].std(), 9))
    fig.text(0.905, height, value_std_dev, fontsize=12, color=COLORS[0])
    height -= spacing
    print value_std_dev

    count = 'samples: {:,}'.format(df[values_colkey].count())
    fig.text(0.905, height, count, fontsize=12, color=COLORS[0])
    height -= spacing
    print count

    value_duration = 'duration: {}'.format(time_delta(df, timestamp_colkey))
    fig.text(0.905, height, value_duration, fontsize=12, color=COLORS[0])
    height -= spacing
    print value_duration

    mean_fmt_str = "{:0,.%if}" % (int(options.ydigits) + 1)
    mean = ("mean: " + mean_fmt_str).format(round(df[values_colkey].mean(), 9))
    fig.text(0.905, height, mean, fontsize=13, fontweight='bold', color=COLORS[0])
    height -= spacing
    print mean

    fig.savefig(options.outfile, bbox_inches='tight')


# Globals
y_fmt_str = None


def main():
    stderr_log("mplot running...\n")
    parser = argparse.ArgumentParser()
    parser.add_argument('infile', nargs='?')
    parser.add_argument('outfile', nargs='?')
    parser.add_argument('-t',
                        '--title',
                        dest='title',
                        action='store',
                        help='title to be used in the chart')
    parser.add_argument('-y',
                        '--ydigits',
                        dest='ydigits',
                        action='store',
                        default="0",
                        help='Number of least significant digits in the Y labels')
    parser.add_argument('-a',
                        '--rolling-average-window',
                        dest='avg_window',
                        type=int,
                        action='store',
                        help='Apply a rolling-average with a window of N data points')
    parser.add_argument('-s',
                        '--skip',
                        dest='skip',
                        action='store',
                        type=int,
                        help='Number of initial data points to skip over')
    parser.add_argument('-d',
                        '--drop',
                        dest='drop',
                        action='store',
                        type=int,
                        help='Number of trailing data points to drop')

    options = parser.parse_args()

    global y_fmt_str
    y_fmt_str = "{:0,.%sf}" % options.ydigits

    if options.infile is None:
        print "use -h for help"
        sys.exit(-1)

    if options.outfile is None:
        extensionless = options.infile.split('.')[0]
        options.outfile = extensionless + '.png'

    if options.title is None:
        options.title = options.infile

    plot(options)

if __name__ == '__main__':
    main()

Si7021-logger.py

#!/usr/bin/env python

# Si7021-logger.py: read temperature / humidity data from an Arduino and log as CSV.
#
# linux usage examples:
# ./Si7021-logger.py /dev/ttyACM0
# ./Si7021-logger.py /dev/ttyUSB0
#
# mac usage examples:
# ./Si7021-logger.py /dev/tty.usbserial-DN02TIYO

import serial
import sys
import struct

# some useful notes on python dates and times:
# http://avilpage.com/2014/11/python-unix-timestamp-utc-and-their.html
import time


# CRC-8 implementation adapted from https://gist.github.com/hypebeast/3833758
class Crc8:
    def __init__(self):
        self.crcTable = (
            0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, 0x38,
            0x3f, 0x36, 0x31, 0x24, 0x23, 0x2a, 0x2d, 0x70, 0x77,
            0x7e, 0x79, 0x6c, 0x6b, 0x62, 0x65, 0x48, 0x4f, 0x46, 
            0x41, 0x54, 0x53, 0x5a, 0x5d, 0xe0, 0xe7, 0xee, 0xe9, 
            0xfc, 0xfb, 0xf2, 0xf5, 0xd8, 0xdf, 0xd6, 0xd1, 0xc4, 
            0xc3, 0xca, 0xcd, 0x90, 0x97, 0x9e, 0x99, 0x8c, 0x8b, 
            0x82, 0x85, 0xa8, 0xaf, 0xa6, 0xa1, 0xb4, 0xb3, 0xba, 
            0xbd, 0xc7, 0xc0, 0xc9, 0xce, 0xdb, 0xdc, 0xd5, 0xd2,
            0xff, 0xf8, 0xf1, 0xf6, 0xe3, 0xe4, 0xed, 0xea, 0xb7, 
            0xb0, 0xb9, 0xbe, 0xab, 0xac, 0xa5, 0xa2, 0x8f, 0x88, 
            0x81, 0x86, 0x93, 0x94, 0x9d, 0x9a, 0x27, 0x20, 0x29, 
            0x2e, 0x3b, 0x3c, 0x35, 0x32, 0x1f, 0x18, 0x11, 0x16, 
            0x03, 0x04, 0x0d, 0x0a, 0x57, 0x50, 0x59, 0x5e, 0x4b, 
            0x4c, 0x45, 0x42, 0x6f, 0x68, 0x61, 0x66, 0x73, 0x74, 
            0x7d, 0x7a, 0x89, 0x8e, 0x87, 0x80, 0x95, 0x92, 0x9b,
            0x9c, 0xb1, 0xb6, 0xbf, 0xb8, 0xad, 0xaa, 0xa3, 0xa4,
            0xf9, 0xfe, 0xf7, 0xf0, 0xe5, 0xe2, 0xeb, 0xec, 0xc1, 
            0xc6, 0xcf, 0xc8, 0xdd, 0xda, 0xd3, 0xd4, 0x69, 0x6e, 
            0x67, 0x60, 0x75, 0x72, 0x7b, 0x7c, 0x51, 0x56, 0x5f, 
            0x58, 0x4d, 0x4a, 0x43, 0x44, 0x19, 0x1e, 0x17, 0x10, 
            0x05, 0x02, 0x0b, 0x0c, 0x21, 0x26, 0x2f, 0x28, 0x3d, 
            0x3a, 0x33, 0x34, 0x4e, 0x49, 0x40, 0x47, 0x52, 0x55, 
            0x5c, 0x5b, 0x76, 0x71, 0x78, 0x7f, 0x6a, 0x6d, 0x64, 
            0x63, 0x3e, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2c, 0x2b, 
            0x06, 0x01, 0x08, 0x0f, 0x1a, 0x1d, 0x14, 0x13, 0xae, 
            0xa9, 0xa0, 0xa7, 0xb2, 0xb5, 0xbc, 0xbb, 0x96, 0x91,
            0x98, 0x9f, 0x8a, 0x8d, 0x84, 0x83, 0xde, 0xd9, 0xd0, 
            0xd7, 0xc2, 0xc5, 0xcc, 0xcb, 0xe6, 0xe1, 0xe8, 0xef,
            0xfa, 0xfd, 0xf4, 0xf3
        )
    
    def crc(self, msg):
        runningCRC = 0
        for c in msg:
            runningCRC = self.crcByte(runningCRC, c)
        return runningCRC

    def crcByte(self, oldCrc, byte):
        res = self.crcTable[oldCrc & 0xFF ^ byte & 0xFF];
        return res


# parse a packet of bytes from the arduino and log the values to stdout as CSV.
# if the CRC didn't match, return false.
def parse_and_send_bytes(bytes):
    # unpack those 9 bytes as two floats and an 8-bit CRC
    (temp_c, humidity, crc) = struct.unpack('ffB', bytes)

    # verify the CRC
    if crc != crc8.crc(bytes[:8]):
        return False

    now = time.time()

    # the FTDI chip seems to buffer up a certain amount of outgoing bytes if there is no
    # listener to recieve them.  thus, when we first run this script, we will get an initial
    # flood of queued readings.  throw those away.
    global last_sample_timestamp
    if now - last_sample_timestamp < 0.1:
        return True
    last_sample_timestamp = now

    # print out the results in CSV format
    sys.stdout.write("%0.3f,%0.4f,%0.4f\n" % (now, temp_c, humidity))
    sys.stdout.flush()
    return True


# globals
last_sample_timestamp = time.time()


if __name__ == "__main__":
    # 9600, 8N1
    ser = serial.Serial(
        port = sys.argv[1], # e.g. /dev/ttyACM0 or /dev/ttyUSB0
        baudrate = 9600,
        bytesize=serial.EIGHTBITS,
        stopbits = serial.STOPBITS_ONE,
        parity = serial.PARITY_NONE,
        timeout = 10
    )

    crc8 = Crc8()
    bytes = bytearray([0,0,0,0,0,0,0,0,0])
    print "timestamp,temp_c,humidity"

    while True:
        # read in 9 bytes from the arduino and process the message
        for i in range(9):
            bytes[i] = ser.read(1)
        success = parse_and_send_bytes(bytes)

        # if CRC failed, try shifting in one byte at a time until we get a good result.
        # it is normal to see this error once during startup.
        if success == False:
            sys.stderr.write("ERROR: bad CRC, discarding result!\n")
            while success == False:
                for i in range(8):
                    bytes[i] = bytes[i+1]
                bytes[8] = ser.read(1)
                success = parse_and_send_bytes(bytes)

split-env-csv.sh

#!/bin/bash

set -e -o pipefail

cd "${1}"

cat env.csv | ../csv-cull.py 0,1 > temp_c.csv
cat env.csv | ../csv-cull.py 0,2 > humidity.csv

start-logs.sh

#!/bin/bash

set -e -o pipefail

now=`date +%s`
subdir=${now}-${1}
mkdir -p ${subdir}

./34401A-listen.py /dev/ttyUSB0 $1 >> ${subdir}/data.csv &
echo $! >> ${subdir}/pids

./Si7021-logger.py /dev/ttyACM0 >> ${subdir}/env.csv &
echo $! >> ${subdir}/pids

stop-logs.sh

#!/bin/bash

set -e -o pipefail

cat $1 | xargs -n1 kill