How to use: numpy, matplotlib, pandas, scipy

data_interp.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint: disable=missing-docstring
import platform
import sys

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

if platform.system() == 'Darwin':
  matplotlib.use('TkAgg')


def _interp(xvalues, yvalues, kind='cubic', step=0.01):
  from scipy import interpolate

  # https://docs.scipy.org/doc/scipy/reference/tutorial/interpolate.html
  xnew = np.arange(xvalues[0], xvalues[-1], step)
  func = interpolate.interp1d(xvalues, yvalues, kind=kind)

  return xnew, func(xnew)


def _plot(x, y, fig_name='fig.png', show_secs=0, interp_kind=None,
    interp_step=0.01, ax_kwargs=None, plot_kwargs=None, grid=True,
    legend_kwargs=None, legend_noline=True):
  fig, ax = plt.subplots()

  if interp_kind:
    x, y = _interp(x, y, kind=interp_kind, step=interp_step)

  if plot_kwargs:
    ax.plot(x, y, **plot_kwargs)
  else:
    ax.plot(x, y)

  if ax_kwargs:
    ax.set(**ax_kwargs)

  if grid:
    ax.grid()

  if legend_kwargs:
    leg = ax.legend(**legend_kwargs)
  else:
    leg = ax.legend()
  if legend_noline:
    leg.get_frame().set_linewidth(0.0)

  if fig_name:
    fig.savefig(fig_name)
    print('save figure to {}'.format(fig_name))

  if show_secs > 0:
    plt.show(block=False)
    plt.pause(show_secs)
    plt.close()
  elif show_secs < 0:
    plt.show()


def _load(path):
  print("Load: {}".format(path))
  # id, (data, timestamp)
  dtype = {'names': ('data', 'timestamp'), 'formats': ('i4', 'f8')}
  return np.loadtxt(path, dtype=dtype, delimiter=",", skiprows=1,
                    usecols=(1, 2))


def _main():
  if len(sys.argv) < 2:
    sys.exit('python data_interp.py *.txt')

  datas = _load(sys.argv[1])
  if datas.size <= 0:
    sys.exit('Datas not found: {}'.format(sys.argv[1]))

  # https://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes
  ax_kwargs= {
      'xlabel': 'timestamp (s)',
      'ylabel': 'data',
      # 'ylim': (0, 20),
  }

  # https://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.plot.html#matplotlib-axes-axes-plot
  plot_kwargs = {
      'label': 'data',
      'color': '#ed0dd9',
      'linestyle': '-',
      'solid_joinstyle': 'round',
  }

  # https://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.legend.html#matplotlib.axes.Axes.legend
  legend_kwargs = {
      'bbox_to_anchor': (1, 1.1),
      'edgecolor': None,
  }

  _plot(datas['timestamp'], datas['data'], fig_name=None, show_secs=-1,
      interp_kind='cubic', interp_step=0.01,
      ax_kwargs=ax_kwargs, plot_kwargs=plot_kwargs, grid=True,
      legend_kwargs=legend_kwargs, legend_noline=True)


if __name__ == "__main__":
  _main()

data_plot.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint: disable=missing-docstring
import sys

import matplotlib.pyplot as plt
import numpy as np


def _plot(paths, nonzero):
  fig, ax = plt.subplots()

  for i, p in enumerate(paths):
    print("Load: {}".format(p))

    # id, (data), timestamp
    datas = np.loadtxt(p, dtype=np.int32, delimiter=",", skiprows=1, usecols=(1))

    # nonzero
    if nonzero:
      datas_nonzero = datas[datas.nonzero()]
      print("  size_nonzero: {}, size_zero: {}".format(
            datas_nonzero.size, datas.size - datas_nonzero.size))
      datas = datas_nonzero
    else:
      print("  size: {}".format(datas.size))

    # average
    data_avg = np.mean(datas)
    # data_avg = np.average(datas)

    # standard deviation
    # data_std = np.std(datas)
    # sample standard deviation
    data_std = np.std(datas, ddof=1)

    print("  avg: {:.2f}, std: {:.2f}, sum: {}".format(
          data_avg, data_std, np.sum(datas)))

    ax.plot(range(len(datas)), datas, label=str(i))

  ax.legend()
  plt.show()


def _parse_args():
  import argparse
  parser = argparse.ArgumentParser()

  parser.add_argument(
      "--nonzero",
      action="store_true",
      help="show datas without zero value: %(default)s")

  args, unknown = parser.parse_known_args()

  print("Args")
  print("  nonzero: {}".format(args.nonzero))

  return args, unknown


def _main():
  args, paths = _parse_args()

  if not paths:
    sys.exit("python data_plot.py *.txt ...")

  _plot(paths, nonzero=args.nonzero)


if __name__ == "__main__":
  _main()

stamp_diff.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint: disable=missing-docstring
import sys

import matplotlib.pyplot as plt
import numpy as np


def _plot(path, xlabel_per_n=None):
  print("Load: {}".format(path))
  stamps = np.loadtxt(path, dtype=np.float64, delimiter=",", skiprows=1, usecols=(2))
  if stamps.size <= 0:
    return

  fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 6))
  fig.subplots_adjust(left=0.1, right=0.9, bottom=0.15, top=0.9, wspace=0.2)

  # timestamp diff

  stamps_diff = np.diff(stamps)

  ax1.plot(range(len(stamps_diff)), stamps_diff, label="timestamp diff (s)")
  ax1.set_xlabel("avg: {:.3f} s".format(np.mean(stamps_diff)), labelpad=15)
  ax1.legend()

  # count per second

  stamps_int = np.array(stamps, dtype='int')
  stamps_int = stamps_int - stamps_int[0]
  import pandas as pd
  stamps_s = pd.Series(data=stamps_int)
  stamps_s = stamps_s.value_counts(sort=False)
  # print(stamps_s)

  ax2 = plt.subplot(1, 2, 2)
  stamps_s.plot.bar(label="count per second", rot=0)
  ax2.set_xlabel("avg: {:.3f}".format(np.mean(stamps_s.values)), labelpad=15)
  if xlabel_per_n:
    for n, label in enumerate(ax2.xaxis.get_ticklabels()):
      if n % xlabel_per_n != 0:
        label.set_visible(False)
  ax2.legend()

  # show

  plt.show()


def _main():
  if len(sys.argv) < 2:
    sys.exit("python stamp_diff.py *.txt")
  _plot(sys.argv[1])


if __name__ == "__main__":
  _main()

1. python data_plot.py data0.txt

2. python data_plot.py data*.txt

3. python data_interp.py data0.txt

4. python stamp_diff.py data0.txt

1. data0.txt

id, data, timestamp
0, 55, 1592207702.688805
1, 41, 1592207702.783134
2, 57, 1592207702.883619
3, 59, 1592207702.980597
4, 58, 1592207703.08313
5, 41, 1592207703.183011
6, 52, 1592207703.281802
7, 46, 1592207703.387034
8, 41, 1592207703.482777
9, 72, 1592207703.582849
11, 46, 1592207703.783618
12, 59, 1592207703.883357
13, 56, 1592207703.983407
14, 57, 1592207704.082893
15, 64, 1592207704.183343
17, 45, 1592207704.381702
18, 45, 1592207704.4813
19, 52, 1592207704.582907
20, 42, 1592207704.68303
21, 55, 1592207704.78293

2. data1.txt

id, data, timestamp
0, 49, 1592207702.688805
1, 60, 1592207702.783134
2, 45, 1592207702.883619
3, 51, 1592207702.980597
4, 49, 1592207703.08313
5, 58, 1592207703.183011
6, 49, 1592207703.281802
7, 54, 1592207703.387034
8, 59, 1592207703.482777
9, 43, 1592207703.582849
11, 46, 1592207703.783618
12, 43, 1592207703.883357
13, 69, 1592207703.983407
14, 54, 1592207704.082893
15, 58, 1592207704.183343
17, 58, 1592207704.381702
18, 49, 1592207704.4813
19, 59, 1592207704.582907
20, 55, 1592207704.68303
21, 59, 1592207704.78293