Dexcom .csv merge

README.md

Dexcom .csv merge

A simple tool to merge several Dexcom Studio .csv/.txt export files into a single file.

NB: Now being further developed in iPancreas-dexcom.

dexcom_merge.py

# usage: dexcom_merge.py [-h] [-o OUTPUT_FILE] [-p DIR_PATH]
#
# Merge a set of Dexcom .csv exports (from Dexcom Studio) into one .csv file.
#
# optional arguments:
#   -h, --help            show this help message and exit
#   -o OUTPUT_FILE, --output_file OUTPUT_FILE
#                         path and/or name of output file
#   -p DIR_PATH, --path DIR_PATH
#                         path to the directory where all your Dexcom Studio
#                         .csv exports are stored
#
# Copyright (c) 2014, Jana E. Beck
# Contact: jana.eliz.beck@gmail.com
# License: GPLv3 (http://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import print_function

import argparse
import csv
import os

class DexcomSet:

  def __init__(self, files):

    self.set = set([])

    self.files = files

    self.total_so_far = 0

    for f in self.files:
      self._add_rows_from_file(f)

  def _add_rows_from_file(self, this_file):
    """Add the rows from a file to the DexcomSet."""

    count = 0
    missing = 0

    with open(this_file, 'rU') as f:
      rdr = csv.reader(f, delimiter='\t')
      # exclude header
      rdr.next()
      for row in rdr:
        # set uses a hash internally and hash keys must be immutable types
        # i.e., tuple, not list
        # also replacing first two cols with '' because want to avoid duplicates with rows that have ID info
        row[0] = ''
        row[1] = ''
        self.set.add(tuple(row))
        count += 1

    print("%i readings in %s." %(count, this_file))
    print("%i items in DexcomSet." %(len(self.set)))
    if len(self.set) - self.total_so_far != count:
      duplicates = ((self.total_so_far + count) - len(self.set))
      print("%i duplicate records in this file." %(duplicates))
    print()
    self.total_so_far = len(self.set)

  def _sort(self):
    """Sort the DexcomSet by GlucoseInternalTime."""

    return sorted(self.set, key=lambda t: t[2])

  def print_set(self, header, output_file = 'merged-dexcom.csv'):
    """Print the DexcomSet row-by-row to file."""

    count = 0

    with open(output_file, 'w') as f:
      wrtr = csv.writer(f, delimiter='\t')
      print("### Writing to new file %s..." %(output_file))
      print()
      wrtr.writerow(header)
      for item in self._sort():
        wrtr.writerow(list(item))
        count += 1

    print("%i non-duplicate records printed to %s." %(count, output_file))
    print()

def get_header(this_file):
  """Get the header of a Dexcom file."""

  with open(this_file, 'rU') as f:
    rdr = csv.reader(f, delimiter='\t')
    return rdr.next()

def get_file_list(path = ""):
  """Get the list of files with .csv or .txt extensions in the target directory."""

  all_files = []

  for root, dirs, files in os.walk(path):
    all_files += [os.path.join(root, f) for f in files if (f.endswith('.txt') or f.endswith('.csv') and not (f.startswith('$') or f.startswith('._')))]

  return all_files

def main():

  parser = argparse.ArgumentParser(description='Merge a set of Dexcom .csv exports (from Dexcom Studio) into one .csv file.')

  parser.add_argument('-c', '--csv', action= 'store_true', help='comma- (instead of tab-)delimited output')
  parser.add_argument('-o', '--output_file', action = 'store', dest = "output_file", help='path and/or name of output file')
  parser.add_argument('-p', '--path', action = 'store', dest = "dir_path", help='path to the directory where all your Dexcom Studio .csv exports are stored')

  args = parser.parse_args()

  try:
    files = get_file_list(args.dir_path)
  except TypeError:
    files = get_file_list()

  expected_header = ['PatientInfoField', 'PatientInfoValue', 'GlucoseInternalTime', 'GlucoseDisplayTime', 'GlucoseValue', 'MeterInternalTime', 'MeterDisplayTime', 'MeterValue', 'EventLoggedInternalTime', 'EventLoggedDisplayTime', 'EventTime', 'EventType', 'EventDescription']
  to_remove = []
  for f in files:
    if get_header(f) != expected_header:
      print()
      print("!!! This file doesn't look like a Dexcom file: \n%s \nI'm skipping it. :(" %(f))
      print()
      to_remove.append(f)

  [files.remove(f) for f in to_remove]
  try:
    header = get_header(files[0])
  except IndexError:
    print()
    print("Sorry, I couldn't find any Dexcom files in this directory. Try giving me a path to the directory where you've stored them.")
    print()
    exit(0)

  print('### Merging the following files:')
  [print(f) for f in files]
  print()

  dex = DexcomSet(files)

  try:
    dex.print_set(header, args.output_file)
  except TypeError:
    dex.print_set(header)

if __name__ == '__main__':
  main()

thumbnail.png