clean-wpd-data.js

#!/usr/bin/env node
const program = require('commander')
const { promisify } = require('util')
const fs = require('fs')
const csv = require('csv')
const moment = require('moment')
const {
  map,
  filter
} = require('lodash')

program
  .option('-i, --in <in>', 'Input CSV')
  .option('-o, --out <out>', 'Output CSV')
  .parse(process.argv)

async function main () {
  if (!program.in) throw new Error('No input file specified')
  if (!program.out) throw new Error('No output file specified')

  console.log(`Reading from: ${program.in}`)
  const inputCsv = await promisify(fs.readFile)(program.in, { encoding: 'utf8' })
  const parsedCsv = await promisify(csv.parse)(inputCsv, { columns: true })

  // Fix dates to ISO format that python strftime expects (6 digit microsecond, not 3)
  const processedCsv = map(parsedCsv, (r => {
    const ReceivedDateTime = moment.utc(r.ReceivedDateTime).format('Y-MM-DD[T]HH:mm:ss.SSSSSS')
    return Object.assign({}, r, { ReceivedDateTime })
  }))

  const filteredCsv = filter(processedCsv, (r => !moment(r.ReceivedDateTime).isAfter(moment())))
  console.log(`Filtered ${parsedCsv.length - filteredCsv.length} records`)

  console.log(`Writing to: ${program.out}`)
  const stringifiedCsv = await promisify(csv.stringify)(filteredCsv, { header: true })
  await promisify(fs.writeFile)(program.out, stringifiedCsv, { encoding: `utf8` })
}

main()