upbot.js

var fs = require('fs');
var soda = require('soda-js');
var moment = require('moment');

var trim = function(str) { return str.replace(/^\s+|\s+$/g, ''); }

// following is from http://stackoverflow.com/questions/8493195/how-can-i-parse-a-csv-string-with-javascript
function csvToArray(text) {
    var re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/;
    var re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;
    // Return NULL if input string is not well formed CSV string.
    if (!re_valid.test(text)) return null;
    var a = [];                     // Initialize array to receive values.
    text.replace(re_value, // "Walk" the string using replace with callback.
        function(m0, m1, m2, m3) {
            // Remove backslash from \' in single quoted values.
            if      (m1 !== undefined) a.push(m1.replace(/\\'/g, "'"));
            // Remove backslash from \" in double quoted values.
            else if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"'));
            else if (m3 !== undefined) a.push(m3);
            return ''; // Return empty string.
        });
    // Handle special case of empty last value.
    if (/,\s*$/.test(text)) a.push('');
    return a;
};

// here we read in the conf file and update all the files it indicates.
fs.readFile(__dirname + '/config.json', function(error, configText)
{
  if (error) { console.error(error); return; }

  var config = JSON.parse(configText);
  for (var i = 0; i < config.updates.length; i++)
  {
    (function(update)
    {
      console.log('Updating ' + update.id + ' from ' + update.file + '...');

      fs.readFile(update.file, function(error, data)
      {
        if (error) { console.error(error); return; }

        var lines = data.toString().split('\n');
        var parsedData = [];
        for (var j = 0; j < lines.length; j++)
        {
          var fields = csvToArray(lines[j]);
          if (fields.length === 0) { console.warn('Warning: skipping empty row in file ' + update.file); continue; }
          if (fields.length !== update.columns.length) { console.error('Expected ' + update.columns.length + ' columns for ' + update.file + ' but got ' + fields.length + '!'); return; }

          var datum = {};
          parsedData.push(datum);

          for (var k = 0; k < fields.length; k++)
          {
            var col = update.columns[k];
            var field = fields[k];

            if (col.replace != null)
              field = field.replace(new RegExp(col.search, 'ig'), col.replace);

            if (col.moment != null)
              field = moment(field, col.moment.in).format(col.moment.out);

            datum[col.name] = trim(field);
          }
        }

        if (parsedData.length === 0) { console.log('No relevant data found in file ' + update.file); return; }

        var producer = new soda.Producer(update.cname, {
          username: config.account,
          password: config.password,
          apiToken: config.token
        });

        producer.operation()
          .withDataset(update.id)
          .add(parsedData)
            .on('success', function(result) { console.log('Successfully wrote ' + parsedData.length + ' rows from ' + update.file + ' to ' + update.id + '.'); console.log(result); })
            .on('error', function(error) { console.error('Error writing ' + parsedData.length + ' rows from ' + update.file + ' to ' + update.id + '!'); console.error(error); });

        console.log('Archiving data from ' + update.file + ' to ' + update.archive + '...');
        fs.appendFile(update.archive, data, function(error)
        {
          if (error) { console.error('Error writing archive file ' + update.archive + ': ' + error); }

          fs.truncate(update.file, 0, function(error)
          {
            if (error)
              console.error('Error truncating data file ' + update.file + ': ' + error);
            else
              console.log('Successfully archived ' + update.file + ' to ' + update.archive);
          });
        })
      });
    })(config.updates[i]);
  }
});