proof of concept, gets data from Nightscout, sends it to OpenHumans

oh.js

/*
  openhumans upload API

TOKEN=example_token
URL=http://www.openhumans.org/api/project/upload/?access_token=$TOKEN
METADATA='{"tags": ["survey", "diet", "csv"], "description": "Diet survey questions and responses", "md5": "156da7fc980988c51682374436849943"}'

http --verbose --form POST $URL \
  project_member_id='12345678' \
    metadata=$METADATA \
      data_file@./test-file.txt
*/

var META_TEMPLATE = {
  tags: []
, description: "Diabetes data"
, md5: ""
};
var _ = require('lodash');
var request = require('request');
var es = require('event-stream');
var validUrl = require('valid-url').isWebUri;
var qs = require('qs');
var url = require('url');
var crypto = require('crypto');

function guess_ns_url (input) {
  var guesses = [
    validUrl(input)
  , validUrl('https://' + input)
  ];
  function exist (i) { return !!i; }
  return guesses.filter(exist).shift( );

}

var NS = {
  entries: '/api/v1/entries.json'
, treatments: '/api/v1/treatments.json'
};



function generateNightScoutStreams (spec) {
  var input = url.parse(guess_ns_url(spec.url));

  function getEntries ( ) {
    var query = { find: { } };
    query.count = 100000;
    if (spec.after) {
      query.find = _.merge(query.find, {dateString: { "$gte": spec.after } });
    }
    if (spec.before) {
      query.find = _.merge(query.find, {dateString: { "$lte": spec.before } });
    }

    var endpoint = _.merge(input,
      { pathname: NS.entries
      , path: NS.entries
      , search: qs.stringify(query, { encode: false })
      });
    var req = url.parse(url.format(endpoint));
    return req;

  }

  function getTreatments ( ) {
    var query = { find: { } };
    query.count = 100000;
    if (spec.after) {
      query.find = _.merge(query.find, {timestamp: { "$gte": spec.after } });
    }
    if (spec.before) {
      query.find = _.merge(query.find, {timestamp: { "$lte": spec.before } });
    }

    var endpoint = _.merge(input,
      { pathname: NS.treatments
      , path: NS.treatments
      , search: qs.stringify(query, { encode: false })
      });
    var req = url.parse(url.format(endpoint));
    return req;

  }

  function iter (item, next) {
    // console.log(item);
    request.get({url: item.href, json: true}, function rx (err, resp, body) {
      next(err, { req: item, res: body });
    })
  }

  function pipeline (readers) {
    return es.pipeline(readers, es.map(iter));
  }

  function stream (done) {
    return pipeline(generate( )).pipe(es.writeArray(done));
  }

  function generate ( ) {
    var urls = [ getEntries( ), getTreatments( ) ];
    return es.readArray(urls);
  }

  generate.iter = iter;
  generate.pipeline = pipeline;
  generate.stream = stream;
  generate.entries = getEntries;
  generate.treatments = getTreatments;

  return generate;


}


function uploader (ctx) {

  var base = ctx.OH_BASE || 'http://unix:oh-debug.socket:';
  var path = ctx.path || '/api/project/upload';

  function formData (id, payload, opts) {
    var metadata = _.clone(META_TEMPLATE);
    // metadata.tags.push('diabetes');
    // metadata.tags.push('nightscout');
    metadata.tags = metadata.tags.concat(opts.tags || [ ])
    metadata.creation_date = opts.creation_date || (new Date( ).toISOString( ));

    if (opts.description) {
      metadata.description = opts.description;
    }

    if (opts.start_date) {
      metadata.start_date = opts.start_date;
    }

    if (opts.end_date) {
      metadata.end_date = opts.end_date;
    }

    if (opts.md5) {
      metadata.md5 = opts.md5;
    }

    var form = {
      project_member_id: id
    , data_file: payload
    , metadata: JSON.stringify(metadata)
    };
    return form;
  }

  function upload (opts, payload, done) {
    var form = formData(ctx.person_id, payload, opts)
    var urlish = url.parse(base + path);
    urlish.search = qs.stringify({access_token: ctx.access_token});
    urlish = url.parse(url.format(urlish));
    // console.log(urlish, form);

    function rx (err, resp, body) {
      done(err, resp.statusMessage);
    }

    request.post({url: urlish.href, formData: form, json: false}, rx);

  }

  return upload;


}

module.exports = uploader;

if (!module.parent) {
  var yargs = require('yargs');
  var argv = yargs.usage('$0 <nightscout>')
    .option('oh_base', {
      default: 'http://www.openhumans.org'
    , help: 'OH Base URL'
    })
    .option('oh_path', {
      default: '/api/project/upload'
    , help: 'OH Url Path endpoint'
    })
    .option('project_member_id', {
      default: ''
    , help: 'OH project member id'

    })
    .option('before', {
      default: ''
    , help: 'select events occuring before this date'

    })
    .option('after', {
      default: ''
    , help: 'select events occuring after this date'

    })
    .option('access_token', {
      default: ''
    , help: 'OH Access Token'

    })
    .help( )
    .argv
    ;

  var urlish = argv._[0];
  // console.log(argv);
  var nightscout = generateNightScoutStreams({
      url: urlish
    , before: argv.before
    , after:  argv.after
  });

  var archiver = uploader({
    access_token: argv.access_token
  , person_id: argv.project_member_id
  , OH_BASE: argv.oh_base
  , path: argv.oh_path
  });

  function getTime (item) {
    return item.timestamp || item.dateString || item.created_at;
  }

  var iterator = es.map(function iter (item, next) {
    var opt = {
      tags: ['nightscout', 'diabetes', 'json', item.req.path]
    , description: item.req.pathname
    };

    if (item.res && item.res.length > 0) {
      opt.start_date = getTime(item.res[0]);
      opt.end_date = getTime(item.res[item.res.length - 1]);
      if (opt.start_date > opt.end_date) {
        opt.start_date = opt.end_date;
        opt.end_date = getTime(item.res[0]);
      }
    }
    var data = JSON.stringify(item.res);
    opt.md5 = crypto.createHash('md5').update(data).digest('hex');

    archiver(opt, data, next);
  });

  function finished (err, uploaded) {
    console.log("ARCHIVED", err, uploaded);
  }

  if (urlish) {
    nightscout.stream(function (err, results) {
      // console.log("FROM NIGHTSCOUT", arguments);
      var sources = es.readArray(results);
      es.pipeline(sources, iterator, es.writeArray(finished));
    });
  }
}