Time/Geo notes and script to parse notes and save to file or load to ElasticSearch

note-load.js

// Parse a summary to extract title, tags, location and start and end
parseNoteSummary = function(text) {
  var result = {
    title: '',
    tags: []
  };
  var ourtext = text;
  regex = / #([\w-\.]+)/;
  while(ourtext.search(regex)!=-1) {
    var out = ourtext.match(regex)[1];
    result.tags.push(out);
    ourtext = ourtext.replace(regex, '');
  }
  regex = / ?@([^@]+)@/;
  if(ourtext.search(regex)!=-1) {
    var match = ourtext.match(regex);
    result['location'] = match[1];
    ourtext = ourtext.replace(regex, ' $1');
  }

  var startEnd = [];

  var time = ourtext.split(': ')[0];
  var startEnd = time.split(' to ');

  regex = / ?\^([^^]+)\^/;
  // alternative based on ^...^
  while(ourtext.search(regex)!=-1) {
    var out = ourtext.match(regex)[1];
    startEnd.push(out);
    ourtext = ourtext.replace(regex, ' $1');
  }
  if (startEnd.length>=1) {
    result['start'] = startEnd[0];
  }
  if (startEnd.length>=2) {
    result['end'] = startEnd[1];
  }

  result.title = ourtext.trim();
  return result;
};

// Parse a date into ISO 8601 format (yyyy-mm-dd) using datejs library
// 
// @return: parsed date in yyyy-mm-dd format or null if could not parse.
//
// TODO: at the moment will always provide mm and dd even if not in input
parseDate = function(date) {
  if (date) {
    var parsedDate = Date.parse(date);
    if (parsedDate) {
      return parsedDate.toString('yyyy-MM-dd');
    }
  }
}

exports.load = function(filepath, dest) {
  var fs = require('fs');
  var text = fs.readFileSync(filepath, 'utf8');
  var parsed = text.split('\n\n').map(function(note) {
    return parseNoteSummary(note);
  });
  fs.writeFileSync('data.json', JSON.stringify(parsed));
  if (dest) {
    count = 1;
    parsed.forEach(function(note) {
      note.id = count;
      // upsert(dest, note);
      count++;
      console.log('uploaded: ' + note.title);
    });
  } else {
    console.log(parsed);
  }
}

function upsert(endpoint, doc) {
  console.log(endpoint);
  var http = require('http');
  var url = require('url');
  var parts = url.parse(endpoint);
  var options = {
    host: parts.hostname,
    path: parts.path,
    port: parts.port,
    method: 'POST'
  };
  console.log(options);

  var req = http.request(options, function(res) {
    console.log('STATUS: ' + res.statusCode);
    console.log('HEADERS: ' + JSON.stringify(res.headers));
    res.setEncoding('utf8');
    res.on('data', function (chunk) {
      console.log('BODY: ' + chunk);
    });
  });
  req.write(JSON.stringify(doc));
  req.end();

notes.txt

2012-04-13T21:00 to 2012-04-14T06:00: fly from @London@ to @Sao Paulo@

2012-04-14: code with Pedro on Recline.

2012-04-15T06:30: leave at 6am on Hacker Bus on 16h trip from @Sao Paulo@ to @Brasilia@. Make major improvements to Recline.

2012-04-16 to 2012-04-18: attend Open Government Partnership meeting in Brasilia presenting in two sessions and moderating another

2012-04-19: attend GIFT (Global Initiative on Financial Transparency) meeting with fellow OKFN-ers Lucy Chambers and Velichka Dimitrova in order to present Technology and Transparency in relation to Participatory Finance report

2012-04-19T16:00: fly from Brasilia to Sao Paulo

2012-04-19T20:00: present about Open Knowledge Foundation at evening workshop organized by Everton of Open Knowledge Foundation Brasil

2012-04-20: visit Camara de Sao Paulo (municipal government of Sao Paulo) and meet with Police Neto (President of Camara)

2012-04-21 to 2012-04-22: participate in 2 day hack day at Casa da Cultura Digital with Daniela da Silva and others

2012-04-23: Dados Abertos Sao Paulo meetup

2012-04-25: meet with Perlmongers and Tiago Rondon re OpenSpending for Sao Paulo and

2012-04-26: visit INPE and give a presentation on the Open Knowledge Foundation. Discuss PyBossa and the ForestWatchers project.

2012-04-27: spend the day with IT3S discussing CSO needs, CKAN and Mootiro

2012-04-28: Papo con Dados - half-day meeting on #OpenData and civic issues ~ 15 attendees. Produce ...

2012-04-29: meet with Patrizia and the Duke for a half-day coding session

2012-05-01: visit Paulo Markun and discuss open data, visualization and civic issues.

2012-05-02T13:00: Fly from @Sao Paulo@ to @Rio de Janerio@ for Rio conference on Services for Cities and the LLGA ceremony

2012-05-02T17:30: LLGA ceremony begins

2012-05-03: all day at Living Labs event. Give a 1m pitch and present at session on ...

2012-05-03T19:00: Dados Abertos meet up in @Rio de Janerio@ co-hosted with Ibase and co-organized with Natalia

2012-05-04: meet with ... from IPEA

2012-05-041500: attend FGV and meet with Ronaldo Lemos and colleagues. Given a presentation on the Open Knowledge Foundation and Open Knowledge Foundation Brasil

2012-05-04T05:45 to 2012-04-04T21:00: 19h flight from @Rio de Janerio@ to @San Francisco@

2012-05-05: meet with Max Ogden, Jessica Lord and Randall Leeds for an afternoon of coding and discussion at @noisebridge@

2012-05-06: meet with Max Ogden and ... . Visit Wikimedia and meet with Dario Tarobelli. Meet with Lisa ... from Common Crawl, visit CfA offices.

2012-06-08 to 2012-06-10: attend ONEF 2012

2012-06-11: meet with CfA and give a short presentation on the Open Knowledge Foundation. Meet with Hypothes.is to discuss next steps around the Annotator, AnnotateIt.org and Hypothes.is. Plan to meet with David Huynh but unfortunately last minute changes mean this is impossible.

2012-05-11T19:00 to 2012-05-12T12:30: flight back to @UK@