.gitignore

input.csv

import.js

// How to run this script:
//
// Install node
// > cd /path/to/import.js
// edit the api details below (username, password and baseURL)
// > npm install csv restler underscore async
// > cp your_file_to_import.csv input.csv
// > node import.js

var csv     = require('csv'),
    url     = require('url'),
    restler = require('restler'),
    _       = require('underscore'),
    async   = require('async');

// create a service constructor for very easy API wrappers a la HTTParty...
var PopItClient = restler.service(
  function() {
    this.defaults.username = 'evdb@ecclestoad.co.uk';
    this.defaults.password = 'ctzfgVHG';
  },
  {
    baseURL: 'http://billit.127-0-0-1.org.uk:3000/api/v1/',

    // need this so that we can send the body as a JSON string - postJson does
    // not seem to be working with a service :(
    headers: {'Content-Type': 'application/json' },
  }
);

var popit_api = new PopItClient();


// Entries in the CSV that have not been added to db yet
// 'profesion', 'Distrito', 'pacto', 'voto_nro', 'carrera', 'establecimiento',
// 'circunscripcion', 'region', 'sigla_region',

// store all the entries in hashes keyed on a name (assuming no duplicates)
var persons       = {};
var positions     = [];
var organisations = {};


csv()
  .fromPath(__dirname+'/input.csv', {columns: true})
  .on('data',function(data, index){

    // console.log( data );
      
    var person = {
      name: data['nombre completo UpperCase'],

      // profile: {
      //   gender: data.sexo,
      //   date_of_birth: data.fecha_nacimiento,
      // },
      
      // names: [
      //   {
      //     name: data['nombre 1'],
      //     comment: "name used as author in bills",
      //   },
      // ],
      
      images: [
        { url: data.foto },
      ],
    };

    // don't bother if we don't have a name - probably a blank line in csv
    if ( ! person.name) return;

    // create the contact details
    var contact_details = _.filter(
      [
        { kind: "Email", value: data.mail ,}
      ],
      function (detail) { return detail.kind && detail.value; }
    );
    person.contact_details = contact_details;
    
    // create all the links
    var person_links = _.filter(
      [
        { "comment": "Website", "url": data.web },
        { "comment": "Declaration Interes", "url": data.declaracion_interes },
        { "comment": "Declaration Patrimonio", "url": data.declaracion_patrimonio },
      ],
      function (link) { return link.url; }
    );
    if ( data.twitter ) {
      person_links.push({ "comment": "Twitter", "url": 'http://twitter.com/' + data.twitter });
    }
    if ( data.facebook) {
      person_links.push({ "comment": "Facebook", "url": 'http://facebook.com/' + data.facebook });
    }
    person.links = person_links;
    
    // store the person for later sending to the API
    persons[person.name] = person;
    

    // if not an independent add party membership
    if (data.partido_sigla != 'IND') {

      organisations[data.Partido] = { name: data.Partido };

      positions.push({
        title:        'Party Member',
        organisation: data.Partido, 
        person:       person.name,
      });
    }

    organisations['Government'] = { name: 'Government' };
    positions.push({
      title:         data.senador_diputado,
      organisation: 'Government', // possibly this could be something better
      person:        person.name,
    });


  })
  .on('error',function(error){
    console.log(error.message);
  })
  .on('end', send_data_to_api);


function send_data_to_api () {
  
  // Go through all the persons and enter them, then the orgs, then the positions
  async.series(
    [
      send_people_to_api,
      send_organisations_to_api,
      send_positions_to_api,
    ],
    function (err) {
      if (err) throw err;
      console.log("ALL DONE");
    }
  );  
}

function post_to_api ( path, doc, cb ) {

  console.log('POSTING to %s', path);
  // console.log(doc);

  popit_api
    .post( path, { data: JSON.stringify(doc) } )
    .on('complete', function(data, response) {
      // console.log(data);
      // console.log(data.result._id);
      // console.log('request complete');
      cb(null, data.result._id);
    })
    .on('fail', function(data, response) {
      cb(data, null);
    });      
}

function send_people_to_api (sending_done) {
  async.forEachSeries(
    _.values(persons),
    function ( person, done ) {
      post_to_api('person', person, function (err, id) {
        person.id = id;
        done(err);
      });
    },
    sending_done
  );
}

function send_organisations_to_api (sending_done) {
  async.forEachSeries(
    _.values(organisations),
    function ( org, done ) {
      post_to_api('organisation', org, function (err, id) {
        org.id = id;
        done(err);
      });
    },
    sending_done
  );
}

function send_positions_to_api (sending_done) {
  async.forEachSeries(
    positions,
    function ( position, done ) {

      // replace the person and org with the correct object id
      position.person       = persons[position.person].id;
      position.organisation = organisations[position.organisation].id;

      post_to_api('position', position, done);
    },
    
    sending_done
  );
}