EasyPost tracker history to CSV

easypost-scrape.js

const async = require('async');
const Easypost = require('node-easypost');
const csv = require('fast-csv');
const fs = require('fs');
const stream = require('stream');


var start_date = new Date('2017-03-01');
var easypost_key = 'fixme';

var csvStream = csv.format({headers: true, quoteColumns: true, quoteHeaders: true});
var writeStream = fs.createWriteStream('./shipments.csv');
writeStream.on('finish', function() {
        console.log('(write stream finished)');
});
csvStream.on('finish', function() {
    console.log('(CSV stream finished)');
});
csvStream.pipe(writeStream);
csvStream.write([
        "id",
        "created",
        "carrier",
        "estimate delivery",
        "status",
        "updated",
        "update message",
        "city",
        "state",
        "zip"
        ]);

var excelDate = function(date) {
    var d = new Date(date);
    return ('00' + (d.getMonth()+1)).slice(-2) + '-'
        + ('00' + d.getDate()).slice(-2) + '-'
        + d.getFullYear() + 'T'
        + ('00' + d.getHours()).slice(-2) + ':'
        + ('00' + d.getMinutes()).slice(-2) + ':'
        + ('00' + d.getSeconds()).slice(-2);
}

function print_result(tracker) {
    console.log(tracker.id + '   ' + tracker.created_at);
    tracker.tracking_details.forEach(function(details) {
        csvStream.write([
            tracker.id,
            excelDate(tracker.created_at),
            tracker.carrier,
            excelDate(tracker.est_delivery_date),
            details.status,
            excelDate(details.datetime),
            details.message,
            details.tracking_location.city,
            details.tracking_location.state,
            details.tracking_location.zip
        ]);
    });

}


const api = new Easypost(easypost_key);
api.Tracker.all({
  page_size: 100
},
function(err,results) {
    var last_updated;
    var lastid;
    results.trackers.forEach(function(tracker) {
        print_result(tracker);
        lastid = tracker.id;
        last_updated = tracker.updated_at;
    });
    console.log('\nloop ended at ' + last_updated + '\n');

    async.whilst(
        function() {
            return( new Date(last_updated) > start_date );
        },
        function(callback) {
            console.log('fetch');
            api.Tracker.all({
                page_size: 100,
                before_id: lastid,
                //start_datetime: '2017-02-01T00:00:00Z'
            },function(err,results) {
                if( err || !results || results.trackers.length === 0 ) {
                    lastid = undefined;
                    console.log('empty results');
                    console.log(err);
                    console.log(results);
                    callback(1);
                } else {
                    results.trackers.forEach(function(tracker) {
                        print_result(tracker);

                        last_updated = tracker.updated_at;
                        lastid = tracker.id;
                    });
                    console.log('\nloop ended at ' + last_updated + '\n');
                    callback(null);
                }
            });
        },
        function(err,n) {
            console.log('\ndone.');
            console.log(err);
            console.log(n);
            console.log('\n');
            csvStream.end();
        }
    )
});