Test s3 read after write consistency

index.js

#!/usr/bin/env node

const AWS = require('aws-sdk');
const http = require('http');


//Data from TOKIO region
//Upload data
const upload_tokio = {
    bucketName: 'XX',
    bucketRegion: 'ap-northeast-1',
    basePath: 'ZZ/ap-northeast-1/media_' + (new Date()).getTime().toString() +"_"
};

//Data from US-WEST-2 region
//Upload data
const upload_oregon = {
    bucketName: 'XX',
    bucketRegion: 'us-west-2',
    basePath: 'ZZ/us-west-2/media_' + (new Date()).getTime().toString() +"_"
};

//Set updload data
const upload_data = upload_tokio;

//Obj Data
const objData = Buffer.alloc(1024 * 1024);

//Download data
const donwload_base_url = 'http://playback.bcovlive.io';

//Upload delay
const upload_delay_ms = 4 * 1000;

//
const verbose = 0;

//RUN THE TEST
let n = 0;

test_consistency(n);


//FUNCTIONS

function test_consistency (num) {

    //Create new S3 connection
    let s3 = createNewS3(upload_data.bucketRegion);

    let path_obj = upload_data.basePath + num.toString();

    let start_upload_ms = (new Date()).getTime();

    upload(s3, upload_data.bucketName, path_obj, objData, function (err, data) {
        if (err) {
            consoleOut('error', err);

            //Try again
            setTimeout(test_consistency, upload_delay_ms, n++);
        }
        let end_upload_ms = (new Date()).getTime();
        let start_download = end_upload_ms;

        if (verbose > 0)
            consoleOut('log', 'Uploaded: ' + path_obj + '. In: ' + (end_upload_ms - start_upload_ms) + 'ms');

        let url = donwload_base_url + '/' + path_obj;

        //Download
        getaDattpGet(url, function (err, data){
            if (err)
                consoleOut('error', err);

            let end_download = (new Date()).getTime();

            if (verbose > 0)
                consoleOut('log', 'Downloaded: ' + path_obj + '. In: ' + (end_download - start_download) + 'ms');

            if ((verbose <= 0) && ((n % 10) === 0))
                consoleOut('log', 'Uploaded & Downloaded successfully: ' + n.toString() + ' times');

            //Continue
            setTimeout(test_consistency, upload_delay_ms, n++);
        });

    })
}

function consoleOut(type, message) {
    let msg = (new Date()).toISOString() + ' ' + message;
    if (type === 'error')
        console.error(msg);
    else
        console.log(msg);
}

function upload (s3, bucketName, objPath, objData, callback) {
    const s3_params = {Bucket: bucketName, Key: objPath, Body: objData};

    s3.upload(s3_params, function(err, data) {
        callback(err, data);
    });
}

function createNewS3(region) {

    var s3_params = {
        apiVersion: '2006-03-01',
        signatureVersion: 'v4'
    };

    if ((typeof (region) === 'string') && (region.trim() != ""))
        s3_params['region'] = region; //From http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property

    //To run in the new AWS regions we need to force signature V4
    return new AWS.S3(s3_params);
}

function getaDattpGet(url, callback) {

    return http.get(url, function(response) {
        // Continuously update stream with data
        let body = '';
        response.on('data', function(d) {
            body += d;
        });
        response.on('end', function() {
            if (response.statusCode < 300) {// OK
                // Data reception is done, do whatever with it!
                callback(null, body);
            }
            else {
                let str = "Code: " + response.statusCode;
                str = str + "\nURL: " + url;
                str = str + "\nHeaders: " + JSON.stringify(response.headers);
                str = str + "\nBody: " + body;

                var err = new Error(str);
                callback(err, body);
            }
        });
        response.on('error', function (err) { //HTTP ERROR
            callback(err, body);
        });
    });
}