Skip to content

Instantly share code, notes, and snippets.

@camilosampedro
Last active July 17, 2019 01:25
Show Gist options
  • Save camilosampedro/cdc6a568c58ed9e5e8c326c6606adcbb to your computer and use it in GitHub Desktop.
Save camilosampedro/cdc6a568c58ed9e5e8c326c6606adcbb to your computer and use it in GitHub Desktop.
// https://stackoverflow.com/a/33755463/4474204
// https://stackoverflow.com/a/33755463/4474204
var fs = require("fs");
var parse = require("csv-parse");
var async = require("async");
var AWS = require("aws-sdk");
var csv_filename = "data.csv";
rs = fs.createReadStream(csv_filename);
var ddb = new AWS.DynamoDB();
parser = parse(
{
columns: true,
delimiter: ","
},
function(err, data) {
var split_arrays = [],
size = 25;
while (data.length > 0) {
console.log("Pushing data");
split_arrays.push(data.splice(0, size));
}
split_arrays = split_arrays.map(l => {
return l.map(d => {
return {
PutRequest: {
Item: {
email: {
S: d.email
},
registeredDate: {
S: d.registeredDate
},
country: {
S: d.country
},
entryCode: {
S: d.entryCode
},
status: {
S: d.status
}
}
}
};
});
});
data_imported = false;
chunk_no = 1;
async.each(
split_arrays,
function(item_data, callback) {
console.log("item_data", item_data);
console.log("callback", callback);
ddb.batchWriteItem(
{
RequestItems: {
"applicants-2019-07-16": item_data
}
},
function(err, res, cap) {
console.log("done going next");
if (err == null) {
console.log("Success chunk #" + chunk_no);
data_imported = true;
} else {
console.log(err);
console.log("Fail chunk #" + chunk_no);
data_imported = false;
}
chunk_no++;
callback();
}
);
},
function() {
// run after loops
console.log("all data imported....");
}
);
}
);
rs.pipe(parser);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment