Skip to content

Instantly share code, notes, and snippets.

@johnib
Last active January 25, 2023 18:30
Show Gist options
  • Star 4 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save johnib/14b090d0883ee2d2fb56dd3ac3c82f9b to your computer and use it in GitHub Desktop.
Save johnib/14b090d0883ee2d2fb56dd3ac3c82f9b to your computer and use it in GitHub Desktop.
Copy S3 data to DynamoDB
let path = require('path');
let aws = require('aws-sdk');
let s3 = new aws.S3();
let zlib = require('zlib');
let docClient = new aws.DynamoDB.DocumentClient();
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
event.Records.forEach(record => {
const params = {
Bucket: record.s3.bucket.name,
Key: record.s3.object.key
};
const isGzip = path.extname(params.Key) === ".gz";
let readStream = s3.getObject(params).createReadStream();
readStream = isGzip ? readStream.pipe(zlib.createGunzip()) : readStream;
readStream.on('data', data => {
// transform to string
// seperate lines, remember that the first / last lines could be partial of the previous / next 'data' event
const dynamoDbDocument = {
// csv parse each line
};
const dynamoDbParams = {
TableName: "Table Name",
Item: dynamoDbDocument
};
docClient.put(params, function (err, data) {
if (err) {
console.error("Unable to add item. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Added item:", JSON.stringify(data, null, 2));
}
});
});
readStream.on('end', () => {
// there are several IO operations that need to occur before calling this callback
// ending the execution this way will not work.
// use promises and wait on all of them before ending the execution
callback(null, '');
});
});
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment