Skip to content

Instantly share code, notes, and snippets.

@lusentis
Created December 13, 2016 16:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save lusentis/dd14c626195067d182d3bbe510be3bb3 to your computer and use it in GitHub Desktop.
Save lusentis/dd14c626195067d182d3bbe510be3bb3 to your computer and use it in GitHub Desktop.
lambda (draft) to replicate a dynamodb table to S3
export function replicator (replicatorInternalId, sourceLogicalName, destinationPhysicalName, destinationArn) {
if (replicatorInternalId === false) {
return {};
}
const resourcesSuffix = sourceLogicalName[0].toUpperCase() + sourceLogicalName.slice(1) + '' + replicatorInternalId;
const lambdaInlineCode = `
'use strict';
console.log('Loading function');
const destinationDynamoRegion = '${destinationArn.split(':')[3]}';
const AWS = require('aws-sdk');
const dynamo = new AWS.DynamoDB({ region: destinationDynamoRegion });
const s3 = new AWS.S3({});
const unwrapFirstValue = obj => {
const firstKey = Object.keys(obj)[0];
return obj[firstKey];
};
exports.handler = (event, context, callback) => {
console.log('Received event:', JSON.stringify(event, null, 2));
return Promise.all(event.Records.map(record => {
if (record.eventName !== 'MODIFY' && record.eventName !== 'INSERT') {
return;
}
const image = record.dynamodb.NewImage;
const key = record.dynamodb.Keys;
const updates = {};
const s3Key = Object.keys(key).map(k => unwrapFirstValue(key[k])).join('/');
return s3.putObject({
Bucket: '\u0024{BucketReplicator}',
Key: '${resourcesSuffix}/' + s3Key + '.json',
Body: JSON.stringify(image, null, 2)
}).promise()
.then(() => {
Object.keys(image).forEach(attributeName => {
if (Object.keys(key).indexOf(attributeName) !== -1) {
// skip the primary key
return;
}
updates[attributeName] = {
Action: 'PUT',
Value: image[attributeName]
};
});
const updateParams = {
TableName: '${destinationPhysicalName}',
Key: key,
AttributeUpdates: updates,
};
console.log('updateParams', updateParams);
return dynamo.updateItem(updateParams).promise();
});
}))
.then(result => {
console.log('final result', result);
callback(null, 'Successfully processed', event.Records.length, 'records.');
})
.catch(err => {
console.log('Error running replicator:', err.message, err.stack);
callback(err.message);
});
};
`;
const roleTemplate = {
[`LambdaRoleForReplicator${resourcesSuffix}`]: {
'Type': 'AWS::IAM::Role',
'Properties': {
'AssumeRolePolicyDocument': {
'Version': '2012-10-17',
'Statement': [{
'Effect': 'Allow',
'Principal': {
'Service': ['lambda.amazonaws.com']
},
'Action': ['sts:AssumeRole']
}]
},
'Path': '/',
'Policies': [{
'PolicyName': 'dawson-policy',
'PolicyDocument': {
'Version': '2012-10-17',
'Statement': [{
'Effect': 'Allow',
'Action': [
'dynamodb:DescribeStream',
'dynamodb:GetRecords',
'dynamodb:GetShardIterator',
'dynamodb:ListStreams'
],
'Resource': [{ 'Fn::GetAtt': [sourceLogicalName, 'StreamArn'] }]
}, {
'Effect': 'Allow',
'Action': [
'dynamodb:UpdateItem'
],
'Resource': [destinationArn]
},
{
'Effect': 'Allow',
'Action': [
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents'
],
'Resource': { 'Fn::Sub': 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*' } // eslint-disable-line
},
{
'Effect': 'Allow',
'Action': [
's3:PutObject'
],
'Resource': { 'Fn::Sub': 'arn:aws:s3:::${BucketReplicator}/*' } // eslint-disable-line
}]
}
}]
}
}
};
const lambdaTemplate = {
[`LambdaReplicator${resourcesSuffix}`]: {
'Type': 'AWS::Lambda::Function',
'Properties': {
'Handler': 'index.handler',
'Role': { 'Fn::GetAtt': [`LambdaRoleForReplicator${resourcesSuffix}`, 'Arn'] },
'Code': { ZipFile: { 'Fn::Sub': lambdaInlineCode } },
'Runtime': 'nodejs4.3',
'MemorySize': 256,
'Timeout': 10
}
}
};
const eventMappingTemplate = {
[`LambdaEventSourceMappingReplicator${resourcesSuffix}`]: {
'Type': 'AWS::Lambda::EventSourceMapping',
'Properties': {
'BatchSize': 1,
'Enabled': true,
'EventSourceArn': {
'Fn::GetAtt': [sourceLogicalName, 'StreamArn']
},
'FunctionName': { 'Ref': `LambdaReplicator${resourcesSuffix}` },
'StartingPosition': 'TRIM_HORIZON'
}
}
};
const bucketTemplate = {
BucketReplicator: { // we keep one single bucket per whole account
'Type': 'AWS::S3::Bucket',
'Properties': {
'VersioningConfiguration': { 'Status': 'Enabled' },
'LifecycleConfiguration': {
'Rules': [{
'Prefix': '*',
'Status': 'Enabled',
'NoncurrentVersionExpirationInDays': 90,
'NoncurrentVersionTransition': {
'StorageClass': 'STANDARD_IA',
'TransitionInDays': 30
}
}]
}
}
}
};
return {
...roleTemplate,
...lambdaTemplate,
...eventMappingTemplate,
...bucketTemplate
};
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment