Skip to content

Instantly share code, notes, and snippets.

@issacg
Last active February 25, 2016 21:48
Show Gist options
  • Save issacg/614e27db1c070d9bcfa9 to your computer and use it in GitHub Desktop.
Save issacg/614e27db1c070d9bcfa9 to your computer and use it in GitHub Desktop.
Script to replicate vault storage from S3 to consul. Expects consul-agent on localhost:8500 and for AWS credentials to be set via environment/credential file/EC2 role
var _ = require('lodash'),
Promise = require('bluebird'),
consul = require('consul'),
logger = require('log4js').getDefaultLogger(),
AWS = require('aws-sdk');
var bucket = 'my.vault.main',
prefix = '',
region = 'us-west-2',
cprefix = 'local/vlt/vlt-main';
var s3 = new Promise.promisifyAll(new AWS.S3({
apiVersion: '2012-06-01',
region: region,
sslEnabled: true
}));
var consul = require('consul')(),
kv = Promise.promisifyAll(consul.kv);
function listObjects() {
logger.debug("Scanning bucket [" + [bucket,prefix].join("/") + "]");
var params = {
Bucket: bucket,
Prefix: prefix
};
var res = {};
var more = true;
return s3.listObjectsAsync(params).then(function(data) {
more = data.IsTruncated;
_.each(data.Contents, function(obj) {
res[obj.Key] = {Key:obj.Key,LastModified:obj.LastModified,ETag:obj.ETag,Size:obj.Size};
});
if (more) {
params.Marker = _.last(data.Contents)['Key'];
return Promise.delay(1).then(getBatch());
} else {
return Promise.resolve();
}
}).then(function() {
logger.info("" + [bucket,prefix].join("/") + " contains " + Object.keys(res).length + " keys");
return Promise.resolve(res);
}).catch(function(err) {
logger.error(err);
});
}
function clearconsul() {
logger.info("Deleting old " + cprefix + " namespace");
return kv.delAsync({key:cprefix,recurse:true}).catch(function(err) {
logger.warn(err);
return Promise.resolve();
})
}
function write2consul(keys) {
return Promise.map(_.keys(keys), function(key) {
key = keys[key];
logger.debug("Writing " + [cprefix,key.Key].join("/"));
s3.getObjectAsync({Bucket: bucket, Key: key.Key}).then(function(data) {
return kv.setAsync({key:[cprefix,key.Key].join("/"), value:data.Body});
}).catch(function(err) {return Promise.reject(err)});
}, {concurrency: 10});
}
clearconsul().then(listObjects).then(write2consul).catch(function(err) {
logger.error(err);
}).done(function() {
logger.info("Finished");
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment