Created
November 13, 2015 07:59
-
-
Save michaelwittig/7394988bca1dfe7e011b to your computer and use it in GitHub Desktop.
Delete all keys / versions from S3 bucket without API throtteling (flow control)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var MAX_CONCURRENT_API_CALLS = 12; | |
var BUCKET = "bucket-name"; | |
var AWS = require("aws-sdk"); | |
var s3 = new AWS.S3(); | |
var async = require("async"); | |
var queue = async.queue(function(task, cb) { | |
console.log(task.action); | |
if (task.action === "list") { | |
s3.listObjectVersions({ | |
"Bucket": BUCKET, | |
"KeyMarker": task.nextKeyMarker, | |
"VersionIdMarker": task.nextVersionIdMarker | |
}, function(err, data) { | |
if (err) { | |
cb(err); | |
} else { | |
var tasks = data.Versions.map(function(version) { | |
return {"action": "delete", "key": version.Key, "versionId": version.VersionId}; | |
}); | |
queue.push(tasks); | |
if (data.IsTruncated === true) { | |
queue.push({"action": "list", "nextKeyMarker": data.NextKeyMarker, "nextVersionIdMarker": data.NextVersionIdMarker}); | |
} | |
cb(); | |
} | |
}); | |
} else if (task.action === "delete") { | |
s3.deleteObject({ | |
"Bucket": BUCKET, | |
"Key": task.key, | |
"VersionId": task.versionId | |
}, cb); | |
} else { | |
cb(new Error("unsupported action " + task.action)); | |
} | |
}, MAX_CONCURRENT_API_CALLS); | |
queue.push({"action": "list"}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment