Skip to content

Instantly share code, notes, and snippets.

@andrewrk
Last active August 29, 2015 14:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save andrewrk/ccbca312c2df8a4db204 to your computer and use it in GitHub Desktop.
Save andrewrk/ccbca312c2df8a4db204 to your computer and use it in GitHub Desktop.
aws sdk ETag does not match MD5 checksum
var AWS = require('aws-sdk');
var fs = require('fs');
var crypto = require('crypto');
var client = new AWS.S3({
accessKeyId: process.env.S3_KEY,
secretAccessKey: process.env.S3_SECRET,
});
var s3Path = "1tmp1/node_modules/chem-cli/node_modules/watchify2/node_modules/browserify/node_modules/browser-builtins/node_modules/http-browserify/example/json-stream/node_modules/JSONStream/test/fixtures/all_npm.json";
var localFile = "test_out.json";
var s3Params = {
Bucket: process.env.S3_BUCKET,
Key: s3Path,
};
var count = 1;
downloadOnce();
function downloadOnce() {
doTheDownload(function(err) {
if (err) throw err;
console.log("downloaded", count++);
downloadOnce();
});
}
function doTheDownload(cb) {
var request = client.getObject(s3Params);
var response = request.createReadStream();
var outStream = fs.createWriteStream(localFile);
var hash = crypto.createHash('md5');
var errorOccurred = false;
var eTag = "";
response.on('error', handleError);
outStream.on('error', handleError);
request.on('httpHeaders', function(statusCode, headers, resp) {
if (statusCode < 300) {
var contentLength = parseInt(headers['content-length'], 10);
eTag = headers.etag || "";
} else {
handleError(new Error("http status code " + statusCode));
}
});
hash.on('data', function(digest) {
if (!compareETag(eTag, digest)) {
console.log("eTag", eTag, "digest", digest.toString('hex'), "path:", localFile);
handleError(new Error("ETag does not match MD5 checksum"));
}
});
outStream.on('close', function() {
if (errorOccurred) return;
cb();
});
response.pipe(outStream);
response.pipe(hash);
function handleError(err) {
if (errorOccurred) return;
errorOccurred = true;
cb(err);
}
}
function compareETag(eTag, md5Buffer) {
eTag = eTag.replace(/^\s*'?\s*"?\s*(.*?)\s*"?\s*'?\s*$/, "$1");
var hex = md5Buffer.toString('hex');
return eTag === hex;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment