Last active
August 29, 2015 14:03
-
-
Save andrewrk/ccbca312c2df8a4db204 to your computer and use it in GitHub Desktop.
aws sdk ETag does not match MD5 checksum
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var AWS = require('aws-sdk'); | |
var fs = require('fs'); | |
var crypto = require('crypto'); | |
var client = new AWS.S3({ | |
accessKeyId: process.env.S3_KEY, | |
secretAccessKey: process.env.S3_SECRET, | |
}); | |
var s3Path = "1tmp1/node_modules/chem-cli/node_modules/watchify2/node_modules/browserify/node_modules/browser-builtins/node_modules/http-browserify/example/json-stream/node_modules/JSONStream/test/fixtures/all_npm.json"; | |
var localFile = "test_out.json"; | |
var s3Params = { | |
Bucket: process.env.S3_BUCKET, | |
Key: s3Path, | |
}; | |
var count = 1; | |
downloadOnce(); | |
function downloadOnce() { | |
doTheDownload(function(err) { | |
if (err) throw err; | |
console.log("downloaded", count++); | |
downloadOnce(); | |
}); | |
} | |
function doTheDownload(cb) { | |
var request = client.getObject(s3Params); | |
var response = request.createReadStream(); | |
var outStream = fs.createWriteStream(localFile); | |
var hash = crypto.createHash('md5'); | |
var errorOccurred = false; | |
var eTag = ""; | |
response.on('error', handleError); | |
outStream.on('error', handleError); | |
request.on('httpHeaders', function(statusCode, headers, resp) { | |
if (statusCode < 300) { | |
var contentLength = parseInt(headers['content-length'], 10); | |
eTag = headers.etag || ""; | |
} else { | |
handleError(new Error("http status code " + statusCode)); | |
} | |
}); | |
hash.on('data', function(digest) { | |
if (!compareETag(eTag, digest)) { | |
console.log("eTag", eTag, "digest", digest.toString('hex'), "path:", localFile); | |
handleError(new Error("ETag does not match MD5 checksum")); | |
} | |
}); | |
outStream.on('close', function() { | |
if (errorOccurred) return; | |
cb(); | |
}); | |
response.pipe(outStream); | |
response.pipe(hash); | |
function handleError(err) { | |
if (errorOccurred) return; | |
errorOccurred = true; | |
cb(err); | |
} | |
} | |
function compareETag(eTag, md5Buffer) { | |
eTag = eTag.replace(/^\s*'?\s*"?\s*(.*?)\s*"?\s*'?\s*$/, "$1"); | |
var hex = md5Buffer.toString('hex'); | |
return eTag === hex; | |
} | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment