-
-
Save trentm/80409ded30b4e0418e9c to your computer and use it in GitHub Desktop.
Node fails to download entire (large) file via http/https when `agent: false` is used. Fails on 0.10, 0.12, 4.1.1.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /* | |
| * Download a file using basic `http[s].request` with `agent: false`. | |
| * | |
| * Usage: | |
| * node download-sans-agent.js URL OUTPUT-FILE | |
| * | |
| * Example: | |
| * node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file | |
| * | |
| * Running ^^ in a loop until if fails: | |
| * | |
| * while true; do echo ""; | |
| * node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file | |
| * retval=$? | |
| * \ls -l out.file | |
| * [[ $retval == 0 ]] || break; | |
| * done | |
| * | |
| * Or as a one-liner: | |
| * | |
| * while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done | |
| */ | |
| var assert = require('assert'); | |
| var fs = require('fs'); | |
| var https = require('https'); | |
| var http = require('http'); | |
| var mod_url = require('url'); | |
| var util = require('util'); | |
| function fatal(/* args ... */) { | |
| console.log('download-sans-agent fatal error:', | |
| util.format.apply(null, arguments)); | |
| process.exit(1); | |
| } | |
| // --- mainline | |
| if (process.argv.length !== 4) { | |
| console.error('Usage: node download-sans-agent.js URL OUTPUT-FILE'); | |
| process.exit(2); | |
| } | |
| var urlStr = process.argv[2]; | |
| var outFile = process.argv[3]; | |
| // Parse the URL, default to 'http://' prefix if not given. | |
| if (! /^https?:\/\//.test(urlStr)) { | |
| urlStr = 'http://' + urlStr; | |
| } | |
| var url = mod_url.parse(urlStr); | |
| assert.ok(['https:', 'http:'].indexOf(url.protocol) !== -1, | |
| 'unexpected URL protocol: ' + url.protocol); | |
| console.log('Downloading %s to %s', urlStr, outFile); | |
| var proto = url.protocol === 'https:' ? https : http; | |
| var req = proto.request({ | |
| method: 'GET', | |
| hostname: url.hostname, | |
| port: url.port, | |
| path: url.path, | |
| // XXX | |
| agent: false | |
| }, function (res) { | |
| if (res.statusCode !== 200) { | |
| fatal('unexpected status code: ' + res.statusCode); | |
| } | |
| var expectedBytes = Number(res.headers['content-length']); | |
| console.log('Expected content-length: %d', expectedBytes); | |
| var numBytes = 0; | |
| var outStream = fs.createWriteStream(outFile); | |
| res.on('error', function (err) { | |
| fatal('response error: ' + err); | |
| }); | |
| outStream.on('error', function (err) { | |
| fatal('outStream error: ' + err); | |
| }); | |
| res.on('data', function (chunk) { | |
| numBytes += chunk.length; | |
| outStream.write(chunk); | |
| }) | |
| res.on('end', function () { | |
| console.log('Note: response "end" event'); | |
| outStream.end(); | |
| }); | |
| outStream.on('close', function () { | |
| console.log('Note: outStream "close" event'); | |
| }) | |
| outStream.on('finish', function () { | |
| console.log('Finished writing to "%s" (%d bytes)', outFile, numBytes); | |
| if (numBytes !== expectedBytes) { | |
| fatal('Incorrect number of bytes downloaded: expected %d, ' + | |
| 'downloaded %d (missing %d bytes)', expectedBytes, | |
| numBytes, expectedBytes - numBytes); | |
| } | |
| }); | |
| }); | |
| req.on('error', function (err) { | |
| fatal('request error: ' + err); | |
| }); | |
| req.end(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ node --version | |
| v0.10.40 | |
| $ while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:05 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9076491 bytes) | |
| download-sans-agent fatal error: Incorrect number of bytes downloaded: expected 9544550, downloaded 9076491 (missing 468059 bytes) | |
| -rw-r--r-- 1 trentm staff 9076491 Sep 24 11:05 out.file | |
| $ while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (8986379 bytes) | |
| download-sans-agent fatal error: Incorrect number of bytes downloaded: expected 9544550, downloaded 8986379 (missing 558171 bytes) | |
| -rw-r--r-- 1 trentm staff 8986379 Sep 24 11:07 out.file | |
| $ while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9039627 bytes) | |
| download-sans-agent fatal error: Incorrect number of bytes downloaded: expected 9544550, downloaded 9039627 (missing 504923 bytes) | |
| -rw-r--r-- 1 trentm staff 9039627 Sep 24 11:07 out.file | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ node --version | |
| v4.1.1 | |
| $ while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:33 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:33 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (8438895 bytes) | |
| download-sans-agent fatal error: Incorrect number of bytes downloaded: expected 9544550, downloaded 8438895 (missing 1105655 bytes) | |
| -rw-r--r-- 1 trentm staff 8438895 Sep 24 11:33 out.file | |
| $ while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:36 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (8232715 bytes) | |
| download-sans-agent fatal error: Incorrect number of bytes downloaded: expected 9544550, downloaded 8232715 (missing 1311835 bytes) | |
| -rw-r--r-- 1 trentm staff 8232715 Sep 24 11:36 out.file |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:10 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:11 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:11 out.file | |
| Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file | |
| Expected content-length: 9544550 | |
| Note: response "end" event | |
| Finished writing to "out.file" (9544550 bytes) | |
| Note: outStream "close" event | |
| -rw-r--r-- 1 trentm staff 9544550 Sep 24 11:11 out.file | |
| ... |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment