Node fails to download entire (large) file via http/https when `agent: false` is used. Fails on 0.10, 0.12, 4.1.1. See related https://gist.github.com/trentm/80409ded30b4e0418e9c with an equivalent script that uses 'data' events instead of `.pipe`.

download-with-pipe-sans-agent.js

/*
 * Download a file using basic `http[s].request` with `agent: false`.
 *
 * Usage:
 *      node download-sans-agent.js URL OUTPUT-FILE
 *
 * Example:
 *      node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file
 *
 * Running ^^ in a loop until if fails:
 *
 *      while true; do echo "";
 *          node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file
 *          retval=$?
 *          \ls -l out.file
 *          [[ $retval == 0 ]] || break;
 *      done
 *
 * Or as a one-liner:
 *
 *      while true; do echo ""; node download-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done
 */

var assert = require('assert');
var fs = require('fs');
var https = require('https');
var http = require('http');
var mod_url = require('url');
var util = require('util');


function fatal(/* args ... */) {
    console.log('download-sans-agent fatal error:',
        util.format.apply(null, arguments));
    process.exit(1);
}


// --- mainline

if (process.argv.length !== 4) {
    console.error('Usage: node download-sans-agent.js URL OUTPUT-FILE');
    process.exit(2);
}
var urlStr = process.argv[2];
var outFile = process.argv[3];

// Parse the URL, default to 'http://' prefix if not given.
if (! /^https?:\/\//.test(urlStr)) {
    urlStr = 'http://' + urlStr;
}
var url = mod_url.parse(urlStr);
assert.ok(['https:', 'http:'].indexOf(url.protocol) !== -1,
    'unexpected URL protocol: ' + url.protocol);

console.log('Downloading %s to %s', urlStr, outFile);
var proto = url.protocol === 'https:' ? https : http;
var req = proto.request({
    method: 'GET',
    hostname: url.hostname,
    port: url.port,
    path: url.path,
    // XXX
    agent: false
}, function (res) {
    if (res.statusCode !== 200) {
        fatal('unexpected status code: ' + res.statusCode);
    }

    var expectedBytes = Number(res.headers['content-length']);
    console.log('Expected content-length: %d', expectedBytes);

    var numBytes = 0;
    var outStream = fs.createWriteStream(outFile);

    res.on('error', function (err) {
        fatal('response error: ' + err);
    });
    outStream.on('error', function (err) {
        fatal('outStream error: ' + err);
    });

    res.pipe(outStream);

    outStream.on('finish', function () {
        numBytes = fs.statSync(outFile).size;
        console.log('Finished writing to "%s" (%d bytes)', outFile, numBytes);
        if (numBytes !== expectedBytes) {
            fatal('Incorrect number of bytes downloaded: expected %d, ' +
                'downloaded %d (missing %d bytes)', expectedBytes,
                numBytes, expectedBytes - numBytes);
        }
    });
});

req.on('error', function (err) {
    fatal('request error: ' + err);
});

req.end();

fails-with-node-4.1.1.out

$ node -v
v4.1.1



$ while true; do echo ""; node download-with-pipe-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done

Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file
Expected content-length: 9544550
Finished writing to "out.file" (9544550 bytes)
-rw-r--r--  1 trentm  staff  9544550 Sep 24 12:19 out.file

Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file
Expected content-length: 9544550
Finished writing to "out.file" (8941323 bytes)
download-sans-agent fatal error: Incorrect number of bytes downloaded: expected 9544550, downloaded 8941323 (missing 603227 bytes)
-rw-r--r--  1 trentm  staff  8941323 Sep 24 12:19 out.file



$ while true; do echo ""; node download-with-pipe-sans-agent.js https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz out.file; retval=$?; \ls -l out.file; [[ $retval == 0 ]] || break; done

Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file
Expected content-length: 9544550
Finished writing to "out.file" (9544550 bytes)
-rw-r--r--  1 trentm  staff  9544550 Sep 24 12:21 out.file

Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file
Expected content-length: 9544550
Finished writing to "out.file" (9544550 bytes)
-rw-r--r--  1 trentm  staff  9544550 Sep 24 12:21 out.file

Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file
Expected content-length: 9544550
Finished writing to "out.file" (9544550 bytes)
-rw-r--r--  1 trentm  staff  9544550 Sep 24 12:21 out.file

Downloading https://nodejs.org/dist/v4.1.1/node-v4.1.1-darwin-x64.tar.gz to out.file
Expected content-length: 9544550
Finished writing to "out.file" (8793867 bytes)
download-sans-agent fatal error: Incorrect number of bytes downloaded: expected 9544550, downloaded 8793867 (missing 750683 bytes)
-rw-r--r--  1 trentm  staff  8793867 Sep 24 12:21 out.file