Socket Hang up problem - A sample to generate the problem.

test-simple-http-data.js

/*
 * This is a simple HTTP data generator for testing.
 *
 */

var http = require('http');

var counter = 0;
http.createServer(function (req, res) {
    var start = new Date();
    var myCounter = counter++;

    var timeout = 50;	// default timeout.  Mimic how much time it takes to run a process.
    
    // Controlled long response time causing timeout in client side. 
    if ( (myCounter % 20000) == 0) {
        console.log('-------- reach 20000 ------------');
        timeout = 600000;	// 10 minutes
    }

    // give it some timeout
    setTimeout(function() {
        var output = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n';

                output += '<myData>ABCDE'+myCounter+'</myData>\n';
                
        console.log("output: " + myCounter);
        
        res.writeHead(200, {'Content-Type': 'application/xml'});
        res.write(output);
        res.end();
    }, timeout);


}).listen(3015);

console.log('Server running at port 3015');

test-simple-http-server.js

var http = require('http');

// option1 is the from Google.  GoogleAPI usually has less timeout and causing less issue. 

//https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=Paris%20Hilton&callback=foo&context=bar
var agent1 = http.getAgent('ajax.googleapis.com',80);
agent1.maxSockets = 2000;

var options1 = {
	agent:agent1,
    host: 'ajax.googleapis.com',
    port: 80,
    path: '/ajax/services/search/web?v=1.0&q=Paris%20Hilton&callback=foo&context=bar',
    method: 'GET',
    headers:{
		"Connection":"keep-alive"
	}
};

// option2 is your own server that feeds data.
var agent2 = http.getAgent('localhost', 80);
agent2.maxSockets = 2000;
var options2 = {
		agent:agent2,
		host:'localhost',
		port:80,
		path:'/',
		method:'GET',
		headers:{
				"Connection":"keep-alive"
			}
	};

// option3 is my AWS micro instance.   It has controlled timeout for ever 20000 request.
var agent3 = http.getAgent('50.19.237.122', 3015);
agent3.maxSockets = 2000;
var options3 = {
		agent:agent3,
		host:'50.19.237.122',
		port:3015,
		path:'/',
		method:'GET',
		headers:{
				"Connection":"keep-alive"
			}	
	};

// set which 
var workingOption = options3;

var counter = 0;

var server = http.createServer(function (req, res) {
	var myCounter = counter ++;
	console.log("Start: counter: " + myCounter);
	
    var start = new Date();	// just for timing.
    
    var clientRequest = http.request(workingOption, function(response) {
        
    	var result = '';
        response.setEncoding('utf8');
        res.writeHead(response.statusCode, {'Content-Type': 'application/xml'});
        response.on('data', function (chunk) {
        	result += chunk;
            if (!res.write(chunk)) { // if write failed, the stream is choking
                response.pause(); // tell the incoming stream to wait until output stream drained
            }
        }).on('end', function () {
            var end = new Date();
            console.log("End: counter: " + myCounter + ' Time: ' + (end-start) + " chunk: " + result.substring(0, 20));
            res.end();
        });
    });
    clientRequest.on('error', function(e) {
    	console.error("ERROR: " + JSON.stringify(e));
    	console.log("ERROR: " + JSON.stringify(e));
    });
    
    
    clientRequest.end();
});

server.listen(3204);

console.log('Server running at port 3204');

var http = require('http');

var query = 'v=12345';

var agent = http.getAgent('myremote.com', 80);
agent.maxSockets = 10;

var options = {
agent:agent,
host:'myremote.com',
port:80,
path:'/getcall?'+query,
method:'POST',
headers:{
"host":'myremote.com',
"user-agent": 'node.js',
"Connection":"keep-alive",
"Keep-Alive": "timeout=10, max=10"
}
};

var callMe = function(counter) {
var client = http.request(options,
function(res) {
console.log(res.statusCode + " : " + counter);
console.log("sock: "+ counter + " :lenght:" + agent.sockets.length);

            if (agent.sockets.length>(agent.maxSockets/10*9)) {
                console.log("keep: "+ counter + " :keepalive is false:");
                client.shouldKeepAlive = false;
            }
            res.on('data', function(chunk) {
                console.log("data: " + counter + " " + chunk);
                //client.destroy();
            });
            res.on('end', function() {
                console.log("done: " + counter + ":socket:" + agent.sockets.length + " :queue:" + agent.queue.length 
                            + ' :finished:' + client.finished
                            + ' :_last:' + client._last
                            + ' :writable:' + client.writable);
                console.log(client.socket);

            });
          }
);
client.end();

}
var index = 0;
setInterval(function() {
console.log("Call Set Interval");
for (var i=0; i<100;index++, i++ ) {
if (index <= 300) {
callMe(index);
}
}
}, 10000);

var cappedArray = [];
var cappedMax = 10;

var allSocketsCleanup = function () {
for (var i=0; i< agent.sockets.length; i++) {
console.log("destroying");
if (!agent.sockets[i]._httpMessage) {
agent.sockets[i].destroy();
}
}
};

setInterval(function() {
console.log("check agent.sockets.length" + agent.sockets.length);
var socketLength = agent.sockets.length;

if (cappedArray.length <= cappedMax) {
    cappedArray.push(socketLength);
    if (socketLength >= agent.maxSockets) {
        allSocketsCleanup();
    }
} else {
    cappedArray.push(socketLength);
    var prevSocketLength = cappedArray.shift();

    if (prevSocketLength == socketLength) {
        allSocketsCleanup();
        console.log('current length ' + agent.sockets.length);
    }
}

},500);

for the above code, it works on the original 0.4.9 http.js not the fix one. (not sure about the http-fix.js has the same socket problem yet).

key point:

1. for http connection that close to the 90% of the maxium socket, I switch it to "Connection":"close" in order to avoid connection being max out & keep-alive.
2. I have a routine to constantly check the # of sockets. If the # of sockets do not go down, or max out, then I will start force destroying socket (that have no httpMessage).

I have run some test and haven't seen any error thrown. and I can see the # of socket go back down.

Use this http.js and https.js will fix the problem.

• copy them to your nodejs/lib folder. replace the old ones.
• rebuild node again.