Socket Hang up problem - A sample to generate the problem.

test-simple-http-data.js

/*
 * This is a simple HTTP data generator for testing.
 *
 */

var http = require('http');

var counter = 0;
http.createServer(function (req, res) {
    var start = new Date();
    var myCounter = counter++;

    var timeout = 50;	// default timeout.  Mimic how much time it takes to run a process.
    
    // Controlled long response time causing timeout in client side. 
    if ( (myCounter % 20000) == 0) {
        console.log('-------- reach 20000 ------------');
        timeout = 600000;	// 10 minutes
    }

    // give it some timeout
    setTimeout(function() {
        var output = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n';

                output += '<myData>ABCDE'+myCounter+'</myData>\n';
                
        console.log("output: " + myCounter);
        
        res.writeHead(200, {'Content-Type': 'application/xml'});
        res.write(output);
        res.end();
    }, timeout);


}).listen(3015);

console.log('Server running at port 3015');

test-simple-http-server.js

var http = require('http');

// option1 is the from Google.  GoogleAPI usually has less timeout and causing less issue. 

//https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=Paris%20Hilton&callback=foo&context=bar
var agent1 = http.getAgent('ajax.googleapis.com',80);
agent1.maxSockets = 2000;

var options1 = {
	agent:agent1,
    host: 'ajax.googleapis.com',
    port: 80,
    path: '/ajax/services/search/web?v=1.0&q=Paris%20Hilton&callback=foo&context=bar',
    method: 'GET',
    headers:{
		"Connection":"keep-alive"
	}
};

// option2 is your own server that feeds data.
var agent2 = http.getAgent('localhost', 80);
agent2.maxSockets = 2000;
var options2 = {
		agent:agent2,
		host:'localhost',
		port:80,
		path:'/',
		method:'GET',
		headers:{
				"Connection":"keep-alive"
			}
	};

// option3 is my AWS micro instance.   It has controlled timeout for ever 20000 request.
var agent3 = http.getAgent('50.19.237.122', 3015);
agent3.maxSockets = 2000;
var options3 = {
		agent:agent3,
		host:'50.19.237.122',
		port:3015,
		path:'/',
		method:'GET',
		headers:{
				"Connection":"keep-alive"
			}	
	};

// set which 
var workingOption = options3;

var counter = 0;

var server = http.createServer(function (req, res) {
	var myCounter = counter ++;
	console.log("Start: counter: " + myCounter);
	
    var start = new Date();	// just for timing.
    
    var clientRequest = http.request(workingOption, function(response) {
        
    	var result = '';
        response.setEncoding('utf8');
        res.writeHead(response.statusCode, {'Content-Type': 'application/xml'});
        response.on('data', function (chunk) {
        	result += chunk;
            if (!res.write(chunk)) { // if write failed, the stream is choking
                response.pause(); // tell the incoming stream to wait until output stream drained
            }
        }).on('end', function () {
            var end = new Date();
            console.log("End: counter: " + myCounter + ' Time: ' + (end-start) + " chunk: " + result.substring(0, 20));
            res.end();
        });
    });
    clientRequest.on('error', function(e) {
    	console.error("ERROR: " + JSON.stringify(e));
    	console.log("ERROR: " + JSON.stringify(e));
    });
    
    
    clientRequest.end();
});

server.listen(3204);

console.log('Server running at port 3204');

I look into http.js and here I think I got the bug for why socket broken / or cannot establish socket when run for long time:

Agent.prototype._establishNewConnection = function() {
var self = this;
assert(this.sockets.length < this.maxSockets); <------

So, what happen is that, when I run the node.js for long time. The # of socket in the connection pool will increase. However, overtime, the # of sockets in the pool will not drop back to what the initial number is.

Therefore, if there are lot of requests, and it reach the maximum, new request cannot establish the connection. In fact, this is normal.

HOWEVER, if there are no more request (silence) for a while, and the # of socket stay at the maxium socket allowed or even above, then there is no way to establish new connection even the sockets in the pool are not doing anything. so, the node.js server will stuck unless restarting server.

May be need to run self._cycle(); before the assert?

I dig deeper.. and the code doesn't stop at assert(this.sockets.length < this.maxSockets); <------
when #of sockets >= this.maxSockets;

it stop even before reaching that code.
in _cycle = function() {

...

if (!haveConnectingSocket && this.sockets.length < this.maxSockets) { <----
this._establishNewConnection();
}

}

That's where prohibit the code from establish new connection.

I think the bug is in the socket checking code :

for (var i = 0; i < this.sockets.length; i++) {
var socket = this.sockets[i];
// If the socket doesn't already have a message it's sending out
// and the socket is available for writing or it's connecting.
// In particular this rules out sockets that are closing.
if (!socket._httpMessage &&
((socket.writable && socket.readable) || socket._httpConnecting)) {
debug('Agent found socket, shift');
// We found an available connection!
this.queue.shift(); // remove first from queue.
assert(first._queue === this.queue);
first._queue = null;

  first.assignSocket(socket);
  httpSocketSetup(socket);
  self._cycle(); // try to dispatch another
  return;
}

if (socket._httpConnecting) haveConnectingSocket = true;

}

I think

if (!socket._httpMessage &&
((socket.writable && socket.readable) || socket._httpConnecting)) {

doesn't catch the case that if the socket is in the connection pool, but actually doing nothing.

I believe some sockets do not have the proper stage set after the request is done. and those sockets will not deplete forever.

so.. there are two issues:

1. some sockets cannot free up. ( i called it ghost)
2. if over time, the # of ghost socket reaches the maxSockets, then no more new socket can be created, nor any ghost socket will be clean up. ( there is a check in http.js that do socket clean up when maxSocket is reach, but not for the ghost).

var http = require('http');

var query = 'v=12345';

var agent = http.getAgent('myremote.com', 80);
agent.maxSockets = 10;

var options = {
agent:agent,
host:'myremote.com',
port:80,
path:'/getcall?'+query,
method:'POST',
headers:{
"host":'myremote.com',
"user-agent": 'node.js',
"Connection":"keep-alive",
"Keep-Alive": "timeout=10, max=10"
}
};

var callMe = function(counter) {
var client = http.request(options,
function(res) {
console.log(res.statusCode + " : " + counter);
console.log("sock: "+ counter + " :lenght:" + agent.sockets.length);

            if (agent.sockets.length>(agent.maxSockets/10*9)) {
                console.log("keep: "+ counter + " :keepalive is false:");
                client.shouldKeepAlive = false;
            }
            res.on('data', function(chunk) {
                console.log("data: " + counter + " " + chunk);
                //client.destroy();
            });
            res.on('end', function() {
                console.log("done: " + counter + ":socket:" + agent.sockets.length + " :queue:" + agent.queue.length 
                            + ' :finished:' + client.finished
                            + ' :_last:' + client._last
                            + ' :writable:' + client.writable);
                console.log(client.socket);

            });
          }
);
client.end();

}
var index = 0;
setInterval(function() {
console.log("Call Set Interval");
for (var i=0; i<100;index++, i++ ) {
if (index <= 300) {
callMe(index);
}
}
}, 10000);

var cappedArray = [];
var cappedMax = 10;

var allSocketsCleanup = function () {
for (var i=0; i< agent.sockets.length; i++) {
console.log("destroying");
if (!agent.sockets[i]._httpMessage) {
agent.sockets[i].destroy();
}
}
};

setInterval(function() {
console.log("check agent.sockets.length" + agent.sockets.length);
var socketLength = agent.sockets.length;

if (cappedArray.length <= cappedMax) {
    cappedArray.push(socketLength);
    if (socketLength >= agent.maxSockets) {
        allSocketsCleanup();
    }
} else {
    cappedArray.push(socketLength);
    var prevSocketLength = cappedArray.shift();

    if (prevSocketLength == socketLength) {
        allSocketsCleanup();
        console.log('current length ' + agent.sockets.length);
    }
}

},500);

for the above code, it works on the original 0.4.9 http.js not the fix one. (not sure about the http-fix.js has the same socket problem yet).

key point:

1. for http connection that close to the 90% of the maxium socket, I switch it to "Connection":"close" in order to avoid connection being max out & keep-alive.
2. I have a routine to constantly check the # of sockets. If the # of sockets do not go down, or max out, then I will start force destroying socket (that have no httpMessage).

I have run some test and haven't seen any error thrown. and I can see the # of socket go back down.

Use this http.js and https.js will fix the problem.

• copy them to your nodejs/lib folder. replace the old ones.
• rebuild node again.