-
-
Save murvinlai/1032413 to your computer and use it in GitHub Desktop.
/* | |
* This is a simple HTTP data generator for testing. | |
* | |
*/ | |
var http = require('http'); | |
var counter = 0; | |
http.createServer(function (req, res) { | |
var start = new Date(); | |
var myCounter = counter++; | |
var timeout = 50; // default timeout. Mimic how much time it takes to run a process. | |
// Controlled long response time causing timeout in client side. | |
if ( (myCounter % 20000) == 0) { | |
console.log('-------- reach 20000 ------------'); | |
timeout = 600000; // 10 minutes | |
} | |
// give it some timeout | |
setTimeout(function() { | |
var output = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'; | |
output += '<myData>ABCDE'+myCounter+'</myData>\n'; | |
console.log("output: " + myCounter); | |
res.writeHead(200, {'Content-Type': 'application/xml'}); | |
res.write(output); | |
res.end(); | |
}, timeout); | |
}).listen(3015); | |
console.log('Server running at port 3015'); |
var http = require('http'); | |
// option1 is the from Google. GoogleAPI usually has less timeout and causing less issue. | |
//https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=Paris%20Hilton&callback=foo&context=bar | |
var agent1 = http.getAgent('ajax.googleapis.com',80); | |
agent1.maxSockets = 2000; | |
var options1 = { | |
agent:agent1, | |
host: 'ajax.googleapis.com', | |
port: 80, | |
path: '/ajax/services/search/web?v=1.0&q=Paris%20Hilton&callback=foo&context=bar', | |
method: 'GET', | |
headers:{ | |
"Connection":"keep-alive" | |
} | |
}; | |
// option2 is your own server that feeds data. | |
var agent2 = http.getAgent('localhost', 80); | |
agent2.maxSockets = 2000; | |
var options2 = { | |
agent:agent2, | |
host:'localhost', | |
port:80, | |
path:'/', | |
method:'GET', | |
headers:{ | |
"Connection":"keep-alive" | |
} | |
}; | |
// option3 is my AWS micro instance. It has controlled timeout for ever 20000 request. | |
var agent3 = http.getAgent('50.19.237.122', 3015); | |
agent3.maxSockets = 2000; | |
var options3 = { | |
agent:agent3, | |
host:'50.19.237.122', | |
port:3015, | |
path:'/', | |
method:'GET', | |
headers:{ | |
"Connection":"keep-alive" | |
} | |
}; | |
// set which | |
var workingOption = options3; | |
var counter = 0; | |
var server = http.createServer(function (req, res) { | |
var myCounter = counter ++; | |
console.log("Start: counter: " + myCounter); | |
var start = new Date(); // just for timing. | |
var clientRequest = http.request(workingOption, function(response) { | |
var result = ''; | |
response.setEncoding('utf8'); | |
res.writeHead(response.statusCode, {'Content-Type': 'application/xml'}); | |
response.on('data', function (chunk) { | |
result += chunk; | |
if (!res.write(chunk)) { // if write failed, the stream is choking | |
response.pause(); // tell the incoming stream to wait until output stream drained | |
} | |
}).on('end', function () { | |
var end = new Date(); | |
console.log("End: counter: " + myCounter + ' Time: ' + (end-start) + " chunk: " + result.substring(0, 20)); | |
res.end(); | |
}); | |
}); | |
clientRequest.on('error', function(e) { | |
console.error("ERROR: " + JSON.stringify(e)); | |
console.log("ERROR: " + JSON.stringify(e)); | |
}); | |
clientRequest.end(); | |
}); | |
server.listen(3204); | |
console.log('Server running at port 3204'); |
var http = require('http');
var query = 'v=12345';
var agent = http.getAgent('myremote.com', 80);
agent.maxSockets = 10;
var options = {
agent:agent,
host:'myremote.com',
port:80,
path:'/getcall?'+query,
method:'POST',
headers:{
"host":'myremote.com',
"user-agent": 'node.js',
"Connection":"keep-alive",
"Keep-Alive": "timeout=10, max=10"
}
};
var callMe = function(counter) {
var client = http.request(options,
function(res) {
console.log(res.statusCode + " : " + counter);
console.log("sock: "+ counter + " :lenght:" + agent.sockets.length);
if (agent.sockets.length>(agent.maxSockets/10*9)) {
console.log("keep: "+ counter + " :keepalive is false:");
client.shouldKeepAlive = false;
}
res.on('data', function(chunk) {
console.log("data: " + counter + " " + chunk);
//client.destroy();
});
res.on('end', function() {
console.log("done: " + counter + ":socket:" + agent.sockets.length + " :queue:" + agent.queue.length
+ ' :finished:' + client.finished
+ ' :_last:' + client._last
+ ' :writable:' + client.writable);
console.log(client.socket);
});
}
);
client.end();
}
var index = 0;
setInterval(function() {
console.log("Call Set Interval");
for (var i=0; i<100;index++, i++ ) {
if (index <= 300) {
callMe(index);
}
}
}, 10000);
var cappedArray = [];
var cappedMax = 10;
var allSocketsCleanup = function () {
for (var i=0; i< agent.sockets.length; i++) {
console.log("destroying");
if (!agent.sockets[i]._httpMessage) {
agent.sockets[i].destroy();
}
}
};
setInterval(function() {
console.log("check agent.sockets.length" + agent.sockets.length);
var socketLength = agent.sockets.length;
if (cappedArray.length <= cappedMax) {
cappedArray.push(socketLength);
if (socketLength >= agent.maxSockets) {
allSocketsCleanup();
}
} else {
cappedArray.push(socketLength);
var prevSocketLength = cappedArray.shift();
if (prevSocketLength == socketLength) {
allSocketsCleanup();
console.log('current length ' + agent.sockets.length);
}
}
},500);
for the above code, it works on the original 0.4.9 http.js not the fix one. (not sure about the http-fix.js has the same socket problem yet).
key point:
- for http connection that close to the 90% of the maxium socket, I switch it to "Connection":"close" in order to avoid connection being max out & keep-alive.
- I have a routine to constantly check the # of sockets. If the # of sockets do not go down, or max out, then I will start force destroying socket (that have no httpMessage).
I have run some test and haven't seen any error thrown. and I can see the # of socket go back down.
Use this http.js and https.js will fix the problem.
- copy them to your nodejs/lib folder. replace the old ones.
- rebuild node again.
so.. there are two issues: