Skip to content

Instantly share code, notes, and snippets.

@dpetukhov
Created January 20, 2012 02:40
Show Gist options
  • Save dpetukhov/1644619 to your computer and use it in GitHub Desktop.
Save dpetukhov/1644619 to your computer and use it in GitHub Desktop.
var fs = require('fs'),
async = require('async'),
util = require('util'),
request = require('request');
var proxies = [
'190.73.142.241:8080',
'110.77.228.254:3128',
'110.77.228.254:3128',
'68.111.122.185:6515',
'91.146.48.245:8080',
'110.77.228.254:3128',
'208.52.91.231:80',
'208.52.91.231:80',
'91.146.48.245:8080',
'68.111.122.185:6515',
'41.35.48.222:8080',
'189.21.218.50:3128',
'82.114.82.60:8080',
'41.35.48.222:8080',
'187.115.76.111:3128',
'187.115.76.111:3128',
'87.98.128.183:80',
'190.242.61.49:8080',
'189.21.218.50:3128',
'82.114.82.60:8080',
'82.114.82.60:8080',
'190.242.61.49:8080',
'189.21.218.50:3128',
'87.98.128.183:80',
'208.52.91.231:80',
'190.242.61.49:8080',
'69.141.241.119:6515',
'187.115.76.111:3128',
'218.213.235.158:80',
'87.98.128.183:80',
'186.202.60.70:8080',
'222.255.0.228:6515',
'69.141.241.119:6515',
'95.110.227.197:808',
'50.135.82.255:6515',
'202.181.232.9:8080',
'222.255.0.228:6515',
'95.110.227.197:808',
'112.196.4.168:8080',
'69.141.241.119:6515',
'69.141.241.119:6515',
'177.39.130.125:3128',
'41.158.128.180:80',
'50.135.82.255:6515',
'41.158.128.180:80',
'69.145.228.57:6515',
'69.145.228.57:6515',
'69.145.228.57:6515',
'69.145.228.57:6515',
'69.145.228.57:6515',
'122.155.0.135:8888',
'118.97.209.218:8080',
'118.97.103.82:8080',
'193.87.168.98:3128',
'189.26.122.154:80',
'218.210.199.252:3128',
'186.227.166.11:80',
'88.84.223.210:3128 ',
'118.96.31.91:3128',
'61.19.127.131:8080',
'118.97.209.218:8080 ',
'115.124.67.241:8181',
'180.243.235.40:8080',
'148.222.11.80:80',
'217.156.48.24:8080',
'202.162.223.106:8080',
'201.30.133.40:3128',
'213.165.180.91:8080',
'159.148.213.228:3128',
'222.124.5.2:8080',
'222.124.5.2:8080',
'222.124.5.2:8080',
'92.247.20.233:80',
'92.247.20.228:80',
'82.103.117.185:3128',
'202.148.29.243:8000',
'82.103.117.185:3128',
'222.124.5.2:8080',
'92.247.20.233:80',
'92.247.20.228:80',
'82.103.117.185:3128',
'202.148.29.243:8000',
'82.103.117.185:3128',
'199.195.141.231:3128',
'199.195.141.231:3128',
'193.138.185.51:3128',
'85.92.159.84:8080',
'186.250.3.20:3128',
'189.3.225.99:3128',
'111.93.166.202:8080',
'195.178.34.206:8080',
'187.72.145.53:8080',
'77.92.157.60:8080',
'118.97.117.138:8080',
'115.111.191.125:8888',
'210.0.205.70:80',
'187.5.159.218:80',
'77.92.157.60:8080',
'190.228.110.195:8080',
'196.1.178.254:3128',
'175.41.131.201:3128',
'66.178.57.4:80',
'115.111.191.125:8888',
'210.0.205.70:80',
'89.102.237.74:8080',
'200.195.145.14:3128',
'187.5.159.218:80',
'190.228.110.195:8080',
'196.1.178.254:3128',
'175.41.131.201:3128',
'66.178.57.4:80',
'184.22.122.89:3128',
'175.41.130.237:3128',
'64.152.0.216:81',
'175.41.131.201:3128',
'79.143.176.232:8080',
'89.102.237.74:8080',
'175.41.130.237:3128',
'79.143.176.232:8080',
'64.152.0.216:81',
'79.143.176.232:8080',
'175.41.130.237:3128',
'184.22.122.89:3128',
'175.41.130.237:3128',
'64.152.0.216:81',
'79.143.176.232:8080',
'184.22.122.89:3128',
'175.41.130.237:3128',
'64.152.0.216:81',
'79.143.176.232:8080',
'64.152.0.216:81',
'79.143.176.232:8080',
'79.143.176.232:8080'
];
// Worker with 40 parallels
var work_queue = async.queue(function(task, callback){
request({
'method': 'GET',
'proxy': 'http://'+proxies[Math.floor(Math.random()*(proxies.length-1))],
'url': 'http://ya.ru',
'jar': false,
'pool': false,
'followRedirect': false,
'headers': {'Connection':'close'},
'timeout': 10000, // timeout for each try
},
function(){
console.log('Queue len', work_queue.length(),
'running', work_queue.running());
callback();
}
);
}, 40);
work_queue.drain = function() {
util.debug('Work end');
}
for (var i = 0; i < 1000; i++){
work_queue.push({});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment