em_300_gets.rb

require 'eventmachine'
require 'em-http'

c = 4
urls = DATA.readlines.map {|s| s.strip }
pending = urls.size
before = Time.now

puts "urls: #{pending}"
puts "concurrency: #{c}"

EM.run do
  q = EM::Queue.new
  q.push(*urls)

  worker = lambda do |url|
    http = EM::HttpRequest.new("http://#{url}/").get(:timeout => 10)
  
    http.callback do
      pending -= 1
      EM.stop if pending < 1
      q.pop worker
    end

    http.errback do
      puts "#{url} #{http.error}"
    end

    http.errback do
      pending -= 1
      EM.stop if pending < 1
      q.pop worker
    end
  end

  c.times do
    q.pop worker
  end

  EM.add_periodic_timer(2) do
    puts "#{q.size}"
  end
end

puts "finished in #{Time.now - before}"

# The first 300 from Alexa's top 500 domains:
# http://www.alexa.com/topsites
__END__
ebay.com
bing.com
google.fr
microsoft.com
yandex.ru
163.com
google.co.jp
linkedin.com
myspace.com
google.com.br
craigslist.org
conduit.com
flickr.com
fc2.com
mail.ru
google.it
imdb.com
vkontakte.ru
livejasmin.com
google.es
googleusercontent.com
mozilla.com
rapidshare.com
soso.com
go.com
sohu.com
bbc.co.uk
apple.com
aol.com
youku.com
doubleclick.com
xvideos.com
google.com.mx
bp.blogspot.com
ask.com
pornhub.com
adobe.com
google.ca
google.ru
paypal.com
cnn.com
orkut.com.br
mediafire.com
tudou.com
youporn.com
sogou.com
espn.go.com
photobucket.com
hotfile.com
xhamster.com
4shared.com
about.com
orkut.com
megaupload.com
ameblo.jp
livejournal.com
imageshack.us
google.co.id
hao123.com
cnet.com
rakuten.co.jp
ebay.de
google.com.tr
megavideo.com
google.com.au
google.cn
uol.com.br
files.wordpress.com
livedoor.com
orkut.co.in
thepiratebay.org
google.pl
nytimes.com
alibaba.com
ifeng.com
tube8.com
tianya.cn
ebay.co.uk
google.com.sa
globo.com
mixi.jp
godaddy.com
twitpic.com
renren.com
cpxinteractive.com
hi5.com
digg.com
dailymotion.com
odnoklassniki.ru
kaixin001.com
optmd.com
yieldmanager.com
redtube.com
google.nl
xtendmedia.com
deviantart.com
amazon.de
ku6.com
taringa.net
netflix.com
goo.ne.jp
weather.com
cnzz.com
ezinearticles.com
xnxx.com
mywebsearch.com
ehow.com
spiegel.de
rediff.com
tumblr.com
amazon.co.jp
filestube.com
google.com.ar
aweber.com
zedo.com
xunlei.com
google.co.th
clicksor.com
google.com.eg
bankofamerica.com
answers.com
rambler.ru
sourceforge.net
ning.com
google.com.pk
google.co.za
huffingtonpost.com
nicovideo.jp
youjizz.com
chinaz.com
wordpress.org
fifa.com
bit.ly
badoo.com
secureserver.net
wretch.cc
partypoker.com
download.com
addthis.com
amazon.co.uk
depositfiles.com
adultfriendfinder.com
terra.com.br
maktoob.com
bild.de
torrentz.com
indiatimes.com
vk.com
onet.pl
chase.com
118114.cn
angege.com
56.com
free.fr
tinypic.com
vimeo.com
gmx.net
digitalpoint.com
wikimedia.org
nk.pl
blogfa.com
comcast.net
stumbleupon.com
douban.com
alipay.com
xinhuanet.com
orange.fr
statcounter.com
typepad.com
tagged.com
dell.com
xvideoslive.com
fastclick.com
dailymail.co.uk
mybrowserbar.com
gougou.com
archive.org
ameba.jp
web.de
hubpages.com
google.be
nifty.com
domaintools.com
hp.com
2ch.net
people.com.cn
google.co.ve
reference.com
metacafe.com
wikia.com
libero.it
naver.com
foxnews.com
hulu.com
pconline.com.cn
wp.pl
allegro.pl
zynga.com
xing.com
guardian.co.uk
zol.com.cn
4399.com
skype.com
mop.com
marca.com
google.com.vn
google.gr
imagevenue.com
google.com.co
squidoo.com
soufun.com
fastbrowsersearch.com
isohunt.com
google.at
netlog.com
clickbank.com
walmart.com
istockphoto.com
amazonaws.com
fbcdn.net
biglobe.ne.jp
liveinternet.ru
opendns.com
google.com.my
telegraph.co.uk
youdao.com
google.se
wsj.com
gamespot.com
spankwire.com
linkbucks.com
soku.com
leboncoin.fr
mozilla.org
126.com
scribd.com
51.la
ign.com
hatena.ne.jp
yelp.com
kaskus.us
tripadvisor.com
google.ch
google.ro
vnexpress.net
adultadworld.com
ig.com.br
uimserv.net
ucoz.ru
megaporn.com
tweetphoto.com
58.com
keezmovies.com
tom.com
reddit.com
joomla.org
leo.org
cricinfo.com
z5x.net
google.com.ua
booking.com
rutracker.org
mashable.com
seznam.cz
360buy.com
infoseek.co.jp
narod.ru
hardsextube.com
google.com.pe
cj.com
mlb.com
51.com
match.com
mapquest.com
brothersoft.com
usps.com
google.pt
pornhublive.com
csdn.net
babylon.com
2345.com
sakura.ne.jp
avg.com
geocities.jp
softonic.com
w3schools.com
icio.us
yesky.com
google.cl
ups.com
repubblica.it
salesforce.com
admagnet.net
google.com.ph
love21cn.com
reuters.com
engadget.com
expedia.com
att.com
fileserve.com