Crystal downloader script

downloader.cr

require "uri"
require "http/client"

def fetch_url(url)
  uri = URI.parse(url)
  client = HTTP::Client.new(uri: uri).tap do |c|
    c.connect_timeout = 5.seconds
    c.read_timeout = 5.seconds
  end
  client.get(uri.to_s)
end

urls = ["http://3blmedia.com/News/GM-Herman-Miller-and-Green-Standards-RePurpose-Program-Sizzles", "http://allafrica.com/stories/201611220169.html", "http://au.news.yahoo.com/a/-/latest/33293227/man-on-video-wanted-in-perth-murder/", "http://beijing.qianlong.com/2016/1122/1138384.shtml", "http://bleacherreport.com/articles/2677527-green-laser-appears-to-be-shined-on-brock-osweiler-during-texans-vs-raiders", "http://blog.eastmoney.com/b1091054775506808", "http://china.caixin.com/2016-11-21/101009731.html", "http://cin.247sports.com/Bolt/Report-AJ-Green-has-a-torn-hamstring-49138760", "http://cincysportszone.com/does-this-dividend-diamond-have-upside-according-to-the-street-sl-green-realty-corp-nyseslg/69487/", "http://cn.reuters.com/article/idCNL4S1DN2AC", "http://constructionreviewonline.com/2016/11/zimbabwe-to-construct-40-8mw-solar-plant/", "http://deadline.com/2016/11/netflix-anne-of-green-gables-adaptation-adds-cast-1201858311/", "http://english.almanar.com.lb/108711", "http://feedproxy.google.com/~r/TheNextWeb/~3/lO4kTtTDUXM/", "http://feeds.foxnews.com/~r/foxnews/most-popular/~3/OsMET5sMU1A/too-much-good-thing-foods-that-can-be-toxic-if-eat-them-in-excess.html", "http://feeds.inc.com/~r/home/updates/~3/CflutGmpQ6w/5-design-tips-that-will-enhance-your-social-media-images.html", "http://finance.huanqiu.com/roll/2016-11/9716176.html", "http://finance.southcn.com/f/2016-11/22/content_160187463.htm", "http://focustaiwan.tw/news/asoc/201611210014.aspx", "http://gd.sina.com.cn/news/b/2016-11-22/detail-ifxxwrwh4884290.shtml", "http://gegu.stock.cnfol.com/geguzixun/20161122/23851956.shtml", "http://gov.hebnews.cn/2016-11/22/content_6083477.htm", "http://host.madison.com/wsj/sports/college/football/badgers-football-ashwaubenon-native-emmit-carpenter-thriving-as-gophers-kicker/article_ef3431f4-c129-57d2-b778-8d953f24e1d7.html", "http://house.syd.com.cn/system/2016/11/22/011236090.shtml", "http://house.xinmin.cn/fczx/2016/11/22/30620405.html", "http://indiatoday.intoday.in/story/train-accident-new-revelation-by-gateman/1/816684.html", "http://jacksonville.com/metro/2016-11-20/3-injured-fire-and-explosion-green-cove-springs-marina", "http://js.ifeng.com/a/20161122/5166052_0.shtml", "http://kuaixun.stcn.com/2016/1122/12953965.shtml", "http://legal.china.com.cn/2016-11/22/content_39758027.htm", "http://lnt.ma/declaration-de-marrakech-promouvoir-marches-de-capitaux-verts-afrique/", "http://londonist.com/london/transport/how-to-make-the-most-of-the-24-hour-northern-line", "http://mashable.com/2016/11/21/we-need-green-day/", "http://meangreenblog.dentonrc.com/2016/11/unt-drexel-running-blog.html/", "http://metro.co.uk/2016/11/22/christmas-2016-12-fantastic-festive-shows-and-outings-to-treat-the-kids-to-this-december-6243931/", "http://money.ycwb.com/2016-11/22/content_23592148.htm"]
channel = Channel(Int32).new
urls.size.times do |i|
  spawn do
    url = urls[i].chomp
    puts "START: #{url}"
    begin
      response = fetch_url(url)
      puts "DONE: #{url}"
    rescue
      puts "ERROR: #{url}"
    end
    channel.send(i)
  end
end

urls.size.times do |i|
  channel.receive
end