gistfile1.txt

olaf@bm-mc-01:~$ sudo awk -F"\"" '{print $6}' /var/log/nginx/metacpan-web/access.log | sort | uniq -dc | sort -nr | head -20
  11915 CCBot/2.0 (http://commoncrawl.org/faq/)
  11452 Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
   7767 Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
   5423 Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
   4194 Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
   2537 Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
   2498 Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
   2366 Mozilla/5.0 (X11; Linux x86_64; rv:10.0.4) Gecko/20100101 Firefox/10.0.4 Iceweasel/10.0.4
   2259 Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
   1825 Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
   1362 Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)
   1318 Python-urllib/2.7
    846 Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.21; +http://software.schmorp.de/pkg/AnyEvent)
    572 perlmodules.net 0.1
    532 Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
    507 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)
    424 Uptimer
    423 Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
    423 Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.15; +http://software.schmorp.de/pkg/AnyEvent)
    407 Panopta v1.1