Check if reverse dns lookup matches a known CDN. (used for unblocking in blacklists)

cdn-reverse-check.rb

#/usr/bin/env ruby

# github/shadowbq - 2014
# GNU GENERAL PUBLIC LICENSE Version 2, June 1991

# Things to match.. 
# CDN List provided by - (23 Dec 2014) https://github.com/WPO-Foundation/webpagetest/blob/master/agent/wpthook/cdn.h

require 'rubygems'
require 'net/dns'
require 'open-uri'

CDN_PROVIDER = [
  [".akamai.net", "Akamai"],
  [".akamaiedge.net", "Akamai"],
  [".akamaihd.net", "Akamai"],
  [".edgesuite.net", "Akamai"],
  [".edgekey.net", "Akamai"],
  [".srip.ne", "Akamai"],
  [".akamaitechnologies.com", "Akamai"],
  [".akamaitechnologies.fr", "Akamai"],
  [".llnwd.net", "Limelight"],
  ["edgecastcdn.net", "Edgecast"],
  [".systemcdn.net", "Edgecast"],
  [".transactcdn.net", "Edgecast"],
  [".v1cdn.net", "Edgecast"],
  [".v2cdn.net", "Edgecast"],
  [".v3cdn.net", "Edgecast"],
  [".v4cdn.net", "Edgecast"],
  [".v5cdn.net", "Edgecast"],
  ["hwcdn.net", "Highwinds"],
  [".simplecdn.net", "Simple CDN"],
  [".instacontent.net", "Mirror Image"],
  [".footprint.net", "Level 3"],
  [".ay1.b.yahoo.com", "Yahoo"],
  [".yimg.", "Yahoo"],
  [".yahooapis.com", "Yahoo"],
  [".google.", "Google"],
  ["googlesyndication.", "Google"],
  ["youtube.", "Google"],
  [".googleusercontent.com", "Google"],
  ["googlehosted.com", "Google"],
  [".gstatic.com", "Google"],
  [".insnw.net", "Instart Logic"],
  [".inscname.net", "Instart Logic"],
  [".internapcdn.net", "Internap"],
  [".cloudfront.net", "Amazon CloudFront"],
  [".netdna-cdn.com", "NetDNA"],
  [".netdna-ssl.com", "NetDNA"],
  [".netdna.com", "NetDNA"],
  [".cotcdn.net", "Cotendo CDN"],
  [".cachefly.net", "Cachefly"],
  ["bo.lt", "BO.LT"],
  [".cloudflare.com", "Cloudflare"],
  [".afxcdn.net", "afxcdn.net"],
  [".lxdns.com", "ChinaNetCenter"],
  [".att-dsa.net", "AT&T"],
  [".vo.msecnd.net", "Windows Azure"],
  [".voxcdn.net", "VoxCDN"],
  [".bluehatnetwork.com", "Blue Hat Network"],
  [".swiftcdn1.com", "SwiftCDN"],
  [".cdngc.net", "CDNetworks"],
  [".gccdn.net", "CDNetworks"],
  [".panthercdn.com", "CDNetworks"],
  [".fastly.net", "Fastly"],
  [".nocookie.net", "Fastly"],
  [".gslb.taobao.com", "Taobao"],
  [".gslb.tbcache.com", "Alimama"],
  [".mirror-image.net", "Mirror Image"],
  [".yottaa.net", "Yottaa"],
  [".cubecdn.net", "cubeCDN"],
  [".r.cdn77.net", "CDN77"],
  [".incapdns.net", "Incapsula"],
  [".bitgravity.com", "BitGravity"],
  [".r.worldcdn.net", "OnApp"],
  [".r.worldssl.net", "OnApp"],
  ["tbcdn.cn", "Taobao"],
  [".taobaocdn.com", "Taobao"],
  [".ngenix.net", "NGENIX"],
  [".pagerain.net", "PageRain"],
  [".ccgslb.com", "ChinaCache"],
  ["cdn.sfr.net", "SFR"],
  [".azioncdn.net", "Azion"],
  [".azioncdn.com", "Azion"],
  [".azion.net", "Azion"],
  [".cdncloud.net.au", "MediaCloud"],
  [".rncdn1.com", "Reflected Networks"],
  [".cdnsun.net", "CDNsun"],
  [".mncdn.com", "Medianova"],
  [".mncdn.net", "Medianova"],
  [".mncdn.org", "Medianova"],
  ["cdn.jsdelivr.net", "jsDelivr"],
  [".nyiftw.net", "NYI FTW"],
  [".nyiftw.com", "NYI FTW"],
  [".resrc.it", "ReSRC.it"],
  [".zenedge.net", "Zenedge"],
  [".lswcdn.net", "LeaseWeb CDN"],  
  [".revcn.net", "Rev Software"],
  [".revdn.net", "Rev Software"],
  [".1e100.net", "Google"]
]


verbose = true

res = Net::DNS::Resolver.new
res.nameservers = ["4.2.2.2","4.2.2.3","4.2.2.4"]

iplist_cdn = []
unresolved = 0

## Malc0de.com http://malc0de.com/bl/IP_Blacklist.txt
## They list whoeva.. google, CDNs, etc..
## Rebuild their list so it's sane.. 

# Truncate to zero length or create file for writing.
open('./ipblack_list.txt', 'w+') do |file|
  file << open('http://malc0de.com/bl/IP_Blacklist.txt').read
end

#Truncate to zero length or create file for writing.
File.open("./ipblack_list_new.txt", "w+") do |newlist|
  File.readlines('./ipblack_list.txt').each do |line|

  	ip = line.strip
  	is_cdn = false

  	begin
  	  packet = res.search(ip)
      reverse_hostname = packet.answer[0].ptr
      puts reverse_hostname if verbose
  	rescue
  	  reverse_hostname = ""
  	  unresolved = unresolved + 1 
  	end

    if reverse_hostname != ""
      CDN_PROVIDER.each do |cdn| 
        if reverse_hostname.match(cdn[0])
          iplist_cdn << ip
          is_cdn = true
          break
        end
      end
      newlist.puts(ip) unless is_cdn 
      newlist.flush
    else
      newlist.puts(ip)
      newlist.flush
    end
  end
end

puts "CDN IPs: #{iplist_cdn.size}" if verbose
puts "Unresolved: #{unresolved}" if verbose

Now included in github.com/shadowbq/enricher