ipfilter.dat generating script

ipfilter.py

import sys, urllib2, zlib, os.path

urls = ['http://www.bluetack.co.uk/config/ads-trackers-and-bad-pr0n.gz',
'http://www.bluetack.co.uk/config/bogon.gz',
'http://www.bluetack.co.uk/config/dshield.gz',
'http://www.bluetack.co.uk/config/fornonlancomputers.gz',
'http://www.bluetack.co.uk/config/hijacked.gz',
'http://www.bluetack.co.uk/config/Microsoft.gz',
'http://www.bluetack.co.uk/config/spyware.gz',
'http://www.bluetack.co.uk/config/iana-multicast.gz',
'http://www.bluetack.co.uk/config/iana-private.gz',
'http://www.bluetack.co.uk/config/iana-reserved.gz',
'http://www.bluetack.co.uk/config/level1.gz']
output = 'ipfilter.dat'
ipfilter = []

def ip2int(ip):
  exp = 3
  intip = 0
  for quad in ip.split('.'):
    intip += (int(quad) * (256 ** exp))
    exp = exp - 1
  return intip

def import_old(file):
  if not os.path.isfile(file): return
  print "Importing " + file + "..."
  f = open(file, 'r')
  lines = f.readlines()
  f.close()
  i = 0
  while i < len(lines):
    ip1 = ip2int(lines[i][:15])
    ip2 = ip2int(lines[i][18:33])
    ipfilter.append([lines[i][:15], lines[i][18:33], lines[i][42:-1].ljust(10), ip1, ip2, ip2-ip1])
    i += 1

READ_BLOCK_SIZE = 1024*8
def get_gzipped(url):
  out = ''
  request = urllib2.Request(url)
  request.add_header('Accept-encoding', 'gzip,deflate')
  response = urllib2.urlopen(request)
  d = zlib.decompressobj(16+zlib.MAX_WBITS)
  while True:
    data = response.read(READ_BLOCK_SIZE)
    if not data: break
    out += d.decompress(data)
  return out

def ip_format(ip):
  return "{0:03d}.{1:03d}.{2:03d}.{3:03d}".format(*map(int, ip.split('.')))


import_old(output)
import_old('n' + output)

for url in urls:
  print "Downloading " + url + "..."
  i = 0
  lines = get_gzipped(url).split('\n')
  print "Processing " + str(len(lines)) + " entries..."
  for row in lines:
    if len(row) < 20: break
    i += 1
    sys.stdout.write(str(i) + "/" + str(len(lines)) + " (" + str(int(100/(1.0*len(lines)/i))) + "%)\r")
    sys.stdout.flush()
    desc, ips = row.split(':')
    ips = map(ip_format, ips.split('-'))
    ip1 = ip2int(ips[0])
    ip2 = ip2int(ips[1])
    ipfilter.append([ips[0], ips[1], desc[0:10].ljust(10), ip1, ip2, ip2-ip1])

print "Sorting...".ljust(40)
ipfilter.sort(key=lambda k: k[5], reverse=True)
ipfilter.sort(key=lambda k: k[3])
print "Filtering..."
for e in enumerate(ipfilter):
  sys.stdout.write(str(e[0]+1) + "/" + str(len(ipfilter)) + " (" + str(int(100/(1.0*len(ipfilter)/(e[0]+1)))) + "%)\r")
  sys.stdout.flush()
  while e[0] < len(ipfilter)-1 and (e[1][3] == ipfilter[e[0]+1][3] or ipfilter[e[0]+1][4] <= e[1][4]):
    ipfilter.pop(e[0]+1)

print "Writing output...".ljust(40)
f = open(output + '_new.txt', 'w')
for row in ipfilter:
  f.write(row[0] + ' - ' + row[1] + ' , 000 , ' + row[2] + "\n")
f.close()