ip-scan.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from twisted.internet import defer, reactor, task
from twisted.web.client import getPage
from ipaddress import ip_address
import os
import csv
import json
import logging
import requests

seed_url = 'http://ftp.apnic.net/stats/apnic/delegated-apnic-latest'
api_url = 'http://int.dpool.sina.com.cn/iplookup/iplookup.php?format=json&ip=%s'

def work():
    
    def seed():
        fname = 'delegated-apnic-latest'
        if not os.path.exists(fname):
            with open(fname, 'w') as f:
                f.write(requests.get(seed_url).text)
        with open(fname) as f:
            for line in f:
                if ('|ipv4|' in line) and ('*' not in line):
                    ip, cnt = line.split('|')[3:5]
                    ip_start = ip_address(unicode(ip))
                    ip_end = ip_address(int(ip_start)+int(cnt)-1)
                    yield (str(ip_start), str(ip_end))

    def parse(data, ip_start, ip_end, d):
        obj = json.loads(data)
        if obj['ret']==1:
            obj = {k:v.encode('utf-8') for k,v in obj.iteritems() if k!='ret'}
            start = obj['start']
            end = obj['end']
            wrt.writerow(obj)
            logging.debug('<<< %s', start)
            if ip_address(unicode(end))<ip_address(unicode(ip_end)):
                logging.debug('... %s %s', end, ip_end)
                ip_start = str(ip_address(int(ip_address(unicode(end)))+1))
                getPage(api_url%ip_start).addCallback(parse, ip_start, ip_end, d)
            else:
                d.callback('DONE')
        else:
            logging.warn('!!! %s', data)

    for ip_start,ip_end in seed():
        logging.debug('>>> %s', ip_start)
        url = 'http://int.dpool.sina.com.cn/iplookup/iplookup.php?format=json&ip={}'.format(ip_start)
        d = defer.Deferred()
        getPage(api_url%ip_start).addCallback(parse, ip_start, ip_end, d)
        yield d

def finish(ign=None):

    reactor.stop()
    logging.info('*** BYE ***')

def run(jobs=100):

    deferreds = []
    coop = task.Cooperator()
    w = work()
    for i in xrange(jobs):
        d = coop.coiterate(w)
        deferreds.append(d)
    dl = defer.DeferredList(deferreds)
    dl.addCallback(finish)

if __name__=='__main__':

    logging.basicConfig(format='%(asctime)s\t[%(levelname)s]\t%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level='DEBUG')
    wrt = csv.DictWriter(open('ip-scan.csv', 'w'), ['start', 'end', 'country', 'province', 'city', 'district', 'isp', 'type', 'desc'])
    wrt.writeheader()
    run()
    reactor.run()