phone-scan.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from twisted.internet import defer, reactor, task
from twisted.web.client import getPage
import json
import redis
import logging
 
ps = [
    130,131,132,133,134,135,136,137,138,139,
    150,151,152,153,154,155,156,157,158,159,
    170,176,177,178,
    180,181,182,183,184,185,186,187,188,189,
]

def work():

    logging.info('*** HEY ***')

    def exists(num):

        return rdb.sismember('phone:all', num)

    def update(obj):

        num = obj['num']
        rdb.sadd('phone:all', num)
        if obj['code']==0:
            logging.debug('+++ %s', num)
            rdb.sadd('phone:good', num)
            rdb.hmset('phone:data:%s'%num, obj['data'])
        else:
            logging.debug('--- %s', num)
            rdb.sadd('phone:bad', num)

    def parse(result, num):

        try:
            obj = json.loads(result)
            obj['num'] = num
            update(obj)
        except:
            logging.warn('??? %s', num)

    for i in ps:
        for j in xrange(0, 10**4):
            num = '{0}{1:04d}0000'.format(i, j)
            if exists(num):
                logging.debug('... %s', num)
                continue
            url = 'http://cx.shouji.360.cn/phonearea.php?number={}'.format(num)
            d = getPage(url)
            d.addCallback(parse, num)
            yield d

def finish(ign=None):

    reactor.stop()
    logging.info('*** BYE ***')

def run(jobs=100):

    deferreds = []
    coop = task.Cooperator()
    w = work()
    for i in xrange(jobs):
        d = coop.coiterate(w)
        deferreds.append(d)
    dl = defer.DeferredList(deferreds)
    dl.addCallback(finish)
 
if __name__=='__main__':

    logging.basicConfig(format='%(asctime)s\t[%(levelname)s]\t%(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level='INFO')
    rdb = redis.StrictRedis(host='localhost', port=6379, db=14)
    run()
    reactor.run()