advancedxy.com's blog related code

convert_text_chunzhen_bin.py

#coding:utf-8
import sys,struct,socket,array,datetime

def ip2str(ip):
    return str(ip>>24)+'.'+str((ip>>16)&0xffL)+'.' \
        +str((ip>>8)&0xffL)+'.'+str(ip&0xffL)

def str2ip(s):
    (ip,) = struct.unpack('!I',socket.inet_aton(s))
    return ((ip>>24)&0xffL)|((ip&0xffL)<<24) \
        |((ip>>8)&0xff00L)|((ip&0xff00L)<<8)

def ip_integer_from_string(s):
    "Convert dotted IPv4 address to integer."
    return reduce(lambda a,b: a<<8 | b, map(int, s.split(".")))

def ip_integer_to_string(ip):
     "Convert 32-bit integer to dotted IPv4 address."
     return ".".join(map(lambda n: str(ip>>n & 0xFF), [24,16,8,0]))

if __name__ == "__main__":
    record_file = open("record_part","wb")
    text_db = open("raw_latest_complete_merged",'rb')
    index_file = open("index_part","wb")
    index_count = 0
    for index,line in enumerate(text_db.readlines()):
        line = line.decode("utf-8")
        items = line.split(";")
        start,prefixlen = items[0].split('/')
        print start,
        end = ip_integer_from_string(start) + 2**(32-int(prefixlen)) -1
        #end = str2ip(ip_integer_to_string(end))
        start = ip_integer_from_string(start)
        #start = str2ip(start)
        # start = ip_integer_from_string(start)
        start = start
        end = end
        print ip_integer_to_string(end)
        #print ip2str(start),ip2str(end)
        offset = record_file.tell() + 8
        # print hex(offset)
        #offset will be represented as 3 byte ,so we need to split the offset into 2 parts.
        highbyte = (offset & 0x00ff0000) >> 16
        # print hex(highbyte)
        # print hex(offset)
        middlebyte = (offset & 0x0000ff00) >> 8
        lowbyte = offset & 0x000000ff
        print hex(offset), hex(highbyte), hex(middlebyte), hex(lowbyte)
        index_file.write(struct.pack('IBBB',start,lowbyte,middlebyte,highbyte))
        #write the ip record, we use the 0x01 mode
        record_file.write(struct.pack('I',end))
        #write the redirect mode
        area = items[2]
        isp = items[4]
        if items[1] != u'中国':
            area = items[1]
            isp = u'亚马逊'
        area = bytes(area.encode('GBK'))
        isp = bytes(isp.encode('GBK'))
        record_file.write(struct.pack(len(area)*'s', *area))
        record_file.write(struct.pack('B',0x0))
        record_file.write(struct.pack(len(isp)*'s', *isp))
        record_file.write(struct.pack('B',0x0))
        index_count = index
    index_count = index_count + 1
    offset = record_file.tell() + 8
    highbyte = (offset & 0xff0000) > 16
    low2bytes = offset & 0xffff
    start = ip_integer_from_string('255.255.255.0')
    end = ip_integer_from_string('255.255.255.255')
    index_file.write(struct.pack('IHB',start,low2bytes,highbyte))
    record_file.write(struct.pack('I',end))
    area = 'lalala'
    record_file.write(struct.pack(len(area)*'s', *area))
    record_file.write(struct.pack('B',0x0))
    record_file.write(struct.pack('B',0x0))
    index_offset_start = record_file.tell() + 8
    index_offset_end = index_offset_start + 7 * index_count
    header_file = open("header_part",'wb')
    header_file.write(struct.pack('I',index_offset_start))
    header_file.write(struct.pack("I",index_offset_end))

query.py

#!/usr/bin/python2
# -*- coding: utf-8 -*-

import requests
import sys
base_taobao_url = "http://ip.taobao.com/service/getIpInfo.php"
base_sina_url = "http://int.dpool.sina.com.cn/iplookup/iplookup.php"

def query_ip(ip, base_url = None):
    if base_url == None:
        url_list = [ base_taobao_url, base_sina_url ]
    else:
        url_list = [ base_url ]
    payload = {"ip":ip}
    resultL = []
    for url in url_list:
        if url == base_sina_url:
            payload["format"] = "json"
        else:
            payload.pop("format", 0)
        result = requests.get(url, params=payload)
        if result.status_code == 200:
            json = result.json()
            if url == base_taobao_url and not json["code"]:
                resultL.append(json["data"])
            if url == base_sina_url and json["ret"] == 1:
                resultL.append(json)
        else:
            print "There is an error, maybe ask to fast"
    if resultL:
        rjson = reduce(lambda d1,d2: dict(d1.items() + { k:v for k,v in d2.items() if v }.items()), resultL)
    else:
        rjson = reduce(lambda d1,d2: dict(d1.items() + { k:v for k,v in d2.items() if v }.items()), resultL,{})
    if rjson:
        if not rjson.get("province", ""):
            rjson["province"] = rjson.get("region", "")
            rjson["city"] = rjson.get("city", "")
            rjson["isp"] = rjson.get("isp", "")
        #print rjson
        return rjson

if __name__ == "__main__":
    if len(sys.argv) >= 2:
        ip = sys.argv[1]
    else:
        ip = "8.8.8.8"
    print   query_ip(ip, base_sina_url)