weibo.py

import struct

p = struct.Struct("<QQ")
i = 0
out = open('weibo.bin', 'wb')
with open('weibo.txt', 'r', errors='ignore') as f:
    f.readline()
    while True:
        line = f.readline()
        if not line:
            break
        try:
            s = line.strip().split()
            uin, mobile = s[-1], s[0]
            uin = int(uin)
            mobile = int(mobile) - 13000000000
            # if mobile > 0xFFFFFFFF:
            #     print("big mobile",mobile)
            # print(qq, mobile)
            out.write(p.pack(uin, mobile))
            i = i + 1
            if i%1000000==0:
                print(i)
                out.flush()
        except Exception as e:
            print(e, line)
out.close()
print('done')

import struct

high_mobile_dict = {}

p = struct.Struct("<LQ")
i = 0
out = open('qq.bin', 'wb')
with open('qq.txt', 'r', errors='ignore') as f:
    f.readline()
    while True:
        line = f.readline()
        if not line:
            break
        try:
            s = line.strip().split('----')
            qq, mobile = s[0], s[-1]
            qq = int(qq)
            if qq < 10000:
                print("small qq",qq)
                continue
            if qq > 0xFFFFFFFF:
                print("big qq",qq)
                continue
            mobile = int(mobile)
            if mobile < 13000000000:
                print("small mobile",qq)
                continue
            if mobile > 0xFFFFFFFFFF:
                print("big mobile",qq)
                continue
            # high_mobile = mobile // 100000000
            # low_mobile = mobile % 100000000
            # if high_mobile not in high_mobile_dict:
            #     high_mobile_dict[high_mobile] = len(high_mobile_dict)
            # print(qq, mobile, high_mobile, low_mobile, low_mobile | (high_mobile_dict[high_mobile] << 27))
            out.write(p.pack(qq, mobile)[0:-3])
            i = i + 1
            if i%1000000==0:
                print(i)
                out.flush()
                # break
        except Exception as e:
            print(e, line)
            # break
out.close()
# print(len(high_mobile_dict), high_mobile_dict)
print('done')