hosts - A mix of working google, twitter, wikipedia ips and no-ads

check_ssl.py

from socket import socket, error, setdefaulttimeout
import ssl
import re
import time
import sys
import os
from hashlib import md5
import json

os.chdir(os.path.split(sys.argv[0])[0])
fr = 'NOTHING'
ip_list = []

if sys.argv[1].endswith('.txt'):
    # ip_list = re.findall('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', open(sys.argv[1]).read())
    ip_list = set()
    _ip = ''
    for l in open(sys.argv[1]).readlines():
        _ = re.findall('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', l)
        if _:
            _ip = _[0]
            continue
        else:
            if 'open' in l or 'filtered' in l:
                ip_list.add(_ip)
                _ip = ''
else:
    ip_list = sys.argv[1:]
    if ip_list[-1].endswith('.txt'):
        ip_list.pop()
#ip_list = re.findall('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', fr)
cert_dict = {}
ip_cert_dict = {}
cert_ip_dict = {}

setdefaulttimeout(2.0)
print('%d hosts up' % len(ip_list))
for ip in ip_list:
    if ip.replace('.', '-') + '.googlebot.com' in fr:
        continue
    trytime = 1
    while True:
        try:
            s = socket()
            c = ssl.wrap_socket(s, cert_reqs=ssl.CERT_REQUIRED, ca_certs=r'./cacert.pem')
            c.connect((ip, 443))
        except Exception as e:
            print(e)
            print('sleep')
            #time.sleep(3)
        else:
            break
        c.close()
        s.close()
        trytime -= 1
        if trytime <= 0:
            print('error on %s' % ip)
            c = None
            break
    if not c:
        continue
    if 'subjectAltName' not in c.getpeercert():
        continue
    _cert = c.getpeercert()
    all_domain = map(lambda x:x[1], _cert['subjectAltName'])
    if 'serialNumber' not in _cert:
        sn = '!%s!' % md5(c.getpeercert(True)).hexdigest()
    else:
        sn = c.getpeercert()['serialNumber']
    cn = c.getpeercert()['subject'][-1][0][1]
    if sn not in cert_dict:
        cert_dict[sn] = (cn, all_domain)

    if sn not in cert_ip_dict:
        cert_ip_dict[sn] = [ip]
    else:
        cert_ip_dict[sn].append(ip)

    ip_cert_dict[ip] = sn

    print('%s => %s %s%s' % (ip, cn, sn, '' if len(all_domain) == 1 else ('(%d)' % len(all_domain))))
if len(sys.argv)>2 and sys.argv[-1].endswith('txt'):
    r = open(sys.argv[-1], 'w')
else:
    r = open('ssl_result.txt', 'w')

print('Cert List:')
r.write('Cert List:')
for k in cert_dict:
    s = '[' + k + '] ' + cert_dict[k][0] + '\n' + ','.join(cert_dict[k][1]) + \
         '\n' + ','.join(cert_ip_dict[k]) + \
         '\n=============\n'
    print(s)
    r.write(s)

sn_keys = cert_dict.keys()
for i in range(len(sn_keys)):
    incl = []
    for j in range(i+1, len(sn_keys)):
        if set(cert_dict[sn_keys[i]][1]) >= set(cert_dict[sn_keys[j]][1]):
            incl.append(sn_keys[j])
    if incl:
        s = sn_keys[i] + ' including: ' + ','.join(incl)
        print(s)
        r.write(s)

print('Cert List:')
r.write('Cert List:')

print('IP to Cert:')
r.write('IP to Cert:')
for k in ip_cert_dict:#print each ip -> cert
    s = k + '=>' + ip_cert_dict[k] + '\n'
    print(s)
    r.write(s)

r.close()

test_fix_googlecn.py

from __future__ import print_function
import httplib2
import re
import os
pref = "203.208"
fixto = "173.194.72.98"
hosts = open(r'd:/hosts').read().split('\n')
old = ''
if os.path.exists(r'z:/hosts'):
    old = open('z:/hosts').read()
output = open(r'z:/hosts', 'w')
old = map(lambda x:x.split()[1] if x else '', old.split('\n'))
h = httplib2.Http()
h.follow_redirects = False
for line in hosts:
    if not line:
        output.write('\n')
    if line.startswith(pref):
        ip, host = line.split()
        if host in old:
            print('%s already done' % host)
            output.write(line + '\n')
            continue
        ret = h.request('http://%s' % ip, headers = {'Host':host})
        print('%s:%s' % (host, ret[0]['status']))
        if ret[0]['status'] == '404':
            output.write('%s    %s\n' % (fixto, host))
            print('%s:fixed' % host)
            continue
    output.write(line + '\n')
output.flush()

test_googlecn.py

hosts='''173.194.72.104    0.docs.google.com
173.194.72.104    0.drive.google.com
173.194.72.104    1.docs.google.com
173.194.72.104    1.drive.google.com
173.194.72.104    10.docs.google.com
173.194.72.104    10.drive.google.com
173.194.72.104    11.docs.google.com
173.194.72.104    11.drive.google.com
173.194.72.104    12.docs.google.com
173.194.72.104    12.drive.google.com
173.194.72.104    13.docs.google.com
173.194.72.104    13.drive.google.com
173.194.72.104    14.docs.google.com
173.194.72.104    14.drive.google.com
173.194.72.104    15.docs.google.com
173.194.72.104    15.drive.google.com
173.194.72.104    16.docs.google.com
173.194.72.104    16.drive.google.com
173.194.72.104    2.docs.google.com
173.194.72.104    2.drive.google.com
173.194.72.104    3.docs.google.com
173.194.72.104    3.drive.google.com
173.194.72.104    4.docs.google.com
173.194.72.104    4.drive.google.com
173.194.72.104    5.docs.google.com
173.194.72.104    5.drive.google.com
173.194.72.104    6.docs.google.com
173.194.72.104    6.drive.google.com
173.194.72.104    7.docs.google.com
173.194.72.104    7.drive.google.com
173.194.72.104    8.docs.google.com
173.194.72.104    8.drive.google.com
173.194.72.104    9.docs.google.com
173.194.72.104    9.drive.google.com
173.194.72.104    admin.google.com
173.194.72.104    answers.google.com
173.194.72.104    appengine.google.com
173.194.72.104    apps.google.com
173.194.72.104    appspot.l.google.com
173.194.72.104    bks0.books.google.com
173.194.72.104    bks1.books.google.com
173.194.72.104    bks10.books.google.com
173.194.72.104    bks2.books.google.com
173.194.72.104    bks3.books.google.com
173.194.72.104    bks4.books.google.com
173.194.72.104    bks5.books.google.com
173.194.72.104    bks6.books.google.com
173.194.72.104    bks7.books.google.com
173.194.72.104    bks8.books.google.com
173.194.72.104    bks9.books.google.com
173.194.72.104    blogsearch.google.com
173.194.72.104    books.google.com
173.194.72.104    browserchannel-docs.l.google.com
173.194.72.104    browserchannel-spreadsheets.l.google.com
173.194.72.104    browsersync.google.com
173.194.72.104    browsersync.l.google.com
173.194.72.104    buzz.google.com
173.194.72.104    cache.l.google.com
173.194.72.104    cache.pack.google.com
173.194.72.104    calendar.google.com
173.194.72.104    cbk0.google.com
173.194.72.104    cbk1.google.com
173.194.72.104    cbk2.google.com
173.194.72.104    cbk3.google.com
173.194.72.104    cbks0.google.com
173.194.72.104    cbks1.google.com
173.194.72.104    cbks2.google.com
173.194.72.104    cbks3.google.com
173.194.72.104    chart.apis.google.com
173.194.72.104    chrome.google.com
173.194.72.104    cloud.google.com
173.194.72.104    code.google.com
173.194.72.104    code.l.google.com
173.194.72.104    csi.l.google.com
173.194.72.104    desktop.google.com
173.194.72.104    desktop.l.google.com
173.194.72.104    desktop2.google.com
173.194.72.104    developers.google.com
173.194.72.104    ditu.google.com
173.194.72.104    dl.google.com
173.194.72.104    dl.l.google.com
173.194.72.104    dl-ssl.google.com
173.194.72.104    docs.google.com
173.194.72.104    docs0.google.com
173.194.72.104    docs1.google.com
173.194.72.104    docs2.google.com
173.194.72.104    docs3.google.com
173.194.72.104    docs4.google.com
173.194.72.104    docs5.google.com
173.194.72.104    docs6.google.com
173.194.72.104    docs7.google.com
173.194.72.104    docs8.google.com
173.194.72.104    docs9.google.com
173.194.72.104    drive.google.com
173.194.72.104    drive0.google.com
173.194.72.104    drive1.google.com
173.194.72.104    drive2.google.com
173.194.72.104    drive3.google.com
173.194.72.104    drive4.google.com
173.194.72.104    drive5.google.com
173.194.72.104    drive6.google.com
173.194.72.104    drive7.google.com
173.194.72.104    drive8.google.com
173.194.72.104    drive9.google.com
173.194.72.104    earth.google.com
173.194.72.104    encrypted.google.com
173.194.72.104    encrypted-tbn.l.google.com
173.194.72.104    encrypted-tbn0.google.com
173.194.72.104    encrypted-tbn1.google.com
173.194.72.104    encrypted-tbn2.google.com
173.194.72.104    encrypted-tbn3.google.com
173.194.72.104    feedburner.google.com
173.194.72.104    feedproxy.google.com
173.194.72.104    finance.google.com
173.194.72.104    fusion.google.com
173.194.72.104    geoauth.google.com
173.194.72.104    gg.google.com
173.194.72.104    ghs.google.com
173.194.72.104    ghs.l.google.com
173.194.72.104    ghs46.google.com
173.194.72.104    ghs46.l.google.com
173.194.72.104    googleapis.l.google.com
173.194.72.104    googleapis-ajax.google.com
173.194.72.104    googleapis-ajax.l.google.com
173.194.72.104    googlecode.l.google.com
173.194.72.104    google-public-dns-a.google.com
173.194.72.104    google-public-dns-b.google.com
173.194.72.104    goto.google.com
173.194.72.104    groups.google.com
173.194.72.104    groups.l.google.com
173.194.72.104    groups-beta.google.com
173.194.72.104    gxc.google.com
173.194.72.104    id.google.com
173.194.72.104    id.l.google.com
173.194.72.104    images.google.com
173.194.72.104    images.l.google.com
173.194.72.104    investor.google.com
173.194.72.104    jmt0.google.com
173.194.72.104    kh.google.com
173.194.72.104    kh.l.google.com
173.194.72.104    khm.google.com
173.194.72.104    khm.l.google.com
173.194.72.104    khm0.google.com
173.194.72.104    khm1.google.com
173.194.72.104    khm2.google.com
173.194.72.104    khm3.google.com
173.194.72.104    khmdb.google.com
173.194.72.104    khms.google.com
173.194.72.104    khms.l.google.com
173.194.72.104    khms0.google.com
173.194.72.104    khms1.google.com
173.194.72.104    khms2.google.com
173.194.72.104    khms3.google.com
173.194.72.104    labs.google.com
173.194.72.104    large-uploads.l.google.com
173.194.72.104    lh2.google.com
173.194.72.104    lh2.l.google.com
173.194.72.104    lh3.google.com
173.194.72.104    lh4.google.com
173.194.72.104    lh5.google.com
173.194.72.104    lh6.google.com
173.194.72.104    linkhelp.clients.google.com
173.194.72.104    local.google.com
173.194.72.104    m.google.com
173.194.72.104    map.google.com
74.125.21.98    maps.google.com
74.125.21.98    maps.l.google.com
74.125.21.98    maps-api-ssl.google.com
173.194.72.104    mars.google.com
173.194.72.104    mobile.l.google.com
173.194.72.104    mobile-gtalk.l.google.com
173.194.72.104    mobilemaps.clients.google.com
173.194.72.104    mt.google.com
173.194.72.104    mt.l.google.com
173.194.72.104    mt0.google.com
173.194.72.104    mt1.google.com
173.194.72.104    mt2.google.com
173.194.72.104    mt3.google.com
173.194.72.104    mtalk.google.com
173.194.72.104    mts.google.com
173.194.72.104    mts.l.google.com
173.194.72.104    mts0.google.com
173.194.72.104    mts1.google.com
173.194.72.104    mts2.google.com
173.194.72.104    mts3.google.com
173.194.72.104    music.google.com
173.194.72.104    music-streaming.l.google.com
173.194.72.104    mw1.google.com
173.194.72.104    mw2.google.com
203.208.41.144    news.google.com
203.208.41.144    news.l.google.com
173.194.72.104    pack.google.com
173.194.72.104    photos.google.com
173.194.72.104    photos-ugc.l.google.com
173.194.72.104    picasa.google.com
173.194.72.104    picasaweb.google.com
173.194.72.104    picasaweb.l.google.com
173.194.72.104    places.google.com
173.194.72.104    play.google.com
173.194.72.104    productforums.google.com
173.194.72.104    profiles.google.com
173.194.72.104    reader.google.com
173.194.72.104    safebrowsing.cache.l.google.com
173.194.72.104    safebrowsing.clients.google.com
173.194.72.104    safebrowsing.google.com
173.194.72.104    safebrowsing-cache.google.com
173.194.72.104    sb.google.com
173.194.72.104    sb.l.google.com
173.194.72.104    sb-ssl.google.com
173.194.72.104    sb-ssl.l.google.com
203.208.41.144    scholar.google.com
173.194.72.104    scholar.l.google.com
173.194.72.104    script.google.com
173.194.72.104    services.google.com
173.194.72.104    sites.google.com
173.194.72.104    sketchup.google.com
173.194.72.104    sketchup.l.google.com
173.194.72.104    spreadsheet.google.com
173.194.72.104    spreadsheets.google.com
173.194.72.104    spreadsheets.l.google.com
173.194.72.104    spreadsheets0.google.com
173.194.72.104    spreadsheets1.google.com
173.194.72.104    spreadsheets2.google.com
173.194.72.104    spreadsheets3.google.com
173.194.72.104    spreadsheets4.google.com
173.194.72.104    spreadsheets5.google.com
173.194.72.104    spreadsheets6.google.com
173.194.72.104    spreadsheets7.google.com
173.194.72.104    spreadsheets8.google.com
173.194.72.104    spreadsheets9.google.com
173.194.72.104    spreadsheets-china.l.google.com
173.194.72.104    suggestqueries.google.com
173.194.72.104    suggestqueries.l.google.com
173.194.72.104    support.google.com
173.194.72.104    tbn0.google.com
173.194.72.104    tbn1.google.com
173.194.72.104    tbn2.google.com
173.194.72.104    tbn3.google.com
173.194.72.104    toolbar.google.com
173.194.72.104    toolbarqueries.clients.google.com
173.194.72.104    toolbarqueries.google.com
173.194.72.104    toolbarqueries.l.google.com
173.194.72.104    tools.google.com
173.194.72.104    tools.l.google.com
173.194.72.104    translate.google.com
173.194.72.104    trends.google.com
173.194.72.104    upload.docs.google.com
173.194.72.104    upload.drive.google.com
173.194.72.104    uploads.code.google.com
173.194.72.104    uploadsj.clients.google.com
173.194.72.104    v3.cache1.c.docs.google.com
173.194.72.104    video.google.com
173.194.72.104    video-stats.l.google.com
173.194.72.104    voice.google.com
173.194.72.104    wallet.google.com
173.194.72.104    wire.l.google.com
173.194.72.104    writely.google.com
173.194.72.104    writely.l.google.com
173.194.72.104    writely-china.l.google.com
173.194.72.104    writely-com.l.google.com
173.194.72.104    www.l.google.com
173.194.72.104    www2.l.google.com
173.194.72.104    www3.l.google.com
173.194.72.104    www4.l.google.com
173.194.72.104    ytstatic.l.google.com
173.194.72.104    keep.google.com
173.194.72.104    0.client-channel.google.com
173.194.72.104    android.clients.google.com
74.125.21.98    www.google.com
173.194.72.104    accounts.l.google.com
173.194.72.104    checkout.l.google.com
173.194.72.104    sandbox.google.com
173.194.72.104    wifi.google.com
173.194.72.104    wifi.l.google.com
173.194.72.104    clients.l.google.com
173.194.72.104    clients1.google.com
173.194.72.104    clients2.google.com
173.194.72.104    clients3.google.com
173.194.72.104    clients4.google.com
173.194.72.104    clients5.google.com
173.194.72.104    clients7.google.com
74.125.21.115    checkout.google.com
74.125.21.84    accounts.google.com
173.194.72.104    m.google.com
74.125.21.51    adwords.google.com
173.194.72.104    talk.google.com
173.194.72.104    google.com

173.194.72.104    plus.google.com
173.194.72.104    plus.url.google.com
173.194.72.104    plusone.google.com

203.208.41.145    mail.google.com
74.125.25.189    filetransferenabled.mail.google.com
74.125.25.189    chatenabled.mail.google.com

173.194.72.104    apis.google.com
173.194.72.104    clients6.google.com
74.125.202.95    talkgadget.google.com

203.208.41.145    0-open-opensocial.googleusercontent.com
203.208.41.145    0-focus-opensocial.googleusercontent.com
203.208.41.145    1-focus-opensocial.googleusercontent.com
203.208.41.145    1-open-opensocial.googleusercontent.com
203.208.41.145    1-ps.googleusercontent.com
203.208.41.145    2-focus-opensocial.googleusercontent.com
203.208.41.145    2-open-opensocial.googleusercontent.com
203.208.41.145    2-ps.googleusercontent.com
203.208.41.145    3-focus-opensocial.googleusercontent.com
203.208.41.145    3-ps.googleusercontent.com
203.208.41.145    3hdrrlnlknhi77nrmsjnjr152ueo3soc-a-calendar-opensocial.googleusercontent.com
203.208.41.145    3-open-opensocial.googleusercontent.com
203.208.41.145    4-ps.googleusercontent.com
203.208.41.145    4fjvqid3r3oq66t548clrdj52df15coc-a-oz-opensocial.googleusercontent.com
203.208.41.145    53rd6p0catml6vat6qra84rs0del836d-a-oz-opensocial.googleusercontent.com
203.208.41.145    59cbv4l9s05pbaks9v77vc3mengeqors-a-oz-opensocial.googleusercontent.com
203.208.41.145    8kubpeu8314p2efdd7jlv09an9i2ljdo-a-oz-opensocial.googleusercontent.com
203.208.41.145    adstvca8k2ooaknjjmv89j22n9t676ve-a-oz-opensocial.googleusercontent.com
203.208.41.145    a-oz-opensocial.googleusercontent.com
203.208.41.145    blogger.googleusercontent.com
203.208.41.145    bt26mravu2qpe56n8gnmjnpv2inl84bf-a-oz-opensocial.googleusercontent.com
203.208.41.145    clients1.googleusercontent.com
203.208.41.145    clients2.googleusercontent.com
203.208.41.145    clients3.googleusercontent.com
203.208.41.145    clients4.googleusercontent.com
203.208.41.145    clients5.googleusercontent.com
203.208.41.145    clients6.googleusercontent.com
203.208.41.145    clients7.googleusercontent.com
203.208.41.145    code-opensocial.googleusercontent.com
203.208.41.145    debh8vg7vd93bco3prdajidmm7dhql3f-a-oz-opensocial.googleusercontent.com
203.208.41.145    doc-00-7o-docs.googleusercontent.com
203.208.41.145    doc-08-7o-docs.googleusercontent.com
203.208.41.145    doc-0c-7o-docs.googleusercontent.com
203.208.41.145    doc-0g-7o-docs.googleusercontent.com
203.208.41.145    doc-0s-7o-docs.googleusercontent.com
203.208.41.145    doc-10-7o-docs.googleusercontent.com
203.208.41.145    doc-14-7o-docs.googleusercontent.com
203.208.41.145    feedback.googleusercontent.com
203.208.41.145    googlehosted.l.googleusercontent.com
203.208.41.145    gp0.googleusercontent.com
203.208.41.145    gp1.googleusercontent.com
203.208.41.145    gp2.googleusercontent.com
203.208.41.145    gp3.googleusercontent.com
203.208.41.145    gp4.googleusercontent.com
203.208.41.145    gp5.googleusercontent.com
203.208.41.145    gp6.googleusercontent.com
203.208.41.145    hsco54a20sh11q9jkmb51ad2n3hmkmrg-a-oz-opensocial.googleusercontent.com
203.208.41.145    i8brh95qor6r54nkl52hidj2ggcs4jgm-a-oz-opensocial.googleusercontent.com
203.208.41.145    images1-focus-opensocial.googleusercontent.com
203.208.41.145    images2-focus-opensocial.googleusercontent.com
203.208.41.145    images3-focus-opensocial.googleusercontent.com
203.208.41.145    images4-focus-opensocial.googleusercontent.com
203.208.41.145    images5-focus-opensocial.googleusercontent.com
203.208.41.145    images6-focus-opensocial.googleusercontent.com
203.208.41.145    images7-focus-opensocial.googleusercontent.com
203.208.41.145    images8-focus-opensocial.googleusercontent.com
203.208.41.145    images9-focus-opensocial.googleusercontent.com
203.208.41.145    images-docs-opensocial.googleusercontent.com
203.208.41.145    images-oz-opensocial.googleusercontent.com
203.208.41.145    k6v18tjr24doa89b55o3na41kn4v73eb-a-oz-opensocial.googleusercontent.com
203.208.41.145    lh1.googleusercontent.com
203.208.41.145    lh2.googleusercontent.com
203.208.41.145    lh3.googleusercontent.com
203.208.41.145    lh4.googleusercontent.com
203.208.41.145    lh5.googleusercontent.com
203.208.41.145    lh6.googleusercontent.com
203.208.41.145    mail-attachment.googleusercontent.com
203.208.41.145    music.googleusercontent.com
203.208.41.145    music-onebox.googleusercontent.com
203.208.41.145    oauth.googleusercontent.com
203.208.41.145    ob7f2qc0i50kbjnc81vkhgmb5hsv7a8l-a-oz-opensocial.googleusercontent.com
203.208.41.145    ode25pfjgmvpquh3b1oqo31ti5ibg5fr-a-calendar.opensocial.googleusercontent.com
203.208.41.145    qhie5b8u979rnch1q0hqbrmbkn9estf7-a-oz-opensocial.googleusercontent.com
203.208.41.145    r70rmsn4s0rhk6cehcbbcbfbs31pu0va-a-oz-opensocial.googleusercontent.com
203.208.41.145    rbjhe237k979f79d87gmenp3gejfonu9-a-oz-opensocial.googleusercontent.com
203.208.41.145    s1.googleusercontent.com
203.208.41.145    s2.googleusercontent.com
203.208.41.145    s3.googleusercontent.com
203.208.41.145    s4.googleusercontent.com
203.208.41.145    s5.googleusercontent.com
203.208.41.145    s6.googleusercontent.com
203.208.41.145    spreadsheets-opensocial.googleusercontent.com
203.208.41.145    static.googleusercontent.com
203.208.41.145    t.doc-0-0-sj.sj.googleusercontent.com
203.208.41.145    themes.googleusercontent.com
203.208.41.145    translate.googleusercontent.com
203.208.41.145    u807isd5egseeabjccgcns005p2miucq-a-oz-opensocial.googleusercontent.com
203.208.41.145    upt14k1i2veesusrda9nfotcrbp9d7p5-a-oz-opensocial.googleusercontent.com
203.208.41.145    webcache.googleusercontent.com
203.208.41.145    www.googleusercontent.com
203.208.41.145    www-calendar-opensocial.googleusercontent.com
203.208.41.145    www-fc-opensocial.googleusercontent.com
203.208.41.145    www-focus-opensocial.googleusercontent.com
203.208.41.145    www-gm-opensocial.googleusercontent.com
203.208.41.145    www-kix-opensocial.googleusercontent.com
203.208.41.145    www-open-opensocial.googleusercontent.com
203.208.41.145    www-opensocial.googleusercontent.com
203.208.41.145    www-opensocial-sandbox.googleusercontent.com
203.208.41.145    www-oz-opensocial.googleusercontent.com
203.208.41.145    www-hangouts-opensocial.googleusercontent.com
203.208.41.145    www-onepick-opensocial.googleusercontent.com
203.208.41.145    books.googleusercontent.com
203.208.41.145    ci0.googleusercontent.com
203.208.41.145    ci1.googleusercontent.com
203.208.41.145    ci2.googleusercontent.com
203.208.41.145    ci3.googleusercontent.com
203.208.41.145    ci4.googleusercontent.com
203.208.41.145    ci5.googleusercontent.com
203.208.41.145    ci6.googleusercontent.com
203.208.41.145    doc-0g-9c-docs.googleusercontent.com

74.125.22.133    lh1.ggpht.com
74.125.22.133    lh2.ggpht.com
74.125.22.133    lh3.ggpht.com
74.125.22.133    lh4.ggpht.com
74.125.22.133    lh5.ggpht.com
74.125.22.133    lh6.ggpht.com
74.125.22.133    nt0.ggpht.com
74.125.22.133    nt1.ggpht.com
74.125.22.133    nt2.ggpht.com
74.125.22.133    nt3.ggpht.com
74.125.22.133    nt4.ggpht.com
74.125.22.133    nt5.ggpht.com

74.125.202.120    csi.gstatic.com
74.125.202.120    g0.gstatic.com
74.125.202.120    g1.gstatic.com
74.125.202.120    g2.gstatic.com
74.125.202.120    g3.gstatic.com
74.125.202.120    maps.gstatic.com
74.125.202.120    mt0.gstatic.com
74.125.202.120    mt1.gstatic.com
74.125.202.120    mt2.gstatic.com
74.125.202.120    mt3.gstatic.com
74.125.202.120    mt4.gstatic.com
74.125.202.120    mt5.gstatic.com
74.125.202.120    mt6.gstatic.com
74.125.202.120    mt7.gstatic.com
74.125.202.120    ssl.gstatic.com
203.208.41.144    t0.gstatic.com
203.208.41.144    t1.gstatic.com
203.208.41.144    t2.gstatic.com
203.208.41.144    t3.gstatic.com
74.125.202.120    www.gstatic.com

74.125.202.95    ajax.googleapis.com
74.125.202.95    chart.googleapis.com
74.125.202.95    fonts.googleapis.com
74.125.202.95    maps.googleapis.com
74.125.202.95    mt0.googleapis.com
74.125.202.95    mt1.googleapis.com
74.125.202.95    mt2.googleapis.com
74.125.202.95    mt3.googleapis.com
74.125.202.95    redirector-bigcache.googleapis.com
74.125.202.95    translate.googleapis.com
74.125.202.95    www.googleapis.com
74.125.202.95    content.googleapis.com

74.125.26.141    appspot.com
74.125.26.141    chrometophone.appspot.com
74.125.26.141    evolutionofweb.appspot.com
74.125.26.141    googcloudlabs.appspot.com
74.125.26.141    gv-gadget.appspot.com
74.125.26.141    moderator.appspot.com
74.125.26.141    newsfeed-dot-latest-dot-rovio-ad-engine.appspot.com
74.125.26.141    productideas.appspot.com
74.125.26.141    project-slingshot-gp.appspot.com
74.125.26.141    r2303.latest.project-slingshot-hr.appspot.com
74.125.26.141    r3085-dot-latest-dot-project-slingshot-gp.appspot.com
74.125.26.141    r3091-dot-latest-dot-project-slingshot-gp.appspot.com
74.125.26.141    r3101-dot-latest-dot-project-slingshot-gp.appspot.com
74.125.26.141    r3269-dot-latest-dot-project-slingshot-gp.appspot.com
74.125.26.141    r3432-dot-latest-dot-project-slingshot-hr.appspot.com
74.125.26.141    r4681-dot-latest-dot-project-slingshot-hr.appspot.com
74.125.26.141    r7647-dot-latest-dot-project-slingshot-hr.appspot.com
74.125.26.141    wcproxyx.appspot.com
74.125.26.141    balsamiqgdrive.appspot.com
74.125.26.141    www.appspot.com

74.125.26.82    autoproxy-gfwlist.googlecode.com
74.125.26.82    chromium.googlecode.com
74.125.26.82    closure-library.googlecode.com
74.125.26.82    earth-api-samples.googlecode.com
74.125.26.82    gmaps-samples-flash.googlecode.com
74.125.26.82    google-code-feed-gadget.googlecode.com
74.125.26.82    smarthosts.googlecode.com

173.194.72.104    blogsearch.google.cn
203.208.41.145    ditu.google.cn
173.194.72.104    gg.google.cn
173.194.72.104    id.google.cn
173.194.72.104    maps.gstatic.cn
173.194.72.104    m.google.cn
173.194.72.104    mt.google.cn
173.194.72.104    mt0.google.cn
173.194.72.104    mt1.google.cn
173.194.72.104    mt2.google.cn
173.194.72.104    mt3.google.cn
203.208.41.144    news.google.cn
203.208.41.144    scholar.google.cn
173.194.72.104    translate.google.cn
173.194.72.104    www.google.cn
173.194.72.104    www.gstatic.cn
173.194.72.104    clients1.google.com.hk
173.194.72.104    accounts.google.com.hk
173.194.72.104    blogsearch.google.com.hk
173.194.72.104    books.google.com.hk
173.194.72.104    desktop.google.com.hk
173.194.72.104    encrypted.google.com.hk
173.194.72.104    groups.google.com.hk
173.194.72.104    gxc.google.com.hk
173.194.72.104    id.google.com.hk
173.194.72.104    images.google.com.hk
173.194.72.104    m.google.com.hk
74.125.21.98    maps.google.com.hk
203.208.41.144    news.google.com.hk
173.194.72.104    picasaweb.google.com.hk
173.194.72.104    plus.url.google.com.hk
203.208.41.144    scholar.google.com.hk
173.194.72.104    toolbar.google.com.hk
173.194.72.104    toolbarqueries.google.com.hk
173.194.72.104    translate.google.com.hk
173.194.72.104    translate.google.com.hk
173.194.72.104    wenda.google.com.hk
74.125.21.98    www.google.com.hk
173.194.72.104    accounts.youtube.com
173.194.72.104    apiblog.youtube.com
173.194.72.104    help.youtube.com
173.194.72.104    insight.youtube.com
173.194.72.104    m.youtube.com
173.194.72.104    ytimg.l.google.com
173.194.72.104    www.youtube.com
173.194.72.104    i.ytimg.com
173.194.72.104    i1.ytimg.com
173.194.72.104    i2.ytimg.com
173.194.72.104    i3.ytimg.com
173.194.72.104    i4.ytimg.com
173.194.72.104    s.ytimg.com
173.194.72.104    android.googlesource.com
173.194.72.104    gerrit.googlesource.com
173.194.72.104    auth.keyhole.com
173.194.72.104    chrome.angrybirds.com
173.194.72.104    developer.android.com
173.194.72.104    domains.googlesyndication.com
173.194.72.104    earthengine.googlelabs.com
173.194.72.104    feeds.feedburner.com
173.194.72.104    g.co
173.194.72.104    goo.gl
173.194.72.104    golang.org
173.194.72.104    listen.googlelabs.com
203.208.41.145    m.googlemail.com
173.194.72.104    market.android.com
173.194.72.104    ngrams.googlelabs.com
173.194.72.104    www.googleadservices.com
173.194.72.104    www.googlelabs.com
173.194.72.104    www.googlesource.com
173.194.72.104    www.orkut.com
173.194.72.104    www.blogger.com
173.194.72.104    accounts.blogger.com
173.194.72.104    www.panoramio.com
173.194.72.104    panoramio.com
173.194.72.104    static.panoramio.com
173.194.72.104    tpc.googlesyndication.com
173.194.72.104    uberproxy.corp.google.com
173.194.72.104    developer.chrome.com
74.125.21.97    ssl.google-analytics.com
173.194.72.104    www.google-analytics.com
'''
target = "203.208.41.148"
pref = "203.208"
import httplib2
import re
import os
h = httplib2.Http()
h.follow_redirects = False
output = []
# compare lst1 to lst2
check_hamming = lambda lst1, lst2:sum(map(lambda x,y:1 if x==y else 0, lst1, lst2))
blacklisted = []
def check_blacklist(host):
    host = host.split('.')
    for b in blacklisted:
        b = b.split('.')
        if len(host) == len(b):
            if len(host) - check_hamming(host, b) < 2 and \
            ('google' not in host or host.index('google') > 1) and \
            'www' not in host: # admit *.google.com, www.*.* being blacklisted
                return True, '.'.join(b)
    return False, None
record_out = lambda x:output.append(x) or open('hosts.txt', 'a').write('%s\n' % x)
done = []
if os.path.exists('hosts.txt'):
    done = map(lambda x:x.split()[1] if x else '', open('hosts.txt').read().split('\n'))
for line in hosts.split('\n'):
    if not line:
        record_out('')
        continue
    ip, host = line.split()
    if host in done:
        print('skip %s' % host)
        continue
    if ip.startswith(pref):
        record_out(line)
        continue
    b = check_blacklist(host)
    if b[0]:
        print('%s similar to %s. blacklisted' % (host ,b[1]))
        record_out(line)
        continue
    try:
        ret = h.request('http://%s' % target, headers = {'Host':host})
    except httplib2.RedirectLimit:
        record_out(line)
        continue
    code = ret[0]['status']
    if code in ['200', '302']:
        print('%s: good!!!' % host)
        record_out('%s    %s' % (target, host))
    elif ret[1] == '<html><body><h1>404 Not Found</h1></body></html>\n':
        print('%s: no medicine to save' % host)
        record_out(line)
    elif code == '301' and len(re.findall('<A HREF="[https]+\://%s/.*?">here</A>' % host, ret[1])) >0:#or href="/xx/xx" un-done
        print('%s: local redirect, good!!!' % host)
        record_out('%s    %s' % (target, host))
    elif '200'<code<'400':
        if raw_input('%s: code is %s\nShow content? >' % (host, code)) == 'y':
            idx = 0
            while True:
                print(ret[1][idx:min(idx + 1500, len(ret[1])) + 1])
                if (raw_input('More:') or 'y') != 'y' or idx > len(ret[1]):
                    break
                idx += 1500
            if raw_input('use cn ip? >') == 'y':
                record_out('%s    %s' % (target, host))
            else:
                record_out(line)
        else:
            record_out(line)
    else:
        # if raw_input('%s:code is %s. use cn ip? >' % (host, code)) == 'y':
        #     record_out('%s    %s' % (target, host))
        # else:
        print('%s:code is %s. auto skip >' % (host, code))
        record_out(line)
        blacklisted.append(host)

#open('hosts.txt', 'w').write('\n'.join(output))