config.py

config.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

#config for haaretz premium checks
#host to check
#edgecast
# host = '93.184.219.20';

#Akamai
host = '82.166.201.147';
#check_site='heb'
check_site='eng'






#urls to monitor
url = {'heb': 'http://www.haaretz.co.il/news/world/america/.premium-1.2577467', 'eng': 'http://www.haaretz.com/news/diplomacy-defense/.premium-1.640043'}

#Rusr limit
Rusr_limit={'heb': 9, 'eng': 6}

userid = 12345

key = 'Paywall4all'

#allowed referer array
referers = {}
referers['heb'] = ['www.google.ca', 'news.google.ca', 'webcache.googleusercontent.com', 'www.twitter.com', 'www.themarker.com', 'www.usit.co.il', 'link4u.co.il', 'www.mouse.co.il', 'mobile.twitter.com', 'bitly.com', 'htz.li', 'paid.outbrain.com', '.google.co', '.start.co.il', '.wikipedia.org', 'http://tools.wmflabs.org/', '.facebook.com']
referers['eng'] = ['reblaze.com', '.google.co', '.google.ca', '.redit.com', 'drudgereport.com', 'facebook.com', 'forward.com', '.wikipedia.org', '.reddit.com', 'http://t.co', 'linkedin.co', 'http://ht.ly', 'gawker.com', 'www.stumbleupon.com', 'www.tumblr.com', 'www.buzzfeed.com', 'www.thedailybeast.com', 'www.huffingtonpost.com', 'webcache.google', 'twitter.com']



#allowed useragent array
user_agents = {}
user_agents['heb'] = ['facebook', 'bing', 'twitter', 'AdsBot-Google', 'Feedfetcher-Google', 'Googlebot', 'msnbot', 'Yahoo', 'outbrain', 'Mediapartners']
user_agents['eng'] = ['facebook', 'bing', 'twitter', 'adsbot-google', 'feedfetcher-google', 'msnbot', 'yahoo', 'mediapartners', 'outbrain', 'Googlebot']




# premium string, if this string exist the article is blocked
string = {'heb': 'כתבה זו זמינה למנויים בלבד','eng': 'register with haaretz'}

# ---- cookies ----
#cookie user details
l_c = {'eng':'engsso','heb':'tmsso'}

#cookie premium users
p_c = {'eng':'Hdc','heb':'Htz'}

#cookie register users

test.py

#! /usr/bin/python
# -*- coding: utf-8 -*-


import config
import urllib2
import hashlib
from termcolor import colored

site=config.check_site
host=config.host
url=config.url[site]
domain = url.split('/')[2]
uri=url[url.index('/',7):]
if domain == 'www.haaretz.com':
    site = 'eng'
elif domain == 'www.haaretz.co.il':
    site = 'heb'
else:
    raise Exception('Error: bad url')









# check unknown user 
def check_unknown_user():
    # return true if unknown user get the blocked page
    u="http://" + host  + uri
    headers= {'Host': domain}
    request = urllib2.Request(u, None, headers)
    socket = urllib2.urlopen(request)
    if socket.code == 200:
        html=socket.read()
        if (html.find(config.string[site]) > 0):
            return True
        else:
            return False
    else:
        raise Exception('Error: got status: ' + socket.code + 'on url: ' + u)
        



# check referer 
def check_referer():
    # return true if reqest get the can view open content if recived from white list referer
    u="http://" + host  + uri
    headers= {'Host': domain}
    failed = []
    success = []
    for referer in config.referers[site]:
        headers['Referer'] = 'http://'+referer + '/haaretz_test'
        request = urllib2.Request(u, None, headers)
        socket = urllib2.urlopen(request)
        if socket.code == 200:
            html=socket.read()
            if (html.find(config.string[site]) > 0):
                # we find the blocked text
                failed.append(referer)
            else:
                success.append(referer)
        else:
            raise Exception('Error: got status: ' + socket.code + 'on url: ' + u)
    if  failed:
        msg = "Warning: white listed referer get blocked content, referer: " + str(failed)
        return False
    else:
        return True
    
    
                
                
                
            
    

# check user-agent
def check_user_agent():
    # return true if reqest get the can view open content if recived from white list referer
    u="http://" + host  + uri
    headers= {'Host': domain}
    failed = []
    success = []
    for user_agent in config.user_agents[site]:
        headers['User-Agent'] = 'haaretz-test-UA: '+user_agent 
        request = urllib2.Request(u, None, headers)
        socket = urllib2.urlopen(request)
        if socket.code == 200:
            html=socket.read()
            if (html.find(config.string[site]) > 0):
                # we find the blocked text
                failed.append(user_agent)
            else:
                success.append(user_agent)
        else:
            raise Exception('Error: got status: ' + socket.code + 'on url: ' + u)
    if  failed:
        msg = "Warning: white listed User-Agent get blocked content, User-Agent: " + str(failed)
        return False
    else:
        return True
    
    
                
            



# check registerd users
def check_registered():
    limit=config.Rusr_limit[site] 
    if check_registered_user(limit-1) and not check_registered_user(limit):
        return True
    else:
        return False
        

def check_registered_user(rusr):
    u="http://" + host  + uri
    headers= {'Host': domain}
    cookie = config.l_c[site]+'=userId%3D'+str(config.userid)+';'+ config.p_c[site] + 'Rusr=' + str(rusr)  
    headers['Cookie']= cookie
    request = urllib2.Request(u, None, headers)
    socket = urllib2.urlopen(request)
    if socket.code == 200:
        html=socket.read()
        if (html.find(config.string[site]) > 0):
            #registerd user limit >= : " + str(rusr) 
            return False
        else:
            return True
    else:
        raise Exception('Error: got status: ' + socket.code + 'on url: ' + u)
        



# check paying user

def check_paying_user():
    u="http://" + host  + uri
    puser = hashlib.md5(str(config.userid)+config.key).hexdigest()
    headers= {'Host': domain}
    cookie = config.l_c[site]+'=userId%3D'+str(config.userid)+';'+ config.p_c[site] + 'Pusr=' + puser
    headers['Cookie']= cookie
    request = urllib2.Request(u, None, headers)
    socket = urllib2.urlopen(request)
    if socket.code == 200:
        html=socket.read()
        if (html.find(config.string[site]) > 0): 
            #Paying user blocked"
            return False
        else:            
            return True
    else:
        raise Exception('Error: got status: ' + socket.code + 'on url: ' + u)
        




# check print edition

print 'Premium checks...'


print('Check unknown user (need to be blocked)........'),
if check_unknown_user():
    print colored('Pass', 'green')
else:
    print colored('Fail', 'red')
    
print('Check referer .........'),
if check_referer():
    print colored('Pass', 'green')
else:
    print colored('Fail', 'red')
    

print('Check User-Agent .........'),
if check_user_agent():
    print colored('Pass', 'green')
else:
    print colored('Fail', 'red')
        

print('Check rgistered users .........'),
if check_registered():
    print colored('Pass', 'green')
else:
    print colored('Fail', 'red')
    


print('Check Paying user .........'),
if check_paying_user():
    print colored('Pass', 'green')
else:
    print colored('Fail', 'red')
    

Premium checks...
Check unknown user (need to be blocked)........ Pass
Check referer ......... Pass
Check User-Agent ......... Pass
Check rgistered users ......... Pass
Check Paying user ......... Pass