get_page.py

# coding:utf-8

import json
import base64
import urllib
import urllib2
import time
import tweepy
from bs4 import BeautifulSoup


def authentication(username,password,url):
    p_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    p_mgr.add_password(None,url,username,password)
    opener = urllib2.build_opener(urllib2.HTTPSHandler(),urllib2.HTTPBasicAuthHandler(p_mgr))
    urllib2.install_opener(opener)
    request = urllib2.Request(url,"\r\n")
    response = urllib2.urlopen(request)
    return response


def create_oauth(consumer_key,consumer_secret,access_key,access_secret):
    auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
    auth.set_access_token(access_key,access_secret)
    return auth


if __name__=='__main__':
    consumer_key = ''
    consumer_secret = ''
    access_key = ''
    access_secret = ''

    url = ""
    username = ""
    password = ""
    board_url = url + "/2016/boards/new.html"

    old_list = []
    new_list = []

    fr = open("./link_list",'r')
    lines = fr.readlines()
    fr.close()
 
    for line in lines:
        old_list.append(line.replace('\n',''))

    res = authentication(username,password,board_url)
    data = ''.join(res.readlines())
    data = data.decode('shift_jisx0213')

    soup = BeautifulSoup(data,'html.parser')
    
    for img in soup.find_all("a"):
        new_list.append(img['href'])

    fw = open('./link_list','w')
    for line in new_list:
        fw.write(line+"\n")
    fw.close()
    
    auth = create_oauth(consumer_key,consumer_secret,access_key,access_secret)
    api = tweepy.API(auth)
    me = api.me().screen_name
 
    difference = list(set(new_list)-set(old_list))
    
    for line in difference:
        each_url = url+line
        res = authentication(username,password,each_url)
        data = ''.join(res.readlines())
        data = convert_to_utf8(data)
        soup = BeautifulSoup(data,'html.parser')
        
        text = soup.find_all("body")[0].get_text()
        api.send_direct_message(me,text=text)
        time.sleep(1)