blogger

blogger.py

import urllib.request
import bs4
import time


def fetch_page(url):
    html = urllib.request.urlopen(url).read().decode('utf-8')
    soup = bs4.BeautifulSoup(html)
    return soup


def get_content(soup):
    content = soup.find('div', class_='post-body entry-content')
    for i in content.find_all('div', class_='amazlet-box'):
        i.extract()
    return content.text


def get_next(soup):
    try:
        next_url = soup.find('a', class_='blog-pager-older-link').get('href')
        return next_url
    except:
        raise


if __name__ == "__main__":
    blog_url = 'http://awakara.blogspot.jp/'
    s = fetch_page(blog_url)
    u = get_next(s)
    t = get_content(s)

    while u != None:
        s = fetch_page(u)
        t += get_content(s)
        try:
            u = get_next(s)
        except:
            break

        time.sleep(1)

    t.replace(u'\xa0', u' ')

    f = open('text.txt', 'w')
    f.write(t)
    f.close()