Text extraction

requirements

beautifulsoup4==4.3.2
python-readability
requests

text_extraction.py

import bs4
import requests
from readability import readability


def get_summary(url):
    response = requests.get(url)
    document = readability.Document(response.content)
    body = bs4.BeautifulSoup(document.summary()).get_text()

    return {'text': body, 'title': document.short_title()}