Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Extracting article contents
# Requires: requests and readability-lxml ( (pip install requests readability-lxml)
import requests
def extract_article_contents(url):
Fetch the main body of content
Returns the html, a plaintext version, the title and subtitle
from readability.readability import Document
# TODO: Needs lots of exception handling
html = requests.get(url).text
doc = Document(html)
summary_html = doc.summary()
# Fairly naive: Pull out the contents of paragraph tags
paragraphs_html = re.findall('<p>(.*?)</p>', summary_html)
paragraphs = [re.sub('<(.*?)>', '', para) for para in paragraphs_html]
summary_text = '\n\n'.join(paragraphs)
# Hack for sites without p tags
if not summary_text:
# Replace line breaks with double new lines (paragraphs)
summary_text = re.sub('<(br.*?)>', '\n\n', summary_html)
# Remove all html
summary_text = re.sub('<(.*?)>', ' ', summary_text)
# Remove whitespace
summary_text = re.sub('\n\n\n+', '\n\n', summary_text)
summary_text = re.sub('[ \t\r\f\v][ \t\r\f\v][ \t\r\f\v]+', ' ', summary_text)
summary_text = summary_text.strip()
title = doc.short_title()
subtitle = doc.title().replace(title, '').strip(' |-:/')
# TODO: Return a named tuple
return summary_html, summary_text, title, subtitle
# Extract the article text using readability
html, text, title, subtitle = extract_article_contents(url)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment