random sentence generator

random_sentence_generator.py

import re

from bs4 import BeautifulSoup
from markdown import markdown

from lightsteem.client import Client

import markovify


def markdown_to_text(markdown_string):
    """ Converts a markdown string to plaintext """

    # md -> html -> text since BeautifulSoup can extract text cleanly
    html = markdown(markdown_string)

    # remove code snippets
    html = re.sub(r'<pre>(.*?)</pre>', ' ', html)
    html = re.sub(r'<code>(.*?)</code >', ' ', html)

    # extract text
    soup = BeautifulSoup(html, "html.parser")
    text = ''.join(soup.findAll(text=True))

    return text


def clear_noise(text):
    TAG_RE = re.compile(r'<[^>]+>')
    text = TAG_RE.sub('', text)
    text = markdown_to_text(text)
    return text


def random_post_generator(username):
    c = Client()
    print("Fetching main posts...")
    main_posts = c.get_discussions_by_blog({"limit": 100, "tag": username})
    main_comments = c.get_discussions_by_comments({"limit": 100, "start_author": username})
    all_text_data = ""
    for post in main_posts + main_comments:
        if post["author"] != username:
            continue

        all_text_data += clear_noise(post["body"])

    # build the model
    text_model = markovify.Text(all_text_data)

    for i in range(0, 10):
        print(text_model.make_short_sentence(140))


if __name__ == '__main__':
    random_post_generator('ned')