newspaper-server.py

import argparse
import functools
import sys
import urllib.parse
import xmlrpc.server

import newspaper
import selenium.webdriver

_requires_webdriver = set()

argument_parser = argparse.ArgumentParser()
argument_parser.add_argument('--port', type=int, default=8100)


@functools.lru_cache(1)
def get_webdriver():
    return selenium.webdriver.Chrome()


def get_newspaper_article_html(url, html=None):
    netloc = urllib.parse.urlsplit(url).netloc
    article = newspaper.Article(url, keep_article_html=True)
    if netloc in _requires_webdriver:
        webdriver = get_webdriver()
        webdriver.get(url)
        html = webdriver.execute_script('return document.documentElement.outerHTML')
        article.download(html)
        article.parse()
        return article.article_html

    article.download(html)
    article.parse()
    if not article.article_html:
        _requires_webdriver.add(netloc)
        return get_newspaper_article_html(url, None)

    return article.article_html


def main(arguments):
    server = xmlrpc.server.SimpleXMLRPCServer(('localhost', arguments.port))
    server.register_function(get_newspaper_article_html)

    try:
        print(f'Newspaper server is running on http://localhost:{arguments.port}/...')
        server.serve_forever()
    except KeyboardInterrupt:
        print('Shutting down...')


if __name__ == '__main__':
    arguments = argument_parser.parse_args()
    sys.exit(main(arguments))