genlist.py

from bs4 import BeautifulSoup
from urllib.request import urlopen
import sys


def gen_url(baseurl="https://www.progressivedatajobs.org/",
            page=1):
    if page == 1:
        return(baseurl)
    else:
        return(baseurl + "page%s/" % page)


def gen_list(url):
    soup = BeautifulSoup(urlopen(url))
    links = soup.find_all('h2', "archive__item-title")
    return([gen_html(link) for link in links])


def gen_html(link):
    url = link.find('a').get('href')
    title = link.get_text().strip()
    return("<p><a href='%s'>%s</a></p>" % (url, title))

if __name__ == "__main__":

    if len(sys.argv) == 1:
        num_pages = 1
    else:
        num_pages = int(sys.argv[1])

    links = []
    for i in range(1, num_pages + 1):
        links = links + gen_list(gen_url(page=i))

    with open('recentposts.txt', 'w') as f:
        for link in links:
            f.write("%s\n" % link)