https://www.facebook.com/groups/1781724435404945/permalink/2562204574023590/

sitemap.py

#!/usr/bin/env python3

import re
import requests
import datetime
import dateutil.parser
from bs4 import BeautifulSoup

# url = "http://reflectivedisequilibrium.blogspot.com/sitemap.xml"
# url = "https://terrytao.wordpress.com/sitemap.xml"
# url = "https://deluks917.wordpress.com/sitemap.xml"
# url = "https://nostalgebraist.tumblr.com/sitemap.xml"
# url = "https://grognor.tumblr.com/sitemap.xml"
# url = "https://www.gwern.net/sitemap.xml"
url = "http://www.overcomingbias.com/sitemap.xml"
# url = "https://blog.jessriedel.com/sitemap.xml"

def find_all_urls(feed_url):
    headers = {'User-Agent': 'Mozilla/5.0 (Ubuntu; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0'}
    r = requests.get(feed_url, headers=headers)
    soup = BeautifulSoup(r.content, "lxml-xml")
    result = []
    for url in soup.find_all('url'):
        link = url.loc.text
        if parse_url_date(link):
            result.append((link, parse_url_date(link)))
        elif url.lastmod:
            result.append((link, dateutil.parser.parse(url.lastmod.text)))
        else:
            result.append((link, None))
    for sitemap in soup.find_all('sitemap'):
        link = sitemap.loc.text
        result.extend(find_all_urls(link))
    return result

def parse_url_date(url):
    m1 = re.match(r'.*/(\d\d\d\d/\d\d/\d\d)/', url)
    m2 = re.match(r'.*/(\d\d\d\d/\d\d)/', url)
    if m1:
        return dateutil.parser.parse(m1.group(1))
    if m2:
        return dateutil.parser.parse(m2.group(1))
    return None

print("""<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0">
<channel>""")

for u, d in find_all_urls(url):
    print(" <item>")
    url_parts = u.split('/')
    if url_parts[-1]:
        title = url_parts[-1].replace("-", " ").replace("_", " ")
    else:
        title = url_parts[-2].replace("-", " ").replace("_", " ")
    print("  <title>{}</title>".format(title))
    print("  <link>{}</link>".format(u))
    if d:
        print("  <pubDate>{}</pubDate>".format(d.strftime("%a, %d %b %Y %H:%M:%S %z")))
    print(" </item>")

print("""</channel>
</rss>""")