Create a gist now

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Parse and dump a sitemap (using Python)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
Inspired by Craig Addyman (http://www.craigaddyman.com/parse-an-xml-sitemap-with-python/)
Enhanced by Viktor Petersson (http://viktorpetersson.com) / @vpetersson
Enhanced by Jari Turkia (https://blog.hqcodeshop.fi/) / @HQJaTu
"""
from bs4 import BeautifulSoup
import requests
from urllib.parse import urlparse
def get_sitemap(url):
get_url = requests.get(url)
if get_url.status_code == 200:
return get_url.text
else:
print('Unable to fetch sitemap: %s.' % url)
def process_sitemap(s):
soup = BeautifulSoup(s, 'lxml')
result = []
for loc in soup.findAll('loc'):
result.append(loc.text)
return result
def is_sub_sitemap(url):
parts = urlparse(url)
if parts.path.endswith('.xml') and 'sitemap' in parts.path:
return True
else:
return False
def parse_sitemap(s):
sitemap = process_sitemap(s)
result = []
while sitemap:
candidate = sitemap.pop()
if is_sub_sitemap(candidate):
sub_sitemap = get_sitemap(candidate)
for i in process_sitemap(sub_sitemap):
sitemap.append(i)
else:
result.append(candidate)
return result
def main():
sitemap = get_sitemap('https://www.cloudsigma.com/sitemap.xml')
url_count = 0
for url in parse_sitemap(sitemap):
url_count += 1
print("%5d) %s" % (url_count, url))
print("-end-of-list-")
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment