Download yahoo answers sitemap

yahoo_sitemap.py

import requests
import xml.etree.ElementTree as ET
import json
import multiprocessing

answer_urls = []

def do_work(url):
	print(url)
	urls = []
	r = requests.get(url)
	file_root = ET.fromstring(r.text)
	for child in file_root:
		for loc in child:
			urls.append(loc.text)
	return urls

if __name__ == "__main__":
	allxml_requests = requests.get('https://answers.yahoo.com/sitemaps/sitemap-us.xml')
	root = ET.fromstring(allxml_requests.text)
	urls_top_level = []
	for sitemap in root:
		for s in sitemap:
			if s.tag == '{http://www.sitemaps.org/schemas/sitemap/0.9}loc':
				urls_top_level.append(s.text)

	the_pool = multiprocessing.Pool(int(multiprocessing.cpu_count()))
	for results in the_pool.imap_unordered(do_work, urls_top_level):
		answer_urls = answer_urls + results
		
		print(len(answer_urls))
	the_pool.close()
	the_pool.join()

	json.dump(answer_urls,open('yahoo_answers.json','w'),indent=2)