salgo60/ScrapeAdelsVapen.py

## ScrapeAdelsVapen.py
import requests
import urllib.request
import time
from bs4 import BeautifulSoup
def cleanupstring(s):
    out_s = s
    while '  ' in out_s:
        out_s = out_s.strip().replace('  ', ' ')
    return out_s, len(s)-len(out_s)

urls = [
    'https://www.adelsvapen.com/wiki/index.php?title=Special%3AAlla+sidor&from=&to=&namespace=0&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Bauman&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Cruus+af+Edeby+nr+69&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Fr%C3%B6lich+nr+49&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Hadorph+nr+2243&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Lagerheim+nr+326&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=M%C3%B6rner+nr+91&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Rosensk%C3%B6ld+nr+287&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Stiernsparre+nr+1819&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Von+Bradke&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Von+Kr%C3%A6mer+nr+1959&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Von+Staal&hideredirects=1',
    'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Witte+von+Wittenheim&hideredirects=1']
adelsvapen = 0
for url in urls:
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    soup.findAll('a')
    one_a_tag = soup.findAll('a')[0]
    #    print ( one_a_tag )
    #   link = one_a_tag['href']
    time.sleep(1)
    for i in range(1, len(soup.findAll('a'))):
        one_a_tag = soup.findAll('a')[i]
        one_a_tag_href = one_a_tag['href']
        if  "/genealogi/" in one_a_tag_href \
                and not "Adelsvapen-Wiki" in one_a_tag_href \
                and not "Special"  in one_a_tag_href \
                and not "Huvudsida"  in one_a_tag_href:
            #print(one_a_tag.text,"|",one_a_tag.text.replace(" ","_").replace("%C3%84","Ä"),"|",one_a_tag_href.replace("/genealogi/",""))
            print(one_a_tag.text.replace(" ","_").replace("%C3%84","Ä"))

        #print(cleanupstring(one_a_tag.text.strip('\n').strip('\t').replace("\n","")), ";", one_a_tag['href'].replace(".htm",""))
	import requests
	import urllib.request
	import time
	from bs4 import BeautifulSoup
	def cleanupstring(s):
	out_s = s
	while ' ' in out_s:
	out_s = out_s.strip().replace(' ', ' ')
	return out_s, len(s)-len(out_s)

	urls = [
	'https://www.adelsvapen.com/wiki/index.php?title=Special%3AAlla+sidor&from=&to=&namespace=0&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Bauman&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Cruus+af+Edeby+nr+69&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Fr%C3%B6lich+nr+49&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Hadorph+nr+2243&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Lagerheim+nr+326&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=M%C3%B6rner+nr+91&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Rosensk%C3%B6ld+nr+287&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Stiernsparre+nr+1819&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Von+Bradke&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Von+Kr%C3%A6mer+nr+1959&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Von+Staal&hideredirects=1',
	'https://www.adelsvapen.com/wiki/index.php?title=Special:Alla_sidor&from=Witte+von+Wittenheim&hideredirects=1']
	adelsvapen = 0
	for url in urls:
	response = requests.get(url)
	soup = BeautifulSoup(response.text, "html.parser")
	soup.findAll('a')
	one_a_tag = soup.findAll('a')[0]
	# print ( one_a_tag )
	# link = one_a_tag['href']
	time.sleep(1)
	for i in range(1, len(soup.findAll('a'))):
	one_a_tag = soup.findAll('a')[i]
	one_a_tag_href = one_a_tag['href']
	if "/genealogi/" in one_a_tag_href \
	and not "Adelsvapen-Wiki" in one_a_tag_href \
	and not "Special" in one_a_tag_href \
	and not "Huvudsida" in one_a_tag_href:
	#print(one_a_tag.text,"\|",one_a_tag.text.replace(" ","_").replace("%C3%84","Ä"),"\|",one_a_tag_href.replace("/genealogi/",""))
	print(one_a_tag.text.replace(" ","_").replace("%C3%84","Ä"))

	#print(cleanupstring(one_a_tag.text.strip('\n').strip('\t').replace("\n","")), ";", one_a_tag['href'].replace(".htm",""))