Text mining and scrapping from Wikipedia (List of Scientists)

wiki_text_mining.py

import wikipedia
import pandas as pd
import string

page = wikipedia.page("List of chemists")
scientists_dict = {"scientist": [], "summary": [], "birth_year": []}
for p in page.links[:2]:
	scientist = wikipedia.page(p)
	scientists_dict["scientist"].append(p)
	summary = scientist.summary
	scientists_dict["summary"].append(summary)
	# split check each word is_numeric len == 4, first
	print(p)
	for s in summary.split(" "):
		s = s.translate(str.maketrans('', '', string.punctuation))
		if s.isnumeric() and len(s) == 4:
			scientists_dict["birth_year"].append(s)
			break
	# regex
	# print(scientist.summary)

df = pd.DataFrame(scientists_dict)
df.to_csv("out.csv", index=False)