crawler3.py

df = pd.DataFrame({'Team': first_col,
     'Summary': summary,
     'History': history,
     'Team_Page': teams_links,
     'Location': locations,
     'Stadium': stadiums,
     'Stadiums_Capcity': stadiums_capcity
    })

# crawling the history of those three clubs were somehow tricky so I had to hard code the section names myself
df.loc[df['History'].isnull(),'History'] = wikipedia.WikipediaPage(u'FC Barcelona').section(u'1899–1922: Beginnings'),\
                                           wikipedia.WikipediaPage(u'CD Leganés').section(u'History'),\
                                           wikipedia.WikipediaPage(u'Valencia CF').section(u'History')

#construct a different variation to the team/ club name
Team_alt = pd.Series(['Deportivo Alaves',
            'Athletic Bilbao',
            'Atletico Madrid' ,
            'Barca',
            'RC Celta de Vigo',
            'SD Eibar',
            'RCD Espanyol',
            'Getafe CF',
            'Girona FC',
            'SD Huesca',
            'Leganes',
            'Levante UD',
            'Rayo Vallecano',
            'Real Betis',
            'Real Madrid CF',
            'Real Sociedad',
            'Sevilla FC',
            'Valencia CF',
            'Real Valladolid',
            'Villarreal CF'])
df['Team_alt'] = Team_alt.values

df.to_csv("data/Teams_data.csv", encoding="utf-8", index=False)