Created
December 22, 2018 19:42
-
-
Save dojutsu-user/3de936406aee2c7b107f4c84be2cb464 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def get_organizations_list_with_links(year_link): | |
"""Get all organisations and their links | |
:param year_link: Valid link to the list of organisations of a specific year | |
:type year_link: str | |
:returns: A dictionary with the names of the organisations as keys and their links as values | |
:rtype: dict | |
""" | |
response = get_response(year_link) | |
if response.ok: | |
soup = BeautifulSoup(response.text, 'html.parser') | |
orgs_li = soup.find_all( | |
'li', 'mdl-list__item mdl-list__item--one-line') | |
orgs_dict = {} | |
for orgs_html in orgs_li: | |
org_name = orgs_html.select('a')[0].text.replace('\n', '') | |
relative_link = orgs_html.select('a')[0].get('href') | |
full_link = HOME_PAGE + relative_link | |
orgs_dict[org_name] = full_link | |
return orgs_dict | |
else: | |
print('Something Went Wrong') | |
print(f'Status Code: {response.status_code}') | |
sys.exit(1) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment