maps the labels of Wikidata QIDs to the appropriate Wikipedia pages

WD_WP_disease_mapping.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Sebastian Burgstaller'
__licence__ = 'GPLv3'

import PBB_Core
import pandas as pd
import requests
import os



col_names = ['QID', 'Wikidata label','Wikipedia title', 'Wikipedia pageid', 'has interwiki link']

data_types = {x: object for x in col_names}
data_types.update({'has interwiki link': bool})


append = True

if os.path.isfile('./WD_to_WP_disease_map.csv') and append:
    wd_to_wp_map = pd.read_csv('./WD_to_WP_disease_map.csv', index_col=0, dtype=data_types)
else:
    wd_to_wp_map = pd.DataFrame(columns=col_names)

wd_disease_items = PBB_Core.WDItemList('CLAIM[557] or CLAIM[699]').wditems['items']

print(wd_to_wp_map.dtypes)

print(len(wd_disease_items))
print(len(set(wd_disease_items)))

for count, item in enumerate(wd_disease_items):
    print(item)
    if str(item) in wd_to_wp_map['QID'].values and append:
        print('skipping', item)
        count += 1
        continue

    # print(count, item)
    wd_object = PBB_Core.WDItemEngine(wd_item_id='Q{}'.format(item))
    wd_json = wd_object.wd_json_representation

    wd_to_wp_map.loc[count, 'QID'] = item

    label = ''
    sitelink = ''

    if 'labels' in wd_json:
        if 'en' in wd_json['labels']:
            label = wd_json['labels']['en']['value']
            # print(count, label)
            wd_to_wp_map.loc[count, 'Wikidata label'] = label

    if 'sitelinks' in wd_json:
        if 'enwiki' in wd_json['sitelinks']:
            sitelink = wd_json['sitelinks']['enwiki']['title']
            # print(count, sitelink)

            wd_to_wp_map.loc[count, 'Wikipedia title'] = sitelink
            wd_to_wp_map.loc[count, 'has interwiki link'] = True

    try:
        params = {
            'action': 'query',
            'titles': label,
            'format': 'json'
        }

        reply = requests.get('https://en.wikipedia.org/w/api.php', params=params)

        reply_json = reply.json()
        print(reply_json)

        if label != '':
            for pageid in reply_json['query']['pages']:
                if pageid != '-1':
                    wd_to_wp_map.loc[count, 'Wikipedia pageid'] = pageid
                    wd_to_wp_map.loc[count, 'Wikipedia title'] = reply_json['query']['pages'][pageid]['title']

    except requests.HTTPError as e:
        print(e)

    print(
        'Q{}'.format(wd_to_wp_map.loc[count, 'QID']),
        wd_to_wp_map.loc[count, 'Wikidata label'],
        wd_to_wp_map.loc[count, 'Wikipedia pageid'],
        wd_to_wp_map.loc[count, 'Wikipedia title']
    )

    # if count > 100:
    #     break

    if count % 100 == 0:
            wd_to_wp_map.to_csv('./WD_to_WP_disease_map.csv', index=True, encoding='utf-8', header=True)
            print('count:', count)

wd_to_wp_map.to_csv('./WD_to_WP_disease_map.csv', index=True, encoding='utf-8', header=True)