Some API analysis scripts

chklicenses.py

# check for license inconsistencies from a indicator list via stdin - not terribly efficient

import requests
import sys
import re

response = requests.get('https://api.worldbank.org/v2/en/sources?format=json&per_page=1000')
result = response.json()
sources = [elem['id'] for elem in result[1]]
series = {}

for row in sys.stdin:
    reg = re.match('^.+\s+(\S+)\s+(Open|Restricted)', row)
    if reg is not None:
        series[reg.group(1)] = {}

print series
        
for src in sources:
    sys.stderr.write('Querying source {}\n'.format(src))
    url = 'https://api.worldbank.org/v2/en/sources/{}/indicators?format=json&per_page=10000'.format(src)
    response = requests.get(url)
    result = response.json()
    for elem in result[1]:
        id = elem['id']
        if series.get(id) is not None:
            sys.stderr.write('Getting license for {}/{}\n'.format(src, id))
            metaurl = 'https://api.worldbank.org/v2/en/sources/{}/indicators/{}/metadata?format=json&per_page=10000'.format(src, id)
            license = 'n/a'
            try:
                response2 = requests.get(metaurl)
                result2  = response2.json()
                for field in result2['source'][0]['concept'][0]['variable'][0]['metatype']:
                    if field['id'] == 'License_Type':
                        license = field['value']
                        break
            except:
                license = 'ERROR'

            series[id][src] = license

for k,i in series.iteritems():
    if len(set(i.values())) > 1:
        row = [k]
        row.extend(['{}-{}'.format(k,v) for k,v in i.iteritems()])
        print ','.join(row)

licenses.py

# report license for all indicators (from each indicators primary source ID)
import requests

response = requests.get('https://api.worldbank.org/v2/en/indicators?format=json&per_page=20000')
series = response.json()[1]

for elem in series:
    url = 'https://api.worldbank.org/v2/en/sources/{}/indicator/{}/metadata?format=json'.format(elem['source']['id'], elem['id'])
    response = requests.get(url)
    license = None
    try:
        metadata = response.json()
        for field in metadata['source'][0]['concept'][0]['variable'][0]['metatype']:
            if field['id'] == 'License_Type':
                license = field['value']
                break
    except:
        license = 'ERROR'

    print "%2s %-30s %s" % (elem['source']['id'], elem['id'], license if license else 'n/a')

sources.py

# 1-to-many report of all indicators and sources

import requests
import sys

response = requests.get('https://api.worldbank.org/v2/en/sources?format=json&per_page=1000')
result = response.json()
sources = [elem['id'] for elem in result[1]]
series = {}

for src in sources:
    sys.stderr.write('Querying source {}\n'.format(src))
    url = 'https://api.worldbank.org/v2/en/sources/{}/indicators?format=json&per_page=10000'.format(src)
    response = requests.get(url)
    result = response.json()
    for elem in result[1]:
        id = elem['id']
        if series.get(id) is None:
            series[id] = []

        series[id].append(src)

keys = series.keys()
keys.sort()
for k in keys:
    row = [k, str(len(series[k]))]
    row.extend(series[k])
    print ','.join(row)