import-chrome-compatibility-data.py

#!/usr/bin/env python3

import sys
import json
import urllib.request
import urllib.error
import re
from collections import OrderedDict

import bs4

def parsecompat(s):
    m = re.search(r'(deprecated\s+)?since\s+chrome\s+(\d+)', s, re.I)
    deprecated, version = m.groups()
    return (bool(deprecated), int(version))

def update(sym_data, feature, version_info):
    deprecated, chromever = version_info
    compat_data = sym_data['__compat']

    for key, feature_data in compat_data.items():
        if feature in key:
            break
    else:
        if deprecated:
            return

        compat_data[feature] = feature_data = OrderedDict([
            ("support", OrderedDict([
                (target, {"version_added": None}) for target in ["chrome", "edge", "firefox", "firefox_android", "opera"]
            ]))
        ])

    if deprecated:
        feature_data['status'] = {"obsolete": True}
    else:
        feature_data['support']['chrome']['version_added'] = str(chromever)
        feature_data['support']['opera']['version_added'] = str(max(chromever - 13, 15))

with open(sys.argv[1]) as file:
    data = json.load(file, object_pairs_hook=OrderedDict)

for api, api_data in data['data']['webextensions']['api'].items():
    try:
        response = urllib.request.urlopen('https://developer.chrome.com/extensions/' + api)
    except urllib.error.HTTPError:
        continue

    with response:
        soup = bs4.BeautifulSoup(response.read(), 'lxml')
        for sym, sym_data in api_data.items():
            header = soup.find('h3', string=re.compile(r'^\s*{}\s*$'.format(re.escape(sym))))
            if header:
                compat = header.find_next_sibling(class_='availability')
                if not compat:
                    desc = header.find_next_sibling(class_='description')
                    if desc:
                        compat = desc.find(class_='availability', recursive=False)
                if compat:
                    update(sym_data, 'basic_support', parsecompat(compat.text)) 
                for row in header.parent.find_all('tr'):
                    cells = row.find_all('td')
                    if len(cells) >= 3:
                        compat = cells[2].find(class_='availability', recursive=False)
                        if compat:
                            update(sym_data, cells[1].text.split()[-1], parsecompat(compat.text))

with open(sys.argv[1], 'w') as file:
    json.dump(data, file, indent=2)