Fetch compatibility scores for top 20 most dependended-upon npm packages

fetch-compatibility-score.py

import requests
import json
import pandas as pd
from tqdm.auto import tqdm
from xml.dom import minidom

def get_version_history(package: str):
    res_df = pd.DataFrame()
    url = f"https://registry.npmjs.org/{package}"
    res = json.loads(requests.get(url).text)
    for ver, time in res['time'].items():
        if ver in ('modified', 'created'):
            continue
        res_df = res_df.append({
            "version": ver,
            "time": pd.to_datetime(time, infer_datetime_format=True)
        }, ignore_index=True)
        
    return res_df


def filter_npm_version(package:str, before=None, exclude_pre_versions=True, num=0):
    before_date = pd.to_datetime(before) if before is not None else None
    res_sorted = get_version_history(package).sort_values('time', ascending=False)
    # to walkaround type error
    res_sorted["time"] = pd.to_datetime(res_sorted["time"]).dt.date  
    if exclude_pre_versions:
        res_sorted = res_sorted[~res_sorted['version'].str.contains('-')]
    if before_date is not None:
        res_sorted = res_sorted[res_sorted['time'] < before_date]
    if num > 0:
        return res_sorted.head(num)
    return res_sorted


def get_npm_version_pairs(package: str, before=None, after=None, exclude_pre_versions=True, num=None):
    before_date = pd.to_datetime(before) if before is not None else None
    after_date = pd.to_datetime(after) if after is not None else None
    res_sorted = get_version_history(package).sort_values('time', ascending=False)
    
    # to walkaround type error
    # all we care is date
    res_sorted["time"] = pd.to_datetime(res_sorted["time"]).dt.date
   
    if exclude_pre_versions:
        res_sorted = res_sorted[~res_sorted['version'].str.contains('-')]
    if after_date is not None:  # >=
        res_sorted = res_sorted[res_sorted['time'] >= after_date]
    if before_date is not None:  # <=
        res_sorted = res_sorted[res_sorted['time'] <= before_date]
    if num is not None:  # this does not guarantee we have <num> results; more like .limit <num>
        res_sorted = res_sorted.head(num)

    res_sorted.set_index('version', inplace=True)
    
    len_ver = len(res_sorted)
    vers = list(res_sorted.index)
    ver_pairs = []
    for i in range(len_ver):
        ver_pairs += [{
            "package": package,
            "new_version": vers[i],
            "old_version": vers[j],
            "new_version_date": res_sorted.loc[vers[i], 'time'],
            "old_version_date":res_sorted.loc[vers[j], 'time'],
            "version_distance": j-i
            } for j in range(i+1, len_ver)]
    return ver_pairs


def get_dependabot_compatibility_score_ver(package_manager, dependency_name, oldver, newver) -> str:
    url = f"https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name={dependency_name}&package-manager={package_manager}&previous-version={oldver}&new-version={newver}"
    res = requests.get(url)
    svg_text = res.text
    doc = minidom.parseString(svg_text)
    for ele in doc.getElementsByTagName("text"):
        res = ele.firstChild.nodeValue
        if res != "compatibility":
            return res
        
        
if __name__ == '__main__':
    # top 20 most dependended-upon packages; may not be 100% accu, but whatever
    # source: https://gist.github.com/anvaka/8e8fa57c7ee1350e3491
    packages = ['lodash','chalk','request','commander','react','express','debug','async','fs-extra',
                'moment','prop-types','react-dom','bluebird','underscore','vue','axios','tslib','mkdirp','glob','yargs']
    
    df_res = pd.DataFrame()
    res_pairs = []

    for package in packages:
        pairs = get_npm_version_pairs(package, after="2020-01-01", exclude_pre_versions=True)
        if len(pairs) > 0:
            print(f"[{package}] {len(pairs)} pairs: from {pairs[0]} to {pairs[-1]}")
            res_pairs += pairs

    for pair in tqdm(res_pairs):
        score = get_dependabot_compatibility_score_ver("npm_and_yarn", pair['package'], pair['old_version'], pair['new_version'])
        df_res = df_res.append({
            "compatibility_score": score,
            **pair
        }, ignore_index=True)

    df_res.to_csv('npm-packages-compatibility-scores.csv', index=False)
    print(df_res.head())

npm-packages-compatibility-scores.csv