Script to expand REP 2005 repo list to pkg list

repo2pkg.py

#!/usr/bin/env python3

import re
import requests
import rosdistro
import sys

repofile = sys.argv[1]

ros2distros = ['foxy', 'eloquent']
ros1distros = ['noetic', 'melodic']
rindex_url = 'https://index.ros.org/p/%s/'

# Pull in the list of repos to process
repos = []
with open(repofile, 'r') as f:
    for l in f.readlines():
        repos.append(l)

# Load the rosdistros we'll search
index = rosdistro.get_index(rosdistro.get_index_url())
distros = []
for d in ros2distros + ros1distros:
    distros.append(rosdistro.get_distribution_file(index, d).get_data())

regex = re.compile('(.*)`.*<(http.*)>.*')
for r in repos:
    m = regex.match(r)
    if not m or len(m.groups()) != 2:
        print(r, end='')
        continue
    s = m.groups()[0]
    u = m.groups()[1]
    matched = False
    for d in distros:
        for k, rr in d['repositories'].items():
            try:
                src_url = rr['source']['url']
                if src_url[0:-4] == u:
                    pkgs = None
                    try:
                        pkgs = rr['release']['packages']
                    except:
                        pkgs = [k]
                    for p in pkgs:
                        url = rindex_url % (p)
                        url_valid = False
                        try:
                            ret = requests.get(url)
                            if ret.status_code == 200:
                                url_valid = True
                        except:
                            pass
                        if not url_valid:
                            url = u
                        print('%s`%s/%s <%s>`_'%(s,k,p,url))
                    matched = True
                    break
            except:
                pass
        if matched:
            break
    if not matched:
        print('%s **(UNMATCHED)**'%(r[0:-1]))