Hacky scripts to update the ros-melodic-* AUR PKGBUILD sources to point to github sources vs. ros-gbp

aur-ros-sources-updater.py

import json
import os
import re
import requests
import subprocess
import time
import urllib
import yaml

### uncommment / fill in the below
#gh_oauth='' # a github token for authentication, otherwise calls are throttled
#global_path = 'path/to/dir/ros-melodic-aur' # directory of all of ros-melodic-* pkg dirs
os.chdir(global_path)

#repos = yaml.load(open('./distribution.yaml', 'r').read(), Loader=yaml.BaseLoader)['repositories']
rosdistro_url = 'https://raw.githubusercontent.com/ros/rosdistro/master/melodic/distribution.yaml'
rosdistro = yaml.load(requests.get(rosdistro_url, allow_redirects=True).content,
                      Loader=yaml.BaseLoader)['repositories']

# packages that are missing information or are special cases
skip = ['fcl', 'libviso2', 'viso2_ros', 'opencv3', 'roscpp_git', 'message_filters_git', 'ivcon', 'stage', 'nodelet_tutorial_math',
        'common_tutorials', 'turtle_actionlib', 'pluginlib_tutorials', 'rosbag_migration_rule', 'actionlib_tutorials', 'ompl', 'bfl', 'convex_decomposition', 'mavlink']

def build_aur_dict(path='./'):
    """Assumes we are in top-level directory with all the repo files.
    Build list of all repos in directory; assumes all start with ros-melodic."""
    aur_list = [d for d in os.listdir(path) if os.path.isdir(d) and d.startswith('ros-melodic')]
    aur_list = [d for d in repo_list if 'PKGBUILD' in os.listdir(d)]
    aur_dict = {'{}'.format(pkgname): '{}'.format(pkgname.replace('ros-melodic-', '').replace('-', '_')) for pkgname in aur_list}
    
    return aur_dict

aur_dict = build_aur_dict()

os_dict = {}
for repo in rosdistro:
    """Go through distro, and make entry for each package in a repository"""
    d = rosdistro[repo]
    if 'source' in d:
        src = d['source']['url']
    elif 'release' in d:
        src = d['release']['url']
    target = re.sub('\.git', '', src.split('/')[3] + '/' + src.split('/')[4])
    pkgver = d.get('release', {'version': None}).get('version', None)
    if pkgver:
        pkgver = pkgver.split('-')[0]
    if 'github' in src:
        dl = 'https://github.com/' + target + '/archive/' + pkgver + '.tar.gz' if pkgver else None
    else:
        dl = None
    pkg_list = d.get('release', {'packages': [repo]}).get('packages', [repo])
    for pkg in pkg_list:
        siblings = len(pkg_list)-1
        pkgname = 'ros-melodic-{}'.format(re.sub('_', '-', pkg))
        ros_dict[pkg] = {'repo': repo, 'siblings': siblings, 'pkgname': pkgname, 'src': src, 'pkgver': pkgver, 'dl': dl}
        
def update_src(pkgname):
    os.chdir(global_path)
    os.chdir(pkgname)
    
    pkg = aur_dict[pkgname]
    if pkg in skip:
        print('skipping: {}'.format(pkgname))
        return (pkgname, 'skip')

    d = ros_dict[pkg]
    if not d.get('pkgver', None):
        print('pkgver not in dict: {}'.format(pkgname))
        return (pkgname, 'no_tag')
    
    if not d.get('dl', None):
        print('dl not in dict: {}'.format(pkgname))
        return (pkgname, 'no_dl')
    
    old_pkgver = re.findall("^pkgver=.*", open('PKGBUILD').read(), re.MULTILINE)
    old_dir = re.findall("^_dir=.*", open('PKGBUILD').read(), re.MULTILINE)
    old_src = re.findall("^source=\(.*\)", open('PKGBUILD').read(), re.MULTILINE)
    old_sha = re.findall("^sha256sums=\(.*\)", open('PKGBUILD').read(), re.MULTILINE)

    if all((old_dir, old_src, old_sha, old_pkgver)):
        old_pkgver = old_pkgver[0]
        old_dir = old_dir[0]
        old_src = old_src[0]
        old_sha = old_sha[0]
    else:
        print('getting PKGBUILD lines failed: {}'.format(pkgname))
        return (pkgname, 'multi_pkgbuild')
    
    new_pkgver = "pkgver='{}'".format(d['pkgver'])
    new_dir = '_dir="{}-${{pkgver}}{}"'.format(d['repo'],
                                               '/{}'.format(pkg) if d['siblings'] else '')
    new_src = 'source=("${{pkgname}}-${{pkgver}}.tar.gz"::"{}")'.format(d['dl'])
    
    if old_pkgver == new_pkgver and old_dir == new_dir and old_src == new_src:
        print('already matches: {}'.format(pkgname))
        return (pkgname, 'updated')
    
    print('starting: {}'.format(pkgname))
    fname = '{}-{}.tar.gz'.format(pkg, d['pkgver'])
    try:
        urllib.request.urlretrieve(d['dl'], fname)
    except urllib.error.HTTPError:
        print('download failed: {}'.format(pkgname))
        return (pkgname, 'dl_fail')
        
    sha256 = subprocess.run(['sha256sum', fname], capture_output=True)
    new_sha = "sha256sums=('{}')".format(sha256.stdout.decode('utf-8').split(' ')[0])
    os.remove(fname)
    
    subprocess.call(['git', 'branch', 'src-fix'])
    subprocess.call(['git', 'checkout', 'src-fix'])
    with open('PKGBUILD', 'r') as f:
        lines = f.readlines()
    
    with open('PKGBUILD', 'w') as f:
        for line in lines:
            line = re.sub(re.escape(old_pkgver), new_pkgver, line)
            line = re.sub(re.escape(old_src), new_src, line)
            line = re.sub(re.escape(old_dir), new_dir, line)
            line = re.sub(re.escape(old_sha), new_sha, line)
            f.write(line)
            
    with open('.SRCINFO', "w") as outfile:
        subprocess.call(['makepkg', '--printsrcinfo'], stdout=outfile)
    
    subprocess.call(['git commit -am "updated _dir, sources, and shasums"'], shell=True)
    return (repo, 'updated')

results = [update_src(pkgname) for pkgname in aur_dict]