Mass patch Openstack repos, based on the project deliverables

mass_submit.py

import os
import yaml
import requests
import git
import multiprocessing
import re
import hashlib
from joblib import Parallel, delayed

COMMIT_USER = "User Name"
COMMIT_EMAIL = "user.name@company.com"
COMMIT_MESSAGE = """Commit header

Commit message here
"""

GERRIT_USERNAME = "username"
GERRIT_SERVER = "review.opendev.org"

DELIVERABLES_URL = "https://opendev.org/openstack/releases/raw/branch/master/deliverables/victoria/openstack-ansible-roles.yaml"
REPO_PATH = "/tmp"
BRANCH = "master"
TOPIC = "topic"


def makesum():
    fd = open("/dev/urandom", "rb")
    buf = fd.read(64)
    if len(buf) != 64:
	    raise Exception("Unable to get random data")
    hash = hashlib.sha1()
    hash.update(buf)
    return hash.hexdigest()


def edit_repo(repo):
    dst_path = os.path.join(REPO_PATH, repo, 'zuul.d/project.yaml')
    example_path = os.path.join(REPO_PATH, repo, 'examples/playbook.yml')
    if os.path.isfile(dst_path) and os.path.isfile(example_path):
        with open(dst_path, 'r') as src:
            src_file = src.read()
        dst_file = re.sub(r'(openstack-ansible-linters)$', r'\1-jobs', src_file, flags = re.M)
        with open(dst_path, 'w') as dst:
            dst.write(dst_file)
    if not os.path.isfile(example_path):
        print(f"Repo {repo} does not have example!")

def patch_repo(repo):
    try:
        cloned_repo = git.Repo.clone_from(os.path.join('https://opendev.org', repo), os.path.join(REPO_PATH, repo))
        edit_repo(repo)
        diff = cloned_repo.index.diff(None) + cloned_repo.untracked_files
        change_id = makesum()
        if len(diff) > 0:
            cloned_repo.index.add([chgd.a_path for chgd in diff])
            author = git.Actor(COMMIT_USER, COMMIT_EMAIL)
            committer = git.Actor(COMMIT_USER, COMMIT_EMAIL)
            cloned_repo.index.commit(COMMIT_MESSAGE + f"\n\nChange-Id: I{change_id}",  author=author, committer=committer)
            cloned_repo.create_remote('gerrit', f'ssh://{GERRIT_USERNAME}@{GERRIT_SERVER}:29418/{repo}')
            cloned_repo.git.push("gerrit", f"HEAD:refs/for/{BRANCH}%topic={TOPIC}")
    except Exception as err:
        print(f"Failed to patch repo {repo}. Error raised: {err}")


def main():
    del_data = requests.get(DELIVERABLES_URL)
    deliverables = yaml.load(del_data.text, Loader=yaml.FullLoader)['repository-settings'].keys()
    # patch_repo('openstack/openstack-ansible-haproxy_server')
    num_cores = multiprocessing.cpu_count()
    Parallel(n_jobs=num_cores, prefer="threads")(delayed(patch_repo)(repo) for repo in deliverables)


if __name__ == "__main__":
    main()