get_changelog.py

import json
import sys
import os
import shlex
import subprocess

import searx.engines


ENGINES = searx.engines.load_engines(searx.engines.settings['engines'])

SUBPROCESS_RUN_ENV = {
    "PATH": os.environ["PATH"],
    "LC_ALL": "C",
    "LANGUAGE": "",
}


def subprocess_run(args, **kwargs):
    """Call :py:func:`subprocess.run` and return (striped) stdout.  If returncode is
    non-zero, raise a :py:func:`subprocess.CalledProcessError`.
    """
    if not isinstance(args, (list, tuple)):
        args = shlex.split(args)

    kwargs["env"] = kwargs.get("env", SUBPROCESS_RUN_ENV)
    kwargs["encoding"] = kwargs.get("encoding", "utf-8")
    kwargs["stdout"] = subprocess.PIPE
    kwargs["stderr"] = subprocess.PIPE
    kwargs["env"] = os.environ
    kwargs["cwd"] = os.curdir
    proc = subprocess.run(args, **kwargs)  # pylint: disable=subprocess-run-check
    print(proc.stderr.strip())
    if proc.returncode != 0:
        sys.exit(proc.returncode)
    return proc.stdout.strip()


pr_list = subprocess_run(['gh', 'pr', 'list', '-L', '2000', '--search', 'is:merged', '--json', 'url,title,author,mergedAt,files,labels'])

def get_paths(paths):
    engines = []
    others = []
    for p in paths:
        searx_engines_path = 'searx/engines/'
        if p.startswith(searx_engines_path) and p.endswith('.py') and p != 'searx/engines/__init__.py':
            engines.append(p[len(searx_engines_path):-3])
        elif p == 'searx/data/engines_languages.json':
            continue
        else:
            others.append(p)
    return engines, others

def print_prs(title, prs):
    print(f'\n## {title}\n')
    for pr in prs:
        pr_number = '#' + (pr['url'].split('/')[-1])
        login = pr['author']['login']
        title = pr['title']
        if login in ('github-actions', 'dependabot'):
            continue
        print(f'* {pr_number} - {title} - @{login}')


def str_startswith(s: str, choices):
    for c in choices:
        if s.strip().lower().startswith(c.lower()):
            return True
    return False


def items_in(item_list, ref):
    for item in item_list:
        if item not in ref:
            return False
    return True


def items_startswith(item_list, ref, ref_atleastone = None):
    if ref_atleastone:
        for item in item_list:
            if item in ref_atleastone:
                break
        else:
            return False
    for item in item_list:
        if not str_startswith(item, ref):
            return False
    return True


engine_prs = {}
docs_prs = []
fix_prs = []
mod_prs = []
enh_prs = []
lint_prs = []
themes_prs = []
setup_prs = []
docker_prs = []
settings_prs = []
misc_prs = []
translations_prs = []

label_ids = {
    'MDU6TGFiZWwzMDI3NDAyODg2': 'lint & coding style'
}

LINT_LABEL_ID = 'MDU6TGFiZWwzMDI3NDAyODg2'


for pr in json.loads(pr_list):
    pr['label_ids'] = [ l['id'] for l in pr['labels']]
    pr_number = '#' + pr['url'].split('/')[-1]
    login = pr['author']['login']
    title = pr['title']
    mergedAt = pr['mergedAt']
    if login in ('github-actions', 'dependabot'):
        continue
    
    paths = [ f['path'] for f in pr['files']]
    if items_startswith(paths, ['docs/']) or str_startswith(title, ['[docs]', '[doc]']):
        docs_prs.append(pr)
        continue

    paths = [ p for p in paths if not p.startswith('docs/') ]
    if items_startswith(paths, ['Dockerfile', 'dockerfiles', '.dockerignore', '.gitignore', 'Makefile', 'manage', '.github/'], ['Dockerfile', 'dockerfiles', '.dockerignore']):
        docker_prs.append(pr)
    elif items_startswith(paths, ['utils/', '.github/', 'manage', 'Makefile', 'requirements.txt', 'requirements-dev.txt', 'setup.py', '.config.sh']):
        setup_prs.append(pr)
    elif items_in(paths, ['searx/settings.yml']):
        settings_prs.append(pr)
    elif items_startswith(paths, ['manage', 'Makefile', 'searx/templates/', 'searx/static/']):
        themes_prs.append(pr)
    elif items_startswith(paths, ['.dir-locals.el', '.pylintrc', '.yamllint.rc']) or LINT_LABEL_ID in pr['label_ids']:
        lint_prs.append(pr)
    elif items_startswith(paths, ['searx/translations', 'babel.cfg', '.weblate']):
        translations_prs.append(pr)
    else:
        engines, others = get_paths(paths)
        for engine_name in engines:
            engine_prs.setdefault(engine_name, []).append(pr)

        if not items_startswith(others, ['searx/settings.yml', 'searx/static/themes/oscar/img/icons', 'searx/templates/', 'Makefile', 'manage']):
            if str_startswith(title, ['[fix]', 'fix ']):
                pr['title'] = pr['title']
                fix_prs.append(pr)
            elif str_startswith(title, ['[enh]']):
                pr['title'] = pr['title']
                enh_prs.append(pr)
            elif title.startswith('[mod]'):
                pr['title'] = pr['title']
                mod_prs.append(pr)
            elif str_startswith(title, ['[themes]', '[theme]']):
                themes_prs.append(pr)
            elif str_startswith(title, ['[pylint]', '[yamllint]', 'Pylint ', 'lint ', '[coding-style]']):
                lint_prs.append(pr)
            else:
                misc_prs.append(pr)

print('## Engine updates\n')
for engine_name in sorted (engine_prs.keys()):
    prs = engine_prs[engine_name]
    prs = [ '#' + pr['url'].split('/')[-1] for pr in prs ]
    prs = ' '.join(prs)
    print(f'* {engine_name} ( {prs} )')

print_prs('settings.yml', settings_prs)
print_prs('Misc', misc_prs)
print_prs('Enhancement', enh_prs)
print_prs('Modifications', mod_prs)
print_prs('Fixes', fix_prs)
print_prs('Themes', themes_prs)
print_prs('Documentation', docs_prs)
print_prs('Docker', docker_prs)
print_prs('Setup', setup_prs)
print_prs('Lint', lint_prs)
print_prs('Translations', translations_prs)