Find duplicate executable names in Debian

command_collision.py

#!/usr/bin/python3

import aiohttp
from asyncio import create_task, sleep, wait, Semaphore,\
                    Queue, run, ALL_COMPLETED
from collections import defaultdict
import json
import re

#import random
#rand_class = random.SystemRandom()

async def pkg_gen(session):
    url = "https://packages.debian.org/unstable/allpackages"
    async with session.get(url) as resp:
        async for line in (x.decode() async for x in resp.content):
            match = re.search("^<dt><a href='(.+?)'", line)
            if match:
                yield(match.group(1))
#                if rand_class.random() > 0.95:
#                    await sleep(5)
#                    return

async def p_proc(session, fes, oq, pkg):
    url = "https://packages.debian.org/sid/amd64/{}/filelist".format(pkg)
    async with fes:
        async with session.get(url) as resp:
            text = await resp.text()
        
    inList = False;
    for line in text.split("\n"):
        if inList:
            if "</div>" in line:
                return
            await oq.put((pkg, line))
        else:
            if "\"pfilelist\"" in line:
                inList = True

async def proc_out(oq):
    inc_paths = [
        "/bin/",
        "/sbin/",
        "/usr/bin/",
        "/usr/sbin/",
    ]

    binDict = defaultdict(lambda: [])

    while True:
        (pkg, path) = await oq.get()

        if pkg == None:
            break

        if any((path.startswith(x) for x in inc_paths)):
            exe = path.split("/")[-1]
            binDict[exe].append((pkg, path))

    print(json.dumps(binDict, indent=2, sort_keys=True))

async def main():
    fes = Semaphore(5)
    oq = Queue()

    otask = create_task(proc_out(oq))

    async with aiohttp.ClientSession() as s:
        tasks = [create_task(p_proc(s, fes, oq, x)) async for x in pkg_gen(s)]
        await wait(tasks, return_when=ALL_COMPLETED)

    await oq.put((None, None))

    await wait([otask])

run(main())