PoC web directory scanner using asyncio. It's fast. NOTE: This does not follow any particular HTTP spec and is probably very broken. AIOHTTP would be a more reliable solution, but in my tests it was a lot slower...

dumb.py

import asyncio
from urllib.parse import quote

# dirb time: 1m36.7s
# 32 workers: 3.6s

DOMAIN = "www.google.com"

async def scan_worker(queue):
	while True:
		reader, writer = await asyncio.open_connection(DOMAIN, 80)
		for _ in range(128):
			path = await queue.get()
			try:
				writer.write("GET /{} HTTP/1.1\r\n\r\n".format(quote(path)).encode())
				r = await reader.readuntil(b"\r\n\r\n")
				resp, *headers = r.split(b"\r\n")[:-2]
				length = 0
				for h in headers:
					h = h.decode()
					if h.lower().startswith("content-length:"):
						length = int(h.split(": ")[-1])
				if length:
					await reader.readexactly(length)
				resp = resp.split(b" ")
				if len(resp) >= 3 and resp[1] != b"404":
					print("+ http://{}/{} (CODE:{}|SIZE:{})".format(DOMAIN, path, resp[1].decode(), length))
				queue.task_done()
			except:
				queue.task_done()
				break
		writer.close()
		await writer.wait_closed()

async def main():
	queue = asyncio.Queue()
	dirlist = open("/usr/share/dirb/wordlists/common.txt").read().split("\n")
	for _dir in dirlist:
		queue.put_nowait(_dir)
	
	tasks = []
	for i in range(32):
		task = asyncio.create_task(scan_worker(queue))
		tasks.append(task)
	
	await queue.join()
	
	for task in tasks:
		task.cancel()
	
	await asyncio.gather(*tasks, return_exceptions=True)

asyncio.run(main())