Converts result from https://opendata.rapid7.com/sonar.http/ into masscan-like format

sonar.http_parse.py

#!/usr/bin/env python3
import base64
import sys
import os
try:
	import simplejson as json
except ImportError:
	import json

# Takes decompressed https://opendata.rapid7.com/sonar.http/ scan file on stdin
# Outputs banners in format similar to masscan's line format

BUFFER_LINES = 16384
BANNER_CUTOFF = 1024
PORT = int( os.environ.get("PORT", "80") )

def escape(s):
	return s.replace(b"\x0d", b"\\x0d").replace(b"\x0a", b"\\x0a").replace(b"\x22", b"\\x22")

def process(l):
	l.pop(-1)
	for entry in l:
		try:
			banner = bytearray(base64.b64decode(entry["data"]))
		except:
			sys.stderr.write("Dropping one entry because of exception\n")
			sys.stderr.flush()
			continue
		l = banner.find(b"\r\n\r\n")
		if l == -1:
			l = banner.find(b"\n\n")
		if l == -1:
			continue
		del banner[l+2:]
		if l > BANNER_CUTOFF:
			del banner[BANNER_CUTOFF:]
		banner = escape(banner)
		sys.stdout.buffer.write(b"\nbanner tcp %d %s 0 http " % (PORT, entry["ip"].encode("ascii")))
		sys.stdout.buffer.write(banner)

buf = bytearray(1024)
offset = 0
def append(s):
	global buf, offset
	slen = len(s)
	if offset + slen > len(buf):
		buf.extend(bytearray( offset + slen - len(buf) ))
	buf[offset:offset+slen] = s
	offset += slen
run = True
while run:
	offset = 0
	append(b"[")
	for i in range(BUFFER_LINES):
		line = sys.stdin.buffer.readline()
		if line == b"":
			run = False
			break
		append(line)
		append(b",")
	append(b"null]")
	process(json.loads( str(memoryview(buf[0:offset]), 'ascii') ))

sys.stdout.write("\n")