Wrapper around masscan and nmap to complete full service scans faster.

masscan-to-nmap.py

#!/usr/bin/env python3
"""
This script will run masscan to quickly find open ports on a target scope.
When masscan is complete, it runs a Nmap service scan on discovered ports, 
in a concurrent way (1 nmap process per IP, with a max of 10 concurrent processes).
Then it merges the nmap results in single files (1 .xml, 1 .nmap, 1.gnmap)
This speeds up full port scans dramatically.
"""
import sys, os, time, argparse, subprocess, shutil

from os import path
from copy import deepcopy
from libnmap.parser import NmapParser
from concurrent.futures import ThreadPoolExecutor


try:
	import  xml.etree.cElementTree as ET
except ImportError:
	import xml.etree.ElementTree as ET

parser = argparse.ArgumentParser()
parser.add_argument('targets', action='store', help='ip/ranges to scan or filename containing ip/ranges', default=None)
parser.add_argument('-i', '--interface', action='store', dest='interface', help='Send packets from this network interface', default='eth0')
parser.add_argument('-r', '--rate', action='store', dest='rate', help='Send packet rate (default: 5000)', default='5000')
parser.add_argument('-p', '--ports', action='store', dest='ports', help='Ports to scan with masscan (default: 1-65535 tcp / top 20 udp)', default='T:1-65535,U:53,67-69,123,135,137-139,161-162,445,500,514,520,631,1434,1900,4500,49152')
parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Don\'t actually run commands', default=False)

args = parser.parse_args()

def prex(cmd):
	print("Running", cmd)
	if not args.debug:
		process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
		for line in process.stdout:
			line = line.strip()
			sys.stdout.write(line)
			sys.stdout.write("         \r" if line.startswith("rate:") else "\n")
			sys.stdout.flush()
		process.wait()
		return process.returncode

def merge_nmap_xmls(directory):

	def start_xml(script_start_time):
		scriptargs = ""
		for argnum in range(0, len(sys.argv)):
			scriptargs = scriptargs + " " + str(sys.argv[argnum])
		nmaprun_attribs =  {'scanner' : 'nmap', 'args' : 'nmap', 'start' : str(script_start_time), 'version' : '1.0', 'xmloutputversion' : '1.04'}
		nmaprun = ET.Element('nmaprun', nmaprun_attribs)
		nmaprun_verbose_attribs = {'level' : '0'}
		nmaprun_debug_attribs = {'level' : '0'}
		nmaprun_verbose = ET.SubElement(nmaprun, 'verbose', nmaprun_verbose_attribs)
		nmaprun_debug = ET.SubElement(nmaprun, 'debug', nmaprun_debug_attribs)
		return nmaprun

	def finalise_xml(nmaprun_merged_results, script_start_time):
		nmaprun = nmaprun_merged_results[0]
		total_hosts = nmaprun_merged_results[1]
		total_seconds = nmaprun_merged_results[2]
		total_files = nmaprun_merged_results[3]
		nmaprun_string = ET.tostring(nmaprun)
		total_script_time = int(time.time())-script_start_time;
		summary_string = ('Nmap XML merge done at ' + time.strftime("%c") + "; " + str(total_hosts) + ' total hosts found in ' + str(total_files)  +' files; Merge completed in ' + str(total_script_time) + ' seconds')
		finished_attribs = {}
		finished_attribs["time"] = str(int(time.time()))
		finished_attribs["timestr"] = time.strftime("%c")
		finished_attribs["elapsed"] = str(total_seconds)
		finished_attribs["summary"] = summary_string
		finished_attribs["exit"] = 'success'
		hosts_attribs = {}
		hosts_attribs["up"] = str(total_hosts)
		hosts_attribs["down"] = '0'
		hosts_attribs["total"] = str(total_hosts) 
		runstats = ET.SubElement(nmaprun, 'runstats')
		finished = ET.SubElement(runstats, 'finished', finished_attribs)
		hosts = ET.SubElement(runstats, 'hosts', hosts_attribs)
		return nmaprun

	def merge_hosts(nmaprun, file_list):
		total_hosts = 0
		total_seconds = 0
		bad_file_list = []
		for current_file in file_list:
			try: 
				current_nmap_file_blob = ET.ElementTree(file=current_file);
				for current_host in current_nmap_file_blob.findall('host'):
					#build our stats here
					total_hosts = total_hosts + 1
					total_seconds = (total_seconds + int(current_host.attrib['starttime']) - int(current_host.attrib['endtime']))
					nmaprun.append(deepcopy(current_host))
			except:
				bad_file_list.append(current_file)
		files_processed = len(file_list) - len(bad_file_list)
		nmaprun_merge_results = [nmaprun, total_hosts, total_seconds, len(file_list)]
		return nmaprun_merge_results, bad_file_list, files_processed, total_hosts

	def input_file_list(sources_list):
		file_list = []
		for target in sources_list:
			if os.path.isdir(target) == True:
				dirlist = os.listdir(target)
				for file in dirlist:
					file_list.append(target+file)
			else:
				file_list.append(target)
		return file_list

	def output_results(nmap_file_preamble, nmaprun, merge_job_output):
		bad_file_list = merge_job_output[1]
		files_processed = merge_job_output[2]
		total_hosts = merge_job_output[3]
		output = nmap_file_preamble
		nmaprun_string = ET.tostring(nmaprun).decode()
		output += nmaprun_string
		open('%s.xml' % args.output_fname, 'w').write(output)	

	script_start_time = int(time.time())
	merge_job_output = []
	nmap_file_preamble = ('<?xml version="1.0"?> \n'
							'<!DOCTYPE nmaprun PUBLIC "-//IDN nmap.org//DTD Nmap XML 1.04//EN" "https://svn.nmap.org/nmap/docs/nmap.dtd"> \n'
							'<?xml-stylesheet href="https://svn.nmap.org/nmap/docs/nmap.xsl" type="text/xsl"?> \n'
							)
	file_list = input_file_list([directory])						
	nmaprun_skel = start_xml(script_start_time)
	merge_job_output = merge_hosts(nmaprun_skel, file_list)
	nmaprun_merged_results = merge_job_output[0]
	nmaprun_finalised = finalise_xml(nmaprun_merged_results, script_start_time)
	output_results(nmap_file_preamble, nmaprun_finalised,merge_job_output)

def do_post():

	if path.exists('./nmap-output'):
		print("Merging .nmap files")
		os.system("cat ./nmap-output/*.nmap | grep -Ev '^#|^Host is up|^Service detection|^Scanned at|^Read data files' > %s.nmap" % args.output_fname)

		print("Merging .gnmap files")
		os.system("cat ./nmap-output/*.gnmap | grep /open/ > %s.gnmap" % args.output_fname)

		print("Merging .xml files")
		merge_nmap_xmls('./nmap-output/')

		print("Deleting old files")
		shutil.rmtree("./nmap-output")

def do_nmap():

	input_file = '%s.masscan' % args.output_fname

	file = open(input_file).read().splitlines()
	results = {}
	for line in file:
		if not line.startswith('Timestamp:'):
			continue
		tab = line.split()
		ip = tab[3]
		port = tab[6].split('/')[0]
		proto = tab[6].split('/')[2]

		if ip not in results:
			results[ip] = []

		results[ip].append((port, proto))

	if not path.exists('./nmap-output'):
		os.mkdir('./nmap-output')

	commands = []
	for ip in results:
		tcp = {x[0] for x in results[ip] if x[1] == 'tcp'}
		udp = {x[0] for x in results[ip] if x[1] == 'udp'}
		port_list_tcp = 'T:' + ','.join(tcp) if tcp else ''
		port_list_udp = 'U:' + ','.join(udp) if udp else ''
		port_spec = '-sV'
		port_spec += 'S' if tcp else ''
		port_spec += 'U' if udp else ''
		port_list = ','.join([port_list_tcp, port_list_udp]).strip(",")

		commands.append("sudo nmap %s -Pn -n -vv --open -p %s %s -oA ./nmap-output/result-%s 2>&1" % (port_spec, port_list, ip, ip))

	with open("%s.nmapcmds" % args.output_fname, 'w') as f:
		f.write("\n".join(commands))

	with ThreadPoolExecutor(10) as executor:
	    executor.map(prex, commands)

def do_masscan():

	if path.exists(args.targets):
		prex("sudo masscan --rate=%d --open -e %s -iL %s -p %s --wait 10 -oG %s.masscan 2>&1" % (int(args.rate), args.interface, args.targets, args.ports, args.output_fname))

	else:
		prex("sudo masscan --rate=%d --open -e %s -p %s --wait 10 -oG %s.masscan %s 2>&1" % (int(args.rate), args.interface, args.ports, args.output_fname, args.targets))


if __name__ == "__main__":

	if path.exists(args.targets):
		args.output_fname = path.basename(path.splitext(args.targets)[0])
	else:
		args.output_fname = args.targets.replace('/', '_')

	do_masscan()
	do_nmap()
	do_post()