qualys_report_download.py

## qualys_report_download.py
#!/usr/bin/env python
# coding=utf-8
# Python linter configuration.
# pylint: disable=I0011
# pylint: disable=C0301
# pylint: disable=W0702
"""
    Script to Generate a report on qualys, then download it.

    1st Public Release - Tweaked from Private version, some code might need checking.

    # Nick Bettison - Linickx.com

"""

import sys
import os
import logging
import logging.handlers
import socket
import time
import re
from xml.etree import ElementTree


version = "1.0"

"""
    Variables to chang!
    /Start
"""

# Qualys
apiuser = 'nick'
apipass = 'linickx'
apiurl = 'https://qualysguard.qg3.apps.qualys.com/api/2.0/fo/report/'
apisleep = 10 # Time to sleep between api calls

# Template ID, this is an array so you can have many!
apitemplateid = ["1712399"]

# Only tested with CSV -
apireportformat = "csv"

# Proxy!
proxies_enable = False
proxies = {
    'http': 'http://proxy.local:8080',
    'https': 'http://proxy.local:8080'
}
ssl_verify = True
path = ''

syslog_enable = False
syslog_server = "10.10.10.10"
syslog_port = 514
syslog_protocol = "udp"
syslog_facility = "local0"


"""
    /END
"""

# Logging Setup
logger = logging.getLogger("qualys1")
logger.setLevel(logging.INFO) # Default Loging Level
formatter = logging.Formatter('[%(levelname)s] %(asctime)s %(message)s')

# Log to Console
chandler = logging.StreamHandler()
chandler.setFormatter(formatter)
#chandler.setLevel(logging.DEBUG) # Use to over-ride logging level
# Add to logger
logger.addHandler(chandler)

if syslog_protocol == "udp":
    socktype = socket.SOCK_DGRAM
elif syslog_protocol == "tcp":
    socktype = socket.SOCK_STREAM
else:
    logger.critical("Unknown Syslog Protocol %s", syslog_protocol)
    syslog_enable = False

if syslog_enable:
    # Log to Syslog
    shandler = logging.handlers.SysLogHandler(address = (syslog_server, syslog_port), facility = syslog_facility, socktype = socktype)
    sformatter = logging.Formatter('%(name)s - %(message)s')
    shandler.setFormatter(sformatter)
    #shandler.setLevel(logging.DEBUG) # Use to over-ride logging level
    # Add to logger
    logger.addHandler(shandler)

try:
    import requests # http://docs.python-requests.org/en/master/
except:
    logger.error("import requests failed - type pip install requests")
    logger.debug("Exception: %s", sys.exc_info()[0])
    sys.exit(1)

request_headers = {} # Python Curl Stuff!
request_headers['user-agent'] = 'LINICKX Downloader  / Version ' + version
request_headers['X-Requested-With'] = 'QualysApi'

def qualys_status():
    """
        Returns the status of reports on qualys API
    """
    # -d 'action=list'
    data = {"action": "list"}
    if proxies_enable:
        qr = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)
    else:
        qr = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)

    logger.info("Request status code: %s", qr.status_code)
    qr.raise_for_status() # Lazy - Throw an error for bad status codes
    qroot = ElementTree.fromstring(qr.content)
    logger.debug(qr.content)

    return qroot

def qualys_launch(template_id=None):
    """
        Requests that a report is run via qualys API
    """
    # -d "action=launch&template_id=1611111&output_format=csv"
    data = {"action": "launch", "template_id":template_id, "output_format":apireportformat, "hide_header":"1"}

    report_wait = True # Enable the loop
    while report_wait:
        if proxies_enable:
            r = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)
        else:
            r = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)

        logger.info("Request status code: %s", r.status_code)
        r.raise_for_status() # Lazy - Throw an error for bad status codes
        root = ElementTree.fromstring(r.content)
        logger.debug(r.content)
        if re.match(r'^Max[\s]number[\s]of[\s]allowed(.*)', root[0][1].text):
            logger.info(root[0][1].text)
            logger.info("Sleeping %s seconds", apisleep)
            time.sleep(apisleep)
        else:
            report_wait = False

    try:
        root[0][2]
    except:
        logger.error(root[0][1].text)
        return None

    for item in root[0][2]:
        counter = 0
        for entry in item:
            logger.debug("Checking XML Entry: %s", entry.text)
            counter += 1 # Increment the counter early, so value is saved after IF
            if entry.text == "ID":
                expected_report_id = item[counter].text
                logger.info("Report ID: %s", expected_report_id)
                return expected_report_id

    return None

def qualys_delete(report_id=None):
    """
        Requests that a saved report is deleted
    """
    data = {"action": "delete", "id":report_id}

    if proxies_enable:
        r = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)
    else:
        r = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)

    logger.info("Delete Request status code: %s", r.status_code)
    r.raise_for_status() # Lazy - Throw an error for bad status codes
    root = ElementTree.fromstring(r.content)
    logger.debug(r.content)


def main():
    """
        The main loop, request the reports.
    """

    reports = []
    # Loop 1 launch the reports
    for template_id in apitemplateid:
        expected_report_id = qualys_launch(template_id)

        if expected_report_id is None:
            logger.critical("Something went wrong, report ID not found")
            continue

        the_report = {'expected_report_id':expected_report_id, 'template_id':template_id}
        reports.append(the_report)

    # Loop 2 Check the report status
    for the_report in reports:

        expected_report_id = the_report['expected_report_id']
        template_id = the_report['template_id']
        """
            Take a break, new reports take time...
        """

        logger.debug("Sleeping %s seconds before requesting the report %s", apisleep, template_id)
        time.sleep(apisleep)

        """
            Begin section to request the status of the report
        """

        report_running = True # Enable the loop

        # Loop for checking the status
        while report_running:
            qualys_reports_list = qualys_status()
            number_of_reports = len(qualys_reports_list[0][1])
            logger.debug("%s Reports found", number_of_reports)

            # Loop through the qualys reports
            for report in qualys_reports_list[0][1]:
                try:
                    report_id = report[0].text
                    report_status = report[6][0].text
                except:
                    logger.debug(report[0].text) # This is a weird edge case that I've not resolved!
                    logger.debug(report[6].text)
                    logger.debug("one day I will fix this weird shizzle")
                    continue

                logger.debug("Report: %s Status: %s", report_id, report_status)

                if report_id == expected_report_id: # Found our Report
                    if report_status == "Finished": # It's finished!
                        report_running = False
                        logger.info("Report: %s %s, ready for download", report_id, report_status)
                    else:
                        report_percent = report[6][2].text # Output the progress
                        logger.info("Report %s Still %s, %s percent complete", report_id, report_status, report_percent)

            if report_running: # Only Sleep in long loops
                try:
                    logger.debug("Sleeping %s seconds", apisleep)
                    time.sleep(apisleep)
                except KeyboardInterrupt:
                    logger.critical("CTRL+C Detected, quitting...")
                    sys.exit(1)


    # Loop 3 Download Reports
    for the_report in reports:
        expected_report_id = the_report['expected_report_id']
        template_id = the_report['template_id']


        """
            Begin section to download the report
        """
        filename = path + "qualys_report_" + template_id + "." + apireportformat
        temp_filename = filename + ".tmp"


        # -d 'action=fetch&id=2911119'
        qualys_data = {"action": "fetch", "id":expected_report_id}

        if proxies_enable:
            r = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=qualys_data, stream=True)
        else:
            r = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=qualys_data, stream=True)

        logger.info("Request status code: %s", r.status_code)
        r.raise_for_status() # Lazy - Throw an error for bad status codes


        #if apireportformat == "csv":
        #    # https://stackoverflow.com/a/14114741/1322110
        #    with open(temp_filename, 'wb') as handle:
        #        for block in r.iter_content(1024):
        #            handle.write(block)

        #    # https://stackoverflow.com/a/23615677/1322110
        #    # https://stackoverflow.com/a/27917855/1322110
        #    # Qualys is sh*t and has garbage at the top of the file!!
        #    csv_regex = '^\"IP\".*' # CSV Header Regex
        #    csv_head_found = False # By Default not found

        #    with open(temp_filename, 'r') as f: # Open Source CSV
        #        with open(filename, 'w') as f1: # Wrire Destination CSV
        #            for line in f:
        #                if re.match(csv_regex, line): # Can we match the CSV Header
        #                    csv_head_found = True
        #                if csv_head_found: # Only write destination CSV after the Header is found, causing other top-of-file #garbage to be ignored
        #                    f1.write(line)


        #    logger.info("Report downloaded - %s", filename)
        #    os.remove(temp_filename)
        #else:
        with open(filename, 'wb') as handle:
            for block in r.iter_content(1024):
                handle.write(block)

        qualys_delete(expected_report_id)
        logger.info("FINISHED!")


# If run from interpreter, run main code function.
if __name__ == "__main__":
    main()
	#!/usr/bin/env python
	# coding=utf-8
	# Python linter configuration.
	# pylint: disable=I0011
	# pylint: disable=C0301
	# pylint: disable=W0702
	"""
	Script to Generate a report on qualys, then download it.

	1st Public Release - Tweaked from Private version, some code might need checking.

	# Nick Bettison - Linickx.com

	"""

	import sys
	import os
	import logging
	import logging.handlers
	import socket
	import time
	import re
	from xml.etree import ElementTree


	version = "1.0"

	"""
	Variables to chang!
	/Start
	"""

	# Qualys
	apiuser = 'nick'
	apipass = 'linickx'
	apiurl = 'https://qualysguard.qg3.apps.qualys.com/api/2.0/fo/report/'
	apisleep = 10 # Time to sleep between api calls

	# Template ID, this is an array so you can have many!
	apitemplateid = ["1712399"]

	# Only tested with CSV -
	apireportformat = "csv"

	# Proxy!
	proxies_enable = False
	proxies = {
	'http': 'http://proxy.local:8080',
	'https': 'http://proxy.local:8080'
	}
	ssl_verify = True
	path = ''

	syslog_enable = False
	syslog_server = "10.10.10.10"
	syslog_port = 514
	syslog_protocol = "udp"
	syslog_facility = "local0"


	"""
	/END
	"""

	# Logging Setup
	logger = logging.getLogger("qualys1")
	logger.setLevel(logging.INFO) # Default Loging Level
	formatter = logging.Formatter('[%(levelname)s] %(asctime)s %(message)s')

	# Log to Console
	chandler = logging.StreamHandler()
	chandler.setFormatter(formatter)
	#chandler.setLevel(logging.DEBUG) # Use to over-ride logging level
	# Add to logger
	logger.addHandler(chandler)

	if syslog_protocol == "udp":
	socktype = socket.SOCK_DGRAM
	elif syslog_protocol == "tcp":
	socktype = socket.SOCK_STREAM
	else:
	logger.critical("Unknown Syslog Protocol %s", syslog_protocol)
	syslog_enable = False

	if syslog_enable:
	# Log to Syslog
	shandler = logging.handlers.SysLogHandler(address = (syslog_server, syslog_port), facility = syslog_facility, socktype = socktype)
	sformatter = logging.Formatter('%(name)s - %(message)s')
	shandler.setFormatter(sformatter)
	#shandler.setLevel(logging.DEBUG) # Use to over-ride logging level
	# Add to logger
	logger.addHandler(shandler)

	try:
	import requests # http://docs.python-requests.org/en/master/
	except:
	logger.error("import requests failed - type pip install requests")
	logger.debug("Exception: %s", sys.exc_info()[0])
	sys.exit(1)

	request_headers = {} # Python Curl Stuff!
	request_headers['user-agent'] = 'LINICKX Downloader / Version ' + version
	request_headers['X-Requested-With'] = 'QualysApi'

	def qualys_status():
	"""
	Returns the status of reports on qualys API
	"""
	# -d 'action=list'
	data = {"action": "list"}
	if proxies_enable:
	qr = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)
	else:
	qr = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)

	logger.info("Request status code: %s", qr.status_code)
	qr.raise_for_status() # Lazy - Throw an error for bad status codes
	qroot = ElementTree.fromstring(qr.content)
	logger.debug(qr.content)

	return qroot

	def qualys_launch(template_id=None):
	"""
	Requests that a report is run via qualys API
	"""
	# -d "action=launch&template_id=1611111&output_format=csv"
	data = {"action": "launch", "template_id":template_id, "output_format":apireportformat, "hide_header":"1"}

	report_wait = True # Enable the loop
	while report_wait:
	if proxies_enable:
	r = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)
	else:
	r = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)

	logger.info("Request status code: %s", r.status_code)
	r.raise_for_status() # Lazy - Throw an error for bad status codes
	root = ElementTree.fromstring(r.content)
	logger.debug(r.content)
	if re.match(r'^Max[\s]number[\s]of[\s]allowed(.*)', root[0][1].text):
	logger.info(root[0][1].text)
	logger.info("Sleeping %s seconds", apisleep)
	time.sleep(apisleep)
	else:
	report_wait = False

	try:
	root[0][2]
	except:
	logger.error(root[0][1].text)
	return None

	for item in root[0][2]:
	counter = 0
	for entry in item:
	logger.debug("Checking XML Entry: %s", entry.text)
	counter += 1 # Increment the counter early, so value is saved after IF
	if entry.text == "ID":
	expected_report_id = item[counter].text
	logger.info("Report ID: %s", expected_report_id)
	return expected_report_id

	return None

	def qualys_delete(report_id=None):
	"""
	Requests that a saved report is deleted
	"""
	data = {"action": "delete", "id":report_id}

	if proxies_enable:
	r = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)
	else:
	r = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=data)

	logger.info("Delete Request status code: %s", r.status_code)
	r.raise_for_status() # Lazy - Throw an error for bad status codes
	root = ElementTree.fromstring(r.content)
	logger.debug(r.content)


	def main():
	"""
	The main loop, request the reports.
	"""

	reports = []
	# Loop 1 launch the reports
	for template_id in apitemplateid:
	expected_report_id = qualys_launch(template_id)

	if expected_report_id is None:
	logger.critical("Something went wrong, report ID not found")
	continue

	the_report = {'expected_report_id':expected_report_id, 'template_id':template_id}
	reports.append(the_report)

	# Loop 2 Check the report status
	for the_report in reports:

	expected_report_id = the_report['expected_report_id']
	template_id = the_report['template_id']
	"""
	Take a break, new reports take time...
	"""

	logger.debug("Sleeping %s seconds before requesting the report %s", apisleep, template_id)
	time.sleep(apisleep)

	"""
	Begin section to request the status of the report
	"""

	report_running = True # Enable the loop

	# Loop for checking the status
	while report_running:
	qualys_reports_list = qualys_status()
	number_of_reports = len(qualys_reports_list[0][1])
	logger.debug("%s Reports found", number_of_reports)

	# Loop through the qualys reports
	for report in qualys_reports_list[0][1]:
	try:
	report_id = report[0].text
	report_status = report[6][0].text
	except:
	logger.debug(report[0].text) # This is a weird edge case that I've not resolved!
	logger.debug(report[6].text)
	logger.debug("one day I will fix this weird shizzle")
	continue

	logger.debug("Report: %s Status: %s", report_id, report_status)

	if report_id == expected_report_id: # Found our Report
	if report_status == "Finished": # It's finished!
	report_running = False
	logger.info("Report: %s %s, ready for download", report_id, report_status)
	else:
	report_percent = report[6][2].text # Output the progress
	logger.info("Report %s Still %s, %s percent complete", report_id, report_status, report_percent)

	if report_running: # Only Sleep in long loops
	try:
	logger.debug("Sleeping %s seconds", apisleep)
	time.sleep(apisleep)
	except KeyboardInterrupt:
	logger.critical("CTRL+C Detected, quitting...")
	sys.exit(1)


	# Loop 3 Download Reports
	for the_report in reports:
	expected_report_id = the_report['expected_report_id']
	template_id = the_report['template_id']


	"""
	Begin section to download the report
	"""
	filename = path + "qualys_report_" + template_id + "." + apireportformat
	temp_filename = filename + ".tmp"



	# -d 'action=fetch&id=2911119'
	qualys_data = {"action": "fetch", "id":expected_report_id}

	if proxies_enable:
	r = requests.post(apiurl, proxies=proxies, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=qualys_data, stream=True)
	else:
	r = requests.post(apiurl, verify=ssl_verify, auth=(apiuser, apipass), headers=request_headers, data=qualys_data, stream=True)

	logger.info("Request status code: %s", r.status_code)
	r.raise_for_status() # Lazy - Throw an error for bad status codes


	#if apireportformat == "csv":
	# # https://stackoverflow.com/a/14114741/1322110
	# with open(temp_filename, 'wb') as handle:
	# for block in r.iter_content(1024):
	# handle.write(block)

	# # https://stackoverflow.com/a/23615677/1322110
	# # https://stackoverflow.com/a/27917855/1322110
	# # Qualys is sh*t and has garbage at the top of the file!!
	# csv_regex = '^\"IP\".*' # CSV Header Regex
	# csv_head_found = False # By Default not found

	# with open(temp_filename, 'r') as f: # Open Source CSV
	# with open(filename, 'w') as f1: # Wrire Destination CSV
	# for line in f:
	# if re.match(csv_regex, line): # Can we match the CSV Header
	# csv_head_found = True
	# if csv_head_found: # Only write destination CSV after the Header is found, causing other top-of-file #garbage to be ignored
	# f1.write(line)


	# logger.info("Report downloaded - %s", filename)
	# os.remove(temp_filename)
	#else:
	with open(filename, 'wb') as handle:
	for block in r.iter_content(1024):
	handle.write(block)

	qualys_delete(expected_report_id)
	logger.info("FINISHED!")



	# If run from interpreter, run main code function.
	if __name__ == "__main__":
	main()