vulners_scanner.py

"""
vulners_scanner.py

Copyright 2018 Kir Ermakov (isox@vulners.com), Ilya Govorkov (gmedian@vulners.com)

This file is part of w3af, http://w3af.org/ .

w3af is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation version 2 of the License.

w3af is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with w3af; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

"""
import w3af.core.data.constants.severity as severity

from w3af.core.controllers.plugins.grep_plugin import GrepPlugin
from w3af.core.data.bloomfilter.scalable_bloom import ScalableBloomFilter
from w3af.core.data.quick_match.multi_re import MultiRE
from w3af.core.data.kb.vuln import Vuln
import w3af.core.controllers.output_manager as om
import vulners
import collections
import re

class vulners_scanner(GrepPlugin):
    """
    Find software vulnerabilities using Vulners.com API

    :author: Kir Ermakov (isox@vulners.com), Ilya Govorkov (gmedian@vulners.com)
    """


    def __init__(self):
        GrepPlugin.__init__(self)

        # Vulners shared objects

        self._vulners_api = None
        self.rules_table = None
        self.rules_updated = False

        self._already_visited = ScalableBloomFilter()
        self._vulnerability_cache = {}

    def update_vulners_rules(self):
        # Get fresh rules from Vulners.
        try:
            self.rules_table = self.get_vulners_api().rules()
            # Adapt it for MultiRe structure [(regex,alias)] removing regex duplicated
            regexAliases = collections.defaultdict(list)
            for softwareName in self.rules_table:
                regexAliases[self.rules_table[softwareName].get('regex')] += [softwareName]
            # Now create fast RE filter
            self._multi_re = MultiRE(((regex, regexAliases.get(regex)) for regex in regexAliases), re.IGNORECASE)
        except Exception as e:
            self.rules_table = None
            error_message = 'Vulners plugin failed to init with error: %s'
            om.out.error(error_message % e)

    def get_vulners_api(self):
        # Lazy import. Just not to make it in the __init__.
        if not self._vulners_api:
            self._vulners_api = vulners.Vulners()
        return self._vulners_api

    def check_vulners(self, software_name, software_version, check_type):
        vulnerabilities = {}
        cached_result = self._vulnerability_cache.get((software_name, software_version, check_type))
        if cached_result:
            return cached_result
        # Ask Vulners about vulnerabilities
        if check_type == 'software':
            vulnerabilities = self.get_vulners_api().softwareVulnerabilities(software_name, software_version)
        elif check_type == 'cpe':
            cpe_string = "%s:%s" % (software_name, software_version)
            vulnerabilities = self.get_vulners_api().cpeVulnerabilities(cpe_string.encode())
        self._vulnerability_cache[(software_name, software_version, check_type)] = vulnerabilities
        return vulnerabilities

    def get_severity(self, cvss_score):
        cvss_score = cvss_score * 10
        if cvss_score in range(20, 30):
            return severity.LOW
        elif cvss_score in range(30, 70):
            return severity.MEDIUM
        elif cvss_score in range(70, 100):
            return severity.HIGH
        else:
            return severity.INFORMATION

    def grep(self, request, response):
        """
        Plugin entry point, search for directory indexing.
        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None

        """
        # Lazy update rules if it's first start of the plugin
        if not self.rules_updated:
            self.update_vulners_rules()
            self.rules_updated = True

        # Check if we have downloaded rules well.
        # If there is no rules - something went wrong, time to exit.
        if not self.rules_table:
            return
        # We do not parse non-text output
        if not response.is_text_or_html():
            return
        if response.get_url().get_domain_path() in self._already_visited:
            return

        self._already_visited.add(response.get_url().get_domain_path())

        raw_response = response.dump()

        # Here we will store uniq vulnerability map
        vulnerabilities_summary = {}

        for match, _, regex_comp, software_list in self._multi_re.query(raw_response):
            # If RE matched we have
            detected_version = match.group(1)
            for software_name in software_list:
                mathced_rule = self.rules_table[software_name]
                vulnerabilities_map = self.check_vulners(software_name= mathced_rule['alias'].encode(), software_version= detected_version, check_type= mathced_rule['type'].encode())
                flattened_vulnerability_list = [item for sublist in vulnerabilities_map.values() for item in sublist]
                for bulletin in flattened_vulnerability_list:
                    if bulletin['id'] not in vulnerabilities_summary:
                        vulnerabilities_summary[bulletin['id']] = bulletin

        # Now add KB's for found vulnerabilities
        for bulletin in vulnerabilities_summary.values():
            v = Vuln(bulletin['id'], bulletin['description'] or bulletin.get('sourceData', bulletin['title']), self.get_severity(bulletin.get('cvss', {}).get('score', 0)), response.id,
                     self.get_name())
            v.set_url(response.get_url())
            self.kb_append_uniq(self, 'vulners', v, 'URL')


    def get_long_desc(self):
        """
        :return: A DETAILED description of the plugin functions and features.
        """
        return """
        This plugin greps every softwarer banner and checks vulnerabilities online at vulners.com database.
        """

I like the plugin!

Quick question: Don't you need an API key to consume the API? Seen this in the pypi docs for vulners:

vulners_api = vulners.Vulners(api_key="YOUR_API_KEY_HERE")

Updated it little bit.
For API performance - yep.
When working in anonymous mode w/o api key you will get anonymous ratelimits.
They are OK but using api key is a better way.
But I really don't know how to make user input for that :))

Ah! I was unaware that the API did allow anonymous access.

Regarding the user-configured setting, you can use this as inspiration: https://github.com/andresriancho/w3af/blob/master/w3af/plugins/crawl/web_spider.py#L424-L460

Awesome!
I think I will add it today.
Pull request soon :)

andresriancho/w3af#17166