grab lots and lots of hashes of favicons. Used this for adding hashes to http-favicon.nse this script is really bad and doesnt work right.

masscan-favicon-grab.py

#!/usr/bin/env python3

import os
import sys
import time
import fcntl
import errno
import socket
import hashlib
import requests
import urllib3
from threading import Thread
from queue import Queue


Q = Queue()


def valid_ipv4_address(ip_address):
    """ valid_ipv4_address() - Validate an IPv4 address.
    Args:
        ip_address (str) - IP address to verify.
    Returns:
        True if ip_address is a valid IPv4 address.
        False if ip_address is not a valid IPv4 address.
    """
    try:
        socket.inet_aton(ip_address)
    except socket.error:
        return False
    return True


def log(host, port, message):
    with open("favicon-%s" % sys.argv[1], "a") as logfile:
        logfile.write("%d %s %s\n" %
                      (time.time(), host, message))



def grab_favicon(host, port, timeout=5):
    url = "http://" + host + ":" + str(port) + "/favicon.ico"
    try:
        r = requests.get(url, timeout=timeout)
    except requests.exceptions.ConnectionError:
        #print("connect failed", url)
        return False
    except requests.exceptions.ReadTimeout:
        #print("timed out", url)
        return False
    except requests.exceptions.TooManyRedirects:
        print(" [-] Too many redirects: %s" % url)
        return False
    except requests.exceptions.ContentDecodingError:
        print(" [-] ContentDecodingError: %s" % url)
        return False
    except urllib3.exceptions.LocationValueError:
        print(" [-] LocationValueError: %s" % url)
        return False
    except urllib3.exceptions.ProtocolError:
        print(" [-] ProtocolError: %s" % url)
        return False
    except requests.exceptions.ChunkedEncodingError:
        print(" [-] ChunkedEncodingError: %s" % url)
        return False
    except requests.exceptions.InvalidSchema:
        return False
    except UnicodeError:
        return False
    except AttributeError:
        return False
    favicon_hash = hashlib.md5(r.content).hexdigest()
    #print(favicon_hash, host, len(r.content))
    log(host, port, "%s %d %d" % (favicon_hash, r.status_code, len(r.content)))
    return True


def thread_scan(thread_id, queue):
    while True:
        host, port = queue.get()
        #print(" [*] Scanning %s:%s on thread %d" % (host, str(port), thread_id))
        grab_favicon(host, port, timeout=5)
        queue.task_done()


def process_ip_list(ip_list, port=0):
    for ip in ip_list:
        if valid_ipv4_address(ip):
            Q.put((ip, port))
        else:
            print("[-] invalid ip: %s" % ip)

    print("[+] Scanning %s - %s (%d/%d %d%%) -- Elapsed: %s" %
          (ip_list[0], ip_list[-1],
           progress, ip_count, progress / ip_count * 100,
           display_time(round(time.time() - start_time))))
    for thread in range(32):
        worker = Thread(target=thread_scan, args=(thread, Q))
        worker.setDaemon(True)
        worker.start()

    Q.join()


def linecount(filename):
    with open(filename, "r") as file:
        for i, _ in enumerate(file):
            pass
    return i + 1


intervals = (
    ("weeks", 604800), # 60 * 60 * 24 * 7
    ("days", 86400),   # 60 * 60 * 24
    ("hours", 3600),   # 60 * 60
    ("minutes", 60),
    ("seconds", 1)
)


def display_time(seconds, granularity=2):
    result = []

    if seconds == 0:
        return "0 seconds"

    for name, count in intervals:
        value = seconds // count
        if value:
            seconds -= value * count
            if value == 1:
                name = name.rstrip("s")
            result.append("{} {}".format(value, name))
    return ", ". join(result[:granularity])


print("[+] Scanning ips in %s" % sys.argv[1])
ip_count = linecount(sys.argv[1])
progress = 0
start_time = time.time()

with open(sys.argv[1], "r") as fp:
    count = 0
    cset = []

    for line in fp:
        count += 1
        cset += [line.rstrip()]
        if count % 1000 == 0:
            progress += len(cset)
            process_ip_list(cset, port=80)
            cset = []
    progress += len(cset)
    if len(cset) > 0:
        process_ip_list(cset, 80)

print("[+] Finished. Elapsed: %s" % display_time(time.time() - start_time))