findmyinstance.py

from concurrent.futures import ThreadPoolExecutor
from timeit import default_timer
from urllib.parse import urlparse
import http.client
import json
import statistics
import socket
import time


POOL = {}
HEADERS = {
    'User-Agent': 'searx.space gist FindMyInstance'
}


def get_connection(host):
    connection = POOL.get(host)
    if connection is None:
        connection = http.client.HTTPSConnection(host)
        POOL[connection] = connection
    return connection


def https_get(url):
    p = urlparse(url)
    conn = get_connection(p.hostname)
    conn.request("GET", p.path, headers=HEADERS)
    res = conn.getresponse()
    if res.status != 200:
        raise ValueError()
    return res.read()


def iter_ip(instances):
    for url, instance in instances.items():
        if instance["network_type"] != "normal":
            continue
        if url.endswith(".i2p/"):
            continue
        yield url


def check(url):
    measures = []
    for _ in range(8):
        try:
            before = default_timer()
            https_get(url)
            elapsed_in_sec = default_timer() - before
            elapsed_in_msec = round(elapsed_in_sec * 1000)
            measures.append(elapsed_in_msec)
        except socket.gaierror:
            print('Connection issue to ', url)
        except ValueError:
            print('Error in response', url)
        time.sleep(0.5)
    if measures:
        return url, statistics.median(measures), statistics.mean(measures)
    return url, -1, -1
    

r = https_get("https://searx.space/data/instances.json")
data = json.loads(r)
instances = data["instances"]
with ThreadPoolExecutor(max_workers=10) as executor:
    results = list(executor.map(check, iter_ip(instances)))
results.sort(key=lambda instance: instance[1])
print(f"{'Instance':55} {'Mediane':>8} {'Mean':>8}")
for url, median, mean in results:
    print(f"{url:55} {median:8.0f} {mean:8.0f}")