Querying 50k Gameservers in 100 lines of Python

collect.py

from gevent import monkey
monkey.patch_all()

import logging
from collections import deque

from gevent.pool import Pool
from valve.source.master_server import MasterServerQuerier
from valve.source.a2s import ServerQuerier, NoResponseError
from valve.source.messages import BrokenMessageError

MASTER_HOSTS = deque(['208.64.200.52', '208.64.200.65', '208.64.200.39'])

MASTER_TIMEOUT = 30
SERVER_TIMEOUT = 5
# See: https://python-valve.readthedocs.org/en/latest/master_server.html#valve.source.master_server.MasterServerQuerier.find
VALVE_REGIONS = [u'na', u'sa', u'eu', u'as', u'oc', u'af', u'rest']


pool = Pool(size=1000)


def get_server_stats(address):
    server = ServerQuerier(address, timeout=SERVER_TIMEOUT)

    try:
        ping = server.ping()
        info = server.get_info()

        logging.info(u'Updated {0}:{1} {2} @ {3}ms'.format(
            address[0], address[1], info['server_name'], ping)
        )
        return True

    except (NotImplementedError, NoResponseError, BrokenMessageError):
        pass


def find_servers(host, region):
    count = 0
    greenlets = []
    master = MasterServerQuerier(
        address=(host, 27011), timeout=MASTER_TIMEOUT
    )

    try:
        logging.info('Querying master {0} for region {1}'.format(host, region))

        for address in master.find(region=[region]):
            logging.info('Querying server: {0}'.format(address))
            greenlets.append(pool.spawn(get_server_stats, address))
            count += 1

    except NoResponseError as e:
        # Protocol is UDP so there's no "end"
        if u'Timed out' not in e.message:
            logging.warning('Error querying master server: {0}'.format(e))
    finally:
        logging.info('Found {0} addresses'.format(count))
        return greenlets


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)

    collectors = []

    for region in VALVE_REGIONS:
        collectors.append(pool.spawn(find_servers, MASTER_HOSTS[0], region))
        MASTER_HOSTS.rotate(1)

    greenlets = []

    for collector in collectors:
        greenlets += collector.get()

    logging.info('Counting results...')
    greenlets = [greenlet.get() for greenlet in greenlets]

    successes = filter(None, greenlets)
    logging.info('Collected {0}/{1}'.format(len(successes), len(greenlets)))