Querying 50k Gameservers in 100 lines of Python

collect.py

from gevent import monkey
monkey.patch_all()

import logging

from gevent.pool import Pool
from valve.source.master_server import MasterServerQuerier
from valve.source.a2s import ServerQuerier, NoResponseError
from valve.source.messages import BrokenMessageError

MASTER_HOST = 'hl2master.steampowered.com'
MASTER_TIMEOUT = 60
SERVER_TIMEOUT = 5


pool = Pool(size=1000)


def get_server_stats(address):
    server = ServerQuerier(address, timeout=SERVER_TIMEOUT)

    try:
        ping = server.ping()
        info = server.get_info()

        logging.info(u'Updated {0}:{1} {2} @ {3}ms'.format(
            address[0], address[1], info['server_name'], ping)
        )
        return True

    except (NotImplementedError, NoResponseError, BrokenMessageError):
        pass


def find_servers():
    count = 0
    greenlets = []
    master = MasterServerQuerier(
        address=(MASTER_HOST, 27011), timeout=MASTER_TIMEOUT
    )

    try:
        for address in master.find():
            logging.info('Querying server: {0}'.format(address))
            greenlets.append(pool.spawn(get_server_stats, address))
            count += 1

    except NoResponseError as e:
        # Protocol is UDP so there's no "end"
        if u'Timed out' not in e.message:
            logging.warning('Error querying master server: {0}'.format(e))
    finally:
        logging.info('Found {0} addresses'.format(count))
        return greenlets


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)

    greenlets = find_servers()

    logging.info('Counting results...')
    greenlets = [greenlet.get() for greenlet in greenlets]

    successes = filter(None, greenlets)
    logging.info('Collected {0}/{1}'.format(len(successes), len(greenlets)))