So, Threema, tell me... how many servers do you use?

README.md

Note

On machines where Python 3 is not the default Python runtime, you should use pip3 instead of pip and python3 instead of python.

Prerequisites

$ sudo apt-get install python3 python3-pip

We recommend using the virtualenv package to create an isolated Python environment:

$ sudo pip install virtualenv
$ virtualenv -p python3 threema-servers-venv

You can switch into the created virtual environment threema-servers-venv by running this command:

$ source threema-servers-venv/bin/activate

To deactivate the virtual environment, just run:

$ deactivate

Installation

If you are using a virtual environment, activate it first.

Install the dependencies by running:

$ pip install asyncio aiodns aiohttp

Run It

$ python how-many-servers-does-threema-use.py

or

$ python how-many-servers-does-threema-use.py --geoip

how-many-servers-does-threema-use.py

import asyncio
import collections
import sys

import aiodns
import aiohttp

web_server_domains = ['www.threema.ch', 'threema.ch']
chat_server_domain = 'g-{:02x}.0.threema.ch'
api_server_domains = ['api.threema.ch']
blob_upload_server_domains = ['upload.blob.threema.ch', 'blob-upload.threema.ch']
blob_server_domain = '{:02x}.blob.threema.ch'
my_id_server_domains = ['myid.threema.ch']
avatar_server_domains = ['avatar.threema.ch']
shop_server_domains = ['shop.threema.ch']
gateway_server_domains = ['gateway.threema.ch', 'msgapi.threema.ch']
work_server_domains = ['work.threema.ch']

geo_ip_api = 'http://geoip.nekudo.com/api/{}/en/short'


def counted_host_results(counted_hosts):
    return '\n'.join((
        '{}. {}: {}x'.format(item, host, count)
        for item, (host, count) in enumerate(counted_hosts.items(), start=1)
    ))


@asyncio.coroutine
def query_geo_ip(session, geo_ip_cache, semaphore, host_or_ip):
    yield from semaphore.acquire()
    if host_or_ip not in geo_ip_cache:
        response = yield from session.get(geo_ip_api.format(host_or_ip))
        json = yield from response.json()
        location = '{country[name]}'.format(**json)
        yield from response.release()
        geo_ip_cache[host_or_ip] = location
        yield from asyncio.sleep(0.1)
    semaphore.release()
    return geo_ip_cache[host_or_ip]


@asyncio.coroutine
def query(*args, domain, type_):
    loop, resolver, session, geo_ip_cache, semaphore = args
    try:
        results = yield from resolver.query(domain, type_)
    except aiodns.error.DNSError as exc:
        return []
    else:
        if session is not None:
            geo_ip_results = yield from asyncio.gather(*[
                query_geo_ip(session, geo_ip_cache, semaphore, result.host)
                for result in results
            ], loop=loop)
            results = [
                (result.host, geo_ip_result)
                for result, geo_ip_result in zip(results, geo_ip_results)
            ]
        else:
            results = [(result.host, None) for result in results]
    return results


@asyncio.coroutine
def query_ip4_ip6(*args, domain):
    loop, *_ = args
    results_list = yield from asyncio.gather(*[
        query(*args, domain=domain, type_='A'),
        query(*args, domain=domain, type_='AAAA'),
    ], loop=loop)
    return (result for results in results_list for result in results)


def format_result(result):
    host, location = result
    if location is not None:
        return '{} ({})'.format(host, location)
    else:
        return host


@asyncio.coroutine
def query_all(*args, domains):
    loop, *_ = args
    coroutines = [query_ip4_ip6(*args, domain=domain) for domain in domains]
    results_list = yield from asyncio.gather(*coroutines, loop=loop)
    results = (format_result(result) for results in results_list for result in results)
    counted_hosts = collections.Counter(results)
    return counted_hosts


@asyncio.coroutine
def web_servers(*args):
    counted_hosts = yield from query_all(*args, domains=web_server_domains)
    return '{} web servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def chat_servers(*args):
    domains = [chat_server_domain.format(group)
               for group in range(0x00, 0xff + 1)]
    counted_hosts = yield from query_all(*args, domains=domains)
    return '{} chat servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def api_servers(*args):
    counted_hosts = yield from query_all(*args, domains=api_server_domains)
    return '{} api servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def blob_upload_servers(*args):
    counted_hosts = yield from query_all(*args, domains=blob_upload_server_domains)
    return '{} blob upload servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def blob_servers(*args):
    domains = [blob_server_domain.format(group)
               for group in range(0x00, 0xff + 1)]
    counted_hosts = yield from query_all(*args, domains=domains)
    return '{} blob servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def my_id_servers(*args):
    counted_hosts = yield from query_all(*args, domains=my_id_server_domains)
    return '{} my id servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )
    
    
@asyncio.coroutine
def avatar_servers(*args):
    counted_hosts = yield from query_all(*args, domains=avatar_server_domains)
    return '{} avatar servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def shop_servers(*args):
    counted_hosts = yield from query_all(*args, domains=shop_server_domains)
    return '{} shop servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def gateway_servers(*args):
    counted_hosts = yield from query_all(*args, domains=gateway_server_domains)
    return '{} gateway servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def work_servers(*args):
    counted_hosts = yield from query_all(*args, domains=work_server_domains)
    return '{} work servers\n{}\n'.format(
        len(counted_hosts),
        counted_host_results(counted_hosts),
    )


@asyncio.coroutine
def main(loop):
    geoip = len(sys.argv) > 1 and sys.argv[1] == '--geoip'
    resolver = aiodns.DNSResolver(loop=loop)
    session = aiohttp.ClientSession(loop=loop) if geoip else None
    geo_ip_cache = {}
    semaphore = asyncio.Semaphore(loop=loop)
    print('Threema currently uses...')
    print()

    args = [loop, resolver, session, geo_ip_cache, semaphore]
    results = yield from asyncio.gather(*[
        web_servers(*args),
        chat_servers(*args),
        api_servers(*args),
        blob_upload_servers(*args),
        blob_servers(*args),
        my_id_servers(*args),
        avatar_servers(*args),
        shop_servers(*args),
        gateway_servers(*args),
        work_servers(*args),
    ], loop=loop)
    print('\n'.join(results))
    
    print('(IPv4 and IPv6 counted separately)')
    if session is not None:
        yield from session.close()


if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main(loop))

output-2016-10-10.log

Threema currently uses...

2 web servers
1. 94.126.23.61 (Switzerland): 2x
2. 2a00:1128:1:1::23:61 (Switzerland): 2x

1 chat servers
1. 5.148.175.198 (Switzerland): 256x

1 api servers
1. 5.148.175.215 (Switzerland): 1x

2 blob upload servers
1. 5.148.175.206 (Switzerland): 2x
2. 5.148.175.208 (Switzerland): 2x

2 blob servers
1. 5.148.175.206 (Switzerland): 4x
2. 5.148.175.208 (Switzerland): 3x

2 my id servers
1. 2a02:418:3009:303::216 (Switzerland): 1x
2. 5.148.175.216 (Switzerland): 1x

2 avatar servers
1. 2a02:418:3009:303::211 (Switzerland): 1x
2. 5.148.175.211 (Switzerland): 1x

2 shop servers
1. 2a02:418:3009:303::216 (Switzerland): 1x
2. 5.148.175.216 (Switzerland): 1x

4 gateway servers
1. 2a02:418:3009:303::211 (Switzerland): 1x
2. 2a02:418:3009:303::214 (Switzerland): 1x
3. 5.148.175.211 (Switzerland): 1x
4. 5.148.175.214 (Switzerland): 1x

2 work servers
1. 2a02:418:3009:303::211 (Switzerland): 1x
2. 5.148.175.211 (Switzerland): 1x

(IPv4 and IPv6 counted separately)

What about Gateway servers? msgapi.threema.ch

I've added web, gateway and work server domains.

You might wanna use the HTTPS-Geolocation-API: https://geoip.nekudo.com/
Not that someone fakes the location. 😉