Skip to content

Instantly share code, notes, and snippets.

Forked from csett86/
Created June 20, 2020 04:39
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
What would you like to do?
import argparse
import resource
import aiohttp
import aiodns
import asyncio
from collections import Counter
import datetime
import time
import socket
import pickle
import struct
import psycopg2
async def fetch(session, url, headers=None):
# print(f'trying {url}')
async with session.get(url, ssl=False, headers=headers) as response:
return await response.json(content_type=None)
def ensure_rlimit_file():
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft < 100000 or hard < 100000:
print(f'collection with aiohttp will not work reliably using soft {soft} and hard {hard} open file limit, increase limit for user')
async def homeserver_for_domain(session, resolver, domain):
# for spec see
# boils down to .well-known, _matrix._tcp. SRV, domain:8448
headers = None
res = await fetch(session, f'https://{domain}/.well-known/matrix/server')
homeserver_initial = res['m.server']
if ':' not in homeserver_initial:
homeserver = f'{homeserver_initial}:8448'
homeserver = homeserver_initial
return homeserver, headers, 'well-known'
except Exception as e:
# well-known failed, try SRV
res = await resolver.query(f'_matrix._tcp.{domain}', 'SRV')
homeserver = f'{res[0].host}:{res[0].port}'
# if srv is found, request needs to include Host header for domain, not srv host. is a good example for this
headers = {'Host': domain}
return homeserver, headers, 'SRV'
except Exception as e:
# well-known failed, SRV failed, use domain:8448 as fallback
homeserver = f'{domain}:8448'
return homeserver, headers, 'fallback'
async def version_for_homeserver(session, resolver, domain, homeserver, headers, method, debug=False):
res = await fetch(session, f'https://{homeserver}/_matrix/federation/v1/version', headers)
versions = res['server']
# version number may look like 1.6.1 (abcd,branch,...), remove everything after first space with split()[0]
version_string = '{0}/{1}'.format(versions['name'], versions['version'].split()[0])
if debug:
print(f'{domain} has {homeserver} via {method} with {version_string}')
return version_string
except Exception as e:
if debug:
print(f'{domain} failed with {e} using {method}')
async def version_for_domain(session, resolver, domain, debug):
homeserver, headers, method = await homeserver_for_domain(session, resolver, domain)
return await version_for_homeserver(session, resolver, domain, homeserver, headers, method, debug)
def file_destinations(file_destinations_file='test_destinations.txt'):
with open(file_destinations_file) as f:
destinations = []
for line in f.readlines():
return destinations
def test_destinations():
# matrix: no well-known, srv with port, host header necessary
# no well-known, srv with port, host header not necessary
# well-known with port 443
# well-known without port (thus implicit 8448)
# no well-known, no srv, thus fallback
return ['', '', '', '', '']
def postgres_destinations():
conn = psycopg2.connect('dbname=synapse host=localhost user=matrix_stats password=matrix_stats')
cur = conn.cursor()
cur.execute('SELECT destination FROM destinations;')
destinations = []
for destination in cur:
destination = destination[0]
return destinations
def format_report(versions):
report = ""
total = 0
for _, count in versions.items():
total = total + count
report += '{0}\n'.format(
report += f'{total} homeservers online\n\n'
for version, count in versions.items():
report += f'{count:<4} {version}\n'
return report
def graphite(versions, host='localhost', port=2004):
now = int(time.time())
tuples = ([])
for version, count in list(versions.items())[:15]:
version = version.lower().replace('.', '-').replace('/', '.')
tuples.append((version, (now, count)))
package = pickle.dumps(tuples, 2)
size = struct.pack('!L', len(package))
sock = socket.socket()
sock.connect( (host, port) )
except socket.error:
print(f'Couldnt connect to {host} on port {port}, is running?')
async def main(args):
debug = args.debug
if args.enable_postgres:
destination_func = postgres_destinations
elif args.enable_file_destinations:
destination_func = file_destinations
destination_func = test_destinations
timeout = aiohttp.ClientTimeout(sock_connect=5, sock_read=5)
async with aiohttp.ClientSession(timeout=timeout) as session:
resolver = aiodns.DNSResolver()
tasks = []
for domain in destination_func():
tasks.append(version_for_domain(session, resolver, domain, debug))
versions_initial = Counter(await asyncio.gather(*tasks)).most_common()
# remove "None" version count for unreachable servers
versions = {}
for version, count in versions_initial:
if version is not None:
versions[version] = count
if debug:
print('\n' + format_report(versions), end='')
if args.enable_report:
reportfile = 'reports/report-{0}.txt'.format('seconds'))
with open(reportfile, 'w') as f:
wwwfile = '/var/www/html/mxversions.txt'
with open(wwwfile, 'w') as f:
if args.enable_graphite:
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--enable-postgres", help="use postgres destinations, if not specified, uses internal test destinations", action="store_true")
parser.add_argument("--enable-file-destinations", help="destinations from file test_destinations.txt in current folder, do not use with --enable-postgres", action="store_true")
parser.add_argument("--enable-report", help="write out a report to file", action="store_true")
parser.add_argument("--enable-graphite", help="send result to graphite on localhost:2004 via pickle", action="store_true")
parser.add_argument("--debug", help="print out report for debugging to stdout", action="store_true")
args = parser.parse_args()
loop = asyncio.get_event_loop()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment