Create a gist now

Instantly share code, notes, and snippets.

@bcbwilla /
Last active Aug 29, 2015

What would you like to do?
Overcast Network ( server player count data collector.
import random
import math
import datetime
from pymongo import MongoClient
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from matplotlib.ticker import MultipleLocator
import numpy as np
class PlayerCountPlotter(object):
""" Used to create plots of Overcast Network (OCN) player counts """
def __init__(self,db="ocn_player_count",collection="server_stats",regions=['us','eu']):
self.REGIONS = regions
self.db = db
self.collection = collection = self.prep_data()
def prep_data(self):
"""Get all the data in database and return in organized dictionary
Return format:
{region: {server: [(time,count),...], ...},...},...}
where region is "us" or "eu", server is e.g. "alpha",
time is a timedate object and count is the server player count
at that time.
# get data from database
c = MongoClient()
db = c[self.db]
stats = db[self.collection]
# build data structure
player_counts = {}
for region in self.REGIONS:
player_counts[region] = {}
for s in stats.find():
time = s['time']
totals = {}
for region in self.REGIONS:
totals[region] = [] #region total player count
for network in s['stats']: # e.g. "us-project-ares" or "eu-blitz"
region = network[:2] # e.g. eu or us
network_stats = s['stats'][network]
for server in network_stats: # e.g. "alpha"
# add (time,count) tuple for server
if server in player_counts[region]:
player_counts[region][server] = [(time,network_stats[server])]
# sum all servers for each region to get region total
for region in self.REGIONS:
if 'total' in player_counts[region]:
player_counts[region]['total'] = [(time,sum(totals[region]))]
return player_counts
def get_server_plot_data(self,region_server):
"""Get data and return in plotable format.
Takes full name of server, e.g. "us-alpha"
Returns tuple of lists of time and count
i.e. ([time1,time2,...],[count1,count2,...])
region = region_server.split('-')[0]
server = region_server.split('-')[1]
d =
z = zip(*d[region][server])
x = list(z[0])
y = list(z[1])
return x,y
def make_good_colors(self,n):
""" Let's pick nice plot colors as difficulty as possible.
n is the number of series that is being plotted
COLORS = ['#a6cee3','#1f78b4','#b2df8a','#33a02c','#fb9a99','#e31a1c',
len_c = len(COLORS)
stride = int(math.floor(len_c/n)) if n < len_c else 1
return [c for c in COLORS[::stride]] #phew
def make_average_plots(self,server_list, include_points=False,
filename="", save_path="images/", show=False,
""" Produces hourly averaged 24 hour plots for servers
server_list: network-server names.
["us-alpha", "eu-mini01",...]
include_points: plots all data points (in addition to lines)
filename: output image filename
save_path: where to save images
show: display the dynamic graph using
errorbars: show error bars
color = self.make_good_colors(len(server_list))
fig = plt.figure()
ax = fig.add_subplot(111)
nc = 0 # for setting the color
for server in server_list:
data = self.get_server_plot_data(server)
x = data[0]
y = data[1]
# get averages. lots of kind of messy data maneuvering
# convert all datetime values to hourly bins, rounding down for minutes < 30
xh = []
for dt in x:
if dt.minute < 30:
if dt.hour < 23:
# this will store values for averages
d = [[] for i in range(len(xh))]
for i in range(len(xh)):
time = xh[i]
value = y[i]
avg_y = [np.mean(v) for v in d]
# we now plot x vs. y where x is an hour 0-23 and y is the
# average value at each hour
time = [h for h in range(len(xh))]
if errorbars:
std_y = [np.std(v) for v in d]
if include_points:
if nc == len(color):
nc = 0
nc += 1
ax.set_xlabel('hour of the day (EDT)')
ax.set_ylabel('average player count')
ax.yaxis.grid(b=True, which='major', color='black', alpha='0.1', linestyle='--')
# make legend a reasonable size
if len(server_list) >= 5:
l_size = 8
l_size = 10
leg = ax.legend(prop={'size':l_size},loc=2)
# save image
if not filename:
filename ="%Y-%m-%d--%H-%M-%S")
if show:
A cute little script to gather information about Overcast Network
server player counts. It uses python and mongodb, the cutest possible
technologies that are hip to know right now.
By bcbwilla, an admirably cute little scripter.
import urllib2
import datetime
from pymongo import MongoClient
from bs4 import BeautifulSoup
# get server play page and scrape the info with beautiful soup.
URL = ""
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
page =
html =
soup = BeautifulSoup(html, "html.parser")
tab_panes = soup.find_all("div",{"class":"tab-pane"})
network_stats = {}
ignore = ["main","us", "us-main", "eu", "eu-main"]
for tab_pane in tab_panes:
server_stats = {}
category = tab_pane['id']
if not (category in ignore or "lobby" in category):
rows = tab_pane.find_all("div",{"class":"span8"})
for row in rows:
h4s = row.find_all("h4")
for h4 in h4s:
name = h4.contents[0].strip().lower()
count = int(h4.contents[1].contents[0].strip())
server_stats[name] = count
network_stats[category] = server_stats
data = {'time':, 'stats': network_stats}
# put info in database
client = MongoClient()
db = client.ocn_player_count
s = db.server_stats
""" Run to make some graphs """
from graph import PlayerCountPlotter
# a dictionary of various server categories for plotting
to_plot = {
'network_totals': ['us-total','eu-total'],
'us_main': ['us-alpha','us-beta','us-capture1','us-capture2','us-destroy','us-deathmatch','us-gear','us-control','us-nostalgia','us-primed'],
'us_obj': ['us-capture1','us-capture2','us-destroy'],
'us_killy': ['us-deathmatch','us-gear','us-control'],
'us_tnt': ['us-nostalgia','us-primed'],
'us_mini': ['us-mini01','us-mini02','us-mini03','us-mini04','us-mini05','us-mini06','us-mini07','us-mini08','us-mini09','us-mini10','us-mini11'],
'us_blitz': ['us-chaos','us-cronus','us-rage1','us-rage2'],
'us_gs': ['us-gs1','us-gs2','us-gs3','us-gs4','us-gs5','us-gs6'],
'eu_main': ['eu-alpha','eu-capture','eu-destroy','eu-deathmatch','eu-control','eu-nostalgia'],
'eu_mini': ['eu-mini01','eu-mini02','eu-mini03','eu-mini04','eu-mini05','eu-mini06'],
'eu_gs': ['eu-gs1','eu-gs2','eu-gs3','eu-gs4'],
'eu_blitz': ['eu-chaos','eu-rage1']
pcp = PlayerCountPlotter()
# Plot all the things!
for k,v in to_plot.iteritems():
print "Plotting " + k
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment