Overcast Network (oc.tc) server player count data collector.

graph.py

import random
import math
import datetime

from pymongo import MongoClient
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from matplotlib.ticker import MultipleLocator
import numpy as np


class PlayerCountPlotter(object):
    """ Used to create plots of Overcast Network (OCN) player counts """

    def __init__(self,db="ocn_player_count",collection="server_stats",regions=['us','eu']):
        self.REGIONS = regions
        self.db = db
        self.collection = collection
        self.data = self.prep_data()

    def prep_data(self):
        """Get all the data in database and return in organized dictionary

        Return format:

        {region: {server: [(time,count),...], ...},...},...}

        where region is "us" or "eu", server is e.g. "alpha",
        time is a timedate object and count is the server player count
        at that time.

        """
        # get data from database
        c = MongoClient()
        db = c[self.db]
        stats = db[self.collection]

        # build data structure
        player_counts = {}
        for region in self.REGIONS:
            player_counts[region] = {}

        for s in stats.find():
            time = s['time']

            totals = {}
            for region in self.REGIONS:
    	        totals[region] = [] #region total player count


            for network in s['stats']: # e.g. "us-project-ares" or "eu-blitz"
    	        region = network[:2] # e.g. eu or us
    	        network_stats = s['stats'][network]

    	        for server in network_stats: # e.g. "alpha"

                    # add (time,count) tuple for server
    	            if server in player_counts[region]:
    	                player_counts[region][server].append((time,network_stats[server]))
    	            else:
                        player_counts[region][server] = [(time,network_stats[server])]

                totals[region].append(sum(network_stats.values()))

            # sum all servers for each region to get region total
            for region in self.REGIONS:
    	        if 'total' in player_counts[region]:
                    player_counts[region]['total'].append((time,sum(totals[region])))
    	        else:
                    player_counts[region]['total'] = [(time,sum(totals[region]))]

        return player_counts


    def get_server_plot_data(self,region_server):
        """Get data and return in plotable format.
        Takes full name of server, e.g. "us-alpha"

        Returns tuple of lists of time and count
        i.e. ([time1,time2,...],[count1,count2,...])
        """
        region = region_server.split('-')[0]
        server = region_server.split('-')[1]
        d = self.data
        z = zip(*d[region][server])
        x = list(z[0])
        y = list(z[1])
        return x,y


    def make_good_colors(self,n):
        """ Let's pick nice plot colors as difficulty as possible.
    	n is the number of series that is being plotted
        """
        COLORS = ['#a6cee3','#1f78b4','#b2df8a','#33a02c','#fb9a99','#e31a1c',
    	      '#fdbf6f','#ff7f00','#cab2d6','#6a3d9a','#ffff99','#b15928']
        len_c = len(COLORS)
        stride = int(math.floor(len_c/n)) if n < len_c else 1
        return [c for c in COLORS[::stride]] #phew


    def make_average_plots(self,server_list, include_points=False, 
                           filename="", save_path="images/", show=False,
                           errorbars=False):
        """ Produces hourly averaged 24 hour plots for servers

        server_list: network-server names.
        Ex:
        ["us-alpha", "eu-mini01",...]
    	include_points: plots all data points (in addition to lines)
    	filename: output image filename
    	save_path: where to save images
        show: display the dynamic graph using plot.show()
        errorbars: show error bars

        """
        color = self.make_good_colors(len(server_list))
        fig = plt.figure()
        ax = fig.add_subplot(111)

        nc = 0 # for setting the color
        for server in server_list:
            data = self.get_server_plot_data(server)
    	    x = data[0]
    	    y = data[1]

    	    # get averages. lots of kind of messy data maneuvering
    	    # convert all datetime values to hourly bins, rounding down for minutes < 30
    	    xh = []

    	    for dt in x:
    	        if dt.minute < 30:
    	            xh.append(dt.hour)
    	        else:
                    if dt.hour < 23:
    		        xh.append(dt.hour+1)
                    else:
                        xh.append(0)

    	    # this will store values for averages
    	    d = [[] for i in range(len(xh))]

    	    for i in range(len(xh)):
    	        time = xh[i]
    	        value = y[i]
                d[time].append(value)

    	    avg_y = [np.mean(v) for v in d]
            
    	    # we now plot x vs. y where x is an hour 0-23 and y is the
    	    # average value at each hour
    	    time = [h for h in range(len(xh))]
            ax.plot(time,avg_y,'-o',color=color[nc],label=server)

            if errorbars:
                std_y = [np.std(v) for v in d]
                ax.errorbar(time,avg_y,yerr=std_y)

    	    if include_points:
    	       ax.plot(xh,y,'o',color=color[nc])

    	    if nc == len(color):
    	        nc = 0
            else:
    	        nc += 1

        ax.set_xlabel('hour of the day (EDT)')
        ax.set_ylabel('average player count')
        ax.set_xlim(0,23)
        ax.xaxis.set_major_locator(MultipleLocator(2.0))
        ax.yaxis.grid(b=True, which='major', color='black', alpha='0.1', linestyle='--')

        # make legend a reasonable size
        if len(server_list) >= 5:
            l_size = 8
        else:
            l_size = 10
        leg = ax.legend(prop={'size':l_size},loc=2)
        leg.get_frame().set_alpha(0.5)

        # save image
        if not filename:
            filename = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S")
	plt.title(filename)
        plt.savefig(save_path+filename+'.png')

        if show:
            plt.show()

player_count.py

"""
 A cute little script to gather information about Overcast Network
 server player counts.  It uses python and mongodb, the cutest possible
 technologies that are hip to know right now.

 By bcbwilla, an admirably cute little scripter.
"""

import urllib2
import datetime

from pymongo import MongoClient
from bs4 import BeautifulSoup

# get server play page and scrape the info with beautiful soup.
URL = "http://www.oc.tc/play"

opener = urllib2.build_opener()     
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
page = opener.open(URL)
html = page.read()
page.close()       
soup = BeautifulSoup(html, "html.parser")

tab_panes = soup.find_all("div",{"class":"tab-pane"})

network_stats = {}
ignore = ["main","us", "us-main", "eu", "eu-main"]

for tab_pane in tab_panes:

    server_stats = {}

    category = tab_pane['id']    
    if not (category in ignore or "lobby" in category):
        rows = tab_pane.find_all("div",{"class":"span8"})

        for row in rows:
            h4s = row.find_all("h4")
            for h4 in h4s:
                name = h4.contents[0].strip().lower()
                count = int(h4.contents[1].contents[0].strip())
                server_stats[name] = count

        network_stats[category] = server_stats

data = {'time': datetime.datetime.now(), 'stats': network_stats}

# put info in database
client = MongoClient()
db = client.ocn_player_count
s = db.server_stats
s.insert(data)

run.py

""" Run graph.py to make some graphs """
from graph import PlayerCountPlotter

# a dictionary of various server categories for plotting
to_plot = {
    'network_totals': ['us-total','eu-total'],
    'us_main': ['us-alpha','us-beta','us-capture1','us-capture2','us-destroy','us-deathmatch','us-gear','us-control','us-nostalgia','us-primed'],
    'us_obj': ['us-capture1','us-capture2','us-destroy'],
    'us_killy': ['us-deathmatch','us-gear','us-control'],
    'us_tnt': ['us-nostalgia','us-primed'], 
    'us_mini': ['us-mini01','us-mini02','us-mini03','us-mini04','us-mini05','us-mini06','us-mini07','us-mini08','us-mini09','us-mini10','us-mini11'],
    'us_blitz': ['us-chaos','us-cronus','us-rage1','us-rage2'],
    'us_gs': ['us-gs1','us-gs2','us-gs3','us-gs4','us-gs5','us-gs6'],
    'eu_main': ['eu-alpha','eu-capture','eu-destroy','eu-deathmatch','eu-control','eu-nostalgia'],
    'eu_mini': ['eu-mini01','eu-mini02','eu-mini03','eu-mini04','eu-mini05','eu-mini06'],
    'eu_gs': ['eu-gs1','eu-gs2','eu-gs3','eu-gs4'],
    'eu_blitz': ['eu-chaos','eu-rage1']
    }

pcp = PlayerCountPlotter()

# Plot all the things!
for k,v in to_plot.iteritems():
    print "Plotting " + k
    pcp.make_average_plots(v,filename=k)