Get stats for overall health of a mailing list over time

mailling_list_troll.py

#! /bin/python
import urllib2
import csv
import datetime
import math


class GmaneListStats(object):

    def __init__(self, what_is_recent=30, group='gmane.comp.web.zope.plone.devel'):
        """
        @what_is_recent: how many days to keep in recent tally
        """
        self.data = {}
        # last what_is_recent days activity
        self.days = 0
        self.recent = what_is_recent

        stats_url = 'http://gmane.org/output-rate.php?group=%s' % group

        fp = urllib2.urlopen(stats_url)
        stats = fp.read()
        fp.close()

        for row in csv.reader(stats.split("\n")[1:], delimiter=' '):
            if len(row) < 3:
                continue

            date = row[0]
            messages = int(row[1])
            self.addDate(datetime.datetime.strptime(date, '%Y%m%d'), messages)

    def addDate(self, date, count):
        """
        Add a date to the collection of data. date is
        expected to be a datetime argument. this is un-granularizing
        the data a bit
        """
        if date.year not in self.data:
            self.data[date.year] = [0,0,0,0,0,0,0,0,0,0,0,0]
        year = self.data[date.year]
        year[date.month-1] += count

        today = datetime.datetime.now()
        if (today - date).days <= self.recent:
            self.days += count

    def getTotal(self):
        total = 0
        for year, months in self.data.items():
            total += sum(months)
        return total

    def getYearlyAverage(self):
        return self.getTotal()/len(self.data.keys())

    def getYearlySummary(self):
        return [ (year, self.getMonthlyAverage(year)) for year in sorted(self.data.keys()) ]

    def getMonthlyAverage(self, year=None):
        if not year:
            year = datetime.datetime.now().year

        year_total = sum(self.data[year])

        num_months = 12.0
        # skip months that haven't occured yet
        today = datetime.datetime.now()
        if today.year == year:
            num_months = (today.month - 1.0) + float(today.day)/30.0  # approx
        return math.ceil(float(year_total)/num_months)

    def getCurrentHealth(self):
        return self.days/self.recent

    def getMonthlyStats(self, months=4):
        today = datetime.datetime.now()
        total = 0
        num = 0
        years = sorted(self.data.keys(), reverse=True)
        for year in years:
            # go through months in reverse
            for month in reversed(self.data[year]):
                if month == 0: # hasn't occured yet
                    continue
                total += month
                num += 1
                if num == months:
                    # crunch
                    average = float(total)/float(months)
                    return {'average': average,
                            'stddev': 0,
                           }

    def isTrollRequired(self, months=4):
        """
        A troll is required if the activity in the last troll period months
        In theory this should be a standard deviation but for now I'll go with 30%
        """
        troll_level = self.getMonthlyStats(months)['average']/30.0 * .7
        return self.getCurrentHealth() < troll_level


def report_out(group='gmane.comp.web.zope.plone.devel'):
    stats = GmaneListStats(group=group)
    print "+++++++++++++++++"
    print group
    print "+++++++++++++++++"
    print "Average Yearly: ", stats.getYearlyAverage()
    print "All Years"
    for (year, num) in stats.getYearlySummary():
        print "%s\t%s" % (year, num)
    print "2012 Monthly Average: ", stats.getMonthlyAverage()
    print "Current Health: ", stats.getCurrentHealth()
    print "Stats", stats.getMonthlyStats()
    print "Troll Required?", stats.isTrollRequired() and "YES!" or "All clear"



if __name__ == '__main__':
    report_out(group='gmane.comp.web.zope.plone.user')
    report_out(group='gmane.comp.web.zope.plone.devel')

cheeky!