Analytics

analytics.py

from multiprocessing import Pool
from collections import Counter
from operator import add, itemgetter

import os, sys
import csv

def substrings(s):
    n = len(s)
    for i in xrange(0, n):
        for j in xrange(i+1, n+1):
            yield s[i:j]


def count(string):
    counts = {}

    for substring in substrings(string):
        if len(substring) < 10:
            key = ','.join(substring)
            if key in counts:
                counts[key] += 1
            else:
                counts[key] = 1

    return counts


def read_csv(fin):
    for row in csv.reader(fin, delimiter=',', quotechar='"'):
        yield row[5].split()

if __name__ == '__main__':

    pool = Pool(8)
    counts = pool.map(count, read_csv(sys.stdin))

    counts = reduce(add, map(Counter, counts))

    counts = sorted(counts.iteritems(), key=itemgetter(1))

    for c in counts:
        print '{} :: {}'.format(c[1], c[0])