grams.py

width = 4
top_count = 256

total = 0
freqs = {}
for line in open("/usr/share/dict/words").readlines():
  line = line.rstrip('\n')
  if line.find('\'') >= 0:
    continue
  total += len(line)
  i = len(line)
  while (i >= width):
    sub = line[i-width:i]
    if freqs.has_key(sub):
      freqs[sub] = freqs[sub] + 1
    else:
      freqs[sub] = 1
    i -= width

top = sorted(freqs.items(), key=lambda x: -x[1])
print total
top = top[0:top_count]
sub = 0
for pair in top:
  sub += pair[1]
print sub
print top