counts.py

import re
import csv
import sys

from elasticsearch import Elasticsearch
from elasticsearch.helpers import scan
from collections import Counter

es = Elasticsearch("localhost:9201")
index = "amcat_vu"
setid = 1021
query = {"query": {"term": {"sets": setid}}}

freqs = Counter()
for i, result in enumerate(scan(es, query, index=index)):
    if not i% 1000: print(i, file=sys.stderr)
    doc = result['_source']
    for t in doc['title'], doc['text']:
        freqs.update(re.findall("\w+", t.lower()))

o = csv.writer(sys.stdout)
o.writerow(["n", "word"])
for w, i in freqs.most_common():
    o.writerow([i, w])