grace_report.py

#!/usr/bin/env python
import re
import tabulate

from elasticsearch import Elasticsearch

groups = [
    {'name':'GRACC (osg)', 'regex': r'gracc\.osg\.'},
    {'name':'GRACC (osg) [OLD]', 'regex': r'gracc-osg-[12]'},
    {'name':'GRACC (osg-transfer)', 'regex': r'gracc\.osg-transfer\.'},
    {'name':'GRACC (osg-itb)', 'regex': r'gracc\.osg-itb\.'},
    {'name':'OSG history', 'regex': r'osg-'},
    {'name':'LIGO history', 'regex': r'ligo-'},
    {'name':'Gratia Summary', 'regex': r'gratia-osg-summary'},
    {'name':'GRACC Monitor', 'regex': r'gracc-monitor'},
    {'name':'FIFE', 'regex':r'fife|logstash-fife'},
    {'name':'Marvel', 'regex': r'\.marvel'},
    {'name':'Kibana', 'regex': r'\.kibana'},    
]


class Table(object):
    def __init__(self):
        self.tab = []
        
    def add(self, name, num, size, docs):
        doc_size = 0.0
        if docs > 0:
            doc_size = float(size)/docs
        self.tab.append([name,num,bytes(size,"GB"),docs,bytes(doc_size,"KB")])
            
    def tabulate(self, fmt="simple"):
        headers=["Index Group","# Indices","Size GB","# Docs","Doc Size KB"]
        return tabulate.tabulate(self.tab, headers, tablefmt=fmt)

_bytes = {
    "b": 1.0/8,
    "B": 1,
    "KB": 1024,
    "MB": 1024*1024,
    "GB": 1024*1024*1024,
    "TB": 1024*1024*1024*1024,
}

def bytes(v,unit):
    return v/_bytes.get(unit,1)


def main():
    client = Elasticsearch()
    r=client.indices.stats(metric='docs,store')
    idxs=r['indices'].keys()
    tab = Table()
    for group in groups:
        try:
            g = group['name']
            gre = group['regex']
        except KeyError:
            continue

        gidxs = filter(lambda i: re.match(gre,i),idxs)
        size = 0
        docs = 0
        for i in gidxs:
            size += r['indices'][i]['total']['store']['size_in_bytes']
            docs += r['indices'][i]['primaries']['docs']['count']
            idxs.remove(i)
        tab.add(g,len(gidxs),size,docs)

    # ungrouped
    size = 0
    docs = 0
    for i in idxs:
        size += r['indices'][i]['total']['store']['size_in_bytes']
        docs += r['indices'][i]['primaries']['docs']['count']
    tab.add('other',len(idxs),size,docs)

    # total
    size = r['_all']['total']['store']['size_in_bytes']
    docs = r['_all']['primaries']['docs']['count']
    tab.add('**total**',len(r['indices']),size,docs)

    print(tab.tabulate())
    print('\nNOTE: Sizes include replicas.')
    print('\n\nOther Indices\n'+('-'*80))
    for i in idxs:
        print(i)


if __name__=='__main__':
    main()