Monitor indexing status and log elapsed time and doc count

monitor_indexing.py

import argparse
import time
import requests
import re

parser = argparse.ArgumentParser(description="Monitor and time indexing.")
parser.add_argument('-l', '--hosts', type=str, help='comma seperated list of hosts to connect to.', default='127.0.0.1')
arguments = parser.parse_args()
indexing_status_patt = re.compile(r'<bool name="indexing">([a-z]+)</bool>')
num_docs_patt = re.compile(r'<int name="numDocs">([0-9]*)</int>')


def check_indexing_status(hosts=[]):
    host_status = {}
    num_docs = {}
    for host in hosts:
        resp = requests.get("http://%s:8983/solr/admin/cores?action=STATUS&core=netflows.netflows" % host)
        print resp.text
        m = indexing_status_patt.search(resp.text)
        m1 = num_docs_patt.search(resp.text)
        num_docs[host] = m1.group(1)
        host_status[host] = m.group(1)
    return host_status, num_docs

hosts = arguments.hosts.split(',')

start_time = time.time()
with open("indexing.csv", "w") as f:
    f.write("host,elapsed(sec),num_docs,status\n")
    while True:
        status, num_docs = check_indexing_status(hosts=hosts)
        for host in hosts:
            f.write("%s,%s,%s,%s\n" % (host, str(time.time()-start_time), num_docs[host], status[host]))
        if ['false']*len(hosts) == status.values():
            break
        time.sleep(1)
    latency = time.time() - start_time

    latency = latency / 60.0
    f.write("Latency (min)=%s\n" % str(latency))
    f.flush()