ElasticSearch-Python usable queries

create_kibana_from_file.sh

#!/bin/sh

count="$(cat $1 | jq '.hits.hits | length')"
es_host=$2

i=0
while [ "$i" -lt $count ]; do
    body=$(cat $1 | jq -r --arg v "${i}" '.hits.hits[$v|tonumber]._source');
    type=$(cat $1 | jq -r --arg v "${i}" '.hits.hits[$v|tonumber]._type');
    id=$(cat $1 | jq -r --arg v "${i}" '.hits.hits[$v|tonumber]._id');
    curl -XPUT "$es_host/.kibana/$type/$id" -d "$body"
    i=$(( i + 1 ))
done

elasticsearch_curls.sh

#Empty an Index
curl -XDELETE 'http://localhost:9200/content_store/_query' -d '
{
  "query": {
    "bool": {
      "must": [
        {
          "match_all": {}
        }
      ]
    }
  }
}
'

queries.py

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import TransportError

client = None

# Client Constructor
def es():
  global client
    client = Elasticsearch('127.0.0.1:9200',
                         sniff_on_start=True,
                         max_retries=100,
                         retry_on_timeout=True,
                         sniff_on_connection_fail=True,
                         sniffer_timeout=1)
                         
                         
# Check if index exists
def index_exists(indexName):
  return client.indices.exists(indexName)
  

reindex.py

#!/usr/bin/python

import elasticsearch
import elasticsearch.helpers
import os
from optparse import OptionParser

def main():
    parser = OptionParser()
    parser.add_option("-e", "--esHost1", dest="esHost1",
                      help="ES host for source index - Default value [http://localhost:9200]", type='string', default="http://localhost:9200")
    parser.add_option("-d", "--esHost2", dest="esHost2",
                      help="ES host for target index - Default value [http://localhost:9200]", type='string', default="http://localhost:9200")
    parser.add_option("-s", "--source", dest="source",
                      help="Source index - Default value [index_source]", type='string', default="index_source")
    parser.add_option("-t", "--target", dest="target",
                      help="Target-destination index - Default value [index_target]", type='string', default="index_target")
    parser.add_option("-b", "--bulk", dest="bulk",
                      help="Bulk size - Default value [50]", type='int', default=50)
    (options, args) = parser.parse_args()


    esSrc = elasticsearch.Elasticsearch([options.esHost1],timeout=120)
    esTar = elasticsearch.Elasticsearch([options.esHost2],timeout=120)
    print("You are about to re-index contents of [" + options.source +"] to [" + options.target + "]")
    raw_input('"Press any key to continue or Ctrl+c to cancel..."')
    #esTar.indices.delete(index=options.target, ignore=[400, 404])
    elasticsearch.helpers.reindex(client=esSrc, source_index=options.source, target_index=options.target, target_client=esTar, chunk_size=options.bulk)
    #esSrc.indices.delete(index=options.source, ignore=[400, 404])


if __name__ == '__main__':
    main()

threshold_cache_cleaner.py

#!/usr/bin/python

from optparse import OptionParser
from elasticsearch import Elasticsearch
from elasticsearch import client
from elasticsearch.exceptions import TransportError
from retrying import retry


es_host = ''
threshold = ''

def retry_if_transport_error(exception):
    """Return True if we should retry (in this case when it's an IOError), False otherwise"""
    print "\n\n\n retrying!!!!!"
    return isinstance(exception, TransportError)

# Connect to ES
#TransportError
@retry(stop_max_attempt_number=4, retry_on_exception=retry_if_transport_error)
def es():
    #print [es_host]
    return Elasticsearch([es_host],
                         sniff_on_start=True,
                         max_retries=100,
                         retry_on_timeout=True,
                         sniff_on_connection_fail=True,
                         sniff_timeout=100)

# Get all indexes from cluster and store them in a list
def passed_threshold(threshold):
    cluster_client = client.ClusterClient(es())
    res = cluster_client.stats(human ="true")
    max_heap = float(res['nodes']['jvm']['mem']['heap_max_in_bytes'])
    used_heap = float(res['nodes']['jvm']['mem']['heap_used_in_bytes'])

    print "Max  heap: " + str(max_heap)
    print "Used heap: " + str(used_heap)

    used_percent = round(used_heap/max_heap*100, 2)
    print str(used_percent)
    print "Use Percentage is " + str(used_percent) + " while threshold is: " + str(threshold)

    if used_percent> float(threshold):
        print 'Used percentage exceeds threshold'
        return True
    else:
        print 'Used percentage doesnt exceed threshold, program will exit'
        return False

def clear_cache():
    print 'clearing cache!'
    indices_client = client.IndicesClient(es())
    res = indices_client.clear_cache()
    print 'cache cleared!'
    print res


def main():
    global es_host
    parser = OptionParser()
    parser.add_option("-t", "--threshold", dest="threshold",
                      help="heap threshold for cleaning cache as percent - Default value:80", type='string', default="80")
    parser.add_option("-e", "--eshost", dest="eshost",
                      help="elasticsearch instance location", type='string', default="localhost")
    (options, args) = parser.parse_args()

    es_host = options.eshost

    if passed_threshold(options.threshold):
        clear_cache()




if __name__ == '__main__':
     main()