Migrate between elastic env

elastic-migration.py

#
# Only tested on Kibana 7.1.1 management
#

import argparse
import json
import uuid

from elasticsearch import Elasticsearch

OPEN_DISTRO_INDEX = ".opendistro-alerting-config"
KIBANA_INDEX = ".kibana_1"

def retrieve_documents(esClient, index):
    return esClient.search(index=index, size=1000, body={"query": {"match_all": {}}})["hits"]["hits"]

def backup(documents, name):
    with open(f"{name}.json", 'w') as f:
        json.dump(documents, f)

def retrieve_docs_and_backup(esClient, batchId, host):
    openDistroDocuments = retrieve_documents(esClient, OPEN_DISTRO_INDEX)
    backup(openDistroDocuments, f"{batchId}-{host}-{OPEN_DISTRO_INDEX}")
    kibanaDocuments = retrieve_documents(esClient, KIBANA_INDEX)
    backup(kibanaDocuments, f"{batchId}-{host}-{KIBANA_INDEX}")
    return openDistroDocuments, kibanaDocuments

def strip_url(url):
    return url.lower().replace("http://", "").replace("https://", "").replace(":", "-").replace("/", "-")

def remove(esClient, documents):
    for document in documents:
        print(f"Removing {document['_id']}")
        esClient.delete(index=document["_index"],doc_type=document["_type"],id=document['_id'])

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-fh', '--fromhost', type=str, required=True, help="From what host to migrate too")
    parser.add_argument('-ths', '--tohosts', type=str, required=True, help="List of hosts to migrate too")
    parser.add_argument('-rdbs', '--removedocsbeforesync', action='store_true', help="If fullsync, remove old objects no present in fromhost")
    parser.add_argument('-v', '--verbose', action='store_true', help="Versbose or not")
    args = parser.parse_args()

    batchId = uuid.uuid4()

    fromESClient = Elasticsearch(args.fromhost)
    openDistroDocuments, kibanaDocuments = retrieve_docs_and_backup(fromESClient, batchId, strip_url(args.fromhost))

    toHosts = args.tohosts.split(",")
    for toHost in toHosts:
        toESClient = Elasticsearch(toHost)
        toOpenDistroDocuments, toKibanaDocuments = retrieve_docs_and_backup(toESClient, batchId, strip_url(toHost))
        if args.removedocsbeforesync:
            remove(toESClient, toOpenDistroDocuments)
            remove(toESClient, toKibanaDocuments)

        for documents in [openDistroDocuments, kibanaDocuments]:
            for document in documents:
                print(f"Add/Update {document['_id']}")
                toESClient.index(index=document["_index"],doc_type=document["_type"],id=document["_id"], body=document["_source"])

if __name__ == '__main__':
	main()