insert_new_elasticsearch.py

# This script assumes you have a directory called elasticsearch with your pickled data in files elasticsearch_X.pickle

import pickle
from elasticsearch import Elasticsearch

### CHANGE THESE CONSTANTS TO MATCH THE ELASTICSEARCH CLUSTER YOU'RE MIGRATING TO ###
ES_HOST = "my-aws-hosted-elasticsearch.us-east-1.es.amazonaws.com"
ES_PORT = 80
ES_INDEX = "my_index"
ES_DOC_TYPE = "feed"
# This is the number of pickled files you generated in the save_elastic_data.py script.
PAGES = 2000

es = Elasticsearch([{'host': ES_HOST, 'port': ES_PORT}])

def es_insert(data=None):
    """
    Insert a new feed into elastic search.

    :param data: the data to create the feed with
    :return: None
    """
    try:
        response = es.index(index=ES_INDEX, doc_type=ES_DOC_TYPE, body=data)
    except Exception as ex:
        template = "An exception of type {0} occurred. Arguments:\n{1!r}"
        message = template.format(type(ex).__name__, ex.args)
        error = {}
        error['created'] = False
        error['message'] = message
        return error
    return response

for counter in range(0,PAGES):
    print("Processing page number " + str(counter))
    input = open("elasticsearch/elasticsearch_" + str(counter) + ".pickle", "r")
    hits = pickle.load(input)
    input.close()
    for hit in hits:
        response = es_insert(hit['_source'])
        if response['result'] != 'created':
            print("Failed to migrate record: " + str(response))