elastic restore script

elastic_restore.py

import elasticsearch  # https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.html
import curator  # http://curator.readthedocs.io/en/v3.4.1/examples.html
import sys

PROD_PREFIX = "prod_"
REPOSITORY = "Your repository here
HOST_LIST = [{"host": "Friendly-hostname-here", "port": 9200}]


def main():
    print 'Starting elastic restore job using hosts: ', HOST_LIST
    client = elasticsearch.Elasticsearch(hosts=HOST_LIST)
    prod_indices = get_prod_indices(client)
    print 'Indices to close: ', prod_indices
    close_successful = curator.close_indices(client, prod_indices)

    if not close_successful and prod_indices:
        print sys.stderr.write('Error in closing indices \n')
        return 1

    latest_snapshot = get_most_recent_snapshot(client)
    snapshot_indices = get_snapshot_indices(client, latest_snapshot)
    print 'Running snapshot for: ', latest_snapshot
    restore_successful = recover_snapshot(client, latest_snapshot)

    if not restore_successful:
        print sys.stderr.write('Restore request failed \n')
        return 1

    indices_to_be_deleted = get_indices_not_in_snapshot(prod_indices, snapshot_indices)
    print 'Deleting these indices: ', indices_to_be_deleted
    delete_successful = curator.delete_indices(client, indices_to_be_deleted)

    if not delete_successful and indices_to_be_deleted:
        print sys.stderr.write('Error in deleting indices')
        return 1
    return 0


def get_prod_indices(client):
    indices = curator.get_indices(client)
    prefix_filter = curator.build_filter(kindOf="prefix", value=PROD_PREFIX)
    return curator.apply_filter(items=indices, **prefix_filter)


def get_most_recent_snapshot(client):
    snapshots = curator.get_snapshots(client, REPOSITORY)
    return sorted(snapshots)[-1]


def get_snapshot_indices(client, latest_snapshot):
    snap_details = curator.get_snapshot(client, REPOSITORY, latest_snapshot)
    return snap_details['snapshots'][0]['indices']


def recover_snapshot(client, snapshot):
    payload = {
        "ignore_unavailable": True,
        "rename_pattern": "(.+)",
        "rename_replacement": "prod_$1",
        "include_aliases": False,
        "index_settings": {
            "index.number_of_replicas": 1
        }
    }
    # This will hang until the restore process is complete
    params = {'request_timeout': 86400}
    return client.snapshot.restore(REPOSITORY, snapshot, payload, wait_for_completion=True, master_timeout="1d", params=params)


def get_indices_not_in_snapshot(prod_indices, snapshot_indices):
    prefixed_snapshot_indices = ["prod_"+index for index in snapshot_indices]
    return list(set(prod_indices).difference(prefixed_snapshot_indices))


if __name__ == '__main__':
    exit_code = main()
    sys.exit(exit_code)