Script to setup a single node elastic search and configure both a local (folder) and remote (AWS S3) repository intended to test/restore snapshots

es-local-restore

#!/bin/bash
set -o errexit
set -o pipefail

export AWS_BUCKET="my-bucket-name"
export AWS_REGION="us-east-1"
export AWS_ACCESS_KEY_ID="my-access-key-id"
export AWS_ACCESS_KEY_SECRET="my-access-key-secret"
export AWS_REGION_ENDPOINT="https://s3.$AWS_REGION.amazonaws.com"
export ES_DOCKER_VERSION="elasticsearch-oss:6.3.2"
export ES_LOCAL_REPO_DESTINATION="/es-repo"
export ES_API_PORT=${ES_API_PORT-9200}
export ES_REMOTE_REPO_NAME="my-remote-repo"

main() {
  set_env_vars
  run_local_es
}

run_local_es() {
  #Startup intermediate ES locally
  export es_pid=$(docker run -d \
    -p "${ES_API_PORT}:9200" \
    -p "${ES_TRANSPORT_PORT}:9300" \
    -v "${LOCAL_VOLUME_DESTINATION}:/repo" \
    -e "discovery.type=single-node" \
    -e "path.repo=/repo" \
    docker.elastic.co/elasticsearch/$ES_DOCKER_VERSION)

  echo "Elasticsearch started as container id $es_pid"

  wait_for_es

  echo "Installing s3 repository support"
  docker exec $es_pid bash -c "yes | bin/elasticsearch-plugin install repository-s3"
  echo "Changing /repo permissions"
  docker exec --privileged -u 0 $es_pid chown -R elasticsearch:0 /repo

  echo "Configuring secret keys"
  docker exec $es_pid bash -c "echo ${AWS_ACCESS_KEY_ID} | bin/elasticsearch-keystore add --stdin s3.client.default.access_key"
  docker exec $es_pid bash -c "echo ${AWS_SECRET_ACCESS_KEY} | bin/elasticsearch-keystore add --stdin s3.client.default.secret_key"

  #ES needs to restart after plugin installation
  docker restart $es_pid
  wait_for_es

  #Setup 2 repositories on the local ES single node cluster
  #one called remote_backup for fetching the snapshots from an S3 remote repository
  #one called local_backup for creating a new snapshot in $LOCAL_VOLUME_DESTINATION on localhost
  setup_repositories

  #backup_to_local
  #restore_all_indexes_from_remote ${SNAPSHOT_NAME}
  #restore_index_from_remote ${SNAPSHOT_NAME} ${INDEX_NAME}
}

backup_to_local() {
  curl -XPUT --fail "localhost:${ES_API_PORT}/_snapshot/local_backup/backup-es?wait_for_completion=true"
}

restore_all_indexes_from_remote() {
  snapshot="${1}"
  echo "Restoring from 'localhost:${ES_API_PORT}/_snapshot/remote_backup/${snapshot}'"
  curl -XPOST --fail "localhost:${ES_API_PORT}/_snapshot/remote_backup/${snapshot}/_restore?wait_for_completion=true"
}

restore_index_from_remote() {
  snapshot="${1}"
  index="${2}"
  echo "Restoring Index '${index}' from Snapshot '${snapshot}'"
  set -x
  curl -X POST "localhost:${ES_API_PORT}/_snapshot/remote_backup/${1}/_restore" -H 'Content-Type: application/json' -d'
  {
    "indices": "'${index}'",
    "ignore_unavailable": true,
    "include_global_state": true
  }'
}

setup_repositories() {
  curl --silent "localhost:${ES_API_PORT}/_nodes/reload_secure_settings"

  echo "Creating local repository at http://localhost:${ES_API_PORT}/_snapshot/local_backup"
  curl --fail -XPUT "http://localhost:${ES_API_PORT}/_snapshot/local_backup" \
    -H 'Content-Type: application/json' \
    -d '{
      "type": "fs",
      "settings": {
          "location": "/repo/local_backup",
          "compress": true
      }
    }'

  echo -e "\nCreating remote repository at http://localhost:${ES_API_PORT}/_snapshot/remote_backup"
  curl --fail -XPUT "http://localhost:${ES_API_PORT}/_snapshot/remote_backup" \
    -H 'Content-Type: application/json' \
    -d "{
      \"type\": \"s3\",
      \"settings\": {
        \"bucket\": \"$AWS_BUCKET\",
        \"compress\": \"true\",
        \"base_path\": \"$ES_REMOTE_REPO_NAME\",
        \"region\": \"$AWS_REGION\",
        \"endpoint\": \"$AWS_REGION_ENDPOINT\",
        \"readonly\": \"true\"
      }
    }"
}

wait_for_es() {
  until curl --silent --output /dev/null --fail localhost:${ES_API_PORT}; do
    echo "Waiting for ES"
    sleep 1
  done
}

main "$@"