Shell script interface for Elasticsearch Dump

dump-query-template.env

# Pass headers into Elasticsearch:
# - Authorization can be used to pass BASIC AUTH with a TOKEN. Use
#   a tool of your choice to convert your username/password into
#   the proper token for basic auth.
ED_HEADERS='{"Authorization": "Basic YOUR-BASIC-AUTH-TOKEN-HERE"}'
# Host must end in "/"
# Include ":port-number" if necessary (otherwise, 443 is inferred by https)
ED_HOST="https://your-host.com:port-number/"

dump-query.sh

#!/bin/bash

############################################################################
# This is a frontend to the elasticdump utility.
# Syntax:
# ./dump-query.sh "INDEX-NAME-OR-PATTERN" /path/to/query.json /path/to/result.json
# Example:
# ./dump-query.sh "logstash-*" query-received-message-times.json result-week2.json
# 
# Dependencies:
# - Create an environment file with your host and authorization header.
#   For the structure of the file, copy dump-query-template.env into dump-query.env.
# - Requires Elastic Dump (an NPM package).
#   https://github.com/elasticsearch-dump/elasticsearch-dump#readme
#   Install globally on your workstation with `npm i elasticdump -g`
# 
# Copyright 2022 Eric A. Soto, eric@issfl.com
# 
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the 
# Software is furnished to do so, subject to the following conditions:
# - The above copyright notice and this permission notice shall be 
#   included in all copies or substantial portions of the Software.
# - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
#   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 
#   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 
#   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
#   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
############################################################################

# Command Line Arguments
INDEX="$1"
INQUERY="$2"
OUTFILE="$3"

# Verify we received all the stuff
if [[ -z "${INDEX}" ]] || [[ -z "${INQUERY}" ]] || [[ -z "${OUTFILE}" ]]; then
  echo "Missing parameter."
  echo "Syntax: ./dump-query.sh \"INDEX-NAME-OR-PATTERN\" /path/to/query.json /path/to/result.json"
  exit 1
fi

# Load environment file
set -o allexport
[[ -f dump-query.env ]] && source dump-query.env
set +o allexport

# Constants
# From Environment: 
# - ED_HEADERS
# - ED_HOST
# Batch Limit (will pull in batches of this amount). Edit to suit!
ED_LIMIT=5000

echo "You are about to DUMP records from Elasticsearch."
echo "Host: $ED_HOST"
echo "Index: $INDEX"
echo "Query File: $INQUERY"
echo "Output File: $OUTFILE"
echo ""
# Wait for the user to press any KEY to proceed or allow them to Ctrl+C
read -n1 -rsp $'Press any key to continue or Ctrl+C to exit...\n'

# Do it!
elasticdump \
  --headers "${ED_HEADERS}" \
  --input="${ED_HOST}${INDEX}" \
  --output="${OUTFILE}" \
  --searchBody=@"${INQUERY}" \
  --limit "$ED_LIMIT" \
  --concurrency 3