Script to create a diagnostic for Elasticsearch running with ECK

eck-elasticsearch-diagnostics.sh

#!/bin/bash

show_help()
{
    SCRIPT_NAME=$(basename $0)
    echo "Generate a diagnostics dump of Elasticsearch cluster running on ECK."
    echo
    echo "Syntax: $SCRIPT_NAME -n <K8S_NAMESPACE> -c <ES_CLUSTER> -s <NODESET> [-h|-d]"
    echo "options:"
    echo "-c | --cluster        Specify the Elasticsearch cluster"
    echo "-h | --help           Print command usage"
    echo "-n | --namespace      Specify the kubernetes namespace"
    echo "-s | --nodeset        Specify the cluster nodeSet"
    echo "-o | --output_dir     Base dir to store the diagnostics archive"
    echo "-d | --debug          Enable debug mode"
    echo
}

ES_DIAG_CLUSTER=''
ES_DIAG_K8S_NS=''
ES_DIAG_NODESET_NAME=''
DEBUG=0
ES_DIAG_VALID_CLUSTER=0 
ES_DIAG_DIR=~/.eck-remote-diagnostics/elasticsearch

while true; do
  # echo "\$1:\"$1\" \$2:\"$2\""
  case "$1" in
    -h | --help ) show_help; exit; ;;
    -n | --namespace ) ES_DIAG_K8S_NS="$2"; shift 2 ;;
    -c | --cluster ) ES_DIAG_CLUSTER="$2"; shift 2 ;;
    -s | --nodeset ) ES_DIAG_NODESET_NAME="$2"; shift 2 ;;
    -o | --output_dir ) ES_DIAG_DIR="$2"; shift 2 ;;
    -d | --debug ) DEBUG="1"; shift ;;
    -- ) shift; break ;;
    * ) break ;;
  esac
done

if [ "$DEBUG" == "0" ]; then
  unset DEBUG
fi

echo ""

if [ ! -z "$DEBUG" ]; then
    echo "---------------------------------- DEBUG INFO -----------------------------------"
    echo "ES_DIAG_K8S_NS: $ES_DIAG_K8S_NS"
    echo "ES_DIAG_CLUSTER: $ES_DIAG_CLUSTER"
    echo "ES_DIAG_NODESET_NAME: $ES_DIAG_NODESET_NAME"
    echo "ES_DIAG_DIR: $ES_DIAG_DIR"
    echo "DEBUG: $DEBUG"
    echo ""
fi

if [ -z "$ES_DIAG_CLUSTER" ] || [ -z "$ES_DIAG_K8S_NS" ] || [ -z "$ES_DIAG_NODESET_NAME" ]; then
    echo "ERROR: Must specify the namespace, cluster name and nodeset!"
    show_help
    exit 1
fi

if [ -z "$ES_DIAG_DIR" ] || [ "$ES_DIAG_DIR" == "" ]; then
    echo "ERROR: Must have a valid local output directory for diagnostics!"
    show_help
    exit 1
fi

# Clusters can be found by running: kubectl get elasticsearch -n <NAMESPACE>
ES_DIAG_VALID_CLUSTERS=$(kubectl get elasticsearch -n $ES_DIAG_K8S_NS  -o go-template='{{range .items}}{{.metadata.name}} {{end}}')
# Nodesets Can be obtained by running: kubectl get statefulset -n <NAMESAPCE> -l common.k8s.elastic.co/type=elasticsearch
ES_VALID_NODESET=$(kubectl get sts -n $ES_DIAG_K8S_NS -l common.k8s.elastic.co/type=elasticsearch -l elasticsearch.k8s.elastic.co/cluster-name=${ES_DIAG_CLUSTER} | sed 1d | awk '{print $1}' | sed "s/${ES_DIAG_CLUSTER}-es-//")

function check_valid_cluster {
    for cluster in $ES_DIAG_VALID_CLUSTERS; do
        if [ "$ES_DIAG_CLUSTER" == "$cluster" ]; then return 0; fi
    done
    return 1
}

function check_valid_nodeset {
    for nodeset in $ES_VALID_NODESET; do
        if [ "$ES_DIAG_NODESET_NAME" == "$nodeset" ]; then return 0; fi
    done
    return 1
}

check_valid_cluster
if [ $? -ne 0 ]; then
    echo -e "\nERROR: Valid elasticsearch clusters are:"
    for C in $ES_DIAG_VALID_CLUSTERS; do
        echo "$C"
    done
    echo ""
    exit 1
fi

check_valid_nodeset
if [ $? -ne 0 ]; then
    echo -e "\nERROR: Valid nodesets for $ES_DIAG_CLUSTER are:"
    echo "$ES_VALID_NODESET"
    echo ""
    exit 1
fi

if [ -z $ELASTIC_USER ]; then
    ELASTIC_USER=elastic
fi

if [ -z $ELASTIC_HOST ]; then
    K8S_NODES=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}')
    K8S_NODE=$(echo $K8S_NODES | awk '{print $1}')
    ELASTIC_PORT=$(kubectl get svc -n $ES_DIAG_K8S_NS --selector="elasticsearch.k8s.elastic.co/cluster-name=$ES_DIAG_CLUSTER" -o go-template='{{range .items}}{{range.spec.ports}}{{if .nodePort}}{{.nodePort}}{{"\n"}}{{end}}{{end}}{{end}}')
    ELASTIC_PASS=$(kubectl get secret $ES_DIAG_CLUSTER-es-elastic-user -n $ES_DIAG_K8S_NS -o jsonpath='{.data.elastic}' | base64 -d)
    ELASTIC_HOST=$K8S_NODE:$ELASTIC_PORT
else

    if [ -z $ELASTIC_USER ] || [ -z $ELASTIC_PASS ]; then
        echo "Must specify user/pass via ELASTIC_USER/ELASTIC_PASS when manually specifying ELASTIC_HOST"
        exit 1
    fi

fi

# -----------------------------------


#ES_DAIG_PACKAGE_VERSION=8.2.2 
#PKG_FILENAME=diagnostics-${ES_DAIG_PACKAGE_VERSION}-dist.zip

ES_DAIG_PACKAGE_VERSION=7.1.5
PKG_FILENAME=support-diagnostics-${ES_DAIG_PACKAGE_VERSION}-dist.zip

ES_DIAG_POD_NAME=$ES_DIAG_CLUSTER-es-$ES_DIAG_NODESET_NAME-0

ES_DIAG_PKG_URL=https://github.com/elastic/support-diagnostics/releases/download/${ES_DAIG_PACKAGE_VERSION}/${PKG_FILENAME}
ES_DIAG_PKG_URI=$(echo $ES_DIAG_PKG_URL | grep -Po '\w\K/\w+[^?]+')
ES_DIAG_PKG_NAME=$(basename $ES_DIAG_PKG_URI)
ES_DIAG_PKG_DIR=$(echo $ES_DIAG_PKG_NAME | sed 's/-dist.zip//' )
ES_DIAG_ES_PASS=$(kubectl get secret $ES_DIAG_CLUSTER-es-elastic-user -n $ES_DIAG_K8S_NS -o jsonpath='{.data.elastic}' | base64 -d)
ES_DIAG_OUTPUT_DIR=/tmp/generated-support-diagnostics


if [ ! -z "$DEBUG" ]; then
    echo "---------------------------------- DEBUG INFO -----------------------------------"
    echo "ES_DIAG_VALID_CLUSTERS: $ES_DIAG_VALID_CLUSTERS"
    echo "ES_VALID_NODESET: $(echo $ES_VALID_NODESET | tr '\n' ' ')"
    echo "ELASTIC_USER: $ELASTIC_USER"
    echo "ELASTIC_HOST: $ELASTIC_HOST"
    echo "ES_DIAG_ES_PASS: *********"
    echo "ES_DIAG_PKG_URL: $ES_DIAG_PKG_URL"
    echo "ES_DIAG_DIR: $ES_DIAG_DIR"
    echo "ES_DIAG_PKG_URI: $ES_DIAG_PKG_URI"
    echo "ES_DIAG_PKG_NAME: $ES_DIAG_PKG_NAME"
    echo "ES_DIAG_PKG_DIR: $ES_DIAG_PKG_DIR"
    echo -e "ES_DIAG_OUTPUT_DIR: $ES_DIAG_OUTPUT_DIR\n"
fi


if [ ! -f ${ES_DIAG_PKG_NAME} ]; then    
    echo "Downloading diags package: $ES_DIAG_PKG_URL"
    wget --quiet ${ES_DIAG_PKG_URL} && unzip -qq ${ES_DIAG_PKG_NAME}
fi

echo "Deleting old left over diagnostics packages in $ES_DIAG_OUTPUT_DIR on Elasticsearch host $ELASTIC_HOST"
kubectl exec -it -n $ES_DIAG_K8S_NS $ES_DIAG_POD_NAME -- /bin/bash -c "rm -rf $ES_DIAG_OUTPUT_DIR"


echo "Uploading diagnostics scripts to pod $ES_DIAG_POD_NAME"
kubectl cp -n $ES_DIAG_K8S_NS  ${ES_DIAG_PKG_DIR} $ES_DIAG_POD_NAME:/tmp

# GET ES VERSION

URL="https://${ELASTIC_HOST}/"
ES_VERSION_MAJOR=$(kubectl exec -it -n $ES_DIAG_K8S_NS $ES_DIAG_POD_NAME -- /bin/bash -c "curl -s -k -H 'Content-Type: application/json' -u \"elastic:$ES_DIAG_ES_PASS\" $URL" | jq -r '.version.number' | cut -d'.' -f1)


if [ "$ES_VERSION_MAJOR" -ge "7" ]; then
    JAVA_HOME=/usr/share/elasticsearch/jdk
else
    JAVA_HOME=/opt/jdk-15.0.1+9/
fi


echo "Executing diagnostic..."
kubectl exec -it -n ${ES_DIAG_K8S_NS} $ES_DIAG_POD_NAME -- /bin/bash -c "cd /tmp/${ES_DIAG_PKG_DIR} && \
    export JAVA_HOME=${JAVA_HOME} && \
    ./diagnostics.sh \
    --host localhost \
    --type remote \
    -u $ELASTIC_USER \
    --ptp '$ES_DIAG_ES_PASS' \
    --ssl \
    --noVerify \
    -o $ES_DIAG_OUTPUT_DIR"

if [ ! -z "$DEBUG" ]; then
    echo "---------------------------------- DEBUG INFO -----------------------------------"
    echo -e "Running: kubectl exec -it -n ${ES_DIAG_K8S_NS} ${ES_DIAG_POD_NAME} -- /bin/bash -c \"ls -A ${ES_DIAG_OUTPUT_DIR} | tr '\n' ' '\"\n"
fi

DIAG_ARCHIVE_NAME=$(kubectl exec -it -n ${ES_DIAG_K8S_NS} ${ES_DIAG_POD_NAME} -- /bin/bash -c "ls -A ${ES_DIAG_OUTPUT_DIR} | tr '\n' ' '")
DIAG_ARCHIVE_PREFIX=$(echo $DIAG_ARCHIVE_NAME | cut -d. -f1)

echo "Copying diagnostics archive to local directory"
DIAG_LOCAL_FILENAME=${ES_DIAG_DIR}/${ES_DIAG_CLUSTER}/${DIAG_ARCHIVE_PREFIX}_${ES_DIAG_CLUSTER}_${ES_DIAG_NODESET_NAME}.tar.gz
DIAG_DIR=$(dirname $DIAG_LOCAL_FILENAME)

if [ ! -z "$DEBUG" ]; then
    echo "---------------------------------- DEBUG INFO -----------------------------------"
    echo "DIAG_ARCHIVE_NAME: $DIAG_ARCHIVE_NAME"
    echo "DIAG_ARCHIVE_PREFIX: $DIAG_ARCHIVE_PREFIX"
    echo "DIAG_LOCAL_FILENAME: $DIAG_LOCAL_FILENAME"
    echo -e "DIAG_DIR: $DIAG_DIR\n"
fi

if [ ! -d "$DIAG_DIR" ]; then
    echo "Creating diagnostics directory" && mkdir $DIAG_DIR
fi

if [ ! -z "$DEBUG" ]; then
    echo "---------------------------------- DEBUG INFO -----------------------------------"
    echo -e "Running: kubectl cp -n ${ES_DIAG_K8S_NS} $ES_DIAG_POD_NAME:/tmp/generated-support-diagnostics/${DIAG_ARCHIVE_NAME} ${DIAG_DIR}\n"
fi

kubectl cp -n ${ES_DIAG_K8S_NS} $ES_DIAG_POD_NAME:/tmp/generated-support-diagnostics/${DIAG_ARCHIVE_NAME} ${DIAG_LOCAL_FILENAME}

echo "Diagnostics archive copied to ${DIAG_LOCAL_FILENAME}"
echo ""