Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
#!/bin/bash
# DIR stores the path to the directory where this script is located
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# ETCD stores the command used to run etcd
ETCD="/usr/local/bin/etcd"
# ETCDCTL stores the command used to run etcdctl
ETCDCTL="/usr/local/bin/etcdctl"
# we use this to choose the path to the etcd/etcdctl binaries, but etcd cannot
# be started with this environment variable set, so we must unset it here
unset ETCD_VERSION
# ETCD_CLUSTERING_OPERATION_CREATE is used to signal that we intend on creating
# a new etcd cluster
ETCD_CLUSTERING_OPERATION_CREATE="create"
# ETCD_CLUSTERING_OPERATION_JOIN is used to signal that we intend on joining an
# existing etcd cluster
ETCD_CLUSTERING_OPERATION_JOIN="join"
# ETCD_CLUSTERING_OPERATION will hold either "create" or "join", a value which
# is later used by the start function
ETCD_CLUSTERING_OPERATION=""
# create_new_cluster starts a new etcd cluster with the current task as the
# single member.
# NOTE: in practice this function is only ever called for etcd-0-peer, and this
# only happens when installing the framework.
function create_new_cluster() {
# give some info about the current operation
printf "creating new etcd cluster with ${TASK_NAME} as the single member...\n"
# export ETCD_NAME (the name of the etcd-0-peer)
export ETCD_NAME="infra${POD_INSTANCE_INDEX}"
# export ETCD_INITIAL_CLUSTER (comprised only of etcd-0-peer)
export ETCD_INITIAL_CLUSTER="infra${POD_INSTANCE_INDEX}=https://${TASK_NAME}.${FRAMEWORK_HOST}:${ETCD_LISTEN_PEER_PORT}"
# export ETCD_INITIAL_CLUSTER_STATE (this is a brand new cluster)
export ETCD_INITIAL_CLUSTER_STATE="new"
# start the etcd process
start_etcd
}
# get_clustering_operation detects whether we should create a new cluster or
# join an existing one.
function get_clustering_operation() {
# give some info about the current operation
printf "detecting whether to create a new etcd cluster or join an existing one...\n"
# instances of etcd having indexes greater than 0 (etcd-1-peer,
# etcd-2-peer, ...) are not allowed to create a cluster anew - they
# should always join an existing one
if [[ "${POD_INSTANCE_INDEX}" -gt "0" ]];
then
printf "joining an existing etcd cluster...\n"
ETCD_CLUSTERING_OPERATION=${ETCD_CLUSTERING_OPERATION_JOIN}
return 0
fi
# the current pod instance is etcd-0-peer, and so we need to understand
# whether we should create a new cluster (such as when the framework is
# being installed for the first time) or join an existing one - such as when
# the etcd-0-peer task has failed or has been replaced. in order to do that
# we try to get the list of peers from etcd.${FRAMEWORK_HOST}. if we get
# at least one member, we join the cluster. otherwise, we retry up to a
# maximum of 5 times and after that we create a new cluster.
# ETCD_MEMBER_COUNT_RETRIES is a counter for the number of
# retries of "etcdctl member list" made so far
local ETCD_MEMBER_COUNT_RETRIES=0
# ETCD_MEMBER_COUNT_MAX_RETRIES is the maximum number of retries
# of "etcdctl member list" to make before creating a new cluster
local ETCD_MEMBER_COUNT_MAX_RETRIES=5
# try to populate ETCD_MEMBER_COUNT taking into account the fact that
# "etcdctl member list" may fail for a myriad of reasons (e.g., networking)
# even though a cluster is already present
until (( ${ETCD_MEMBER_COUNT_RETRIES} >= ${ETCD_MEMBER_COUNT_MAX_RETRIES} ));
do
# list etcd members and count the number of lines
ETCD_MEMBER_LIST=$(
${ETCDCTL} \
--cert=etcd-crt.pem \
--key=etcd-key.pem \
--cacert=ca-crt.pem \
--endpoints=https://etcd.${FRAMEWORK_VIP_HOST}:${ETCD_LISTEN_CLIENT_PORT} member list)
# if "etcdctl member list" was successful, we should join the existing
# cluster
if [[ $? == 0 ]];
then
# give some info about the current operation
printf "joining an existing etcd cluster...\n${ETCD_MEMBER_LIST}\n"
# "etcd member list" succeeded, which means that at least one etcd
# peer is active and that we should join an existing cluster
ETCD_CLUSTERING_OPERATION=${ETCD_CLUSTERING_OPERATION_JOIN}
# exit the function
return 0
fi
# give some info about the current operation
printf "failed to list members\...\n"
# increment ETCD_MEMBER_COUNT_RETRIES
ETCD_MEMBER_COUNT_RETRIES=$((ETCD_MEMBER_COUNT_RETRIES+1))
# if ETCD_MEMBER_COUNT_MAX_RETRIES hasn't been reached yet, we should
# sleep until the next retry
if (( ${ETCD_MEMBER_COUNT_RETRIES} < ${ETCD_MEMBER_COUNT_MAX_RETRIES} ));
then
# give some info about the current operation
printf "sleeping...\n"
# wait a litle between the next retry (1s -> 2s -> 4s -> 8s -> 16s)
sleep $((2**${ETCD_MEMBER_COUNT_RETRIES}))
# continue onto the next iteration
continue
fi
# at this point "etcd member list" has failed everytime, which means
# (most probably) that there are no active etcd peers and that we should
# create a new cluster
# give some info about the current operation
printf "failed to list members. creating a new etcd cluster...\n"
# set the current operation as "create"
ETCD_CLUSTERING_OPERATION=${ETCD_CLUSTERING_OPERATION_CREATE}
done
}
# health_check performs a health check on the current peer by checking if it is
# listed as "healthy" in "etcdctl endpoint health".
function health_check()
{
# check the current peer's health using "etcdctl endpoint health"
local ETCD_PEER_IS_HEALTHY
ETCD_PEER_IS_HEALTHY=$(${ETCDCTL} \
--cert=etcd-crt.pem \
--key=etcd-key.pem \
--cacert=ca-crt.pem \
--endpoints=https://etcd-${POD_INSTANCE_INDEX}-peer.${FRAMEWORK_HOST}:${ETCD_LISTEN_CLIENT_PORT} \
endpoint health | grep -c "is healthy")
# return accordingly
if [[ "${ETCD_PEER_IS_HEALTHY}" == "1" ]];
then
# the peer is healthy
exit 0
else
# the peer is not healthy
exit 1
fi
}
# join_existing_cluster attempts to perform an "etcd member add" operation on
# the existing cluster. it then starts etcd.
function join_existing_cluster()
{
# check if a previous data-dir exists, in which case we don"t need to run
# "etcdctl member add" and can just start etcd
if [[ -d "${DIR}/${ETCD_DATA_DIR}/member" ]];
then
# give some info about the current operation
printf "data-dir is not empty. starting etcd...\n"
# start etcd
start_etcd
# exit the function
return 0
fi
# at this point no data-dir exists, so we must run "etcdctl member add"
# before starting etcd
# give some info about the current operation
printf "data-dir is empty. running \"etcdctl member add\"...\n"
# wait for the vip to resolve
wait_for_dns_to_resolve "etcd.${FRAMEWORK_VIP_HOST}"
# add the new member, storing the output
local ETCD_MEMBER_ADD_OUTPUT
ETCD_MEMBER_ADD_OUTPUT=$(${ETCDCTL} \
--cert=etcd-crt.pem \
--key=etcd-key.pem \
--cacert=ca-crt.pem \
--endpoints=https://etcd.${FRAMEWORK_VIP_HOST}:${ETCD_LISTEN_CLIENT_PORT} \
member add \
--peer-urls=https://etcd-${POD_INSTANCE_INDEX}-peer.${FRAMEWORK_HOST}:${ETCD_LISTEN_PEER_PORT} \
infra${POD_INSTANCE_INDEX})
# check whether "etcd member add" was successful
if [[ $? != 0 ]];
then
# the call to "etcd member add" failed, and so we should fail the task
# and wait for it to be relaunched
printf "failed to add member etcd-${POD_INSTANCE_INDEX}-peer to cluster...\n${ETCD_MEMBER_ADD_OUTPUT}\n"
# fail the task
exit 1
else
# grep the output for the relevant lines and export the
# necessary environment variables. this will set the following
# environment variables:
# - ETCD_INITIAL_CLUSTER
# - ETCD_INITIAL_CLUSTER_STATE
# - ETCD_NAME
eval export $(echo "${ETCD_MEMBER_ADD_OUTPUT}" | grep ETCD_)
# start etcd
start_etcd
fi
}
# readines_check performs a readiness check on the current peer (i.e. checks
# whether the framework scheduler can proceed with the next task).
function readiness_check()
{
# grab a list of current cluster members
local ETCD_MEMBER_LIST
ETCD_MEMBER_LIST=$(${ETCDCTL} \
--cert=etcd-crt.pem \
--key=etcd-key.pem \
--cacert=ca-crt.pem \
--endpoints=https://etcd-${POD_INSTANCE_INDEX}-peer.${FRAMEWORK_HOST}:${ETCD_LISTEN_CLIENT_PORT} member list)
# check the status of the "etcdctl member list command"
if [[ $? != 0 ]];
then
# at this point the current peer is not ready
# give some info about the current operation
printf "readiness_check: failed to list the members of the cluster...\n"
# the peer is not ready
exit 1
fi
# count the number of current cluster members
local ETCD_CLUSTER_MEMBER_COUNT
ETCD_CLUSTER_MEMBER_COUNT=$(echo "${ETCD_MEMBER_LIST}" | wc -l)
# count the number of current cluster members that are healthy
local ETCD_HEALTHY_MEMBER_COUNT
ETCD_HEALTHY_MEMBER_COUNT=$(${ETCDCTL} \
--cert=etcd-crt.pem \
--key=etcd-key.pem \
--cacert=ca-crt.pem \
--endpoints=https://etcd-${POD_INSTANCE_INDEX}-peer.${FRAMEWORK_HOST}:${ETCD_LISTEN_CLIENT_PORT} \
endpoint health --cluster | grep -c " is healthy")
# if all existing members are healthy...
if (( ${ETCD_CLUSTER_MEMBER_COUNT} > 0 )) && [[ "${ETCD_HEALTHY_MEMBER_COUNT}" == "${ETCD_CLUSTER_MEMBER_COUNT}" ]];
then
# give some info about the current operation
printf "readiness_check: ok...\n"
# ... the readiness check should pass...
exit 0
else
# give some info about the current operation
printf "readiness_check: failed...\nETCD_CLUSTER_MEMBER_COUNT: %s\nETCD_HEALTHY_MEMBER_COUNT: %s\n" ETCD_CLUSTER_MEMBER_COUNT ETCD_HEALTHY_MEMBER_COUNT
# ... otherwise, it should fail.
exit 1
fi
}
# recover performs recovery steps on the current peer
function recover()
{
# give some info about the current operation
printf "recovering etcd-${POD_INSTANCE_INDEX}-peer...\n"
# wait for the vip to resolve
wait_for_dns_to_resolve "etcd.${FRAMEWORK_VIP_HOST}"
# fetch the list of members in the cluster
local ETCD_MEMBER_LIST
ETCD_MEMBER_LIST=$(
${ETCDCTL} \
--cert=etcd-crt.pem \
--key=etcd-key.pem \
--cacert=ca-crt.pem \
--endpoints=https://etcd.${FRAMEWORK_VIP_HOST}:${ETCD_LISTEN_CLIENT_PORT} member list)
if [[ $? != 0 ]];
then
# the call to "etcd member list" failed, and so we should fail the task
# and wait for it to be relaunched
printf "failed to list the members of the cluster\n"
# exit the task
exit 1
else
# wipe any pre-existing data
rm -rf data-dir/member
# the call to "etcd member list" succeeded, so we should attempt to
# remove any previously existing instance of this peer from the member
# list
# extract the dead peer"s id from the output
local ETCD_DEAD_PEER_ID
ETCD_DEAD_PEER_ID=$(echo "${ETCD_MEMBER_LIST}" | awk -F "," "/etcd-${POD_INSTANCE_INDEX}-peer/ { print \$1 }")
if [[ "${ETCD_DEAD_PEER_ID}" == "" ]];
then
# the current peer is not known by the cluster, so we're good
return 0
else
# give some info about the current operation
printf "removing etcd-${POD_INSTANCE_INDEX}-peer (id ${ETCD_DEAD_PEER_ID})...\n"
# the current peer is already known by the cluster, so we must
# remove it
${ETCDCTL} \
--cert=etcd-crt.pem \
--key=etcd-key.pem \
--cacert=ca-crt.pem \
--endpoints https://etcd.${FRAMEWORK_VIP_HOST}:${ETCD_LISTEN_CLIENT_PORT} member remove ${ETCD_DEAD_PEER_ID}
if [[ $? != 0 ]];
then
# the call to "etcd member remove" failed, and so we should fail the
# task and wait for it to be relaunched
exit 1
fi
fi
fi
}
# start determines whether the current peer should create a new cluster or join
# an existing one.
function start()
{
# wait for the name of the current task to resolve in order to prevent etcd
# from crashing before it reaches the "started" state (which later
# originates cluster id mismatch errors)
wait_for_dns_to_resolve "etcd-${POD_INSTANCE_INDEX}-peer.${FRAMEWORK_HOST}"
# detect whether we need to create a new cluster or join an existing one
get_clustering_operation
# act accordingly
case "${ETCD_CLUSTERING_OPERATION}" in
${ETCD_CLUSTERING_OPERATION_CREATE})
create_new_cluster
;;
${ETCD_CLUSTERING_OPERATION_JOIN})
join_existing_cluster
;;
esac
}
# start_etcd starts etcd, replacing the current script.
function start_etcd()
{
# give some info about the current operation
printf "starting etcd...\n"
# start etcd (replacing the current script)
exec $ETCD \
--name=infra${POD_INSTANCE_INDEX} \
--cert-file=etcd-crt.pem \
--key-file=etcd-key.pem \
--client-cert-auth \
--trusted-ca-file=ca-crt.pem \
--peer-cert-file=etcd-crt.pem \
--peer-key-file=etcd-key.pem \
--peer-trusted-ca-file=ca-crt.pem \
--listen-peer-urls=https://${MESOS_CONTAINER_IP}:${ETCD_LISTEN_PEER_PORT} \
--initial-advertise-peer-urls=https://etcd-${POD_INSTANCE_INDEX}-peer.${FRAMEWORK_HOST}:${ETCD_LISTEN_PEER_PORT} \
--listen-client-urls=https://${MESOS_CONTAINER_IP}:${ETCD_LISTEN_CLIENT_PORT} \
--advertise-client-urls=https://etcd-${POD_INSTANCE_INDEX}-peer.${FRAMEWORK_HOST}:${ETCD_LISTEN_CLIENT_PORT} \
--log-output=stdout \
--quota-backend-bytes=${ETCD_QUOTA_BACKEND_BYTES} \
--election-timeout=${ETCD_ELECTION_TIMEOUT} \
--heartbeat-interval=${ETCD_HEARTBEAT_INTERVAL} \
--enable-v2=false
}
# wait_for_dns_to_resolve waits for the name passed as the first parameter to
# resolve before returning. if the name doesn't resolve after 5 retries with an
# exponential backoff, it causes the script to exit.
function wait_for_dns_to_resolve()
{
# give some info about the current operation
printf "waiting for \"${1}\" to resolve...\n"
# DNS_NAME holds the name which resolution we want to wait for. we will wait
# for it to resolve.
local DNS_NAME="${1}"
# DNS_RESOLVE_RETRIES holds the number of retries done so far.
local DNS_RESOLVE_RETRIES=0
# DNS_RESOLVE_MAX_RETRIES holds the maximum number of retries that will be
# made.
local DNS_RESOLVE_MAX_RETRIES=5
# loop until DNS_NAME resolves or DNS_RESOLVE_MAX_RETRIES is reached
while (( ${DNS_RESOLVE_RETRIES} <= ${DNS_RESOLVE_MAX_RETRIES} ));
do
# check if DNS_NAME resolves
host "${DNS_NAME}"
# check host's exit code
if [[ $? == 0 ]];
then
# give some info about the current operation
printf "${DNS_NAME} successfully resolved...\n"
# exit the function
return 0
fi
# at this point, dns resolution has failed, so we should check whether
# we should retry or end the loop
# give some info about the current operation
printf "${DNS_NAME} doesn't resolve yet...\n"
# increment the counter
DNS_RESOLVE_RETRIES=$((DNS_RESOLVE_RETRIES+1))
# if this is not the last retry...
if (( ${DNS_RESOLVE_RETRIES} < ${DNS_RESOLVE_MAX_RETRIES} ));
then
# give some info about the current operation
printf "sleeping...\n"
# wait a litle between the next retry (1s -> 2s -> 4s -> 8s -> 16s)
sleep $((2**${DNS_RESOLVE_RETRIES}))
# continue onto the next iteration
continue
fi
# at this point, dns resolution has failed everytime, so we should exit the
# task
# give some info about the current operation
echo "${DNS_NAME} didn't resolve after $((DNS_RESOLVE_MAX_RETRIES)) retries..."
# exit the task
exit 1
done
}
# parse the first argument and run the corresponding command
case "$1" in
"health-check")
health_check
;;
"start")
start;
;;
"readiness-check")
readiness_check
;;
"recover")
recover;
;;
*)
exit 1;
;;
esac
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment