Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Shelley monitoring
#!/bin/bash
#---------------------------------------------------------------------
# File: setup_shelley_monitoring.sh
# Created: 2019/10/17
# Creator: ilap
#=====================================================================
# DESCRIPTION:
#
# This script downloads and configures the required files
# for monitoring a Shelley node by using grafana/prometheus.
#
clean_up () {
echo "Cleaning up...." >&2
rm -rf "${TMP_DIR}"
RES=$1
exit ${RES:=127}
}
message() {
echo -e "$*" >&2
exit 127
}
get_idx () {
case $OSTYPE in
"darwin"*)
IDX=0
;;
"linux-gnu"*)
if [[ $HOSTTYPE == *"x86_64"* ]]; then
IDX=1
elif [[ $HOSTTYPE == *"arm"* ]]; then
IDX=2
else
message "The $HOSTTYPE is not supported"
fi
;;
*)
message "The \"$OSTYPE\" OS is not supported"
;;
esac
echo $IDX
}
dl() {
DL_URL="${1}"
OUTPUT="${TMP_DIR}/`basename \"${DL_URL}\"`"
shift
TO_DIR="${@}"
case ${DL} in
*"wget"*)
wget --no-check-certificate --output-document="${OUTPUT}" "${DL_URL}";;
*)
( cd ${TMP_DIR} && curl -JOL "${DL_URL}" --silent );;
esac
}
#### MAIN
if [ -z "$1" ] ; then
message "usage: `basename $0` <project path> # e.g. ... ./Shelley"
fi
PROJ_DIR="`pwd`/${1}"
PROJ_NAME="`basename ${1}`"
export TMP_DIR=`mktemp -d "/tmp/${PROJ_NAME}.XXXXXXXX"`
CURL=`which curl`
WGET=`which wget`
JCLI=`which jcli`
DL=${CURL:=$WGET}
if [ -z "$DL" -o -z "`which pip3`" -o -z "$JCLI" ]; then
message 'You need to have 'wget' or 'curl', 'jcli' and 'pip3' to be installed\nand accessable by PATH environment to continue...\nExiting.'
fi
# Obtain parameters.
IP=127.0.0.1
PORT=3101
export IP PORT
while :
do
read -p "What is the ip of the node (default:${IP})? " ip
read -p "What is the port of the REST api of the node running on ${IP:="${ip}"}'s (default: ${PORT})? " port
echo "Is this correct? http://${ip:-"${IP}"}:${port:-"${PORT}"}"
read -p "Do you want to continue? [Y/n/q] " answer
case ${answer:="Y"} in
[yY]*)
IP=${ip:-"${IP}"}
PORT=${port:-"${PORT}"}
break;;
[nN]* )
continue;;
[qQ]* )
exit;;
* )
echo "Please enter [yY](es), [nN](o) or [qQ](quit).";;
esac
done
ARCHS=("darwin-amd64" "linux-amd64" "linux-armv6")
IDX=`get_idx`
PROM_VER=2.13.0
PROM_URL="https://github.com/prometheus/prometheus/releases/download/v${PROM_VER}/prometheus-${PROM_VER}.${ARCHS[IDX]}.tar.gz"
GRAF_VER=6.4.3
GRAF_URL="https://dl.grafana.com/oss/release/grafana-${GRAF_VER}.${ARCHS[IDX]}.tar.gz"
NIX_URL="https://raw.githubusercontent.com/input-output-hk/jormungandr-nix/master/nixos/jormungandr-monitor"
MON_PY="monitor.py"
GRAF_DB="grafana.json"
trap clean_up SIGHUP SIGINT SIGQUIT SIGTRAP SIGABRT SIGTERM
echo -e "Downloading prometheus..." >&2
dl $PROM_URL
echo -e "Downloading grafana..." >&2
dl $GRAF_URL
echo -e "Downloading jormungandr monitoring scripts for prometheus" >&2
dl ${NIX_URL}/$MON_PY
dl ${NIX_URL}/$GRAF_DB
cd ${TMP_DIR} && mkdir -p "${PROJ_DIR}"/{exporters,prometheus,grafana}
PROM_DIR="${PROJ_DIR}/prometheus"
GRAF_DIR="${PROJ_DIR}/grafana"
tar zxC "${PROM_DIR}" -f *prome*gz --strip-components 1
tar zxC "${GRAF_DIR}" -f *graf*gz --strip-components 1
echo -e "Configuring components" >&2
cp -pr ${MON_PY} "${PROJ_DIR}"/exporters && chmod +x "${PROJ_DIR}"/exporters/*
cp -pr ${GRAF_DB} "${PROJ_DIR}"/
sed -i -e 's#@jcli@#'"$JCLI"'#g' "${PROJ_DIR}"/exporters/${MON_PY}
sed -i -e 's@\(/usr/bin/env python\)@\13@' "${PROJ_DIR}"/exporters/${MON_PY}
pip3 install ipython python-dateutil prometheus_client >/dev/null
cd ${PROJ_DIR}
EXPORTER_IP=localhost
EXPORTER_PORT=9100
sed -i -e 's@\(^scrape_configs:.*\)@\1\
- job_name: '\''jormungandr'\''\
static_configs:\
- targets: ['\'${EXPORTER_IP}:${EXPORTER_PORT}\'']@g' "${PROM_DIR}"/prometheus.yml
cat > start_all.sh <<EOF
#!/bin/bash
#1. exporter
PORT=${EXPORTER_PORT}
JORMUNGANDR_API="http://${IP}:${PORT}/api"
# Addresses to monitor e.g.
# ADDRESSES="ta1..... ta1..... ta1...."
ADDRESSES=""
export PORT JORMUNGANDR_API ADDRESSES
"${PROJ_DIR}"/exporters/monitor.py &
sleep 3
#2. Prometheus
"${PROM_DIR}"/prometheus --config.file="${PROM_DIR}"/prometheus.yml &
sleep 3
#3. Grafana
#vi conf/defaults.ini
cd "${GRAF_DIR}"
./bin/grafana-server web
EOF
chmod a+rx start_all.sh
echo -e "Installation completed
You need to do the following to configure grafana:
0. Start the required services by \"./${PROJ_NAME}/start_all.sh\"
- Keep in mind this startup script is very simple and has limited capabilities
- check the monitor script (http://${EXPORTER_IP}:${EXPORTER_PORT})
1. Login to grafana as admin/admin (http://localhost:3000)
2. Add "prometheus" (all lowercase) datasource (http://localhost:9090)
3. Create a new dashboard by importing 'grafana.json' (left plus sign).
Enjoy...
" >&2
clean_up 0
@ilap

This comment has been minimized.

Copy link
Owner Author

ilap commented Oct 17, 2019

Just created but have not tested.

@ilap

This comment has been minimized.

Copy link
Owner Author

ilap commented Oct 17, 2019

Ok, the Revision 6 has been tested on macOS, Ubuntu and Raspberry Pi4.

@rickymac68

This comment has been minimized.

Copy link

rickymac68 commented Oct 20, 2019

This worked perfectly. Mac OSX Catalina 10.15. Thank you ilap

@ilap

This comment has been minimized.

Copy link
Owner Author

ilap commented Oct 20, 2019

You're welcome.:)

@mark-stopka

This comment has been minimized.

Copy link

mark-stopka commented Oct 21, 2019

The elif directive has a wrong indentation, I know it's just cosmetic, but since it's so easy to fix... :)

        if [[ $HOSTTYPE == *"x86_64"* ]]; then
            IDX=1
            elif [[ $HOSTTYPE == *"arm"* ]]; then
            IDX=2
        else
@ilap

This comment has been minimized.

Copy link
Owner Author

ilap commented Oct 21, 2019

Fixed.:)

@ssainball

This comment has been minimized.

Copy link

ssainball commented Nov 9, 2019

Very good !!

@zcryptox

This comment has been minimized.

Copy link

zcryptox commented Jan 11, 2020

Great Tool... All node HW specific metrics aren't puling any data (CPU, network, Memory) - Jormungnadr metrics working fine... on Mac OSX

@lsc3

This comment has been minimized.

Copy link

lsc3 commented Jan 13, 2020

I'm using Mac OS Mojave and having problems with the 'ss' command used in monitor.py. Seems Mac doesn't provide "ss"? I see RickyMac68 above said it's working perfectly for him. Have I missed something? I have rudimentary Python skills but don't understand the "@lsof@", "@wc@" and "@ss@ in the monitor.py script. I just replaced the first two with direct paths to the commands and that seemed to fix it for them. But don't have an "ss" command. Any help would be greatly appreciated. Thank you.

@lsc3

This comment has been minimized.

Copy link

lsc3 commented Jan 13, 2020

Well ... I think I've things working for now. Thanks for the script. It helped a lot.

@zcryptox

This comment has been minimized.

Copy link

zcryptox commented Jan 13, 2020

@Isc3 can you share the changes you made on OSX? thx

@lsc3

This comment has been minimized.

Copy link

lsc3 commented Jan 13, 2020

I never got 'ss' working. But for now I can get some of these metrics into prometheus and grafana.

I had to pip3 install prometheus_client and py-dateutil python modules.

I basically changed the bash env to python3, commented out everything having to do with "ss". And changed the @lsof@ and @wc@ instances to full paths to each. This just allowed me to run 'python3 monitor.py'. I already have Prometheus/Grafana running. I didn't use the start_all.sh script.
I think that's all of it.
Hopefully this helps.

#!/usr/bin/env python3 <-------- changed from python to python3

JORMUNGANDR_API = os.getenv('JORMUNGANDR_RESTAPI_URL',
os.getenv('JORMUNGANDR_API', 'http://127.0.0.1:<MY_NODE_LISTEN_PORT>/api')) <------- node rest listen port

NODE_METRICS = [
"blockRecvCnt",
"connections",
"lastBlockDate",
"lastBlockEpoch",
"lastBlockFees",
"lastBlockHash",
"lastBlockHeight",
"lastBlockSlot",
"lastBlockSum",
"lastBlockTime",
"lastBlockTx",
"recvq", <------------------ commented out
"txRecvCnt",
"uptime",

Decorate function with metric.
@JORMUNGANDR_METRICS_REQUEST_TIME.time()
def process_jormungandr_metrics():
# Process jcli returned metrics
metrics = jcli_rest(['node', 'stats', 'get'])
lsof = subprocess.Popen(
('/usr/sbin/lsof', '-nPi', ':3000', '-sTCP:ESTABLISHED'), <---------------- Changed @lsof@ to direct path
stdout=subprocess.PIPE)
wc = subprocess.check_output(('/usr/bin/wc', '-l'), stdin=lsof.stdout). <-------------- Changed @wc@ to direct path
lsof.wait()
metrics['connections'] = int(wc, 10)

ss = subprocess.run(('@ss@', '-plntH', '( sport = :3000 )'), stdout=subprocess.PIPE).  <--------------comment out
recvq = ss.stdout.split()[1]                <--------------comment out
metrics['recvq'] = int(recvq, 10)       <--------------comment out

def jcli_rest(args):
flags = ['--host', JORMUNGANDR_API, '--output-format', 'json']
params = ['/usr/local/bin/jcli', 'rest', 'v0'] + args + flags <----------- Put in direct path to my jcli binary
result = subprocess.run(params, stdout=subprocess.PIPE)
return json.loads(result.stdout)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.