Skip to content

Instantly share code, notes, and snippets.

@xuhaiL
Forked from fvigotti/purge_prom_pushgateway.sh
Created July 18, 2022 09:38
Show Gist options
  • Save xuhaiL/51f95f196e249eb5af5246dc23afb01e to your computer and use it in GitHub Desktop.
Save xuhaiL/51f95f196e249eb5af5246dc23afb01e to your computer and use it in GitHub Desktop.
this is a sample script that clean old metrics from pushgateway, not very clean but working anyway, used as sample in a issue
trap 'echo "got sigterm" ; exit 0' SIGTERM
EXPIRATION_SECONDS=${EXPIRATION_SECONDS:-900}
PGW_URL=${PGW_URL:-http://pushgateway}
function convert_to_standardnotation(){
# convert number from scientific notation to standar d( ie '1.5383780136826127e+09' )
printf '%.0f' $1
}
function extract_pushgateway_variable(){
local -r _METRIC=$1
local -r _VARNAME=$2
#echo 'push_time_seconds{instance="10.32.32.7",job="bk_jenkins"} 1.5383802210997093e+09' | sed -r 's/.*instance="([^"]*).*/\1/g'
echo $_METRIC | sed -r "s/.*${_VARNAME}=\"([^\"]*).*/\\1/g"
# sample usage :
# extract_pushgateway_variable 'push_time_seconds{instance="10.32.32.7",job="bk_jenkins"} 1.5383802210997093e+09' 'instance'
}
function check_metric_line(){
local -r _line=$1
METRIC_TIME=$(echo $_line | awk '{print $2}' )
#echo "mtime = $_line -> $METRIC_TIME "
METRIC_TIME=$(convert_to_standardnotation $METRIC_TIME)
#echo "$CURRENT_TIME - $METRIC_TIME "
METRIC_AGE_SECONDS=$((CURRENT_TIME-METRIC_TIME))
if [ "$METRIC_AGE_SECONDS" -gt "$EXPIRATION_SECONDS" ]; then
metricInstance=$(extract_pushgateway_variable "$_line" 'instance')
metricJob=$(extract_pushgateway_variable "$_line" 'job')
echo "[INFO] job should be deleted $metricJob - $metricInstance age: $METRIC_AGE_SECONDS "
curl -s -X DELETE "$PGW_URL/metrics/job/${metricJob}/instance/${metricInstance}"
fi
}
function check_expired_metric_loop(){
export CURRENT_TIME=$(date +%s)
METRICS_LIST=$(curl -s $PGW_URL/metrics | egrep "^push_time_seconds")
echo "$METRICS_LIST" | while read -r line || [[ -n "$line" ]]; do
check_metric_line "$line"
done
sleep $((EXPIRATION_SECONDS / 3 ))
}
while : ; do
check_expired_metric_loop
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment