Skip to content

Instantly share code, notes, and snippets.

@bahamat
Last active April 25, 2024 18:43
Show Gist options
  • Save bahamat/b77284bc08d772308ae7d1c49c53cecb to your computer and use it in GitHub Desktop.
Save bahamat/b77284bc08d772308ae7d1c49c53cecb to your computer and use it in GitHub Desktop.
Shell script to pull CMON metrics from Joyent Triton (https://www.joyent.com/triton)
#!/bin/bash
# shellcheck disable=SC2154
if [[ -n "$TRACE" ]]; then
export PS4='[\D{%FT%TZ}] ${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
set -o xtrace
fi
export PATH=/opt/pkg/bin:/opt/local/bin:/opt/custom/bin:$PATH
call() {
curl --connect-timeout 10 $insecure "${header:--i}" -s --compressed --cert "$cert" --key "$key" "https://${1}/${2}"
exit $?
}
do_genconfig() {
local datacenters
local dc_names
if [[ -n $gz ]]; then
printf 'Prometheus does not support discovery of CNs.\n'
# shellcheck disable=SC2016
printf 'Use file_sd and `cmon -g targets` instead.\n'
exit 1
fi
datacenters=$(get_datacenters)
# shellcheck disable=SC2207
dc_names=( $(get_dcnames) )
cat << EOF
global:
scrape_interval: 10s
evaluation_interval: 8s
# scrape_timeout is set to the global default 10s
scrape_configs:
- job_name: prometheus
scrape_interval: 5s
static_configs:
- targets: ['localhost:9090']
EOF
for dc in "${dc_names[@]}"; do
dc_url="$(json -Ma -c "this.key==\"$dc\"" value <<< "$datacenters")"
print_prom_job "$dc" "$dc_url"
done
}
do_keys() {
keydir=$(get_cmonkeydir)
mkdir -p ~/.triton/cmon/"${keydir}"
cd ~/.triton/cmon/"${keydir}" || return
export SDC_ACCOUNT="$user"
triton -p "${profile}" -a "$SDC_ACCOUNT" profile cmon-certgen -y
}
do_targets() {
local dcname
local kind='cns'
local objs
dcname="$(get_dcname)"
if [[ -z $gz ]]; then
printf 'Prometheus supports triton_sd for containers directly.\n'
# shellcheck disable=2016
printf 'Use `cmon genconfig` instead.\n'
exit 1
fi
# shellcheck disable=2207
objs=( $(call "${socket:?}" "v1/${gz}discover" | json -Ha "$kind" | json -a -o jsony-0) )
# shellcheck disable=2207
targets=(
$(for o in "${objs[@]}"; do
read -r -a u <<< "$(json -ga server_uuid server_hostname <<< "$o")"
printf '{"targets":'
printf '["%s.%s"],' "${u[0]}" "$socket"
printf '"labels":'
printf '{"server_uuid":"%s","hostname":"%s","datacenter":"%s"' "${u[0]}" "${u[1]}" "$dcname"
printf '}}\n'
done)
)
t=$(join , "${targets[@]}")
printf '[%s]' "$t" | json
}
get_cmonkeydir () {
local url
profile_j=$(triton -p "$profile" profile get -j)
url=$(json url <<< "$profile_j" | sed -E 's/^https?//g' | tr -d :/ | tr '.' _)
echo "${user}@${url}"
}
get_datacenters() {
triton -p "$profile" datacenters -j
}
get_dcname() {
cut -d '.' -f 2 <<< "$socket"
}
get_dcnames() {
local datacenters
datacenters="$(get_datacenters)"
json -Ma key <<< "$datacenters"
}
get_profile() {
triton -p "$profile" profile get -j
}
function join {
local IFS="$1"; shift; echo "$*";
}
parse_cmon_url () {
# shellcheck disable=2207
parts=( $(tr -d '/' <<< "$1" | tr ':' ' ') )
echo "${parts[@]}"
}
print_prom_job () {
local dc="$1"
local url="$2"
# shellcheck disable=2207
local cmon=($(parse_cmon_url "$(triton -a "$user" -p "$profile" -U "$url" services -j | json cmon)"))
local scheme="${cmon[0]}"
local endpoint="${cmon[1]}"
# Not used
#local port="${cmon[2]}"
cat << EOF
- job_name: ${user}_${dc}
scheme: ${scheme}
tls_config:
cert_file: ${cert}
key_file: ${key}
${config_insecure}
triton_sd_configs:
- account: ${user}
dns_suffix: ${endpoint}
endpoint: ${endpoint}
version: 1
tls_config:
cert_file: ${cert}
key_file: ${key}
${config_insecure}
relabel_configs:
- source_labels: [__meta_triton_machine_alias]
target_label: alias
- source_labels: [__meta_triton_machine_id]
target_label: zonename
- source_labels: [__param_datacenter_name]
target_label: datacenter_name
replacement: ${dc}
- source_labels: [__param_account]
target_label: account_name
replacement: ${user}
EOF
}
strip_proto() {
#sed 's#https://##' <<< "$1"
echo "${1/*:\/\/}"
}
usage () {
printf '%s [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] targets\n' "${0##*/}"
printf '%s [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] [ -G groups ] discover\n' "${0##*/}"
printf '%s [ -p profile ] [ -c certficate ] [ -k key ] metrics vm_uuid|server_uuid\n' "${0##*/}"
printf '%s [ -p profile ] -u username keysetup\n' "${0##*/}"
printf '%s [ -p profile ] -u username genconfig\n' "${0##*/}"
printf '\n -g\tDiscover Compute Nodes (gz metrics)\n'
exit "$1"
}
while getopts "e:c:k:p:u:G:ghiH" options; do
case $options in
H) header=' ';;
c) cert="${OPTARG}";;
e) socket="${OPTARG}";;
g) gz="gz/";;
G) groups="?groups=${OPTARG}" ;;
i)
insecure="-k";
config_insecure="insecure_skip_verify: true";
;;
k) key="${OPTARG}";;
p) profile="${OPTARG}";;
u) user="${OPTARG}";;
h) usage 0;;
*) usage 1;;
esac
done
shift $(( OPTIND - 1 ))
if [[ -z $user ]] || [[ -z $socket ]]; then
# Default profile
profile_json=$(get_profile)
fi
if [[ -n $profile ]]; then
profile=$(json name <<< "$profile_json")
fi
if [[ -z $user ]]; then
user=$(json account <<< "$profile_json")
fi
if [[ -z $socket ]]; then
socket=$(triton -p "$profile" services -j | json cmon)
fi
socket=$(strip_proto "$socket")
action="$1"
inst="$2"
if [[ -z $cert ]] || [[ -z $key ]]; then
keydir=$(get_cmonkeydir)
fi
# Use certs generated by triton profile cmon-certgen
if [[ -z $cert ]]; then
cert=~/.triton/cmon/"${keydir}"/cmon-${user}-cert.pem
fi
if [[ -z $key ]]; then
key=~/.triton/cmon/"${keydir}"/cmon-${user}-key.pem
fi
if [[ -z $socket ]]; then
printf 'Profile "%s" does not list a cmon endpoint\n' "$profile"
exit 1
fi
case $action in
disco|discover)
call "${socket:?}" "v1/${gz}discover${groups}"
;;
metrics)
call "${inst:?}.${socket}" metrics
;;
certgen|keysetup)
do_keys
;;
targets)
do_targets
;;
genconfig)
if [[ -z $user ]]; then
printf 'user is required for config generation\n'
usage 1
fi
do_genconfig;
;;
*)
printf 'Not implemented\n'
;;
esac
@bahamat
Copy link
Author

bahamat commented Oct 27, 2017

Discovery

$ cmon -p east1a discover | json -H 
{
  "containers": [
    {
      "server_uuid": "44454c4c-5300-1037-8044-b8c04f524432",
      "source": "Bootstrapper",
      "vm_alias": "e3e1aa6b",
      "vm_brand": "joyent",
      "vm_image_uuid": "1f32508c-e6e9-11e6-bc05-8fea9e979940",
      "vm_owner_uuid": "72804dc9-afef-4228-af55-2e973063865b",
      "vm_uuid": "e3e1aa6b-ad2c-cdc1-8b7d-df5c8bbdda70",
      "cached_date": 1508548301529
    }
  ]
}

@bahamat
Copy link
Author

bahamat commented Oct 27, 2017

Metrics. This works for either compute nodes or instances. It doesn't matter if the uuid passed is a server_uuid or vm_uuid.

$ cmon -p east1a metrics e3e1aa6b-ad2c-cdc1-8b7d-df5c8bbdda70 | head -15
HTTP/1.1 200 OK
Content-Type: text/plain
Content-Length: 4526
Date: Fri, 27 Oct 2017 17:54:58 GMT
Server: cmon
x-request-id: 76789960-9433-4703-a548-86abe3147538
x-response-time: 85
x-server-name: 75f44180-07b1-4992-aec3-ca9ec28d3eab
Connection: keep-alive

# HELP time_of_day System time in seconds since epoch
# TYPE time_of_day counter
time_of_day 1509126898642
# HELP net_agg_packets_in Aggregate inbound packets
# TYPE net_agg_packets_in counter

@bahamat
Copy link
Author

bahamat commented Oct 27, 2017

Pipe metrics to exposition for json format.

$ cmon -p east1a metrics e3e1aa6b-ad2c-cdc1-8b7d-df5c8bbdda70 | exposition | json | head -15
[
  {
    "name": "time_of_day",
    "metrics": [
      {
        "value": "1509127000839"
      }
    ],
    "help": "System time in seconds since epoch",
    "type": "COUNTER"
  },
  {
    "name": "net_agg_packets_in",
    "metrics": [
      {

@bahamat
Copy link
Author

bahamat commented Nov 6, 2017

Compute Node discovery for Global Zone Metrics

> cmon -i -g -c ./cmon-admin-cert.pem -k ./cmon-admin-key.pem -e https://cmon.vt-coal-1.ext.example.com:9163 discover | json
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 523
Date: Thu, 07 May 2020 15:57:53 GMT
Server: cmon
x-request-id: 25f78b09-c3ce-4418-add0-1f8161e1e3c3
x-response-time: 83
x-server-name: f2cccad9-5684-4a29-9106-75f717c4dc1f
Connection: keep-alive

{
  "cns": [
    {
      "server_uuid": "564d2b64-8862-278f-6181-cea59c4ec30d",
      "server_hostname": "headnode"
    },
    {
      "server_uuid": "298697d2-edde-4a3e-a921-3758b569621a",
      "server_hostname": "VC298697D2"
    },
    {
      "server_uuid": "33a55038-8383-45d5-9c5a-3e44ca2bebb6",
      "server_hostname": "VC33A55038"
    },
    {
      "server_uuid": "7b2fffab-cc38-4bb0-8713-edb9e97b8aba",
      "server_hostname": "VC7B2FFFAB"
    },
    {
      "server_uuid": "8ee0a055-fb1c-429b-822a-72f8b011405f",
      "server_hostname": "VC8EE0A055"
    },
    {
      "server_uuid": "eb8adc80-4ef1-4e5f-84a0-eb1094a1746d",
      "server_hostname": "VCEB8ADC80"
    }
  ]
}

@bahamat
Copy link
Author

bahamat commented May 7, 2020

Generate a json file to be used with Prometheus' file_sd.

> cmon -i -g -c ./cmon-admin-cert.pem -k ./cmon-admin-key.pem -e https://cmon.vt-coal-1.ext.example.com:9163 targets
[
  {
    "targets": [
      "564d2b64-8862-278f-6181-cea59c4ec30d.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "headnode",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "298697d2-edde-4a3e-a921-3758b569621a.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC298697D2",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "33a55038-8383-45d5-9c5a-3e44ca2bebb6.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC33A55038",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "7b2fffab-cc38-4bb0-8713-edb9e97b8aba.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC7B2FFFAB",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "8ee0a055-fb1c-429b-822a-72f8b011405f.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC8EE0A055",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "eb8adc80-4ef1-4e5f-84a0-eb1094a1746d.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VCEB8ADC80",
      "datacenter": "vt-coal-1"
    }
  }
]
> 

@bahamat
Copy link
Author

bahamat commented Oct 18, 2021

All options

% cmon -h
cmon [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] targets
cmon [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] [ -G groups ] discover
cmon [ -p profile ] [ -c certficate ] [ -k key ] metrics vm_uuid|server_uuid
cmon [ -p profile ] -u username keysetup
cmon [ -p profile ] -u username genconfig

    -g	Discover Compute Nodes (gz metrics)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment