Reproduce Telegraf issue #5962

telegraf_prometheus_dog_statsd_bug.sh

# PREP:
mkdir -p /tmp/telegraf_debug/{prometheus,statsd,telegraf}

# RUN TELEGRAF AGENT
cat <<EOF > /tmp/telegraf_debug/telegraf/telegraf.conf
[agent]
  interval = "10s"
  flush_interval = "10s"
  debug = true
  logfile = "/tmp/telegraf_debug/telegraf/telegraf.log"

[[inputs.prometheus]]
  urls = ["http://localhost:9090/metrics"]
  [inputs.prometheus.tagdrop]
    interface = ["all"]

[[inputs.statsd]]
  metric_separator = "_"
  parse_data_dog_tags = true
  service_address = ":8126"

[[outputs.prometheus_client]]
  expiration_interval = "30s"
  listen = ":9091"
  path = "/v1/metrics"
  string_as_label = false
EOF

telegraf --config /tmp/telegraf_debug/telegraf/telegraf.conf >/dev/null 2>&1 &

# CREATE PROMETHEUS ENDPOINT
cat<<EOF > /tmp/telegraf_debug/prometheus/metrics
net_bytes_recv{dc="alpha",host="server1",interface="docker0"} 126742965
net_bytes_recv{dc="alpha",host="server1",interface="eth0"} 8.40904966256
net_bytes_sent{dc="alpha",host="server1",interface="docker0"} 57160501
net_bytes_sent{dc="alpha",host="server1",interface="eth0"} 346307013621
net_drop_in{dc="alpha",host="server1",interface="eth0"} 5
net_icmp_inaddrmasks{dc="alpha",host="server1",interface="all"} 48
net_icmp_indestunreachs{dc="alpha",host="server1",interface="all"} 203406
net_icmp_inechoreps{dc="alpha",host="server1",interface="all"} 690894
net_icmp_inechos{dc="alpha",host="server1",interface="all"} 43144
net_icmp_inerrors{dc="alpha",host="server1",interface="all"} 14187
net_icmp_inmsgs{dc="alpha",host="server1",interface="all"} 1.770706e+06
net_icmp_intimeexcds{dc="alpha",host="server1",interface="all"} 833117
net_icmp_intimestampreps{dc="alpha",host="server1",interface="all"} 1
net_icmp_intimestamps{dc="alpha",host="server1",interface="all"} 96
EOF

cd /tmp/telegraf_debug/prometheus
python -m SimpleHTTPServer 9090 2>/dev/null &

# TEST TAGDROP
while [ 1 ]; do
  echo "Testing net_* metrics for interface=\"all\""
  curl -s localhost:9091/v1/metrics  | grep '^net_' | grep 'interface="all"' || echo Test passing
  sleep 5; clear
done &

# At this point you are running a Prometheus endpoint with metrics and a Telegraf
# agent configured to listen to statsd (parsing dog statsd tags) and scrape it.
# In the foreground, we are scraping the Telegraf prometheus_client output plugin 
# and we can see the the 'all' interface is not in the results, so tagdrop works.

# LET'S BREAK IT: IN ANOTHER SHELL, RUN THIS
while [ 1 ]; do
  cat /tmp/telegraf_debug/prometheus/metrics \
  | sed 's/^/statsd_/g;s/{/ #/g;s/"//g;s/=/:/g;s/}//g' \
  | awk '{print $1":"$3"|g|"$2}' \
  | while read line; do
    echo -n "${line}" | nc -4u -w0 localhost 8126
  done
  sleep 1
done

# GO BACK TO YOUR ORIGINAL SHELL AND WAIT FOR THE RESULTS TO UPDATE