Wavefront Proxy + Telegraf Agent + RabbitMQ

rabbit.yml

apiVersion: rabbitmq.com/v1beta1
kind: RabbitmqCluster
metadata:
  annotations:
    prometheus.io/port: "15692"
    prometheus.io/scrape: "true"
  name: bunny
spec:
  image: rabbitmq:3.8
  replicas: 3

telegraf.conf

# Telegraf Configuration
#
# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
# file would generate.
#
# Environment variables can be used anywhere in this config file, simply surround
# them with ${}. For strings the variable must be within quotes (ie, "${STR_VAR}"),
# for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR})

# Configuration for telegraf agent
[agent]
  ## Default data collection interval for all inputs
  ### Recommended value by RabbitMQ team ###
  interval = "30s"

  ##### Defaults #####

  ## Rounds collection interval to 'interval'
  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
  round_interval = true

  ## Telegraf will send metrics to outputs in batches of at most
  ## metric_batch_size metrics.
  ## This controls the size of writes that Telegraf sends to output plugins.
  metric_batch_size = 1000

  ## Maximum number of unwritten metrics per output.  Increasing this value
  ## allows for longer periods of output downtime without dropping metrics at the
  ## cost of higher maximum memory usage.
  metric_buffer_limit = 10000

  ## Collection jitter is used to jitter the collection by a random amount.
  ## Each plugin will sleep for a random time within jitter before collecting.
  ## This can be used to avoid many plugins querying things like sysfs at the
  ## same time, which can have a measurable effect on the system.
  collection_jitter = "0s"

  ## Default flushing interval for all outputs. Maximum flush_interval will be
  ## flush_interval + flush_jitter
  flush_interval = "10s"
  ## Jitter the flush interval by a random amount. This is primarily to avoid
  ## large write spikes for users running a large number of telegraf instances.
  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
  flush_jitter = "0s"

  ### Set any of these if necessary ###
  ## Log at debug level.
  # debug = false
  ## Log only error level messages.
  # quiet = false

  ## Override default hostname, if empty use os.Hostname()
  ### May be interesting to set this to something unique-ish to identify the source? ###
  hostname = ""
  ## If set to true, do no set the "host" tag in the telegraf agent.
  omit_hostname = false


# Configuration for Wavefront server to send metrics to
[[outputs.wavefront]]
  ## Url for Wavefront Direct Ingestion or using HTTP with Wavefront Proxy
  ## If using Wavefront Proxy, also specify port. example: http://proxyserver:2878
  url = "http://<wavefront-proxy-service-name>.<namespace-name>:<port>"

  ## prefix for metrics keys
  ### perhaps useful to filter metrics? ###
  #prefix = "my.specific.prefix."

  ## point tags to use as the source name for Wavefront (if none found, host will be used)
  ### May be worth exploring if setting some of these helps building Wavefront queries ###
  #source_override = ["hostname", "address", "agent_host", "node_host"]

[[inputs.prometheus]]
  ## Metric version controls the mapping from Prometheus metrics into
  ## Telegraf metrics.  When using the prometheus_client output, use the same
  ## value in both plugins to ensure metrics are round-tripped without
  ## modification.
  ##
  ##   example: metric_version = 1; deprecated in 1.13
  ##            metric_version = 2; recommended version

  # Maybe is ok to use 2?
  # metric_version = 1

  ## Scrape Kubernetes pods for the following prometheus annotations:
  ## - prometheus.io/scrape: Enable scraping for this pod
  ## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to
  ##     set this to 'https' & most likely set the tls config.
  ## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
  ## - prometheus.io/port: If port is not 9102 use this annotation
  monitor_kubernetes_pods = true
  ## Restricts Kubernetes monitoring to a single namespace
  ##   ex: monitor_kubernetes_pods_namespace = "default"
  monitor_kubernetes_pods_namespace = "${POD_NAMESPACE}"

telegraf.yml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: telegraf-agent
  name: telegraf-agent
spec:
  replicas: 1
  selector:
    matchLabels:
      app: telegraf-agent
  template:
    metadata:
      labels:
        app: telegraf-agent
    spec:
      containers:
      - name: telegraf-agent
        image: telegraf:1.15
        imagePullPolicy: IfNotPresent
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        securityContext:
          privileged: false
        volumeMounts:
        - name: telegraf-conf
          mountPath: "/etc/telegraf/"
      serviceAccountName: telegraf-agent
      volumes:
      - name: telegraf-conf
        configMap:
          name: telegraf-conf
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pod-reader
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
  - list
  - watch
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: telegraf-agent
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: telegraf-agent-pod-reader
subjects:
- kind: ServiceAccount
  name: telegraf-agent
roleRef:
  kind: Role
  name: pod-reader
  apiGroup: rbac.authorization.k8s.io

wavefront-proxy.yml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: wavefront-proxy
    name: wavefront-proxy
  name: wavefront-proxy
  namespace: rabbitmq-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: wavefront-proxy
  template:
    metadata:
      labels:
        app: wavefront-proxy
    spec:
      hostname: some-proxy-perhaps-changeme
      containers:
      - name: wavefront-proxy
        image: wavefronthq/proxy:8.3
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 1
            memory: "800Mi"
          requests:
            cpu: 1
            memory: "800Mi"
        env:
        - name: WAVEFRONT_URL
          value: https://<your-instance>.wavefront.com/api/
        - name: JAVA_HEAP_USAGE
          value: "512m"
        - name: WAVEFRONT_TOKEN
          valueFrom:
            secretKeyRef:
              name: wavefront-token
              key: token
        ports:
        - containerPort: 2878
          protocol: TCP
        securityContext:
          privileged: false
---
apiVersion: v1
kind: Service
metadata:
  name: wavefront-proxy
  namespace: rabbitmq-system
  labels:
    app: wavefront-proxy
spec:
  ports:
  - name: wavefront
    port: 2878
    protocol: TCP
  selector:
    app: wavefront-proxy