Kamon, prometheus, grafana in Kubernetes

k8s-descriptor.yml

apiVersion: v1
kind: Namespace
metadata:
  name: monitoring
  labels:
    name: monitoring
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: statsd-exporter
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: statsd-exporter
    spec:
      containers:
      - name: statsd-exporter
        image: prom/statsd-exporter
        imagePullPolicy: Always
        ports:
        - containerPort: 9102
        - containerPort: 9125
        args: [
        ]
---
apiVersion: v1
kind: Service
metadata:
  name: statsd-exporter-svc
  labels:
    app: statsd-exporter
spec:
  ports:
  - name: udp
    port: 8125
    protocol: UDP
    targetPort: 9125
  - name: tcp
    port: 9102
    protocol: TCP
    targetPort: 9102
  selector:
    app: statsd-exporter
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: prometheus
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      containers:
      - image: prom/prometheus
        imagePullPolicy: Always
        name: prometheus
        volumeMounts:
        - name: prom-conf-vol
          mountPath: /etc/prometheus
        ports:
        - containerPort: 9090
      volumes:
      - name: prom-conf-vol
        configMap:
          name: prometheus-conf
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus-svc
  labels:
    app: prometheus
spec:
  type: LoadBalancer
  ports:
  - port: 9090
    targetPort: 9090
  selector:
    app: prometheus
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: grafana
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: grafana
    spec:
      containers:
      - name: grafana
        image: grafana/grafana
        imagePullPolicy: Always
        ports:
        - containerPort: 3000
        env:
        - name: GF_SECURITY_ADMIN_PASSWORD
          valueFrom:
            secretKeyRef:
              name: grafana-admin-secret
              key: password.txt
        args: [
        ]
---
apiVersion: v1
kind: Service
metadata:
  name: grafana-svc
  labels:
    app: grafana
spec:
  type: LoadBalancer
  ports:
  - port: 80
    targetPort: 3000
  selector:
    app: grafana

kamon-datadog.conf

kamon {
  ...
  datadog {
    hostname = "statsd-exporter-svc.monitoring.svc.cluster.local"
    port = 8125
    ...
  }
  default-tags {
    service-name = "my-service"
    env = "dev"
  }
}

prometheus.yml

global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.
  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    monitor: 'codelab-monitor'
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'
    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s
    static_configs:
      - targets: ['statsd-exporter-svc:9102'] 

setup.sh

# creating secret for grafana credentials
$ echo -n "grafana_admin_password" > ./password.txt

$ kubectl --namespace monitoring create secret generic grafana-admin-secret --from-file=./password.txt
secret "grafana-admin-secret" created

# create configmap for prometheus
$ kubectl --namespace monitoring create configmap prometheus-conf --from-file prometheus.yml

# create pods and services
$ kubectl --namespace monitoring apply -f k8s-descriptor.yml

# In the Grafana UI add Prometheus as a data source
# DONE!

update-prometheus-config.sh

# to update prometheus config
# save updated config to prometheus.yml
$ kubectl --namespace monitoring create configmap prometheus-conf --from-file prometheus.yml -o yaml --dry-run | kubectl --namespace monitoring replace -f -

# restart prometheus
$ kubectl --namespace monitoring scale --replicas=0 deployment/prometheus
$ kubectl --namespace monitoring scale --replicas=1 deployment/prometheus

# DONE!

See corresponding blog post https://medium.com/jeroen-rosenberg/from-monolith-to-microservice-architecture-on-kubernetes-part-4-monitoring-health-checks-32c441741ca6