Medium blog post code

01-prometheus-repo.sh

helm repo add stable https://kubernetes-charts.storage.googleapis.com
helm repo up

02-create-prometheus-props.sh

cat > prom.yaml <<PROPS
server:
  # Can't use PVs with "Deployments" (1)
  persistentVolume:
    enabled: false
  # Limit how many resources Prometheus can use
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 500m
      memory: 512Mi
PROPS

03-apply-prometheus.sh

helm upgrade --install \
               --values prom.yaml \
               --version 11.6.0 \
               prometheus \
               stable/prometheus

04-access-prometheus-ui.sh

kubectl port-forward svc/prometheus-server 8080:80

05-prometheus-increase-retention.yaml

server:
  # Keep the metrics for 3 months
  retention: 90d
  # Can't use PVs with "Deployments" (1)
  persistentVolume:
    enabled: false
  # Limit how many resources Prometheus can use
  resources:
    limits:
      cpu: 750m
      memory: 12Gi  # (2)
    requests:
      cpu: 500m
      memory: 12Gi  # (2)

06-prometheus-persistence.yaml

server:
  # Keep the metrics for 3 months
  retention: 90d
  # Can't use PVs with "Deployments" (1)
  persistentVolume:
    enabled: true
  statefulSet:  # (3)
    enabled: true
  # Limit how many resources Prometheus can use
  resources:
    limits:
      cpu: 750m
      memory: 12Gi  # (2)
    requests:
      cpu: 500m
      memory: 12Gi  # (2)

07-thanos-repo.sh

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo up

08-create-thanos-props.sh

cat > thanos.yaml <<PROPS
objstoreConfig: |-
  type: s3
  config:
    bucket: thanos
    endpoint: {{ include "thanos.minio.fullname" . }}.default.svc.cluster.local:9000
    access_key: minio
    secret_key: minio123
    insecure: true
querier:
  dnsDiscovery:
    enabled: true
    sidecarsService: prometheus-server
    sidecarsNamespace: default
storegateway:
  enabled: true
minio:
  enabled: true
  accessKey:
    password: "minio"
  secretKey:
    password: "minio123"
  defaultBuckets: "thanos"
PROPS

09-apply-thanos.sh

helm upgrade --install \
               --values thanos.yaml \
               --version 1.1.0 \
               thanos \
               bitnami/thanos

10-prometheus-thanos-sidecar.yaml

server:
  # Keep the metrics for 3 months
  retention: 90d
  # Can't use PVs with "Deployments" (1)
  persistentVolume:
    enabled: true
  statefulSet:  # (3)
    enabled: true
  # required by the Thanos sidecar
  global:
    external_labels:
      cluster: a
  extraArgs:
    storage.tsdb.min-block-duration: 2h
    storage.tsdb.max-block-duration: 2h
  service:
    gRPC:
      enabled: true
  sidecarContainers:
  - name: thanos-sc
    image: bitnami/thanos:0.12.2
    imagePullPolicy: IfNotPresent
    args:
    - sidecar
    - --prometheus.url=http://localhost:9090
    - --grpc-address=0.0.0.0:10901
    - --http-address=0.0.0.0:10902
    - --tsdb.path=/data/
    - --objstore.config=$(OBJSTORE_CONFIG)
    env:
    - name: OBJSTORE_CONFIG
      valueFrom:
        secretKeyRef:
          name: thanos-objstore-secret
          key: objstore.yml
    ports:
    - name: grpc
      containerPort: 10901
      protocol: TCP
    - name: http
      containerPort: 10902
      protocol: TCP
    volumeMounts:
    - mountPath: /data
      name: storage-volume
  # Limit how many resources Prometheus can use
  resources:
    limits:
      cpu: 750m
      memory: 12Gi  # (2)
    requests:
      cpu: 500m
      memory: 12Gi  # (2)

11-access-thanos-ui.sh

kubectl port-forward svc/thanos-querier 9090

12-thanos-compaction.yaml

objstoreConfig: |-
  type: s3
  config:
    bucket: thanos
    endpoint: {{ include "thanos.minio.fullname" . }}.default.svc.cluster.local:9000
    access_key: minio
    secret_key: minio123
    insecure: true
querier:
  dnsDiscovery:
    enabled: true
    sidecarsService: prometheus-server
    sidecarsNamespace: default
compactor:  # (5)
  enabled: true
  retentionResolutionRaw: 30d  # Full resolution
  retentionResolution5m: 60d   # 1 data point every 5m
  retentionResolution1h: 10y   # 1 data point per hour
storegateway:
  enabled: true
minio:
  enabled: true
  accessKey:
    password: "minio"
  secretKey:
    password: "minio123"
  defaultBuckets: "thanos"

13-prometheus-reduce-retention.yaml

server:
  retention: 8h
# Can't use PVs with "Deployments" (1)
  persistentVolume:
    enabled: true
  statefulSet:  # (3)
    enabled: true
  # required by the Thanos sidecar
  global:
    external_labels:
      cluster: a
  extraArgs:
    storage.tsdb.min-block-duration: 2h
    storage.tsdb.max-block-duration: 2h
  service:
    gRPC:
      enabled: true
  sidecarContainers:
  - name: thanos-sc
    image: bitnami/thanos:0.12.2
    imagePullPolicy: IfNotPresent
    args:
    - sidecar
    - --prometheus.url=http://localhost:9090
    - --grpc-address=0.0.0.0:10901
    - --http-address=0.0.0.0:10902
    - --tsdb.path=/data/
    - --objstore.config=$(OBJSTORE_CONFIG)
    env:
    - name: OBJSTORE_CONFIG
      valueFrom:
        secretKeyRef:
          name: thanos-objstore-secret
          key: objstore.yml
    ports:
    - name: grpc
      containerPort: 10901
      protocol: TCP
    - name: http
      containerPort: 10902
      protocol: TCP
    volumeMounts:
    - mountPath: /data
      name: storage-volume
  # Limit how many resources Prometheus can use
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 500m
      memory: 512Mi

14-prometheus-ha-deployment.yaml

# Disable the default reloader.
# Thanos sidecar will be doing this now
configmapReload:
  prometheus:
    enabled: false
server:
  replicaCount: 3
  retention: 8h
  # Can't use PVs with "Deployments" (1)
  persistentVolume:
    enabled: true
  statefulSet:  # (3)
    enabled: true
    headless:
      gRPC:
        enabled: true
  # required by the Thanos sidecar
  global:
    external_labels:
      cluster: a
      replica: $(REPLICA)
  extraArgs:
    storage.tsdb.min-block-duration: 2h
    storage.tsdb.max-block-duration: 2h
  service:
    gRPC:
      enabled: true
  sidecarContainers:
  - name: thanos-sc
    image: bitnami/thanos:0.12.2
    imagePullPolicy: IfNotPresent
    args:
    - sidecar
    - --prometheus.url=http://localhost:9090
    - --grpc-address=0.0.0.0:10901
    - --http-address=0.0.0.0:10902
    - --tsdb.path=/data/
    - --objstore.config=$(OBJSTORE_CONFIG)
    - --reloader.config-file=/etc/config/prometheus.yml
    - --reloader.config-envsubst-file=/templated/config/prometheus.yml
    - --reloader.rule-dir=/etc/config
    env:
    - name: OBJSTORE_CONFIG
      valueFrom:
        secretKeyRef:
          name: thanos-objstore-secret
          key: objstore.yml
    - name: REPLICA
      valueFrom:
        fieldRef:
          fieldPath: metadata.name
    ports:
    - name: grpc
      containerPort: 10901
      protocol: TCP
    - name: http
      containerPort: 10902
      protocol: TCP
    volumeMounts:
    - mountPath: /data
      name: storage-volume
    - name: config-volume
      mountPath: /etc/config
      readOnly: true
    - name: templated-config
      mountPath: /templated/config
  extraVolumes:
  - name: templated-config
    emptyDir:
      medium: Memory
  configPath: /templated/config/prometheus.yml
  extraVolumeMounts:
  - name: templated-config
    mountPath: /templated/config
  # Limit how many resources Prometheus can use
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 500m
      memory: 512Mi

15-thanos-more-sidecars.yaml

objstoreConfig: |-
  type: s3
  config:
    bucket: thanos
    endpoint: {{ include "thanos.minio.fullname" . }}.default.svc.cluster.local:9000
    access_key: minio
    secret_key: minio123
    insecure: true
querier:
  dnsDiscovery:
    enabled: true
    sidecarsService: prometheus-server-headless
    sidecarsNamespace: default
compactor:  # (5)
  enabled: true
  retentionResolutionRaw: 30d  # Full resolution
  retentionResolution5m: 60d   # 1 data point every 5m
  retentionResolution1h: 10y   # 1 data point per hour
storegateway:
  enabled: true
minio:
  enabled: true
  accessKey:
    password: "minio"
  secretKey:
    password: "minio123"
  defaultBuckets: "thanos"

16-prometheus-remote-write.yaml

server:
  replicaCount: 3
  retention: 8h
  remoteWrite:
  - http://thanos-write:9090
  # Can't use PVs with "Deployments" (1)
  persistentVolume:
    enabled: false
# Limit how many resources Prometheus can use
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 500m
      memory: 512Mi