Skip to content

Instantly share code, notes, and snippets.

@bhavin192
Last active June 3, 2022 07:25
Show Gist options
  • Save bhavin192/6e98f6a9ad34b49f4346792652af8fda to your computer and use it in GitHub Desktop.
Save bhavin192/6e98f6a9ad34b49f4346792652af8fda to your computer and use it in GitHub Desktop.
Modified yaml files for Prometheus+Thanos setup
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: mockmetrics
spec:
replicas: 1
template:
metadata:
name: mockmetrics
labels:
app: mockmetrics
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8080"
spec:
containers:
- name: mockmetrics
image: docker.io/bhavin192/mockmetrics:prom-bulk
imagePullPolicy: Always
ports:
- containerPort: 8080
resources:
limits:
cpu: 50m
memory: 28Mi
requests:
cpu: 50m
memory: 28Mi
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: prometheus-gcs
labels:
app: prometheus-gcs
thanos-peer: "true"
spec:
serviceName: "prometheus"
replicas: 2
selector:
matchLabels:
app: prometheus-gcs
thanos-peer: "true"
template:
metadata:
labels:
app: prometheus-gcs
thanos-peer: "true"
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "10902"
spec:
## Commented out because Minikube has only one node, should be commented in for any production setup
# affinity:
# podAntiAffinity:
# requiredDuringSchedulingIgnoredDuringExecution:
# - labelSelector:
# matchExpressions:
# - key: app
# operator: In
# values:
# - prometheus
# topologyKey: kubernetes.io/hostname
serviceAccountName: prometheus-server
containers:
- name: prometheus
image: quay.io/prometheus/prometheus:v2.5.0
args:
- "--storage.tsdb.retention=24h"
- "--config.file=/etc/prometheus-shared/prometheus.yml"
- "--storage.tsdb.path=/var/prometheus"
- "--storage.tsdb.min-block-duration=2h"
- "--storage.tsdb.max-block-duration=2h"
- "--web.enable-lifecycle"
ports:
- name: prom-http
containerPort: 9090
volumeMounts:
- name: config-shared
mountPath: /etc/prometheus-shared
- name: data
mountPath: /var/prometheus
- name: thanos-sidecar
# Always use explicit image tags (release or master-<date>-sha) instead of ambigous `latest` or `master`.
image: improbable/thanos:v0.1.0
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /creds/gcs-credentials.json
args:
- "sidecar"
- "--log.level=debug"
- "--tsdb.path=/var/prometheus"
- "--prometheus.url=http://127.0.0.1:9090"
- "--cluster.peers=thanos-peers.monitoring.svc.cluster.local:10900"
# NOTE: This is required to be added in GCS prior startup of this.
- "--gcs.bucket=thanos-store"
- "--reloader.config-file=/etc/prometheus/prometheus.yml.tmpl"
- "--reloader.config-envsubst-file=/etc/prometheus-shared/prometheus.yml"
ports:
- name: sidecar-http
containerPort: 10902
- name: grpc
containerPort: 10901
- name: cluster
containerPort: 10900
volumeMounts:
- name: data
mountPath: /var/prometheus
- name: gcs-credentials
mountPath: /creds/
- name: config-shared
mountPath: /etc/prometheus-shared
- name: config
mountPath: /etc/prometheus
volumes:
- name: config
configMap:
name: prometheus-config-gcs
- name: config-shared
emptyDir: {}
- name: data
emptyDir: {}
- name: gcs-credentials
secret:
defaultMode: 420
# NOTE: gcs-credentials secret with single file gcs-credentials.json (GCS service account) is required.
secretName: gcs-credentials
---
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config-gcs
data:
prometheus.yml.tmpl: |-
global:
external_labels:
monitor: prometheus
replica: '$(HOSTNAME)'
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- "127.0.0.1:9090"
- job_name: kubelets
kubernetes_sd_configs:
- role: node
- job_name: kube_pods
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_label_app]
action: keep
regex: mockmetrics
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: (.+?)(?::\d+)?;(\d+)
replacement: ${1}:${2}
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_pod_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
# Scrapes the endpoint lists for the main Prometheus endpoints
- job_name: kube_endpoints
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- action: keep
source_labels: [__meta_kubernetes_service_label_app]
regex: prometheus
- action: replace
source_labels: [__meta_kubernetes_service_label_app]
target_label: job
- action: replace
target_label: prometheus
source_labels: [__meta_kubernetes_service_label_prometheus]
---
apiVersion: v1
kind: Service
metadata:
labels:
app: prometheus-gcs
name: prometheus-gcs
spec:
externalTrafficPolicy: Cluster
ports:
- port: 9090
protocol: TCP
targetPort: prom-http
name: http-prometheus
- port: 10902
protocol: TCP
targetPort: sidecar-http
name: http-sidecar-metrics
selector:
app: prometheus-gcs
sessionAffinity: None
type: NodePort
status:
loadBalancer: {}
---
apiVersion: v1
kind: Service
metadata:
name: thanos-peers
spec:
type: ClusterIP
clusterIP: None
ports:
- name: cluster
port: 10900
targetPort: cluster
selector:
# Useful endpoint for gathering all thanos components for common gossip cluster.
thanos-peer: "true"
# Source: prometheus/templates/server-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app: prometheus
name: prometheus-server
---
# Source: prometheus/templates/server-clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
labels:
app: prometheus
name: prometheus-server
rules:
- apiGroups:
- ""
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- apiGroups:
- "extensions"
resources:
- ingresses/status
- ingresses
verbs:
- get
- list
- watch
- nonResourceURLs:
- "/metrics"
verbs:
- get
---
# Source: prometheus/templates/server-clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
labels:
app: prometheus
name: prometheus-server
subjects:
- kind: ServiceAccount
name: prometheus-server
namespace: monitoring
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: thanos-query
labels:
app: thanos-query
thanos-peer: "true"
spec:
replicas: 2
selector:
matchLabels:
app: thanos-query
thanos-peer: "true"
template:
metadata:
labels:
app: thanos-query
thanos-peer: "true"
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "10902"
spec:
containers:
- name: thanos-query
# Always use explicit image tags (release or master-<date>-sha) instead of ambigous `latest` or `master`.
image: improbable/thanos:v0.1.0
args:
- "query"
- "--log.level=debug"
- "--cluster.peers=thanos-peers.monitoring.svc.cluster.local:10900"
- "--query.replica-label=replica"
ports:
- name: http
containerPort: 10902
- name: grpc
containerPort: 10901
- name: cluster
containerPort: 10900
livenessProbe:
httpGet:
path: /-/healthy
port: http
---
apiVersion: v1
kind: Service
metadata:
labels:
app: thanos-query
name: thanos-query
spec:
externalTrafficPolicy: Cluster
ports:
- port: 9090
nodePort: 30909
protocol: TCP
targetPort: http
name: http-query
selector:
app: thanos-query
sessionAffinity: None
type: NodePort
# type: LoadBalancer
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: thanos-store
spec:
serviceName: "thanos-store"
replicas: 1
selector:
matchLabels:
app: thanos
thanos-peer: "true"
template:
metadata:
labels:
app: thanos
thanos-peer: "true"
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "10902"
spec:
containers:
- name: thanos-store
# Always use explicit image tags (release or master-<date>-sha) instead of ambigous `latest` or `master`.
image: improbable/thanos:v0.1.0
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /creds/gcs-credentials.json
args:
- "store"
- "--log.level=debug"
- "--data-dir=/var/thanos/store"
- "--cluster.peers=thanos-peers.monitoring.svc.cluster.local:10900"
# NOTE: This is required to be added in GCS prior startup of this.
- "--gcs.bucket=thanos-store"
ports:
- name: http
containerPort: 10902
- name: grpc
containerPort: 10901
- name: cluster
containerPort: 10900
volumeMounts:
- mountPath: /creds/
name: gcs-credentials
readOnly: true
- name: data
mountPath: /var/thanos/store
volumes:
- name: data
emptyDir: {}
- name: gcs-credentials
secret:
defaultMode: 420
# NOTE: gcs-credentials secret with single file gcs-credentials.json is required.
secretName: gcs-credentials
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment