prometheus-operator-helm.md

Prometheus-operator installation with Helm

• Prometheus, Grafana, kube-state-metrics, prometheus-node-exporter on Kubernetes

helm Chart

• https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack

Prerequisites

• Kubernetes 1.19+
• Helm 3.2.0+
• A persistent storage resource and RW access to it
• Kubernetes StorageClass for dynamic provisioning

Document

• https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#configuration

helm update

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update prometheus-community

value-override

cat <<EOF > values-override.yaml
--- 
prometheus:
  prometheusSpec: 
    scrapeInterval: 5s
    externalLabels: 
      cluster: "crio-k8s-1.25"
    replicas: 1
grafana:
  adminPassword: prom-operator
EOF

Install

helm install prometheus-stack prometheus-community/kube-prometheus-stack \
  --create-namespace \
  --namespace monitoring \
  -f values-override.yaml
  

clusterIP to NodePort

kubectl patch svc prometheus-stack-grafana -n monitoring --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"},{"op":"replace","path":"/spec/ports/0/nodePort","value":32071}]'

kubectl patch svc prometheus-stack-kube-prom-prometheus -n monitoring --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"},{"op":"replace","path":"/spec/ports/0/nodePort","value":30097}]'

Connect

instance_public_ip="$(curl ifconfig.me --silent)"
echo "https://$instance_public_ip:32071"
echo "ID: admin"
echo "PW: " $(kubectl get secret --namespace monitoring prometheus-stack-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo)

(Option) Thanos

helm Chart

• https://github.com/bitnami/charts/tree/master/bitnami/thanos

Prerequisites

• Kubernetes 1.19+
• Helm 3.2.0+
• A persistent storage resource and RW access to it
• Kubernetes StorageClass for dynamic provisioning

helm update

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update

create namespace

apiVersion: v1
kind: Namespace
metadata:
  name: monitoring

create thanos-sidecar-secret (objectstore-config)

cat <<EOF > thanos-sidecar-secret.yml
type: s3
config:
  bucket: thanos
  endpoint: thanos-minio:9000
  access_key: fJyQ816ERv
  secret_key: jnvmHb9hc6A9qa7Da9FEHpA2WqDIMBoehC66xiXF
  insecure: true
EOF

kubectl create secret generic thanos-objstore-config --from-file=objstore.yml=thanos-sidecar-secret.yml
kubectl create secret generic thanos-objstore-config -n monitoring --from-file=objstore.yml=thanos-sidecar-secret.yml

cat <<EOF > prom-thanos-sidecar.yaml
--- 
kubeControllerManager:
  service:
    port: 10257
    targetPort: 10257
  serviceMonitor:
    https: 'true'
    insecureSkipVerify: 'true'
kubeScheduler:
  service:
    port: 10259
    targetPort: 10259
  serviceMonitor:
    https: 'true'
    insecureSkipVerify: 'true'
prometheus:
  prometheusSpec: 
    scrapeInterval: 5s
    externalLabels: 
      cluster: "crio-v1-25-k8s"
    replicas: 1
    thanos: 
      minTime: -3h
      objectStorageConfig: 
        key: objstore.yml
        name: thanos-objstore-config
    # additionalScrapeConfigs: |
    #   - job_name: "ceph-exporter"
    #     static_configs:
    #       - targets: ['rook-ceph-mgr.rook-ceph.svc.cluster.local:9283']
  thanosServiceExternal: 
    enabled: true
    type: NodePort
    nodePort: 32073
# grafana:
  # additionalDataSources:
  # - name: Prometheus
  #   type: prometheus
  #   url: http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090/
  #   basicAuth: false
  #   basicAuthUser:
  #   basicAuthPassword:
  #   isDefault: true
  #   editable: true
  #   jsonData:
  #     graphiteVersion: "1.1"
  #     tlsAuth: false
  #     tlsAuthWithCACert: false    
EOF

helm update

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

Install

helm install prometheus-stack prometheus-community/kube-prometheus-stack \
  --create-namespace \
  --namespace monitoring \
  --set grafana.adminPassword=taking \
  --set prometheus.thanos.create=true \
  -f prom-thanos-sidecar.yaml

kubectl patch svc prometheus-stack-grafana -n monitoring --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"},{"op":"replace","path":"/spec/ports/0/nodePort","value":32071}]'

values-override.yaml

cat <<EOF > values-override.yaml
# global:
#   storageClass: local-storage
objstoreConfig: |-
  type: s3
  config:
    bucket: thanos
    endpoint: {{ include "thanos.minio.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:9000
    #endpoint: thanos-minio:9000
    access_key: minio
    secret_key: minio123
    insecure: true
query:
  enabled: true
  dnsDiscovery:
    sidecarsService: kube-prometheus-prometheus-thanos
    sidecarsNamespace: monitoring
  stores:
    -  prometheus-stack-kube-prom-thanos-discovery:10901
queryFrontend:
  enabled: true
  config: |-
    type: IN-MEMORY
    config:
      max_size: 512MB
      max_size_items: 100
      validity: 100s
  extraFlags: 
  - --query-range.split-interval=24h
  - --query-range.max-retries-per-request=3
  - --query-frontend.log-queries-longer-than=60s
  service:
    type: NodePort
    nodePorts:
      http: 32548
bucketweb:
  enabled: true
compactor:
  enabled: true
  #retentionResolutionRaw: 30d
  #retentionResolution5m: 30d
  #retentionResolution1h: 10y
  ## Compaction 실행주기 설정 
  #consistencyDelay: 30m
storegateway:
  enabled: true
  #config: |-
  #  type: IN-MEMORY
  #  config:
  #    max_size: 250MB
  #    max_item_size: 125MB
ruler:
  enabled: true
  alertmanagers:
    - http://prometheus-stack-kube-prom-alertmanager:9093
  #evalInterval: 1m
  #clusterName: thanos
  config: |-
    groups:
      - name: "metamonitoring"
        rules:
          - alert: "PrometheusDown"
            expr: absent(up{prometheus="monitoring/prometheus-operator"})
minio:
  enabled: true
  auth:
    rootUser: minio
    rootPassword: minio123
  defaultBuckets: thanos
EOF

Install

helm install thanos bitnami/thanos \
  --create-namespace \
  --namespace monitoring \
  --set global.storageClass=ceph-block \
  -f values-override.yaml

update

cat <<EOF > thanos-querier.yaml
querier:
 stores:
  - IP:PORT

queryFrontend:
 service:
  type: NodePort
  nodePort: 32548
EOF

helm upgrade thanos -f thanos-querier.yaml bitnami/thanos -n monitoring

ServiceMonitor

cat <<'EOF' | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
  labels:
    team: rook
spec:
  namespaceSelector:
    matchNames:
      - rook-ceph
  selector:
    matchLabels:
      app: rook-ceph-mgr
      rook_cluster: rook-ceph
      ceph_daemon_id: a
  endpoints:
  - port: http-metrics
    path: /metrics
    interval: 5s
EOF

(Option) 멀티클러스터 시 thanos 수집

Prometheus 설치 후

cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: prometheus-stack-kube-prom-thanos-discovery
  namespace: monitoring
spec:
  ports:
  - name: grpc
    port: 10901
    protocol: TCP
    nodePort: 32073
    targetPort: grpc
  - name: http
    port: 10902
    protocol: TCP
    targetPort: http
  selector:
    app.kubernetes.io/name: prometheus
    prometheus: prometheus-stack-kube-prom-prometheus
  sessionAffinity: None
  type: NodePort
status:
  loadBalancer: {}
EOF

(Option) kubernetes v1.24 이상

cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Secret
metadata:
  name: crio-k8s
  namespace: kube-system
  annotations:
    kubernetes.io/service-account.name: crio-k8s
type: kubernetes.io/service-account-token
EOF

kubectl get secrets -n kube-system
kubectl describe secrets/crio-k8s -n kube-system | grep token

(Option) Thanos Down 해결 방법

## kube-proxy
kubectl edit cm kube-proxy -n kube-system
metricsBindAddress: "" -> metricsBindAddress: 0.0.0.0:10249
kubectl rollout restart daemonset/kube-proxy -n kube-system



## etcd
/etc/kubernetes/manifests/etcd.yaml
127.0.0.1:2381 > 0.0.0.0:2381

## scheduler
/etc/kubernetes/manifests/kube-scheduler.yaml
bind-address=127.0.0.1 > bind-address=0.0.0.0


## controller-manager
/etc/kubernetes/manifests/kube-controller-manager.yaml
bind-address=127.0.0.1 > bind-address=0.0.0.0