Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?

Prometheus-operator installation with Helm

  • Prometheus, Grafana, kube-state-metrics, prometheus-node-exporter on Kubernetes

helm Chart

Prerequisites

  • Kubernetes 1.19+
  • Helm 3.2.0+
  • A persistent storage resource and RW access to it
  • Kubernetes StorageClass for dynamic provisioning

Document

helm update

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

Install

helm install prometheus-stack prometheus-community/kube-prometheus-stack \
  --create-namespace \
  --namespace prometheus-stack \
  --set grafana.adminPassword=prom-operator

image

clusterIP to NodePort
kubectl patch svc prometheus-stack-grafana -n prometheus-stack --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"},{"op":"replace","path":"/spec/ports/0/nodePort","value":32071}]'
Connect
instance_public_ip="$(curl ifconfig.me --silent)"
echo "https://$instance_public_ip:32071"
echo "ID: admin"
echo "PW: " $(kubectl get secret --namespace prometheus-stack prometheus-stack-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo)

image

@taking
Copy link
Author

taking commented Jul 6, 2022

(Option) Thanos

helm Chart

Prerequisites

  • Kubernetes 1.19+
  • Helm 3.2.0+
  • A persistent storage resource and RW access to it
  • Kubernetes StorageClass for dynamic provisioning

helm update

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update

create namespace

apiVersion: v1
kind: Namespace
metadata:
  name: monitoring

create thanos-sidecar-secret (objectstore-config)

cat <<EOF > thanos-sidecar-secret.yml
type: s3
config:
  bucket: thanos
  endpoint: thanos-minio:9000
  access_key: fJyQ816ERv
  secret_key: jnvmHb9hc6A9qa7Da9FEHpA2WqDIMBoehC66xiXF
  insecure: true
EOF

kubectl create secret generic thanos-objstore-config --from-file=objstore.yml=thanos-sidecar-secret.yml
kubectl create secret generic thanos-objstore-config -n monitoring --from-file=objstore.yml=thanos-sidecar-secret.yml
cat <<EOF > prom-thanos-sidecar.yaml
--- 
prometheus:
  prometheusSpec: 
    scrapeInterval: 5s
    externalLabels: 
      cluster: "openstack-210-207-104-191"
    replicas: 1
    thanos: 
      minTime: -3h
      objectStorageConfig: 
        key: objstore.yml
        name: thanos-objstore-config
    # additionalScrapeConfigs: |
    #   - job_name: "ceph-exporter"
    #     static_configs:
    #       - targets: ['rook-ceph-mgr.rook-ceph.svc.cluster.local:9283']
  thanosService: 
    enabled: true
    type: ClusterIP    
# grafana:
  # additionalDataSources:
  # - name: Prometheus
  #   type: prometheus
  #   url: http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090/
  #   basicAuth: false
  #   basicAuthUser:
  #   basicAuthPassword:
  #   isDefault: true
  #   editable: true
  #   jsonData:
  #     graphiteVersion: "1.1"
  #     tlsAuth: false
  #     tlsAuthWithCACert: false    
EOF

helm update

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

Install

helm install prometheus-stack prometheus-community/kube-prometheus-stack \
  --create-namespace \
  --namespace monitoring \
  --set grafana.adminPassword=innogrid \
  --set prometheus.thanos.create=true \
  -f prom-thanos-sidecar.yaml
kubectl patch svc prometheus-stack-grafana -n monitoring --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"},{"op":"replace","path":"/spec/ports/0/nodePort","value":32071}]'

values-override.yaml

cat <<EOF > values-override.yaml
# global:
#   storageClass: local-storage
objstoreConfig: |-
  type: s3
  config:
    bucket: thanos
    endpoint: thanos-minio:9000
    access_key: fJyQ816ERv
    secret_key: jnvmHb9hc6A9qa7Da9FEHpA2WqDIMBoehC66xiXF
    insecure: true  
query:
  enabled: true
  stores:
    - prometheus-stack-kube-prom-thanos-discovery:10901
queryFrontend:
  enabled: true
  config: |-
    type: IN-MEMORY
    config:
      max_size: 512MB
      max_size_items: 100
      validity: 100s
  extraFlags: 
  - --query-range.split-interval=24h
  - --query-range.max-retries-per-request=3
  - --query-frontend.log-queries-longer-than=60s
  service:
    type: NodePort
    nodePorts:
      http: 32072
compactor:
  enabled: true
  retentionResolutionRaw: 30d
  retentionResolution5m: 30d
  retentionResolution1h: 10y
  # Compaction 실행주기 설정 
  consistencyDelay: 30m
storegateway:
  enabled: true
  config: |-
    type: IN-MEMORY
    config:
      max_size: 250MB
      max_item_size: 125MB
ruler:
  enabled: true
  alertmanagers:
    - http://prometheus-stack-kube-prom-alertmanager:9093
  evalInterval: 1m
  clusterName: thanos
  config: |-
    groups:
      - name: "metamonitoring"
        rules:
          - alert: "PrometheusDown"
            expr: absent(up{prometheus="monitoring/prometheus-operator"})
minio:
  enabled: true
  auth:
    rootUser: fJyQ816ERv
    rootPassword: jnvmHb9hc6A9qa7Da9FEHpA2WqDIMBoehC66xiXF
  defaultBuckets: thanos
EOF

Install

helm install thanos bitnami/thanos \
  --create-namespace \
  --namespace monitoring \
  --set global.storageClass=ceph-block \
  -f values-override.yaml

@taking
Copy link
Author

taking commented Jul 6, 2022

ServiceMonitor

cat <<'EOF' | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
  labels:
    team: rook
spec:
  namespaceSelector:
    matchNames:
      - rook-ceph
  selector:
    matchLabels:
      app: rook-ceph-mgr
      rook_cluster: rook-ceph
      ceph_daemon_id: a
  endpoints:
  - port: http-metrics
    path: /metrics
    interval: 5s
EOF

@taking
Copy link
Author

taking commented Jul 15, 2022

(Option) 멀티클러스터 시 thanos 수집

Prometheus 설치 후

cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: prometheus-stack-kube-prom-thanos-discovery
  namespace: monitoring
spec:
  ports:
  - name: grpc
    port: 10901
    protocol: TCP
    nodePort: 32073
    targetPort: grpc
  - name: http
    port: 10902
    protocol: TCP
    targetPort: http
  selector:
    app.kubernetes.io/name: prometheus
    prometheus: prometheus-stack-kube-prom-prometheus
  sessionAffinity: None
  type: NodePort
status:
  loadBalancer: {}
EOF

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment