Using OpenTelemetry Collector with Clickhouse

README.md

Clickhouse & OTel Collector

clickhouse network security

The username/networks/ip under spec.configuration.users was the key to getting things to work.
It seems that "10.0.0.0/16" isn't enough. Allowing all networks solved the problem.

Otel Collector and sensitive data

Secrets can be mounted to environment variables under the extraEnvs key.

The environment variable can then be referenced in the config with the syntax ${env:VARIABLE_NAME}.

otel-collector-helm-value-overrides.yaml

mode: deployment

serviceMonitor:
  enabled: true

ports:
  metrics:
    enabled: true
    containerPort: 8889
    servicePort: 8889
    protocol: TCP

resources:
  requests:
    cpu: 100m
    memory: 256Mi
  limits:
    cpu: 250m
    memory: 512Mi

securityContext:
  allowPrivilegeEscalation: false

config:
  extensions:
    health_check: {}
    zpages:
      endpoint: 0.0.0.0:55679
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318
    prometheus:
      config:
        scrape_configs:
          - job_name: opentelemetry-collector
            scrape_interval: 10s
            static_configs:
              - targets:
                  - 0.0.0.0:8888
  exporters:
    debug:
      verbosity: detailed
    prometheus:
      endpoint: 0.0.0.0:8889
    otlp:
      endpoint: "jaeger-collector.observability.svc:4317"
      tls:
        insecure: true  
    clickhouse:
      endpoint: tcp://clickhouse-simple-02:9000?dial_timeout=10s&compress=lz4
      database: otel
      ttl_days: 3
      username: ${env:CLICKHOUSE_USER}
      password: ${env:CLICKHOUSE_PASSWORD}
      logs_table_name: otel_logs
      traces_table_name: otel_traces
      metrics_table_name: otel_metrics
      timeout: 5s
      retry_on_failure:
        enabled: true
        initial_interval: 5s
        max_interval: 30s
        max_elapsed_time: 300s
  processors:
    batch:
      timeout: 1s
  connectors:
    spanmetrics:
      histogram:
        explicit:
          buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
      dimensions:
        - name: http.method
          default: GET
        - name: http.status_code
      dimensions_cache_size: 1000
      aggregation_temporality: 'AGGREGATION_TEMPORALITY_CUMULATIVE'
  service:
    extensions: [ health_check, zpages ]
    telemetry:
      logs:
        level: debug
      metrics:
        address: ":8888"
    pipelines:
      logs:
        exporters: [ debug, clickhouse ]
        processors: [ batch ]
        receivers: [ otlp ]
      metrics:
        exporters: [ debug, prometheus, clickhouse ]
        processors: [ batch ]
        receivers: [ otlp, spanmetrics, prometheus ]
      traces:
        receivers: [ otlp ]
        processors: [ batch ]
        exporters: [ otlp, debug, spanmetrics, clickhouse ]
extraEnvs:
  - name: CLICKHOUSE_USER
    valueFrom:
      secretKeyRef:
        name: clickhouse-user-otel
        key: userName
  - name: CLICKHOUSE_PASSWORD
    valueFrom:
      secretKeyRef:
        name: clickhouse-user-otel
        key: password            
  

simple-02.yaml

apiVersion: clickhouse.altinity.com/v1
kind: ClickHouseInstallation
metadata:
  name: simple-02
  namespace: observability
spec:
  configuration:
    users:
      otel/password: "YOUR_VALUE_HERE or better use hash"
      otel/networks/ip:
        - "10.0.0.0/16"
        - "0.0.0.0/0"
        - "::/0"
      otel/allow_databases/database:
        - default
        - otel
        - system
        - INFORMATION_SCHEMA
        - information_schema
    clusters:
      - name: simple
        layout:
          replicasCount: 1
          shardsCount: 1
  defaults:
    templates:
      dataVolumeClaimTemplate: data-volume-template
      logVolumeClaimTemplate: log-volume-template
  templates:
    volumeClaimTemplates:
      - name: data-volume-template
        spec:
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi
      - name: log-volume-template
        spec:
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 100Mi