Skip to content

Instantly share code, notes, and snippets.

@pasdam

pasdam/otel-collector-config.yaml

Created September 5, 2023 15:40
Show Gist options
  • Save pasdam/d7e3a71ac48f943e13b1f7b4894000ed to your computer and use it in GitHub Desktop.
Save pasdam/d7e3a71ac48f943e13b1f7b4894000ed to your computer and use it in GitHub Desktop.
Download ZIP
Opentelemetry collector config to send telemetry to clickhouse
Raw
otel-collector-config.yaml
connectors:
spanmetrics:
histogram:
explicit:
buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
dimensions:
- name: http.method
default: GET
- name: http.status_code
exemplars:
enabled: true
exclude_dimensions: ['status.code']
dimensions_cache_size: 1000
aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE"
metrics_flush_interval: 15s
receivers:
tcplog/docker:
listen_address: "0.0.0.0:2255"
operators:
- type: regex_parser
regex: '^<([0-9]+)>[0-9]+ (?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P<container_id>\S+) (?P<container_name>\S+) [0-9]+ - -( (?P<body>.*))?'
timestamp:
parse_from: attributes.timestamp
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
- type: move
from: attributes["body"]
to: body
- type: remove
field: attributes.timestamp
# please remove names from below if you want to collect logs from them
- type: filter
id: signoz_logs_filter
expr: 'attributes.container_name matches "^playground-signoz-(logspout|load-hotrod|frontend|alertmanager|query-service|otel-collector|clickhouse-1)-1"'
jaeger:
protocols:
grpc:
endpoint: 0.0.0.0:14250
thrift_http:
endpoint: 0.0.0.0:14268
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
batch:
send_batch_size: 10000
send_batch_max_size: 11000
timeout: 10s
transform:
error_mode: ignore
log_statements:
- context: log
statements:
- merge_maps(cache, ExtractPatterns(body, "^^(?P<timestamp>[^\t]+)\t(?P<severity>[^\t]+)\t(?P<source>[^\t]+)\t(?P<message>[^\t]+)\t(?P<json_msg>[^\t]+)$"), "upsert") where IsMatch(attributes["container_name"], "playground-signoz-hotrod-1")
- set(body, cache["message"])
- merge_maps(cache, ParseJSON(cache["json_msg"]), "upsert") where IsMatch(cache["json_msg"], "^\\{.*\\}$")
- set(attributes["service"], cache["service"])
- set(severity_text, cache["severity"])
- set(trace_id.string, cache["trace_id"])
- set(span_id.string, cache["span_id"])
attributes:
actions:
- action: insert
key: loki.attribute.labels
value: container_name
exporters:
clickhouse:
endpoint: tcp://clickhouse-1:9000?dial_timeout=10s&compress=lz4
database: otel
ttl_days: 3
logs_table_name: otel_logs
traces_table_name: otel_traces
metrics_table_name: otel_metrics
timeout: 5s
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s
logging:
verbosity: detailed
sampling_initial: 5
sampling_thereafter: 200
service:
telemetry:
logs:
level: debug
pipelines:
logs:
receivers:
- tcplog/docker
processors:
- transform
- attributes
- batch
exporters:
- clickhouse
- logging
metrics:
receivers:
- spanmetrics
processors:
- batch
exporters:
- clickhouse
traces:
receivers:
- jaeger
processors:
- batch
exporters:
- clickhouse
- spanmetrics
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment