dataops-sre

#!/usr/bin/env python3
import requests
import requests.exceptions
import os, time, json, datetime

#Regex to match following
#kafka.log:type=Log,name=Size,topic=connect-offsets-webtrekk-connect-kafka-to-s3-stg-2018-10-09,partition=19
#{'topic': 'connect-offsets-webtrekk-connect-kafka-to-s3-stg-2018-10-09', 'partition': '19'}
REGEX = r'.*topic=(?P<topic>.*),partition=(?P<partition>\d+)'

dataops-sre

#!/usr/bin/env python3
import requests
import requests.exceptions
import os, time, json, datetime

#Regex to match following
#kafka.log:type=Log,name=Size,topic=connect-offsets-webtrekk-connect-kafka-to-s3-stg-2018-10-09,partition=19
#{'topic': 'connect-offsets-webtrekk-connect-kafka-to-s3-stg-2018-10-09', 'partition': '19'}
REGEX = r'.*topic=(?P<topic>.*),partition=(?P<partition>\d+)'

dataops-sre

#!/usr/bin/env python3
import requests
import requests.exceptions
import os, time, json, datetime

#Regex to match following
#kafka.log:type=Log,name=Size,topic=connect-offsets-webtrekk-connect-kafka-to-s3-stg-2018-10-09,partition=19
#{'topic': 'connect-offsets-webtrekk-connect-kafka-to-s3-stg-2018-10-09', 'partition': '19'}
REGEX = r'.*topic=(?P<topic>.*),partition=(?P<partition>\d+)'

dataops-sre

annotations:
  ad.datadoghq.com/kafka-connect.logs: '[{"log_processing_rules":[{"name":"exclude_noise_for_committing_offsets","pattern":"Committing offsets","type":"exclude_at_match"},{"name":"exclude_noise_for_empty_flushing","pattern":"flushing 0 outstanding messages for offset commit","type":"exclude_at_match"},{"name":"new_log_start_with_date","pattern":"\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])\\s\\d{2}:\\d{2}:\\d{2}","type":"multi_line"}],"path":"/var/log/kafka/server.log","service":"kafka-connect","source":"kafka","sourcecategory":"connect","tags":["env:local"],"type":"file"}]'
  # label for existing template on file
  ad.datadoghq.com/kafka-connect.check_names: '["kafka-connect-394f2"]'  # becomes instance tag in datadog
  ad.datadoghq.com/kafka-connect.init_configs: '[{"collect_default_metrics":false,"conf":[{"include":{"attribute":{"incoming-byte-rate":{"alias":"jmx.kafka.connect.incoming_byte_rate","metric_type":"gauge"},"outgoing-byte-rate":{"alias":"jmx.kafka.connect.outgoing_byte_rat

dataops-sre

instances:
  - jmx_url: service:jmx:rmi:///jndi/rmi://%%host%%:9999/jmxrmi
    name: "kafka-connect"
    tags:
    - kafka:connect
    - env:"dev"
    - app:"kafka-connect"
logs:
  - type: file
    path: /var/log/kafka/server.log

dataops-sre

apiVersion: v1
kind: ConfigMap
metadata:
  name: datadog-config
  namespace: monitoring
data:
  probe.sh: |
    #!/bin/sh
    set -e