Skip to content

Instantly share code, notes, and snippets.

@sahilsk
Last active August 2, 2022 08:43
Show Gist options
  • Save sahilsk/6f5a900858f3719ffdd61148d9bda06e to your computer and use it in GitHub Desktop.
Save sahilsk/6f5a900858f3719ffdd61148d9bda06e to your computer and use it in GitHub Desktop.
Burrow configuration with slack and pagerduty notification
[general]
pidfile="burrow.pid"
stdout-logfile="burrow.out"
[logging]
filename="/opt/burrow/logs/burrow.log"
level="info"
maxsize=100
maxbackups=10
maxage=10
use-localtime=false
use-compression=true
[zookeeper]
servers=["x.x.x.x:2181","y.y.y.y:2181","z.z.z.z.140:2181"]
timeout=6
[client-profile.myclient]
kafka-version="2.2.0"
[httpserver.mylistener]
address=":8080"
timeout=300
[storage.mystorage]
class-name="inmemory"
[evaluator.mystorage]
class-name="caching"
[cluster.my-cluster]
class-name="kafka"
servers=["p.p.p.p:9092","q.q.q.q:9092","r.r.r.r:9092"]
client-profile="myclient"
[consumer.myconsumers]
class-name="kafka"
cluster="my-cluster"
servers=["p.p.p.p:9092","q.q.q.q:9092","r.r.r.r:9092"]
[notifier.pagerduty]
class-name="http"
interval=300
threshold=3
send-close=true
method-open="POST"
method-close="POST"
extras={integration_key='xxxxxx'}
template-open="/opt/burrow/config/default-pg-post.tmpl"
template-close="/opt/burrow/config/default-pg-delete.tmpl"
url-open="https://events.pagerduty.com/v2/enqueue"
url-close="https://events.pagerduty.com/v2/enqueue"
[notifier.slacknotify]
class-name="http"
interval=300
threshold=3
send-close=true
template-open='/opt/burrow/config/default-slack-post.tmpl'
template-close='/opt/burrow/config/default-slack-delete.tmpl'
url-open='https://hooks.slack.com/services/xxxx'
url-close='https://hooks.slack.com/services/xxxx'
{
"payload": {
"summary": "[{{.Result.Cluster}}]A kafka consumer is no longer lagging: {{ .Result.TotalLag}}",
"timestamp": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
"source": "xxx-prod-burrow",
"severity": "info",
"component": "{{.Result.Cluster}}",
"group": "{{.Result.Group}}",
"total_lag": "{{ .Result.TotalLag}}"
},
"routing_key": "{{ index .Extras "integration_key"}}",
"dedup_key": "{{.Result.Group}}",
"event_action": "resolve",
"client": "burrow-hostname",
"client_url": "http://x.x.x.x:8080/v3/kafka/{{.Result.Cluster}}"
}
{
"payload": {
"summary": "[{{.Result.Cluster}}]Kafka Consumer Lag Detected: {{len .Result.Partitions}} partitions have problems",
"timestamp": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
"source": "xxxx-prod-burrow",
"severity": "warning",
"component": "kafka",
"group": "{{.Result.Group}}",
"class": "Consumer Lag",
"custom_details": {
"cluster": "{{.Result.Cluster}}",
"consumer_group": "{{.Result.Group}}",
"total_lag": "{{ .Result.TotalLag}}",
"status": "{{.Result.Status.String}}",
"complete": "{{.Result.Complete}}",
"errors": "{{len .Result.Partitions}} partitions have problems",
"start": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
"partitions": "{{range .Result.Partitions}} {{.Status.String}} {{.Topic}}:{{.Partition}} ({{.Start.Timestamp}}, {{.Start.Offset}}, {{.Start.Lag}}) -> ({{.End.Timestamp}}, {{.End.Offset}}, {{.End.Lag}}), \n{{end}}"
}
},
"routing_key": "{{ index .Extras "integration_key"}}",
"dedup_key": "{{.Result.Group}}",
"links": [{
"href": "https://github.com/linkedin/Burrow/wiki/HTTP-Endpoint",
"text": "Burrow endpoints for troubleshooting"
}],
"event_action": "trigger",
"client": "burrow-hostname",
"client_url": "http://x.x.x.x:8080/v3/kafka/{{.Result.Cluster}}"
}
{
"attachments": [{
"color": "good",
"title": "A kafka consumer is no longer lagging",
"fields": [{
"title": "Group",
"value": "{{ .Group }}",
"short": false
}, {
"title": "Cluster",
"value": "{{ .Cluster }}",
"short": true
}, {
"title": "Total Lag",
"value": "{{ .Result.TotalLag}}",
"short": true
}, {
"title": "Start",
"value": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
"short": true
}]
}]
}
{
"attachments": [{
"color": "danger",
"title": "A kafka consumer is lagging behind!",
"fields": [{
"title": "Group",
"value": "{{ .Group }}",
"short": false
}, {
"title": "Cluster",
"value": "{{ .Cluster }}",
"short": true
}, {
"title": "Total Lag",
"value": "{{ .Result.TotalLag}}",
"short": true
}, {
"title": "Start",
"value": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}",
"short": true
}, {
"title": "Errors",
"value": "{{len .Result.Partitions}} partitions have problems",
"short": true
}, {
"title": ".....",
"value": "{{range .Result.Partitions}}{{.Status.String}} {{.Topic}}:{{.Partition}} ({{.Start.Timestamp}}, {{.Start.Offset}}, {{.Start.Lag}}) -> ({{.End.Timestamp}}, {{.End.Offset}}, {{.End.Lag}})\n{{end}}",
"short": false
}]
}]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment