Skip to content

Instantly share code, notes, and snippets.

@hreidar
Created September 18, 2020 20:42
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hreidar/da7d86337250960448c6011920d071fb to your computer and use it in GitHub Desktop.
Save hreidar/da7d86337250960448c6011920d071fb to your computer and use it in GitHub Desktop.
Vector swimlanes config
[sources.p2-logs-from_ingress]
type = "kafka"
bootstrap_servers = "k8s-c4-m1:30656,k8s-c4-m2:30656,k8s-c4-m3:30656"
group_id = "p2-vector-parser"
topics = ["P2_OSS_AND_IP.R"]
# split messages to swimlanes
[transforms.splitting_types]
inputs = ["p2-logs-from_ingress"]
type = "swimlanes"
[transforms.splitting_types.lanes.oss]
type = "check_fields"
"type.eq" = "oss"
[transforms.splitting_types.lanes.ipv4]
type = "check_fields"
"type.eq" = "ipv4_lease_log"
[transforms.splitting_types.lanes.ipv6]
type = "check_fields"
"type.eq" = "ipv6_lease_log"
[transforms.input-oss-grok]
inputs = ["splitting_types.oss"]
...
[transforms.ipv4-grok]
inputs = ["splitting_types.ipv4"]
...
[transforms.ipv6-regex]
inputs = ["splitting_types.ipv6"]
...
...
# some more transforms here, filter, drop (all of them work seperatly before swimline was intoduced
...
[sinks.p2-oss-to_kafka]
type = "kafka"
inputs = ["oss-drop_fields"]
bootstrap_servers = "k8s-c4-m1:30656,k8s-c4-m2:30656,k8s-c4-m3:30656"
topic = "P2_OSS.P"
encoding = "json"
[sinks.p2-ipv4-to_kafka]
type = "kafka"
inputs = ["ipv4-drop_fields"]
bootstrap_servers = "k8s-c4-m1:30656,k8s-c4-m2:30656,k8s-c4-m3:30656"
topic = "P2_IPV4.P"
encoding = "json"
[sinks.p2-ipv6-to_kafka]
type = "kafka"
inputs = ["ipv6-drop_fields"]
bootstrap_servers = "k8s-c4-m1:30656,k8s-c4-m2:30656,k8s-c4-m3:30656"
topic = "P2_IPV6.P"
encoding = "json"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment