Created
June 19, 2024 02:18
-
-
Save darkn3rd/4684fa311bbd83c8385734e81deb43f3 to your computer and use it in GitHub Desktop.
My Datadog Pipelines Example (experimental)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
variable "pipelines" { | |
description = "List of pipelines with their processors" | |
type = list(any) | |
} | |
resource "datadog_logs_custom_pipeline" "pipelines" { | |
for_each = { for idx, pipeline in var.pipelines : idx => pipeline } | |
name = each.value.name | |
is_enabled = each.value.is_enabled | |
filter { | |
query = each.value.filter.query | |
} | |
dynamic "processor" { | |
for_each = each.value.processors | |
content { | |
${processor.type} { | |
${processor.type == "arithmetic_processor" ? "expression" : ""} = processor.expression | |
${processor.type == "arithmetic_processor" ? "target" : ""} = processor.target | |
${processor.type == "arithmetic_processor" ? "is_replace_missing" : ""} = processor.is_replace_missing | |
${processor.type == "attribute_remapper" ? "sources" : ""} = processor.sources | |
${processor.type == "attribute_remapper" ? "source_type" : ""} = processor.source_type | |
${processor.type == "attribute_remapper" ? "target" : ""} = processor.target | |
${processor.type == "attribute_remapper" ? "target_type" : ""} = processor.target_type | |
${processor.type == "attribute_remapper" ? "target_format" : ""} = processor.target_format | |
${processor.type == "attribute_remapper" ? "preserve_source" : ""} = processor.preserve_source | |
${processor.type == "attribute_remapper" ? "override_on_conflict" : ""} = processor.override_on_conflict | |
${processor.type == "grok_parser" ? "samples" : ""} = processor.samples | |
${processor.type == "grok_parser" ? "source" : ""} = processor.source | |
${processor.type == "grok_parser" ? "grok" : ""} = processor.grok | |
name = processor.name | |
is_enabled = processor.is_enabled | |
} | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
pipelines = [ | |
{ | |
name = "sample pipeline 1" | |
is_enabled = true | |
filter = { query = "source:foo" } | |
processors = [ | |
{ | |
type = "arithmetic_processor" | |
expression = "(time1 - time2)*1000" | |
target = "my_arithmetic" | |
is_replace_missing = true | |
name = "sample arithmetic processor" | |
is_enabled = true | |
}, | |
{ | |
type = "attribute_remapper" | |
sources = ["db.instance"] | |
source_type = "tag" | |
target = "db" | |
target_type = "attribute" | |
target_format = "string" | |
preserve_source = true | |
override_on_conflict = false | |
name = "sample attribute processor" | |
is_enabled = true | |
} | |
# Add other processors as needed | |
] | |
}, | |
{ | |
name = "sample pipeline 2" | |
is_enabled = true | |
filter = { query = "source:python" } | |
processors = [ | |
{ | |
type = "grok_parser" | |
samples = ["sample log 1"] | |
source = "message" | |
grok = { | |
support_rules = "" | |
match_rules = "Rule %%{word:my_word2} %%{number:my_float2}" | |
} | |
name = "sample grok parser" | |
is_enabled = true | |
} | |
# Add other processors as needed | |
] | |
} | |
# Add other pipelines as needed | |
] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment