My Datadog Pipelines Example (experimental)

main.tf

variable "pipelines" {
  description = "List of pipelines with their processors"
  type = list(any)
}

resource "datadog_logs_custom_pipeline" "pipelines" {
  for_each = { for idx, pipeline in var.pipelines : idx => pipeline }

  name       = each.value.name
  is_enabled = each.value.is_enabled

  filter {
    query = each.value.filter.query
  }

  dynamic "processor" {
    for_each = each.value.processors
    content {
      ${processor.type} {
        ${processor.type == "arithmetic_processor" ? "expression" : ""}            = processor.expression
        ${processor.type == "arithmetic_processor" ? "target" : ""}                = processor.target
        ${processor.type == "arithmetic_processor" ? "is_replace_missing" : ""}    = processor.is_replace_missing
        ${processor.type == "attribute_remapper" ? "sources" : ""}                 = processor.sources
        ${processor.type == "attribute_remapper" ? "source_type" : ""}             = processor.source_type
        ${processor.type == "attribute_remapper" ? "target" : ""}                  = processor.target
        ${processor.type == "attribute_remapper" ? "target_type" : ""}             = processor.target_type
        ${processor.type == "attribute_remapper" ? "target_format" : ""}           = processor.target_format
        ${processor.type == "attribute_remapper" ? "preserve_source" : ""}         = processor.preserve_source
        ${processor.type == "attribute_remapper" ? "override_on_conflict" : ""}    = processor.override_on_conflict
        ${processor.type == "grok_parser" ? "samples" : ""}                        = processor.samples
        ${processor.type == "grok_parser" ? "source" : ""}                         = processor.source
        ${processor.type == "grok_parser" ? "grok" : ""}                           = processor.grok
        name       = processor.name
        is_enabled = processor.is_enabled
      }
    }
  }
}

mydefs.tfvars

pipelines = [
  {
    name       = "sample pipeline 1"
    is_enabled = true
    filter     = { query = "source:foo" }
    processors = [
      {
        type                  = "arithmetic_processor"
        expression            = "(time1 - time2)*1000"
        target                = "my_arithmetic"
        is_replace_missing    = true
        name                  = "sample arithmetic processor"
        is_enabled            = true
      },
      {
        type                  = "attribute_remapper"
        sources               = ["db.instance"]
        source_type           = "tag"
        target                = "db"
        target_type           = "attribute"
        target_format         = "string"
        preserve_source       = true
        override_on_conflict  = false
        name                  = "sample attribute processor"
        is_enabled            = true
      }
      # Add other processors as needed
    ]
  },
  {
    name       = "sample pipeline 2"
    is_enabled = true
    filter     = { query = "source:python" }
    processors = [
      {
        type                  = "grok_parser"
        samples               = ["sample log 1"]
        source                = "message"
        grok                  = {
          support_rules = ""
          match_rules   = "Rule %%{word:my_word2} %%{number:my_float2}"
        }
        name                  = "sample grok parser"
        is_enabled            = true
      }
      # Add other processors as needed
    ]
  }
  # Add other pipelines as needed
]