Corelight custom pipelines when using Elastic Agent & the Custom TCP integration

Corelight custom pipelines when using Elastic Agent & the Custom TCP integration

# Creates and updates BT agent policy
resource "null_resource" "corelight_ingest_pipelines" {
  depends_on = [ec_deployment.blue_teams_deployments]
  count      = var.deployment_count

  triggers = {
    always_run = "${timestamp()}"
  }

  provisioner "local-exec" {
    command = "./resources/elastic/elasticsearch/scripts/load_ingest_pipelines.sh -e ${ec_deployment.blue_teams_deployments[count.index].elasticsearch.https_endpoint} -u ${ec_deployment.blue_teams_deployments[count.index].elasticsearch_username} -s ${ec_deployment.blue_teams_deployments[count.index].elasticsearch_password} -i ./resources/corelight/ingest_pipelines/ecs-mapping/pipeline"
  }
}


resource "elasticstack_elasticsearch_ingest_pipeline" "corelight_main_custom" {
  depends_on = [ec_deployment.blue_teams_deployments]
  count      = var.deployment_count

  name        = "corelight-ecs-main-pipeline@custom"
  description = "JSON decoder pipeline for Corelight logs"

  // processors can be defined in different way
  processors = [
    // using the jsonencode function, which is the recommended way if you want to provide JSON object by yourself
    jsonencode({
      set = {
        field = "original_index"
        copy_from = "_index"
      },
      remove = {
        description = "Remove event.dataset"
        field       = "event.dataset"
        "ignore_missing" = true
      },
      json = {
        field       = "message"
        add_to_root = true
      }
    })
  ]
  on_failure = [
    jsonencode({
        set = {
            field = "error.message"
            value = "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}"
        }
    })
  ]
  elasticsearch_connection {
    endpoints = ["${ec_deployment.blue_teams_deployments[count.index].elasticsearch.https_endpoint}"]
    username  = ec_deployment.blue_teams_deployments[count.index].elasticsearch_username
    password  = ec_deployment.blue_teams_deployments[count.index].elasticsearch_password
  }
}


resource "elasticstack_elasticsearch_ingest_pipeline" "corelight_main_final_custom" {
  depends_on = [ec_deployment.blue_teams_deployments]
  count      = var.deployment_count


  name        = "corelight-ecs-postprocess-final-main-pipeline@custom"
  description = "JSON decoder pipeline for Corelight logs"

  // processors can be defined in different way
  processors = [
    jsonencode({
      set = {
        field = "_index"
        value = "{{{original_index}}}"
      }
    }),

    jsonencode({  
      set = {
        field = "data_stream.type"
        value = "logs"
      }
    }),
    jsonencode({
      set = {
        field = "data_stream.dataset"
        value = "corelight"
      }
    }),
    jsonencode({
      set = {
        field = "event.category"
        value = "network"
      }
    }),
    jsonencode({
      set = {
        field = "event.kind"
        value = "event"
      }
    }),
    jsonencode({
      set = {
        field = "event.type"
        value = "connection"
      }
    }),
    jsonencode({
      convert = {
        field = "dns.answers.ttl"
        type = "long"
        ignore_missing = true
        ignore_failure = true
      }
    }),
    jsonencode({
      convert = {
        field = "dns.id"
        type = "string"
        ignore_missing = true
      }
    }),
    jsonencode ({
      remove = {
        field = [
          "tags",
          "original_index",
          "message"
        ]
        "ignore_missing" = true
      }

    })
  ]

  elasticsearch_connection {
    endpoints = ["${ec_deployment.blue_teams_deployments[count.index].elasticsearch.https_endpoint}"]
    username  = ec_deployment.blue_teams_deployments[count.index].elasticsearch_username
    password  = ec_deployment.blue_teams_deployments[count.index].elasticsearch_password
  }
}


resource "elasticstack_elasticsearch_ingest_pipeline" "corelight_main_failures_custom" {
  depends_on = [ec_deployment.blue_teams_deployments]
  count      = var.deployment_count


  name        = "corelight-ecs-postprocess-parse_failures-pipeline@custom"
  description = "JSON decoder pipeline for Corelight logs"

  // processors can be defined in different way
  processors = [
    jsonencode({
      append = {
        field = "tags"
        value = [
          "failure-custom"
        ]
      }
    })
  ]

  elasticsearch_connection {
    endpoints = ["${ec_deployment.blue_teams_deployments[count.index].elasticsearch.https_endpoint}"]
    username  = ec_deployment.blue_teams_deployments[count.index].elasticsearch_username
    password  = ec_deployment.blue_teams_deployments[count.index].elasticsearch_password
  }
}