logstash config

logstash

input {
    beats {
        type => beats
        port => 5060
    }
}

filter {
    dissect {
        mapping => {
            "message" => "%{timestamp} | %{level} | %{thread} | %{category} | %{message}"
        }
    }

    date {
        match => ["timestamp", "ISO8601", "YYYY-MM-dd HH:mm:ss,SSS"]
    }

    mutate {
        copy => {
            "message" => "message_copy"
        }
    }

    grok {
        match => {
            "message_copy" => ["%{JSON:payload_raw}"]
        }
        pattern_definitions => {
            "JSON" => "{.*$"
        }
    }

    json {
        source => "payload_raw"
        target => "payload"
    }

    mutate {
        remove_field => ["payload_raw", "message_copy"]
    }

    grok {
        match => {
            "message" => [
                "Final userId : %{GREEDYDATA:userId} "
            ]
        }
        add_tag => ["user_id_processed"]
    }

    grok {
        match => {
            "message" => [
                "Form ABC API loginId %{GREEDYDATA:loginId} classNo %{GREEDYDATA:classNo} "
            ]
        }
        add_tag => ["login_id_processed"]
    }
}

output {
    stdout {
        codec => rubydebug
    }
    elasticsearch {
        hosts => ["https://elastic_search_instance.com:9999"]
        user => "elastic11"
        password => "pass11"
        index => "test-api-node1-%{+YYYY.MM.dd}"
    }
}


# sample data
2019-10-24 14:37:37,329 | INFO  | -nio-8300-exec-7 | .s.g.s.APIServiceImpl | Response from API (API response message): {
      "ContainerId": "c1e77181-1f1c-4355-9f0e-33b5fad6a308",
      "UserId": "82cc5179-3785-45cf-9cff-00fe90896274",
      "LoginId": "testloginId",
      "ClassNo": "a54df09f-5ee6-4a4c-a8b2-87004aaa3540"
}

2019-10-24 14:37:37,329 | INFO  | -nio-8300-exec-7 | .s.g.s.APIServiceImpl | Final userId : 82cc5179-3785-45cf-9cff-00fe90896274
2019-10-24 14:37:37,329 | INFO  | -nio-8300-exec-7 | .s.g.s.APIServiceImpl | Form ABC API loginId testloginId classNo a54df09f-5ee6-4a4c-a8b2-87004aaa3540