logstash.conf

input {
    file {
        path => "./logs/*.log"
        start_position => "beginning"
    }
   
}

filter {

    grok {
        add_tag => ["valid"]
        match => { "message" => "%{TIMESTAMP_ISO8601:time} \[%{LOGLEVEL:level}\] %{GREEDYDATA:contents}" }
    }

    if "valid" not in [tags] {            
        drop {}
    }
    
    mutate {
        remove_tag => [ "valid" ]
    }

    date {
        
        match => ["time", "ISO8601"]
        remove_field => ["time"]
    }
    
    # for remove that duplicate log
    fingerprint {
        method => "SHA1"
        key => "KEY"
    }

}

output {
    elasticsearch {
        hosts => ["localhost:9200"]
        index => "log"
        document_id => "%{fingerprint}"
        
    }
    stdout { codec => rubydebug }
}