Dropping duplicate events in Logstash

dedup.conf

# Dropping duplicate events in Logstash
#
# Explanation:
# - Add a hashed field with the anonymize filter (it's fast)
# - ES docs are unique per index/docid, duplicates will be overwritten
# - Set ES "document_id" field when submitting
#
# Caveats:
# - Adds a nonsense field to your events. Annoying but harmless.
# - Make sure that whatever field(s) you hash are unique!

filter {
    mutate {
        add_field => ["docId", "%{message}"]
    }
    anonymize {
        fields => ["docId"]
        key => "pianoPlayingMonkey"
    }
}

output {
    elasticsearch {
        document_id => "%{docId}"
    }
}