experimental checksumming for logstash events as a filter

checksum.rb

require "logstash/filters/base"
require "logstash/namespace"
require "yaml"

class LogStash::Filters::Checksum < LogStash::Filters::Base

  config_name "checksum"
  plugin_status "experimental"

  # A list of keys to use in creating the string to checksum
  # Keys will be sorted before building the string
  # keys and values will then be concatenated with pipe delimeters
  # and checksummed
  config :keys, :validate => :array, :default => ["@message", "@source_host", "@timestamp", "@source_path", "@type", "@source"]

  config :algorithm, :validate => ["md5", "sha128", "sha256", "sha384"], :default => "sha256"

  public
  def register
    require 'openssl'
    @to_checksum = ""
  end

  public
  def filter(event)
    return unless filter?(event)

    @logger.debug("Running checksum filter", :event => event)

    @keys.sort.each do |k|
      @logger.debug("Adding key to string", :current_key => k)
      @to_checksum << "|#{k}|#{event[k]}"
    end
    @to_checksum << "|"
    @logger.debug("Final string built", :to_checksum => @to_checksum)

    digested_string = OpenSSL::Digest.hexdigest(@algorithm, @to_checksum)
    @logger.debug("Digested string", :digested_string => digested_string)
    event.fields['logstash_checksum'] = digested_string
  end
end

cli.sh

jvstratusmbp :: ~/development/logstash ‹master*› » ruby --1.9 bin/logstash -f logstash.conf    
{:args=>["agent", "-f", "logstash.conf"]}
{:run=>"agent"}
{:remaining=>[]}
doneargs
Using experimental plugin 'checksum'. This plugin is untested and may change in the future. For more information about plugin statuses, see http://logstash.net/docs/1.1.0.1/plugin-status  {"level":"warn"}
foobarbangbaz

event.json

{
"@source":"stdin://jvstratusmbp.local/",
"@type":"stdin",
"@tags":[],
"@fields":{
"logstash_checksum":"34092978fb4055baa980815768bddc5342d601b59b48cd147ee44447ceff6929"
},
"@timestamp":"2012-06-02T07:00:48.045000Z",
"@source_host":"jvstratusmbp.local",
"@source_path":"/",
"@message":"foobarbangbaz"
}

logstash.conf

input { stdin { type => 'stdin' } }
filter { checksum { } }
output { stdout { debug => true debug_format => "json" }}

• sorting keys makes sense
• I agree on making keys a list of fields.

Nice work (like everything else in Logstash). Thanks!

If you need to calculate hash for a single field, you could use the following ruby code:

    ruby {
      code => "event.to_hash.merge!('message_hash' => OpenSSL::Digest.hexdigest('md5', event['message']))"
    }