Logstash Pipeline for filtering and parsing OpnSense Filterlog Output

logstash_pipeline_opnsense_filterlog.md

• I applied these rules according to https://github.com/opnsense/ports/blob/master/opnsense/filterlog/files/description.txt

input {
  tcp {
    port => 5000
    type => syslog
  }
  udp {
    port => 5000
    type => syslog
  }
}

filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }

  if [syslog_program] == "filterlog" {
    grok {
      match => { "message" => "[^:]*: %{GREEDYDATA:filterlog_csv}"}
    }

    mutate {
      split => {"filterlog_csv" => "," }
    }

    mutate {
      add_field => {
        "filter_rulenr" =>      "%{[filterlog_csv][0]}"
        "filter_subrulenr" =>   "%{[filterlog_csv][1]}"
        "filter_anchorname" =>  "%{[filterlog_csv][2]}"
        "filter_label" =>       "%{[filterlog_csv][3]}"
        "filter_interface" =>   "%{[filterlog_csv][4]}"
        "filter_reason" =>      "%{[filterlog_csv][5]}"
        "filter_action" =>      "%{[filterlog_csv][6]}"
        "filter_dir" =>         "%{[filterlog_csv][7]}"
        "filter_ipversion" =>   "%{[filterlog_csv][8]}"
        "filter_tos" =>         "%{[filterlog_csv][9]}"
        "filter_ecn" =>         "%{[filterlog_csv][10]}"
        "filter_ttl" =>         "%{[filterlog_csv][11]}"
        "filter_id" =>          "%{[filterlog_csv][12]}"
        "filter_offset" =>      "%{[filterlog_csv][13]}"
        "filter_flags" =>       "%{[filterlog_csv][14]}"
        "filter_protonum" =>    "%{[filterlog_csv][15]}"
        "filter_protoname" =>   "%{[filterlog_csv][16]}"
        "filter_length" =>      "%{[filterlog_csv][17]}"
        "filter_src" =>         "%{[filterlog_csv][18]}"
        "filter_dst" =>         "%{[filterlog_csv][19]}"
      }
    }
    if [filter_protoname] == "tcp" {
      mutate {
        add_field => {
          "filter_prot_srcport" => "%{[filterlog_csv][20]}"
          "filter_prot_dstport" => "%{[filterlog_csv][21]}"
          "filter_prot_datalen" => "%{[filterlog_csv][22]}"
          "filter_prot_flags" =>   "%{[filterlog_csv][23]}"
          "filter_prot_seq" =>     "%{[filterlog_csv][24]}"
          "filter_prot_ack" =>     "%{[filterlog_csv][25]}"
          "filter_prot_window" =>  "%{[filterlog_csv][26]}"
          "filter_prot_urg" =>     "%{[filterlog_csv][27]}"
          "filter_prot_options" => "%{[filterlog_csv][28]}"
        }
      }
    }
    if [filter_protoname] == "udp" {
      mutate {
        add_field => {
          "filter_prot_srcport" => "%{[filterlog_csv][20]}"
          "filter_prot_dstport" => "%{[filterlog_csv][21]}"
          "filter_prot_datalen" => "%{[filterlog_csv][22]}"
        }
      }  
    }
    mutate {
      remove_field => [ "filterlog_csv" ]
      remove_field => [ "syslog_message" ]
      remove_field => [ "message" ]
    }
    geoip {
      source => "filter_src"
    }
 
  }
}