logstash configuration file

mini.conf

input {
  beats {
    # The port to listen on
    port => 5055
    
    # ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    # ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"

  }
}

filter {
  if [log_type] == "nmon" {
    grok {
      match => {"message" => "%{WORD:stat},%{WORD:snap},%{BASE10NUM:UserP},%{BASE10NUM:SysP},%{BASE10NUM:WaitP},%{BASE10NUM:IdleP},%{DATA},%{INT:cpu}"}
    }
    mutate {
      convert => {
        "UserP" => "float"
        "SysP" => "float"
        "WaitP" => "float"
        "IdleP" => "float"
        "cpu" => "integer"
      }
    }
  }

  if [log_type] == "iwd_access" or [log_type] == "psm_access" {
    grok {
      match => {"message" => "%{DATA:local_interface} %{DATA:request_originator}?%{SPACE}?*\[%{DATA:time_stamp}\] %{NUMBER:remote_port}:%{NUMBER:local_port} %{GREEDYDATA} \"%{WORD:verb} %{URI:resource} %{DATA:param}\" %{DATA:response_code} %{DATA:response_size} %{GREEDYDATA:response_time}"}
    }

     mutate {
      convert => {
        "remote_port" => "integer"
        "local_port" => "integer"
        "response_size" => "integer"
        "response_code" => "integer"
        "response_time" => "integer"
        }
    }
    date {
      match => [ "time_stamp", "dd/MMM/YYYY:HH:mm:ss Z"]
      remove_field => [ "time_stamp" ]
    }

  }

  if [log_type] == "ipas.async" {
    grok {
      match => {"message" => "%{DATA:jvm} \[%{DATA:time_stamp}\] %{DATA:pid} %{DATA:class} \| %{GREEDYDATA:body}"}
      add_field => {"fulltime" => "%{time_stamp} UTC"}
    }
    if [shipper] == "fit-4" {
    date {
      match => [ "time_stamp", "MM-dd-yy HH:mm:ss"]
      timezone => "Etc/GMT"
      remove_field => [ "time_stamp", "full_time" ]
    }
    }


    else if [shipper] == "pasw-1" {

    date {
      match => [ "time_stamp", "MM-dd-yy HH:mm:ss"]
      timezone => "EST"
      remove_field => [ "time_stamp", "full_time" ]
    }


    }
#    mutate {
#            convert => {
#                "pid" => "integer"
#           }
#    }
  }

  if [log_type] == "ipas.placement" {
    grok {
          match => {"message" => ".*?{APAVMDataImpl id={APAID id=%{DATA:VMPlacementID}, ty={%{DATA:PlacementType}}, dn=%{DATA:VMPlacementName}}.*?u=N.*?res=%{BASE10NUM:CPUReservation}.*?usg=%{BASE10NUM:CPUUsage}.*?u=MB.*?res=%{BASE10NUM:RAMReservation}.*?sf.s=%{DATA:state}}.*"}
          match => {"message" => ".*?{APAPMDataImpl id={APAID id=%{DATA:CNPlacementID}, ty={%{DATA:PlacementType}}, dn=%{DATA:CNPlacementName}}.*?u=N.*?cap=%{BASE10NUM:CPUCap}, pUsage=%{BASE10NUM:CPUUsed}, Headroom=%{BASE10NUM:CPUHeadroom}.*?u=MB.*?cap=%{BASE10NUM:RAMCap}, pUsage=%{BASE10NUM:RAMUsed}, Headroom=%{BASE10NUM:RAMHeadroom}}.*?rpf.pc=\[.*?%{DATA:VMPlacementIDs}\].*? sf.p=%{DATA:Power_State}, sf.s=%{DATA:State},.*"}
        }
        mutate {
          convert => {
                "CPUReservation" => "float"
                "CPUUsage" => "float"
                "RAMReservation" => "float"
                "CPUCap" => "float"
                "CPUUsed" => "float"
                "CPUHeadroom" => "float"
                "RAMCap" => "float"
                "RAMUsed" => "float"
                "RAMHeadroom" => "float"

          }
        }
    if [PlacementType] == "APAPMType" {
      ruby {
        code => "
          begin
            if event['VMPlacementIDs']
              event['VMPlacementIDs'] = event['VMPlacementIDs'].scan(/id\=(.*?)\,/).flatten
            end
          end
          "}
   }
  }
  if "_grokparsefailure" in [tags] or "_dateparsefailure" in [tags] {
    drop {}
  }

}

output {
#  if [log_type] == "ipas.placement"  {
#    stdout {  codec => rubydebug }
#  }
  elasticsearch {
    hosts => ["es1", "es2"]
    sniffing => true
    #workers => 8
    index => "psm-%{+YYYY.MM.dd}"
  }
}