filter

gistfile1.txt

filter {
        if [type] == "syslog" {
                grok {
         match => [ "message", "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:@timestamp} (?<hostname>[a-zA-Z0-9\-_]+[.][a-zA-Z0-9\-_\.]+) %{GREEDYDATA:raw_message}" ]
                                }
        }
		if ([message] =~ /,SYSTEM,/) {
			mutate {
				add_tag => "SYSTEM"
				}
			}
		if ([message] =~ /,TRAFFIC,/) {
			mutate {
				add_tag => "TRAFFIC"
				}
			}
    if [tag] == "SYSTEM" {
	csv {
		source => "raw_message"
		columns => [ "FUTURE_USE", "ReceiveTime","SerialNum","Type","Threat-ContentType","Log-Level","GeneratedTime","VirtualSystem","EventID","Object","Module","Module2","LogType","Severity","Description","LogNumber","SequenceNumber","dg_hier_level_1","dg_hier_level_2","dg_hier_level_3","dg_hier_level_4","VirtualSystemName","DeviceName" ]
      }
	} else if [tag] == "TRAFFIC" {
		csv {
			source => "raw_message"
			columns => [ "ReceiveTime","SerialNumber","Type","Threat-ContentType","FUTURE_USE","GeneratedTime","SourceIP","DestinationIP","NATSourceIP","NATDestinationIP","RuleName","SourceUser","DestinationUser","Application","VirtualSystem","SourceZone","DestinationZone","InboundInterface","OutboundInterface","LogAction","FUTURE_USE","SessionID","Repeat Count","SourcePort","DestinationPort","NATSourcePort","NATDestinationPort","Flags","Protocol","Action","Bytes","BytesSent","BytesReceived","Packets","StartTime","ElapsedTimeInSec","Category","FUTURE_USE","SequenceNumber","ActionFlags","SourceLocation","DestinationLocation","FUTURE_USE","PacketsSent","PacketsReceived","SessionEndReason","dg_hier_level_1","dg_hier_level_2","dg_hier_level_3","dg_hier_level_4","VirtualSystemName","DeviceName","ActionSource","SourceVM-UUID","DestinationVM-UUID","TunnelID-IMSI","MonitorTag-IMEI","ParentSessionID","ParentStartTime","TunnelType" ]
			}
		mutate {
		convert => [ "Bytes", "integer" ]
		convert => [ "BytesReceived", "integer" ]
		convert => [ "BytesSent", "integer" ]
		convert => [ "ElapsedTimeInSec", "integer" ]
		convert => [ "geoip.area_code", "integer" ]
		convert => [ "geoip.dma_code", "integer" ]
		convert => [ "geoip.latitude", "float" ]
		convert => [ "geoip.longitude", "float" ]
		convert => [ "NATDestinationPort", "integer" ]
		convert => [ "NATSourcePort", "integer" ]
		convert => [ "Packets", "integer" ]
		convert => [ "PacketsReceived", "integer" ]
		convert => [ "PacketsSent", "integer" ]
		convert => [ "SequenceNumber", "integer" ]
		gsub => [ "Rule", " ", "_",
				"Application", "( |-)", "_" ]
		remove_field => [ "message", "raw_message" ]
		}
		if [SourceAddress] and [SourceAddress] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
			geoip {
				source => "SourceAddress"
				target => "SourceGeo"
				}
			}
		if ([SourceGeo.location] and [SourceGeo.location] =~ "0,0") {
			mutate {
				replace => [ "SourceGeo.location", "" ]
				}
			}
		if [DestinationAddress] and [DestinationAddress] !~ "(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)|(^169\.254\.)" {
			geoip {
				source => "DestinationAddress"
				target => "DestinationGeo"
			}
		}
		if ([DestinationGeo.location] and [DestinationGeo.location] =~ "0,0") {
			mutate {
				replace => [ "DestinationAddress.location", "" ]
			}
		}
	}
}