Skip to content

Instantly share code, notes, and snippets.

@somoza
Forked from mesimeris/grok-patterns
Last active August 31, 2015 17:57
Show Gist options
  • Save somoza/d9294df61e8d02aaa9db to your computer and use it in GitHub Desktop.
Save somoza/d9294df61e8d02aaa9db to your computer and use it in GitHub Desktop.
LOGSTASH: syslog listener filtering with grok patterns and applying useful tags
# NOTE: These patterns take into account the additional log-line information passed to the logstash listener from rsyslog. YMMV.
DHCPD ((%{SYSLOGTIMESTAMP:timestamp})\s*(%{HOSTNAME:hostname})\s*dhcpd\S+\s*(%{WORD:dhcp_action})?.*[for|on] (%{IPV4:dhcp_client_ip})?.*[from|to] (%{COMMONMAC:dhcp_client_mac})?.*via (%{USERNAME:interface}))
IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s*(%{HOSTNAME:nf_host})\s*kernel\S+\s*(%{WORD:nf_action})?.*IN=(%{USERNAME:nf_in_interface})?.*OUT=(%{USERNAME:nf_out_interface})?.*MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*))
DNS ((%{MONTHDAY:day})-(%{MONTH:month})-(%{YEAR:year}) (%{TIME:timestamp}) client (%{IPV4:dns_client_ip})#(%{NONNEGINT:dns_uuid})?.*query: (%{HOSTNAME:dns_dest}) (%{WORD:dns_type}) (%{WORD:dns_record})?.*(%{IPV4:dns_server}))
PGSQL ((%{SYSLOGTIMESTAMP:pgsql_timestamp}) (%{HOSTNAME:pgsql_hostname})?.*SAST >(%{WORD:pgsql_severity}): (%{GREEDYDATA:pgsql_message}))
SQUID ((%{DATA:squid_timestamp_unix}) (%{INT:squid_time_elapsed_ms}) (%{IPV4:squid_client_ip}) (%{DATA:squid_action})/(%{INT:squid_reply_code}) (%{INT:squid_request_size}) (%{WORD:squid_method}) (%{URIPROTO:squid_request_protocol})://(%{DATA:squid_request_hostname}) - (%{WORD:squid_heirarchy})/(%{IPV4:squid_dest_ip}) (%{WORD:squid_content_type})/(%{WORD:squid_content_object}))
RADIUSINFO ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_messsage}))
RADIUSERROR ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{DATA:radius_module}): (%{GREEDYDATA:radius_messsage}))
RADIUSAUTHOKMAC ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}) (%{WORD}) (%{GREEDYDATA:radius_mac})\))
RADIUSAUTHOK ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))
RADIUSAUTHFAIL ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}): \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))
input {
syslog {
type => syslog
port => 514
}
}
filter {
# IPTABLES DROP/REJECT/ACCEPT
if [program] == "firewall-drop" {
grok {
break_on_match => true
match => { "message" => "DROP: " }
add_tag => "iptables"
add_tag => "iptables-drop"
}
}
if [program] == "firewall-reject" {
grok {
break_on_match => true
match => { "message" => "REJECT: " }
add_tag => "iptables"
add_tag => "iptables-reject"
}
}
if [program] == "firewall-accept" {
grok {
break_on_match => true
match => { "message" => "ACCEPT: " }
add_tag => "iptables"
add_tag => "iptables-accept"
}
}
if ("iptables" in [tags]) {
grok {
break_on_match => true
patterns_dir => "/etc/logstash/grok/iptables.pattern"
match => { "message" => "%{IPTABLES}" }
}
}
# SQUID
if [program] == "squid-access-log" {
grok {
patterns_dir => "/etc/logstash/grok/squid.pattern"
match => { message => "%{SQUID}" }
add_tag => "squid"
}
}
# DHCPD
if [program] == "dhcpd" {
grok {
break_on_match => true
match => { "message" => "DHCPDISCOVER" }
add_tag => "dhcp"
add_tag => "dhcp-discover"
}
grok {
break_on_match => true
match => { "message" => "DHCPOFFER" }
add_tag => "dhcp"
add_tag => "dhcp-offer"
}
grok {
break_on_match => true
match => { "message" => "DHCPREQUEST" }
add_tag => "dhcp"
add_tag => "dhcp-request"
}
grok {
break_on_match => true
match => { "message" => "DHCPDACK" }
add_tag => "dhcp"
add_tag => "dhcp-acknowledge"
}
grok {
break_on_match => true
match => { "message" => "DHCPNAK" }
add_tag => "dhcp"
add_tag => "dhcp-nak"
}
grok {
break_on_match => true
match => { "message" => "DHCPDECLINE" }
add_tag => "dhcp"
add_tag => "dhcp-decline"
}
grok {
break_on_match => true
match => { "message" => "DHCPRELEASE" }
add_tag => "dhcp"
add_tag => "dhcp-release"
}
grok {
break_on_match => true
match => { "message" => "DHCPINFORM" }
add_tag => "dhcp"
add_tag => "dhcp-inform"
}
}
if ("dhcp" in [tags]) {
grok {
break_on_match => true
patterns_dir => "/etc/logstash/grok/dhcpd.pattern"
match => { "message" => "%{DHCPD}" }
}
}
# DNS
if [program] == "dns-queries" {
grok {
break_on_match => true
patterns_dir => "/etc/logstash/grok/named.pattern"
match => { "message" => "%{DNS}" }
}
}
# PGSQL
if [program] == "pgsql" {
grok {
match => { "message" => ">PANIC: " }
add_tag => "pgsql"
add_tag => "pgsql-panic"
}
grok {
match => { "message" => ">ERROR: " }
add_tag => "pgsql"
add_tag => "pgsql-error"
}
grok {
match => { "message" => ">FATAL: " }
add_tag => "pgsql"
add_tag => "pgsql-fatal"
}
grok {
patterns_dir => "/etc/logstash/grok/pgsql.pattern"
match => { "message" => ">LOG: " }
add_tag => "pgsql"
add_tag => "pgsql-log"
}
}
if ("pgsql" in [tags]) {
grok {
patterns_dir => "/etc/logstash/grok/pgsql.pattern"
match => { "message" => "%{PGSQL}" }
}
}
# RADIUS
if [program] == "radius" {
grok {
match => { "message" => " Info: " }
add_tag => "radius"
add_tag => "radius-info"
}
grok {
match => { "message" => " Auth: Login OK" }
add_tag => "radius"
add_tag => "radius-auth-ok"
}
grok {
match => { "message" => " Auth: Login incorrect" }
add_tag => "radius"
add_tag => "radius-auth-fail"
}
grok {
match => { "message" => " Error: " }
add_tag => "radius"
add_tag => "radius-error"
}
}
if ("radius-info" in [tags]) {
grok {
patterns_dir => "/etc/logstash/grok/radius.pattern"
match => { "message" => "%{RADIUSINFO}" }
}
}
if ("radius-auth-ok" in [tags]) {
grok {
match => { "message" => " cli " }
add_tag => "radius-mac"
}
}
if ("radius-mac" in [tags]) {
grok {
break_on_match => true
patterns_dir => "/etc/logstash/grok/radius.pattern"
match => { "message" => "%{RADIUSAUTHOKMAC}" }
remove_tag => "radius-auth-ok"
}
}
if ("radius-auth-ok" in [tags]) {
grok {
patterns_dir => "/etc/logstash/grok/radius.pattern"
match => { "message" => "%{RADIUSAUTHOK}" }
}
}
if ("radius-auth-fail" in [tags]) {
grok {
patterns_dir => "/etc/logstash/grok/radius.pattern"
match => { "message" => "%{RADIUSAUTHFAIL}" }
}
}
if ("radius-error" in [tags]) {
grok {
patterns_dir => "/etc/logstash/grok/radius.pattern"
match => { "message" => "%{RADIUSERROR}" }
}
}
}
output {
redis {
host => "127.0.0.1"
data_type => "list"
key => "logstash"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment