LOGSTASH: syslog listener filtering with grok patterns and applying useful tags

grok-patterns

# NOTE: These patterns take into account the additional log-line information passed to the logstash listener from rsyslog. YMMV.

DHCPD ((%{SYSLOGTIMESTAMP:timestamp})\s*(%{HOSTNAME:hostname})\s*dhcpd\S+\s*(%{WORD:dhcp_action})?.*[for|on] (%{IPV4:dhcp_client_ip})?.*[from|to] (%{COMMONMAC:dhcp_client_mac})?.*via (%{USERNAME:interface}))
IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s*(%{HOSTNAME:nf_host})\s*kernel\S+\s*(%{WORD:nf_action})?.*IN=(%{USERNAME:nf_in_interface})?.*OUT=(%{USERNAME:nf_out_interface})?.*MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*))
DNS ((%{MONTHDAY:day})-(%{MONTH:month})-(%{YEAR:year}) (%{TIME:timestamp}) client (%{IPV4:dns_client_ip})#(%{NONNEGINT:dns_uuid})?.*query: (%{HOSTNAME:dns_dest}) (%{WORD:dns_type}) (%{WORD:dns_record})?.*(%{IPV4:dns_server}))
PGSQL ((%{SYSLOGTIMESTAMP:pgsql_timestamp}) (%{HOSTNAME:pgsql_hostname})?.*SAST >(%{WORD:pgsql_severity}):  (%{GREEDYDATA:pgsql_message}))
SQUID ((%{DATA:squid_timestamp_unix}) (%{INT:squid_time_elapsed_ms}) (%{IPV4:squid_client_ip}) (%{DATA:squid_action})/(%{INT:squid_reply_code}) (%{INT:squid_request_size}) (%{WORD:squid_method}) (%{URIPROTO:squid_request_protocol})://(%{DATA:squid_request_hostname}) - (%{WORD:squid_heirarchy})/(%{IPV4:squid_dest_ip}) (%{WORD:squid_content_type})/(%{WORD:squid_content_object}))
RADIUSINFO ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_messsage}))
RADIUSERROR ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{DATA:radius_module}): (%{GREEDYDATA:radius_messsage}))
RADIUSAUTHOKMAC ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}) (%{WORD}) (%{GREEDYDATA:radius_mac})\))
RADIUSAUTHOK ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))
RADIUSAUTHFAIL ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}): \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))

logstash-rsyslog.conf

input {
  syslog {
    type => syslog
    port => 514
  }
}

filter {

# IPTABLES DROP/REJECT/ACCEPT

  if [program] == "firewall-drop" {
    grok {
        break_on_match => true
        match => { "message" => "DROP: " }
	add_tag => "iptables"
	add_tag => "iptables-drop"
	}
  }
  if [program] == "firewall-reject" {
    grok {
        break_on_match => true
        match => { "message" => "REJECT: " }
	add_tag => "iptables"	
	add_tag => "iptables-reject"
	}
  }
  if [program] == "firewall-accept" {
    grok {
        break_on_match => true
        match => { "message" => "ACCEPT: " }
	add_tag => "iptables"
	add_tag => "iptables-accept"
	}
  }

  if ("iptables" in [tags]) {
    grok {
	break_on_match => true
	patterns_dir => "/etc/logstash/grok/iptables.pattern"
        match => { "message" => "%{IPTABLES}" }
         }
  }

# SQUID

  if [program] == "squid-access-log" {
    grok {
        patterns_dir => "/etc/logstash/grok/squid.pattern"
        match => { message => "%{SQUID}" }
        add_tag => "squid"
    }
  }


# DHCPD

  if [program] == "dhcpd" {
    grok {
	break_on_match => true
	match => { "message" => "DHCPDISCOVER" }
	add_tag => "dhcp"
	add_tag => "dhcp-discover"
	}
    grok {
        break_on_match => true
        match => { "message" => "DHCPOFFER" }
        add_tag => "dhcp"
        add_tag => "dhcp-offer"
        }
    grok {
        break_on_match => true
        match => { "message" => "DHCPREQUEST" }
        add_tag => "dhcp"
        add_tag => "dhcp-request"
        }
    grok {
        break_on_match => true
        match => { "message" => "DHCPDACK" }
        add_tag => "dhcp"
        add_tag => "dhcp-acknowledge"
        }
    grok {
        break_on_match => true
        match => { "message" => "DHCPNAK" }
        add_tag => "dhcp"
        add_tag => "dhcp-nak"
        }
    grok {
        break_on_match => true
        match => { "message" => "DHCPDECLINE" }
        add_tag => "dhcp"
        add_tag => "dhcp-decline"
        }
    grok {
        break_on_match => true
        match => { "message" => "DHCPRELEASE" }
        add_tag => "dhcp"
        add_tag => "dhcp-release"
        }
    grok {
        break_on_match => true
        match => { "message" => "DHCPINFORM" }
        add_tag => "dhcp"
        add_tag => "dhcp-inform"
        }
  }
    if ("dhcp" in [tags]) {
      grok {
	  break_on_match => true
	  patterns_dir => "/etc/logstash/grok/dhcpd.pattern"
	  match => { "message" => "%{DHCPD}" }
         }
    }

# DNS

  if [program] == "dns-queries" {
    grok {
	break_on_match => true
	patterns_dir => "/etc/logstash/grok/named.pattern"
	match => { "message" => "%{DNS}" }
	}
  }

# PGSQL

  if [program] == "pgsql" {
    grok {
      match => { "message" => ">PANIC: " }
      add_tag => "pgsql"
      add_tag => "pgsql-panic"
    }
    grok {
      match => { "message" => ">ERROR: " }
      add_tag => "pgsql"
      add_tag => "pgsql-error"
    }
    grok {
      match => { "message" => ">FATAL: " }
      add_tag => "pgsql"
      add_tag => "pgsql-fatal"
    }
    grok {
      patterns_dir => "/etc/logstash/grok/pgsql.pattern"
      match => { "message" => ">LOG: " }
      add_tag => "pgsql"
      add_tag => "pgsql-log"
    }
  }
    if ("pgsql" in [tags]) {
      grok {
	patterns_dir => "/etc/logstash/grok/pgsql.pattern"
        match => { "message" => "%{PGSQL}" }
         }
    }

# RADIUS

  if [program] == "radius" {
    grok {
      match => { "message" => " Info: " }
      add_tag => "radius"
      add_tag => "radius-info"
    }
    grok {
      match => { "message" => " Auth: Login OK" }
      add_tag => "radius"
      add_tag => "radius-auth-ok"
    }
    grok {
      match => { "message" => " Auth: Login incorrect" }
      add_tag => "radius"
      add_tag => "radius-auth-fail"
    }
    grok {
      match => { "message" => " Error: " }
      add_tag => "radius"
      add_tag => "radius-error"
    }
  }
    if ("radius-info" in [tags]) {
      grok {
        patterns_dir => "/etc/logstash/grok/radius.pattern"
        match => { "message" => "%{RADIUSINFO}" }
      }
    }
    if ("radius-auth-ok" in [tags]) {
      grok {
          match => { "message" => " cli " }
          add_tag => "radius-mac"
      }
    }
    if ("radius-mac" in [tags]) {
      grok {
            break_on_match => true
            patterns_dir => "/etc/logstash/grok/radius.pattern"
            match => { "message" => "%{RADIUSAUTHOKMAC}" }
            remove_tag => "radius-auth-ok"
          }
      }
    if ("radius-auth-ok" in [tags]) {
      grok {
        patterns_dir => "/etc/logstash/grok/radius.pattern"
        match => { "message" => "%{RADIUSAUTHOK}" }
      }
    }
    if ("radius-auth-fail" in [tags]) {
      grok {
        patterns_dir => "/etc/logstash/grok/radius.pattern"
        match => { "message" => "%{RADIUSAUTHFAIL}" }
      }
    }
    if ("radius-error" in [tags]) {
      grok {
        patterns_dir => "/etc/logstash/grok/radius.pattern"
        match => { "message" => "%{RADIUSERROR}" }
      }
    }
}  
output {
    redis {
      host => "127.0.0.1"
      data_type => "list"
      key => "logstash"
    }
}