Skip to content

Instantly share code, notes, and snippets.

@nipunarora

nipunarora/gist:13c240f866fef12e6e7b

Created April 30, 2015 20:25
Show Gist options
  • Save nipunarora/13c240f866fef12e6e7b to your computer and use it in GitHub Desktop.
Save nipunarora/13c240f866fef12e6e7b to your computer and use it in GitHub Desktop.
Download ZIP
Configuration
Raw
gistfile1.txt
#Logstash Header Start
input {
stdin {
type => "access_log_train"
}
}
#Logstash Header End
filter {
if [type] == "access_log_train" {
#Patterns Start:1
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P1F1} HTTP / %{BASE16NUM:P1F2} \. %{BASE16NUM:P1F3} \" %{BASE16NUM:P1F4} \- %{BASE16NUM:P1F5} %{BASE16NUM:P1F6} %{BASE16NUM:P1F7} User \- agent : Java / %{BASE16NUM:P1F8} \. %{BASE16NUM:P1F9} \. %{BASE16NUM:P1F10} _ %{BASE16NUM:P1F11} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "1" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P3F1} HTTP / %{BASE16NUM:P3F2} \. %{BASE16NUM:P3F3} \" %{BASE16NUM:P3F4} \- %{BASE16NUM:P3F5} %{BASE16NUM:P3F6} %{BASE16NUM:P3F7} User \- agent : Java / %{BASE16NUM:P3F8} \. %{BASE16NUM:P3F9} \. %{BASE16NUM:P3F10} _ %{BASE16NUM:P3F11} \- icedtea Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "3" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P4F1} HTTP / %{BASE16NUM:P4F2} \. %{BASE16NUM:P4F3} \" %{BASE16NUM:P4F4} \- %{BASE16NUM:P4F5} %{BASE16NUM:P4F6} %{BASE16NUM:P4F7} User \- agent : Mozilla / %{BASE16NUM:P4F8} \. %{BASE16NUM:P4F9} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "4" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P6F1} HTTP / %{BASE16NUM:P6F2} \. %{BASE16NUM:P6F3} \" %{BASE16NUM:P6F4} %{NOTSPACE:P6NS1} %{BASE16NUM:P6F5} %{BASE16NUM:P6F6} %{BASE16NUM:P6F7} User \- agent : Mozilla / %{BASE16NUM:P6F8} \. %{BASE16NUM:P6F9} \( compatible ; MSIE %{BASE16NUM:P6F10} \. %{BASE16NUM:P6F11} ; Windows NT %{BASE16NUM:P6F12} \. %{BASE16NUM:P6F13} (.*); Trident / %{BASE16NUM:P6F14} \. %{BASE16NUM:P6F15} \) Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "6" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P7F1} HTTP / %{BASE16NUM:P7F2} \. %{BASE16NUM:P7F3} \" %{BASE16NUM:P7F4} \- %{BASE16NUM:P7F5} %{BASE16NUM:P7F6} %{BASE16NUM:P7F7} User \- agent : Mozilla / %{BASE16NUM:P7F8} \. %{BASE16NUM:P7F9} \( compatible ; MSIE %{BASE16NUM:P7F10} \. %{BASE16NUM:P7F11} ; Windows NT %{BASE16NUM:P7F12} \. %{BASE16NUM:P7F13} ; InfoPath \. %{BASE16NUM:P7F14} ; \. NET CLR %{BASE16NUM:P7F15} \. %{BASE16NUM:P7F16} \. %{BASE16NUM:P7F17} ; \. NET CLR %{BASE16NUM:P7F18} \. %{BASE16NUM:P7F19} \. %{BASE16NUM:P7F20} ; MS \- RTC LM %{BASE16NUM:P7F21} ; \. NET CLR %{BASE16NUM:P7F22} \. %{BASE16NUM:P7F23} \. %{BASE16NUM:P7F24} \. %{BASE16NUM:P7F25} ; \. NET CLR %{BASE16NUM:P7F26} \. %{BASE16NUM:P7F27} \. %{BASE16NUM:P7F28} \) Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "7" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P2F1} HTTP / %{BASE16NUM:P2F2} \. %{BASE16NUM:P2F3} \" %{BASE16NUM:P2F4} %{BASE16NUM:P2F5} %{BASE16NUM:P2F6} %{BASE16NUM:P2F7} %{BASE16NUM:P2F8} User \- agent : Java / %{BASE16NUM:P2F9} \. %{BASE16NUM:P2F10} \. %{BASE16NUM:P2F11} _ %{BASE16NUM:P2F12} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "2" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* HTTP / %{BASE16NUM:P5F1} \. %{BASE16NUM:P5F2} \" %{BASE16NUM:P5F3} \- %{BASE16NUM:P5F4} %{BASE16NUM:P5F5} %{BASE16NUM:P5F6} User \- agent : Java / %{BASE16NUM:P5F7} \. %{BASE16NUM:P5F8} \. %{BASE16NUM:P5F9} _ %{BASE16NUM:P5F10} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "5" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P8F1} HTTP / %{BASE16NUM:P8F2} \. %{BASE16NUM:P8F3} \" %{BASE16NUM:P8F4} \- %{BASE16NUM:P8F5} %{BASE16NUM:P8F6} %{BASE16NUM:P8F7} User \- agent : %{WORD:P8W1} / %{BASE16NUM:P8F8} \. (.*)%{BASE16NUM:P8F9} \. %{BASE16NUM:P8F10} (.*)%{NOTSPACE:P8NS1} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "8" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET http : / / \[ support \. siteshell \. jp \] : %{BASE16NUM:P11F1} / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P11F2} HTTP / %{BASE16NUM:P11F3} \. %{BASE16NUM:P11F4} \" %{BASE16NUM:P11F5} \- %{BASE16NUM:P11F6} %{BASE16NUM:P11F7} %{BASE16NUM:P11F8} User \- agent : Mozilla / %{BASE16NUM:P11F9} \. %{BASE16NUM:P11F10} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "11" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET http : / / (\d)+ \. (\d)+ \. (\d)+ \. (\d)+ : %{BASE16NUM:P10F1} / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P10F2} HTTP / %{BASE16NUM:P10F3} \. %{BASE16NUM:P10F4} \" %{BASE16NUM:P10F5} \- %{BASE16NUM:P10F6} %{BASE16NUM:P10F7} %{BASE16NUM:P10F8} User \- agent : Mozilla / %{BASE16NUM:P10F9} \. %{BASE16NUM:P10F10} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "10" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET http : / / (\d)+ \. (\d)+ \. (\d)+ \. (\d)+ : %{BASE16NUM:P12F1} / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P12F2} HTTP / %{BASE16NUM:P12F3} \. %{BASE16NUM:P12F4} \" %{BASE16NUM:P12F5} \- %{BASE16NUM:P12F6} %{BASE16NUM:P12F7} %{BASE16NUM:P12F8} User \- agent : Mozilla / %{BASE16NUM:P12F9} \. %{BASE16NUM:P12F10} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "12" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET / HTTP / %{BASE16NUM:P13F1} \. %{BASE16NUM:P13F2} \" %{BASE16NUM:P13F3} %{BASE16NUM:P13F4} %{BASE16NUM:P13F5} %{BASE16NUM:P13F6} %{BASE16NUM:P13F7} User \- agent : Mozilla / %{BASE16NUM:P13F8} \. %{BASE16NUM:P13F9} \( (.*)%{NOTSPACE:P13NS1} %{NOTSPACE:P13NS2} ; (.*)\. (.*)(\S) (.*)%{WORD:P13W1} / %{NOTSPACE:P13NS3} \. (.*)Cookie : (.*)Referer : \-" ]
add_tag => [ "pattern", "13" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} (.*)HTTP / %{BASE16NUM:P14F1} \. %{BASE16NUM:P14F2} \" %{BASE16NUM:P14F3} %{NOTSPACE:P14NS1} %{BASE16NUM:P14F4} %{BASE16NUM:P14F5} %{BASE16NUM:P14F6} User \- agent : \- Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "14" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" GET (.*)/ (.*)%{WORD:P15W1} HTTP / %{BASE16NUM:P15F1} \. %{BASE16NUM:P15F2} \" %{BASE16NUM:P15F3} %{BASE16NUM:P15F4} %{BASE16NUM:P15F5} %{BASE16NUM:P15F6} %{BASE16NUM:P15F7} User \- agent : (.*)%{WORD:P15W2} / %{BASE16NUM:P15F8} \. (.*)Cookie : (.*)Referer : \-" ]
add_tag => [ "pattern", "15" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{HLA_TS_1:ts1} \" CONNECT %{NOTSPACE:P16NS1} \. %{NOTSPACE:P16NS2} \. %{NOTSPACE:P16NS3} \. %{NOTSPACE:P16NS4} : %{BASE16NUM:P16F1} HTTP / %{BASE16NUM:P16F2} \. %{BASE16NUM:P16F3} \" %{BASE16NUM:P16F4} %{BASE16NUM:P16F5} %{BASE16NUM:P16F6} %{BASE16NUM:P16F7} %{BASE16NUM:P16F8} User \- agent : \- Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "16" ]
tag_on_failure => []
}
}
if "pattern" not in [tags] {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "(\d)+ \. (\d)+ \. (\d)+ \. (\d)+ \- - \[ %{BASE16NUM:P9F1} / %{BASE16NUM:P9F2} / (.*): (time) : %{BASE16NUM:P9F3} \" GET http : / / support \. siteshell \. jp : %{BASE16NUM:P9F4} / SiteShellServer / update \. do \? serialID = \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* \* & version = %{BASE16NUM:P9F5} HTTP / %{BASE16NUM:P9F6} \. %{BASE16NUM:P9F7} \" %{BASE16NUM:P9F8} %{NOTSPACE:P9NS1} %{BASE16NUM:P9F9} %{BASE16NUM:P9F10} %{BASE16NUM:P9F11} User \- agent : Mozilla / %{BASE16NUM:P9F12} \. %{BASE16NUM:P9F13} Cookie : \- Referer : \-" ]
add_tag => [ "pattern", "9" ]
tag_on_failure => []
}
}
# Patterns End:17
seq {
field => "seq"
}
date {
match => [ "ts1", "yyyyMMddHHmmss", "yyyy / MM / dd HH : mm : ss", "yyyy - MM - dd HH : mm : ss", "yyyy - MM - dd't'HH : mm : ss . SSS'z'" ]
}
}
}
#Logstash Tail Start
output {
if "pattern" not in [tags] {
file{
codec => json_lines
path => unmatched_patterns
}
}
if "pattern" in [tags] {
file{
codec => json_lines
path => matched_patterns
}
}
}
#Logstash Tail End
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment