Skip to content

Instantly share code, notes, and snippets.

@cdenneen

cdenneen/agent.conf

Last active December 20, 2015 17:29
Show Gist options
  • Save cdenneen/6168983 to your computer and use it in GitHub Desktop.
Save cdenneen/6168983 to your computer and use it in GitHub Desktop.
Download ZIP
Multiline adding tag even when not match
Raw
agent.conf
input {
tcp {
port => 3516
type => json
format => json
}
}
filter {
mutate {
type => json
gsub => [
"datetime", "[ \t]$", "",
"process", "[ \t]{2,}", "",
"area", "[ \t]{2,}", "",
"category", "[ \t]{2,}", "",
"level", "[ \t]{2,}", ""
]
rename => [ "message", "@message" ]
add_tag => "%{app}"
add_tag => "%{tags}"
}
multiline {
pattern => "^\.\.\."
add_tag => [ "extra_line" ]
what => previous
}
date {
match => [ "datetime", "MM/dd/YYYY HH:mm:ss.SS", "MM/dd/YYYY HH:mm:ss.SS*" ]
}
mutate {
remove => [ "SourceModuleName", "SourceModuleType", "EventReceivedTime", "tags", "app", "datetime" ]
}
}
output {
stdout {
message => "%{@timestamp} - %{@tags} - %{@message}"
# debug => true
}
}
Raw
run output
nc localhost 3516 < sample.json
2013-08-06T19:02:51.400Z - spuls,Production,multiline,extra_line
2013-08-06T19:02:51.410Z - spuls,Production,extra_line
2013-08-06T19:03:01.460Z - spuls,Production,multiline,extra_line
2013-08-06T19:03:01.460Z - spuls,Production,extra_line
2013-08-06T19:03:01.470Z - spuls,Production,extra_line
2013-08-06T19:03:01.470Z - spuls,Production,extra_line
2013-08-06T19:03:01.490Z - spuls,Production,extra_line
2013-08-06T19:03:01.490Z - spuls,Production,extra_line
2013-08-06T19:03:01.500Z - spuls,Production,extra_line
2013-08-06T19:03:01.500Z - spuls,Production
nc localhost 3516 < sample.json
2013-08-06T19:02:51.400Z - spuls,Production,multiline,extra_line - <?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema"><soap:Body><GetListItems xmlns="http://schemas.microsoft.com/sharepoint/soap/"><listName>ePix incoming mail (Dev, QA and Stg)</listName><viewName /><query><Query xmlns=""><Where><Gt><FieldRef Name="Created" /><Value Type="DateTime" IncludeTimeValue="TRUE">2013-08-05T15:29:29Z</Value></Gt></Where><OrderBy><FieldRef Name="Modified" Ascending="TRUE" /></OrderBy></Query></query><viewFields><ViewFields xmlns="">*</ViewFields></viewFields><rowLimit>10000</rowLimit><queryOptions><QueryOptions xmlns=""><ViewAttributes Scope="RecursiveAll" /></QueryOptions></queryOptions><webID /></GetListItems>...
...</soap:Body></soap:Envelope>
2013-08-06T19:02:51.410Z - spuls,Production,extra_line - Site=/technology/NewsRoom
2013-08-06T19:03:01.460Z - spuls,Production,multiline,extra_line - <?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema"><soap:Body><GetListItems xmlns="http://schemas.microsoft.com/sharepoint/soap/"><listName>ePix incoming mail (Dev, QA and Stg)</listName><viewName /><query><Query xmlns=""><Where><Gt><FieldRef Name="Created" /><Value Type="DateTime" IncludeTimeValue="TRUE">2013-08-05T15:29:29Z</Value></Gt></Where><OrderBy><FieldRef Name="Modified" Ascending="TRUE" /></OrderBy></Query></query><viewFields><ViewFields xmlns="">*</ViewFields></viewFields><rowLimit>10000</rowLimit><queryOptions><QueryOptions xmlns=""><ViewAttributes Scope="RecursiveAll" /></QueryOptions></queryOptions><webID /></GetListItems>...
...</soap:Body></soap:Envelope>
2013-08-06T19:03:01.460Z - spuls,Production,extra_line - Site=/technology/NewsRoom
2013-08-06T19:03:01.470Z - spuls,Production,extra_line - Entering monitored scope (Request (GET:http://sp.ap.org:80/technology/NewsRoom/elvis/Lists/elvistix20/elvistix3.aspx))
2013-08-06T19:03:01.470Z - spuls,Production,extra_line - Name=Request (GET:http://sp.ap.org:80/technology/NewsRoom/elvis/Lists/elvistix20/elvistix3.aspx)
2013-08-06T19:03:01.490Z - spuls,Production,extra_line - Leaving Monitored Scope (EnsureListItemsData). Execution Time=14.4028027416207
2013-08-06T19:03:01.490Z - spuls,Production,extra_line - Leaving Monitored Scope (Request (POST:http://sp.ap.org:80/technology/NewsRoom/photo/_vti_bin/lists.asmx)). Execution Time=39.4360018084237
2013-08-06T19:03:01.500Z - spuls,Production,extra_line - Site=/technology/NewsRoom
2013-08-06T19:03:01.500Z - spuls,Production - Leaving Monitored Scope (PostResolveRequestCacheHandler). Execution Time=15.1526369431969
Raw
sample.json
{"EventReceivedTime":"2013-08-06 17:21:17","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:02:51.40 ","process":"w3wp.exe (0x18C8) ","tid":"0x16E8","area":"SharePoint Foundati
on ","category":"General ","eventid":"fbv6","level":"Medium ","message":"<?xml version=\"1.0\" encoding=\"utf-8\"?><soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" xmlns:xsi=\"h
ttp://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\"><soap:Body><GetListItems xmlns=\"http://schemas.microsoft.com/sharepoint/soap/\"><listName>ePix incoming mail (Dev, QA and Stg)</listName><vi
ewName /><query><Query xmlns=\"\"><Where><Gt><FieldRef Name=\"Created\" /><Value Type=\"DateTime\" IncludeTimeValue=\"TRUE\">2013-08-05T15:29:29Z</Value></Gt></Where><OrderBy><FieldRef Name=\"Modified\" Ascending=\"TRUE\" /></Order
By></Query></query><viewFields><ViewFields xmlns=\"\">*</ViewFields></viewFields><rowLimit>10000</rowLimit><queryOptions><QueryOptions xmlns=\"\"><ViewAttributes Scope=\"RecursiveAll\" /></QueryOptions></queryOptions><webID /></Get
ListItems>...","correlation":"07f58f32-ef76-401e-8e40-a82bcc520a95","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:17","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:02:51.40*","process":"w3wp.exe (0x18C8) ","tid":"0x16E8","area":"SharePoint Foundati
on ","category":"General ","eventid":"fbv6","level":"Medium ","message":"...</soap:Body></soap:Envelope>","correlation":"07f58f32-ef76-401e-8e40-a82bcc520a95","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:17","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:02:51.41 ","process":"w3wp.exe (0x18C8) ","tid":"0x16E8","area":"SharePoint Foundati
on ","category":"Logging Correlation Data ","eventid":"xmnv","level":"Medium ","message":"Site=/technology/NewsRoom","correlation":"07f58f32-ef76-401e-8e40-a82bcc520a95","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.46 ","process":"w3wp.exe (0x18C8) ","tid":"0x1FF4","area":"SharePoint Foundati
on ","category":"General ","eventid":"fbv6","level":"Medium ","message":"<?xml version=\"1.0\" encoding=\"utf-8\"?><soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" xmlns:xsi=\"h
ttp://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\"><soap:Body><GetListItems xmlns=\"http://schemas.microsoft.com/sharepoint/soap/\"><listName>ePix incoming mail (Dev, QA and Stg)</listName><vi
ewName /><query><Query xmlns=\"\"><Where><Gt><FieldRef Name=\"Created\" /><Value Type=\"DateTime\" IncludeTimeValue=\"TRUE\">2013-08-05T15:29:29Z</Value></Gt></Where><OrderBy><FieldRef Name=\"Modified\" Ascending=\"TRUE\" /></Order
By></Query></query><viewFields><ViewFields xmlns=\"\">*</ViewFields></viewFields><rowLimit>10000</rowLimit><queryOptions><QueryOptions xmlns=\"\"><ViewAttributes Scope=\"RecursiveAll\" /></QueryOptions></queryOptions><webID /></Get
ListItems>...","correlation":"6858b873-5323-40a5-a073-7d2ba554f23f","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.46*","process":"w3wp.exe (0x18C8) ","tid":"0x1FF4","area":"SharePoint Foundati
on ","category":"General ","eventid":"fbv6","level":"Medium ","message":"...</soap:Body></soap:Envelope>","correlation":"6858b873-5323-40a5-a073-7d2ba554f23f","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.46 ","process":"w3wp.exe (0x18C8) ","tid":"0x1FF4","area":"SharePoint Foundati
on ","category":"Logging Correlation Data ","eventid":"xmnv","level":"Medium ","message":"Site=/technology/NewsRoom","correlation":"6858b873-5323-40a5-a073-7d2ba554f23f","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.47 ","process":"w3wp.exe (0x18C8) ","tid":"0x03CC","area":"SharePoint Foundati
on ","category":"Monitoring ","eventid":"nasq","level":"Medium ","message":"Entering monitored scope (Request (GET:http://sp.ap.org:80/technology/NewsRoom/elvis/Lists/elvistix20/elvistix3.aspx))","tags":
"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.47 ","process":"w3wp.exe (0x18C8) ","tid":"0x03CC","area":"SharePoint Foundati
on ","category":"Logging Correlation Data ","eventid":"xmnv","level":"Medium ","message":"Name=Request (GET:http://sp.ap.org:80/technology/NewsRoom/elvis/Lists/elvistix20/elvistix3.aspx)","correlation":"7f5e3bc3-7535-
450f-a1dd-95d7867d71c6","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.49 ","process":"w3wp.exe (0x18C8) ","tid":"0x1FF4","area":"SharePoint Foundati
on ","category":"Monitoring ","eventid":"b4ly","level":"High ","message":"Leaving Monitored Scope (EnsureListItemsData). Execution Time=14.4028027416207","correlation":"6858b873-5323-40a5-a073-7d2ba554
f23f","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.49 ","process":"w3wp.exe (0x18C8) ","tid":"0x1FF4","area":"SharePoint Foundati
on ","category":"Monitoring ","eventid":"b4ly","level":"Medium ","message":"Leaving Monitored Scope (Request (POST:http://sp.ap.org:80/technology/NewsRoom/photo/_vti_bin/lists.asmx)). Execution Time=39.4
360018084237","correlation":"6858b873-5323-40a5-a073-7d2ba554f23f","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.50 ","process":"w3wp.exe (0x18C8) ","tid":"0x03CC","area":"SharePoint Foundati
on ","category":"Logging Correlation Data ","eventid":"xmnv","level":"Medium ","message":"Site=/technology/NewsRoom","correlation":"7f5e3bc3-7535-450f-a1dd-95d7867d71c6","tags":"Production","app":"spuls"}
{"EventReceivedTime":"2013-08-06 17:21:19","SourceModuleName":"spuls","SourceModuleType":"im_file","datetime":"08/06/2013 15:03:01.50 ","process":"w3wp.exe (0x18C8) ","tid":"0x03CC","area":"SharePoint Foundati
on ","category":"Monitoring ","eventid":"b4ly","level":"High ","message":"Leaving Monitored Scope (PostResolveRequestCacheHandler). Execution Time=15.1526369431969","correlation":"7f5e3bc3-7535-450f-a1
dd-95d7867d71c6","tags":"Production","app":"spuls"}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment