Skip to content

Instantly share code, notes, and snippets.

@donnydavis

donnydavis/pfsense-2.4-extractor.json

Created March 30, 2018 19:06
Show Gist options
  • Save donnydavis/290da149f20c6e0b0abecfb232463c3e to your computer and use it in GitHub Desktop.
Save donnydavis/290da149f20c6e0b0abecfb232463c3e to your computer and use it in GitHub Desktop.
Download ZIP
pfsense 2.4 graylog Extractors
Raw
pfsense-2.4-extractor.json
{
"extractors": [
{
"title": "pfSense filterlog: IPv6 TCP",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,Class,FlowLabel,HopLimit,Protocol,ProtocolID,Length,SourceIP,DestIP,SourcePort,DestPort,DataLength,TCPFlags,Sequence,ACK,Window,URG,Options"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),6,.*,tcp,.*$"
},
{
"title": "pfSense filterlog: IPv4 UDP",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,SourcePort,DestPort,DataLength"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,(?i)UDP,.*$"
},
{
"title": "pfSense filterlog: IPv4 TCP",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,SourcePort,DestPort,DataLength,TCPFlags,Sequence,ACK,Window,URG,Options"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,tcp,.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP Unreachable Protocol",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_DestIP,ICMP_ProtocolID"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,unreachproto,.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP Echo",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_ID,ICMP_Sequence"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,(request|reply),.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP Unreachable Port",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_DestIP,ICMP_ProtocolID,ICMP_Port"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,unreachport,.*$"
},
{
"title": "pfSense filterlog: IPv6 UDP",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,Class,FlowLabel,HopLimit,Protocol,ProtocolID,Length,SourceIP,DestIP,SourcePort,DestPort,DataLength"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),6,.*,(?i)UDP,.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP Unreachable Other",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_Description"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,(unreach|timexceed|paramprob|redirect|maskreply),.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP Need Frag",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_DestIP,ICMP_MTU"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,needfrag,.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP TStamp Reply",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_ID,ICMP_Sequence,ICMP_otime,ICMP_rtime,ICMP_ttime"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,tstampreply,.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP TStamp",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_ID,ICMP_Sequence"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,tstamp,.*$"
},
{
"title": "pfSense filterlog: IPv4 ICMP Default",
"extractor_type": "regex",
"converters": [
{
"type": "csv",
"config": {
"column_header": "RuleNumber,SubRuleNumber,Anchor,Tracker,Interface,Reason,Action,Direction,IPVersion,TOS,ECN,TTL,ID,Offset,Flags,ProtocolID,Protocol,Length,SourceIP,DestIP,ICMP_Type,ICMP_Description"
}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "FilterData",
"extractor_config": {
"regex_value": "^filterlog:\\s+(.*)$"
},
"condition_type": "regex",
"condition_value": "^filterlog:\\s+.*,(in|out),4,.*,icmp,.*,(?!(request|reply|unreachproto|unreachport|unreach|timexceed|paramprob|redirect|maskreply|needfrag|tstamp|tstampreply)),.*$"
},
{
"title": "Puppet error - DevOps",
"extractor_type": "regex",
"converters": [],
"order": 1,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "puppetError",
"extractor_config": {
"regex_value": "(err)"
},
"condition_type": "regex",
"condition_value": "^\\d+-+\\d+-\\d+\\s+\\d+:+\\d+:\\d+\\s+-\\d+\\s+Puppet+\\s+\\(+err+\\)"
},
{
"title": "Snort Source IP",
"extractor_type": "regex",
"converters": [],
"order": 34,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "src_ip",
"extractor_config": {
"regex_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s(\\S+):\\d+\\s->\\s\\S+:\\d+"
},
"condition_type": "regex",
"condition_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
{
"title": "Snort Priority",
"extractor_type": "regex",
"converters": [],
"order": 33,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "snort_priority",
"extractor_config": {
"regex_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s(\\d)\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
"condition_type": "regex",
"condition_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
{
"title": "Snort Message",
"extractor_type": "regex",
"converters": [],
"order": 31,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "snort_message",
"extractor_config": {
"regex_value": "\\[\\d+:\\d+:\\d\\] (.*) \\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
"condition_type": "regex",
"condition_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
{
"title": "Snort Classification",
"extractor_type": "regex",
"converters": [],
"order": 32,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "snort_classification",
"extractor_config": {
"regex_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:(.+)\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
"condition_type": "regex",
"condition_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
{
"title": "Snort Protocol",
"extractor_type": "regex",
"converters": [],
"order": 35,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "snort_protocol",
"extractor_config": {
"regex_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{(\\S+)\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
"condition_type": "regex",
"condition_value": "\\[\\d+:\\d+:\\d\\].*\\[Classification:.+\\]\\s\\[Priority:\\s\\d\\]\\s\\{\\S+\\}\\s\\S+:\\d+\\s->\\s\\S+:\\d+"
},
{
"title": "Puppet: Puppet Configuration Version",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 3,
"cursor_strategy": "copy",
"source_field": "full_message",
"target_field": "puppetConfigVersion",
"extractor_config": {
"regex_value": "^\\d+-+\\d+-\\d+\\s+\\d+:+\\d+:\\d+\\s+-\\d+\\s+Puppet+\\s+\\(+info+\\)+:\\s+Applying+\\s+configuration+\\s+version+\\s+'(\\d[0-9]{0,9})+'"
},
"condition_type": "regex",
"condition_value": "^\\d+-+\\d+-\\d+\\s+\\d+:+\\d+:\\d+\\s+-\\d+\\s+Puppet+\\s+\\(+info+\\)+:\\s+Applying+\\s+configuration+\\s+version+\\s+'(\\d[0-9]{0,9})+'"
},
{
"title": "Time Request from message IIS",
"extractor_type": "regex",
"converters": [],
"order": 8,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "request_time",
"extractor_config": {
"regex_value": "^\\d+-\\d+-\\d+\\s+(\\d+:\\d+:\\d+)"
},
"condition_type": "regex",
"condition_value": "^\\d+-\\d+-\\d+\\s+(\\d+:\\d+:\\d+)+\\s+\\d+.\\d+.\\d+.\\d+"
},
{
"title": "pfSense Action",
"extractor_type": "regex",
"converters": [],
"order": 24,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "action",
"extractor_config": {
"regex_value": ".* (pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
"condition_type": "regex",
"condition_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
{
"title": "Puppet main Module name",
"extractor_type": "split_and_index",
"converters": [],
"order": 9,
"cursor_strategy": "copy",
"source_field": "full_message",
"target_field": "puppetModule",
"extractor_config": {
"index": 3,
"split_by": "/"
},
"condition_type": "regex",
"condition_value": "^\\d+-\\d+-\\d+\\s+\\d+:\\d+:\\d+\\s+-\\d+\\s+\\/+Stage+\\[+main+\\]+\\/(.*?)+\\/"
},
{
"title": "pfSense Interface",
"extractor_type": "regex",
"converters": [],
"order": 25,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "interface",
"extractor_config": {
"regex_value": ".* (?:pass|block) .* on ([^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
"condition_type": "regex",
"condition_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
{
"title": "Puppet: Puppet run time - DevOps",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 2,
"cursor_strategy": "copy",
"source_field": "full_message",
"target_field": "puppetRunTime",
"extractor_config": {
"regex_value": "Finished catalog run in ((\\d[0-9]{0,9}\\.[0-9]{0,9})) seconds"
},
"condition_type": "regex",
"condition_value": "^\\d+-+\\d+-\\d+\\s+\\d+:+\\d+:\\d+\\s+-\\d+\\s+Puppet+\\s+\\(+notice+\\)+:\\s+Finished+\\s+catalog+\\s+run+\\s+in+\\s+(\\d[0-9]{0,9}\\.[0-9]{0,9})+\\s+seconds"
},
{
"title": "pfSense Source IP",
"extractor_type": "regex",
"converters": [],
"order": 27,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "srcip",
"extractor_config": {
"regex_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
"condition_type": "regex",
"condition_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
{
"title": "pfSense Protocol",
"extractor_type": "regex",
"converters": [],
"order": 26,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "protocol",
"extractor_config": {
"regex_value": ".* (?:pass|block) .* on (?:[^:]+):.* (TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
"condition_type": "regex",
"condition_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
{
"title": "pfSense Source Port",
"extractor_type": "regex",
"converters": [],
"order": 28,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "srcport",
"extractor_config": {
"regex_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
"condition_type": "regex",
"condition_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
{
"title": "Sudo Executor",
"extractor_type": "regex",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "sudo_executor",
"extractor_config": {
"regex_value": "sudo:\\s+(\\S+)\\s+:"
},
"condition_type": "string",
"condition_value": "sudo:"
},
{
"title": "pfSense Destination Port",
"extractor_type": "regex",
"converters": [],
"order": 30,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "destport",
"extractor_config": {
"regex_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(\\d*): (?:.*)"
},
"condition_type": "regex",
"condition_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
{
"title": "pfSense Destination IP",
"extractor_type": "regex",
"converters": [],
"order": 29,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "destip",
"extractor_config": {
"regex_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
"condition_type": "regex",
"condition_value": ".* (?:pass|block) .* on (?:[^:]+):.* (?:TCP|UDP|IGMP|ICMP) .* (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*) [<|>] (?:\\d+\\.\\d+\\.\\d+\\.\\d+)\\.?(?:\\d*): (?:.*)"
},
{
"title": "Sudo Command",
"extractor_type": "regex",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "sudo_command",
"extractor_config": {
"regex_value": "sudo:.+COMMAND=(.+);?"
},
"condition_type": "string",
"condition_value": "sudo:"
},
{
"title": "Sudo Command User",
"extractor_type": "regex",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "sudo_command_user",
"extractor_config": {
"regex_value": "sudo:.+USER=(\\S+)"
},
"condition_type": "string",
"condition_value": "sudo:"
},
{
"title": "Level/Severity",
"extractor_type": "regex",
"converters": [
{
"type": "syslog_pri_level",
"config": {}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "level",
"extractor_config": {
"regex_value": "\\d <(.+)>"
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "HTTP method",
"extractor_type": "regex",
"converters": [
{
"type": "lowercase",
"config": {}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "http_method",
"extractor_config": {
"regex_value": "method=(.+?)(\\s|$)"
},
"condition_type": "string",
"condition_value": "method="
},
{
"title": "Path",
"extractor_type": "regex",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "path",
"extractor_config": {
"regex_value": "path=(.+?)(\\s|$)"
},
"condition_type": "string",
"condition_value": "path="
},
{
"title": "Request duration",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "request_duration",
"extractor_config": {
"regex_value": "duration=(.+?)(\\s|$)"
},
"condition_type": "string",
"condition_value": "duration"
},
{
"title": "Facility",
"extractor_type": "regex",
"converters": [
{
"type": "syslog_pri_facility",
"config": {}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "facility",
"extractor_config": {
"regex_value": "\\d <(.+)>"
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "View duration",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "view_duration",
"extractor_config": {
"regex_value": "view=(.+?)(\\s|$)"
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "DB Duration",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "db_duration",
"extractor_config": {
"regex_value": "db=(.+?)(\\s|$)"
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "Remote Address",
"extractor_type": "regex",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "remote_addr",
"extractor_config": {
"regex_value": "nginx:\\s+(\\S+)"
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Request ID",
"extractor_type": "regex",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "request_id",
"extractor_config": {
"regex_value": "\\[([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\\]"
},
"condition_type": "regex",
"condition_value": "\\[([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\\]"
},
{
"title": "Request Timestamp",
"extractor_type": "regex",
"converters": [
{
"type": "date",
"config": {
"date_format": "dd/MMM/YYYY:HH:mm:ss Z"
}
}
],
"order": 2,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "timestamp",
"extractor_config": {
"regex_value": "nginx:.+?\\[(.+?)\\]"
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Request Verb",
"extractor_type": "regex",
"converters": [],
"order": 3,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "request_verb",
"extractor_config": {
"regex_value": "nginx:.+\\[.+\\] \"(\\S+)"
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Remote User",
"extractor_type": "regex",
"converters": [],
"order": 1,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "remote_user",
"extractor_config": {
"regex_value": "nginx: \\S+ - (\\S+)"
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Request Path",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 4,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "request_path",
"extractor_config": {
"regex_value": "nginx:.+?\"\\S+ (\\S+).+\""
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "HTTP Version",
"extractor_type": "regex",
"converters": [],
"order": 5,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "http_version",
"extractor_config": {
"regex_value": "nginx:.+HTTP/(\\S+)\""
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Response Status",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 6,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "response_status",
"extractor_config": {
"regex_value": "nginx:.+?HTTP/\\S+\" (\\d+)"
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "HTTP Referer",
"extractor_type": "regex",
"converters": [],
"order": 9,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "http_referer",
"extractor_config": {
"regex_value": "nginx:.+?HTTP/\\S+\" \\d+ \\d+ \"(.+?)\""
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Response Bytes",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 7,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "response_bytes",
"extractor_config": {
"regex_value": "nginx:.+?HTTP/\\S+\" \\d+ (\\d+)"
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "HTTP User Agent",
"extractor_type": "regex",
"converters": [],
"order": 8,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "http_user_agent",
"extractor_config": {
"regex_value": "nginx:.+?HTTP/\\S+\" \\d+ \\d+ \".+?\" \"(.+?)\""
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Connection requests",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 11,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "connection_requests",
"extractor_config": {
"regex_value": "connection_requests=(.+?)\\|"
},
"condition_type": "regex",
"condition_value": ".+connection_requests=.+"
},
{
"title": "Connection ID",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 10,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "connection_id",
"extractor_config": {
"regex_value": "connection=(.+?)\\|"
},
"condition_type": "regex",
"condition_value": ".+connection=.+"
},
{
"title": "Response time",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 12,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "millis",
"extractor_config": {
"regex_value": "millis=(.+?)>"
},
"condition_type": "regex",
"condition_value": ".+millis=.+"
},
{
"title": "Message",
"extractor_type": "regex",
"converters": [],
"order": 13,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "message",
"extractor_config": {
"regex_value": "nginx:.+?\\\"(\\S+.+HTTP\\/\\S+)\\\" \\d+"
},
"condition_type": "regex",
"condition_value": "^\\S+\\s+nginx:"
},
{
"title": "Response Status",
"extractor_type": "regex",
"converters": [
{
"type": "numeric",
"config": {}
}
],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "response_status",
"extractor_config": {
"regex_value": "status=(.+?)(\\s|$)"
},
"condition_type": "string",
"condition_value": "status="
},
{
"title": "Remote Address",
"extractor_type": "regex",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "remote_addr",
"extractor_config": {
"regex_value": "\\[(\\b(?:\\d{1,3}\\.){3}\\d{1,3}\\b)\\]"
},
"condition_type": "regex",
"condition_value": "\\[(\\b(?:\\d{1,3}\\.){3}\\d{1,3}\\b)\\]"
}
],
"version": "2.4.3"
}
@donnydavis
Copy link
Author

This is a first stab at the Extractors I could find from around the interwebs, and complied into one hopefully usable extractor set

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment