Skip to content

Instantly share code, notes, and snippets.

@jalogisch
Last active June 21, 2016 10:35
Show Gist options
  • Save jalogisch/83cbe6f1140794c91f979de1324aff02 to your computer and use it in GitHub Desktop.
Save jalogisch/83cbe6f1140794c91f979de1324aff02 to your computer and use it in GitHub Desktop.
when using https://gist.github.com/untergeek/0373ee85a41d03ae1b78 to send data via amqp to graylog you some extractor to have your data clean. All after extractor 4 are not needed (but present as i use them for dovecot logs)
{
"extractors": [
{
"title": "1_extract_json_message",
"extractor_type": "json",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "full_message",
"target_field": "",
"extractor_config": {
"key_separator": "_",
"list_separator": ", ",
"kv_separator": "="
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "2_key_value_from_message",
"extractor_type": "copy_input",
"converters": [
{
"type": "tokenizer",
"config": {}
}
],
"order": 1,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "message",
"extractor_config": {},
"condition_type": "none",
"condition_value": ""
},
{
"title": "3_copy_host_to_source",
"extractor_type": "copy_input",
"converters": [
{
"type": "lowercase",
"config": {}
}
],
"order": 2,
"cursor_strategy": "copy",
"source_field": "host",
"target_field": "source",
"extractor_config": {},
"condition_type": "none",
"condition_value": ""
},
{
"title": "5_split_lit",
"extractor_type": "split_and_index",
"converters": [],
"order": 4,
"cursor_strategy": "copy",
"source_field": "lip",
"target_field": "lip",
"extractor_config": {
"index": 1,
"split_by": ","
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "6_split_rip",
"extractor_type": "split_and_index",
"converters": [],
"order": 5,
"cursor_strategy": "copy",
"source_field": "rip",
"target_field": "rip",
"extractor_config": {
"index": 1,
"split_by": ","
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "4_copy_extracted_timestamp_2_timestamp",
"extractor_type": "copy_input",
"converters": [
{
"type": "flexdate",
"config": {
"time_zone": "Etc/UTC"
}
}
],
"order": 3,
"cursor_strategy": "copy",
"source_field": "@timestamp",
"target_field": "timestamp",
"extractor_config": {},
"condition_type": "none",
"condition_value": ""
},
{
"title": "7_split_method",
"extractor_type": "split_and_index",
"converters": [],
"order": 6,
"cursor_strategy": "copy",
"source_field": "method",
"target_field": "method",
"extractor_config": {
"index": 1,
"split_by": ","
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "8_split_mpid",
"extractor_type": "split_and_index",
"converters": [],
"order": 7,
"cursor_strategy": "copy",
"source_field": "mpid",
"target_field": "mpid",
"extractor_config": {
"index": 1,
"split_by": ","
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "9_extract_user",
"extractor_type": "grok",
"converters": [],
"order": 8,
"cursor_strategy": "copy",
"source_field": "user",
"target_field": "",
"extractor_config": {
"grok_pattern": "<%{GREEDYDATA:user}>"
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "10_extract_session",
"extractor_type": "grok",
"converters": [],
"order": 9,
"cursor_strategy": "copy",
"source_field": "session",
"target_field": "",
"extractor_config": {
"grok_pattern": "<%{GREEDYDATA:session}>"
},
"condition_type": "none",
"condition_value": ""
}
],
"version": "2.0.0-SNAPSHOT"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment