Skip to content

Instantly share code, notes, and snippets.

@sherry-ger
Last active May 28, 2019 19:54
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sherry-ger/20a7430eb27bdc7a3c64e21e42960b07 to your computer and use it in GitHub Desktop.
Save sherry-ger/20a7430eb27bdc7a3c64e21e42960b07 to your computer and use it in GitHub Desktop.
Correlations and Alerts
DELETE _xpack/watcher/watch/interesting_hosts
PUT _xpack/watcher/watch/interesting_hosts
{
"trigger": {
"schedule": {
"interval": "5m"
}
},
"input": {
"chain": {
"inputs": [
{
"ip_input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"filebeat-*"
],
"body": {
"query": {
"bool": {
"filter": [
{
"range": {
"@timestamp": {
"gte": "now-5m"
}
}
}
]
}
},
"aggs": {
"remoteip": {
"terms": {
"field": "nginx.access.remote_ip"
},
"aggs": {
"beathost": {
"terms": {
"field": "beat.hostname"
}
}
}
}
},
"size": 0
}
}
}
}
},
{
"lookup": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"notable_hosts"
],
"body": {
"query": {
"terms": {
"geoip.ip": [
"{{#ctx.payload.ip_input.aggregations.remoteip.buckets}}{{key}}",
"{{/ctx.payload.ip_input.aggregations.remoteip.buckets}}"
]
}
}
}
}
}
}
}
]
}
},
"condition": {
"compare" : { "ctx.payload.lookup.hits.total" : { "gt" : 0 }}
},
"transform": {
"script": {
"lang": "painless",
"source": """
def first=ctx.payload.lookup.hits.hits.stream().map(p -> p._source.geoip.ip).collect(Collectors.toList());
def second=ctx.payload.ip_input.aggregations.remoteip.buckets.stream().map(
e->{
return [ 'indicator': e.key, 'host': e.beathost.buckets.stream().map(h -> h.key).collect(Collectors.toList()) ]
}).filter(p->first.contains(p.indicator)).collect(Collectors.toList());
return second;"""
}
},
"actions": {
"notify-slack": {
"throttle_period": "5m",
"slack": {
"account": "team1",
"message": {
"from": "watcher",
"to": [
"@sherry"
],
"text": "System Monitoring",
"attachments": [
{
"title": "Correlations Found",
"text": "Please investigate these indicators and hosts:\n{{#ctx.payload._value}}{{.}}\n\n{{/ctx.payload._value}}",
"color": "danger"
}
]
}
}
}
}
}
POST _xpack/watcher/watch/interesting_hosts/_execute
@sherry-ger
Copy link
Author

#indexing action of multiple documents

PUT _xpack/watcher/watch/interesting_hosts
{
  "trigger": {
    "schedule": {
      "interval": "5m"
    }
  },
  "input": {
    "chain": {
      "inputs": [
        {
          "ip_input": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "filebeat-*"
                ],
                "body": {
                  "query": {
                    "bool": {
                      "filter": [
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "now-3y"
                            }
                          }
                        },
                        {
                          "term": {
                            "fileset.name": "access"
                          }
                        }
                      ]
                    }
                  },
                  "aggs": {
                    "remoteip": {
                      "terms": {
                        "field": "nginx.access.remote_ip"
                      },
                      "aggs": {
                        "beathost": {
                          "terms": {
                            "field": "beat.hostname"
                          }
                        }
                      }
                    }
                  },
                  "size": 0
                }
              }
            }
          }
        },
        {
          "lookup": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "notable_hosts"
                ],
                "body": {
                  "query": {
                    "terms": {
                      "geoip.ip": [
                        "{{#ctx.payload.ip_input.aggregations.remoteip.buckets}}{{key}}",
                        "{{/ctx.payload.ip_input.aggregations.remoteip.buckets}}"
                      ]
                    }
                  }
                }
              }
            }
          }
        }
      ]
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.lookup.hits.total": {
        "gt": 0
      }
    }
  },
  "transform": {
    "script": {
      "lang": "painless",
      "source": """
      def first=ctx.payload.lookup.hits.hits.stream().map(p -> p._source.geoip.ip).collect(Collectors.toList()); 
      def second=ctx.payload.ip_input.aggregations.remoteip.buckets.stream().map(
      e->{ 
        return [ 'indicator': e.key, 'host': e.beathost.buckets.stream().map(h -> h.key).collect(Collectors.toList()) ]      
        }).filter(p->first.contains(p.indicator)).collect(Collectors.toList()); 
        return second;
"""
    }
  },
  "actions": {
    "index_payload": {
      "transform": {
        "script": "return ['_doc':ctx.payload._value];"
      },
      "index": {
        "index": "test_watcher_index",
        "doc_type": "_doc"
      }
    }
  }
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment