Sample Elasticsearch Scroll Query

extract_traffic.py

from elasticsearch import Elasticsearch, helpers
import csv

import logging
logging.basicConfig()

target_file = "traffic.csv"

es=Elasticsearch("your.es.host.example.tld", timeout=480)

query_body = """
{
    "size": 1500,
    "fields" : ["@timestamp", "src_geoip.country_name", "protocol", "dstport"],
    "query": {
        "filtered": {
            "query": {
                "match_all": {}
            },
            "filter": {
                "terms": {
                    "srcip": ["8.8.8.8", "8.8.4.4]
                },
                "bool": {
                    "must": [
                      {
                          "range": {
                              "@timestamp": {
                                "gt" : "now-3w",
                                "lt" : "now"
                              }
                          }
                      },
                      {
                           "exists": {
                             "field": "dstport"
                           }
                      },
                      {
                           "term": {
                             "srczone" : "external"
                           }
                      }
                     ]
                }
            }
        }
    }
}
"""
csvfile = open(target_file, 'wb')


class DictUnicodeProxy(object):
    def __init__(self, d):
        self.d = d
    def __iter__(self):
        return self.d.__iter__()
    def get(self, item, default=None):
        i = self.d.get(item, default)
        if isinstance(i, list):
            i = i[0]
        if isinstance(i, unicode):
            return i.encode('utf-8')
        return i


scanResp = helpers.scan(client=es, query=query_body, scroll="90m", index="", timeout="10m")
headers = ['@timestamp','src_geoip.country_name','protocol','dstport']
csvwriter = csv.DictWriter(csvfile, delimiter='\t', fieldnames=headers)
csvwriter.writeheader()

for resp in scanResp:
    csvwriter.writerow(DictUnicodeProxy(resp['fields']))

csvfile.close()