copy all data in elasticsearch to a local file using pyes

elastic_search_copy.py

import pyes
import simplejson as json

SOURCE =['SERVER:9200']

sconn = pyes.ES(SOURCE)

def scroll_gen(index):
    q = '{"query":{"match_all":{}}, "size": 15000}'
    s = sconn.search_raw(json.loads(q), scroll="5m", indices=index)
    scroll_id = s._scroll_id
    yield s.hits.hits

    scroll = sconn.search_scroll(scroll_id)
    while scroll.hits.hits:
        yield scroll.hits.hits
        scroll = sconn.search_scroll(scroll_id)
        scroll_id = scroll._scroll_id
        # print "scroll_id:  %s" % scroll_id

def copy_data():
    index = 'index_name'
    appendfile = open(index + '-copy.json', 'w')
    
    sgen = scroll_gen(index)
    for count, res in enumerate(sgen):
        for records, hit in enumerate(res):
            appendfile.write(json.dumps(dat) + '\n')
        if count % 1000 == 0:
             print count

def main():
    copy_data()

if __name__ == '__main__':
    main()