Import cowrie json logs to elasticsearch

import_cowrie_json_elasticsearch.py

import os
import json
from elasticsearch import Elasticsearch
# import aiohttp
# from aiomultiprocess import Process, Pool
# import multiprocessing
import asyncio
import time
import uuid
es = Elasticsearch()

async def post_json(payload):
    # return await request("GET", url)
    return es.index(index='cowrie-logs',doc_type='document', body=json.loads(json.dumps(payload)))


async def send_file(file_name):
    print(file_name)
    tasks = []
    with open(file_name) as f:
        tasks = [post_json(line) for line in f.readlines()]
    start = time.time()
    results = asyncio.gather(*tasks)
    results = await results
    print(results)
    end = time.time()
    total_time = end - start
    # req_per_sec = total_time/60
    print("Num of results = ", len(list(results)))
    print("time = ", total_time)

async def all_files():
    import os
    files = []
    for root, dirs, files in os.walk("."):
        files = [await send_file(os.path.relpath(os.path.join(root, f), ".")) for f in files if '.json' in f]

    results = asyncio.gather(*files)
    results = await results

# if __name__ == "__main__":

if __name__ == "__main__":
    loop = asyncio.get_event_loop()
    loop.run_until_complete(all_files())
#     loop.run_until_complete(run_aiomultiprocess(req_num=5000))
    loop.close()