Kibana data structure translation

ingest.py

#!/usr/bin/env python

import json
import os
import sys

from elasticsearch import Elasticsearch

def main(argv):

    es = Elasticsearch()
    index = 'rhel7'
    results = {}
    scores = {}

    # results directory
    directory = argv[1]

    #scores file
    scores_file = argv[2]

    # build results data structures
    for root, dirs, files in os.walk(directory):
        for subdir in dirs:
            print('+ walking ' + subdir)
            for r, d, f in os.walk(directory + '/' + subdir):
                for filename in f:
                    print('++ trying ' + filename)
                    try:
                        with open(directory + '/' + subdir + '/' + filename, 'r') as jsonfile:
                            jsondata = json.load(jsonfile)
                        rpm_name = filename.replace('.json', '')
                        if isinstance(jsondata['results'], dict):
                            for binary in jsondata['results'].keys():
                                result_name = rpm_name + '->' + jsondata['results'][binary]['filename']
                                results[result_name] = {}
                                results[result_name]['metadata'] = jsondata['metadata']
                                if jsondata['results'][binary]['complexity'].get('r2aa'):
                                    if jsondata['results'][binary]['complexity']['r2aa'].get('afCc'):
                                        if jsondata['results'][binary]['complexity']['r2aa']['afCc'] is not None:
                                            results[result_name]['complexity'] = jsondata['results'][binary]['complexity']
                                else:
                                    results[result_name]['complexity'] = 'error in data'
                                results[result_name]['filename'] = jsondata['results'][binary]['filename']
                                results[result_name]['find-libc-functions'] = jsondata['results'][binary]['find-libc-functions']
                                results[result_name]['hardening-check'] = jsondata['results'][binary]['hardening-check']
                                results[result_name]['report-functions'] = jsondata['results'][binary]['report-functions']
                                results[result_name]['rpm'] = jsondata['results'][binary]['rpm']
                    except Exception as e:
                        continue

    with open(scores_file, 'r') as f:
        scores_data = json.load(f)
    for score in scores_data.keys():
        rpm_name = score.replace('.json', '')
        for binary in scores_data[score]['binary_scores'].keys():
            result_name = rpm_name + '->' + binary
            if results.get(result_name):
                print('++ matched ' + result_name)
                results[result_name]['binary_score'] = scores_data[score]['binary_scores'][binary]
            else:
                print('++ NO MATCH!')

    for key in results.keys():
        result = results[key]
        res = es.index(index=index, doc_type='ctl-result', body=result)



if __name__ == "__main__":
    main(sys.argv)

Invocation: python ingest.py ctl-results/rhel/7 rhel7