Analyses logs data copied from kibana to reconstruct the history of a single key storing JSON on S3

kibana_s3_history.py

"""
This script analyses logs data copied from kibana to reconstruct
the history of a single key storing JSON on S3.
It prints out the first value of the key, followed by coloured diffs
between consecutive values. To use the script:

Open kibana
Open Network tab in Chrome console
Search for the following in kibana:
"'Key': 'key_here'"
(replace key_here but keep all quotes)
Find latest network request starting with _msearch
Right-click > Copy > Copy response
Paste into file
Replace path below with file path
"""

import difflib
import json
import re
from ast import literal_eval

with open('change_path_here') as f:
    data = json.load(f)

sessions = []
for hit in data['responses'][0]['hits']['hits']:
    match = re.search('Calling s3:put_object with (.+)', hit['_source']['message'])
    if match:
        message = match.group(1)
        body = json.loads(literal_eval(message)['Body'].decode('utf8'))
        sessions.append(body)


class Colours(object):
    reset = '\033[0m'
    bold = '\033[1m'
    red = '\033[31m'
    green = '\033[32m'
    cyan = '\033[36m'


def diff_strings(str1, str2):
    if str1 == str2:
        result = 'Results are equal'
    else:
        lines1 = str1.splitlines()
        lines2 = str2.splitlines()

        diff = difflib.unified_diff(lines1, lines2,
                                    n=4, lineterm='')

        def color_diff():
            for line in diff:
                if line.startswith('+'):
                    yield Colours.green + line + Colours.reset
                elif line.startswith('-'):
                    yield Colours.red + line + Colours.reset
                elif line.startswith('@'):
                    yield Colours.cyan + line + Colours.reset
                else:
                    yield line

        result = '\n'.join(color_diff())
    print(result)


def pretty(d):
    return json.dumps(d, indent=4, sort_keys=True)


sessions.reverse()
print(pretty(sessions[0]))
for s1, s2 in zip(sessions, sessions[1:]):
    print('-' * 80)
    diff_strings(pretty(s1), pretty(s2))