Skip to content

Instantly share code, notes, and snippets.

@alexmojaki
Created August 28, 2017 12:25
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save alexmojaki/12d17ab3fbca7bd688195bb30786e2c5 to your computer and use it in GitHub Desktop.
Analyses logs data copied from kibana to reconstruct the history of a single key storing JSON on S3
"""
This script analyses logs data copied from kibana to reconstruct
the history of a single key storing JSON on S3.
It prints out the first value of the key, followed by coloured diffs
between consecutive values. To use the script:
Open kibana
Open Network tab in Chrome console
Search for the following in kibana:
"'Key': 'key_here'"
(replace key_here but keep all quotes)
Find latest network request starting with _msearch
Right-click > Copy > Copy response
Paste into file
Replace path below with file path
"""
import difflib
import json
import re
from ast import literal_eval
with open('change_path_here') as f:
data = json.load(f)
sessions = []
for hit in data['responses'][0]['hits']['hits']:
match = re.search('Calling s3:put_object with (.+)', hit['_source']['message'])
if match:
message = match.group(1)
body = json.loads(literal_eval(message)['Body'].decode('utf8'))
sessions.append(body)
class Colours(object):
reset = '\033[0m'
bold = '\033[1m'
red = '\033[31m'
green = '\033[32m'
cyan = '\033[36m'
def diff_strings(str1, str2):
if str1 == str2:
result = 'Results are equal'
else:
lines1 = str1.splitlines()
lines2 = str2.splitlines()
diff = difflib.unified_diff(lines1, lines2,
n=4, lineterm='')
def color_diff():
for line in diff:
if line.startswith('+'):
yield Colours.green + line + Colours.reset
elif line.startswith('-'):
yield Colours.red + line + Colours.reset
elif line.startswith('@'):
yield Colours.cyan + line + Colours.reset
else:
yield line
result = '\n'.join(color_diff())
print(result)
def pretty(d):
return json.dumps(d, indent=4, sort_keys=True)
sessions.reverse()
print(pretty(sessions[0]))
for s1, s2 in zip(sessions, sessions[1:]):
print('-' * 80)
diff_strings(pretty(s1), pretty(s2))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment