azure-waf-logs-parser.py

import json
import os
import sys
import argparse
import traceback

def json_load_all(buf):
    while True:
        try:
            yield json.loads(buf)
        except json.JSONDecodeError as err:
            yield json.loads(buf[:err.pos])
            buf = buf[err.pos:]
        else:
            break

def process_file(inp, output_dict):
    try:
        lines = inp.readlines() 
        for line in lines:
            line = line.strip()  # Strips the newline character 
            log_element = json.loads(line)
            if log_element["properties"]["ruleId"] not in output_dict:
                print("NEW RULE ID:" + log_element["properties"]["ruleId"])
                output_dict[log_element["properties"]["ruleId"]] = {
                    "requestUris": [log_element["properties"]["requestUri"]],
                    "messages": [log_element["properties"]["message"]],
                    "count": 1
                }
            else:
                if (log_element["properties"]["requestUri"] not in output_dict[log_element["properties"]["ruleId"]]["requestUris"]):
                    output_dict[log_element["properties"]["ruleId"]]["requestUris"].append(log_element["properties"]["requestUri"])
                if (log_element["properties"]["message"] not in output_dict[log_element["properties"]["ruleId"]]["messages"]):
                    output_dict[log_element["properties"]["ruleId"]]["messages"].append(log_element["properties"]["message"])
                count = output_dict[log_element["properties"]["ruleId"]]["count"] + 1
                output_dict[log_element["properties"]["ruleId"]]["count"] = count
    except Exception:
        traceback.print_exc()

parser = argparse.ArgumentParser(description="WAF logs parser")
parser.add_argument('action', help='The action to take possible values are: flat-file and process. e.g.: waf-parser.py process "." or waf-parser.py process "sample.json" or waf-parser.py flat-file "sample.json" ')
parser.add_argument('file-path', help='Name of json file to process/flat. "." to flat the whole json files in the current folder')
args = parser.parse_args()
print(args)

if args.action == "flat-file":
    file = getattr(args, 'file-path')
    # TODO: parse the case of . and flat all files in current dir
    print("You asked to flat the json file: " + file)
    if ('.json' not in file):
        sys.exit('ERROR: Must provide a .json file as second parameter')

    print(os.path.join(".", file))
    with open(file) as inp, open('flat_'+ file, 'w') as out:
        for obj in json_load_all(inp.read()):
            json.dump(obj, out)
            print(file=out)
        print('Written flat json file to: ' + 'flat_'+ file)
elif args.action == "process":
    output_dict = {}
    with open('output.json', 'w') as out:
        file = getattr(args, 'file-path')
        if file == ".":
            # Process all the files in the current dir
            for file in os.listdir("."):
                if file.endswith(".json"):
                    print("Processing file: " + file)
                    with open(file) as inp:
                        process_file(inp, output_dict)
        else:
            # Process the file passed as second parameter
            if ('.json' not in file):
                sys.exit('ERROR: Must provide a .json file as second parameter')
            with open(file) as inp:
                output_dict = {}
                process_file(inp, output_dict)
        
        json.dump(output_dict, out)
        # print(output_dict) # print to stdout