Skip to content

Instantly share code, notes, and snippets.

@imShakil
Created August 26, 2023 10:47
Show Gist options
  • Save imShakil/8e3ba6794aeff55bbbbf6095a3c1013d to your computer and use it in GitHub Desktop.
Save imShakil/8e3ba6794aeff55bbbbf6095a3c1013d to your computer and use it in GitHub Desktop.
Qwiklabs Assessment: Log Analysis Using Regular Expressions
#!/usr/bin/env python3
import re
import csv
import operator
errors = {}
stats = {}
errors_pattern = r"ERROR ([\w \']*) "
stats_pattern = r"([A-Z]{3,5})\s([\w ]*).*\(([\w\.\w ]*)\)"
logs = open("syslog.log", "r").readlines()
logs = [log.strip() for log in logs]
def sortResult(res, k=None, rev=False):
sortedResult = sorted(res.items(), key=k, reverse=rev)
return sortedResult
def ranking_of_errors():
for log in logs:
res = re.search(errors_pattern, log)
if res:
if res[1] in errors.keys():
errors[res[1]] += 1
else:
errors[res[1]] = 1
def user_usage():
for log in logs:
res = re.search(stats_pattern, log)
if res:
if res[3] in stats.keys():
stats[res[3]][res[1]] += 1
else:
stats[res[3]] = {}
stats[res[3]]["INFO"] = stats[res[3]]["ERROR"] = 0
stats[res[3]][res[1]] += 1
if __name__ == "__main__":
ranking_of_errors()
user_usage()
errors = sortResult(errors, k=operator.itemgetter(1), rev=True)
stats = sortResult(stats)
#print(errors)
#print(stats)
# create error_message.csv
def create_error_message_csv():
with open("error_message.csv", "w") as file:
csv_file = csv.writer(file)
csv_file.writerow(['Error', 'Count'])
for item in errors:
csv_file.writerow(item)
file.close()
def create_user_statics_csv():
with open("user_statistics.csv", "w") as file:
csv_file = csv.writer(file)
csv_file.writerow(['Username', 'INFO', "ERROR"])
for user, item in stats:
csv_file.writerow([user, item['INFO'], item['ERROR']])
file.close()
create_error_message_csv()
create_user_statics_csv()
@imShakil
Copy link
Author

imShakil commented Aug 28, 2023

Modified code to count both of error message and user stats

def collect_logs():
    pattern = r"(ERROR|INFO)\s([\w \']*).*\(([\w\.\w ]*)\)"
    for log in logs:
        res = re.search(pattern, log) # find 3 groups, 1: ERROR | INFO, 2: Message, 3: username
        if res:
            logLevel, logMessage, userName = res.groups()

            # count logMessage
            if logLevel == 'ERROR':
                if logMessage in errors.keys():
                    errors[logMessage] += 1
                else:
                    errors[logMessage] = 1

            # count user statistics
            if userName in stats.keys():
                stats[userName][logLevel] += 1
            else:
                stats[userName] = {}
                stats[userName]["INFO"] = stats[userName]["ERROR"] = 0
                stats[userName][logLevel] += 1

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment