Created
March 31, 2020 04:58
-
-
Save vijayakshit/168787b0d292c4879f53599d62200f6a to your computer and use it in GitHub Desktop.
This gist allows you to create a sqlite3 table from similar loglines/strings to analyse the same better
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import csv | |
import sqlite3 | |
from sqlite3 import Error | |
########## Instructions For Use ############# | |
# 0.Using a venv is recomended | |
# 1.Install sqlite3( pip3 install sqlite3 ) | |
# 2.Create a empty .db file in the vicinity | |
# 3.Copy Logs onto a .log file in the vicinity | |
# 4.Add Properties Config depnding on the Need | |
# 5.Run The Script | |
# 6.Connect to the db using a sql client and enjoy | |
########### Config ####################### | |
properties = [ | |
{ | |
"name": "user_id", | |
"substringStart":"/user/", | |
"substringEnd":"/", | |
}, | |
{ | |
"name": "ge_id", | |
"substringStart":"/ge/", | |
"substringEnd":"/", | |
} | |
] | |
INPUT_LOG_FILE_PATH = "./logs_copy.log" | |
DB_PATH = "/Users/akshitvijay/Desktop/ge/tempe/db/ge.db" | |
########### Constants/Globals ########### | |
CONN = None | |
ALL_EVENTS = [] | |
TABLE_NAME = "ge_logs" | |
########### Functions ############ | |
def create_connection(): | |
global DB_PATH,CONN | |
if CONN != None: | |
return | |
try: | |
CONN = sqlite3.connect(DB_PATH) | |
except Error as e: | |
print(e) | |
def commit_and_close_connection(): | |
global CONN | |
if CONN == None: | |
return | |
CONN.commit() | |
CONN.close() | |
def create_table(): | |
global TABLE_NAME, CONN | |
sql_create_table_query = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " ( id integer PRIMARY KEY" | |
for obj_property in properties: | |
sql_create_table_query = sql_create_table_query + " ," + obj_property["name"] + " text" | |
sql_create_table_query = sql_create_table_query + ");" | |
print(sql_create_table_query) | |
try: | |
c = CONN.cursor() | |
c.execute(sql_create_table_query) | |
except Error as e: | |
print(e) | |
def extract_logs(): | |
global ALL_EVENTS | |
with open(INPUT_LOG_FILE_PATH,"r") as f: | |
all_lines = f.readlines() | |
for line in all_lines: | |
event = {} | |
for obj_property in properties: | |
prop_value = line.split(obj_property["substringStart"])[1].split(obj_property["substringEnd"])[0] | |
event[obj_property["name"]] = prop_value | |
ALL_EVENTS.append(event) | |
def insert_into_table(row): | |
global TABLE_NAME, CONN | |
column_names = "" | |
values = "" | |
for property_obj in properties: | |
column_names = column_names + property_obj["name"] + "," | |
values = values +"'"+ row[property_obj["name"]] +"'," | |
column_names = column_names[:-1] | |
values = values[:-1] | |
insert_query = "INSERT INTO "+TABLE_NAME+" ("+ column_names +")" + " VALUES("+ values +")" | |
print(insert_query) | |
cur = CONN.cursor() | |
cur.execute(insert_query) | |
return cur.lastrowid | |
def write_logs_to_table(): | |
global ALL_EVENTS | |
for EVENT in ALL_EVENTS: | |
insert_into_table(EVENT) | |
def select_all(): | |
global TABLE_NAME, CONN | |
cur = CONN.cursor() | |
cur.execute("Select * from "+TABLE_NAME) | |
rows = cur.fetchall() | |
for row in rows: | |
print(row) | |
######## Main ########### | |
if __name__ == '__main__': | |
create_connection() | |
create_table() | |
extract_logs() | |
write_logs_to_table() | |
#select_all() | |
commit_and_close_connection() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment