Skip to content

Instantly share code, notes, and snippets.

@vijayakshit
Created March 31, 2020 04:58
Show Gist options
  • Save vijayakshit/168787b0d292c4879f53599d62200f6a to your computer and use it in GitHub Desktop.
Save vijayakshit/168787b0d292c4879f53599d62200f6a to your computer and use it in GitHub Desktop.
This gist allows you to create a sqlite3 table from similar loglines/strings to analyse the same better
import json
import csv
import sqlite3
from sqlite3 import Error
########## Instructions For Use #############
# 0.Using a venv is recomended
# 1.Install sqlite3( pip3 install sqlite3 )
# 2.Create a empty .db file in the vicinity
# 3.Copy Logs onto a .log file in the vicinity
# 4.Add Properties Config depnding on the Need
# 5.Run The Script
# 6.Connect to the db using a sql client and enjoy
########### Config #######################
properties = [
{
"name": "user_id",
"substringStart":"/user/",
"substringEnd":"/",
},
{
"name": "ge_id",
"substringStart":"/ge/",
"substringEnd":"/",
}
]
INPUT_LOG_FILE_PATH = "./logs_copy.log"
DB_PATH = "/Users/akshitvijay/Desktop/ge/tempe/db/ge.db"
########### Constants/Globals ###########
CONN = None
ALL_EVENTS = []
TABLE_NAME = "ge_logs"
########### Functions ############
def create_connection():
global DB_PATH,CONN
if CONN != None:
return
try:
CONN = sqlite3.connect(DB_PATH)
except Error as e:
print(e)
def commit_and_close_connection():
global CONN
if CONN == None:
return
CONN.commit()
CONN.close()
def create_table():
global TABLE_NAME, CONN
sql_create_table_query = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " ( id integer PRIMARY KEY"
for obj_property in properties:
sql_create_table_query = sql_create_table_query + " ," + obj_property["name"] + " text"
sql_create_table_query = sql_create_table_query + ");"
print(sql_create_table_query)
try:
c = CONN.cursor()
c.execute(sql_create_table_query)
except Error as e:
print(e)
def extract_logs():
global ALL_EVENTS
with open(INPUT_LOG_FILE_PATH,"r") as f:
all_lines = f.readlines()
for line in all_lines:
event = {}
for obj_property in properties:
prop_value = line.split(obj_property["substringStart"])[1].split(obj_property["substringEnd"])[0]
event[obj_property["name"]] = prop_value
ALL_EVENTS.append(event)
def insert_into_table(row):
global TABLE_NAME, CONN
column_names = ""
values = ""
for property_obj in properties:
column_names = column_names + property_obj["name"] + ","
values = values +"'"+ row[property_obj["name"]] +"',"
column_names = column_names[:-1]
values = values[:-1]
insert_query = "INSERT INTO "+TABLE_NAME+" ("+ column_names +")" + " VALUES("+ values +")"
print(insert_query)
cur = CONN.cursor()
cur.execute(insert_query)
return cur.lastrowid
def write_logs_to_table():
global ALL_EVENTS
for EVENT in ALL_EVENTS:
insert_into_table(EVENT)
def select_all():
global TABLE_NAME, CONN
cur = CONN.cursor()
cur.execute("Select * from "+TABLE_NAME)
rows = cur.fetchall()
for row in rows:
print(row)
######## Main ###########
if __name__ == '__main__':
create_connection()
create_table()
extract_logs()
write_logs_to_table()
#select_all()
commit_and_close_connection()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment