cosven/extract_sql.py

## extract_sql.py
#!/usr/bin/env python3

import re


def get_key_value(key, line):
    """get key's value in log line"""
    p = re.compile(r'\[{}\='.format(key))
    m = p.search(line)
    if m is None:
        raise SystemExit("get key({})'s value failed".format(key))
    stack = []
    start = m.end()
    end = start
    for i, c in enumerate(line[start:]):
        if c == '[':
            stack.append(c)
            continue
        if c == ']':
            if stack:
                stack.pop()
            else:
                end = start + i + 1
                break
    return line[start:end - 1]


def parse_line(line):
    """parse sql statement from line

    :return: (conn_id, sql) tuple
    """
    sql = get_key_value('sql', line)
    conn = get_key_value('conn', line)
    return conn, sql


def extract_lines(filepath):
    """extract lines from log file"""
    # match log time prefix: [2019/11
    line_start_p = re.compile(r'\[\d{4}\/\d{2}')
    with open(filepath) as f:
        line = None
        for broken_line in f:
            if line_start_p.match(broken_line) is None:
                assert line is not None
                line += broken_line
            else:  # brand new line arrived
                # check if there exists old line
                if line is not None:
                    yield line
                line = broken_line


if __name__ == '__main__':
    import sys

    log_filename = sys.argv[1]
    for line in extract_lines(log_filename):
        if 'GENERAL_LOG' not in line:
            continue
        conn_id, sql = parse_line(line)
        print(conn_id, sql)
	#!/usr/bin/env python3

	import re


	def get_key_value(key, line):
	"""get key's value in log line"""
	p = re.compile(r'\[{}\='.format(key))
	m = p.search(line)
	if m is None:
	raise SystemExit("get key({})'s value failed".format(key))
	stack = []
	start = m.end()
	end = start
	for i, c in enumerate(line[start:]):
	if c == '[':
	stack.append(c)
	continue
	if c == ']':
	if stack:
	stack.pop()
	else:
	end = start + i + 1
	break
	return line[start:end - 1]


	def parse_line(line):
	"""parse sql statement from line

	:return: (conn_id, sql) tuple
	"""
	sql = get_key_value('sql', line)
	conn = get_key_value('conn', line)
	return conn, sql


	def extract_lines(filepath):
	"""extract lines from log file"""
	# match log time prefix: [2019/11
	line_start_p = re.compile(r'\[\d{4}\/\d{2}')
	with open(filepath) as f:
	line = None
	for broken_line in f:
	if line_start_p.match(broken_line) is None:
	assert line is not None
	line += broken_line
	else: # brand new line arrived
	# check if there exists old line
	if line is not None:
	yield line
	line = broken_line


	if __name__ == '__main__':
	import sys

	log_filename = sys.argv[1]
	for line in extract_lines(log_filename):
	if 'GENERAL_LOG' not in line:
	continue
	conn_id, sql = parse_line(line)
	print(conn_id, sql)