Skip to content

Instantly share code, notes, and snippets.

@millerdev millerdev/pglogslim.py
Last active Nov 10, 2017

Embed
What would you like to do?
#! /usr/bin/env python3
# filter postgres logs to remove parameters of very large queries
#
# https://gist.github.com/millerdev/def20a42d78717174cd5722a9de2055a
import gzip
import re
import signal
import sys
# HACK prevent "BrokenPipeError: [Errno 32] Broken pipe" output on stderr
signal.signal(signal.SIGPIPE, lambda n, f: sys.exit())
PARAM_RE = re.compile(r"(\{.+?\}|ARRAY\[.+?\])")
WHERE_RE = re.compile(
r'(OR \(""locations_sqllocation""\.""rght"" <= \d+ '
r'AND ""locations_sqllocation""\.""lft"" >= \d+ '
r'AND ""locations_sqllocation""\.""tree_id"" = \d+\) ?)+'
)
def param_summary(match):
return "{%s values}" % (match.group(0).count(",") + 1)
def main():
if len(sys.argv) == 1:
msg = "Usage: {} (LOG_FILE | -) [LOG_FILE ...] > NEW_LOG_FILE\n"
sys.stderr.write(msg.format(*sys.argv))
sys.exit()
for name in sys.argv[1:]:
sys.stderr.write("processing {}\n".format(name))
if name == '-':
process_log(sys.stdin)
elif name.endswith(".gz"):
with gzip.open(name, mode='rt', encoding="utf-8") as log:
process_log(log)
else:
with open(name, encoding="utf-8") as log:
process_log(log)
def process_log(log):
for line in log:
if len(line) > 600:
line = PARAM_RE.sub(param_summary, line)
line = WHERE_RE.sub("OR ...", line)
sys.stdout.write(line)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.