Skip to content

Instantly share code, notes, and snippets.

View gullevek's full-sized avatar

Clemens Schwaighofer gullevek

View GitHub Profile
@gullevek
gullevek / double_byte_string_format.py
Last active December 3, 2022 06:37
Python class to shorten double byte string and set correct adapted format length for output print
#!/usr/bin/env python3
"""
formatting with double width characters
"""
import unicodedata
def shorten_string_cjk(intput_string, width, placeholder='..'):
@gullevek
gullevek / New New Index Bloat Query
Last active July 25, 2022 17:35 — forked from mbanck/New New Index Bloat Query
This version uses pg_stats and not pg_statistics so any user can use this query
WITH btree_index_atts AS (
SELECT nspname, relname, reltuples, relpages, indrelid, relam,
regexp_split_to_table(indkey::text, ' ')::smallint AS attnum,
indexrelid as index_oid
FROM pg_index
JOIN pg_class ON pg_class.oid=pg_index.indexrelid
JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
JOIN pg_am ON pg_class.relam = pg_am.oid
WHERE pg_am.amname = 'btree'
),
@gullevek
gullevek / human_readable_bytes_to_number.js
Last active May 20, 2021 02:09
javascript to convert human readable bytes (eg 10MB) to a number (10,485,760)
/**
* Convert a string with B/K/M/etc into a byte number
* @param {String|Number} bytes Any string with B/K/M/etc
* @return {String|Number} A byte number, or original string as is
*/
function stringByteFormat(bytes)
{
// if anything not string return
if (!(typeof bytes === 'string' || bytes instanceof String)) {
return bytes;
@gullevek
gullevek / query_to_csv.perl
Last active February 18, 2021 02:50
Writes all data from a given query to a csv file. Usefull for reading out large batches of data to avoid memory over usage. Uses async and CURSOR. Python version only uses CURSOR as named cursor and async do not work at the same time.
#!/usr/bin/perl
# AUTHOR: Clemens Schwaighofer
# DATE: 2015/8/6
# DSCRIPTION:
# Runs a query from a file or command line and outputs the data to a CSV file
# Runs query async/and CURSOR
use strict;
use warnings;
@gullevek
gullevek / file_size_check.sh
Created February 7, 2020 02:32
Loops over files in a folder and checks if they changed. If they stay the same in 3 loops of 5s wait the script will end
#!/bin/bash
folder=${1};
diff=1;
sleeptime=5;
if [ ! -d "${folder}" ]; then
echo "Folder: ${folder} not found";
exit;
fi;
@gullevek
gullevek / php-error-log.json
Last active December 2, 2019 01:21
lnav php error log parser
{
"php_log": {
"title": "PHP error log format",
"description": "Error Log format used by PHP (and PHP-FPM)",
"url": "http://www.php.net/",
"regex": {
"general": {
"pattern": "^\\[(?<date>\\d{2}-\\w{3}-\\d{4}) (?<timestamp>\\d{2}:\\d{2}:\\d{2}) (?<timezone>\\w+(\\/\\w+)?)\\] (?<level>PHP [\\w ]+): (?<body>.*)"
},
"dump": {
@gullevek
gullevek / web-dev-log.json
Created November 29, 2019 07:56
lnav log parser for web development customized
{
"web_dev": {
"title": "Web Development Log",
"description": "Development log from CoreLibs\\Basic Class",
"url": "http://www.egplusww.jp/",
"regex": {
"general": {
"pattern": "^\\[(?<date>\\d{4}-\\d{2}-\\d{2}) (?<timestamp>\\d{2}:\\d{2}:\\d{2}\\.\\d{8})\\] \\[(?<server>[\\w.]+)\\] \\[(?<sourcefile>\\/([A-Za-z0-9-_.]+\\/)+([A-Za-z0-9-_. ]+\\.(inc|php)))\\] \\[(?<puid>[\\w]+)\\] \\{(?<namespaceclass>[\\w]+(\\\\[\\w]+)*)\\} <(?<muid>[\\w ]+)> - (?<body>.*)"
}
},
@gullevek
gullevek / rsync-transfer-log.json
Created November 29, 2019 07:54
lnav rsync custom transfer log parser
{
"rsync": {
"title": "rsync log",
"description": "rsync transfer and progress logs",
"url": "http://www.egplusww.jp/",
"regex": {
"rsync": {
"pattern": "^(?<date>\\d{4}/\\d{2}/\\d{2}) (?<timestamp>\\d{2}:\\d{2}:\\d{2}) \\[(?<opid>\\d+)\\] (?<body>.*)"
},
"progress": {
@gullevek
gullevek / filesize_per_year.sh
Created July 6, 2018 01:11
Find files and group the file size sum per year
find ./ -type f -print0 | xargs -0 stat -f '%Sm %z' -t "%Y" | awk 'function human(x) {s=" B KB MB GB TB EB PB YB ZB"; while (x>=1024 && length(s)>1){x/=1024; s=substr(s,4)} s=substr(s,1,4); xf=(s==" B ")?"%d ":"%.2f"; return sprintf( xf"%s", x, s);}{sum[$1]+= $2;}END{for (date in sum){printf ("%s %s\n", date, human(sum[date]));}}' | sort
@gullevek
gullevek / progress_data_import.sql
Last active May 8, 2018 02:48
PostgreSQL function to monitor progress for select -> insert flow
CREATE OR REPLACE FUNCTION copy_progress(from_date DATE, to_date DATE, p_precision INT DEFAULT 1) RETURNS "varchar"
AS $$
DECLARE
status VARCHAR; -- return status
my_rec RECORD; -- transfer record
-- progress info
pos INT := 1; -- current position
row_count INT := 0; -- overall row count (max)
percent NUMERIC; -- output percent
old_percent NUMERIC := 0.0; -- previous percent