|
#!/usr/bin/env bash |
|
|
|
# Senghan Bright |
|
# Delta Projects |
|
|
|
|
|
# TODO: read predefined searches from a .INI file to make management easier for people? |
|
# or sync directly with what already exists in Kibana |
|
declare -A el_predefined_searches |
|
el_predefined_searches=( |
|
# [bidrequests]='' |
|
['delta-java-app']='' |
|
# [delta-metrics]='' |
|
['filebeat']='' |
|
['filebeat-*:airflow']='beat.hostname: *airflow* AND source: *airflow*' |
|
['filebeat-*:airflow_slimer']='beat.hostname: *airflow* AND source:*slimer_batch_*_dag*' |
|
['flink']='' |
|
['fluentd-*:dogfight']='host:dogfight* AND source:dogfight' |
|
['fluentd-*:haproxy']='host:dflb-ssl* AND ident:haproxy' |
|
['fluentd-*:mailgw_postfix']='host:mailgw* AND ident:postfix*' |
|
['fluentd-*:trackbox']='host:trackbox* AND source:trackbox-logs' |
|
# [googlebidrequests]='' |
|
# [logstash]='' |
|
['metricbeat']='' |
|
['rancher-default']='' |
|
['rancher-local']='' |
|
) |
|
|
|
declare -a fzf_options |
|
# fzf_options=('--layout=default' '--inline-info' '--no-sort' '--no-clear' '--ansi' '--no-border' '--preview-window down:1') |
|
fzf_options=('--layout=default' '--inline-info' '--no-sort' '--no-clear' '--ansi' '--no-border') |
|
|
|
# these bindings are passive and do not alter the menu state |
|
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" |
|
declare -A fzf_navigation_keys |
|
fzf_navigation_keys=( |
|
[page-down]='ctrl-f' |
|
[page-up]='ctrl-b' |
|
[top]='ctrl-t' |
|
[kill-line]='ctrl-k' |
|
[toggle-preview]='?' |
|
) |
|
# ["execute-silent(source $script_dir/toggle_elktail.sh; toggle_elktail)"]='ctrl-p' |
|
|
|
|
|
# these bindings will exit the menu and perform an action |
|
# user can set environment variables to override default bindings |
|
# (or values read from a .INI file will override bindings) |
|
declare -A fzf_postmenuaction_keys |
|
fzf_postmenuaction_keys=( |
|
['change_source']="${TBANA_SOURCE_KEY:-tab}" |
|
['navigate_back']="${TBANA_BACK_KEY:-esc}" |
|
['select_date']="${TBANA_DATE_KEY:-f1}" |
|
) |
|
|
|
declare -a HOSTS_CACHE |
|
declare -a MENU_SOURCES=('HOSTS_CACHE' 'EL_SEARCHES') |
|
ACTIVE_MENU=0 |
|
|
|
elktail_bin=/usr/local/bin/elktail |
|
elktail_initial_entries=5000 |
|
|
|
el_host='elasticsearch.service.consul:9200' |
|
el_log_format='%@timestamp %host %level %container_name%ident %message%short_message%log' |
|
|
|
COLOR_RED='\x1b[38;5;1m' |
|
COLOR_GREEN='\x1b[38;5;6m' |
|
COLOR_YELLOW='\x1b[38;5;2m' |
|
COLOR_GREY='\x1b[38;5;8m' |
|
COLOR_RESET='\x1b[0m' |
|
|
|
|
|
# Usage: cut "string" "delimiter" "field" |
|
cut() { |
|
local string=$1 |
|
local delimiter=$2 |
|
local field=$3 |
|
|
|
local result |
|
IFS=$delimiter read -r -a result <<< "$string" |
|
printf '%s' "${result[field]}" |
|
} |
|
|
|
|
|
# Usage: regex "string" "regex" |
|
regex() { |
|
[[ $1 =~ $2 ]] && printf '%s\n' "${BASH_REMATCH[1]}" |
|
} |
|
|
|
|
|
cycle_menu() { |
|
local source_total |
|
source_total=$(( ${#MENU_SOURCES[@]} - 1 )) |
|
((ACTIVE_MENU++)) |
|
if [[ $ACTIVE_MENU -gt $source_total ]]; then |
|
ACTIVE_MENU=0 |
|
fi |
|
} |
|
|
|
generate_help_text() { |
|
local help_text='' |
|
IFS=$',' |
|
for binding in $fzf_binds; do |
|
key=$(cut "$binding" ':' 0) |
|
action=$(cut "$binding" ':' 1) |
|
help_text+="\o033[32m$key\o033[39m$action " |
|
done |
|
unset IFS |
|
|
|
echo "$help_text" |
|
} |
|
|
|
|
|
generate_fzf_binds() { |
|
local -n array_reference=$1 |
|
local key |
|
local fzf_bindings='' |
|
|
|
for action in "${!array_reference[@]}"; do |
|
key=${array_reference[$action]} |
|
if [[ $1 =~ ^fzf_navigation* ]]; then |
|
fzf_bindings+="$key:$action," # navigation |
|
else |
|
fzf_bindings+="$key:execute(echo $key)+accept," # post-menu action |
|
fi |
|
done |
|
# remove trailing comma |
|
echo "${fzf_bindings::-1}" |
|
} |
|
|
|
|
|
# |
|
get_predefined_searches() { |
|
local menu_entries='' |
|
for array_key in "${!el_predefined_searches[@]}"; do |
|
menu_entries+=$(format_el_query "$array_key") |
|
done |
|
echo -en "$menu_entries" | sort |
|
} |
|
|
|
|
|
# generates a list of dates from today -> ((today - 365 days)) |
|
generate_dates() { |
|
local start_date=$1 |
|
local generated_date |
|
local date_list |
|
local date_suffix |
|
|
|
local count |
|
count=0 |
|
while [ $count -lt 363 ]; do |
|
generated_date=$(date +"%d-%b-%Y" -d "$start_date -$count days" ) |
|
date_list+="$generated_date\n" |
|
|
|
# optimisation: Generate dates we know each month has without using `date` |
|
if [[ $generated_date == '28-'* ]]; then |
|
date_suffix=${generated_date:2:9} |
|
for d in {27..1}; do |
|
date_list+="${d}${date_suffix}\n" |
|
((count++)) |
|
[ $count -gt 363 ] && break |
|
done |
|
fi |
|
((count++)) |
|
done |
|
echo "$date_list" |
|
} |
|
|
|
|
|
# cache the output of `knife node list` |
|
get_hosts() { |
|
local cache=$1 |
|
local cache_file |
|
local nodes |
|
|
|
cache_file="$script_dir/.hosts_cache" |
|
# is the $cache var empty? |
|
if [[ -z $cache ]]; then |
|
# check if there's a cache file on disk |
|
if [[ $(find "$cache_file" -mmin -$(( 24 * 60 )) 2>/dev/null) ]]; then |
|
cache=$(<"$cache_file") |
|
else |
|
nodes=$(knife node list) |
|
# strip the domain |
|
cache=${nodes//.se-ix.delta.prod/} |
|
# store cache on disk |
|
echo -n "$cache" > "$cache_file" |
|
fi |
|
fi |
|
echo -n "${cache[*]}" |
|
} |
|
|
|
|
|
# Show menu of predefined queries or hosts |
|
show_fzf_menu() { |
|
local content=$1 |
|
|
|
# add any options sent as additional parameters |
|
declare -a options |
|
options=("${fzf_options[@]}") |
|
if [ -n "${*:2}" ]; then |
|
options+=("${@:2}") |
|
fi |
|
|
|
local bindings |
|
bindings=$(generate_fzf_binds fzf_navigation_keys) |
|
bindings+=",$(generate_fzf_binds fzf_postmenuaction_keys)" |
|
# bindings+=',ctrl-o:execute(/usr/local/bin/dialog --print-text-only "hello world" 8 52; sleep 2)' |
|
bindings+=',ctrl-o:execute(tput smcup; vim "~/.skhdrc")' |
|
echo -en "$content" | fzf "${options[@]}" --bind "$bindings" |
|
} |
|
|
|
|
|
show_fzf_logview() { |
|
local log=1 |
|
} |
|
|
|
# |
|
select_date() { |
|
local date_list |
|
local remaining_dates |
|
local date_search_start |
|
local date_search_end |
|
local preview_title |
|
|
|
preview_title="\\${COLOR_RED}search-range:\\${COLOR_RESET}" |
|
date_list=$(generate_dates "$(date +%F)") |
|
date_search_start=$(show_fzf_menu "$date_list") |
|
|
|
# remove all dates before $date_search_start |
|
remaining_dates=$(regex "$date_list" "(.*$date_search_start).*") |
|
# display the list in reverse so dates close to $date_search_start are easy to select |
|
date_search_end=$(show_fzf_menu "$remaining_dates" --tac) |
|
} |
|
|
|
|
|
format_el_query() { |
|
local input=$1 |
|
local index_name |
|
local query_pattern |
|
if [[ $input == *":"* ]]; then |
|
index_name=$(cut "$input" ':' 0) |
|
query_pattern=$(cut "$input" ':' 1) |
|
else |
|
index_name=$input |
|
query_pattern='' |
|
fi |
|
# index_doc_count=$(curl -s "http://elasticsearch.service.consul:9200/$index_name/_count" | jq '.count') |
|
|
|
# apply formatting and color |
|
echo "${COLOR_RED}[${COLOR_RESET}${index_name}${COLOR_RED}]${COLOR_RESET} ${COLOR_GREEN}${query_pattern}${COLOR_RESET}\n" |
|
} |
|
|
|
|
|
offset_date() { |
|
local epoch_time=$1 |
|
local offset=$2 |
|
local offset_epoch |
|
offset_epoch=$(("$epoch_time" + "$offset")) |
|
# return date in human readable format |
|
date -d @$offset_epoch '+%Y-%m-%dT%H:%M+02:00' |
|
} |
|
|
|
|
|
init() { |
|
echo "Generating hosts cache..." |
|
HOSTS_CACHE=$(get_hosts "$HOSTS_CACHE") |
|
# TODO: also cache our highlighted predefined searches (just as a var) |
|
EL_SEARCHES=$(get_predefined_searches) |
|
clear |
|
} |
|
|
|
main() { |
|
local menu_content |
|
local selected_line |
|
local key_pressed |
|
|
|
menu_content=${MENU_SOURCES[$ACTIVE_MENU]} |
|
fzf_output=$(show_fzf_menu "${!menu_content}") |
|
|
|
# split output |
|
key_pressed=${fzf_output%%$'\n'*} |
|
selected_line=${fzf_output#*$'\n'} |
|
|
|
# Handle action keys |
|
case $key_pressed in |
|
${TBANA_BACK_KEY:-esc}) |
|
echo "OHMAN" |
|
exit |
|
;; |
|
${TBANA_SOURCE_KEY:-tab}) |
|
cycle_menu |
|
main |
|
;; |
|
${TBANA_DATE_KEY:-f1}) |
|
select_date |
|
echo "OOT" |
|
exit |
|
;; |
|
esac |
|
|
|
|
|
# do we really have a choice? |
|
if [ -n "$selected_line" ]; then |
|
local el_index_pattern |
|
local el_query |
|
local el_query_pattern |
|
|
|
case ${MENU_SOURCES[ACTIVE_MENU]} in |
|
HOSTS_CACHE) |
|
el_index_pattern='fluentd-[0-9]' |
|
el_query="host:$selected_line" |
|
;; |
|
EL_SEARCHES) |
|
el_index_pattern=$(cut "$selected_line" ' ' 0) |
|
el_index_pattern=$(regex "$el_index_pattern" '\[(.*)\]') # remove braces |
|
|
|
el_query_pattern=$(cut "$selected_line" ' ' 1) |
|
if [[ -n $el_query_pattern ]]; then |
|
el_query=${el_predefined_searches[${el_index_pattern}:${el_query_pattern}]} |
|
else |
|
el_query='*' |
|
fi |
|
;; |
|
esac |
|
|
|
edit_key=${FZF_CTRL_R_EDIT_KEY:-ctrl-space} |
|
exec_key=${FZF_CTRL_R_EXEC_KEY:-ctrl-x} |
|
back_key=${FZF_CTRL_R_BACK_KEY:-esc} |
|
|
|
|
|
logentry_selection=$( |
|
# TODO: improve this. |
|
# highlight date-time column, application and message(message can be colored for warn/error) |
|
$elktail_bin -n $elktail_initial_entries -url $el_host -f "$el_log_format" -i "$el_index_pattern" \ |
|
"$el_query" | sed \ |
|
-e "s/^\([0-9\-]*T[0-9:\.\+Z]*\)/$COLOR_GREY\1$COLOR_RESET/" \ |
|
-e "s/\b\(ERROR\)\b/$COLOR_RED\1$COLOR_RESET/" \ |
|
-e "s/\b\(WARN\)\b/$COLOR_YELLOW\1$COLOR_RESET/" | |
|
$fzf_command --header "$el_index_pattern::$el_query" --ansi \ |
|
--preview 'echo :: {3..}' --tac --preview-window down:1:hidden \ |
|
--bind "$(generate_fzf_binds fzf_navigation_keys),$back_key:execute(kill -INT $(pgrep elktail) 2>/dev/null; echo $back_key)+accept" \ |
|
--expect "$edit_key","$exec_key") |
|
|
|
key_pressed=${logentry_selection%%$'\n'*} |
|
selection=${logentry_selection#*$'\n'} |
|
select_source_key=${FZF_CTRL_R_BACK_KEY:-tab} |
|
|
|
case $key_pressed in |
|
$edit_key) result=$(echo "edit");; |
|
$exec_key) result=$(echo "exec");; |
|
$select_source_key) echo "BOO" && exit 0;; |
|
$back_key) main;; |
|
esac |
|
|
|
exit 0 |
|
|
|
local entry_time |
|
local entry_time_epoch |
|
entry_time=$(cut "$log_selection" ' ' 0) |
|
# convert to epoch time |
|
entry_time_epoch=$(date -d "$entry_time" +%s) |
|
|
|
# NOTE: elktail ignores "+02:00" time specifier? |
|
# offset the time manually |
|
local log_begin |
|
local log_end |
|
log_begin=$(offset_date "$entry_time_epoch" -300) # 5mins before |
|
log_end=$(offset_date "$entry_time_epoch" 300) # 5mins after |
|
# TODO: make the context log default to showing the selected hoost |
|
|
|
$elktail_bin -url $el_host -a "$log_begin" -b "$log_end" -f "$el_log_format" -n 10000 -i "$el_index_pattern" "$el_query" | less --tabs=4 --no-init --LONG-PROMPT --ignore-case --quit-if-one-screen --RAW-CONTROL-CHARS |
|
fi |
|
} |
|
|
|
# MAIN |
|
init |
|
main |
|
exit 0 |
