sanketsudake/dump-analyzer

## dump-analyzer
#!/usr/bin/env bash
###
# This script is used to analyze the dump files generated by the Fission CI.
###
set -o errexit
set -o nounset
set -o pipefail
if [[ -n "${TRACE-}" ]]; then
    set -o xtrace
fi
REPO="fission/fission"

function exit_error() {
    echo "$1"
    exit 1
}

function doit() {
    echo "! $*"
    "$@"
}

function check_context() {
    if [ -z "$DUMP_CONTEXT" ]; then
        exit_error "DUMP_CONTEXT not set"
    fi
}

function list_context() {
    for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do
        echo "==== $dump ===="
        echo "-- Fission Version --"
        cat $dump/fission-version/fission-version.txt | grep "server:\|client\:\|Version"
        echo "-- K8s Version --"
        cat $dump/kubernetes-version/kubernetes-version.txt | grep gitVersion
    done
}

function list_kind() {
    for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do
        echo "==== $dump ===="
        cat $dump/kind-version.txt
        echo -e "\n"
    done
}

function extract() {
    ARTIFACT_PATH=.dumps/$RUN_ID
    if [ ! -d "$ARTIFACT_PATH" ]; then
        doit mkdir -p "$ARTIFACT_PATH"
    fi
    echo $ARTIFACT_PATH
    # Don't download if artifact already exits for run
    if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then
        doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH"
    fi
    if [ -d "$ARTIFACT_PATH"/extract ]; then
        doit rm -r "$ARTIFACT_PATH"/extract
    fi
    DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip')
    for dump in $DUMPS; do
        doit unzip -q $dump -d "$ARTIFACT_PATH"/extract
    done
}

function info() {
    check_context
    echo "Dump context: $DUMP_CONTEXT"
    doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt
    doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt
}

function race_conditions() {
    check_context
    # doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/"
    LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f | grep log)
    for logfile in $LOG_FILES; do
        out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile")
        if [ "$out" != "" ]; then
            shortname=$(basename $logfile)
            echo "Trace from $shortname"
            sed -n '/DATA RACE/,/==================$/p' "$logfile"
        fi
    done
}

function error_logs() {
    check_context
    # ack '"level":"error"' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | go-slearch -pts,caller,msg,error,logger,errorVerbose | sort
    ack -i 'error' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | sort
}

function usage {
    echo "./$(basename "$0") [OPTIONS]"
    echo "Utilities related to fission dump analysis"
    echo "
Options:
    -h              Show usage
    -x [run_id]     Download and extract dump locally
    -l [run_id]    List dump files for a run
    -k [run_id]    List kind exports in a run

    Following options required DUMP_CONTEXT variable set.

    -i              Display dump info
    -r              Find all race conditions in dump
    -e              Find all errors in logs"
    exit 3
}

# list of arguments expected in the input
optstring=":hxirlek"

if [[ ${#} -eq 0 ]]; then
    usage
fi

while getopts ${optstring} arg; do
    case ${arg} in
    h)
        echo "showing usage!"
        usage
        ;;
    x)
        RUN_ID=$2
        if [ -z "$RUN_ID" ]; then
            exit_error "run id not mentioned"
        fi
        extract
        ;;
    l)
        RUN_ID=$2
        if [ -z "$RUN_ID" ]; then
            exit_error "run id not mentioned"
        fi
        list_context
        ;;
    i)
        info
        ;;
    r)
        race_conditions
        ;;
    e)
        error_logs
        ;;
    k)
        RUN_ID=$2
        if [ -z "$RUN_ID" ]; then
            exit_error "run id not mentioned"
        fi
        list_kind
        ;;
    :)
        echo "$0: Must supply an argument to -$OPTARG." >&2
        exit 1
        ;;
    ?)
        echo "Invalid option: -${OPTARG}."
        exit 2
        ;;
    esac
done

## fission-dump-analyze.md

      
    Raw
  

              fission-dump-analyze.md
            
          
    Dump analyzer

Downloading dump

$ ./dump-analyzer -x 3179903728
! mkdir -p .dumps/3179903728
.dumps/3179903728
! gh run download 3179903728 -R fission/fission -D .dumps/3179903728
! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.19.16/fission-dump_1664866624.zip -d .dumps/3179903728/extract
! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.21.12/fission-dump_1664866628.zip -d .dumps/3179903728/extract
! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.20.15/fission-dump_1664866662.zip -d .dumps/3179903728/extract
Check all dumps

./dump-analyzer -l 3179903728
==== .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304 ====
-- Fission Version --
client:
    Version: v0.0.0
server:
    Version: v0.0.0
-- K8s Version --
gitVersion: v1.21.12
==== .dumps/3179903728/extract/e7febf06-79ce-4eb7-bba0-3fc66aa4bf542096519315 ====
-- Fission Version --
client:
    Version: v0.0.0
server:
    Version: v0.0.0
-- K8s Version --
gitVersion: v1.19.16
==== .dumps/3179903728/extract/3b339dc2-e9dd-4f7a-a721-41e16d9d248f4152710428 ====
-- Fission Version --
client:
    Version: v0.0.0
server:
    Version: v0.0.0
-- K8s Version --
gitVersion: v1.20.15
We can see that the dump is from 3 different versions of k8s.
To explore specific dump set DUMP_CONTEXT environment variable.
$ export DUMP_CONTEXT=.dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304

$ ./dump-analyzer -i
Dump context: .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304
! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/fission-version/fission-version.txt
client:
  fission/core:
    BuildDate: "2022-10-04T06:28:54Z"
    GitCommit: 9e74b01
    Version: v0.0.0
server:
  fission/core:
    BuildDate: "2022-10-04T06:29:01Z"
    GitCommit: 9e74b01
    Version: v0.0.0
! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/kubernetes-version/kubernetes-version.txt
buildDate: "2022-05-19T20:02:29Z"
compiler: gc
gitCommit: 696a9fdd2a58340e61e0d815c5769d266fca0802
gitTreeState: clean
gitVersion: v1.21.12
goVersion: go1.16.15
major: "1"
minor: "21"
platform: linux/amd64

# See all errors in the dump
$ ./dump-analyzer -e

# If you want to see error in specific dump you can also grep

$ grep -rin "string" $DUMP_CONTEXT

# OR

$ ack "string" $DUMP_CONTEXT
Kind logs

 ./dump-analyzer -k 3179903728
==== .dumps/3179903728/kind-logs-3179903728-v1.20.15 ====
kind v0.14.0 go1.18.2 linux/amd64

==== .dumps/3179903728/kind-logs-3179903728-v1.19.16 ====
kind v0.14.0 go1.18.2 linux/amd64

==== .dumps/3179903728/kind-logs-3179903728-v1.21.12 ====
kind v0.14.0 go1.18.2 linux/amd64
	#!/usr/bin/env bash
	###
	# This script is used to analyze the dump files generated by the Fission CI.
	###
	set -o errexit
	set -o nounset
	set -o pipefail
	if [[ -n "${TRACE-}" ]]; then
	set -o xtrace
	fi
	REPO="fission/fission"

	function exit_error() {
	echo "$1"
	exit 1
	}

	function doit() {
	echo "! $*"
	"$@"
	}

	function check_context() {
	if [ -z "$DUMP_CONTEXT" ]; then
	exit_error "DUMP_CONTEXT not set"
	fi
	}

	function list_context() {
	for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do
	echo "==== $dump ===="
	echo "-- Fission Version --"
	cat $dump/fission-version/fission-version.txt \| grep "server:\\|client\:\\|Version"
	echo "-- K8s Version --"
	cat $dump/kubernetes-version/kubernetes-version.txt \| grep gitVersion
	done
	}

	function list_kind() {
	for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do
	echo "==== $dump ===="
	cat $dump/kind-version.txt
	echo -e "\n"
	done
	}

	function extract() {
	ARTIFACT_PATH=.dumps/$RUN_ID
	if [ ! -d "$ARTIFACT_PATH" ]; then
	doit mkdir -p "$ARTIFACT_PATH"
	fi
	echo $ARTIFACT_PATH
	# Don't download if artifact already exits for run
	if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then
	doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH"
	fi
	if [ -d "$ARTIFACT_PATH"/extract ]; then
	doit rm -r "$ARTIFACT_PATH"/extract
	fi
	DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip')
	for dump in $DUMPS; do
	doit unzip -q $dump -d "$ARTIFACT_PATH"/extract
	done
	}

	function info() {
	check_context
	echo "Dump context: $DUMP_CONTEXT"
	doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt
	doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt
	}

	function race_conditions() {
	check_context
	# doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/"
	LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f \| grep log)
	for logfile in $LOG_FILES; do
	out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile")
	if [ "$out" != "" ]; then
	shortname=$(basename $logfile)
	echo "Trace from $shortname"
	sed -n '/DATA RACE/,/==================$/p' "$logfile"
	fi
	done
	}

	function error_logs() {
	check_context
	# ack '"level":"error"' $DUMP_CONTEXT \| grep -v "404" \| cut -d':' -f 3- \| go-slearch -pts,caller,msg,error,logger,errorVerbose \| sort
	ack -i 'error' $DUMP_CONTEXT \| grep -v "404" \| cut -d':' -f 3- \| sort
	}

	function usage {
	echo "./$(basename "$0") [OPTIONS]"
	echo "Utilities related to fission dump analysis"
	echo "
	Options:
	-h Show usage
	-x [run_id] Download and extract dump locally
	-l [run_id] List dump files for a run
	-k [run_id] List kind exports in a run

	Following options required DUMP_CONTEXT variable set.

	-i Display dump info
	-r Find all race conditions in dump
	-e Find all errors in logs"
	exit 3
	}

	# list of arguments expected in the input
	optstring=":hxirlek"

	if [[ ${#} -eq 0 ]]; then
	usage
	fi

	while getopts ${optstring} arg; do
	case ${arg} in
	h)
	echo "showing usage!"
	usage
	;;
	x)
	RUN_ID=$2
	if [ -z "$RUN_ID" ]; then
	exit_error "run id not mentioned"
	fi
	extract
	;;
	l)
	RUN_ID=$2
	if [ -z "$RUN_ID" ]; then
	exit_error "run id not mentioned"
	fi
	list_context
	;;
	i)
	info
	;;
	r)
	race_conditions
	;;
	e)
	error_logs
	;;
	k)
	RUN_ID=$2
	if [ -z "$RUN_ID" ]; then
	exit_error "run id not mentioned"
	fi
	list_kind
	;;
	:)
	echo "$0: Must supply an argument to -$OPTARG." >&2
	exit 1
	;;
	?)
	echo "Invalid option: -${OPTARG}."
	exit 2
	;;
	esac
	done