Script to record maxmimum GPU memory usage of the first card using nvidia-smi in a time period.

MaxGpuMem.sh

#!/bin/bash
max=0
while true; do
	tmp=$(nvidia-smi | grep 11019MiB | awk '{split($0,a,"|"); print a[3]}' | awk '{split($0, a, "/"); print a[1]}' | awk 'NR==1{print $1}' | awk '{split($0, a, "M"); print a[1]}')
	if [[ "$tmp" > "$max" ]]; then
		max=$tmp
	fi
	echo $max "MiB"
	sleep .5;
done