Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Put log of every Yarn run into different file
#!/usr/bin/env bash
# Start all hadoop daemons. Run this on master node.
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
rm_logfile_name=yarn-yarn-resourcemanager-`hostname`.log
nm_logfile_name=yarn-yarn-nodemanager-`hostname`.log
echo "----------------------Clean logs-----------------------"
#Clean log content of last run
if [ -f "$bin"/../logs/$rm_logfile_name ]; then
> "$bin"/../logs/"$rm_logfile_name"
if [ -s "$bin"/../logs/"$rm_logfile_name" ]; then
rm "$bin"/../logs/"$rm_logfile_name"
else
echo "RM log file is cleaned."
fi
fi
if [ -f "$bin"/../logs/$nm_logfile_name ]; then
> "$bin"/../logs/"$nm_logfile_name"
if [ -s "$bin"/../logs/"$nm_logfile_name" ]; then
rm "$bin"/../logs/"$nm_logfile_name"
else
echo "NM log file is cleaned."
fi
fi
# start yarn resourcemanager
if [ -f "$bin"/yarn-daemon.sh ]; then
echo "----------------------Starting resourcemanager-----------------------"
"$bin"/yarn-daemon.sh start resourcemanager
fi
# start hdfs secondarynamenode
if [ -f "$bin"/yarn-daemon.sh ]; then
echo "----------------------Starting nodemanager-----------------------"
"$bin"/yarn-daemon.sh start nodemanager
fi
#!/usr/bin/env bash
# Stop all hadoop daemons. Run this on master node.
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
rm_logfile_name="$bin"/../logs/yarn-yarn-resourcemanager-`hostname`.log
nm_logfile_name="$bin"/../logs/yarn-yarn-nodemanager-`hostname`.log
rm_logfile_location="$bin"/../logs/rm_log_archive
nm_logfile_location="$bin"/../logs/nm_log_archive
rm_log_basename="resourcemanager-log-"
nm_log_basename="nodemanager-log-"
# stop yarn resourcemanager
if [ -f "$bin"/yarn-daemon.sh ]; then
echo "----------------------Stoping resourcemanager-----------------------"
"$bin"/yarn-daemon.sh stop resourcemanager
fi
# stop hdfs secondarynamenode
if [ -f "$bin"/yarn-daemon.sh ]; then
echo "----------------------Stop nodemanager-----------------------"
"$bin"/yarn-daemon.sh stop nodemanager
fi
echo "----------------------Moving logs to log archive-----------------------"
# Copy RM log of last run to rm log archive
if [ -f "$rm_logfile_name" ]; then
#echo "Original log file is exist."
#Moving log content to defined directory
if [ -d "$rm_logfile_location" ]; then
#echo "User defined rm log directory is exist."
if [[ -n $(find $rm_logfile_location -maxdepth 0 -empty) ]]; then
firstRunLogName="$rm_log_basename"'001.log'
#echo "First run, log name is: ", $firstRunLogName
cat "$rm_logfile_name" > "$rm_logfile_location"/"$firstRunLogName"
echo "Writing log to", $firstRunLogName
else
# echo "Directory is not empty."
file=`ls "$rm_logfile_location"|sort -g -r|head -n1`
lastRunLogFullName=$(basename $file)
extension=${lastRunLogFullName##*.}
lastRunFileName=${lastRunLogFullName%.*}
IFS='-' read -ra ARRAY <<< "$lastRunFileName"
preVersion=${ARRAY[@]:(-1)}
preVersionToDigital=${preVersion#0}
((newVersion=10#$preVersionToDigital+1));
padding_newVersion=$(printf "%03d" $newVersion)
newLogFileName="$rm_log_basename""$padding_newVersion"'.'"$extension"
cat "$rm_logfile_name" > "$rm_logfile_location"/"$newLogFileName"
echo "Writing log to", $newLogFileName
fi
fi
fi
# Copy NM log of last run to nm log archive
if [ -f "$nm_logfile_name" ]; then
#echo "Original log file is exist."
#Moving log content to defined directory
if [ -d "$nm_logfile_location" ]; then
#echo "User defined rm log directory is exist."
if [[ -n $(find $nm_logfile_location -maxdepth 0 -empty) ]]; then
firstRunLogName="$nm_log_basename"'001.log'
#echo "First run, log name is: ", $firstRunLogName
cat "$nm_logfile_name" > "$nm_logfile_location"/"$firstRunLogName"
echo "Writing log to", $firstRunLogName
else
# echo "Directory is not empty."
file=`ls "$nm_logfile_location"|sort -g -r|head -n1`
lastRunLogFullName=$(basename $file)
extension=${lastRunLogFullName##*.}
lastRunFileName=${lastRunLogFullName%.*}
IFS='-' read -ra ARRAY <<< "$lastRunFileName"
preVersion=${ARRAY[@]:(-1)}
preVersionToDigital=${preVersion#0}
((newVersion=10#$preVersionToDigital+1));
padding_newVersion=$(printf "%03d" $newVersion)
newLogFileName="$nm_log_basename""$padding_newVersion"'.'"$extension"
cat "$nm_logfile_name" > "$nm_logfile_location"/"$newLogFileName"
echo "Writing log to", $newLogFileName
fi
fi
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment