A bash script recommending a Hadoop memory configuration depending on CPU cores and available physical RAM.

memory-config

#/bin/bash
 
phymem=$(free -m|awk '/^Mem:/{print $2}')
echo "Total physical memory on this machine is $phymem MB"

if [ -n "$1" ]
then
	phymem=$1
	echo "Total physical memory set to $phymem MB"
fi
 
numcores=$(nproc)
echo "Total cores is" $numcores

if [ -n "$2" ]
then
	numcores=$2
	echo "Available cores set to $numcores"
fi
 
systemres=$(expr $phymem / 8)
echo "Recommended reserved system memory $systemres MB"
 
availram=$(expr $phymem - $systemres)
echo "Recommended available system memory $availram MB"
 
mincontainersize=256
if [ "$phymem" -gt 4096 ]
then
	mincontainersize=512
fi
if [ "$phymem" -gt 8192 ]
then
	mincontainersize=1024
fi
if [ "$phymem" -gt 24576 ]
then
	mincontainersize=2048
fi
echo "Recommended minimum container size $mincontainersize MB"
 
numcontainer=$numcores
if [ "$(expr $availram / $mincontainersize)" -lt "$numcores" ]
then
	numcontainers=$(expr $phymem / mincontainersize)
fi
echo "Recommended maximum number of containers $numcontainer"
 
rampercontainer=$mincontainersize
if [ "$(expr $availram / $numcontainer)" -gt "$mincontainersize" ]
then
	rampercontainer=$(expr $availram / $numcontainer)
fi
echo "Recommended memory per container is $rampercontainer"
 
yarn_nodemanager_resource_memory_mb=$(expr $numcontainer \* $rampercontainer)
yarn_scheduler_minimum_allocation_mb=$rampercontainer
yarn_scheduler_maximum_allocation_mb=$(expr $numcontainer \* $rampercontainer)
mapreduce_map_memory_mb=$rampercontainer
mapreduce_reduce_memory_mb=$(expr 2 \* $rampercontainer)
mapreduce_map_java_opts=$(expr 8 \* $mapreduce_map_memory_mb / 10)
mapreduce_reduce_java_opts=$(expr 8 \* $mapreduce_reduce_memory_mb / 10)
yarn_app_mapreduce_am_resource_mb=$(expr 2 \* $rampercontainer)
yarn_app_mapreduce_am_command_opts=$(expr 8 \* $yarn_app_mapreduce_am_resource_mb / 10)
 
echo "Recommended configuration:"
echo "--------------------------"
echo "yarn-site.xml    yarn.nodemanager.resource.cpu-vcores  $numcores"
echo "yarn-site.xml    yarn.nodemanager.resource.memory-mb   $yarn_nodemanager_resource_memory_mb"
echo "yarn-site.xml    yarn.scheduler.minimum-allocation-mb  $yarn_scheduler_minimum_allocation_mb"
echo "yarn-site.xml    yarn.scheduler.maximum-allocation-mb  $yarn_scheduler_maximum_allocation_mb"
echo "yarn-site.xml    yarn.app.mapreduce.am.resource.mb     $yarn_app_mapreduce_am_resource_mb"
echo "yarn-site.xml    yarn.app.mapreduce.am.command-opts    -Xmx${yarn_app_mapreduce_am_command_opts}m"
echo "mapred-site.xml  mapreduce.map.memory.mb               $mapreduce_map_memory_mb"
echo "mapred-site.xml  mapreduce.reduce.memory.mb            $mapreduce_reduce_memory_mb"
echo "mapred-site.xml  mapreduce.map.java.opts               -Xmx${mapreduce_map_java_opts}m"
echo "mapred-site.xml  mapreduce.reduce.java.opts            -Xmx${mapreduce_reduce_java_opts}m"