Install openfoam 6.0 in HPC with Intel compiler and MPI

readme.md

install of 6 in SuperMike2 HPC

Reference:

• [1] https://software.intel.com/content/www/us/en/develop/articles/recipe-compile-openfoam-with-intel-compiler-for-intel-xeon-scalable-processors.html
• [2] https://thelinuxcluster.com/category/application/openfoam/

Step1. Compiler and MPI library setup

qsub -I -l nodes=1:ppn=16,walltime=02:00:00 -q workq -A hpc_cdss01

#Download and unpack
cd /project/cs16/tma5/OpenFOAM
git clone https://github.com/OpenFOAM/OpenFOAM-6.git
git clone https://github.com/OpenFOAM/ThirdParty-6.git

cd ThirdParty-6
mkdir download
wget -P download "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz"
tar -zxf download/metis-5.1.0.tar.gz

#load library
module unload mpich
module unload intel/18.0.0
module load INTEL/18.0.0
module load impi/2018.0.128
module load cmake/3.7.2/INTEL-18.0.0

export MPI_ROOT=/home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/intel64

#Check arch version, it should be x86_64 in SuperMike II
uname -m

#compile need to set directory
export PROJECT=/project/cs16/tma5

#Set parallel compiling
export WM_NCOMPPROCS=16

Step2. Compiling parameters setup

cd $PROJECT/OpenFOAM6
1. vi etc/config.sh/mpi 
	case "$WM_MPLIB" in - INTELMPI 
	export MPI_ROOT=/home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/intel64 

2. vi etc/config.sh/settings 
	a. case $WM_ARCH in Linux 
		case `uname -m` in > x86_64 
			case $WM_ARCH_OPTION in 
				i ) 32 
				ii) 64 
					> export WM_CC='icc' 
					> export WM_CXX='icpc' 
					> export WM_CFLAGS='-O2 -no-prec-div -m64 -fPIC'
					> export WM_CXXFLAGS='-O2 -no-prec-div -m64 -fPIC -std=c++0x' 

3. vi etc/bashrc 
	> export FOAM_INST_DIR=$PROJECT/OpenFOAM
	> export WM_COMPILER=Icc 
	> export WM_MPLIB=INTELMPI 

#Setup building environment
source /home/compilers/Intel/parallel_studio_xe_2018.0/compilers_and_libraries/linux/bin/compilervars.sh intel64
source /home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/bin64/mpivars.sh
source /project/cs16/tma5/OpenFOAM/OpenFOAM-6/etc/bashrc WM_LABEL_SIZE=64 WM_COMPILER_TYPE=system


#Then save an alias in the personal .bashrc file, simply by running the following command:
echo "alias of6='source $PROJECT/OpenFOAM/OpenFOAM-6/etc/bashrc $FOAM_SETTINGS'" >> $HOME/.bashrc

Step3. build scotch and metis with Intel compiling parameters

cd $WM_THIRD_PARTY_DIR/etc/wmakeFiles/scotch/
vi Makefile.inc.i686_pc_linux2.shlib-OpenFOAM
	>AR = icc
	>CCS = icc
	>CCP = mpicc
	>CFLAGS = $(WM_CFLAGS) -O3 -fPIC -DCOMMON_FILE_COMPRESS_GZ -DCOMMON_RANDOM_FIXED_SEED -DSCOTCH_RENAME -Drestrict=__restrict -I$(MPI_ROOT)/include -L$(MPI_ROOT)/lib 

cd $WM_THIRD_PARTY_DIR/metis-5.1.0
vi Makefile
	> openmp = -qopenmp
	> cc = mpiicc

cd $WM_THIRD_PARTY_DIR

#Setup building environment
source /home/compilers/Intel/parallel_studio_xe_2018.0/compilers_and_libraries/linux/bin/compilervars.sh intel64
source /home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/bin64/mpivars.sh
source /project/cs16/tma5/OpenFOAM/OpenFOAM-6/etc/bashrc WM_LABEL_SIZE=64 WM_COMPILER_TYPE=system

./Allwmake -j $WM_NCOMPPROCS > log.make 2>&1
wmRefresh

Step4. Build OpenFoam with Intel MPI

cd $WM_PROJECT_DIR
vi wmake/rules/linux64Icc/mplibINTELMPI
	> PINC  = -isystem $(MPI_ARCH_PATH)/include 
	> PLIBS = -L$(MPI_ARCH_PATH)/lib -lmpi
wmRefresh

#For faster compile using 16 cores in SuperMike II
export WM_NCOMPPROCS=16

#!!! unload the intelmpi for my hpc setting to use the self compiled mpi
#Setup building environment
source /home/compilers/Intel/parallel_studio_xe_2018.0/compilers_and_libraries/linux/bin/compilervars.sh intel64
source /home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/bin64/mpivars.sh
source /project/cs16/tma5/OpenFOAM/OpenFOAM-6/etc/bashrc WM_LABEL_SIZE=64 WM_COMPILER_TYPE=system

./Allwmake -j $WM_NCOMPPROCS > log.make 2>&1

#------!!!Testing!!!--------
of6
icoFoam -help

run case

#------run case-------------
module unload mpich
module unload intel/18.0.0
module load INTEL/18.0.0
module load impi/2018.0.128
module load cmake/3.7.2/INTEL-18.0.0


source /home/compilers/Intel/parallel_studio_xe_2018.0/compilers_and_libraries/linux/bin/compilervars.sh intel64
source /home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/bin64/mpivars.sh
source /project/cs16/tma5/OpenFOAM/OpenFOAM-6/etc/bashrc WM_LABEL_SIZE=64 WM_COMPILER_TYPE=system