- Follow the guide @ https://github.com/mcovalt/FEniCS-MKL/blob/master/Dockerfile
- HPC command https://info.hpc.sussex.ac.uk/hpc-guide/resource.html
- guide2 https://wiki.math.uwaterloo.ca/fluidswiki/index.php?title=FEniCS
Enter a node ssh mike012
Login:
ssh -X bwang31@qbc.loni.org
srun -t 5:00:00 -n48 -N1 -A loni_mp2020 -p bigmem --pty /bin/bash
srun -t 5:00:00 -n48 -N1 -A loni_mp2020 -p checkpt --pty /bin/bash
#Using node for faster compile
srun -t 4:00:00 -n24 -N1 -A loni_mp2020 -p workq --pty /bin/bash
export NGS_ROOT=/project/karsten/bwang31/ngsolve
mkdir -p $NGS_ROOT && cd $NGS_ROOT
module purge
module load python/3.7.6
module load cmake/3.16.2/intel-19.0.5
module load gcc/9.3.0
export CFLAGS='-O3'
export CXXFLAGS='-O3'
export FCFLAGS='-O3'
export F77FLAGS='-O3'
export F90FLAGS='-O3'
export MPICCFLAGS='-O3'
export MPICXXFLAGS='-O3'
export MPIF77FLAGS='-O3'
export MPIF90FLAGS='-O3'
export MKL_NUM_THREADS=1
export OMP_NUM_THREADS=1
cd $NGS_ROOT
wget -P download "https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.6.tar.gz"
mkdir -p deps && cd deps
tar -zxf ../download/openmpi-3.1.6.tar.gz
cd $NGS_ROOT/deps/openmpi-3.1.6
#without-verbs is used to disable " OpenFabrics device error "
#ref: https://www.open-mpi.org/faq/?category=openfabrics
./configure --prefix=$NGS_ROOT/deps/openmpi-3.1.6/openmpi-lib
make all install
export OPENMPI_PATH=$NGS_ROOT/deps/openmpi-3.1.6/openmpi-lib
export PATH=$PATH:$OPENMPI_PATH/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$OPENMPI_PATH/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/packages/python/3.7.6/lib/
#small dependency for ngsolve
pip install --user mpi4py
pip install petsc petsc4py #this is only required for ngsolve-petsc module
export BASEDIR=$NGS_ROOT
cd $BASEDIR && git clone https://github.com/NGSolve/ngsolve.git ngsolve-src
cd $BASEDIR/ngsolve-src && git submodule update --init --recursive
mkdir $BASEDIR/ngsolve-build
mkdir $BASEDIR/ngsolve-install
#We need to define this first to avoid error while building
export NETGENDIR="${BASEDIR}/ngsolve-install/bin"
export PATH=$NETGENDIR:$PATH
export PYTHONPATH=$NETGENDIR/../`python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1,0,''))"`:$PYTHONPATH
cd $BASEDIR/ngsolve-build
cmake -DCMAKE_INSTALL_PREFIX=${BASEDIR}/ngsolve-install ${BASEDIR}/ngsolve-src \
-DCMAKE_BUILD_TYPE=RELEASE \
-DBUILD_STUB_FILES=OFF \
-DPYTHON_EXECUTABLE:FILEPATH=$(which python) \
-DPYTHON_INCLUDE_DIR=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
-DPYTHON_LIBRARY=$(python -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \
-DUSE_GUI=OFF \
-DUSE_UMFPACK=ON \
-DUSE_MUMPS=ON \
-DUSE_HYPRE=OFF \
-DUSE_MPI=ON \
-DMKL_SDL=OFF \
-DUSE_MKL=ON \
-DLAPACK_LIBRARIES=${MKLROOT}/lib/intel64 \
-DCMAKE_CXX_FLAGS="-O3 -std=c++17" \
-DCMAKE_C_FLAGS="-O3"
make # parallel build may fail
make install
/project/karsten/bwang31/ngsolve/bashrc.sh
export NGS_ROOT=/project/karsten/bwang31/ngsolve
module purge
module load python/3.7.6
module load cmake/3.16.2/intel-19.0.5
module load gcc/9.3.0
export CFLAGS='-O3'
export CXXFLAGS='-O3'
export FCFLAGS='-O3'
export F77FLAGS='-O3'
export F90FLAGS='-O3'
export MPICCFLAGS='-O3'
export MPICXXFLAGS='-O3'
export MPIF77FLAGS='-O3'
export MPIF90FLAGS='-O3'
export MKL_NUM_THREADS=1
export OMP_NUM_THREADS=1
export OPENMPI_PATH=$NGS_ROOT/deps/openmpi-3.1.6/openmpi-lib
export PATH=$PATH:$OPENMPI_PATH/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$OPENMPI_PATH/lib
export BASEDIR=$NGS_ROOT
export NETGENDIR="${BASEDIR}/ngsolve-install/bin"
export PATH=$NETGENDIR:$PATH
export PYTHONPATH=$NETGENDIR/../`python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1,0,''))"`:$PYTHONPATH
cd /work/bwang31/ngsolve
source /project/karsten/bwang31/ngsolve/bashrc.sh
cd /work/bwang31/ngsolve
cp ${BASEDIR}/ngsolve-install/share/ngsolve/py_tutorials/mpi/mpi_navierstokes.py .
# Copy the following modified mpi_naiversoktes
# Run simulation
#--mca btl '^openib' is used to disable openfabrics error
mpirun -np 8 ngspy mpi_navierstokes.py --mca btl '^openib'
#Union VTK into a single time-series XDMF
#Must using the following fname format
#file_name = f"{output_path}/vtkout_time{t:.3f}_proc{rank} "
python unionSolution.py --dir navierstokes_output
#View xdmf and h5 using paraview