Skip to content

Instantly share code, notes, and snippets.

@BinWang0213

BinWang0213/SubJobs.py

Last active August 22, 2021 11:17
Show Gist options
  • Save BinWang0213/9492b60f3127e2aec0c17937c6b96459 to your computer and use it in GitHub Desktop.
Save BinWang0213/9492b60f3127e2aec0c17937c6b96459 to your computer and use it in GitHub Desktop.
Download ZIP
Compile fenics with Intel Compiler and MPI in SuperMikeII HPC Cluster
Raw
readme.md

Fenics 2019.1.0 install on SuperMike II

Follow the guide @ https://github.com/mcovalt/FEniCS-MKL/blob/master/Dockerfile HPC command https://info.hpc.sussex.ac.uk/hpc-guide/resource.html

Environment setup

#Using node for faster compile
qsub -I -l nodes=1:ppn=16,walltime=04:00:00 -q workq -A hpc_hpc_cdss01

#Enter a node
ssh mike012

export FENICS_ROOT=/project/cs16/tma5/fenics
#export FENICS_ROOT=/project/cs16/tma5/fenics_bigmemt

cd $FENICS_ROOT

module purge

module load git/2.17
module load python/3.6.4-anaconda
#module load boost/1.63.0/INTEL-18.0.0
#module load gcc/6.4.0
module load intel/18.0.0
module load cmake/3.7.2/INTEL-18.0.0
module load impi/2018.0.128
source /home/compilers/Intel/parallel_studio_xe_2018.0/compilers_and_libraries/linux/bin/compilervars.sh intel64
source /home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/bin64/mpivars.sh

Build Cmake

wget -P download https://www.cmake.org/files/v3.9/cmake-3.9.0.tar.gz
tar -xzf download/cmake-3.9.0.tar.gz
env CC=mpiicc CXX=mpiicpc ./bootstrap

Build Petsc with mumps, scotch, etc

wget -P download "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.14.0.tar.gz"
tar -zxf download/petsc-3.14.0.tar.gz

cd petsc-3.14.0

export PETSC_DIR=$PWD
export PETSC_ARCH=centos-intelmkl

./configure \
        --with-cc=mpiicc \
        --with-cxx=mpiicpc \
        --with-fc=mpiifort \
        --with-shared-libraries \
        --with-debugging=0 \
        --with-c-support \
        --with-cxx-dialect=C++11 \
        --known-mpi-shared-libraries=1 \
        --COPTFLAGS="-O3 -xHost -ip -std=gnu11" \
        --CXXOPTFLAGS="-O3 -xHost -ip" \
        --FOPTFLAGS="-O3 -xHost -ip" \
        --with-blas-include=$MKLROOT/include \
        --with-blas-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
        --with-lapack-include=$MKLROOT/include \
        --with-lapack-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
        --with-blacs-include=$MKLROOT/include \
        --with-blacs-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
        --with-scalapack-include=$MKLROOT/include \
        --with-scalapack-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
        --with-mkl_pardiso-dir=$MKLROOT/ \
        --with-mkl_cpardiso-dir=$MKLROOT/ \
        --download-mumps \
        --download-hypre \
        --download-metis \
        --download-parmetis \
        --download-suitesparse \
        --download-ptscotch \
        --prefix=petsc-lib

make PETSC_DIR=$PWD PETSC_ARCH=centos-intelmkl all
make PETSC_DIR=$PWD PETSC_ARCH=centos-intelmkl install

export PETSC_DIR=$FENICS_ROOT/petsc-3.14.0/petsc-lib
export PETSC_ARCH=""

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.14.0/petsc-lib/

Build HDF5

cd $FENICS_ROOT

wget -P download "https://www.zlib.net/zlib-1.2.11.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/lib-external/szip/2.1.1/src/szip-2.1.1.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.21/src/hdf5-1.8.21.tar.gz"
wget -P download "https://gitlab.com/libeigen/eigen/-/archive/3.3.8/eigen-3.3.8.tar.bz2"

tar -zxf download/zlib-1.2.11.tar.gz
tar -zxf download/szip-2.1.1.tar.gz
tar -zxf download/hdf5-1.8.21.tar.gz
tar -xvf download/eigen-3.3.8.tar.bz2

Build zlib

cd $FENICS_ROOT/zlib-1.2.11

export CC=icc
export CFLAGS='-O3 -xHost -ip -fPIC'
./configure --prefix=$PWD/zlib-lib

make -j 8 && make install 
export ZLIB_DIR=$FENICS_ROOT/zlib-1.2.11/zlib-lib

Build szlib

cd $FENICS_ROOT/szip-2.1.1

export CC=icc
export CFLAGS='-O3 -xHost -ip'
./configure --prefix=$PWD/szlib-lib

make -j 8 && make install 
export SZIP_DIR=$FENICS_ROOT/szip-2.1.1/szlib-lib

Build hdf5

cd $FENICS_ROOT/hdf5-1.8.21/

CC=mpiicc ./configure \
        --enable-parallel \
        --with-zlib=${ZLIB_DIR} \
        --with-szlib=${SZIP_DIR} \
        --enable-build-mode=production \
        --prefix=$PWD/hdf5-lib
make -j 8 && make install 
export HDF5_DIR=$FENICS_ROOT/hdf5-1.8.21/hdf5-lib

Build Boost

cd $FENICS_ROOT

wget -P download "https://sourceforge.net/projects/boost/files/boost/1.60.0/boost_1_60_0.tar.gz"
tar -zxf download/boost_1_60_0.tar.gz

cd boost_1_60_0/tools/build
./bootstrap.sh --with-toolset=intel-linux
./b2 install --prefix=$FENICS_ROOT/boost_1_60_0
export PATH=$FENICS_ROOT/boost_1_60_0/bin:${PATH}
cd $FENICS_ROOT/boost_1_60_0
b2 -j 16 \
   --with-filesystem \
   --with-iostreams \
   --with-math \
   --with-program_options \
   --with-system \
   --with-thread \
   --with-timer \
   --with-regex \
   --build-dir=boost-build \
      threading=multi \
      variant=release \
      address-model=64 \
      toolset=intel \
      stage
export BOOST_DIR=$FENICS_ROOT/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/boost_1_60_0/
export PATH=$FENICS_ROOT/boost_1_60_0/bin:${PATH}

Build fenics

export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION

rm -rf $FENICS_INSTALL
mkdir -p $FENICS_INSTALL && cd $FENICS_INSTALL

git clone --branch=$FENICS_VERSION https://bitbucket.org/fenics-project/fiat.git
git clone --branch=$FENICS_VERSION https://bitbucket.org/fenics-project/dijitso.git
git clone --branch=$FENICS_VERSION https://bitbucket.org/fenics-project/ufl.git
git clone --branch=$FENICS_VERSION.post0 https://bitbucket.org/fenics-project/ffc.git
git clone --branch=$FENICS_VERSION.post0 https://bitbucket.org/fenics-project/dolfin.git
git clone --branch=$FENICS_VERSION https://bitbucket.org/fenics-project/mshr.git

chmod u+w ./*/.git/objects/pack/*
mkdir -p $FENICS_INSTALL/dolfin/build
mkdir -p $FENICS_INSTALL/mshr/build

Install python virtual env and fenics python packages

cd $FENICS_INSTALL
python3 -m pip install --user virtualenv
python3 -m virtualenv --no-download $FENICS_INSTALL
source $FENICS_INSTALL/bin/activate
pip install ply
pip install numpy

LDSHARED='icc -shared' \
CC=icc \
pip3 --no-cache-dir install matplotlib --no-binary :all:

pip3 --no-cache-dir install \
        fenics==${FENICS_VERSION} \
        mpi4py==3.0.1 \
        petsc4py==3.13.0 \
        sympy==1.1.1 \
        --no-binary :all:

Build pybind11

cd /project/cs16/tma5/fenics
wget -P download "https://github.com/pybind/pybind11/archive/v2.2.3.tar.gz"
tar -xvf download/v2.2.3.tar.gz

cd $FENICS_ROOT/pybind11-2.2.3/
mkdir build && cd build

source $FENICS_INSTALL/bin/activate

cmake -DPYBIND11_TEST=off \
      -DCMAKE_INSTALL_PREFIX=../pybind11-lib \
      -DPYBIND11_CPP_STANDARD=-std=c++11 ..

make -j 8 && make install 
cd ..
export PYBIND11_DIR=$FENICS_ROOT/pybind11-2.2.3/pybind11-lib

Build dolfin

export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION

cd $FENICS_INSTALL/dolfin/build

source $FENICS_INSTALL/bin/activate

export EIGEN3_DIR=$FENICS_ROOT/eigen-3.3.8/
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$EIGEN3_DIR

#May need to run follwoing script 2 times
CC=MPIICC cmake .. \
        -DCMAKE_BUILD_TYPE="Release" \
        -DCMAKE_INSTALL_PREFIX=../fenics-lib \
        -DCMAKE_CXX_COMPILER=mpiicpc \
        -DCMAKE_CXX_FLAGS="-O3 -xHost -ip -mkl -std=c++11 -gxx-name=$( which g++ )" \
        -DCMAKE_C_COMPILER=mpiicc \
        -DCMAKE_C_FLAGS="-O3 -xHost -ip -mkl -std=gnu11" \
        -DCMAKE_Fortran_COMPILER=mpiifort \
        -DCMAKE_VERBOSE_MAKEFILE=ON \
        -DHDF5_C_COMPILER_EXECUTABLE=${HDF5_DIR}/bin/h5pcc \
        -DHDF5_hdf5_LIBRARY_RELEASE=${HDF5_DIR}/lib/libhdf5.so \
        -DDOLFIN_ENABLE_UMFPACK=ON \
        -DDOLFIN_ENABLE_CHOLMOD=ON \
        -DZLIB_LIBRARY=${ZLIB_DIR}/lib/libz.a \
        -DZLIB_INCLUDE_DIR=${ZLIB_DIR}/include \
        -DDOLFIN_ENABLE_TRILINOS=OFF \
        -DDOLFIN_ENABLE_BENCHMARKS=ON \
        -DDOLFIN_AUTO_DETECT_MPI=ON \
        -DMPI_CXX_COMPILER=$( which mpiicpc ) \
        -DMPI_C_COMPILER=$( which mpiicc ) \
        -DHDF5_LIBRARIES=${HDF5_DIR}/lib/libhdf5.so \
        -DHDF5_C_COMPILER_EXECUTABLE=${HDF5_DIR}/bin/h5pcc \
        -DHDF5_hdf5_LIBRARY_RELEASE=${HDF5_DIR}/lib/libhdf5.so \
        -DPYTHON_EXECUTABLE:FILEPATH=$( which python3 ) \
        -DBOOST_ROOT=${BOOST_DIR}

make -j 16 && make install

export DOLFIN_DIR=$FENICS_INSTALL/dolfin/fenics-lib/

cd ../python

CC=icc \
CXX=icpc \
pip3 --no-cache-dir install . --no-binary :all:

Build mshr

cd $FENICS_INSTALL/mshr/build
cmake  .. \
      -DCMAKE_BUILD_TYPE=Release \
      -DCMAKE_INSTALL_PREFIX=../mshr-lib \
      -DCMAKE_CXX_COMPILER=mpiicpc \
      -DCMAKE_CXX_FLAGS="-O3 -xHost -ip -mkl -std=c++11 -gxx-name=$( which g++ )" \
      -DCMAKE_C_COMPILER=mpiicc \
      -DCMAKE_C_FLAGS="-O3 -xHost -ip -mkl -std=gnu11"

make -j 16 && make install

export MSHR_DIR=$FENICS_INSTALL/mshr/mshr-lib/

cd ../python

CC=icc \
CXX=icpc \
pip3 --no-cache-dir install . --no-binary :all:

Usage

#!/bin/bash

export FENICS_ROOT=/project/cs16/tma5/fenics
export FENICS_VERSION=2019.1.0 
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION

module purge

module load git/2.17
module load python/3.6.4-anaconda
module load boost/1.63.0/INTEL-18.0.0
module load gcc/6.4.0
module load intel/18.0.0
module load cmake/3.7.2/INTEL-18.0.0
module load impi/2018.0.128

source /home/compilers/Intel/parallel_studio_xe_2018.0/compilers_and_libraries/linux/bin/compilervars.sh intel64
source /home/compilers/Intel/parallel_studio_xe_2018.0/impi/2018.0.128/bin64/mpivars.sh

source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf
source $FENICS_INSTALL/bin/activate

export PETSC_DIR=$FENICS_ROOT/petsc-3.14.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.14.0/petsc-lib/

export ZLIB_DIR=$FENICS_ROOT/zlib-1.2.11/zlib-lib
export SZIP_DIR=$FENICS_ROOT/szip-2.1.1/szlib-lib
export HDF5_DIR=$FENICS_ROOT/hdf5-1.8.21/hdf5-lib
export PYBIND11_DIR=$FENICS_ROOT/pybind11-2.2.3/pybind11-lib
export EIGEN3_DIR=$FENICS_ROOT/eigen-3.3.8/

export BOOST_DIR=/home/packages/boost/1.63.0/INTEL-18.0.0/
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$EIGEN3_DIR

export DOLFIN_DIR=$FENICS_INSTALL/dolfin/fenics-lib/
export MSHR_DIR=$FENICS_INSTALL/mshr/mshr-lib/
Raw
readme_leopart.md

Build Leopart, Geopart

Compile and install LEoPart

git clone https://bitbucket.org/jakob_maljaars/leopart.git
cd leopart/source/cpp
cmake .
make
cd ../..
python3 setup.py install

Compile and install dolfin_dg

git clone https://bitbucket.org/nate-sime/dolfin_dg.git
cd dolfin_dg
python3 setup.py install

Compile and install geopart

git clone https://bitbucket.org/nate-sime/geopart.git
cd geopart
python3 setup.py install
Raw
createQsubFiles.py
import shutil
import os
NumNodes=3
NumCores=40
allocation_name="hpc_hpc_cdss01"
RunTime=1 #hours
fenics_solver_dir="/work/tma5/fenics/solvers"
def createBashScript(script_name,case_dir):
#Create a running bash script for a dir
#case_dir="/work/tma5/fenics/Case2"
#script_name="RunCase"
sh_file=""
#Interpreter line
sh_file+="#!/bin/bash\n"
#PBS lines
sh_file+=f"#PBS -A {allocation_name}\n"
sh_file+=f"#PBS -q workq\n"
sh_file+=f"#PBS -l nodes={NumNodes}:ppn=16\n"
sh_file+=f"#PBS -l walltime={int(RunTime)}:00:00\n"
sh_file+=f"#PBS -V\n"
sh_file+=f"#PBS -j oe\n"
sh_file+=f"#PBS -N fenics_{script_name}\n"
sh_file+="\n"
#Setting fenics env
sh_file+="module purge\n"
sh_file+="source /project/cs16/tma5/fenics/bashrc.sh\n"
sh_file+="\n"
#Checking env
sh_file+="echo Checking mpi env...\n"
sh_file+="which icc\n"
sh_file+="which mpirun\n"
sh_file+="\n"
#Enter case dir
sh_file+=f"cd {case_dir}\n"
sh_file+="\n"
sh_file+="echo Cleaning cache...\n"
clean_file_types=["xml",'txt','vtu','h5','pvd']
for t in clean_file_types:
sh_file+=f"rm -r *.{t}\n"
sh_file+="\n"
sh_file+="echo Running Solver...\n"
sh_file+=f"python3 {os.path.join(fenics_solver_dir,'Step1.gmsh2hdf.py')}\n"
sh_file+=f"mpirun -np {NumCores} python3 {os.path.join(fenics_solver_dir,'Step2.Stokes3D_AMG.py')}\n"
sh_file+=f"python3 {os.path.join(fenics_solver_dir,'Step3.postProcessing.py')}\n"
sh_file+="\n"
sh_file+="echo Simulation complete...\n"
file_writer = open(f"{script_name}.sh","w")
file_writer.write(sh_file)
file_writer.close()
os.chmod(f"{script_name}.sh", 0o777)
#Find all dir in the current path
case_dirs = []
case_names = []
search_path=os.getcwd()+"/Cases"
print(f"------------Searching case dirs from {search_path}--------")
for root, dirs, files in os.walk(search_path, topdown=False):
search_dir=os.path.basename(root)
for name in dirs:
#print(search_dir,name)
case_dirs.append(os.path.join(root, name))
case_names+=[search_dir+"__"+name]
#Create script for each folder
caseID=0
for d,n in zip(case_dirs,case_names):
print(caseID,n,d)
createBashScript(n,d)
caseID+=1
Raw
SubJobs.py
import shutil
import glob, os
import subprocess
#Find all job files
os.chdir("./")
bash_files=[]
for file in glob.glob("*.sh"):
#print(file)
bash_files+=[file]
#Submit all jobs
for i in range(len(bash_files)):
#if(i>1):
# break
bashCommand = f"qsub {bash_files[i]}"
print(f"[{i}] Submit job {bashCommand}")
process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
output, error = process.communicate()
print(output)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment