- Follow the guide @ https://github.com/mcovalt/FEniCS-MKL/blob/master/Dockerfile
- HPC command https://info.hpc.sussex.ac.uk/hpc-guide/resource.html
- guide2 https://wiki.math.uwaterloo.ca/fluidswiki/index.php?title=FEniCS
Enter a node ssh mike012
#Using node for faster compile
srun -t 4:00:00 -n24 -N1 -A loni_mp2020 -p workq --pty /bin/bash
export FENICS_ROOT=/project/karsten/bwang31/fenics
#export FENICS_ROOT=/project/karsten/bwang31/fenics_bigmemt
export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
mkdir -p $FENICS_INSTALL && cd $FENICS_INSTALL
cd $FENICS_ROOT
module purge
module load gcc/9.3.0
### Build mpich with gcc (build only once)
```bash
cd $FENICS_ROOT
wget -P download "http://www.mpich.org/static/downloads/3.3.2/mpich-3.3.2.tar.gz"
tar -zxf download/mpich-3.3.2.tar.gz
cd mpich-3.3.2
mkdir -p build && cd build
../configure --prefix=$HOME/mpich-3.3 \
--enable-fast=O2 \
CC=gcc \
CXX=g++ \
FC=gfortran \
F77=gfortran \
F90='' \
CFLAGS='' \
CXXFLAGS='' \
FFLAGS='' \
FCFLAGS='' \
F90FLAGS='' \
F77FLAGS=''
make -j24 && make install
export PACKAGES_DIR=$HOME
export PATH=$PACKAGES_DIR/mpich-3.3/bin:$PATH
export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export F90=mpif90
export C_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$CPLUS_INCLUDE_PATH
export FPATH=$PACKAGES_DIR/mpich-3.3/include:$FPATH
export MANPATH=$PACKAGES_DIR/mpich-3.3/share/man:$MANPATH
export LD_LIBRARY_PATH=$PACKAGES_DIR/mpich-3.3/lib:$LD_LIBRARY_PATH
#test install
git clone https://gist.github.com/a5e1d66886bc17b0f7936466739cc287.git
cd a5e1d66886bc17b0f7936466739cc287
mpicc -o cpi_parallel cpi_parallel.c
mpirun -np 8 ./cpi_parallel
#Setup compiling environement
source $FENICS_ROOT/bashrc.sh
#!/bin/bash
### fenics Environment Profile
# GCC 9.3.0
# MPICH 3.3
module purge
module load gcc/9.3.0
export PACKAGES_DIR=$HOME
export PATH=$PACKAGES_DIR/mpich-3.3/bin:$PATH
export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export F90=mpif90
export C_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$CPLUS_INCLUDE_PATH
export FPATH=$PACKAGES_DIR/mpich-3.3/include:$FPATH
export MANPATH=$PACKAGES_DIR/mpich-3.3/share/man:$MANPATH
export LD_LIBRARY_PATH=$PACKAGES_DIR/mpich-3.3/lib:$LD_LIBRARY_PATH
export FENICS_ROOT=/project/karsten/bwang31/fenics
export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
cd $FENICS_ROOT
cd $FENICS_ROOT
wget -P download wget https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz
tar zxvf download/cmake-3.12.3.tar.gz
cd cmake-3.*
./bootstrap --prefix=$HOME/cmake-3.12.3
make -j24
make install
#Create symlink cmake3 to avoid confliction with sys cmake
ln -s /home/bwang31/cmake-3.12.3/bin/cmake /home/bwang31/cmake-3.12.3/bin/cmake3
ln -s /home/bwang31/cmake-3.12.3/bin/ccmake /home/bwang31/cmake-3.12.3/bin/ccmake3
cd $FENICS_ROOT
wget -P download https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
bash download/Miniconda3-latest-Linux-x86_64.sh
Enter->yes->yes
#Create a virtual env
cd $FENICS_INSTALL
mkdir -p python-env && cd python-env
python3 -m pip install --user virtualenv
python3 -m virtualenv --no-download $FENICS_INSTALL/python-env
#Enter the virtual env (use it later on)
source $FENICS_INSTALL/python-env/bin/activate
cd $FENICS_ROOT
wget -P download "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.14.0.tar.gz"
tar -zxf download/petsc-3.14.0.tar.gz
cd petsc-3.14.0
export PETSC_DIR=$PWD
export PETSC_ARCH=centos-gcc-opt
./configure \
--with-cc=mpicc \
--with-cxx=mpicxx \
--with-fc=mpifort \
--with-shared-libraries \
--with-debugging=0 \
--with-c-support \
--with-cxx-dialect=C++11 \
--known-mpi-shared-libraries=1 \
--download-fblaslapack \
--download-scalapack \
--download-mumps \
--download-hypre \
--download-metis \
--download-parmetis \
--download-suitesparse \
--download-ptscotch \
--download-superlu_dist \
--prefix=petsc-lib
make PETSC_DIR=$PWD PETSC_ARCH=centos-gcc-opt all
make PETSC_DIR=$PWD PETSC_ARCH=centos-gcc-opt install
export PETSC_DIR=$FENICS_ROOT/petsc-3.14.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.14.0/petsc-lib/
#test petsc
#https://www.mcs.anl.gov/petsc/documentation/tutorials/HandsOnExercise.html
cd /src/ksp/ksp/tutorials
make ex50
mpiexec -n 1 ./ex50 -da_grid_x 4 -da_grid_y 4 -mat_view
mpiexec -n 4 ./ex50 -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type superlu_dist -ksp_monitor -ksp_view
mpiexec -n 4 ./ex50 -da_grid_x 1025 -da_grid_y 1025 -pc_type mg -pc_mg_levels 9 -ksp_monitor
- zlib
- szip
- HDF5
- eigen
- boost
- pybind11
cd $FENICS_ROOT
wget -P download "https://www.zlib.net/zlib-1.2.11.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/lib-external/szip/2.1.1/src/szip-2.1.1.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.21/src/hdf5-1.8.21.tar.gz"
wget -P download "https://gitlab.com/libeigen/eigen/-/archive/3.3.8/eigen-3.3.8.tar.bz2"
wget -P download "https://sourceforge.net/projects/boost/files/boost/1.60.0/boost_1_60_0.tar.gz"
wget -P download "https://github.com/pybind/pybind11/archive/v2.2.3.tar.gz"
mkdir -p deps && cd deps
tar -zxf ../download/zlib-1.2.11.tar.gz
tar -zxf ../download/szip-2.1.1.tar.gz
tar -zxf ../download/hdf5-1.8.21.tar.gz
tar -xvf ../download/eigen-3.3.8.tar.bz2
tar -xvf ../download/boost_1_60_0.tar.gz
tar -xvf ../download/v2.2.3.tar.gz
########
# zlib #
########
cd $FENICS_ROOT/deps/zlib-1.2.11
./configure --prefix=$PWD/zlib-lib
make -j24 && make install
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib
########
# szip #
########
cd $FENICS_ROOT/deps/szip-2.1.1
./configure --prefix=$PWD/szlib-lib
make -j24 && make install
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib
########
# HDF5 #
########
cd $FENICS_ROOT/deps/hdf5-1.8.21/
CC=mpicc ./configure \
--enable-parallel \
--with-zlib=${ZLIB_DIR} \
--with-szlib=${SZIP_DIR} \
--enable-build-mode=production \
--prefix=$PWD/hdf5-lib
make -j24 && make install
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.8.21/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib
#test hdf5, require LD_LIBRARY_PATH has zlib and szip
cd $FENICS_ROOT/deps
git clone https://github.com/mathaefele/parallel_HDF5_hands-on.git
$HDF5_DIR/bin/h5pcc -o phdf5_ex4 phdf5_ex4.c
mpirun -n 16 ./phdf5_ex4
#########
# Eigen #
#########
cd $FENICS_ROOT/deps/eigen-3.3.8
mkdir -p build && cd build
cmake3 .. -DCMAKE_INSTALL_PREFIX=../Eigen-lib
make install
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/
#########
# Boost #
#########
cd $FENICS_ROOT/deps
#build b2 builder
cd boost_1_60_0/tools/build
./bootstrap.sh --with-toolset=gcc
./b2 install --prefix=$FENICS_ROOT/deps/boost_1_60_0
export PATH=$FENICS_ROOT/deps/boost_1_60_0/bin:${PATH}
#build boost library using the builder
cd $FENICS_ROOT/deps/boost_1_60_0
b2 -j 24 \
--with-filesystem \
--with-iostreams \
--with-math \
--with-program_options \
--with-system \
--with-thread \
--with-timer \
--with-regex \
--build-dir=boost-build \
toolset=gcc \
stage
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/
############
# pybind11 #
############
cd $FENICS_ROOT/deps/pybind11-2.2.3/
mkdir -p build && cd build
cmake3 -DPYBIND11_TEST=off \
-DCMAKE_INSTALL_PREFIX=../pybind11-lib \
-DPYBIND11_CPP_STANDARD=-std=c++11 ..
make -j24 && make install
export PYBIND11_DIR=$FENICS_ROOT/deps/pybind11-2.2.3/pybind11-lib
- https://fenics.readthedocs.io/en/latest/installation.html#stable-version
- LONI only support run fenics in parallel, so we need to enable
-DDOLFIN_SKIP_BUILD_TESTS=TRUE
and run code usingmpirun
#Start to install fenics
pip3 install fenics-ffc --upgrade
pip3 install mpi4py==3.0.1 petsc4py==3.13.0 --no-binary :all:
cd $FENICS_INSTALL
DOLFIN_VERSION=$(python3 -c"import ffc; print(ffc.__version__)")
git clone --branch=$DOLFIN_VERSION https://bitbucket.org/fenics-project/dolfin
git clone --branch=$FENICS_VERSION https://bitbucket.org/fenics-project/mshr
mkdir -p dolfin/build && cd dolfin/build
cmake3 .. \
-DDOLFIN_SKIP_BUILD_TESTS=TRUE \
-DCMAKE_BUILD_TYPE="Release" \
-DCMAKE_INSTALL_PREFIX=../fenics-lib \
-DLAPACK_LIBRARIES=${PETSC_DIR}/lib/libflapack.a \
-DBLAS_LIBRARIES=${PETSC_DIR}/lib/libfblas.a
make -j24 && make install && cd ../..
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf
mkdir -p mshr/build && cd mshr/build
cmake3 .. \
-DCMAKE_BUILD_TYPE="Release" \
-DCMAKE_INSTALL_PREFIX=../mshr-lib
make -j24 && make install && cd ../..
#!!Please replace cmake by cmake3 in setup.py
cd dolfin/python && pip3 install . && cd ../..
cd mshr/python && pip3 install . && cd ../..
#test install (!!LONI only support run fenics in parallel)
cd $FENICS_INSTALL/dolfin/python/demo
python3 generate-demo-files.py
cd documented/cahn-hilliard
mpirun -n 1 python3 demo_cahn-hilliard.py
mpirun -n 10 python3 demo_cahn-hilliard.py
#fenics hdf5 test
#Replace following code into demo.py
file = XDMFFile("output.xdmf")
file.write(u.split()[0],t)
mpirun -n 24 python3 demo_cahn-hilliard.py
- Stokes HDG Solver https://bitbucket.org/nate-sime/geopart/src/master/
#Leopart
cd $FENICS_INSTALL
mkdir 3rdPartyLib && cd 3rdPartyLib
git clone https://bitbucket.org/jakob_maljaars/leopart.git
cd leopart/source/cpp
cmake3 .
make -j24 && cd ../..
pip3 --no-cache-dir install .
#dolfin_dg
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/dolfin_dg.git
cd dolfin_dg
pip3 --no-cache-dir install .
#Geopart
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/geopart.git
cd geopart
pip3 --no-cache-dir install .
$FENICS_ROOT/bashrc.sh
#!/bin/bash
### fenics Environment Profile
# GCC 9.3.0
# MPICH 3.3
#Setup basic fenics environment variables
export FENICS_ROOT=/project/karsten/bwang31/fenics
export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
cd $FENICS_ROOT
module purge
module load gcc/9.3.0
#Setup MPI variables
export PACKAGES_DIR=$HOME
export PATH=$PACKAGES_DIR/mpich-3.3/bin:$PATH
export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export F90=mpif90
export C_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$CPLUS_INCLUDE_PATH
export FPATH=$PACKAGES_DIR/mpich-3.3/include:$FPATH
export MANPATH=$PACKAGES_DIR/mpich-3.3/share/man:$MANPATH
export LD_LIBRARY_PATH=$PACKAGES_DIR/mpich-3.3/lib:$LD_LIBRARY_PATH
#Enter fenics python virtual env
source $FENICS_INSTALL/python-env/bin/activate
#Setup dependencies path
export PETSC_DIR=$FENICS_ROOT/petsc-3.14.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.14.0/petsc-lib/
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.8.21/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/
export PYBIND11_DIR=$FENICS_ROOT/deps/pybind11-2.2.3/pybind11-lib
#Setup fenics env
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf
Now add this bash into ~/.bashrc so we can setup fenics simply by fenics2019
echo '# fenics' >> ~/.bashrc
echo 'alias fenics2019="source /project/bwang31/fenics/bashrc.sh"' >> ~/.bashrc