Skip to content

Instantly share code, notes, and snippets.

@BinWang0213

BinWang0213/readme.md

Last active November 27, 2020 04:49
Show Gist options
  • Save BinWang0213/e1ce39832d0fe03a6d2fd3cbdf1110d1 to your computer and use it in GitHub Desktop.
Save BinWang0213/e1ce39832d0fe03a6d2fd3cbdf1110d1 to your computer and use it in GitHub Desktop.
Download ZIP
Compile fenics in QueenBee-3 (mpiexec has to be used even to run in serial)
Raw
readme.md

Fenics 2019.1.0 install on QueenBee-3

Enter a node ssh mike012

MPI and compiling env setup

#Using node for faster compile
srun -t 4:00:00 -n24 -N1 -A loni_mp2020 -p workq --pty /bin/bash

export FENICS_ROOT=/project/karsten/bwang31/fenics
#export FENICS_ROOT=/project/karsten/bwang31/fenics_bigmemt
export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION

mkdir -p $FENICS_INSTALL && cd $FENICS_INSTALL
cd $FENICS_ROOT

module purge
module load gcc/9.3.0

### Build mpich with gcc (build only once)
```bash
cd $FENICS_ROOT
wget -P download "http://www.mpich.org/static/downloads/3.3.2/mpich-3.3.2.tar.gz"
tar -zxf download/mpich-3.3.2.tar.gz

cd mpich-3.3.2
mkdir -p build && cd build

../configure --prefix=$HOME/mpich-3.3 \
--enable-fast=O2 \
CC=gcc \
CXX=g++ \
FC=gfortran \
F77=gfortran \
F90='' \
CFLAGS='' \
CXXFLAGS='' \
FFLAGS='' \
FCFLAGS='' \
F90FLAGS='' \
F77FLAGS=''

make -j24 && make install

export PACKAGES_DIR=$HOME
export PATH=$PACKAGES_DIR/mpich-3.3/bin:$PATH
export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export F90=mpif90
export C_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$CPLUS_INCLUDE_PATH
export FPATH=$PACKAGES_DIR/mpich-3.3/include:$FPATH
export MANPATH=$PACKAGES_DIR/mpich-3.3/share/man:$MANPATH
export LD_LIBRARY_PATH=$PACKAGES_DIR/mpich-3.3/lib:$LD_LIBRARY_PATH


#test install
git clone https://gist.github.com/a5e1d66886bc17b0f7936466739cc287.git
cd a5e1d66886bc17b0f7936466739cc287
mpicc -o cpi_parallel cpi_parallel.c
mpirun -np 8 ./cpi_parallel

Create compiling environment script

#Setup compiling environement

source $FENICS_ROOT/bashrc.sh

#!/bin/bash
### fenics Environment Profile
# GCC 9.3.0
# MPICH 3.3

module purge
module load gcc/9.3.0

export PACKAGES_DIR=$HOME
export PATH=$PACKAGES_DIR/mpich-3.3/bin:$PATH
export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export F90=mpif90
export C_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$CPLUS_INCLUDE_PATH
export FPATH=$PACKAGES_DIR/mpich-3.3/include:$FPATH
export MANPATH=$PACKAGES_DIR/mpich-3.3/share/man:$MANPATH
export LD_LIBRARY_PATH=$PACKAGES_DIR/mpich-3.3/lib:$LD_LIBRARY_PATH


export FENICS_ROOT=/project/karsten/bwang31/fenics
export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
cd $FENICS_ROOT

Install cmake (Once for a new user)

cd $FENICS_ROOT
wget -P download wget https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz
tar zxvf download/cmake-3.12.3.tar.gz
cd cmake-3.*
./bootstrap --prefix=$HOME/cmake-3.12.3
make -j24
make install

#Create symlink cmake3 to avoid confliction with sys cmake
ln -s /home/bwang31/cmake-3.12.3/bin/cmake /home/bwang31/cmake-3.12.3/bin/cmake3
ln -s /home/bwang31/cmake-3.12.3/bin/ccmake /home/bwang31/cmake-3.12.3/bin/ccmake3

Install python3 (Once for a new user)

cd $FENICS_ROOT
wget -P download https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
bash download/Miniconda3-latest-Linux-x86_64.sh
Enter->yes->yes

#Create a virtual env
cd $FENICS_INSTALL
mkdir -p python-env && cd python-env
python3 -m pip install --user virtualenv
python3 -m virtualenv --no-download $FENICS_INSTALL/python-env

#Enter the virtual env (use it later on)
source $FENICS_INSTALL/python-env/bin/activate

Build Petsc with mumps, scotch, etc

cd $FENICS_ROOT
wget -P download "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.14.0.tar.gz"
tar -zxf download/petsc-3.14.0.tar.gz

cd petsc-3.14.0

export PETSC_DIR=$PWD
export PETSC_ARCH=centos-gcc-opt

./configure \
        --with-cc=mpicc \
        --with-cxx=mpicxx \
        --with-fc=mpifort \
        --with-shared-libraries \
        --with-debugging=0 \
        --with-c-support \
        --with-cxx-dialect=C++11 \
        --known-mpi-shared-libraries=1 \
        --download-fblaslapack \
        --download-scalapack \
        --download-mumps \
        --download-hypre \
        --download-metis \
        --download-parmetis \
        --download-suitesparse \
        --download-ptscotch \
        --download-superlu_dist \
        --prefix=petsc-lib

make PETSC_DIR=$PWD PETSC_ARCH=centos-gcc-opt all
make PETSC_DIR=$PWD PETSC_ARCH=centos-gcc-opt install

export PETSC_DIR=$FENICS_ROOT/petsc-3.14.0/petsc-lib
export PETSC_ARCH=""

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.14.0/petsc-lib/

#test petsc
#https://www.mcs.anl.gov/petsc/documentation/tutorials/HandsOnExercise.html
cd /src/ksp/ksp/tutorials
make ex50
mpiexec -n 1 ./ex50  -da_grid_x 4 -da_grid_y 4 -mat_view
mpiexec -n 4 ./ex50  -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type superlu_dist -ksp_monitor -ksp_view
mpiexec -n 4 ./ex50 -da_grid_x 1025 -da_grid_y 1025 -pc_type mg -pc_mg_levels 9 -ksp_monitor

Build dependencies for fenics

  • zlib
  • szip
  • HDF5
  • eigen
  • boost
  • pybind11
cd $FENICS_ROOT

wget -P download "https://www.zlib.net/zlib-1.2.11.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/lib-external/szip/2.1.1/src/szip-2.1.1.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.21/src/hdf5-1.8.21.tar.gz"
wget -P download "https://gitlab.com/libeigen/eigen/-/archive/3.3.8/eigen-3.3.8.tar.bz2"
wget -P download "https://sourceforge.net/projects/boost/files/boost/1.60.0/boost_1_60_0.tar.gz"
wget -P download "https://github.com/pybind/pybind11/archive/v2.2.3.tar.gz"

mkdir -p deps && cd deps
tar -zxf ../download/zlib-1.2.11.tar.gz
tar -zxf ../download/szip-2.1.1.tar.gz
tar -zxf ../download/hdf5-1.8.21.tar.gz
tar -xvf ../download/eigen-3.3.8.tar.bz2
tar -xvf ../download/boost_1_60_0.tar.gz
tar -xvf ../download/v2.2.3.tar.gz

########
# zlib #
########
cd $FENICS_ROOT/deps/zlib-1.2.11
./configure --prefix=$PWD/zlib-lib
make -j24 && make install 
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib

########
# szip #
########
cd $FENICS_ROOT/deps/szip-2.1.1
./configure --prefix=$PWD/szlib-lib
make -j24 && make install 
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib

########
# HDF5 #
########
cd $FENICS_ROOT/deps/hdf5-1.8.21/
CC=mpicc ./configure \
        --enable-parallel \
        --with-zlib=${ZLIB_DIR} \
        --with-szlib=${SZIP_DIR} \
        --enable-build-mode=production \
        --prefix=$PWD/hdf5-lib
make -j24 && make install
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.8.21/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib

#test hdf5, require LD_LIBRARY_PATH has zlib and szip
cd $FENICS_ROOT/deps
git clone https://github.com/mathaefele/parallel_HDF5_hands-on.git
$HDF5_DIR/bin/h5pcc -o phdf5_ex4 phdf5_ex4.c
mpirun -n 16 ./phdf5_ex4

#########
# Eigen #
#########
cd $FENICS_ROOT/deps/eigen-3.3.8
mkdir -p build && cd build
cmake3 .. -DCMAKE_INSTALL_PREFIX=../Eigen-lib
make install
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/

#########
# Boost #
#########
cd $FENICS_ROOT/deps

#build b2 builder
cd boost_1_60_0/tools/build
./bootstrap.sh --with-toolset=gcc
./b2 install --prefix=$FENICS_ROOT/deps/boost_1_60_0
export PATH=$FENICS_ROOT/deps/boost_1_60_0/bin:${PATH}

#build boost library using the builder
cd $FENICS_ROOT/deps/boost_1_60_0
b2 -j 24 \
   --with-filesystem \
   --with-iostreams \
   --with-math \
   --with-program_options \
   --with-system \
   --with-thread \
   --with-timer \
   --with-regex \
   --build-dir=boost-build \
      toolset=gcc \
      stage
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/

############
# pybind11 #
############
cd $FENICS_ROOT/deps/pybind11-2.2.3/
mkdir -p build && cd build

cmake3 -DPYBIND11_TEST=off \
      -DCMAKE_INSTALL_PREFIX=../pybind11-lib \
      -DPYBIND11_CPP_STANDARD=-std=c++11 ..

make -j24 && make install 
export PYBIND11_DIR=$FENICS_ROOT/deps/pybind11-2.2.3/pybind11-lib

Build fenics

#Start to install fenics
pip3 install fenics-ffc --upgrade
pip3 install mpi4py==3.0.1 petsc4py==3.13.0 --no-binary :all:

cd $FENICS_INSTALL
DOLFIN_VERSION=$(python3 -c"import ffc; print(ffc.__version__)")
git clone --branch=$DOLFIN_VERSION https://bitbucket.org/fenics-project/dolfin
git clone --branch=$FENICS_VERSION https://bitbucket.org/fenics-project/mshr

mkdir -p dolfin/build && cd dolfin/build
cmake3 .. \
        -DDOLFIN_SKIP_BUILD_TESTS=TRUE \
        -DCMAKE_BUILD_TYPE="Release" \
        -DCMAKE_INSTALL_PREFIX=../fenics-lib \
        -DLAPACK_LIBRARIES=${PETSC_DIR}/lib/libflapack.a \
        -DBLAS_LIBRARIES=${PETSC_DIR}/lib/libfblas.a
make -j24 && make install && cd ../..

source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf


mkdir -p mshr/build  && cd mshr/build
cmake3 .. \
        -DCMAKE_BUILD_TYPE="Release" \
        -DCMAKE_INSTALL_PREFIX=../mshr-lib
make -j24 && make install && cd ../..

#!!Please replace cmake by cmake3 in setup.py
cd dolfin/python && pip3 install . && cd ../..
cd mshr/python   && pip3 install . && cd ../..

#test install (!!LONI only support run fenics in parallel)
cd $FENICS_INSTALL/dolfin/python/demo
python3 generate-demo-files.py
cd documented/cahn-hilliard
mpirun -n 1 python3 demo_cahn-hilliard.py
mpirun -n 10 python3 demo_cahn-hilliard.py

#fenics hdf5 test
#Replace following code into demo.py
file = XDMFFile("output.xdmf")
    file.write(u.split()[0],t)
mpirun -n 24 python3 demo_cahn-hilliard.py

Build 3rdparty fenics library

#Leopart
cd $FENICS_INSTALL
mkdir 3rdPartyLib && cd 3rdPartyLib

git clone https://bitbucket.org/jakob_maljaars/leopart.git
cd leopart/source/cpp
cmake3 .
make -j24 && cd ../..
pip3 --no-cache-dir install .

#dolfin_dg
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/dolfin_dg.git
cd dolfin_dg
pip3 --no-cache-dir install .

#Geopart
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/geopart.git
cd geopart
pip3 --no-cache-dir install .

Fenics 2019.1.0 usage in LONI

Updated fenics env bash script

$FENICS_ROOT/bashrc.sh

#!/bin/bash
### fenics Environment Profile
# GCC 9.3.0
# MPICH 3.3

#Setup basic fenics environment variables
export FENICS_ROOT=/project/karsten/bwang31/fenics
export FENICS_VERSION=2019.1.0
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
cd $FENICS_ROOT

module purge
module load gcc/9.3.0

#Setup MPI variables
export PACKAGES_DIR=$HOME

export PATH=$PACKAGES_DIR/mpich-3.3/bin:$PATH
export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export F90=mpif90
export C_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$PACKAGES_DIR/mpich-3.3/include:$CPLUS_INCLUDE_PATH
export FPATH=$PACKAGES_DIR/mpich-3.3/include:$FPATH
export MANPATH=$PACKAGES_DIR/mpich-3.3/share/man:$MANPATH
export LD_LIBRARY_PATH=$PACKAGES_DIR/mpich-3.3/lib:$LD_LIBRARY_PATH


#Enter fenics python virtual env
source $FENICS_INSTALL/python-env/bin/activate

#Setup dependencies path
export PETSC_DIR=$FENICS_ROOT/petsc-3.14.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.14.0/petsc-lib/
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.8.21/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/
export PYBIND11_DIR=$FENICS_ROOT/deps/pybind11-2.2.3/pybind11-lib

#Setup fenics env
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf

Now add this bash into ~/.bashrc so we can setup fenics simply by fenics2019

echo '# fenics' >> ~/.bashrc
echo 'alias fenics2019="source /project/bwang31/fenics/bashrc.sh"' >> ~/.bashrc
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment