Skip to content

Instantly share code, notes, and snippets.

@BinWang0213

BinWang0213/readme.md

Last active July 21, 2021 18:24
Show Gist options
  • Save BinWang0213/8fdb6aa964cb31d058df007209a32574 to your computer and use it in GitHub Desktop.
Save BinWang0213/8fdb6aa964cb31d058df007209a32574 to your computer and use it in GitHub Desktop.
Download ZIP
Compile fenics-64 with intel compiler in QueenBee-3 (mpiexec has to be used even to run in serial)
Raw
readme.md

Fenics 2019.1.0 install on QueenBee-3 with MKL

Enter a node ssh mike012

Login:

ssh -X bwang31@qbc.loni.org
srun -t 5:00:00 -n48 -N1 -A loni_mp2020 -p bigmem --pty /bin/bash
srun -t 5:00:00 -n48 -N1 -A loni_mp2020 -p checkpt --pty /bin/bash

MPI and compiling env setup

#Using node for faster compile
srun -t 4:00:00 -n24 -N1 -A loni_mp2020 -p workq --pty /bin/bash

export FENICS_ROOT=/project/karsten/bwang31/fenics-opt
export FENICS_VERSION=2019.1.0-64
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION

mkdir -p $FENICS_INSTALL && cd $FENICS_INSTALL
cd $FENICS_ROOT

module purge
module load intel/19.0.5
module load intel-mpi/2019.5.281

#Setup MPI variables
export CC=icc
export CXX=icpc
export FC=ifort
export F77=ifort
export F90=ifort
export MPICC=mpiicc
export MPICXX=mpiicpc
export MPIF77=mpiifort
export MPIF90=mpiifort
export CFLAGS='-O3 -xHost'
export CXXFLAGS='-O3 -xHost'
export FCFLAGS='-O3 -xHost'
export F77FLAGS='-O3 -xHost'
export F90FLAGS='-O3 -xHost'
export MPICCFLAGS='-O3 -xHost'
export MPICXXFLAGS='-O3 -xHost'
export MPIF77FLAGS='-O3 -xHost'
export MPIF90FLAGS='-O3 -xHost'
export OMP_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export OPENBLAS_VERBOSE=0

Install cmake (Once for a new user, dont't install again)

cd $FENICS_ROOT
wget -P download wget https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz
tar zxvf download/cmake-3.12.3.tar.gz
cd cmake-3.*
./bootstrap --prefix=$HOME/cmake-3.12.3
make -j24
make install

#Create symlink cmake3 to avoid confliction with sys cmake
ln -s /home/bwang31/cmake-3.12.3/bin/cmake /home/bwang31/cmake-3.12.3/bin/cmake3
ln -s /home/bwang31/cmake-3.12.3/bin/ccmake /home/bwang31/cmake-3.12.3/bin/ccmake3

Install python3 (only install it once)

cd $FENICS_ROOT
wget -P download "https://www.python.org/ftp/python/3.8.6/Python-3.8.6.tar.xz"
tar -xf download/Python-3.8.6.tar.xz
cd Python-3.8.6
#fwrapv for icc https://bugs.python.org/issue40223
./configure CFLAGS='-fwrapv' --enable-optimizations --with-ensurepip=install --prefix=$PWD/python-intel
make -j24 && make install

export PATH=$FENICS_ROOT/Python-3.8.6/python-intel/bin:$PATH

#Create a virtual env
cd $FENICS_INSTALL
mkdir -p python-env-64 && cd python-env-64
python3 -m pip install --user virtualenv
python3 -m virtualenv --no-download $FENICS_INSTALL/python-env-64

#Enter the virtual env (use it later on)
source $FENICS_INSTALL/python-env-64/bin/activate
#Exit the virtural evn
deactivate

Install numpy with MKL (only install it once)

cd $FENICS_ROOT
wget -P download "https://github.com/numpy/numpy/releases/download/v1.20.3/numpy-1.20.3.tar.gz"
tar -zxf download/numpy-1.20.3.tar.gz

cd numpy-1.20.3
cp site.cfg{.example,}
#edit site.cfg to include mkl
# library_dirs = /opt/intel/compilers_and_libraries_2018/linux/mkl/lib/intel64
# include_dirs = /opt/intel/compilers_and_libraries_2018/linux/mkl/include
# libraries = mkl_rt

#Modify self.cc_exe line in numpy/distutils/intelccompiler.py
# add -std=gnu11 to self.cc_exe
self.cc_exe = ('icc -O3 -g -fPIC -fp-model strict -fomit-frame-pointer -xhost -std=gnu11 -{}').format(mpopt)

python setup.py config --compiler=intelem build_clib --compiler=intelem build_ext --compiler=intelem install

#check
cd ..
import numpy as np
print(np.ones((4, 4)) + np.eye(4))

Build Petsc with mumps, scotch, etc

cd $FENICS_ROOT
wget -P download "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.15.0.tar.gz"
tar -zxf download/petsc-3.15.0.tar.gz

cd petsc-3.15.0

export OPENBLAS_NUM_THREADS=1
export OPENBLAS_VERBOSE=0

## 64-bit int
export PETSC_DIR=$PWD
export PETSC_ARCH=centos-openblas-opt-64

./configure \
    PETSC_ARCH=centos-openblas-opt-64 \
    --with-cc=mpiicc \
     --with-cxx=mpiicpc \
     --with-fc=mpiifort \
     --with-shared-libraries \
     --with-debugging=0 \
     --with-c-support \
     --with-cxx-dialect=C++11 \
     --known-mpi-shared-libraries=1 \
     --COPTFLAGS="-O3 -xHost -std=gnu11" \
     --CXXOPTFLAGS="-O3 -xHost" \
     --FOPTFLAGS="-O3 -xHost" \
    --with-64-bit-indices=yes \
    --with-debugging=0 \
    --with-fortran-bindings=no \
    --with-shared-libraries \
    --download-openblas \
    --download-metis \
    --download-parmetis \
    --download-ptscotch \
    --download-mumps \
    --download-scalapack \
    --download-suitesparse \
    --download-superlu_dist \
    --with-scalar-type=real \
    --prefix=petsc-lib

make PETSC_DIR=$PWD PETSC_ARCH=centos-openblas-opt-64 all
make PETSC_DIR=$PWD PETSC_ARCH=centos-openblas-opt-64 install

#Build petsc4py, remember enter the virtualenv-64
cd src/binding/petsc4py
PETSC_ARCH=centos-openblas-opt-64 pip3 install --no-cache-dir .

export PETSC_DIR=$FENICS_ROOT/petsc-3.15.0/petsc-lib
export PETSC_ARCH=""

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/lib

Build Petsc with MKL mumps, scotch, etc (20% faster than openblas)

cd $FENICS_ROOT
wget -P download "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.15.0.tar.gz"
tar -zxf download/petsc-3.15.0.tar.gz

cd petsc-3.15.0


## 64-bit int
export PETSC_DIR=$PWD
export PETSC_ARCH=centos-mkl-opt-64

#ptscotch must be installed for 12X faster serial fenics

./configure \
    PETSC_ARCH=centos-mkl-opt-64 \
    --with-cc=mpiicc \
    --with-cxx=mpiicpc \
    --with-fc=mpiifort \
    --with-shared-libraries \
    --with-debugging=0 \
    --with-c-support \
    --with-cxx-dialect=C++11 \
    --known-mpi-shared-libraries=1 \
    --COPTFLAGS="-O3 -xHost -mkl -std=gnu11" \
    --CXXOPTFLAGS="-O3 -xHost -mkl" \
    --FOPTFLAGS="-O3 -xHost -mkl" \
    --with-blas-include=$MKLROOT/include \
    --with-blas-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
    --with-lapack-include=$MKLROOT/include \
    --with-lapack-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
    --with-blacs-include=$MKLROOT/include \
    --with-blacs-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
    --with-scalapack-include=$MKLROOT/include \
    --with-scalapack-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
    --with-mkl_pardiso-dir=$MKLROOT/ \
    --with-mkl_cpardiso-dir=$MKLROOT/ \
    --with-64-bit-indices=yes \
    --with-debugging=0 \
    --with-fortran-bindings=no \
    --with-shared-libraries \
    --download-metis \
    --download-parmetis \
    --download-ptscotch \
    --download-mumps \
    --download-scalapack \
    --download-suitesparse \
    --download-superlu_dist \
    --with-scalar-type=real \
    --prefix=petsc-lib

make PETSC_DIR=$PWD PETSC_ARCH=centos-mkl-opt-64 all
make PETSC_DIR=$PWD PETSC_ARCH=centos-mkl-opt-64 install

#Build petsc4py, remember enter the virtualenv-64
cd src/binding/petsc4py
PETSC_ARCH=centos-mkl-opt-64 pip3 install --no-cache-dir .

export PETSC_DIR=$FENICS_ROOT/petsc-3.15.0/petsc-lib
export PETSC_ARCH=""

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/lib

Testing petsc

#test petsc
#https://www.mcs.anl.gov/petsc/documentation/tutorials/HandsOnExercise.html
cd src/ksp/ksp/tutorials
make ex50
mpirun -n 1 ./ex50  -da_grid_x 4 -da_grid_y 4 -mat_view
mpirun -n 4 ./ex50  -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type superlu_dist -ksp_monitor -ksp_view
mpirun -n 4 ./ex50  -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type mkl_cpardiso -ksp_monitor -ksp_view
mpirun -n 4 ./ex50  -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type mumps -ksp_monitor -ksp_view
mpirun -n 4 ./ex50 -da_grid_x 1025 -da_grid_y 1025 -pc_type mg -pc_mg_levels 9 -ksp_monitor

make ex52
mpirun -n 1 ./ex52 -m 100 -n 100 -use_petsc_lu
mpirun -n 48 ./ex52 -m 1000 -n 1000 -use_mumps_lu
mpirun -n 48 ./ex52 -m 1000 -n 1000 -use_mumps_ch -mat_type sbaij

#test parmetis
mkdir parmetis_test && cd parmetis_test/
git clone https://github.com/knutson/ParMETIS_Example.git
cd ParMETIS_Example
export METIS_HOME=$PETSC_DIR
#add option -std=c99 in MakeFile
make clean && make && mpirun -np 3 ./a.out

#test petsc4py
cd $FENICS_ROOT/deps
wget -P ../download https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc4py-3.15.0.tar.gz
tar -zxf ../download/petsc4py-3.15.0.tar.gz
cd petsc4py-3.15.0/demo/petsc-examples/ksp
mpirun -np 4 python3 ex2.py
mpirun -np 4 python3 ex23.py
mpirun -np 1 python3 ex23.py

Build dependencies for fenics

  • python3
  • zlib
  • szip
  • HDF5 > 1.10.6, 2GB parallel IO
  • eigen
  • boost
  • pybind11 > 2.4.3, compatible with geopart
cd $FENICS_ROOT

wget -P download "https://www.zlib.net/zlib-1.2.11.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/lib-external/szip/2.1.1/src/szip-2.1.1.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.6/src/hdf5-1.10.6.tar.gz"
wget -P download "https://gitlab.com/libeigen/eigen/-/archive/3.3.8/eigen-3.3.8.tar.bz2"
wget -P download "https://sourceforge.net/projects/boost/files/boost/1.60.0/boost_1_60_0.tar.gz"
wget -P download "https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"

mkdir -p deps && cd deps
tar -zxf ../download/zlib-1.2.11.tar.gz
tar -zxf ../download/szip-2.1.1.tar.gz
tar -zxf ../download/hdf5-1.10.6.tar.gz
tar -xvf ../download/eigen-3.3.8.tar.bz2
tar -xvf ../download/boost_1_60_0.tar.gz

########
# zlib #
########
cd $FENICS_ROOT/deps/zlib-1.2.11
./configure --prefix=$PWD/zlib-lib
make -j24 && make install 
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib

########
# szip #
########
cd $FENICS_ROOT/deps/szip-2.1.1
./configure --prefix=$PWD/szlib-lib
make -j24 && make install 
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib

########
# HDF5 #
########
cd $FENICS_ROOT/deps/hdf5-1.10.6/


CC=mpiicc ./configure \
        --enable-parallel \
        --with-zlib=${ZLIB_DIR} \
        --with-szlib=${SZIP_DIR} \
        --enable-build-mode=production \
        --prefix=$PWD/hdf5-lib
make -j24 && make install
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.10.6/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib

#test hdf5, require LD_LIBRARY_PATH has zlib and szip
cd $FENICS_ROOT/deps
git clone https://github.com/mathaefele/parallel_HDF5_hands-on.git
cd parallel_HDF5_hands-on/phdf5-4/C/solution
$HDF5_DIR/bin/h5pcc -o phdf5_ex4 phdf5_ex4.c
mpirun -n 16 ./phdf5_ex4

#########
# Eigen #
#########
cd $FENICS_ROOT/deps/eigen-3.3.8
mkdir -p build && cd build
cmake3 .. -DCMAKE_INSTALL_PREFIX=../Eigen-lib
make install
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/

#########
# Boost #
#########
cd $FENICS_ROOT/deps

#build b2 builder
cd boost_1_60_0/tools/build
./bootstrap.sh --with-toolset=intel-linux
./b2 install --prefix=$FENICS_ROOT/deps/boost_1_60_0
export PATH=$FENICS_ROOT/deps/boost_1_60_0/bin:${PATH}

#build boost library using the builder
cd $FENICS_ROOT/deps/boost_1_60_0
b2 -j 24 \
   --with-filesystem \
   --with-iostreams \
   --with-math \
   --with-program_options \
   --with-system \
   --with-thread \
   --with-timer \
   --with-regex \
   --build-dir=boost-build \
      toolset=intel \
      stage
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/

############
# pybind11 #
############
cd $FENICS_INSTALL
tar -xvf ../download/v2.4.3.tar.gz
cd pybind11-2.4.3
mkdir -p build && cd build
source $FENICS_INSTALL/python-env-64/bin/activate

cmake3 -DPYBIND11_TEST=off \
      -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
      -DCMAKE_INSTALL_PREFIX=../pybind11-lib \
      -DPYBIND11_CPP_STANDARD=-std=c++11 ..

make -j24 && make install 
export PYBIND11_DIR=$FENICS_INSTALL/pybind11-2.4.3/pybind11-lib

Build fenics 64 bit

cd $FENICS_INSTALL

#Start to install fenics
pip3 install fenics-ffc --upgrade
pip3 install mpi4py==3.0.3  --no-binary :all:

#test petsc4py and mpi4py
cd $FENICS_ROOT/deps
wget -P ../download https://github.com/mpi4py/mpi4py/releases/download/3.0.3/mpi4py-3.0.3.tar.gz
tar -zxf ../download/mpi4py-3.0.3.tar.gz
cd mpi4py-3.0.3/demo
mpirun -n 5 python3 helloworld.py

cd $FENICS_INSTALL
DOLFIN_VERSION=$(python3 -c"import ffc; print(ffc.__version__)")
echo $DOLFIN_VERSION
mkdir -p dolfin $$ cd dolfin
git clone --branch=$DOLFIN_VERSION https://bitbucket.org/fenics-project/dolfin
git clone --branch=${DOLFIN_VERSION%.*} https://bitbucket.org/fenics-project/mshr

mkdir -p dolfin/build && cd dolfin/build
cmake3 .. \
        -DCMAKE_BUILD_TYPE="Release" \
        -DDOLFIN_SKIP_BUILD_TESTS=TRUE \
        -DCMAKE_INSTALL_PREFIX=../fenics-lib \
        -DCMAKE_CXX_COMPILER=mpiicpc \
        -DCMAKE_CXX_FLAGS="-O3 -xHost -ip -std=c++11" \
        -DCMAKE_C_COMPILER=mpiicc \
        -DCMAKE_C_FLAGS="-O3 -xHost -ip" \
        -DCMAKE_Fortran_COMPILER=mpiifort
make -j 48 && make install && cd ../..
export DOLFIN_DIR=$FENICS_INSTALL/dolfin-64/dolfin/fenics-lib/

#Source dolfin lib and exe
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf

mkdir -p mshr/build  && cd mshr/build
cmake3 .. \
        -DCMAKE_BUILD_TYPE="Release" \
        -DCMAKE_INSTALL_PREFIX=../mshr-lib
make -j24 && make install && cd ../..

source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf

#!!Please replace cmake by cmake3 in setup.py!!
cd dolfin/python && pip3 install . --log dolfin.log && cd ../..
cd mshr/python   && pip3 install . --log mshr.log && cd ../..


#test install (!!LONI only support run fenics in parallel)
cd $FENICS_INSTALL/dolfin/python/demo
python3 generate-demo-files.py
cd documented/cahn-hilliard

#Using ParMetis as mesh partitioner (SCOTCH not work well for intel compiler)
parameters["mesh_partitioner"] = "ParMETIS"
solver.parameters["linear_solver"] = "mumps"

#check parameter info
info(parameters,True)


mpirun -n 1 python3 demo_cahn-hilliard.py
mpirun -n 4 python3 demo_cahn-hilliard.py

#fenics hdf5 test
#Replace following code into demo.py
file = XDMFFile("output.xdmf")
    file.write(u.split()[0],t)
mpirun -n 24 python3 demo_cahn-hilliard.py

Build 3rdparty fenics library

#Leopart
cd $FENICS_INSTALL            #32 bit
mkdir 3rdPartyLib && cd 3rdPartyLib

git clone https://bitbucket.org/jakob_maljaars/leopart.git
git clone https://bitbucket.org/nate-sime/leopart.git
git clone https://binwang31@bitbucket.org/binwang31/leopart.git (this is the latest one)
cd leopart/source/cpp


cmake3 . \
        -DCMAKE_CXX_COMPILER=mpiicpc \
        -DCMAKE_CXX_FLAGS="-O3 -xHost -ip -std=c++11" \
        -DCMAKE_C_COMPILER=mpiicc \
        -DCMAKE_C_FLAGS="-O3 -xHost -ip " \
        -DPYTHON_EXECUTABLE:FILEPATH=$(which python3)

make -j 24 && cd ../..
pip3 install .

cd unit_tests
#add following in script
#dolfin.parameters["mesh_partitioner"] = "ParMETIS"
mpirun --np 10 pytest -s test_steady_stokes.py

#dolfin_dg
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/dolfin_dg.git
cd dolfin_dg
pip3 install .

#Geopart
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/geopart.git
git clone --single-branch --branch brinkman https://binwang31@bitbucket.org/binwang31/geopart.git 
cd geopart
pip3 install .

cd test/unit
#add following in script
#dolfin.parameters["mesh_partitioner"] = "ParMETIS"
pip3 install pytest
mpirun --np 8 pytest -s test_brinkman.py

Fenics 2019.1.0 usage in LONI

Updated fenics env bash script

$FENICS_ROOT/bashrc.sh

#!/bin/bash
### fenics Environment Profile
# intel/19.0.5
# intel-mpi/2019.5.281

#Setup basic fenics environment variables
export FENICS_ROOT=/project/karsten/bwang31/fenics-opt
export FENICS_VERSION=2019.1.0-64
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
cd $FENICS_ROOT

module purge
module load intel/19.0.5
module load intel-mpi/2019.5.281

#Setup MPI variables
export CC=icc
export CXX=icpc
export FC=ifort
export F77=ifort
export F90=ifort
export MPICC=mpiicc
export MPICXX=mpiicpc
export MPIF77=mpiifort
export MPIF90=mpiifort
export CFLAGS='-O3 -xHost'
export CXXFLAGS='-O3 -xHost'
export FCFLAGS='-O3 -xHost'
export F77FLAGS='-O3 -xHost'
export F90FLAGS='-O3 -xHost'
export MPICCFLAGS='-O3 -xHost'
export MPICXXFLAGS='-O3 -xHost'
export MPIF77FLAGS='-O3 -xHost'
export MPIF90FLAGS='-O3 -xHost'
export OMP_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export OPENBLAS_VERBOSE=0

#Enter fenics python virtual env
source $FENICS_INSTALL/python-env-64/bin/activate

#Setup dependencies path
export PETSC_DIR=$FENICS_ROOT/petsc-3.15.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/lib
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.10.6/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/
export PYBIND11_DIR=$FENICS_INSTALL/pybind11-2.4.3/pybind11-lib

#Setup fenics env
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf

We can add this bash into ~/.bashrc so we can setup fenics simply by fenics2019

echo '# fenics' >> ~/.bashrc
echo 'alias fenics2019="source /project/bwang31/fenics-opt/bashrc.sh"' >> ~/.bashrc

Run Cases

gmsh,meshio,pygmsh environment

pip3 install pandas
CC=mpiicc pip3 install --no-binary=gmsh gmsh
CC=mpiicc pip3 install --no-binary=h5py h5py
pip3 install pygmsh


module load gcc/8.4.0
module purge
source /project/karsten/bwang31/fenics-opt/bashrc_64.sh

cd /work/bwang31/fenics/

#Gmsh binary dependency, don't load gcc when we we use fenics Expression
module load gcc/8.4.0


echo Cleaning cache...
rm -r *.xml
rm -r *.txt
rm -r *.vtu
rm -r *.h5
rm -r *.pvd
rm -r *.xdmf
rm -r *.npy

SOLVER_PATH=/work/bwang31/fenics/solvers
python3 $SOLVER_PATH/stokes_solver.py --np 48 --mesh_fname hsub40.msh --steps 1 1 0

docker exec -u 0 -it notebook bash

Raw
zgmsh.sh
#Install gmsh module in fenics docker
pip3 install pygmsh --user
pip3 install meshio --user
pip3 install gmsh --user
pip3 install --no-binary=h5py h5py
sudo apt-get -y install libglu1 \
libxcursor-dev \
libxinerama1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment