- Follow the guide @ https://github.com/mcovalt/FEniCS-MKL/blob/master/Dockerfile
- HPC command https://info.hpc.sussex.ac.uk/hpc-guide/resource.html
- guide2 https://wiki.math.uwaterloo.ca/fluidswiki/index.php?title=FEniCS
Enter a node ssh mike012
Login:
ssh -X bwang31@qbc.loni.org
srun -t 5:00:00 -n48 -N1 -A loni_mp2020 -p bigmem --pty /bin/bash
srun -t 5:00:00 -n48 -N1 -A loni_mp2020 -p checkpt --pty /bin/bash
#Using node for faster compile
srun -t 4:00:00 -n24 -N1 -A loni_mp2020 -p workq --pty /bin/bash
export FENICS_ROOT=/project/karsten/bwang31/fenics-opt
export FENICS_VERSION=2019.1.0-64
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
mkdir -p $FENICS_INSTALL && cd $FENICS_INSTALL
cd $FENICS_ROOT
module purge
module load intel/19.0.5
module load intel-mpi/2019.5.281
#Setup MPI variables
export CC=icc
export CXX=icpc
export FC=ifort
export F77=ifort
export F90=ifort
export MPICC=mpiicc
export MPICXX=mpiicpc
export MPIF77=mpiifort
export MPIF90=mpiifort
export CFLAGS='-O3 -xHost'
export CXXFLAGS='-O3 -xHost'
export FCFLAGS='-O3 -xHost'
export F77FLAGS='-O3 -xHost'
export F90FLAGS='-O3 -xHost'
export MPICCFLAGS='-O3 -xHost'
export MPICXXFLAGS='-O3 -xHost'
export MPIF77FLAGS='-O3 -xHost'
export MPIF90FLAGS='-O3 -xHost'
export OMP_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export OPENBLAS_VERBOSE=0
cd $FENICS_ROOT
wget -P download wget https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz
tar zxvf download/cmake-3.12.3.tar.gz
cd cmake-3.*
./bootstrap --prefix=$HOME/cmake-3.12.3
make -j24
make install
#Create symlink cmake3 to avoid confliction with sys cmake
ln -s /home/bwang31/cmake-3.12.3/bin/cmake /home/bwang31/cmake-3.12.3/bin/cmake3
ln -s /home/bwang31/cmake-3.12.3/bin/ccmake /home/bwang31/cmake-3.12.3/bin/ccmake3
cd $FENICS_ROOT
wget -P download "https://www.python.org/ftp/python/3.8.6/Python-3.8.6.tar.xz"
tar -xf download/Python-3.8.6.tar.xz
cd Python-3.8.6
#fwrapv for icc https://bugs.python.org/issue40223
./configure CFLAGS='-fwrapv' --enable-optimizations --with-ensurepip=install --prefix=$PWD/python-intel
make -j24 && make install
export PATH=$FENICS_ROOT/Python-3.8.6/python-intel/bin:$PATH
#Create a virtual env
cd $FENICS_INSTALL
mkdir -p python-env-64 && cd python-env-64
python3 -m pip install --user virtualenv
python3 -m virtualenv --no-download $FENICS_INSTALL/python-env-64
#Enter the virtual env (use it later on)
source $FENICS_INSTALL/python-env-64/bin/activate
#Exit the virtural evn
deactivate
cd $FENICS_ROOT
wget -P download "https://github.com/numpy/numpy/releases/download/v1.20.3/numpy-1.20.3.tar.gz"
tar -zxf download/numpy-1.20.3.tar.gz
cd numpy-1.20.3
cp site.cfg{.example,}
#edit site.cfg to include mkl
# library_dirs = /opt/intel/compilers_and_libraries_2018/linux/mkl/lib/intel64
# include_dirs = /opt/intel/compilers_and_libraries_2018/linux/mkl/include
# libraries = mkl_rt
#Modify self.cc_exe line in numpy/distutils/intelccompiler.py
# add -std=gnu11 to self.cc_exe
self.cc_exe = ('icc -O3 -g -fPIC -fp-model strict -fomit-frame-pointer -xhost -std=gnu11 -{}').format(mpopt)
python setup.py config --compiler=intelem build_clib --compiler=intelem build_ext --compiler=intelem install
#check
cd ..
import numpy as np
print(np.ones((4, 4)) + np.eye(4))
cd $FENICS_ROOT
wget -P download "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.15.0.tar.gz"
tar -zxf download/petsc-3.15.0.tar.gz
cd petsc-3.15.0
export OPENBLAS_NUM_THREADS=1
export OPENBLAS_VERBOSE=0
## 64-bit int
export PETSC_DIR=$PWD
export PETSC_ARCH=centos-openblas-opt-64
./configure \
PETSC_ARCH=centos-openblas-opt-64 \
--with-cc=mpiicc \
--with-cxx=mpiicpc \
--with-fc=mpiifort \
--with-shared-libraries \
--with-debugging=0 \
--with-c-support \
--with-cxx-dialect=C++11 \
--known-mpi-shared-libraries=1 \
--COPTFLAGS="-O3 -xHost -std=gnu11" \
--CXXOPTFLAGS="-O3 -xHost" \
--FOPTFLAGS="-O3 -xHost" \
--with-64-bit-indices=yes \
--with-debugging=0 \
--with-fortran-bindings=no \
--with-shared-libraries \
--download-openblas \
--download-metis \
--download-parmetis \
--download-ptscotch \
--download-mumps \
--download-scalapack \
--download-suitesparse \
--download-superlu_dist \
--with-scalar-type=real \
--prefix=petsc-lib
make PETSC_DIR=$PWD PETSC_ARCH=centos-openblas-opt-64 all
make PETSC_DIR=$PWD PETSC_ARCH=centos-openblas-opt-64 install
#Build petsc4py, remember enter the virtualenv-64
cd src/binding/petsc4py
PETSC_ARCH=centos-openblas-opt-64 pip3 install --no-cache-dir .
export PETSC_DIR=$FENICS_ROOT/petsc-3.15.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/lib
cd $FENICS_ROOT
wget -P download "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.15.0.tar.gz"
tar -zxf download/petsc-3.15.0.tar.gz
cd petsc-3.15.0
## 64-bit int
export PETSC_DIR=$PWD
export PETSC_ARCH=centos-mkl-opt-64
#ptscotch must be installed for 12X faster serial fenics
./configure \
PETSC_ARCH=centos-mkl-opt-64 \
--with-cc=mpiicc \
--with-cxx=mpiicpc \
--with-fc=mpiifort \
--with-shared-libraries \
--with-debugging=0 \
--with-c-support \
--with-cxx-dialect=C++11 \
--known-mpi-shared-libraries=1 \
--COPTFLAGS="-O3 -xHost -mkl -std=gnu11" \
--CXXOPTFLAGS="-O3 -xHost -mkl" \
--FOPTFLAGS="-O3 -xHost -mkl" \
--with-blas-include=$MKLROOT/include \
--with-blas-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
--with-lapack-include=$MKLROOT/include \
--with-lapack-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
--with-blacs-include=$MKLROOT/include \
--with-blacs-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
--with-scalapack-include=$MKLROOT/include \
--with-scalapack-lib="-L$MKLROOT/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl" \
--with-mkl_pardiso-dir=$MKLROOT/ \
--with-mkl_cpardiso-dir=$MKLROOT/ \
--with-64-bit-indices=yes \
--with-debugging=0 \
--with-fortran-bindings=no \
--with-shared-libraries \
--download-metis \
--download-parmetis \
--download-ptscotch \
--download-mumps \
--download-scalapack \
--download-suitesparse \
--download-superlu_dist \
--with-scalar-type=real \
--prefix=petsc-lib
make PETSC_DIR=$PWD PETSC_ARCH=centos-mkl-opt-64 all
make PETSC_DIR=$PWD PETSC_ARCH=centos-mkl-opt-64 install
#Build petsc4py, remember enter the virtualenv-64
cd src/binding/petsc4py
PETSC_ARCH=centos-mkl-opt-64 pip3 install --no-cache-dir .
export PETSC_DIR=$FENICS_ROOT/petsc-3.15.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/lib
#test petsc
#https://www.mcs.anl.gov/petsc/documentation/tutorials/HandsOnExercise.html
cd src/ksp/ksp/tutorials
make ex50
mpirun -n 1 ./ex50 -da_grid_x 4 -da_grid_y 4 -mat_view
mpirun -n 4 ./ex50 -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type superlu_dist -ksp_monitor -ksp_view
mpirun -n 4 ./ex50 -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type mkl_cpardiso -ksp_monitor -ksp_view
mpirun -n 4 ./ex50 -da_grid_x 120 -da_grid_y 120 -pc_type lu -pc_factor_mat_solver_type mumps -ksp_monitor -ksp_view
mpirun -n 4 ./ex50 -da_grid_x 1025 -da_grid_y 1025 -pc_type mg -pc_mg_levels 9 -ksp_monitor
make ex52
mpirun -n 1 ./ex52 -m 100 -n 100 -use_petsc_lu
mpirun -n 48 ./ex52 -m 1000 -n 1000 -use_mumps_lu
mpirun -n 48 ./ex52 -m 1000 -n 1000 -use_mumps_ch -mat_type sbaij
#test parmetis
mkdir parmetis_test && cd parmetis_test/
git clone https://github.com/knutson/ParMETIS_Example.git
cd ParMETIS_Example
export METIS_HOME=$PETSC_DIR
#add option -std=c99 in MakeFile
make clean && make && mpirun -np 3 ./a.out
#test petsc4py
cd $FENICS_ROOT/deps
wget -P ../download https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc4py-3.15.0.tar.gz
tar -zxf ../download/petsc4py-3.15.0.tar.gz
cd petsc4py-3.15.0/demo/petsc-examples/ksp
mpirun -np 4 python3 ex2.py
mpirun -np 4 python3 ex23.py
mpirun -np 1 python3 ex23.py
- python3
- zlib
- szip
- HDF5 > 1.10.6, 2GB parallel IO
- eigen
- boost
- pybind11 > 2.4.3, compatible with geopart
cd $FENICS_ROOT
wget -P download "https://www.zlib.net/zlib-1.2.11.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/lib-external/szip/2.1.1/src/szip-2.1.1.tar.gz"
wget -P download "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.6/src/hdf5-1.10.6.tar.gz"
wget -P download "https://gitlab.com/libeigen/eigen/-/archive/3.3.8/eigen-3.3.8.tar.bz2"
wget -P download "https://sourceforge.net/projects/boost/files/boost/1.60.0/boost_1_60_0.tar.gz"
wget -P download "https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"
mkdir -p deps && cd deps
tar -zxf ../download/zlib-1.2.11.tar.gz
tar -zxf ../download/szip-2.1.1.tar.gz
tar -zxf ../download/hdf5-1.10.6.tar.gz
tar -xvf ../download/eigen-3.3.8.tar.bz2
tar -xvf ../download/boost_1_60_0.tar.gz
########
# zlib #
########
cd $FENICS_ROOT/deps/zlib-1.2.11
./configure --prefix=$PWD/zlib-lib
make -j24 && make install
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib
########
# szip #
########
cd $FENICS_ROOT/deps/szip-2.1.1
./configure --prefix=$PWD/szlib-lib
make -j24 && make install
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib
########
# HDF5 #
########
cd $FENICS_ROOT/deps/hdf5-1.10.6/
CC=mpiicc ./configure \
--enable-parallel \
--with-zlib=${ZLIB_DIR} \
--with-szlib=${SZIP_DIR} \
--enable-build-mode=production \
--prefix=$PWD/hdf5-lib
make -j24 && make install
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.10.6/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib
#test hdf5, require LD_LIBRARY_PATH has zlib and szip
cd $FENICS_ROOT/deps
git clone https://github.com/mathaefele/parallel_HDF5_hands-on.git
cd parallel_HDF5_hands-on/phdf5-4/C/solution
$HDF5_DIR/bin/h5pcc -o phdf5_ex4 phdf5_ex4.c
mpirun -n 16 ./phdf5_ex4
#########
# Eigen #
#########
cd $FENICS_ROOT/deps/eigen-3.3.8
mkdir -p build && cd build
cmake3 .. -DCMAKE_INSTALL_PREFIX=../Eigen-lib
make install
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/
#########
# Boost #
#########
cd $FENICS_ROOT/deps
#build b2 builder
cd boost_1_60_0/tools/build
./bootstrap.sh --with-toolset=intel-linux
./b2 install --prefix=$FENICS_ROOT/deps/boost_1_60_0
export PATH=$FENICS_ROOT/deps/boost_1_60_0/bin:${PATH}
#build boost library using the builder
cd $FENICS_ROOT/deps/boost_1_60_0
b2 -j 24 \
--with-filesystem \
--with-iostreams \
--with-math \
--with-program_options \
--with-system \
--with-thread \
--with-timer \
--with-regex \
--build-dir=boost-build \
toolset=intel \
stage
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/
############
# pybind11 #
############
cd $FENICS_INSTALL
tar -xvf ../download/v2.4.3.tar.gz
cd pybind11-2.4.3
mkdir -p build && cd build
source $FENICS_INSTALL/python-env-64/bin/activate
cmake3 -DPYBIND11_TEST=off \
-DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
-DCMAKE_INSTALL_PREFIX=../pybind11-lib \
-DPYBIND11_CPP_STANDARD=-std=c++11 ..
make -j24 && make install
export PYBIND11_DIR=$FENICS_INSTALL/pybind11-2.4.3/pybind11-lib
- https://fenics.readthedocs.io/en/latest/installation.html#stable-version
- LONI only support run fenics in parallel, so we need to enable
-DDOLFIN_SKIP_BUILD_TESTS=TRUE
and run code usingmpirun
cd $FENICS_INSTALL
#Start to install fenics
pip3 install fenics-ffc --upgrade
pip3 install mpi4py==3.0.3 --no-binary :all:
#test petsc4py and mpi4py
cd $FENICS_ROOT/deps
wget -P ../download https://github.com/mpi4py/mpi4py/releases/download/3.0.3/mpi4py-3.0.3.tar.gz
tar -zxf ../download/mpi4py-3.0.3.tar.gz
cd mpi4py-3.0.3/demo
mpirun -n 5 python3 helloworld.py
cd $FENICS_INSTALL
DOLFIN_VERSION=$(python3 -c"import ffc; print(ffc.__version__)")
echo $DOLFIN_VERSION
mkdir -p dolfin $$ cd dolfin
git clone --branch=$DOLFIN_VERSION https://bitbucket.org/fenics-project/dolfin
git clone --branch=${DOLFIN_VERSION%.*} https://bitbucket.org/fenics-project/mshr
mkdir -p dolfin/build && cd dolfin/build
cmake3 .. \
-DCMAKE_BUILD_TYPE="Release" \
-DDOLFIN_SKIP_BUILD_TESTS=TRUE \
-DCMAKE_INSTALL_PREFIX=../fenics-lib \
-DCMAKE_CXX_COMPILER=mpiicpc \
-DCMAKE_CXX_FLAGS="-O3 -xHost -ip -std=c++11" \
-DCMAKE_C_COMPILER=mpiicc \
-DCMAKE_C_FLAGS="-O3 -xHost -ip" \
-DCMAKE_Fortran_COMPILER=mpiifort
make -j 48 && make install && cd ../..
export DOLFIN_DIR=$FENICS_INSTALL/dolfin-64/dolfin/fenics-lib/
#Source dolfin lib and exe
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf
mkdir -p mshr/build && cd mshr/build
cmake3 .. \
-DCMAKE_BUILD_TYPE="Release" \
-DCMAKE_INSTALL_PREFIX=../mshr-lib
make -j24 && make install && cd ../..
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf
#!!Please replace cmake by cmake3 in setup.py!!
cd dolfin/python && pip3 install . --log dolfin.log && cd ../..
cd mshr/python && pip3 install . --log mshr.log && cd ../..
#test install (!!LONI only support run fenics in parallel)
cd $FENICS_INSTALL/dolfin/python/demo
python3 generate-demo-files.py
cd documented/cahn-hilliard
#Using ParMetis as mesh partitioner (SCOTCH not work well for intel compiler)
parameters["mesh_partitioner"] = "ParMETIS"
solver.parameters["linear_solver"] = "mumps"
#check parameter info
info(parameters,True)
mpirun -n 1 python3 demo_cahn-hilliard.py
mpirun -n 4 python3 demo_cahn-hilliard.py
#fenics hdf5 test
#Replace following code into demo.py
file = XDMFFile("output.xdmf")
file.write(u.split()[0],t)
mpirun -n 24 python3 demo_cahn-hilliard.py
- Stokes HDG Solver https://bitbucket.org/nate-sime/geopart/src/master/
#Leopart
cd $FENICS_INSTALL #32 bit
mkdir 3rdPartyLib && cd 3rdPartyLib
git clone https://bitbucket.org/jakob_maljaars/leopart.git
git clone https://bitbucket.org/nate-sime/leopart.git
git clone https://binwang31@bitbucket.org/binwang31/leopart.git (this is the latest one)
cd leopart/source/cpp
cmake3 . \
-DCMAKE_CXX_COMPILER=mpiicpc \
-DCMAKE_CXX_FLAGS="-O3 -xHost -ip -std=c++11" \
-DCMAKE_C_COMPILER=mpiicc \
-DCMAKE_C_FLAGS="-O3 -xHost -ip " \
-DPYTHON_EXECUTABLE:FILEPATH=$(which python3)
make -j 24 && cd ../..
pip3 install .
cd unit_tests
#add following in script
#dolfin.parameters["mesh_partitioner"] = "ParMETIS"
mpirun --np 10 pytest -s test_steady_stokes.py
#dolfin_dg
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/dolfin_dg.git
cd dolfin_dg
pip3 install .
#Geopart
cd $FENICS_INSTALL/3rdPartyLib
git clone https://bitbucket.org/nate-sime/geopart.git
git clone --single-branch --branch brinkman https://binwang31@bitbucket.org/binwang31/geopart.git
cd geopart
pip3 install .
cd test/unit
#add following in script
#dolfin.parameters["mesh_partitioner"] = "ParMETIS"
pip3 install pytest
mpirun --np 8 pytest -s test_brinkman.py
$FENICS_ROOT/bashrc.sh
#!/bin/bash
### fenics Environment Profile
# intel/19.0.5
# intel-mpi/2019.5.281
#Setup basic fenics environment variables
export FENICS_ROOT=/project/karsten/bwang31/fenics-opt
export FENICS_VERSION=2019.1.0-64
export FENICS_INSTALL=$FENICS_ROOT/fenics$FENICS_VERSION
cd $FENICS_ROOT
module purge
module load intel/19.0.5
module load intel-mpi/2019.5.281
#Setup MPI variables
export CC=icc
export CXX=icpc
export FC=ifort
export F77=ifort
export F90=ifort
export MPICC=mpiicc
export MPICXX=mpiicpc
export MPIF77=mpiifort
export MPIF90=mpiifort
export CFLAGS='-O3 -xHost'
export CXXFLAGS='-O3 -xHost'
export FCFLAGS='-O3 -xHost'
export F77FLAGS='-O3 -xHost'
export F90FLAGS='-O3 -xHost'
export MPICCFLAGS='-O3 -xHost'
export MPICXXFLAGS='-O3 -xHost'
export MPIF77FLAGS='-O3 -xHost'
export MPIF90FLAGS='-O3 -xHost'
export OMP_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export OPENBLAS_VERBOSE=0
#Enter fenics python virtual env
source $FENICS_INSTALL/python-env-64/bin/activate
#Setup dependencies path
export PETSC_DIR=$FENICS_ROOT/petsc-3.15.0/petsc-lib
export PETSC_ARCH=""
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$FENICS_ROOT/petsc-3.15.0/petsc-lib/lib
export ZLIB_DIR=$FENICS_ROOT/deps/zlib-1.2.11/zlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZLIB_DIR/lib
export SZIP_DIR=$FENICS_ROOT/deps/szip-2.1.1/szlib-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SZIP_DIR/lib
export HDF5_DIR=$FENICS_ROOT/deps/hdf5-1.10.6/hdf5-lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF5_DIR/lib
export EIGEN3_INCLUDE_DIR=$FENICS_ROOT/deps/eigen-3.3.8/include
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$FENICS_ROOT/deps/eigen-3.3.8/Eigen-lib/
export BOOST_DIR=$FENICS_ROOT/deps/boost_1_60_0/
export BOOST_ROOT=$FENICS_ROOT/deps/boost_1_60_0/
export PYBIND11_DIR=$FENICS_INSTALL/pybind11-2.4.3/pybind11-lib
#Setup fenics env
source $FENICS_INSTALL/dolfin/fenics-lib/share/dolfin/dolfin.conf
We can add this bash into ~/.bashrc so we can setup fenics simply by fenics2019
echo '# fenics' >> ~/.bashrc
echo 'alias fenics2019="source /project/bwang31/fenics-opt/bashrc.sh"' >> ~/.bashrc
gmsh,meshio,pygmsh environment
pip3 install pandas
CC=mpiicc pip3 install --no-binary=gmsh gmsh
CC=mpiicc pip3 install --no-binary=h5py h5py
pip3 install pygmsh
module load gcc/8.4.0
module purge
source /project/karsten/bwang31/fenics-opt/bashrc_64.sh
cd /work/bwang31/fenics/
#Gmsh binary dependency, don't load gcc when we we use fenics Expression
module load gcc/8.4.0
echo Cleaning cache...
rm -r *.xml
rm -r *.txt
rm -r *.vtu
rm -r *.h5
rm -r *.pvd
rm -r *.xdmf
rm -r *.npy
SOLVER_PATH=/work/bwang31/fenics/solvers
python3 $SOLVER_PATH/stokes_solver.py --np 48 --mesh_fname hsub40.msh --steps 1 1 0
docker exec -u 0 -it notebook bash