TCAD Charon GPU CUDA optimization

General.opts

# This is to avoid a warning from cmake
tcad-charon_ENABLE_PyTrilinos:BOOL=OFF

tcad-charon_TEST_CATEGORIES:STRING="HEAVY"

tcad-charon_VERBOSE_CONFIGURE:BOOL=OFF

tcad-charon_ENABLE_Kokkos:BOOL=ON
tcad-charon_ENABLE_KokkosAlgorithms:BOOL=ON
tcad-charon_ENABLE_KokkosCore:BOOL=ON

tcad-charon_ENABLE_INSTALL_CMAKE_CONFIG_FILES:BOOL=ON
tcad-charon_ENABLE_CHECKED_STL:BOOL=OFF
tcad-charon_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON
tcad-charon_ENABLE_MueLu:BOOL=ON
tcad-charon_ENABLE_Stokhos:BOOL=OFF
tcad-charon_ENABLE_ROL:BOOL=OFF
tcad-charon_ENABLE_SECONDARY_TESTED_CODE:BOOL=OFF

tcad-charon_ENABLE_ALL_PACKAGES:BOOL=OFF
tcad-charon_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=OFF

tcad-charon_ENABLE_SEACASEx1ex2v2:BOOL=OFF
tcad-charon_ENABLE_SEACASEx2ex1v2:BOOL=OFF
tcad-charon_ENABLE_SEACASExomatlab:BOOL=OFF

# Options that are required for Xyce builds and useful for the type 1
# stack.
tcad-charon_ENABLE_Sacado:BOOL=ON
tcad-charon_ENABLE_AztecOO:BOOL=ON
tcad-charon_ENABLE_Ifpack:BOOL=ON

# Options required or useful for tpetra
tcad-charon_ENABLE_Amesos2:BOOL=ON
tcad-charon_ENABLE_Ifpack2:BOOL=ON
tcad-charon_ENABLE_Tempus:BOOL=ON
tcad-charon_ENABLE_Zoltan2:BOOL=ON
tcad-charon_ENABLE_Xpetra:BOOL=ON

tcad-charon_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON

tcad-charon_ENABLE_STKUnit_test_utils:BOOL=OFF
tcad-charon_ENABLE_STKSearch:BOOL=OFF
tcad-charon_ENABLE_STKSearchUtil:BOOL=OFF
tcad-charon_ENABLE_STKUnit_tests:BOOL=OFF
tcad-charon_ENABLE_STKDoc_tests:BOOL=OFF
tcad-charon_ENABLE_STKExprEval:BOOL=OFF

tcad-charon_ENABLE_TESTS:BOOL=OFF
tcad-charon_ENABLE_EXAMPLES:BOOL=OFF

tcad-charon_ENABLE_DAKOTA_DRIVERS:BOOL=ON

tcad-charon_ENABLE_Epetra:BOOL=ON

AztecOO_ENABLE_TEUCHOS_TIME_MONITOR:BOOL=ON

Intrepid2_ENABLE_DEBUG_INF_CHECK:BOOL=OFF

SEACASExodus_ENABLE_MPI:BOOL=OFF

TPL_ENABLE_MPI:BOOL=ON
TPL_ENABLE_Matio:BOOL=OFF
TPL_ENABLE_X11:BOOL=OFF

tcad-charon_ENABLE_Charon:BOOL=ON

Kokkos_ENABLE_SERIAL:BOOL=ON
Kokkos_ENABLE_OPENMP:BOOL=OFF
Kokkos_ENABLE_PTHREAD:BOOL=OFF
Kokkos_ENABLE_CUDA:BOOL=OFF

Charon_ENABLE_TESTS:BOOL=ON
Charon_ENABLE_EXAMPLES:BOOL=ON
Charon_ENABLE_DEBUG:BOOL=ON

CMAKE_VERBOSE_MAKEFILE:BOOL=OFF
CMAKE_SKIP_RULE_DEPENDENCY:BOOL=ON
CMAKE_CXX_STANDARD:STRING=14

TPL_ENABLE_HDF5:BOOL=OFF

TPL_ENABLE_Boost:BOOL=ON
TPL_ENABLE_BoostLib:BOOL=ON

TPL_ENABLE_Netcdf:BOOL=ON

Panzer_ENABLE_TESTS:BOOL=OFF

EpetraExt_ENABLE_HDF5:BOOL=OFF

# Don't generally want to search install path for libraries as that
# will find possibly outdated files.
CMAKE_FIND_NO_INSTALL_PREFIX:BOOL=ON

GIT_EXECUTABLE:FILEPATH="$(which git)"

gpu.opts

##########################################################################################################
# Options file to enable kokkos/cuda and openmp support in charon
##########################################################################################################
# Required/suggested modules:
#    sems-env sems-git/2.10.1 sems-cmake/3.19.1 sems-gcc/7.2.0 sems-cuda/10.1 sems-openmpi/4.0.5
#    sems-hdf5/1.10.6/parallel sems-netcdf/4.7.3/parallel sems-zlib/1.2.8/base sems-boost/1.58.0/base
# then either
#    module load sems-python/3.5.2
# or
#    module swap sems-python/2.7.9 sems-python/3.5.2
#
# Additionally
#   export OMPI_CXX=<base dir>/tcad-charon/Trilinos/packages/kokkos/bin/nvcc_wrapper

tcad-charon_ENABLE_OpenMP:BOOL=ON

#################################
# CUDA/GPU settings
#################################
TPL_ENABLE_CUDA:BOOL=ON
TPL_ENABLE_HDF5:BOOL=ON

Sacado_ENABLE_HIERARCHICAL:BOOL=OFF
Sacado_ENABLE_HIERARCHICAL_DFAD:BOOL=OFF
Sacado_ENABLE_MEMORY_POOL:BOOL=OFF

Tpetra_ENABLE_CUDA:BOOL=ON
Tpetra_INST_SERIAL:BOOL=ON
Tpetra_INST_INT_LONG_LONG:BOOL=ON

# Relocatable code requires a static build
BUILD_SHARED_LIBS:BOOL=ON
Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE:BOOL=OFF

Kokkos_ENABLE_CUDA_UVM:BOOL=ON
Kokkos_ENABLE_CUDA:BOOL=ON
Kokkos_ENABLE_CUDA_LAMBDA:BOOL=ON
Kokkos_ENABLE_OPENMP:BOOL=ON
Kokkos_ENABLE_SERIAL:BOOL=ON
#Kokkos_ARCH_VOLTA70:BOOL=ON
Kokkos_ARCH_TURING75:BOOL=ON
#Kokkos_CUDA_DIR:PATH=/projects/sems/install/rhel7-x86_64/sems/compiler/cuda/10.1/base
Kokkos_CUDA_DIR:PATH=/usr/local/cuda/

Linux.opts

# Shared libs don't seem to work on the Mac when using the GNU
# compilers so only enable them for Linux

BUILD_SHARED_LIBS:BOOL=ON