CMakeLists.txt to build Chroma with QUDA -- see https://github.com/lattice/quda/wiki/Chroma-with-QUDA

CMakeLists.txt

# CMAKE script to build CHROMA with QUDA and QDPJIT

# Copyright (c) 2009-2021, QUDA Developers
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

cmake_minimum_required(VERSION 3.19)
project(CHROMABUILD CXX C CUDA)

# Set a default build type if none was specified
set(default_build_type "Release")

if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
  message(
    STATUS
      "Setting build type to '${default_build_type}' as none was specified.")
  set(CMAKE_BUILD_TYPE
      "${default_build_type}"
      CACHE STRING "Choose the type of build." FORCE)
  # Set the possible values of build type for cmake-gui
  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
                                               "MinSizeRel" "RelWithDebInfo")
endif()
string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)

# some options to select build type
set(CHROMABUILD_QUDA_GPU_ARCH
    sm_70
    CACHE STRING "set the GPU architecture for QUDA (sm_60, sm_70, sm_80)")
option(CHROMABUILD_QDPJIT "Use QDPJIT if true or QDPXX otherwise" TRUE)
option(CHROMABUILD_OMP "Enable OpenMP" TRUE)

# location of dependencies
set(CHROMABUILD_QMP_TAG
    qmp2-5-2
    CACHE STRING "git tag for QMP")
set(CHROMABUILD_QMP_REPOSITORY
    https://github.com/usqcd-software/qmp.git
    CACHE STRING "git repository for QMP")
set(CHROMABUILD_LIBXML2_TAG
    master
    CACHE STRING "git tag for libxml2")
set(CHROMABUILD_LIBXML2_REPOSITORY
    https://github.com/GNOME/libxml2.git
    CACHE STRING "git repository for libxml2")
set(CHROMABUILD_LLVM_TAG
    llvmorg-11.0.0
    CACHE STRING "git tag for llvm")
set(CHROMABUILD_LLVM_REPOSITORY
    https://github.com/llvm/llvm-project.git
    CACHE STRING "git repository for llvm")
set(CHROMABUILD_QDPJIT_TAG
    52960068547f88cc9aaad99a5a19262f9e7d2cc4
    CACHE STRING "git tag for qdpjit")
set(CHROMABUILD_QDPJIT_REPOSITORY
    https://github.com/JeffersonLab/qdp-jit.git
    CACHE STRING "git repository for qdpjit")
set(CHROMABUILD_QDPXX_TAG
    qdp1-46-0
    CACHE STRING "git tag for qdpxx")
set(CHROMABUILD_QDPXX_REPOSITORY
    https://github.com/usqcd-software/qdpxx.git
    CACHE STRING "git repository for qdpxx")
set(CHROMABUILD_QUDA_TAG
    aa7cad2101efd66c7506faf7d26fe8ae080eccac
    CACHE STRING "git tag for qdpjit")
set(CHROMABUILD_QUDA_REPOSITORY
    https://github.com/lattice/quda.git
    CACHE STRING "git repository for quda")
set(CHROMABUILD_CHROMA_TAG
    b65cb0ee475eb93b918b6c1f797bc819e6b4ebf2
    CACHE STRING "git tag for Chroma")
set(CHROMABUILD_CHROMA_REPOSITORY
    https://github.com/JeffersonLab/chroma.git
    CACHE STRING "git repository for Chroma")

mark_as_advanced(CHROMABUILD_LIBXML2_REPOSITORY)
mark_as_advanced(CHROMABUILD_LIBXML2_TAG)
mark_as_advanced(CHROMABUILD_LLVM_REPOSITORY)
mark_as_advanced(CHROMABUILD_LLVM_TAG)
mark_as_advanced(CHROMABUILD_QMP_REPOSITORY)
mark_as_advanced(CHROMABUILD_QMP_TAG)
mark_as_advanced(CHROMABUILD_QDPJIT_REPOSITORY)
mark_as_advanced(CHROMABUILD_QDPJIT_TAG)
mark_as_advanced(CHROMABUILD_QDPXX_REPOSITORY)
mark_as_advanced(CHROMABUILD_QDPXX_TAG)
mark_as_advanced(CHROMABUILD_QUDA_REPOSITORY)
mark_as_advanced(CHROMABUILD_QUDA_TAG)
mark_as_advanced(CHROMABUILD_CHROMA_REPOSITORY)
mark_as_advanced(CHROMABUILD_CHROMA_TAG)

set_property(CACHE CHROMABUILD_QUDA_GPU_ARCH PROPERTY STRINGS sm_60 sm_70 sm_80)
if(CHROMABUILD_OMP)
  find_package(OpenMP)
  set(OMPENABLE --enable-openmp)
  set(MY_OMPFLAGS -D_REENTRANT)
endif()
if(CHROMABUILD_QDPJIT)
  set(CHROMABUILD_PREFIX "QDPJIT_${CHROMABUILD_QUDA_GPU_ARCH}")
else(CHROMABUILD_QDPJIT)
  set(CHROMABUILD_PREFIX "QDPXX_${CHROMABUILD_QUDA_GPU_ARCH}")
endif(CHROMABUILD_QDPJIT)

# find some packages
get_filename_component(cuda_dir "${CMAKE_CUDA_COMPILER}" DIRECTORY)
set(PK_CUDA_HOME ${cuda_dir}/../)
find_package(CUDAToolkit)
mark_as_advanced(CMAKE_CUDA_ARCHITECTURES)
# get some packages to pass around
find_package(MPI REQUIRED)
# find_package(LibXml2)
find_program(MAKE_EXE NAMES gmake nmake make)
find_program(INSTALL_EXE NAMES install)
find_program(AUTORECONF_EXE NAMES autoreconf)
# find_program(AUTOUPDATE_EXE NAMES autoupdate)


# set up external project
include(ExternalProject)
set_directory_properties(PROPERTIES EP_BASE ${CMAKE_CURRENT_BINARY_DIR})

# set up flags for autoconf builds
set(PK_CXXFLAGS
    "${OpenMP_CXX_FLAGS} ${MY_OMPFLAGS} ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} -std=c++14 -fexceptions -frtti"
)
set(PK_CFLAGS
    "${OpenMP_C_FLAGS} ${MY_OMPFLAGS} ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}  -std=gnu99 "
)

# ##############################################################################
# QMP
# ##############################################################################
ExternalProject_Add(
  QMP
  GIT_REPOSITORY ${CHROMABUILD_QMP_REPOSITORY}
  GIT_TAG ${CHROMABUILD_QMP_TAG}
  CONFIGURE_COMMAND
    CC=${MPI_C_COMPILER} CXX=${MPI_CXX_COMPILER} <SOURCE_DIR>/configure
    "INSTALL=${INSTALL_EXE} -C" --with-qmp-comms-type=MPI --prefix=<INSTALL_DIR>
  LOG_INSTALL ON
  LOG_BUILD ON
  LOG_DOWNLOAD ON
  LOG_UPDATE ON
  LOG CONFIGURE ON
  LOG_MERGED_STDOUTERR ON
  LOG_OUTPUT_ON_FAILURE ON
  PREFIX common)

ExternalProject_Add_Step(
  QMP reconf
  COMMAND ${AUTORECONF_EXE} -fi
  WORKING_DIRECTORY <SOURCE_DIR>
  DEPENDERS configure
  DEPENDEES download)

ExternalProject_Get_Property(QMP INSTALL_DIR)
set(QMP_INSTALL_DIR ${INSTALL_DIR})

# ##############################################################################
# libxml2
# ##############################################################################
ExternalProject_Add(
  libxml2
  GIT_REPOSITORY https://github.com/GNOME/libxml2.git
  GIT_SHALLOW TRUE
  GIT_TAG master
  CONFIGURE_COMMAND
    CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} <SOURCE_DIR>/autogen.sh
    --prefix=<INSTALL_DIR> -host=x86_64-unknown-linux --build=x86_64-suse-linux
    --disable-shared --without-zlib --without-python --without-readline
    --without-threads --without-history --without-reader --without-writer
    --with-output --without-ftp --without-http --without-pattern
    --without-catalog --without-docbook --without-iconv --without-schemas
    --without-schematron --without-modules --without-xptr --without-xinclude
  LOG_INSTALL ON
  LOG_BUILD ON
  LOG_DOWNLOAD ON
  LOG_UPDATE ON
  LOG_PATCH ON
  LOG CONFIGURE ON
  LOG_MERGED_STDOUTERR ON
  LOG_OUTPUT_ON_FAILURE ON
  PREFIX common)

ExternalProject_Get_Property(libxml2 INSTALL_DIR)
set(LIBXML2_INSTALL_DIR ${INSTALL_DIR})


if(CHROMABUILD_QDPJIT)

# ##############################################################################
# LLVM for QDPJIT
# ##############################################################################
  ExternalProject_Add(
    llvm
    GIT_REPOSITORY ${CHROMABUILD_LLVM_REPOSITORY}
    GIT_TAG ${CHROMABUILD_LLVM_TAG}
    GIT_SHALLOW TRUE
    SOURCE_SUBDIR llvm
    CMAKE_ARGS -DLLVM_ENABLE_ZLIB=OFF
               -DBUILD_SHARED_LIBS=OFF
               -DLLVM_ENABLE_RTTI=ON
               -DLLVM_ENABLE_TERMINFO=OFF
               -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
               -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
               -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
               -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
               -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
               -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
    CMAKE_CACHE_ARGS -DLLVM_TARGETS_TO_BUILD:STRING=X86;NVPTX
    LOG_INSTALL ON
    LOG_BUILD ON
    LOG_DOWNLOAD ON
    LOG_UPDATE ON
    LOG CONFIGURE ON
    LOG_MERGED_STDOUTERR ON
    LOG_OUTPUT_ON_FAILURE ON
    PREFIX ${CHROMABUILD_PREFIX})

  ExternalProject_Get_Property(llvm INSTALL_DIR)
  set(LLVM_INSTALL_DIR ${INSTALL_DIR})

  # ############################################################################
  # QDPJIT
  # ############################################################################
  ExternalProject_Add(
    QDPJIT
    GIT_REPOSITORY ${CHROMABUILD_QDPJIT_REPOSITORY}
    GIT_TAG ${CHROMABUILD_QDPJIT_TAG}
    CONFIGURE_COMMAND
      <SOURCE_DIR>/configure "INSTALL=${INSTALL_EXE} -C" CC=${MPI_C_COMPILER}
      CXX=${MPI_CXX_COMPILER} "CXXFLAGS=${PK_CXXFLAGS}" "CFLAGS=${PK_CFLAGS}"
      --prefix=<INSTALL_DIR> --with-libxml2=${LIBXML2_INSTALL_DIR}
      --with-qmp=${QMP_INSTALL_DIR} --enable-parallel-arch=parscalar
      --enable-precision=double --enable-largefile --enable-parallel-io
      --disable-cuda-managed-memory --enable-dml-output-buffering
      --disable-generics --with-cuda=${PK_CUDA_HOME}
      --with-llvm=${LLVM_INSTALL_DIR} --enable-llvm11
      --enable-comm-split-deviceinit --host=x86_64-linux-gnu --build=none
      ${OMPENABLE}
    LOG_INSTALL ON
    LOG_BUILD ON
    LOG_DOWNLOAD ON
    LOG_UPDATE ON
    LOG CONFIGURE ON
    LOG_MERGED_STDOUTERR ON
    LOG_OUTPUT_ON_FAILURE ON
    PREFIX ${CHROMABUILD_PREFIX})

  ExternalProject_Add_Step(
    QDPJIT reconf
    COMMAND ${AUTORECONF_EXE} -fi
    WORKING_DIRECTORY <SOURCE_DIR>
    DEPENDERS configure
    DEPENDEES download)

  find_file(
    CUDA_LIBDEVICE name libdevice.10.bc
    HINTS "${CUDAToolkit_LIBRARY_ROOT}/nvvm/libdevice" ${PK_CUDA_HOME}
    NO_DEFAULT_PATH)
  mark_as_advanced(CUDA_LIBDEVICE)

  ExternalProject_Add_Step(
    QDPJIT libdevice
    WORKING_DIRECTORY <SOURCE_DIR>
    COMMAND cmake -E make_directory libdevice_files
    COMMAND cmake -E copy ${CUDA_LIBDEVICE} libdevice_files/libdevice.bc
    COMMAND ./pack_libdevice.sh ./libdevice_files
    DEPENDERS configure reconf
    DEPENDEES download)

  ExternalProject_Add_StepDependencies(QDPJIT build llvm)
  ExternalProject_Add_StepDependencies(QDPJIT build QMP)
  ExternalProject_Add_StepDependencies(QDPJIT build libxml2)

  ExternalProject_Get_Property(QDPJIT INSTALL_DIR)
  set(QDPJIT_INSTALL_DIR ${INSTALL_DIR})

else(CHROMABUILD_QDPJIT)

  # ############################################################################
  # QDPXX
  # ############################################################################
  ExternalProject_Add(
    QDPXX
    GIT_REPOSITORY ${CHROMABUILD_QDPXX_REPOSITORY}
    GIT_TAG ${CHROMABUILD_QDPXX_TAG}
    CONFIGURE_COMMAND
      <SOURCE_DIR>/configure "INSTALL=${INSTALL_EXE} -C" CC=${MPI_C_COMPILER}
      CXX=${MPI_CXX_COMPILER} "CXXFLAGS=${PK_CXXFLAGS}" "CFLAGS=${PK_CFLAGS}"
      --prefix=<INSTALL_DIR> --with-libxml2=${LIBXML2_INSTALL_DIR}
      --with-qmp=${QMP_INSTALL_DIR} --enable-parallel-arch=parscalar
      --enable-precision=double --enable-largefile --enable-parallel-io
      --enable-dml-output-buffering --enable-db-lite --disable-generics
      --host=x86_64-linux-gnu --build=none ${OMPENABLE}
    LOG_INSTALL ON
    LOG_BUILD ON
    LOG_DOWNLOAD ON
    LOG_UPDATE ON
    LOG CONFIGURE ON
    LOG_MERGED_STDOUTERR ON
    LOG_OUTPUT_ON_FAILURE ON
    PREFIX ${CHROMABUILD_PREFIX})

  ExternalProject_Add_Step(
    QDPXX reconf
    COMMAND ${AUTORECONF_EXE} -fi
    WORKING_DIRECTORY <SOURCE_DIR>
    DEPENDERS configure
    DEPENDEES download)

  ExternalProject_Add_StepDependencies(QDPXX build QMP)
  ExternalProject_Add_StepDependencies(QDPXX build libxml2)

  ExternalProject_Get_Property(QDPXX INSTALL_DIR)
  set(QDPXX_INSTALL_DIR ${INSTALL_DIR})

endif(CHROMABUILD_QDPJIT)

# ##############################################################################
# QUDA
# ##############################################################################
ExternalProject_Add(
  QUDA
  GIT_REPOSITORY ${CHROMABUILD_QUDA_REPOSITORY}
  GIT_TAG ${CHROMABUILD_QUDA_TAG}
  CMAKE_ARGS -DQUDA_DIRAC_DEFAULT_OFF=ON
             -DQUDA_DIRAC_CLOVER=ON
             -DQUDA_DIRAC_WILSON=ON
             -DQUDA_DYNAMIC_CLOVER=OFF
             -DQUDA_HETEROGENEOUS_ATOMIC=OFF
             -DQUDA_FORCE_GAUGE=OFF
             -DQUDA_FORCE_HISQ=OFF
             -DQUDA_GAUGE_ALG=OFF
             -DQUDA_GAUGE_TOOLS=OFF
             -DQUDA_GPU_ARCH=${CHROMABUILD_QUDA_GPU_ARCH}
             -DQUDA_INTERFACE_MILC=OFF
             -DQUDA_INTERFACE_CPS=OFF
             -DQUDA_INTERFACE_QDP=ON
             -DQUDA_INTERFACE_TIFR=OFF
             -DQUDA_MAGMA=OFF
             -DQUDA_QMP=ON
             -DQUDA_QMPHOME=${QMP_INSTALL_DIR}
             -DQUDA_MULTIGRID=ON
             -DQUDA_MAX_MULTI_BLAS_N=9
             -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
             -DCMAKE_BUILD_TYPE=DEVEL
             -DQUDA_BUILD_SHAREDLIB=ON
             -DQUDA_BUILD_ALL_TESTS=OFF
             -DQUDA_PROPAGATE_CXX_FLAGS=OFF
             -DQUDA_INTERFACE_QDPJIT=${CHROMABUILD_QDPJIT}
             -DQUDA_QDPJIT=${CHROMABUILD_QDPJIT}
             -DQUDA_QDPJITHOME=${QDPJIT_INSTALL_DIR}
  LOG_INSTALL ON
  LOG_BUILD ON
  LOG_DOWNLOAD ON
  LOG_UPDATE ON
  LOG CONFIGURE ON
  LOG_MERGED_STDOUTERR ON
  LOG_OUTPUT_ON_FAILURE ON
  PREFIX ${CHROMABUILD_PREFIX})

if(CHROMABUILD_QDPJIT)
  ExternalProject_Add_StepDependencies(QUDA build QDPJIT)
endif(CHROMABUILD_QDPJIT)
ExternalProject_Add_StepDependencies(QUDA build QMP)

ExternalProject_Get_Property(QUDA INSTALL_DIR)
set(QUDA_INSTALL_DIR ${INSTALL_DIR})

# ##############################################################################
# CHROMA
# ##############################################################################
set(PK_LIBS "-ldl -lpthread")
set(PK_LDFLAGS
    "-rdynamic ${OMPFLAGS} -L${PK_CUDA_HOME}/lib64 -L${PK_CUDA_HOME}/lib64/stubs -L${PK_CUDA_HOME}/nvvm/lib64"
)

if(CHROMABUILD_QDPJIT)
  set(QDP_INSTALL_DIR ${QDPJIT_INSTALL_DIR})
  set(CHROMA_QDP_FLAGS --enable-jit-clover --enable-quda-deviface)
  set(CHROMA_LDFLAGS
      "${PK_LDFLAGS} -L${LLVM_INSTALL_DIR}/lib -Wl,-rpath,${LLVM_INSTALL_DIR}/lib "
  )
else(CHROMABUILD_QDPJIT)
  set(QDP_INSTALL_DIR ${QDPXX_INSTALL_DIR})
  set(CHROMA_QDP_FLAGS --enable-c-scalarsite-bicgstab-kernels
                       --enable-cpp-wilson-dslash)
  set(CHROMA_LDFLAGS "${PK_LDFLAGS} -Wl,-zmuldefs")
endif(CHROMABUILD_QDPJIT)

ExternalProject_Add(
  Chroma
  GIT_REPOSITORY ${CHROMABUILD_CHROMA_REPOSITORY}
  GIT_TAG ${CHROMABUILD_CHROMA_TAG}
  CONFIGURE_COMMAND
    <SOURCE_DIR>/configure --prefix=<INSTALL_DIR> CC=${MPI_C_COMPILER}
    CXX=${MPI_CXX_COMPILER} CXXFLAGS=${PK_CXXFLAGS} CFLAGS=${PK_CFLAGS}
    "LDFLAGS=${CHROMA_LDFLAGS}" "LIBS=${PK_LIBS}" --with-qdp=${QDP_INSTALL_DIR}
    ${OMPENABLE} --with-qmp=${QMP_INSTALL_DIR} --host=x86_64-linux-gnu
    --build=none--with-cuda=${PK_CUDA_HOME} --with-quda=${QUDA_INSTALL_DIR}
    ${CHROMA_QDP_FLAGS}
  LOG_INSTALL ON
  LOG_BUILD ON
  LOG_DOWNLOAD ON
  LOG_CONFIGURE ON
  LOG_UPDATE ON
  LOG_PATCH ON
  LOG_MERGED_STDOUTERR ON
  LOG_OUTPUT_ON_FAILURE ON
  PREFIX ${CHROMABUILD_PREFIX})

ExternalProject_Add_Step(
  Chroma reconf
  COMMAND ${AUTORECONF_EXE}
  WORKING_DIRECTORY <SOURCE_DIR>
  DEPENDERS configure
  DEPENDEES download update)

if(CHROMABUILD_QDPJIT)
  ExternalProject_Add_StepDependencies(Chroma build QDPJIT)
else(CHROMABUILD_QDPJIT)
  ExternalProject_Add_StepDependencies(Chroma build QDPXX)
endif(CHROMABUILD_QDPJIT)

ExternalProject_Add_StepDependencies(Chroma build QMP)
ExternalProject_Add_StepDependencies(Chroma build QUDA)