gistfile1.txt

{stack-trace}: 4 frames:
0x7fca4415c40c  : hpx::detail::backtrace[abi:cxx11](unsigned long) + 0x9c in /home/zbyerly/work/local_install_Release/lib/libhp
x.so.1
0x7fca4416ff27  : std::__exception_ptr::exception_ptr hpx::detail::get_exception<hpx::exception>(hpx::exception const&, std::__
cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_tra
its<char>, std::allocator<char> > const&, long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
 const&) + 0x97 in /home/zbyerly/work/local_install_Release/lib/libhpx.so.1
0x7fca44170445  : void hpx::detail::throw_exception<hpx::exception>(hpx::exception const&, std::__cxx11::basic_string<char, std
::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<cha
r> > const&, long) + 0x55 in /home/zbyerly/work/local_install_Release/lib/libhpx.so.1
0x7fca441bda1e  : hpx::detail::throw_exception(hpx::error, std::__cxx11::basic_string<char, std::char_traits<char>, std::alloca
tor<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic
_string<char, std::char_traits<char>, std::allocator<char> > const&, long) + 0x4e in /home/zbyerly/work/local_install_Release/l
ib/libhpx.so.1
{env}: 148 entries:
  ACLOCAL_PATH=/opt/eb/software/cmake/3.9.0/share/aclocal
  BASH_FUNC_module()=() {  eval `/usr/bin/modulecmd bash $*`
}
  BOOST_ROOT=/opt/boost/1.64.0-gcc6.4.0/release
  BOOST_VERSION=1.64.0
  CPATH=/opt/eb/software/impi/2017.2.174/include64:/opt/eb/software/hwloc/1.11.3/include:/opt/mn/gcc/6.4.0/include
  DAPL_NETWORK_PROCESS_NUM=1
  DISPLAY=localhost:10.0
  EBDEVELCMAKE=/opt/eb/software/cmake/3.9.0/easybuild/cmake-3.9.0-easybuild-devel
  EBDEVELHWLOC=/opt/eb/software/hwloc/1.11.3/easybuild/hwloc-1.11.3-easybuild-devel
  EBDEVELIMPI=/opt/eb/software/impi/2017.2.174/easybuild/impi-2017.2.174-easybuild-devel
  EBROOTCMAKE=/opt/eb/software/cmake/3.9.0
  EBROOTHWLOC=/opt/eb/software/hwloc/1.11.3
  EBROOTIMPI=/opt/eb/software/impi/2017.2.174
  EBVERSIONCMAKE=3.9.0
  EBVERSIONHWLOC=1.11.3
  EBVERSIONIMPI=2017.2.174
  EDITOR=emacs
  ENVIRONMENT=BATCH
  GFORTRAN_UNBUFFERED_PRECONNECTED=y
  HISTCONTROL=ignoredups
  HISTFILESIZE=10000
  HISTSIZE=10000
  HOME=/home/zbyerly
  HOSTNAME=marvin08
  I_MPI_CMD=mpirun -np 1 -ppn 1 /home/zbyerly/work/dgswem-hpx/builds/build_Release_intel_new/src/dgswem_lgd_dataflow --hpx:thre
ads=16 --hpx:localities=1 --n_timesteps=100 --hpx:print-counter=/arithmetics/mean@/threads{locality#*/worker-thread#*}/time/ave
rage --hpx:print-counter=/threads{locality#0/total}/idle-rate --hpx:print-counter=/arithmetics/mean@/threads{locality#0/worker-
thread#*}/time/overall --hpx:print-counter-destination=/home/zbyerly/work/dgswem-hpx/scripts/run/Sep14 
  I_MPI_EXT_MPI_CALLS_2821=0x7fca4169c5d0
  I_MPI_HYDRA_BOOTSTRAP=slurm
  I_MPI_HYDRA_UUID=f00a0000-2cc5-1b65-2b59-0500033a0a03
  I_MPI_INFO_BRAND=Intel(R) Xeon(R) 
  I_MPI_INFO_CACHE1=0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23
  I_MPI_INFO_CACHE2=0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23
  I_MPI_INFO_CACHE3=0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1
  I_MPI_INFO_CACHES=3
  I_MPI_INFO_CACHE_SHARE=2,2,32
  I_MPI_INFO_CACHE_SIZE=32768,262144,20971520
  I_MPI_INFO_CORE=0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7
  I_MPI_INFO_C_NAME=Unknown
  I_MPI_INFO_DESC=1342177285
  I_MPI_INFO_FLGB=0
  I_MPI_INFO_FLGC=532603903
  I_MPI_INFO_FLGCEXT=0
  I_MPI_INFO_FLGD=-1075053569
  I_MPI_INFO_FLGDEXT=0
  I_MPI_INFO_LCPU=16
  I_MPI_INFO_MODE=263
  I_MPI_INFO_NC=16
  I_MPI_INFO_NP=2
  I_MPI_INFO_NUMA_NODE_MAP=mlx4_0:0
  I_MPI_INFO_NUMA_NODE_NUM=2
  I_MPI_INFO_PACK=0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1
  I_MPI_INFO_SERIAL=E5-2450 0 
  I_MPI_INFO_SIGN=132823
  I_MPI_INFO_STATE=0
  I_MPI_INFO_THREAD=0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
  I_MPI_INFO_VEND=1
  I_MPI_INTERNAL_MEM_POLICY=default
  I_MPI_MPIRUN=mpirun
  I_MPI_NUMBER_OF_MPI_PROCESSES_PER_NODE=1
  I_MPI_PERHOST=allcores
  I_MPI_PIN_INFO=x0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
  I_MPI_PIN_MAPPING=1:0 0
  I_MPI_PM=hydra
  I_MPI_RANK_CMD=/home/zbyerly/work/dgswem-hpx/builds/build_Release_intel_new/src/dgswem_lgd_dataflow --hpx:threads=16 --hpx:lo
calities=1 --n_timesteps=100 --hpx:print-counter=/arithmetics/mean@/threads{locality#*/worker-thread#*}/time/average --hpx:prin
t-counter=/threads{locality#0/total}/idle-rate --hpx:print-counter=/arithmetics/mean@/threads{locality#0/worker-thread#*}/time/
overall --hpx:print-counter-destination=/home/zbyerly/work/dgswem-hpx/scripts/run/Sep14 
  I_MPI_ROOT=/opt/eb/software/impi/2017.2.174
  I_MPI_THREAD_LEVEL=0
  LANG=en_US.UTF-8
  LD_LIBRARY_PATH=/opt/eb/software/ifort/2017.4.196/compilers_and_libraries_2017.4.196/linux/bin/intel64/../../compiler/lib/int
el64:/opt/eb/software/impi/2017.2.174/lib64:/opt/boost/1.64.0-gcc6.4.0/release/lib:/opt/eb/software/hwloc/1.11.3/lib:/opt/mn/gc
c/6.4.0/lib/gcc/x86_64-redhat-linux/6.4.0:/opt/mn/gcc/6.4.0/lib64:/opt/mn/gcc/6.4.0/lib:/opt/eb/software/impi/2017.2.174/compil
ers_and_libraries_2017.2.174/linux/mpi/intel64/lib:/opt/eb/software/impi/2017.2.174/compilers_and_libraries_2017.2.174/linux/mp
i/mic/lib
  LESSOPEN=||/usr/bin/lesspipe.sh %s
  LIBRARY_PATH=/opt/eb/software/ifort/2017.4.196/compilers_and_libraries_2017.4.196/linux/bin/intel64/../../compiler/lib/intel6
4:/opt/eb/software/impi/2017.2.174/lib64:/opt/eb/software/hwloc/1.11.3/lib:/opt/mn/gcc/6.4.0/lib64:/opt/mn/gcc/6.4.0/lib
  LOADEDMODULES=gcc/6.4.0:cmake/3.9.0:hwloc/1.11.3:boost/1.64.0-gcc6.4.0:impi/2017.2.174
  LOGNAME=zbyerly
  LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:su=37;41:sg=30;43:ca=30
;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.l
zh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.l
rz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*
.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;
31:*.cab=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.x
pm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=0
1;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*
.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;3
5:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.axv=01;35:*.anx=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac
=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.axa=00;36:*.oga=00;36:
*.spx=00;36:*.xspf=00;36:
  MAIL=/var/spool/mail/zbyerly
  MANPATH=/opt/eb/software/hwloc/1.11.3/share/man:/opt/mn/gcc/6.4.0/share/man:/usr/share/man:/usr/local/share/man
  MIC_LD_LIBRARY_PATH=/opt/eb/software/impi/2017.2.174/mic/lib
  MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/opt/boost/modules:/opt/modules:/opt/eb/modules/all
  MODULESHOME=/usr/share/Modules
  MPIR_CVAR_CH3_INTERFACE_HOSTNAME=marvin08
  MPIR_CVAR_NEMESIS_ENABLE_CKPOINT=1
  NLSPATH=/opt/eb/software/impi/2017.2.174/idb/intel64/locale/%l_%t/%N
  PATH=/opt/eb/software/impi/2017.2.174/bin64:/opt/eb/software/hwloc/1.11.3/sbin:/opt/eb/software/hwloc/1.11.3/bin:/opt/eb/soft
ware/cmake/3.9.0/bin:/opt/mn/gcc/6.4.0/bin:/home/zbyerly/bin:/home/zbyerly/bin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/bin:/u
sr/local/sbin:/usr/sbin
  PKG_CONFIG_PATH=/opt/eb/software/hwloc/1.11.3/lib/pkgconfig
  PMI_FD=7
  PMI_RANK=0
  PMI_SIZE=1
  PWD=/home/zbyerly/scratch/meshes/wetdrybedslope-xlarge/512-scotchtest
  QTDIR=/usr/lib64/qt-3.3
  QTINC=/usr/lib64/qt-3.3/include
  QTLIB=/usr/lib64/qt-3.3/lib
  QT_GRAPHICSSYSTEM_CHECKED=1
  ROOTGCC=/opt/mn/gcc/6.4.0
  SHELL=/bin/bash
  SHLVL=3
  SLURMD_NODENAME=marvin08
  SLURM_CHECKPOINT_IMAGE_DIR=/var/lib/slurm/checkpoint
  SLURM_CLUSTER_NAME=rostam
  SLURM_CPUS_ON_NODE=16
  SLURM_DISTRIBUTION=cyclic
  SLURM_GTIDS=0
  SLURM_JOBID=1744305
  SLURM_JOB_CPUS_PER_NODE=16
  SLURM_JOB_GID=2000
  SLURM_JOB_ID=1744305
  SLURM_JOB_NAME=run-hpx.sbatch
  SLURM_JOB_NODELIST=marvin08
  SLURM_JOB_NUM_NODES=1
  SLURM_JOB_PARTITION=storm
  SLURM_JOB_UID=3004
  SLURM_JOB_USER=zbyerly
  SLURM_LAUNCH_NODE_IPADDR=10.3.3.58
  SLURM_LOCALID=0
  SLURM_NNODES=1
  SLURM_NODEID=0
  SLURM_NODELIST=marvin08
  SLURM_NODE_ALIASES=(null)
  SLURM_NPROCS=1
  SLURM_NTASKS=1
  SLURM_PRIO_PROCESS=0
  SLURM_PROCID=0
  SLURM_SRUN_COMM_HOST=10.3.3.58
  SLURM_SRUN_COMM_PORT=37760
  SLURM_STEPID=0
  SLURM_STEP_ID=0
  SLURM_STEP_LAUNCHER_PORT=37760
  SLURM_STEP_NODELIST=marvin08
  SLURM_STEP_NUM_NODES=1
  SLURM_STEP_NUM_TASKS=1
  SLURM_STEP_TASKS_PER_NODE=1
  SLURM_SUBMIT_DIR=/home/zbyerly/work/dgswem-hpx/scripts/run/Sep14
  SLURM_SUBMIT_HOST=bob1
  SLURM_TASKS_PER_NODE=1
  SLURM_TASK_PID=2789
  SLURM_TOPOLOGY_ADDR=marvin08
  SLURM_TOPOLOGY_ADDR_PATTERN=node
  SLURM_UMASK=0022
  SRUN_DEBUG=3
  SSH_AUTH_SOCK=/tmp/ssh-lFWyeIK4k8/agent.1944
  SSH_CLIENT=10.3.3.1 47940 22
  SSH_CONNECTION=10.3.3.1 47940 10.3.3.8 22
  SSH_TTY=/dev/pts/0
  TERM=xterm-256color
  TMPDIR=/tmp
  USER=zbyerly
  VERSIONGCC=6.4.0
  XDG_RUNTIME_DIR=/run/user/3004
  XDG_SESSION_ID=8
  _=/opt/eb/software/impi/2017.2.174/bin64/mpiexec.hydra
  _LMFILES_=/opt/modules/gcc/6.4.0:/opt/eb/modules/all/cmake/3.9.0:/opt/eb/modules/all/hwloc/1.11.3:/opt/boost/modules/boost/1.
64.0-gcc6.4.0:/opt/eb/modules/all/impi/2017.2.174
{locality-id}: 0
{hostname}: [ (mpi:0) (tcp:10.3.3.58:7910) ]
{process-id}: 2821
{function}: partitioner::get_pool_name: 
{file}: /home/zbyerly/work/hpx/src/runtime/resource/detail/detail_partitioner.cpp
{line}: 45
{os-thread}: 10, worker-thread#10
{thread-id}: 00007fca24c247a0
{thread-description}: <unknown>
{state}: state_startup
{auxinfo}: 
{config}:
  HPX_WITH_AGAS_DUMP_REFCNT_ENTRIES=OFF
  HPX_WITH_APEX=OFF
  HPX_WITH_ATTACH_DEBUGGER_ON_TEST_FAILURE=OFF
  HPX_WITH_AUTOMATIC_SERIALIZATION_REGISTRATION=ON
  HPX_WITH_CXX14_RETURN_TYPE_DEDUCTION=TRUE
  HPX_WITH_DEPRECATION_WARNINGS=ON
  HPX_WITH_EXECUTION_POLICY_COMPATIBILITY=ON
  HPX_WITH_EXECUTOR_COMPATIBILITY=ON
  HPX_WITH_GOOGLE_PERFTOOLS=OFF
  HPX_WITH_HWLOC=ON
  HPX_WITH_INCLUSIVE_SCAN_COMPATIBILITY=ON
  HPX_WITH_IO_COUNTERS=ON
  HPX_WITH_IO_POOL=ON
  HPX_WITH_ITTNOTIFY=OFF
  HPX_WITH_LOCAL_DATAFLOW_COMPATIBILITY=ON
  HPX_WITH_LOGGING=ON
  HPX_WITH_MORE_THAN_64_THREADS=(
  HPX_WITH_NATIVE_TLS=ON
  HPX_WITH_NETWORKING=ON
  HPX_WITH_PAPI=OFF
  HPX_WITH_PARCELPORT_ACTION_COUNTERS=OFF
  HPX_WITH_PARCELPORT_LIBFABRIC=OFF
  HPX_WITH_PARCELPORT_MPI=true
  HPX_WITH_PARCELPORT_MPI_MULTITHREADED=ON
  HPX_WITH_PARCELPORT_TCP=ON
  HPX_WITH_PARCELPORT_VERBS=OFF
  HPX_WITH_PARCEL_COALESCING=ON
  HPX_WITH_PARCEL_PROFILING=OFF
  HPX_WITH_SCHEDULER_LOCAL_STORAGE=OFF
  HPX_WITH_SPINLOCK_DEADLOCK_DETECTION=OFF
  HPX_WITH_STACKTRACES=ON
  HPX_WITH_SWAP_CONTEXT_EMULATION=OFF
  HPX_WITH_THREAD_BACKTRACE_ON_SUSPENSION=OFF
  HPX_WITH_THREAD_CREATION_AND_CLEANUP_RATES=OFF
  HPX_WITH_THREAD_CUMULATIVE_COUNTS=ON
  HPX_WITH_THREAD_DEBUG_INFO=OFF
  HPX_WITH_THREAD_DESCRIPTION_FULL=OFF
  HPX_WITH_THREAD_GUARD_PAGE=ON
  HPX_WITH_THREAD_IDLE_RATES=true
  HPX_WITH_THREAD_LOCAL_STORAGE=OFF
  HPX_WITH_THREAD_MANAGER_IDLE_BACKOFF=ON
  HPX_WITH_THREAD_QUEUE_WAITTIME=OFF
  HPX_WITH_THREAD_STACK_MMAP=ON
  HPX_WITH_THREAD_STEALING_COUNTS=ON
  HPX_WITH_THREAD_TARGET_ADDRESS=OFF
  HPX_WITH_TIMER_POOL=ON
  HPX_WITH_TRANSFORM_REDUCE_COMPATIBILITY=ON
  HPX_WITH_TUPLE_RVALUE_SWAP=ON
  HPX_WITH_UNWRAPPED_COMPATIBILITY=ON
  HPX_WITH_VALGRIND=OFF
  HPX_WITH_VERIFY_LOCKS=OFF
  HPX_WITH_VERIFY_LOCKS_BACKTRACE=OFF
  HPX_WITH_VERIFY_LOCKS_GLOBALLY=OFF

  HPX_PARCEL_MAX_CONNECTIONS=512
  HPX_PARCEL_MAX_CONNECTIONS_PER_LOCALITY=4
  HPX_AGAS_LOCAL_CACHE_SIZE=4096
  HPX_HAVE_MALLOC=jemalloc
  HPX_PREFIX (configured)=/home/zbyerly/work/local_install_Release
  HPX_PREFIX=/home/zbyerly/work/local_install_Release
{version}: V1.1.0 (AGAS: V3.0), Git: 7eac243d93
{boost}: V1.64.0
{build-type}: release
{date}: Sep 12 2017 12:01:14
{platform}: linux
{compiler}: GNU C++ version 6.4.0
{stdlib}: GNU libstdc++ version 20170704
{what}: pool 1 (zero-based index) requested out of bounds. The partitioner owns only 1 pools: HPX(bad_parameter)

===================================================================================
=   BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES
=   PID 2821 RUNNING AT marvin08
=   EXIT CODE: 134
=   CLEANING UP REMAINING PROCESSES
=   YOU CAN IGNORE THE BELOW CLEANUP MESSAGES
===================================================================================

===================================================================================
=   BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES
=   PID 2821 RUNNING AT marvin08
=   EXIT CODE: 6
=   CLEANING UP REMAINING PROCESSES
=   YOU CAN IGNORE THE BELOW CLEANUP MESSAGES
===================================================================================
   Intel(R) MPI Library troubleshooting guide:
      https://software.intel.com/node/561764
===================================================================================
runtime: 2