agas seg fault

gistfile1.txt

{stack-trace}: 4 frames:
0x2ae2d315921f  : hpx::detail::backtrace(unsigned long) + 0x7f in /home/dmarce1/lib/libhpx.so.0
0x2ae2d315a79d  : boost::exception_ptr hpx::detail::get_exception<hpx::exception>(hpx::exception const&, std::string const&, std::string const&, long, std::string const&) + 0x23d in /home/dmarce1/lib/libhpx.so.0
0x2ae2d315a36c  : void hpx::detail::throw_exception<hpx::exception>(hpx::exception const&, std::string const&, std::string const&, long) + 0x10c in /home/dmarce1/lib/libhpx.so.0
0x2ae2d392bd36  : hpx::agas::server::primary_namespace::resolve_free_list(boost::unique_lock<hpx::lcos::local::spinlock>&, std::list<std::_Rb_tree_iterator<std::pair<hpx::naming::gid_type const, long> >, std::allocator<std::_Rb_tree_iterator<std::pair<hpx::naming::gid_type const, long> > > > const&, std::list<hpx::agas::server::primary_namespace::free_entry, std::allocator<hpx::agas::server::primary_namespace::free_entry> >&, hpx::naming::gid_type const&, hpx::naming::gid_type const&, hpx::error_code&) + 0x1316 in /home/dmarce1/lib/libhpx.so.0
{env}: 104 entries:
  ACLOCAL_PATH=/usr/local/packages/cmake/2.8.12/INTEL-14.0.2/share/aclocal
  BASH_FUNC_module()=() {  eval `/usr/local/packages/Modules/$MODULE_VERSION/bin/modulecmd bash $*`
}
@                                                                                                                                                                                                                    
                                                                                                                                                                                                   1067,1        65%
  LIBRARY_PATH=/home/dmarce1/lib:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/compiler/lib/intel64:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/ipp/../compiler/lib/intel64:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/ipp/lib/intel64:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/compiler/lib/intel64:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/mkl/lib/intel64:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/tbb/lib/intel64/gcc4.4:/usr/local/packages/mvapich2/2.0/INTEL-14.0.2/lib:/usr/local/packages/hwloc/1.10.0/INTEL-14.0.2-mic//lib:/usr/local/compilers/gcc/4.9.0/lib64:/usr/local/compilers/gcc/4.9.0/lib:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/compiler/lib/intel64:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/mkl/lib/intel64
  LM_LICENSE_FILE=/usr/local/compilers/pgi/license.dat
  LOADEDMODULES=gcc/4.9.0:intel/14.0.2:cmake/2.8.12/INTEL-14.0.2:hwloc/1.10.0/INTEL-14.0.2-mic:mvapich2/2.0/INTEL-14.0.2
  LOGNAME=dmarce1
  MAIL=/var/spool/mail/dmarce1
  MANPATH=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/man/en_US:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/man/en_US:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/debugger/gdb/intel64/share/man/:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/debugger/gdb/intel64_mic/share/man/:/usr/local/packages/mvapich2/2.0/INTEL-14.0.2/share/man:/usr/local/packages/hwloc/1.10.0/INTEL-14.0.2-mic//share/man:/usr/local/packages/cmake/2.8.12/INTEL-14.0.2/man:/usr/local/compilers/gcc/4.9.0/share/man:/usr/local/compilers/pgi/linux86-64/man:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/man/en_US:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/man/en_US:
  MIC_LD_LIBRARY_PATH=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/compiler/lib/mic:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/mpirt/lib/mic:/opt/intel/mic/coi/device-linux-release/lib:/opt/intel/mic/myo/lib:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/ipp/lib/lib/mic:/opt/intel/mic/coi/device-linux-release/lib:/opt/intel/mic/myo/lib:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/compiler/lib/mic:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/mkl/lib/mic:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/tbb/lib/mic:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/compiler/lib/mic:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/mkl/lib/mic
  MIC_LIBRARY_PATH=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/compiler/lib/mic:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/mpirt/lib/mic:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/tbb/lib/mic
  MKLROOT=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/mkl
  MOABHOMEDIR=/usr/moab
  MODULEPATH=/usr/local/packages/Modules/modulefiles/apps:/usr/local/packages/Modules/modulefiles/admin:/usr/local/packages/Modules/modulefiles/xsede
  MODULESHOME=/usr/local/packages/Modules/3.2.10
  MODULE_VERSION=3.2.10
  MODULE_VERSION_STACK=3.2.10
  MPIR_CVAR_CH3_INTERFACE_HOSTNAME=smic288
  MPM_LAUNCHER=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/debugger/mpm/bin/start_mpm.sh
  MV2_COMM_WORLD_LOCAL_RANK=0
  MV2_COMM_WORLD_LOCAL_SIZE=1
  MV2_COMM_WORLD_RANK=59
  MV2_COMM_WORLD_SIZE=128
  MV2_IBA_HCA=mlx4_0
  NLSPATH=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/compiler/lib/intel64/locale/%l_%t/%N:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/ipp/lib/intel64/locale/%l_%t/%N:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/mkl/lib/intel64/locale/%l_%t/%N:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/debugger/gdb/intel64_mic/share/locale/%l_%t/%N:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/debugger/gdb/intel64/share/locale/%l_%t/%N:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/compiler/lib/intel64/locale/en_US:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/mkl/lib/intel64/locale/en_US
  OLDPWD=/home/dmarce1
  PATH=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/bin/intel64:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/debugger/gdb/intel64_mic/bin:/usr/lib64/qt-3.3/bin:/usr/local/packages/mvapich2/2.0/INTEL-14.0.2/bin:/usr/local/packages/hwloc/1.10.0/INTEL-14.0.2-mic//bin:/usr/local/packages/cmake/2.8.12/INTEL-14.0.2/bin:/usr/local/compilers/gcc/4.9.0/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/local/compilers/pgi/linux86-64/6.1/bin:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/bin/intel64:/opt/ibutils/bin:/home/dmarce1/bin:/home/dmarce1/bin
  PBS_ENVIRONMENT=PBS_BATCH
  PBS_GPUFILE=/var/spool/torque/aux//48882.smic3gpu
  PBS_JOBCOOKIE=A36A676B679F0FA57227E8601174B793
  PBS_JOBID=48882.smic3
  PBS_JOBNAME=job
  PBS_MOMPORT=15003
  PBS_NODENUM=0
  PBS_NP=2560
  PBS_NUM_NODES=128
  PBS_NUM_PPN=20
  PBS_O_HOME=/home/dmarce1
                                                                                                                                                                                                   1148,3        70%
  PBS_NUM_PPN=20
  PBS_O_HOME=/home/dmarce1
  PBS_O_HOST=smic1.hpc.lsu.edu
  PBS_O_LANG=en_US.UTF-8
  PBS_O_LOGNAME=dmarce1
  PBS_O_MAIL=/var/spool/mail/dmarce1
  PBS_O_PATH=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/bin/intel64:/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/debugger/gdb/intel64_mic/bin:/usr/local/packages/hwloc/1.10.0/INTEL-14.0.2-mic//bin:/usr/local/packages/cmake/2.8.12/INTEL-14.0.2/bin:/usr/local/compilers/gcc/4.9.0/bin:/usr/lib64/qt-3.3/bin:/usr/local/packages/mvapich2/2.0/INTEL-14.0.2/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/local/compilers/pgi/linux86-64/6.1/bin:/usr/local/compilers/Intel/cluster_studio_xe_2013.1.046/composer_xe_2013_sp1.2.144/bin/intel64:/opt/ibutils/bin:/opt/dell/srvadmin/bin:/home/dmarce1/bin:/home/dmarce1/bin:/usr/local/compilers/pgi/linux86-64/6.1/bin:/home/dmarce1/bin
  PBS_O_QUEUE=checkpt
  PBS_O_SHELL=/bin/bash
  PBS_O_WORKDIR=/work/dmarce1/FVFMM/Release
  PBS_QUEUE=checkpt
  PBS_SERVER=smic3
  PBS_TASKNUM=1
  PBS_VERSION=TORQUE-3.0.6
  PBS_VNODENUM=0
  PBS_WALLTIME=21600
  PGI=/usr/local/compilers/pgi
  PKG_CONFIG_PATH=/usr/local/packages/mvapich2/2.0/INTEL-14.0.2/lib/pkgconfig:/usr/local/packages/hwloc/1.10.0/INTEL-14.0.2-mic//lib/pkgconfig:/home/dmarce1/lib/pkgconfig
  PMI_FD=5
  PMI_RANK=59
  PMI_SIZE=128
  PWD=/work/dmarce1/FVFMM/Release
  QTDIR=/usr/lib64/qt-3.3
  QTINC=/usr/lib64/qt-3.3/include
  QTLIB=/usr/lib64/qt-3.3/lib
  SHELL=/bin/bash
  SHLVL=2
  SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass
  SSH_CLIENT=204.90.41.8 46428 22
  SSH_CONNECTION=204.90.41.8 46428 204.90.42.34 22
  TBBROOT=/usr/local/compilers/Intel/parallel_studio-2015u3/composer_xe_2015.3.187/tbb
  USER=dmarce1
  _=/usr/local/packages/mvapich2/2.0/INTEL-14.0.2/bin/mpirun
  _LMFILES_=/usr/local/packages/Modules/modulefiles/apps/gcc/4.9.0:/usr/local/packages/Modules/modulefiles/apps/intel/14.0.2:/usr/local/packages/Modules/modulefiles/apps/cmake/2.8.12/INTEL-14.0.2:/usr/local/packages/Modules/modulefiles/apps/hwloc/1.10.0/INTEL-14.0.2-mic:/usr/local/packages/Modules/modulefiles/apps/mvapich2/2.0/INTEL-14.0.2
  _MODULESBEGINENV_=/home/dmarce1/.modulesbeginenv
{locality-id}: 59
{hostname}: [ (mpi:59) (tcp:127.0.0.1:7969) ]
{process-id}: 18395
{function}: primary_namespace::resolve_free_list
{file}: /work/dmarce1/builds/hpx/src/runtime/agas/server/primary_namespace_server.cpp
{line}: 1012
{os-thread}: 12, worker-thread#12
{thread-id}: 00002ae2dea3f740
{thread-description}: <unknown>
{state}: state_running
{auxinfo}:
{config}:
  HPX_HAVE_NATIVE_TLS=ON
  HPX_HAVE_STACKTRACES=ON
                                                                                                                                                                                                   1181,3        72%
  HPX_HAVE_NATIVE_TLS=ON
  HPX_HAVE_STACKTRACES=ON
  HPX_HAVE_COMPRESSION_BZIP2=OFF
  HPX_HAVE_COMPRESSION_SNAPPY=OFF
  HPX_HAVE_COMPRESSION_ZLIB=OFF
  HPX_HAVE_PARCEL_COALESCING=ON
  HPX_HAVE_PARCELPORT_TCP=ON
  HPX_HAVE_PARCELPORT_MPI=ON (MPICH V3.1, MPI V3.0)
  HPX_HAVE_PARCELPORT_IPC=OFF
  HPX_HAVE_PARCELPORT_IBVERBS=OFF
  HPX_HAVE_VERIFY_LOCKS=OFF
  HPX_HAVE_HWLOC=ON
  HPX_HAVE_ITTNOTIFY=OFF
  HPX_HAVE_RUN_MAIN_EVERYWHERE=OFF
  HPX_LIMIT=5
  HPX_PARCEL_MAX_CONNECTIONS=512
  HPX_PARCEL_MAX_CONNECTIONS_PER_LOCALITY=4
  HPX_INITIAL_AGAS_LOCAL_CACHE_SIZE=256
  HPX_AGAS_LOCAL_CACHE_SIZE_PER_THREAD=32
  HPX_HAVE_MALLOC=tbbmalloc
  HPX_PREFIX (configured)=/work/dmarce1/builds/hpx_build
  HPX_PREFIX=/home/dmarce1
{version}: V0.9.11-trunk (AGAS: V3.0), Git:
{boost}: V1.59.0
{build-type}: release
{date}: Sep 15 2015 14:55:30
{platform}: linux
{compiler}: Intel C++ C++0x mode version 1500
{stdlib}: GNU libstdc++ version 20140422
{what}: primary_namespace::resolve_free_list, failed to resolve gid, gid({0000003c00000001, 0000000002dc13f6}): HPX(internal_server_error)