Example code of HeapStats realtime deadlock detector

README.md

This Gist is for discussion about Bug 3293: REFACTORING Realtime deadlock detector implementation. Discussion issue for this on GitHub is HeapStats #76. We appreciate your comment!

I think we can have two approach to refactor HeapStats realtime deadlock detector:

1. Track all relevant threads through GetObjectMonitorUsage() JVMTI function
   • deadlockDetector.cpp
   • Very simple recursive implementation
   • Only call JVMTI functions
   • Many VMOps might be called if many relevant threads exist at GetObjectMonitorUsage()
2. Call jmm_FindMonitorDeadlockedThreads() to find deadlock
   • deadlockDetector-jmm.cpp
   • Use JMM (HotSpot Monitoring and Management Interface) function to find deadlock
     • http://openjdk.java.net/groups/serviceability/svcjdk.html
   • Execute jmm_FindMonitorDeadlockedThreads() if multiple relevant threads exist ( > 3 )
     • This function will call at safepoint
   • jmm_FindMonitorDeadlockedThreads() is not exported
     • We have to find function symbol through TVMFunctions
     • jmm.h is included in OpenJDK, however it is not provided in distribution package

You can build/execute these samples:

$ export JAVA_HOME=/path/to/jdk
$ make
$ java -agentpath:./libdltest.so <Deadlock App>

JMM version ( libdltest-jmm.so ) does not work because JMM function is not exported from libjvm.so .

deadlockDetector-jmm.cpp

#include <jvmti.h>
#include <jni.h>

#include <stdio.h>


// Does not work.
// This function is not exported from libjvm.so .
extern "C" jobjectArray jmm_FindMonitorDeadlockedThreads(JNIEnv *env);


void JNICALL OnMonitorContendedEnterForDeadlock(
                 jvmtiEnv *jvmti, JNIEnv *env, jthread thread, jobject monitor){
  // Find owner thread
  jvmtiMonitorUsage monitor_usage;
  jvmti->GetObjectMonitorUsage(monitor, &monitor_usage);
  jvmti->Deallocate((unsigned char *)monitor_usage.waiters);
  jvmti->Deallocate((unsigned char *)monitor_usage.notify_waiters);

  if(monitor_usage.owner == NULL){ // Monitor has been released
    return;
  }

  // Check dead lock (compare with oop)
  if(*(void **)monitor_usage.owner == *(void **)thread){ // unlikely condition
    jvmtiThreadInfo thread_info;
    jvmti->GetThreadInfo(thread, &thread_info);
    printf("deadlock detected!: %s\n", thread_info.name);
    jvmti->Deallocate((unsigned char *)thread_info.name);
    return;
  }

  // Check thread state of owner thread
  jint thread_state;
  jvmti->GetThreadState(monitor_usage.owner, &thread_state);
  if((thread_state & JVMTI_THREAD_STATE_BLOCKED_ON_MONITOR_ENTER) == 0){
    // No dead lock because owner thread does not wait at the monitor.
    return;
  }

  // Find dead lock at VM operation
  jobjectArray dl_threads = jmm_FindMonitorDeadlockedThreads(env);
  if(env->GetArrayLength(dl_threads) > 0){
    printf("deadlock detected!\n");
  }

}

JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *vm, char *options, void *reserved){
  jvmtiEnv *jvmti;
  vm->GetEnv((void **)&jvmti, JVMTI_VERSION_1);

  jvmtiCapabilities capabilities = {0};
  capabilities.can_generate_monitor_events = 1;
  capabilities.can_get_monitor_info = 1;
  jvmti->AddCapabilities(&capabilities);

  jvmtiEventCallbacks callbacks = {0};
  callbacks.MonitorContendedEnter = &OnMonitorContendedEnterForDeadlock;
  jvmti->SetEventCallbacks(&callbacks, sizeof(jvmtiEventCallbacks));

  jvmti->SetEventNotificationMode(JVMTI_ENABLE,
                                  JVMTI_EVENT_MONITOR_CONTENDED_ENTER, NULL);
  printf("Agent started.\n");
  return JNI_OK;
}

JNIEXPORT void JNICALL Agent_OnUnload(JavaVM *vm){
  // Do nothing
}

deadlockDetector.cpp

#include <jvmti.h>
#include <jni.h>

#include <stdio.h>


void JNICALL OnMonitorContendedEnterForDeadlock(
                 jvmtiEnv *jvmti, JNIEnv *env, jthread thread, jobject monitor){
  // Set contended monitor oop to jthread as an object tag
  jvmti->SetTag(thread, (jlong)*(void **)monitor);

  // Track all relevant threads
  jobject mon = monitor;
  while(true){
    // Get owner thread of this monitor
    //   CAUTION: GetObjectMonitorUsage() might be executed at safepoint!
    void *owner_oop;
    jvmtiMonitorUsage monitor_usage;
    jvmti->GetObjectMonitorUsage(mon, &monitor_usage);
    jvmti->Deallocate((unsigned char *)monitor_usage.waiters);
    jvmti->Deallocate((unsigned char *)monitor_usage.notify_waiters);

    if(monitor_usage.owner == NULL){ // Monitor has been released
      return;
    }

    owner_oop = *(void **)monitor_usage.owner;

    // Check dead lock (compare with oop)
    if(owner_oop == *(void **)thread){ // unlikely condition
      jvmtiThreadInfo thread_info;
      jvmti->GetThreadInfo(thread, &thread_info);
      printf("deadlock detected!: %s\n", thread_info.name);
      jvmti->Deallocate((unsigned char *)thread_info.name);
      return;
    }

    jthread owner_thread = (jthread)&owner_oop;

    // Check thread state of owner thread
    jint thread_state;
    jvmti->GetThreadState(owner_thread, &thread_state);
    if((thread_state & JVMTI_THREAD_STATE_BLOCKED_ON_MONITOR_ENTER) == 0){
      // No dead lock because owner thread does not wait at the monitor.
      return;
    }

    // Get contended monitor
    void *contended_monitor_oop;
    jvmti->GetTag(owner_thread, (jlong *)&contended_monitor_oop);
    if(contended_monitor_oop == NULL){ // unlikely condition
      // Monitor of owner might be released at this point
      // (caused by OnMonitorContendedEnteredForDeadlock())
      return;
    }
    mon = (jobject)&contended_monitor_oop;
  }

}

void JNICALL OnMonitorContendedEnteredForDeadlock(
                 jvmtiEnv *jvmti, JNIEnv *env, jthread thread, jobject monitor){
  // Remove contended monitor oop from thread
  jvmti->SetTag(thread, 0L);
}

JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *vm, char *options, void *reserved){
  jvmtiEnv *jvmti;
  vm->GetEnv((void **)&jvmti, JVMTI_VERSION_1);

  jvmtiCapabilities capabilities = {0};
  capabilities.can_generate_monitor_events = 1;
  capabilities.can_tag_objects = 1;
  capabilities.can_get_monitor_info = 1;
  jvmti->AddCapabilities(&capabilities);

  jvmtiEventCallbacks callbacks = {0};
  callbacks.MonitorContendedEnter = &OnMonitorContendedEnterForDeadlock;
  callbacks.MonitorContendedEntered = &OnMonitorContendedEnteredForDeadlock;
  jvmti->SetEventCallbacks(&callbacks, sizeof(jvmtiEventCallbacks));

  jvmti->SetEventNotificationMode(JVMTI_ENABLE,
                                  JVMTI_EVENT_MONITOR_CONTENDED_ENTER, NULL);
  jvmti->SetEventNotificationMode(JVMTI_ENABLE,
                                  JVMTI_EVENT_MONITOR_CONTENDED_ENTERED, NULL);

  printf("Agent started.\n");
  return JNI_OK;
}

JNIEXPORT void JNICALL Agent_OnUnload(JavaVM *vm){
  // Do nothing
}

Makefile

.PHONY: clean
CFLAGS = -fPIC -g
LDFLAGS = -lpthread
CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux
TARGETS = libdltest.so libdltest-jmm.so


all: $(TARGETS)

libdltest.so: deadlockDetector.o
	$(CXX) -shared -o $@ $^ $(LDFLAGS)

libdltest-jmm.so: deadlockDetector-jmm.o
	$(CXX) -shared -o $@ $^ $(LDFLAGS)

.cpp.o:
	$(CXX) $(CFLAGS) $(CPPFLAGS) -c $<

clean:
	$(RM) $(TARGETS) *.o