One can easily make JVMTI agent that will trace MethodEntry / MethodExit events and correspondingly increase or decrease stack depth counter. Here is an example of such agent. When the program ends, it will print the maximum recorded Java stack depth.
#include <jvmti.h>
#include <stdint.h>
#include <stdio.h>
static volatile int max_depth = 0;
static int adjust_stack_depth(jvmtiEnv *jvmti, int delta) {
intptr_t depth = 0;
(*jvmti)->GetThreadLocalStorage(jvmti, NULL, (void**)&depth);
(*jvmti)->SetThreadLocalStorage(jvmti, NULL, (const void*)(depth + delta));
return (int)depth;
}
void JNICALL MethodEntry(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread, jmethodID method) {
adjust_stack_depth(jvmti, +1);
}
void JNICALL MethodExit(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread, jmethodID method,
jboolean was_popped_by_exception, jvalue return_value) {
int depth = adjust_stack_depth(jvmti, -1);
if (depth > max_depth) {
max_depth = depth; // TODO: replace with atomic CAS to avoid race condition
}
}
JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *vm, char *options, void *reserved) {
jvmtiEnv* jvmti;
(*vm)->GetEnv(vm, (void**)&jvmti, JVMTI_VERSION_1_0);
jvmtiCapabilities capabilities = {0};
capabilities.can_generate_method_entry_events = 1;
capabilities.can_generate_method_exit_events = 1;
(*jvmti)->AddCapabilities(jvmti, &capabilities);
jvmtiEventCallbacks callbacks = {0};
callbacks.MethodEntry = MethodEntry;
callbacks.MethodExit = MethodExit;
(*jvmti)->SetEventCallbacks(jvmti, &callbacks, sizeof(callbacks));
(*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_METHOD_ENTRY, NULL);
(*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_METHOD_EXIT, NULL);
return 0;
}
JNIEXPORT void JNICALL Agent_OnUnload(JavaVM *vm) {
printf("Max stack depth = %d\n", max_depth);
}
Compile:
gcc -fPIC -shared -I $JAVA_HOME/include -I $JAVA_HOME/include/linux -o libmaxdepth.so maxdepth.c
Run:
java -agentpath:/path/to/libmaxdepth.so MyProgram
However, tracing each method entry and exit is very expensive. A less accurate, but much more efficient alternative would be a sampling profiler which periodically records a stack trace of a running thread, e.g. async-profiler or Java Flight Recorder.