tikv_mem_trace.stp

probe begin {
    warn("Begin to trace jemalloc.\n")
}

probe process.function("malloc").return,
      process.function("mallocx").return{ 
    if (pid() != target()) next

    printf("malloc: %p, bytes: %d\n", $return, @entry($size))
    
    print_ubacktrace()
}

probe process.function("calloc").return {
    if (pid() != target()) next

    printf("malloc: %p, bytes: %d\n", $return, @entry($size) * @entry($num))
    
    print_ubacktrace()  
}

probe process.function("realloc").return,
      process.function("rallocx").return {
    if (pid() != target()) next

    printf("free: %p\n", @entry($ptr))
    printf("malloc: %p, bytes: %d\n", $return, @entry($size))
    
    print_ubacktrace()  
}

probe process.function("free"),
      process.function("sdallocx") {
    if (pid() != target()) next

    if ($ptr == 0) next
   
    printf("free: %p\n", $ptr)
}

probe timer.s(10) {
    exit()
}

/*
probe process.function("xallocx").return{
    if (pid() != target()) next

    // TODO: do we need to print free $ptr here?
    printf("malloc: %p, bytes: %d\n", $return, @entry($size) + @entry($extra))
    
    print_ubacktrace()  
}

probe process.function("dallocx") {
    if (pid() != target()) next

    if ($ptr == 0) next
   
    printf("free: %p\n", $ptr)
}
*/

sudo stap tikv_mem_trace.stp -x tikv_pid > mem.log

sudo stap -DSTP_NO_OVERLOAD tikv_mem_trace.stp -x tikv_pid > mem.log

After dump, we can use https://gist.github.com/siddontang/6ca422aad494bf2968787e9a9d559fa8#file-memtrace_parser-py to parse the log and see which operation uses the most memory and doesn't release.

#!/usr/bin/python

import sys

def parse():
	if len(sys.argv) != 2:
		print 'must pass memory trace file'
		sys.exit(2)

	file_name = sys.argv[1]
	# [addr, size, stack]
	mem_info = [0, 0, ""]
	mem_stats = {}
	
	with open(file_name) as f:
		for line in f:
			line = line.strip()
			if line.startswith("malloc"):
				if mem_info[0] > 0:
					addr = mem_info[0]
					mem_stats[addr] = mem_info
				
				line2 = line.split(",")
				v1 = line2[0].split(":")
				v2 = line2[1].split(":")
				addr = int(v1[1].strip(), 0)
				size = int(v2[1].strip(), 0)
				mem_info = [addr, size, ""]
			elif line.startswith("free"):
				line2 = line.split(":")
				addr = int(line2[1].strip(), 0)
				if addr in mem_stats:
					del mem_stats[addr]
			else:
				mem_info[2] += line + "\n"

	stack_stats = {}
	for m in mem_stats.values():
		stack = m[2]
		if stack not in stack_stats:
			# [size, count, stack]
			stack_stats[stack] = [m[1], 1, stack]
		else:
			info = stack_stats[stack]
			info[0] += m[1]
			info[1] += 1

	for s in sorted(stack_stats.values(), reverse=True):
		print "size: ", s[0], " count: ", s[1], "\n", s[2]

if __name__ == "__main__":
   parse()

https://gist.github.com/siddontang/8d5e4c8029e5b1df481546063842be91