A tool to show how the thread runs on different CPUs

cpu_tenancy.py

#!/usr/bin/python

import sys
import re

# perf record -F 99 -p $1 -e "sched:sched_stat_runtime" -a -- sleep 20
# perf script -F comm,pid,tid,cpu,time | python cpu_tenancy.py
# 
# time-monitor-wo 79618/79634 [007] 11994704.695317:
#   grpc-server-1 79618/79954 [006] 11994704.717969:
#    apply worker 79618/79969 [010] 11994704.718048:

class CpuTime:
    def __init__(self, cpu, time):
        self.cpu = cpu 
        self.last_time = time
        self.total_time = 0.0

    def merge(self, time):
        elapsed = time - self.last_time
        self.total_time += elapsed
        self.last_time = time
        return elapsed

    def __str__(self):
        return "CPU-%d/%dus" % (self.cpu, self.total_time * 1000000)

class TidTime: 
    def __init__(self, tid, name, cpu, time):
        self.name = name
        self.tid = tid 
        self.total_time = 0.0
        self.cpu_times = [CpuTime(cpu, time)]

    def add(self, cpu, time):
        if self.cpu_times[-1].cpu != cpu:
            self.cpu_times.append(CpuTime(cpu, time))
        else:
            elapsed = self.cpu_times[-1].merge(time)
            self.total_time += elapsed

    def __str__(self):
        return "| %s | %s" % (" | ".join([str(t) for t in self.cpu_times]), self.name)


def get_cpu_tenancy_time_by_tid():
    tid_map = dict()
    
    for line in sys.stdin:
        try:
            tokens = re.split("\s+", line.strip())
            process_id = re.search("([0-9]+)/([0-9]+)", tokens[-3].strip())
            if process_id is not None:
                _pid = int(process_id.group(1))
                tid = int(process_id.group(2))

                cpu_id = int(re.search("\[([0-9]+)\]", tokens[-2]).group(1))
                time = float(tokens[-1].strip().strip(':'))

                if tid not in tid_map:
                    name = " ".join(tokens[0:len(tokens) - 3])
                    tid_map[tid] = TidTime(tid, name, cpu_id, time)
                else:
                    tid_map[tid].add(cpu_id, time)                
        except AttributeError:
            print("failed to parse line: " + line)

    return tid_map

if __name__ == "__main__":
    tid_map = get_cpu_tenancy_time_by_tid()
    m = sorted(tid_map.items(), key = lambda kv: kv[1].total_time)

    for v in m:
        print(str(v[1]))

cpu_tenancy.sh

perf record -F 99 -p $1 -e "sched:sched_stat_runtime" -a -- sleep 60
perf script -F comm,pid,tid,cpu,time > tenancy.out
cat tenancy.out | python cpu_tenancy.py

example_output

| CPU-11/10043us | cop-high31
| CPU-16/1999us | CPU-1/153994us | CPU-34/155994us | raftstore-4
| CPU-7/500438us | time-monitor-wo
| CPU-18/533004us | tikv-server
| CPU-34/1126us | CPU-18/178us | CPU-32/5us | CPU-6/161us | CPU-8/962us | CPU-18/0us | CPU-32/407us | CPU-14/521871us | CPU-32/0us | CPU-6/0us | CPU-30/886us | CPU-1/157484us | CPU-10/419us | grpc-server-3
| CPU-6/94us | CPU-8/1265us | CPU-30/1888us | CPU-34/520us | CPU-6/0us | CPU-14/44us | CPU-32/749us | CPU-16/268us | CPU-38/396us | CPU-2/530us | CPU-14/0us | CPU-18/833540us | CPU-38/411us | grpc-server-1
| CPU-10/693us | CPU-12/1813819us | apply worker