peuse_cpu_crash.py

#!/usr/bin/env python3
"""
How to use:

Run this script out of /tmp in a seperate cmd (CMD-A) 
leave this running CRTL+C if you want to kill it of `ps aux` find the PID the `kill -9 [PID]`

Step1: chmod +x peuse_cpu_crash.py
Step2: python3 peuse_cpu_crash.py

Then to view the logs after the crash in a seperate cmd (CMD-B) run the following:

To see the logs interactivly run in other terminal (youre in vi so : q to exit) : 
OptionA: less -N --follow-name +F ~/cpu.log
OptionB: watch -n 1 'tail -n 10 ~/cpu.log'
OptionC: less -N +F ~/cpu_log.log

To print the whole file after crash: 
OptionA: cat ~/cpu.log
"""

import logging
import psutil
import subprocess
import time

from logging.handlers import RotatingFileHandler
from pathlib import Path


logger = logging.getLogger('logger')
logger.setLevel(logging.DEBUG)
log_file_size_bytes = 1024*1024*25  # 25 MB
handler = RotatingFileHandler(str(Path.home()) + '/tmp', maxBytes=log_file_size_bytes, backupCount=10)
format = "%(asctime)s, %(levelname)s, %(message)s"  
formatter = logging.Formatter(fmt=format, datefmt='%Y-%m-%d__%H:%M:%S')
handler.setFormatter(formatter)
logger.addHandler(handler)

t_measurement_sec = 4

while True:
    cpu_percent_cores = psutil.cpu_percent(interval=t_measurement_sec, percpu=True)
    avg = sum(cpu_percent_cores)/len(cpu_percent_cores)
    cpu_percent_overall = avg
    cpu_percent_overall_str = ('%5.2f' % cpu_percent_overall) + '%'
    cpu_percent_cores_str = [('%5.2f' % x) + '%' for x in cpu_percent_cores]
    cpu_percent_cores_str = ', '.join(cpu_percent_cores_str)
    logger.info('       ==> Overall: {} <==,        Individual CPUs: {} '.format(
        cpu_percent_overall_str,
        cpu_percent_cores_str))

"""
https://unix.stackexchange.com/a/295608/114401 
You can change this number if you have too many processes runing past 30 and its too messy
"""
    cmd = ['ps', '-eo', '%cpu,args']#, '|', 'awk', "'$1 >= 30 {print}'"]
    p = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    out, err = p.communicate()
    # print(out.decode("utf-8"))
    lines = out.decode("utf-8").splitlines()
    # separate out the %cpu usage right at the front from the rest of the ps output
    splitlines = [line.split(maxsplit=1) for line in lines]
    # print(splitlines)

    # Convert to a list of [float cpu_pct, str cmd]
    cpu_processes_list = []
    for line in splitlines[1:]: 
        individual_cpu_usage_pct = float(line[0])
        cmd = line[1]
        cpu_processes_list.append([individual_cpu_usage_pct, cmd])
    cpu_processes_list.sort(reverse=True)  # sort highest-cpu-usage first

"""
Obtain a list of the top 10 processes, and another list of the processes > X% cpu usage
You can change this however you want too if you dont like the output
"""

    cpu_processes_top10_list = cpu_processes_list[:10]
    cpu_processes_above_threshold_list = []
    for process in cpu_processes_list:
        CPU_THRESHOLD_PCT = 15
        individual_cpu_usage_pct = process[0]
        if individual_cpu_usage_pct >= CPU_THRESHOLD_PCT:
            cpu_processes_above_threshold_list.append(process)

    OVERALL_CPU_THRESHOLD_PCT = 50
    cpu_processes_list = cpu_processes_above_threshold_list
    if cpu_percent_overall > OVERALL_CPU_THRESHOLD_PCT:
        if len(cpu_processes_top10_list) > len(cpu_processes_above_threshold_list):
            cpu_processes_list = cpu_processes_top10_list

    # Log the high-cpu-usage processes
    handler.setFormatter(None) # remove formatter for these log msgs only
    num = 0
    for process in cpu_processes_list:
        num += 1
        num_str = "%2i" % num
        individual_cpu_usage_pct = process[0]
        individual_cpu_usage_pct_str = ('%5.2f' % individual_cpu_usage_pct) + "%"
        cmd_str = process[1]

        logger.info('    {}/{}) {}, cmd: {}'.format(num_str, len(cpu_processes_list), individual_cpu_usage_pct_str, cmd_str))
    handler.setFormatter(formatter)  # restore format for next logs