README.md

Pod watcher

K8s has metrics server to provide cpu and memory usage of pods. Default 60 seconds, can be changed using --metric-resolution flag. Official are not recommending setting values below 15s, as this is the resolution of metrics calculated by Kubelet.(ref1, ref2)

So I create this tool to watch cpu and memory and log peak usage of pods.

How to install

// instal node 17
bash <(curl -sL https://gist.githubusercontent.com/grieve54706/acd9c1a411adab38c4f61f2b1497769f/raw/d42e76f3f8a3aaadf3055094df8c7dab3c688e2f/upgrade-node.sh)

// reload env var
source ~/.bashrc

// run tool
npx https://gist.github.com/grieve54706/acd9c1a411adab38c4f61f2b1497769f

// remove npx cache if need to use new version
cd ~/.npm/_npx/ && rm -rf *

How to use

TBD

calc_cpu.sh

tstart=$(date +%s%N)
cstart=$(cat /sys/fs/cgroup/cpu/cpuacct.usage)

sleep 1

tstop=$(date +%s%N)
cstop=$(cat /sys/fs/cgroup/cpu/cpuacct.usage)

cpu_usage=$(awk -v cstart="$cstart" -v cstop="$cstop" -v tstart="$tstart" -v tstop="$tstop" 'BEGIN { printf "%.0f", (cstop - cstart) / (tstop - tstart) * 1000 }')

echo $cpu_usage

main.js

#!/usr/bin/env node

import figlet from 'figlet';
import gradient from 'gradient-string';
import { checkbox, confirm } from '@inquirer/prompts';
import shell from 'shelljs';
import chalk from 'chalk';
import fs from 'fs';

async function main() {

    console.log(gradient.retro(figlet.textSync('Pod Watcher')));

    console.log('Welcome to use pod watcher, you can choose which pod you want to watch peak cpu and memory usage of pods.')

    const selectedPods = await checkbox({
        message: 'Select which kind pod you want to watch',
        choices: [
            { name: 'gateway', value: 'gateway' },
            { name: 'sql-engine-coordinator', value: 'sql-engine-coordinator' },
            { name: 'sql-engine-worker', value: 'sql-engine-worker' },
            { name: 'vulcan-sql', value: 'vulcan-sql' },
            { name: 'web', value: 'web' },
            { name: 'tableau-publish-worker', value: 'tableau-publish-worker' },
        ],
    });

    const podNames = [];
    const { stdout } = shell.exec('kubectl get pods -o jsonpath={..metadata.name}', { silent: true })
    for (const podName of stdout.split(' ')) {
        for (const pod of selectedPods) {
            if (podName.startsWith(pod) && !podName.startsWith('vulcan-sql-inject-pat')) {
                podNames.push(podName);
            }
        }
    }
    podNames.sort();

    console.log(podNames);
    const podPeakCPU = new Map();
    const podPeakMemory = new Map();
    let cpuJob;
    let memJob;

    if (await confirm({ message: 'Do you want to ' + chalk.green('START') + ' to watch these memory usages of pods?', default: true })) {
        if (!fs.existsSync('calc_cpu.sh')) {
            console.log('Download cpu script...');
            downloadCPUScript();
        }
        console.log('Copy script into pods...');
        cpScriptIntoPods(podNames);
        cpuJob = startCollectCPU(podNames, podPeakCPU);
        memJob = startCollectMemory(podNames, podPeakMemory);
    }
    else {
        return;
    }

    console.log('Now you can start your process to watch the peak cpu and memory usage of pods, If your process is done, you can stop watcher to look at the results.');

    if (await confirm({ message: 'Do you want to ' + chalk.red('STOP') + ' watcher to look at the results?', default: true })) {
        console.log('Peak of usage of every pod');
        const maxPodNameLength = findMaxPodNameLength(podNames);
        const blank = '   ';
        console.log(`${'NAME'.padEnd(maxPodNameLength)} ${blank} CPU(cores) ${blank} MEMORY(bytes)`);
        podNames.forEach(pod => {
            const cpu_usage = formatCPU(podPeakCPU.get(pod));
            const memory_usage = bytesToMB(podPeakMemory.get(pod));
            console.log(`${pod.padEnd(maxPodNameLength)} ${blank} ${cpu_usage.padEnd('CPU(cores)'.length)} ${blank} ${memory_usage}`);
        });
    }

    clearTimeout(cpuJob);
    clearTimeout(memJob);
}

function downloadCPUScript() {
    shell.exec(`wget https://gist.githubusercontent.com/grieve54706/acd9c1a411adab38c4f61f2b1497769f/raw/da4ecdb005748815a6d9d17facb03b8a1b35b465/calc_cpu.sh`, { silent: true })
}

function cpScriptIntoPods(podNames) {
    for (const podName of podNames) {
        shell.exec(`kubectl cp calc_cpu.sh ${podName}:/`)
    }
}

function startCollectCPU(podNames, podPeakCPU) {
    return setInterval(() => {
        Promise.all(podNames.map(podName => {
            return new Promise((resolve, reject) => {
                collectCPU(podName, podPeakCPU);
                resolve();
            });
        }));
    }, 1000);
}

function collectCPU(podName, podPeakCPU) {
    const { stdout } = shell.exec(`kubectl exec ${podName} -- sh /calc_cpu.sh`, { silent: true })
    const cpu_usage = Number(stdout.trim());
    if (cpu_usage > (podPeakCPU.get(podName) || 0)) {
        podPeakCPU.set(podName, cpu_usage);
    }
}

function startCollectMemory(podNames, podPeakMemory) {
    return setInterval(() => {
        Promise.all(podNames.map(podName => {
            return new Promise((resolve, reject) => {
                collectMemory(podName, podPeakMemory);
                resolve();
            });
        }));
    }, 1000);
}

function collectMemory(podName, podPeakMemory) {
    const { stdout } = shell.exec(`kubectl exec ${podName} -- cat /sys/fs/cgroup/memory/memory.usage_in_bytes`, { silent: true })
    const memory_usage = Number(stdout.trim());
    if (memory_usage > (podPeakMemory.get(podName) || 0)) {
        podPeakMemory.set(podName, memory_usage);
    }
}

function findMaxPodNameLength(podNames) {
    return podNames.reduce((a, b) => {
        return a.length > b.length ? a : b;
    }).length;
}

function formatCPU(cpu_usage) {
    return `${parseFloat(cpu_usage).toFixed(0)}m`
}

function bytesToMB(bytes) {
    return `${parseFloat(bytes / (1024 ** 2)).toFixed(0)}Mi`
}

await main();

package.json

{
  "name": "pod-memory-watcher",
  "version": "1.0.0",
  "description": "",
  "main": "main.js",
  "bin": "./main.js",
  "type": "module",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "Grieve",
  "license": "ISC",
  "dependencies": {
    "@inquirer/prompts": "^3.0.0",
    "chalk": "^5.3.0",
    "figlet": "^1.6.0",
    "gradient-string": "^2.0.2",
    "shelljs": "^0.8.5"
  }
}

upgrade-node.sh

#!/bin/bash

sudo apt-get remove nodejs
sudo apt-get remove npm
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash
chmod +x ~/.nvm/nvm.sh
source ~/.bashrc 
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm install 17
node -v
npm -v