Proxmox VM Utilization + Folding@Home VM Control

api.py

import json
import requests
import time
import warnings
from enum import Enum
from dataclasses import dataclass
from dataclasses_json import dataclass_json
from typing import List, Tuple

warnings.filterwarnings('ignore', message='Unverified HTTPS request')


class VMStatus(Enum):
    stopped = "stopped"
    running = "running"


@dataclass_json
@dataclass
class VM:
    name: str
    vmid: str
    status: VMStatus
    uptime: int


class ProxmoxAPIClient:
    _hostname = None
    _node = None
    _username = None
    _password = None

    _CSRFPreventionToken = None
    _ticket = None
    _auth_timestamp = 0

    def __init__(self, hostname: str, node: str, username: str, password:str, port: int = 8006):
        self._hostname = hostname
        self._node = node
        self._port = port
        self._username = username
        self._password = password

    def _endpoint(self) -> str:
        return f'https://{self._hostname}:{self._port}/api2/json'

    def login(self):
        payload = {'username': self._username, 'password': self._password}
        req = requests.post(self._endpoint() + f'/access/ticket', data=payload, verify=False)
        if req.status_code == 401:
            raise Exception("Invalid Proxmox credentials.")
        response = req.json()
        self._CSRFPreventionToken = response['data']['CSRFPreventionToken']
        self._ticket = response['data']['ticket']
        self._auth_timestamp = time.time()

    def _get(self, url) -> requests.Response:
        if not self._ticket or self._auth_timestamp >= 7200:
            self.login()
        headers = {'CSRFPreventionToken': self._CSRFPreventionToken}
        cookies = {'PVEAuthCookie': self._ticket}
        response = requests.get(self._endpoint() + url, headers=headers, cookies=cookies, verify=False)
        return response

    def _post(self, url, payload: dict = None) -> requests.Response:
        if not self._ticket or (time.time() - self._auth_timestamp) >= 7200:
            self.login()
        headers = {'CSRFPreventionToken': self._CSRFPreventionToken}
        cookies = {'PVEAuthCookie': self._ticket}
        response = requests.post(
            self._endpoint() + url,
            verify=False,
            cookies=cookies,
            headers=headers,
            data=json.dumps(payload) if payload is not None else None
        )
        return response

    def get_vms(self) -> List[VM]:
        response = self._get(f'/nodes/{self._node}/qemu/').json()
        return [VM.from_dict(x) for x in response['data']]

    # An Average of the last N maximum CPU values per interval within a timeframe
    def get_avg_cpu(self, vm: VM, timeframe: str = 'hour', minutes: int = None) -> float:
        if vm.status != VMStatus.running:
            return 0
        response = self._get(f'/nodes/{self._node}/qemu/{vm.vmid}/rrddata?timeframe={timeframe}&cf=MAX').json()
        try:
            cpu_values: List[float] = [x['cpu'] for x in response['data'] if 'cpu' in x]
            if minutes:
                minutes = min(minutes, len(cpu_values))
                cpu_values = cpu_values[-minutes:]
            return sum(cpu_values) / len(cpu_values)
        except Exception as e:
            print(e)

    def get_node_memory(self) -> Tuple[float, float]:
        response = self._get(f'/nodes/{self._node}/rrddata?timeframe=hour&cf=MAX').json()
        results = response['data']
        last_15_minutes = [x for x in results if (time.time() - x['time']) < 900 and 'memtotal' in x]
        mem_used_values = [x['memused'] for x in last_15_minutes]
        average_mem_used = sum(mem_used_values) / len(mem_used_values)
        total_memory = last_15_minutes[-1]['memtotal']
        return average_mem_used, total_memory

    def start_vm(self, vm_id):
        response = self._post(f'/nodes/{self._node}/qemu/{vm_id}/status/start')
        return response.status_code == 200

    def stop_vm(self, vm_id) -> bool:
        response = self._post(f'/nodes/{self._node}/qemu/{vm_id}/status/shutdown')
        return response.status_code == 200

folding_cpu_limiter.py

#!/usr/bin/python3
#
# SETUP:
# apt-get update && apt-get install python3-pip -y
# pip3 install folding-at-home
# pip3 install proxmox-python
# pip3 install dataclasses_json
# mkdir proxmox/ && touch proxmox/__init__.py
# copy api.py file to proxmox/api.py
# chmox +x folding_cpu_limiter.py
# setup cron (eg. "*/5 * * * * /root/folding_cpu_limiter.py")

from folding_at_home.api import FAHClientAPI
from proxmox.api import ProxmoxAPIClient
from typing import Dict

# Proxmox
host: str = '192.168.1.27'
node: str = 'stormy'
username: str = 'root@pam'  # or '<user>@pve'
password: str = '<password>'

# Throttle options
sample_avg_minutes = 5  # average cpu load for last 5 minutes
# pause folding if using any of these
high_priority_vms = ['1', '2']  # must be string
high_priority_cpu_threshold = 10  # percent
# LIGHT if using any of these
low_priority_vms = ['3', '4', '5']  # must be string
low_priority_cpu_threshold = 20  # percent

folding = FAHClientAPI(host='192.168.1.176')
prox = ProxmoxAPIClient(hostname=host, node=node, username=username, password=password)
vms = prox.get_vms()
avg_cpu: Dict[str, float] = {}
for vm in vms:
    cpu_percentage = round(prox.get_avg_cpu(vm, minutes=sample_avg_minutes) * 100, 2)
    avg_cpu[vm.vmid] = cpu_percentage
    print(f'{vm.name} - {vm.status} (Avg CPU: {cpu_percentage}%)')


high_exceed = [key for key, value in avg_cpu.items() if key in high_priority_vms and value >= high_priority_cpu_threshold]
if len(high_exceed) > 0:
    folding.pause()
    print(f'Paused Folding due to VM(s): {high_exceed}')
    exit(0)
else:
    # resume, no effect if already running
    folding.resume()
    print(f'Resumed Folding!')

low_exceed = [key for key, value in avg_cpu.items() if key in low_priority_vms and value >= low_priority_cpu_threshold]
if len(low_exceed) > 0:
    folding.set_power(power_level='LIGHT')
    print('Set Folding power to LIGHT')
else:
    folding.set_power(power_level='FULL')
    print('Set Folding power to FULL')

Setup instructions in folding_cpu_limiter.py, comments at top of file.

Example Output:

# ./folding_cpu_limiter.py 
FoldingHome - VMStatus.running (Avg CPU: 47.11%)
VMExample1 - VMStatus.running (Avg CPU: 2.14%)
VMExample2 - VMStatus.running (Avg CPU: 5.26%)
Resumed Folding!
Set Folding power to FULL