Azure insight gpu metrics

push_gpu_metrics.py

#!/usr/bin/env python
from applicationinsights import TelemetryClient
import subprocess
import sys
import socket

hostname = socket.gethostname()
telemetry_id = sys.argv[1]

tc = TelemetryClient(telemetry_id)
popen = subprocess.Popen('nvidia-smi dmon -d 10', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

keys = None
while True:
    line = popen.stdout.readline().strip()
    print(line)

    if 'Idx' in line:
        continue

    if 'gpu' in line:
        keys = line[1:].split()
        continue
    else:
        vals = map(int, line.split())

    d = dict(zip(keys, vals))
    for k, v in d.items():
        if k != 'gpu':
            aggregate_metric_name = 'aggregate_gpu_%s' % (k)
            standalone_metric_name = '%s_gpu_%s' % (hostname, k)

            print(aggregate_metric_name, v)
            tc.track_metric(aggregate_metric_name, v)
            tc.track_metric(standalone_metric_name, v)
    tc.flush()
    print('sent')