Example of using the monitor_gpu decorator

result.json

{
    "commits": [
        "7ea1152603baef5f0012905f321586399f816958"
    ],
    "dates": [
        "2019-07-11 09:51:36 -0700"
    ],
    "content": [
        {
            "nvml_driver_version": "418.67",
            "nvml_system_nvml_version": "10.418.67",
            "nvml_deviceCount": 1,
            "nvml_unit_count": 0,
            "devices": {
                "Tesla V100-SXM2-32GB": {
                    "nvml_device_board_id": 6656,
                    "nvml_device_multi_gpu_board": 0,
                    "nvml_device_brand": 2,
                    "nvml_device_serial": "0323518083147",
                    "nvml_device_set_cpu_affinite": null,
                    "nvml_device_minor_number": 0,
                    "nvml_device_uuid": "GPU-be7b9ac8-f75e-1960-c52d-85429b4c86b1",
                    "nvml_device_inforom_version": "G503.0203.00.04",
                    "nvml_device_inforam_checksum": 0,
                    "nvml_device_display_mode": 1,
                    "nvml_device_display_active": 0,
                    "nvml_device_persistence_mode": 1,
                    "nvml_device_supported_memory_clocks": [
                        877
                    ],
                    "nvml_device_performance_state": 0,
                    "nvml_device_management_mode": 1,
                    "nvml_device_power_managerment_mode": 1,
                    "nvml_device_power_management_limit": 300000,
                    "nvml_device_power_management_limit_constraints": [
                        150000,
                        300000
                    ],
                    "nvml_device_power_management_default_limit": 300000,
                    "nvml_device_enforced_power_limit": 300000,
                    "nvml_device_power_usage": 43955,
                    "nvml_device_memory_info": {
                        "free": 34058207232,
                        "total": 34058272768,
                        "used": 65536
                    },
                    "nvml_device_bar1_memory_info": {
                        "bar1Free": 34357047296,
                        "bar1Total": 34359738368,
                        "bar1Used": 2691072
                    },
                    "nvml_device_compute_mode": 3,
                    "nvml_device_ecc_mode": [
                        1,
                        1
                    ],
                    "nvml_device_current_ecc_mode": 1,
                    "nvml_device_pending_ecc_mode": 1,
                    "nvml_device_utilization_rates": {
                        "gpu": 0,
                        "memory": 0
                    },
                    "nvml_device_encoder_utilization": [
                        0,
                        167000
                    ],
                    "nvml_device_decoder_utilization": [
                        0,
                        167000
                    ],
                    "nvml_device_pci_replay_counter": 0,
                    "nvml_device_vbios_version": "88.00.43.00.03",
                    "nvml_device_compute_running_processes": [],
                    "nvml_device_grapics_running_processes": [],
                    "nvml_device_supported_event_types": 31,
                    "nvml_device_current_pcie_link_generation": 3,
                    "nvml_device_max_pcie_link_generation": 3,
                    "nvml_device_curr_pcie_link_width": 16,
                    "nvml_device_max_pcie_link_width": 16,
                    "nvml_device_supported_clocks_throttle_reasons": 511,
                    "nvml_device_current_clocks_throttle_reasons": 1,
                    "nvml_device_index": 0,
                    "nvml_device_accounting_mode": 0,
                    "nvml_device_accounting_pids": [],
                    "nvml_device_accounting_buffer_size": 4000
                }
            },
            "SECONDS": "3"
        },
        {
            "nvml_driver_version": "418.67",
            "nvml_system_nvml_version": "10.418.67",
            "nvml_deviceCount": 1,
            "nvml_unit_count": 0,
            "devices": {
                "Tesla V100-SXM2-32GB": {
                    "nvml_device_board_id": 6656,
                    "nvml_device_multi_gpu_board": 0,
                    "nvml_device_brand": 2,
                    "nvml_device_serial": "0323518083147",
                    "nvml_device_set_cpu_affinite": null,
                    "nvml_device_minor_number": 0,
                    "nvml_device_uuid": "GPU-be7b9ac8-f75e-1960-c52d-85429b4c86b1",
                    "nvml_device_inforom_version": "G503.0203.00.04",
                    "nvml_device_inforam_checksum": 0,
                    "nvml_device_display_mode": 1,
                    "nvml_device_display_active": 0,
                    "nvml_device_persistence_mode": 1,
                    "nvml_device_supported_memory_clocks": [
                        877
                    ],
                    "nvml_device_performance_state": 0,
                    "nvml_device_management_mode": 1,
                    "nvml_device_power_managerment_mode": 1,
                    "nvml_device_power_management_limit": 300000,
                    "nvml_device_power_management_limit_constraints": [
                        150000,
                        300000
                    ],
                    "nvml_device_power_management_default_limit": 300000,
                    "nvml_device_enforced_power_limit": 300000,
                    "nvml_device_power_usage": 43955,
                    "nvml_device_memory_info": {
                        "free": 34058207232,
                        "total": 34058272768,
                        "used": 65536
                    },
                    "nvml_device_bar1_memory_info": {
                        "bar1Free": 34357047296,
                        "bar1Total": 34359738368,
                        "bar1Used": 2691072
                    },
                    "nvml_device_compute_mode": 3,
                    "nvml_device_ecc_mode": [
                        1,
                        1
                    ],
                    "nvml_device_current_ecc_mode": 1,
                    "nvml_device_pending_ecc_mode": 1,
                    "nvml_device_utilization_rates": {
                        "gpu": 0,
                        "memory": 0
                    },
                    "nvml_device_encoder_utilization": [
                        0,
                        167000
                    ],
                    "nvml_device_decoder_utilization": [
                        0,
                        167000
                    ],
                    "nvml_device_pci_replay_counter": 0,
                    "nvml_device_vbios_version": "88.00.43.00.03",
                    "nvml_device_compute_running_processes": [],
                    "nvml_device_grapics_running_processes": [],
                    "nvml_device_supported_event_types": 31,
                    "nvml_device_current_pcie_link_generation": 3,
                    "nvml_device_max_pcie_link_generation": 3,
                    "nvml_device_curr_pcie_link_width": 16,
                    "nvml_device_max_pcie_link_width": 16,
                    "nvml_device_supported_clocks_throttle_reasons": 511,
                    "nvml_device_current_clocks_throttle_reasons": 1,
                    "nvml_device_index": 0,
                    "nvml_device_accounting_mode": 0,
                    "nvml_device_accounting_pids": [],
                    "nvml_device_accounting_buffer_size": 4000
                }
            },
            "SECONDS": "3"
        }
    ]
}

test_gpu_decorator.py

#!/usr/bin/env python

from watchme.watchers.gpu.decorators import monitor_gpu
from time import sleep

# Here we create a decorator to monitor "myfunc." Specifically, we:
# - want to use the watcher "decorator" that already exists. If we want to
#   create on the fly, we would set creat=True
# - will record metrics every 3 seconds
# - to have somewhat of an impact on system resources we make a long list
# - we test to ensure that something is returned ("Hello!")

@monitor_gpu('gpu', seconds=3)
def myfunc(iters, pause):
    long_list = []
    print("Generating a long list, pause is %s and iters is %s" % (pause, iters))
    for i in range(iters):
        long_list = long_list + (i*10)*['pancakes']
        print("i is %s, sleeping %s seconds" % (i, pause))
        sleep(pause)

    return len(long_list)

# ensure the function runs when the file is called
if __name__ == '__main__':
    print("Calling myfunc with 2 iters")
    result = myfunc(2, 2)
    print("Result list has length %s" % result)