Databricks clusters VMs used memory percentage KQL

databricks-clusters-vms-used-memory-percentage.kql

let VMs = Heartbeat
| where OSType == 'Linux'
  and Category != 'Azure Monitor Agent'
  and ResourceId != ''
| project Computer, TimeGenerated, ClusterId = substring(Computer, 0, 20), VM = tostring(split(ResourceId, "/")[-1])
| summarize arg_max(TimeGenerated, *) by Computer;
let Clusters = DatabricksClusters
| where ActionName in ('create', 'startResult')
| extend ResponseTxt = parse_json(Response)
| where ResponseTxt.statusCode == 200
| extend ResponseResultTxt = parse_json(tostring(ResponseTxt.result))
| extend RequestParamsTxt = parse_json(RequestParams)
| project ClusterId = tostring(case(ActionName == 'create', ResponseResultTxt.cluster_id, RequestParamsTxt.clusterId)),
ClusterName = tostring(case(ActionName == 'create', RequestParamsTxt.cluster_name, RequestParamsTxt.clusterName))
| distinct ClusterId, ClusterName
| order by ClusterId asc;
Perf
| where ObjectName == 'Memory'
  and CounterName == '% Used Memory'
| join kind=inner (VMs)
  on Computer
| join kind=leftouter (Clusters)
    on ClusterId
| project ClusterVM = strcat(ClusterName, " (", VM, ")"), TimeGenerated, UsedMemoryPercentage = CounterValue
| sort by ClusterVM asc, TimeGenerated asc
| render timechart with (ymax = 100)

monitoring.sh

#!/bin/bash

if [[ -n "$LOG_ANALYTICS_WORKSPACE_ID" && -n "$LOG_ANALYTICS_WORKSPACE_KEY" ]]; then
    wget https://raw.githubusercontent.com/Microsoft/OMS-Agent-for-Linux/master/installer/scripts/onboard_agent.sh \
        && sh onboard_agent.sh -w $LOG_ANALYTICS_WORKSPACE_ID -s $LOG_ANALYTICS_WORKSPACE_KEY \
        && echo '<source>
    type oms_omi
    object_name "Physical Disk"
    instance_regex ".*"
    counter_name_regex ".*"
    interval 5m
</source>

<source>
    type oms_omi
    object_name "Logical Disk"
    instance_regex ".*"
    counter_name_regex ".*"
    interval 5m
</source>

<source>
    type oms_omi
    object_name "Processor"
    instance_regex ".*"
    counter_name_regex ".*"
    interval 30s
</source>

<source>
    type oms_omi
    object_name "Memory"
    instance_regex ".*"
    counter_name_regex ".*"
    interval 30s
</source>' > /etc/opt/microsoft/omsagent/$LOG_ANALYTICS_WORKSPACE_ID/conf/omsagent.d/oms_omi.conf \
        && chown omsagent:omiusers /etc/opt/microsoft/omsagent/$LOG_ANALYTICS_WORKSPACE_ID/conf/omsagent.d/oms_omi.conf \
        && ls -la /etc/opt/microsoft/omsagent/$LOG_ANALYTICS_WORKSPACE_ID/conf/omsagent.d \
        && cat /etc/opt/microsoft/omsagent/$LOG_ANALYTICS_WORKSPACE_ID/conf/omsagent.d/oms_omi.conf
fi