Join prom metrics

join_prom_metrics.md

Example metrics:

# HELP node_hwmon_sensor_label Label for given chip and sensor
# TYPE node_hwmon_sensor_label gauge
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd1",sensor="temp3"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd2",sensor="temp4"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd3",sensor="temp5"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd4",sensor="temp6"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd5",sensor="temp7"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd6",sensor="temp8"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd7",sensor="temp9"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tccd8",sensor="temp10"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tctl",sensor="temp1"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:18_3",label="tdie",sensor="temp2"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd1",sensor="temp3"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd2",sensor="temp4"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd3",sensor="temp5"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd4",sensor="temp6"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd5",sensor="temp7"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd6",sensor="temp8"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd7",sensor="temp9"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tccd8",sensor="temp10"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tctl",sensor="temp1"} 1
node_hwmon_sensor_label{chip="pci0000:00_0000:00:19_3",label="tdie",sensor="temp2"} 1
# HELP node_hwmon_temp_celsius Hardware monitor for temperature (input)
# TYPE node_hwmon_temp_celsius gauge
node_hwmon_temp_celsius{chip="0000:60:03_1_0000:63:00_0",sensor="temp1"} 37
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp1"} 31.25
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp10"} 29.5
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp2"} 31.25
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp3"} 34.75
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp4"} 30.75
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp5"} 31
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp6"} 31.25
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp7"} 31
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp8"} 31
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:18_3",sensor="temp9"} 29.25
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp1"} 31.25
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp10"} 31.75
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp2"} 31.25
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp3"} 28.5
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp4"} 31.5
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp5"} 31
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp6"} 28.5
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp7"} 29.75
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp8"} 30.5
node_hwmon_temp_celsius{chip="pci0000:00_0000:00:19_3",sensor="temp9"} 30.5

example join query:

avg(node_hwmon_temp_celsius * on (sensor,chip,instance) group_left(label) node_hwmon_sensor_label) by (hostname,label)