openshift on azure will not expose the infra nodes by default, therefore we can't access the prometheus ui, so we can use the following script to check the prometheus targets health

promethues_on_osa_get_targets.sh

#!/bin/sh
oc exec -ti prometheus-k8s-0 -c prometheus -- curl -s http://localhost:9090/targets |grep -i '<a id="job'

######## example output #####
#        <a id="job-alertmanager-main" href="#job-alertmanager-main">alertmanager-main (3/3 up)</a>
#        <a id="job-apiserver" href="#job-apiserver">apiserver (3/3 up)</a>
#        <a id="job-cluster-monitoring-operator" href="#job-cluster-monitoring-operator">cluster-monitoring-operator (1/1 up)</a>
#        <a id="job-kube-state-metrics" href="#job-kube-state-metrics">kube-state-metrics (2/2 up)</a>
#        <a id="job-kubelet" href="#job-kubelet">kubelet (210/210 up)</a>
#        <a id="job-node-exporter" href="#job-node-exporter">node-exporter (105/105 up)</a>
#        <a id="job-prometheus-k8s" href="#job-prometheus-k8s">prometheus-k8s (2/2 up)</a>
#        <a id="job-prometheus-operator" href="#job-prometheus-operator">prometheus-operator (1/1 up)</a>

# check for any target failures.
oc exec -ti prometheus-k8s-0 -c prometheus -- curl -s http://localhost:9090/targets |grep -i '<a id="job' |grep -v