Example of using the Cloudera Manager API to poll for YARN health checks and to list long running jobs using a tsquery

get-yarn-long-running-jobs.py

#!/usr/bin/python

## ********************************************************************************
## get-yarn-long-running-jobs.py 
## 
## Usage: ./get-yarn-long-running-jobs.py
## 
## Edit the settings below to connect to your Cluster
##
## ********************************************************************************

import sys
import time
import datetime
from cm_api.api_client import ApiResource

## Settings to connect to the cluster 

cm_host = "toronto"

cm_port = "7180"

cm_login = "admin"

cm_password = "admin"

cluster_name = "onefoursix" 



## define a  tsquery
query = "SELECT application_duration from YARN_APPLICATIONS where application_duration > 12000"


## Connect to CM
api = ApiResource(server_host=cm_host, server_port=cm_port, username=cm_login, password=cm_password, version=11)

## Get Cluster
cluster = None
clusters = api.get_all_clusters()
for c in clusters:
  if c.displayName == cluster_name:
    cluster = c
    break
if cluster is None:
  print "\nError: Cluster '" + cluster_name + "' not found"
  quit(1)


## Get YARN Service
yarn = None
service_list = cluster.get_all_services()
for service in service_list:
  if service.type == "YARN":
    yarn = service
    break
if yarn is None:
  print "Error: Could not locate YARN Service"
  quit(1)

## Get YARN Health Checks
for chk in yarn.healthChecks:
  if chk['summary'] in ['CONCERNING','BAD'] :
    print (chk)
    if chk['name'] == 'alarm:Slow-YARN-Job':
      
      ## define a 30 minute window
      from_time = datetime.datetime.fromtimestamp(time.time() - 1800)
      to_time = datetime.datetime.fromtimestamp(time.time())
      
      ## get the metrics associated with the tsquery
      result = api.query_timeseries(query, from_time, to_time)
      ts_list = result[0]
      
      ## print the metrics
      for ts in ts_list.timeSeries:
        print "--- %s: %s ---" % (ts.metadata.entityName, ts.metadata.metricName)
        for point in ts.data:
          print "%s:\t%s" % (point.timestamp.isoformat(), point.value)