Skip to content

Instantly share code, notes, and snippets.

@MauricioRoman
Created November 26, 2014 18:37
Show Gist options
  • Save MauricioRoman/954135b02e0d0869e2d4 to your computer and use it in GitHub Desktop.
Save MauricioRoman/954135b02e0d0869e2d4 to your computer and use it in GitHub Desktop.
#!/usr/bin/python
import requests, urllib2, simplejson, datetime, time
# Constants
MAX_RETRIES = 15 #Max number of retries when requesting data
TIMEOUT_RETURN = 100 #Timeout in calling API to retrieve results
DELAY = 0.5 #Delay per retry in seconds (increases * no_retries)
MAX_FIELDS = 300 #Max fields to retrieve from field request via API
# Configuration parameters
# Need to change these to your case, as well as remove decorator from logQueryData() once tested
T = 1000 # ms
pivot_field = 'json.response_time'
my_token = '3dc53bxe-4d3d-4a76-w515-c277dic4uc6c'
username = 'johndoe'
password = '123456'
subdomain = 'acme'
time_interval = '15m'
# Decorator function to print parameter names sent to function
# From: http://stackoverflow.com/questions/6200270/
# Used to test before sending data to Loggly
def dump_args(func):
"This decorator dumps out the arguments passed to a function before calling it"
argnames = func.func_code.co_varnames[:func.func_code.co_argcount]
fname = func.func_name
def echo_func(*args,**kwargs):
print fname, "(", ', '.join(
'%s=%r' % entry
for entry in zip(argnames,args[:len(argnames)])+
[("args",list(args[len(argnames):]))]+[("kwargs",kwargs)]) +")"
return echo_func
def getApdex(id, query, pivot_field, percentages, searchFrom, searchTo, timestamp,
subdomain, deployment, username, password, output_key):
""" Calculates Apdex score and stores result back into Loggly
id: a label for the log, for instance, indicating how often the data is fetched
query: dict with queries to generate numerator, denominator, etc. for alculation
percentages: dict with definitions of how metrics are calculated
-- all other terms are self explanatory --
"""
results = {}
#Iterate over queries and get results
for key in query.iterkeys():
search_url = "http://%s.%s/apiv2/fields/%s?q=%s&from=%s&until=%s&facet_size=%d" % (
subdomain, deployment,pivot_field, query[key],searchFrom,searchTo, MAX_FIELDS)
print search_url
status, results[key] = getAPI(search_url, username, password)
print results
# Get last term in pivot field to use as key in log event
pivot = pivot_field.split('.')[-1] + '_perf'
Rs = getApdexPercent(results, percentages, 'Rs')
Rt = getApdexPercent(results, percentages, 'Rt') - Rs
apdex = Rs + Rt / 2.0
metrics_dict = dict( zip( [ 'Rs','Rt','apdex'],
[ round(Rs,1), round(Rt,1), round(apdex,1)] ))
#Send data to your data store
# (Put your own code here)
#Send values back to Loggly
logQueryData(output_key,dict([('timestamp',str(timestamp.isoformat()) ), ('action','analytics'),
('bin',id), (pivot,metrics_dict) ] ))
return 0
def getApdexPercent(results, percentages, key):
percentage = 0.
numerator = results[percentages[key]['numerator']]
denominator = results[percentages[key]['denominator']]
if denominator > 0:
percentage = round( float(numerator) / float(denominator) * 100., 2)
return percentage
# Remove this decorator after testing in order to send data to your Loggly account
@dump_args
def logQueryData(loggly_key, data):
""" Logs a JSON object to Loggly """
log_data = "PLAINTEXT=" + urllib2.quote(simplejson.dumps(data))
# Send log data to Loggly
urllib2.urlopen("https://logs-01.loggly.com/inputs/" + loggly_key + "/tag/queryAPI/", log_data)
def getAPI(search_url, username, password):
""" Fetches data from Loggly via the API """
retries = 0
while retries < MAX_RETRIES:
try:
# We launch the search
r = requests.get(search_url, auth=(username, password), timeout=TIMEOUT_RETURN)
res = r.json()
try:
if res['total_events'] >= 0:
return r.status_code, res['total_events']
else:
return r.status_code, 0
except ValueError as error:
print '%s Error: %s - response: %s\n' % ( datetime.datetime.utcnow(), error, r )
break
except ValueError:
print '%s Error status code: %d\n' % ( datetime.datetime.utcnow(), r.status_code )
retries = retries + 1
time.sleep(DELAY * (retries+1) )
return 0
########## Main Code #############
# The query terms have the T and Y values
query_terms = {
'A': '%s:*' % pivot_field,
'B': '%s:<%d' % (pivot_field, T),
'C': '%s:<%d' % (pivot_field, 4*T)
}
# This dictionary structure can be used for any type of relational metric based on counts
percentages = {'Rs':{
'numerator':'B',
'denominator':'A'
},
'Rt':{
'numerator':'C',
'denominator':'A'
}
}
now = datetime.datetime.utcnow()
getApdex(time_interval, query_terms, pivot_field, percentages, '-'+time_interval, 'now', now,
subdomain, 'loggly.com', username, password, my_token)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment