Skip to content

Instantly share code, notes, and snippets.

@ram-nadella
Last active November 4, 2016 18:08
Show Gist options
  • Save ram-nadella/eb4b5d0d2ee2e17dd4a1956441ea4a7f to your computer and use it in GitHub Desktop.
Save ram-nadella/eb4b5d0d2ee2e17dd4a1956441ea4a7f to your computer and use it in GitHub Desktop.
A helper module that wraps the SignalxFX python module to expose easy metrics usage functions and uses Django signals to flush metrics
import atexit
import logging
import time
from celery.signals import task_postrun
from django.dispatch import receiver
from django.core.signals import request_finished
from django.conf import settings
import signalfx
LOGGER = logging.getLogger(__name__)
# *********
#
# Metrics helper module for instrumenting code and sending data to SignalFx
#
# Usage:
# 1.) import the module where you intend to use it
# from libs.common.metrics import sfx
# 2.) to instrument code
# sfx.increment('email_sent')
#
# *********
def increment(name, value=1, dimensions=None, timestamp=None):
'''increment will increase the value of a counter by 1 (can pass a higher value)
a counter metric can only ever go up
use it for measuring how often something happens
eg. tasks completed, errors occured, emails sent
a counter can be used to derive other metrics, for example rates (eg. bids/sec)
by applying functions on the counter value when displaying in a dashboard
you can skip the timestamp if the increment call happens right around when
the task you are measuring happens
Usage examples:
sfx.increment('user.password.reset')
sfx.increment('sendgrid.error')
sfx.increment('emails.sent', 22)
sfx.increment('user.login.attempt', 1, {result: 'success'})
'''
if value <= 0:
return
counter = prepare_data_point(name, value, dimensions, timestamp)
_counters.append(counter)
def gauge(name, value, dimensions=None, timestamp=None):
'''gauges let you measure the current value of a metric
for example, number of active auctions etc.
NOTE: gauges are the metric to use for measuring time to complete an action but
see the timing() convenience function for this use case
Usage example:
listed_items = db.query(<find the info>)
sfx.gauge('items.listed', listed_items)
'''
gauge = prepare_data_point(name, value, dimensions, timestamp)
_gauges.append(gauge)
def timing(name, start_time, dimensions=None, timestamp=None):
'''a timer can be used for measuring the duration of actions and provides the convenience
of just passing in the start time of the action and the duration will be computed
based on the start time using the call to timing() as the end time
Usage example:
start_time = sfx.get_timestamp_ms()
fancy_operation_to_be_measured()
sfx.timing('fancy_operation', start_time)
'''
current_timestamp = get_timestamp_ms()
duration = current_timestamp - start_time
gauge = prepare_data_point(name, duration, dimensions, current_timestamp)
_gauges.append(gauge)
def prepare_data_point(name, value, dimensions=None, timestamp=None):
'''prepare the data point for sending, add defaults if missing'''
return {
'metric': name,
'value': value,
'dimensions': dimensions or {},
'timestamp': timestamp or get_timestamp_ms()
}
def event(name, category=None, dimensions=None, properties=None, timestamp=None):
'''use for marking events eg. deployments'''
_sfx.send_event(
event_type=name,
category=category,
dimensions=dimensions or {},
properties=properties,
timestamp=timestamp or get_timestamp_ms()
)
def send():
'''send the data to sfx
does not need to be called at each call site that is instrumented, will be called
automatically at the end of django requests and celery jobs'''
try:
# send the values we've accumulated until now
_sfx.send(
counters=_counters,
gauges=_gauges
)
# clear local lists
del _counters[:]
del _gauges[:]
except Exception as e:
# TODO: send to Sentry
LOGGER.info('Exception when sending data to signalfx: %s' % e)
def stop():
'''calls signalfx stop; triggered using atexit
does not need to be called directly
stop will flush any remaining metrics and closes the signalfx sending thread
'''
try:
send()
_sfx.stop()
except Exception as e:
# TODO: send to Sentry
LOGGER.info('Exception when stopping signalfx client: %s' % e)
def get_timestamp_ms():
'''timestamp since epoch in milliseconds'''
return int(time.time() * 1000)
@receiver(request_finished)
def sfx_send_on_django_request_finished(**kwargs):
'''listen to a django request finished signal and send metrics to signalfx'''
send()
# initialize
_signalfx_token = settings.NAUMAN_SIGNALFX_TOKEN
_sfx = signalfx.SignalFx().ingest(_signalfx_token)
_sfx.add_dimensions({
'environment': settings.ENVIRONMENT
})
_counters = []
_gauges = []
atexit.register(stop)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment