A helper module that wraps the SignalxFX python module to expose easy metrics usage functions and uses Django signals to flush metrics

signalfx-pythong-django-signal.py

import atexit
import logging
import time

from celery.signals import task_postrun
from django.dispatch import receiver
from django.core.signals import request_finished
from django.conf import settings
import signalfx


LOGGER = logging.getLogger(__name__)

# *********
#
# Metrics helper module for instrumenting code and sending data to SignalFx
#
# Usage:
# 1.) import the module where you intend to use it
# from libs.common.metrics import sfx
# 2.) to instrument code
# sfx.increment('email_sent')
#
# *********


def increment(name, value=1, dimensions=None, timestamp=None):
    '''increment will increase the value of a counter by 1 (can pass a higher value)
    a counter metric can only ever go up
    use it for measuring how often something happens
    eg. tasks completed, errors occured, emails sent
    a counter can be used to derive other metrics, for example rates (eg. bids/sec)
    by applying functions on the counter value when displaying in a dashboard
    you can skip the timestamp if the increment call happens right around when
    the task you are measuring happens

    Usage examples:
    sfx.increment('user.password.reset')
    sfx.increment('sendgrid.error')
    sfx.increment('emails.sent', 22)
    sfx.increment('user.login.attempt', 1, {result: 'success'})
    '''
    if value <= 0:
        return

    counter = prepare_data_point(name, value, dimensions, timestamp)
    _counters.append(counter)


def gauge(name, value, dimensions=None, timestamp=None):
    '''gauges let you measure the current value of a metric
    for example, number of active auctions etc.
    NOTE: gauges are the metric to use for measuring time to complete an action but
    see the timing() convenience function for this use case

    Usage example:
    listed_items = db.query(<find the info>)
    sfx.gauge('items.listed', listed_items)
    '''
    gauge = prepare_data_point(name, value, dimensions, timestamp)
    _gauges.append(gauge)


def timing(name, start_time, dimensions=None, timestamp=None):
    '''a timer can be used for measuring the duration of actions and provides the convenience
    of just passing in the start time of the action and the duration will be computed
    based on the start time using the call to timing() as the end time

    Usage example:
    start_time = sfx.get_timestamp_ms()
    fancy_operation_to_be_measured()
    sfx.timing('fancy_operation', start_time)
    '''
    current_timestamp = get_timestamp_ms()
    duration = current_timestamp - start_time
    gauge = prepare_data_point(name, duration, dimensions, current_timestamp)
    _gauges.append(gauge)


def prepare_data_point(name, value, dimensions=None, timestamp=None):
    '''prepare the data point for sending, add defaults if missing'''
    return {
        'metric': name,
        'value': value,
        'dimensions': dimensions or {},
        'timestamp': timestamp or get_timestamp_ms()
    }


def event(name, category=None, dimensions=None, properties=None, timestamp=None):
    '''use for marking events eg. deployments'''
    _sfx.send_event(
        event_type=name,
        category=category,
        dimensions=dimensions or {},
        properties=properties,
        timestamp=timestamp or get_timestamp_ms()
    )


def send():
    '''send the data to sfx
    does not need to be called at each call site that is instrumented, will be called
    automatically at the end of django requests and celery jobs'''
    try:
        # send the values we've accumulated until now
        _sfx.send(
            counters=_counters,
            gauges=_gauges
        )

        # clear local lists
        del _counters[:]
        del _gauges[:]
    except Exception as e:
        # TODO: send to Sentry
        LOGGER.info('Exception when sending data to signalfx: %s' % e)


def stop():
    '''calls signalfx stop; triggered using atexit
    does not need to be called directly
    stop will flush any remaining metrics and closes the signalfx sending thread
    '''
    try:
        send()
        _sfx.stop()
    except Exception as e:
        # TODO: send to Sentry
        LOGGER.info('Exception when stopping signalfx client: %s' % e)


def get_timestamp_ms():
    '''timestamp since epoch in milliseconds'''
    return int(time.time() * 1000)


@receiver(request_finished)
def sfx_send_on_django_request_finished(**kwargs):
    '''listen to a django request finished signal and send metrics to signalfx'''
    send()


# initialize
_signalfx_token = settings.NAUMAN_SIGNALFX_TOKEN
_sfx = signalfx.SignalFx().ingest(_signalfx_token)
_sfx.add_dimensions({
    'environment': settings.ENVIRONMENT
})
_counters = []
_gauges = []

atexit.register(stop)