Skip to content

Instantly share code, notes, and snippets.

@jordan-brough
Created June 28, 2012 22:31
  • Star 3 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save jordan-brough/a7f498f84a98af002fcc to your computer and use it in GitHub Desktop.
mvsum for splunk
# put this in $SPLUNK_HOME/etc/apps/search/local/commands.conf
[mvsum]
filename = mvsum.py
streaming = true
retainsevents = true
supports_multivalues = true
supports_getinfo = true
# save this as $SPLUNK_HOME/etc/apps/search/bin/mvsum.py
# make sure to chmod it to be executable
# Sums multi-valued fields within a single result.
# Usage:
# | mvsum response_time as total_response_time
# where "response_time" is a multi-valued numeric field.
import splunk.Intersplunk as si
import sys
import exceptions
def num(s):
try:
return int(s)
except exceptions.ValueError:
return float(s)
isgetinfo, sys.argv = si.isGetInfo(sys.argv)
keywords, options = si.getKeywordsAndOptions()
if len(keywords) != 3 or keywords[1] != "as":
si.parseError("Invalid syntax. Syntax is: mvsum input-field as output-field")
if isgetinfo:
# outputInfo automatically calls sys.exit()
si.outputInfo(True, False, True, False, None, False)
try:
input_field = keywords[0]
output_field = keywords[2]
results, dummyresults, settings = si.getOrganizedResults()
for result in results:
if input_field in result:
field_value = result[input_field]
if field_value:
if isinstance(field_value, list):
nums = [num(x) for x in field_value]
result[output_field] = str(sum(nums))
else:
result[output_field] = int(field_value)
except:
import traceback
stack = traceback.format_exc()
results = si.generateErrorResults("Error : Traceback: " + str(stack))
si.outputResults(results)
@thibodux
Copy link

Hey Jordan, wanted to let you know I took your code and updated it to support Splunk 6.x. Thanks for the inspiration. Here are the update files that have been working fine in 6.x across all platforms.

#commands.conf
[mvsum]
filename       = mvsum.py
type           = python
run_in_preview = false
enableheader   = true
streaming      = true
generating     = false
retainsevents  = true
supports_multivalues = true
supports_getinfo     = true
supports_rawargs     = false
#searchbnf.conf
[mvsum-command]
syntax      = mvsum <input-field> as <output-field>
shortdesc   = Sums up the numeric values in a multi-value field
description = Calculates the sum of numeric multi-value field <input-field> \
              and puts the result in the field <output-field>
usage       = public
example1    = | transaction Resource mvlist="Activity" ... | mvsum Activity as TotalActivity
comment1    = Calculates the total of the metric field 'Activity' for each transaction event\
              and stores the results in the output field TotalActivity
related     = mvjoin mvcombine eval
#mvsum.py

# Sums multi-valued field values and store in a single value output field.
# Usage:
#   | mvsum input-field as output-field
#   where "input-field" is a multi-valued field of numbers and "output-field" is the resulting field name.

import re
import splunk.Intersplunk
import sys
import exceptions

# Helper function used to parse values in the <input-field> field
def getNum(value):
    if value == '' or value is None or value is "NULL":
        value = None
    else:
        try:
            value = float(value)
            if value.is_integer():
                value = int(value)
        except ValueError:
            pass
    return value


# Main logic of mvsum function
def generateResult(results, settings):
    try:

        # get input parameters
        (isgetinfo, sys.argv) = splunk.Intersplunk.isGetInfo(sys.argv)
        if isgetinfo:
            # outputInfo automatically calls sys.exit()
            splunk.Intersplunk.outputInfo(True, False, True, False, None, False)

        # check input arguments for validity (mvsum <input-field> as <output-field>)
        if len(sys.argv) < 4 or sys.argv[2] != "as":
            return splunk.Intersplunk.generateErrorResults(
                "Invalid syntax.  Syntax is: mvsum input-field as output-field")

        # store input arguments
        inputstr = sys.argv[1]
        outputstr = sys.argv[3]

        # assert input-field is specified
        if not inputstr:
            return splunk.Intersplunk.generateErrorResults(
                "mvsum requires an input field to sum up")

        # assert output-field is specified
        if not outputstr:
            return splunk.Intersplunk.generateErrorResults(
                "mvsum requires an output field to store the")

        # loop through results and calculate the new output-field
        # by summing up numeric values in the multi-value input-field
        for result in results:
            if inputstr in result:
                field_value = result[inputstr]
                if field_value:
                    if isinstance(field_value, list):
                        nums = [getNum(x) for x in field_value]
                        try:
                            result[outputstr] = round(sum(nums),3)
                        except:
                            return splunk.Intersplunk.generateErrorResults(
                                "non-numeric data passed in input field")
                    else:
                        result[outputstr] = field_value

    # catch exceptions
    except Exception, e:
        import traceback
        stack = traceback.format_exc()
        results = splunk.Intersplunk.generateErrorResults(
            str(e) + ". Traceback: " + str(stack))

    return results

# Entry point of the code
results, dummyresults, settings = splunk.Intersplunk.getOrganizedResults()
results = generateResult(results, settings)
splunk.Intersplunk.outputResults(results)

@rgalloway3
Copy link

Ryan, I'd like to put your code into an app on Splunkbase. Any objections? I think many people would benefit.

@rgalloway3
Copy link

rgalloway3 commented Aug 17, 2020

Jordan/Ryan,

I updated the code to be compatible with Splunk 8 as well as SCP2.

commands.conf:

[mvsum]
filename = mvsum.py
chunked = true
python.version = python

mvsum.py:

# save this as $SPLUNK_HOME/etc/apps/search/bin/mvsum.py
# make sure to chmod it to be executable

# Sums multi-valued fields within a single result.
# Usage:
#   | mvsum total=total_response_time response_time
#   where "response_time" is a multi-valued numeric field.

from __future__ import absolute_import, division, print_function, unicode_literals
#import app
import os, sys

splunkhome = os.environ['SPLUNK_HOME']
sys.path.append(os.path.join(splunkhome, 'etc', 'apps', 'mvstats', 'lib')) # Update ???
from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators
from splunklib import six

# Helper function to parse values in the <input-field> field
def getNum(value):
    if value == '' or value is None or value is "NULL":
        value = None
    else:
        try:
            value = float(value)
            if value.is_integer():
                value = int(value)
        except ValueError:
            pass
    return value

@Configuration()
class mvsumCommand(StreamingCommand):
    """ Computes the sum of a multi-value field.

    ##Syntax

    .. code-block::
        mvsum total=<field> <mv-field>

    ##Description:

    Computes the sum of the values in a multi-value field.

    ##Example:

    ..code-block::
        index=_internal | stats values(date_hour) as hours | mvsum hours as sum

    """

    result = Option(
        doc='''
        **Syntax:** **result=***<fieldname>*
        **Description:** Name of the field that will hold the computed sum''',
        require=True, validate=validators.Fieldname())

    def stream(self, records):
        self.logger.debug('mvsumCommand: %s', self)  # logs command line

        if len(self.fieldnames) != 1:
            raise Exception('Please specify a field')
		
        for record in records:
            return_value = ''
            for fieldname in self.fieldnames:
                field_value = record[fieldname]
                if field_value:
                    if isinstance(field_value, list):
                        nums = [getNum(x) for x in field_value]
                        return_value = str(sum(nums))
                    else:
                        return_value = str(getNum(field_value))
            record[self.result] = return_value
            yield record

if __name__ == "__main__":
    dispatch(mvsumCommand, sys.argv, sys.stdin, sys.stdout, __name__)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment