Install and configure Telegraf to parse Juju Charm Store API logs and write metrics to Influxdb

telegraf.sh

#!/bin/bash

# Install and configure Telegraf to parse Juju Charm Store API logs
# and write metrics to Influxdb.

set -ex

# Install Telegraf
wget -qO- https://repos.influxdata.com/influxdb.key | sudo apt-key add -
source /etc/lsb-release
echo "deb https://repos.influxdata.com/${DISTRIB_ID,,} ${DISTRIB_CODENAME} stable" | sudo tee /etc/apt/sources.list.d/influxdb.list

sudo apt-get update -y
sudo apt-get install -y telegraf || true

sudo systemctl stop telegraf


# Configure Telegraf to:
# 1. Watch for new log files in /home/ubuntu/logs/**
# 2. Parse the logs using the rules defined here
# 3. Write the metric data to Influxdb
#
# More info about the Telegraf plugins used below can be found at
# https://github.com/influxdata/telegraf/tree/master/plugins

cat >/etc/telegraf/telegraf.conf << 'EOF'
[agent]
  interval = "10ms"
  logfile = "/var/log/telegraf/telegraf.log"
  quiet = false

[[inputs.logparser]]
  # The logparser plugin runs first and tokenizes the log line into fields,
  # using the provided grok pattern(s). All subsequent plugins operate on fields.

  files = ["/home/ubuntu/logs/**"]
  from_beginning = true

  # Exclude some fields we don't care about
  tagexclude = ["path", "host"]

  [inputs.logparser.grok]
    measurement = "charmstore_api_request"
    patterns = [
      "%{COMBINED_LOG_FORMAT} [^\"].*\"%{KVPAIRS:kv}\""
    ]
    custom_patterns = '''
      KVPAIRS [^"].*
      SUCCESS_CODE 2..
      CHARMSTORE /(charmstore|v5)\S+
      COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{CHARMSTORE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{SUCCESS_CODE:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)
      COMBINED_LOG_FORMAT %{COMMON_LOG_FORMAT} "%{DATA:referrer}" "%{DATA:agent}"
    '''

[[processors.strings]]
  # Do some basic string replacements

  order = 1  # run this plugin first
  [[processors.strings.replace]]
    field = "kv"
    old = ","
    new = ""
  [[processors.strings.replace]]
    field = "request"
    old = "%2F"
    new = "/"
  [[processors.strings.replace]]
    field = "request"
    old = "%3A"
    new = ":"

[[processors.parser]]
  # Split each kv pair in the "kv" field out into its own field, where
  # the key becomes the field name, and the value becomes the field value

  order = 2
  parse_fields = ["kv"]
  merge = "override"
  data_format = "logfmt"

[[processors.regex]]
  # Lastly, do regex processing on the fields. These run in the order in
  # which they are defined.

  order = 3
  namepass = ["charmstore_api_request"]

  [[processors.regex.fields]]
    # Apply pattern to the "request" field. If it matches, create a new
    # "namespace" field with the value from the first match subgroup

    key = "request"
    pattern = '^[^~]*~([^/]*)/.*$'
    replacement = "${1}"
    result_key = "namespace"

  [[processors.regex.fields]]
    # If the "environment_uuid" field exists and has a value, create a new
    # "request_type" field, with value "upgrade"

    key = "environment_uuid"
    pattern = '^.*$'
    replacement = "upgrade"
    result_key = "request_type"

  [[processors.regex.fields]]
    # If pattern matches on "request" field, create or update "request_type" field
    # with value "download"

    key = "request"
    pattern = '.*archive(\?.*)?$'
    replacement = "download"
    result_key = "request_type"

  [[processors.regex.fields]]
    # If pattern matches on "request" field, create new "charm_or_bundle" field
    # with value from the first match subgroup

    key = "request"
    pattern = '.*v5/([^/\?]*).*'
    replacement = "${1}"
    result_key = "charm_or_bundle"

  [[processors.regex.fields]]
    # If pattern matches on "request" field, create or update "charm_or_bundle" field
    # with value from the third match subgroup

    key = "request"
    pattern = '.*id=(cs:)?(~[^/]*/)?([^&]*).*'
    replacement = "${3}"
    result_key = "charm_or_bundle"

  [[processors.regex.fields]]
    # If pattern matches on "request" field, create or update "charm_or_bundle" field
    # with value from the first match subgroup

    key = "request"
    pattern = '.*v5/~[^/]*/([^/]*).*'
    replacement = "${1}"
    result_key = "charm_or_bundle"

  [[processors.regex.fields]]
    # If pattern matches on "request" field, create or update "charm_or_bundle" field
    # with value from the first match subgroup

    key = "request"
    ## https://github.com/juju/charmstore/blob/v5/internal/series/series.go#L38
    pattern = '.*v5/((bundle|oneiric|precise|quantal|raring|saucy|trusty|utopic|vivid|wily|xenial|yakkety|zesty|artful|bionic|cosmic|disco|win2012hvr2|win2012hv|win2012r2|win2012|win7|win8|win81|win10|win2016|win2016hv|win2016nano|centos7|kubernetes)/([^/]*)).*'
    replacement = "${1}"
    result_key = "charm_or_bundle"

  [[processors.regex.fields]]
    # If pattern matches on "request" field, create or update "charm_or_bundle" field
    # with value from the first match subgroup

    key = "request"
    ## https://github.com/juju/charmstore/blob/v5/internal/series/series.go#L38
    pattern = '.*v5/~[^/]*/((bundle|oneiric|precise|quantal|raring|saucy|trusty|utopic|vivid|wily|xenial|yakkety|zesty|artful|bionic|cosmic|disco|win2012hvr2|win2012hv|win2012r2|win2012|win7|win8|win81|win10|win2016|win2016hv|win2016nano|centos7|kubernetes)/([^/]*)).*'
    replacement = "${1}"
    result_key = "charm_or_bundle"

  [[processors.regex.fields]]
    # Parse the charm or bundle revision number into a new field

    key = "charm_or_bundle"
    pattern = '.*-(\d*)$'
    replacement = "${1}"
    result_key = "revision"

  [[processors.regex.fields]]
    # Parse the charm or bundle series into a new field

    key = "charm_or_bundle"
    pattern = '^(.*)/.*$'
    replacement = "${1}"
    result_key = "series"

  [[processors.regex.fields]]
    # Remove series from the charm or bundle name, since we have a series field

    key = "charm_or_bundle"
    pattern = '^([^/]*)/(.*)$'
    replacement = "${2}"

  [[processors.regex.fields]]
    # Remove revision from charm or bundle name, since we have a revision field

    key = "charm_or_bundle"
    pattern = '^(.*)-(\d*)$'
    replacement = "${1}"

[[processors.converter]]
  order = 4
  # Remove the kv field, we don't need to store it in the db
  fielddrop = ["kv"]

  # Convert some fields to tags. In Influxdb, tags are indexed, fields are not
  # (see https://docs.influxdata.com/influxdb/v1.7/concepts/key_concepts/#discussion).
  [processors.converter.fields]
    tag = ["cloud", "cloud_region", "provider", "controller_version", "namespace", "request_type", "charm_or_bundle", "revision", "series"]
    # Convert the revision field to an integer
    integer = ["revision"]

  [processors.converter.tags]
    # Convert the revision tag to an integer
    integer = ["revision"]

[[outputs.influxdb]]
  ## The full HTTP or UDP endpoint URL for your InfluxDB instance.
  urls = ["http://x.x.x.x:8086"] # required
  username = "xxxxxxxxxx"
  password = "xxxxxxxxxx"
  ## The target database for metrics (telegraf will create it if not exists).
  database = "mydatabase" # required
  ## Write timeout (for the InfluxDB client), formatted as a string.
  timeout = "5s"
EOF

sudo systemctl start telegraf