influx-patterns

# Captures are a slightly modified version of logstash "grok" patterns, with
#  the format %{<capture syntax>[:<semantic name>][:<modifier>]}
# By default all named captures are converted into string fields.
# Modifiers can be used to convert captures to other types or tags.
# Timestamp modifiers can be used to convert captures to the timestamp of the
#  parsed metric.

# View logstash grok pattern docs here:
#   https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
# All default logstash patterns are supported, these can be viewed here:
#   https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns

# Available modifiers:
#   int
#   string
#   float
#   duration (gets converted to int64 nanoseconds)
#   tag
# Timestamp modifiers:
#   ts-ansic         ("Mon Jan _2 15:04:05 2006")
#   ts-unix          ("Mon Jan _2 15:04:05 MST 2006")
#   ts-ruby          ("Mon Jan 02 15:04:05 -0700 2006")
#   ts-rfc822        ("02 Jan 06 15:04 MST")
#   ts-rfc822z       ("02 Jan 06 15:04 -0700")
#   ts-rfc850        ("Monday, 02-Jan-06 15:04:05 MST")
#   ts-rfc1123       ("Mon, 02 Jan 2006 15:04:05 MST")
#   ts-rfc1123z      ("Mon, 02 Jan 2006 15:04:05 -0700")
#   ts-rfc3339       ("2006-01-02T15:04:05Z07:00")
#   ts-rfc3339nano   ("2006-01-02T15:04:05.999999999Z07:00")
#   ts-httpd         ("02/Jan/2006:15:04:05 -0700")
#   ts-"CUSTOM"

# Test log file pattern, test log looks like this:
#   [04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.432ms
# Breakdown of the DURATION pattern below:
#   NUMBER  is a builtin logstash grok pattern matching float & int numbers.
#   [nuµm]? is a regex specifying 0 or 1 of the characters within brackets.
#   s       is also regex, this match must end in "s".
DURATION %{NUMBER}[nuµm]?s
TESTLOG \[%{HTTPDATE:foo:ts-httpd}\] %{NUMBER:myfloat:float} %{NUMBER:myint:int} %{IPORHOST:clientip:tag} %{DURATION:response_time:duration}

# InfluxDB log file patterns
TIMESTAMP %{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}
CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
INFLUXDB_HTTPD_LOG \[httpd\] %{TIMESTAMP} %{CLIENT:clientip:tag} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp:ts-httpd}\] %{WORD:httpmethod:tag} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion} %{NUMBER:response:tag} (?:%{NUMBER:bytes:int}|-) (?:%{URI:referrer}|-) %{WORD} %{UUID} %{DURATION:response_time:duration}

influxdb.log

 8888888           .d888 888                   8888888b.  888888b.
   888            d88P"  888                   888  "Y88b 888  "88b
   888            888    888                   888    888 888  .88P
   888   88888b.  888888 888 888  888 888  888 888    888 8888888K.
   888   888 "88b 888    888 888  888  Y8bd8P' 888    888 888  "Y88b
   888   888  888 888    888 888  888   X88K   888    888 888    888
   888   888  888 888    888 Y88b 888 .d8""8b. 888  .d88P 888   d88P
 8888888 888  888 888    888  "Y88888 888  888 8888888P"  8888888P"

[run] 2016/06/04 12:09:38 InfluxDB starting, version unknown, branch unknown, commit unknown
[run] 2016/06/04 12:09:38 Go version go1.6.2, GOMAXPROCS set to 4
[run] 2016/06/04 12:09:38 no configuration provided, using default settings
[store] 2016/06/04 12:09:39 Using data dir: /Users/sparrc/.influxdb/data
[subscriber] 2016/06/04 12:09:39 opened service
[monitor] 2016/06/04 12:09:39 Starting monitor system
[monitor] 2016/06/04 12:09:39 'build' registered for diagnostics monitoring
[monitor] 2016/06/04 12:09:39 'runtime' registered for diagnostics monitoring
[monitor] 2016/06/04 12:09:39 'network' registered for diagnostics monitoring
[monitor] 2016/06/04 12:09:39 'system' registered for diagnostics monitoring
[cluster] 2016/06/04 12:09:39 Starting cluster service
[shard-precreation] 2016/06/04 12:09:39 Starting precreation service with check interval of 10m0s, advance period of 30m0s
[snapshot] 2016/06/04 12:09:39 Starting snapshot service
[copier] 2016/06/04 12:09:39 Starting copier service
[admin] 2016/06/04 12:09:39 Starting admin service
[admin] 2016/06/04 12:09:39 Listening on HTTP: [::]:8083
[continuous_querier] 2016/06/04 12:09:39 Starting continuous query service
[httpd] 2016/06/04 12:09:39 Starting HTTP service
[httpd] 2016/06/04 12:09:39 Authentication enabled: false
[httpd] 2016/06/04 12:09:39 Listening on HTTP: [::]:8086
[retention] 2016/06/04 12:09:39 Starting retention policy enforcement service with check interval of 30m0s
[monitor] 2016/06/04 12:09:39 Storing statistics in database '_internal' retention policy 'monitor', at interval 10s
2016/06/04 12:09:39 Sending anonymous usage statistics to m.influxdb.com
[run] 2016/06/04 12:09:39 Listening for signals
[tsm1wal] 2016/06/04 12:09:49 tsm1 WAL starting with 10485760 segment size
[tsm1wal] 2016/06/04 12:09:49 tsm1 WAL writing to /Users/sparrc/.influxdb/wal/_internal/monitor/1
[shard] 2016/06/04 12:09:49 /Users/sparrc/.influxdb/data/_internal/monitor/1 database index loaded in 4.709µs
[query] 2016/06/04 12:09:57 CREATE DATABASE IF NOT EXISTS telegraf
[query] 2016/06/04 12:09:57 WARNING: IF NOT EXISTS is deprecated as of v0.13.0 and will be removed in v1.0
[httpd] 2016/06/04 12:09:57 ::1 - - [04/Jun/2016:12:09:57 +0100] POST /query?db=&q=CREATE+DATABASE+IF+NOT+EXISTS+%22telegraf%22 HTTP/1.1 200 137 - InfluxDBClient df937fc8-2a44-11e6-8001-000000000000 4.693958ms
[tsm1wal] 2016/06/04 12:10:12 tsm1 WAL starting with 10485760 segment size
[tsm1wal] 2016/06/04 12:10:12 tsm1 WAL writing to /Users/sparrc/.influxdb/wal/telegraf/default/2
[shard] 2016/06/04 12:10:12 /Users/sparrc/.influxdb/data/telegraf/default/2 database index loaded in 1.742µshttpd] 2016/06/04 12:41:45 ::1 - - [04/Jun/2016:12:41:45 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 50da745e-2a49-11e6-809f-000000000000 7.854403ms
[httpd] 2016/06/04 12:41:57 ::1 - - [04/Jun/2016:12:41:57 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 580ada5b-2a49-11e6-80a0-000000000000 1.233592ms
[httpd] 2016/06/04 12:42:10 ::1 - - [04/Jun/2016:12:42:10 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 5f3b1d78-2a49-11e6-80a1-000000000000 2.499176ms
[httpd] 2016/06/04 12:42:22 ::1 - - [04/Jun/2016:12:42:22 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 666b6b8d-2a49-11e6-80a2-000000000000 486.038µs
[httpd] 2016/06/04 12:42:34 ::1 - - [04/Jun/2016:12:42:34 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 6d9c16f0-2a49-11e6-80a3-000000000000 4.280412ms
[httpd] 2016/06/04 12:42:46 ::1 - - [04/Jun/2016:12:42:46 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 74cc9e2e-2a49-11e6-80a4-000000000000 2.561105ms
[httpd] 2016/06/04 12:42:58 ::1 - - [04/Jun/2016:12:42:58 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 7bfc6a7b-2a49-11e6-80a5-000000000000 515.854µs
[httpd] 2016/06/04 12:43:04 ::1 - - [04/Jun/2016:12:43:04 +0100] POST /write?consistency=all&db=telegraf&precision=ns&rp=default HTTP/1.1 204 0 - InfluxDBClient 7f92e892-2a49-11e6-80a6-000000000000 1.509814ms
[run] 2016/06/04 13:01:46 Signal received, initializing clean shutdown...
[run] 2016/06/04 13:01:46 Waiting for clean shutdown...
[copier] 2016/06/04 13:01:46 copier listener closed
[snapshot] 2016/06/04 13:01:46 snapshot listener closed
[cluster] 2016/06/04 13:01:46 cluster service accept error: network connection closed
[shard-precreation] 2016/06/04 13:01:46 Precreation service terminating
[continuous_querier] 2016/06/04 13:01:46 continuous query service terminating
[retention] 2016/06/04 13:01:46 retention policy enforcement terminating
[monitor] 2016/06/04 13:01:46 shutting down monitor system
[monitor] 2016/06/04 13:01:46 terminating storage of statistics
[subscriber] 2016/06/04 13:01:47 closed service
[run] 2016/06/04 13:01:47 server shutdown completed

telegraf-logparser.conf

# Stream and parse log file(s).
[[inputs.logparser]]
  ## files to tail.
  ## These accept standard unix glob matching rules, but with the addition of
  ## ** as a "super asterisk". ie:
  ##   "/var/log/**.log"  -> recursively find all .log files in /var/log
  ##   "/var/log/*/*.log" -> find all .log files with a parent dir in /var/log
  ##   "/var/log/apache.log" -> just tail the apache log file
  ##
  ## See https://github.com/gobwas/glob for more examples
  ##
  files = ["$HOME/db/ws/test.log"]
  ## Read file from beginning.
  from_beginning = true

  ## For parsing logstash-style "grok" patterns:
  [inputs.logparser.grok]
    patterns = ["%{INFLUXDB_HTTPD_LOG}", "%{TESTLOG}"]
    custom_pattern_files = ["$HOME/db/ws/influx-patterns"]

test.log

[04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.413ms