lyonzy/README.md

## README.md

      
    Raw
  

              README.md
            
          
    Quick-and-dirty IoT Data Capture for Raspberry Pi

This docker-compose file and conf files configure a small cluster of tools to capture and analyse time-series data on a Raspberry Pi. This is (minimally) tested on the original Pi.
The cluster consists of:

InfluxDb, a time-series database
Grafana, a visualisation and dashboarding package for time-series data
Chronograf, a simpler querying tool for InfluxDb
Telegraf, a tool to route data, in this case used to take messages from an MQTT broker and put them in InfluxDb
Mosquitto, an MQTT broker

Notes:

Grafana is available at port 3000
Chronograf is available at port 8888
You can send messages to Mosquitto on 1883, the standard MQTT port
Use avahi-daemon on your Pi to get a cool mDNS address like raspberrypi.local
User data from the applications is kept in ./data, so mount this somewhere or change the docker-compose.yml file to persist this data
MQTT messages must be in InfluxDb line protocol format
There is no security on anything, so this is really only suitable for home networks. Don't run this on the internet without additional security measures.


## docker-compose.yml
version: '2'

services:
  # influxdb = time series database
  influxdb:
    image: hypriot/rpi-influxdb
    container_name: influxdb
    environment:
      - INFLUXDB_DATA_ENGINE=tsm1
      - INFLUXDB_REPORTING_DISABLED=false
    volumes:
      - ./data/influxdb:/var/lib/influxdb

  # grafana = data visualisation and dashboarding
  grafana:
    image: tcjennings/rpi-grafana
    container_name: grafana
    ports:
      - "3000:3000"
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    volumes:
      - ./data/grafana:/var/lib/grafana
      - ./grafana-datasources.yml/:/etc/grafana/provisioning/datasources/all.yml

  # chronograf = simple influxdb querying
  chronograf:
    image: mbrande3/rpi-chronograf
    container_name: chronograf
    environment:
      - INFLUXDB_URL=http://influxdb:8086
    ports:
      - "8888:8888"
    volumes:
       - ./data/chronograf:/var/lib/chronograf

  # telegraf = forwarding from MQTT to InfluxDB
  telegraf:
    image: swestcott/rpi-telegraf
    container_name: telegraf
    volumes:
       - ./telegraf.conf:/etc/telegraf/telegraf.conf


  # mosquitto = eclipse MQTT broker
  mosquitto:
    image:  mjenz/rpi-mosquitto
    container_name: mosquitto
    ports:
      - "1883:1883"

## grafana-datasources.yml
apiVersion: 1

deleteDatasources:
  - name: influxdb
    orgId: 1

datasources:
  - name: Influxdb
    type: influxdb
    access: proxy
    url: http://influxdb:8086
    database: measurements
    basicAuth: false
    withCredentials: false
    isDefault: true
    editable: false

## telegraf.conf
# Telegraf Configuration
#
# Telegraf is entirely plugin driven. All metrics are gathered from the
# declared inputs, and sent to the declared outputs.
#
# Plugins must be declared in here to be active.
# To deactivate a plugin, comment out the name and any variables.
#
# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
# file would generate.
#
# Environment variables can be used anywhere in this config file, simply prepend
# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)


# Global tags can be specified here in key="value" format.
[global_tags]
  # dc = "us-east-1" # will tag all metrics with dc=us-east-1
  # rack = "1a"
  ## Environment variables can be used as tags, and throughout the config file
  # user = "$USER"


# Configuration for telegraf agent
[agent]
  ## Default data collection interval for all inputs
  interval = "10s"
  ## Rounds collection interval to 'interval'
  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
  round_interval = true

  ## Telegraf will send metrics to outputs in batches of at most
  ## metric_batch_size metrics.
  ## This controls the size of writes that Telegraf sends to output plugins.
  metric_batch_size = 1000

  ## For failed writes, telegraf will cache metric_buffer_limit metrics for each
  ## output, and will flush this buffer on a successful write. Oldest metrics
  ## are dropped first when this buffer fills.
  ## This buffer only fills when writes fail to output plugin(s).
  metric_buffer_limit = 10000

  ## Collection jitter is used to jitter the collection by a random amount.
  ## Each plugin will sleep for a random time within jitter before collecting.
  ## This can be used to avoid many plugins querying things like sysfs at the
  ## same time, which can have a measurable effect on the system.
  collection_jitter = "0s"

  ## Default flushing interval for all outputs. You shouldn't set this below
  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
  flush_interval = "10s"
  ## Jitter the flush interval by a random amount. This is primarily to avoid
  ## large write spikes for users running a large number of telegraf instances.
  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
  flush_jitter = "0s"

  ## By default or when set to "0s", precision will be set to the same
  ## timestamp order as the collection interval, with the maximum being 1s.
  ##   ie, when interval = "10s", precision will be "1s"
  ##       when interval = "250ms", precision will be "1ms"
  ## Precision will NOT be used for service inputs. It is up to each individual
  ## service input to set the timestamp at the appropriate precision.
  ## Valid time units are "ns", "us" (or "µs"), "ms", "s".
  precision = ""

  ## Logging configuration:
  ## Run telegraf with debug log messages.
  debug = false
  ## Run telegraf in quiet mode (error log messages only).
  quiet = false
  ## Specify the log file name. The empty string means to log to stderr.
  logfile = ""

  ## Override default hostname, if empty use os.Hostname()
  hostname = ""
  ## If set to true, do no set the "host" tag in the telegraf agent.
  omit_hostname = false


###############################################################################
#                            OUTPUT PLUGINS                                   #
###############################################################################

# Configuration for sending metrics to InfluxDB
[[outputs.influxdb]]
  ## The full HTTP or UDP URL for your InfluxDB instance.
  ##
  ## Multiple URLs can be specified for a single cluster, only ONE of the
  ## urls will be written to each interval.
  # urls = ["unix:///var/run/influxdb.sock"]
  # urls = ["udp://127.0.0.1:8089"]
  # urls = ["http://127.0.0.1:8086"]
  urls = ["http://influxdb:8086"]
  ## The target database for metrics; will be created as needed.
  database = "measurements"

  ## If true, no CREATE DATABASE queries will be sent.  Set to true when using
  ## Telegraf with a user without permissions to create databases or when the
  ## database already exists.
  # skip_database_creation = false

  ## Name of existing retention policy to write to.  Empty string writes to
  ## the default retention policy.  Only takes effect when using HTTP.
  # retention_policy = ""

  ## Write consistency (clusters only), can be: "any", "one", "quorum", "all".
  ## Only takes effect when using HTTP.
  # write_consistency = "any"

  ## Timeout for HTTP messages.
  # timeout = "5s"

  ## HTTP Basic Auth
  # username = "telegraf"
  # password = "metricsmetricsmetricsmetrics"

  ## HTTP User-Agent
  # user_agent = "telegraf"

  ## UDP payload size is the maximum packet size to send.
  # udp_payload = 512

  ## Optional TLS Config for use on HTTP connections.
  # tls_ca = "/etc/telegraf/ca.pem"
  # tls_cert = "/etc/telegraf/cert.pem"
  # tls_key = "/etc/telegraf/key.pem"
  ## Use TLS but skip chain & host verification
  # insecure_skip_verify = false

  ## HTTP Proxy override, if unset values the standard proxy environment
  ## variables are consulted to determine which proxy, if any, should be used.
  # http_proxy = "http://corporate.proxy:3128"

  ## Additional HTTP headers
  # http_headers = {"X-Special-Header" = "Special-Value"}

  ## HTTP Content-Encoding for write request body, can be set to "gzip" to
  ## compress body or "identity" to apply no encoding.
  # content_encoding = "identity"

  ## When true, Telegraf will output unsigned integers as unsigned values,
  ## i.e.: "42u".  You will need a version of InfluxDB supporting unsigned
  ## integer values.  Enabling this option will result in field type errors if
  ## existing data has been written.
  # influx_uint_support = false


# # Read metrics from MQTT topic(s)
[[inputs.mqtt_consumer]]
#   ## MQTT broker URLs to be used. The format should be scheme://host:port,
#   ## schema can be tcp, ssl, or ws.
   servers = ["tcp://mosquitto:1883"]
#
#   ## MQTT QoS, must be 0, 1, or 2
#   qos = 0
#   ## Connection timeout for initial connection in seconds
#   connection_timeout = "30s"
#
   ## Topics to subscribe to
   topics = [
     "/#",
   ]
#
#   # if true, messages that can't be delivered while the subscriber is offline
#   # will be delivered when it comes back (such as on service restart).
#   # NOTE: if true, client_id MUST be set
#   persistent_session = false
#   # If empty, a random client ID will be generated.
#   client_id = ""
#
#   ## username and password to connect MQTT server.
#   # username = "telegraf"
#   # password = "metricsmetricsmetricsmetrics"
#
#   ## Optional TLS Config
#   # tls_ca = "/etc/telegraf/ca.pem"
#   # tls_cert = "/etc/telegraf/cert.pem"
#   # tls_key = "/etc/telegraf/key.pem"
#   ## Use TLS but skip chain & host verification
#   # insecure_skip_verify = false
#
#   ## Data format to consume.
#   ## Each data format has its own unique set of configuration options, read
#   ## more about them here:
#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
#   data_format = "influx"
	version: '2'

	services:
	# influxdb = time series database
	influxdb:
	image: hypriot/rpi-influxdb
	container_name: influxdb
	environment:
	- INFLUXDB_DATA_ENGINE=tsm1
	- INFLUXDB_REPORTING_DISABLED=false
	volumes:
	- ./data/influxdb:/var/lib/influxdb

	# grafana = data visualisation and dashboarding
	grafana:
	image: tcjennings/rpi-grafana
	container_name: grafana
	ports:
	- "3000:3000"
	environment:
	- GF_AUTH_ANONYMOUS_ENABLED=true
	- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
	volumes:
	- ./data/grafana:/var/lib/grafana
	- ./grafana-datasources.yml/:/etc/grafana/provisioning/datasources/all.yml

	# chronograf = simple influxdb querying
	chronograf:
	image: mbrande3/rpi-chronograf
	container_name: chronograf
	environment:
	- INFLUXDB_URL=http://influxdb:8086
	ports:
	- "8888:8888"
	volumes:
	- ./data/chronograf:/var/lib/chronograf

	# telegraf = forwarding from MQTT to InfluxDB
	telegraf:
	image: swestcott/rpi-telegraf
	container_name: telegraf
	volumes:
	- ./telegraf.conf:/etc/telegraf/telegraf.conf


	# mosquitto = eclipse MQTT broker
	mosquitto:
	image: mjenz/rpi-mosquitto
	container_name: mosquitto
	ports:
	- "1883:1883"
	apiVersion: 1

	deleteDatasources:
	- name: influxdb
	orgId: 1

	datasources:
	- name: Influxdb
	type: influxdb
	access: proxy
	url: http://influxdb:8086
	database: measurements
	basicAuth: false
	withCredentials: false
	isDefault: true
	editable: false
	# Telegraf Configuration
	#
	# Telegraf is entirely plugin driven. All metrics are gathered from the
	# declared inputs, and sent to the declared outputs.
	#
	# Plugins must be declared in here to be active.
	# To deactivate a plugin, comment out the name and any variables.
	#
	# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
	# file would generate.
	#
	# Environment variables can be used anywhere in this config file, simply prepend
	# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
	# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)


	# Global tags can be specified here in key="value" format.
	[global_tags]
	# dc = "us-east-1" # will tag all metrics with dc=us-east-1
	# rack = "1a"
	## Environment variables can be used as tags, and throughout the config file
	# user = "$USER"


	# Configuration for telegraf agent
	[agent]
	## Default data collection interval for all inputs
	interval = "10s"
	## Rounds collection interval to 'interval'
	## ie, if interval="10s" then always collect on :00, :10, :20, etc.
	round_interval = true

	## Telegraf will send metrics to outputs in batches of at most
	## metric_batch_size metrics.
	## This controls the size of writes that Telegraf sends to output plugins.
	metric_batch_size = 1000

	## For failed writes, telegraf will cache metric_buffer_limit metrics for each
	## output, and will flush this buffer on a successful write. Oldest metrics
	## are dropped first when this buffer fills.
	## This buffer only fills when writes fail to output plugin(s).
	metric_buffer_limit = 10000

	## Collection jitter is used to jitter the collection by a random amount.
	## Each plugin will sleep for a random time within jitter before collecting.
	## This can be used to avoid many plugins querying things like sysfs at the
	## same time, which can have a measurable effect on the system.
	collection_jitter = "0s"

	## Default flushing interval for all outputs. You shouldn't set this below
	## interval. Maximum flush_interval will be flush_interval + flush_jitter
	flush_interval = "10s"
	## Jitter the flush interval by a random amount. This is primarily to avoid
	## large write spikes for users running a large number of telegraf instances.
	## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
	flush_jitter = "0s"

	## By default or when set to "0s", precision will be set to the same
	## timestamp order as the collection interval, with the maximum being 1s.
	## ie, when interval = "10s", precision will be "1s"
	## when interval = "250ms", precision will be "1ms"
	## Precision will NOT be used for service inputs. It is up to each individual
	## service input to set the timestamp at the appropriate precision.
	## Valid time units are "ns", "us" (or "µs"), "ms", "s".
	precision = ""

	## Logging configuration:
	## Run telegraf with debug log messages.
	debug = false
	## Run telegraf in quiet mode (error log messages only).
	quiet = false
	## Specify the log file name. The empty string means to log to stderr.
	logfile = ""

	## Override default hostname, if empty use os.Hostname()
	hostname = ""
	## If set to true, do no set the "host" tag in the telegraf agent.
	omit_hostname = false


	###############################################################################
	# OUTPUT PLUGINS #
	###############################################################################

	# Configuration for sending metrics to InfluxDB
	[[outputs.influxdb]]
	## The full HTTP or UDP URL for your InfluxDB instance.
	##
	## Multiple URLs can be specified for a single cluster, only ONE of the
	## urls will be written to each interval.
	# urls = ["unix:///var/run/influxdb.sock"]
	# urls = ["udp://127.0.0.1:8089"]
	# urls = ["http://127.0.0.1:8086"]
	urls = ["http://influxdb:8086"]
	## The target database for metrics; will be created as needed.
	database = "measurements"

	## If true, no CREATE DATABASE queries will be sent. Set to true when using
	## Telegraf with a user without permissions to create databases or when the
	## database already exists.
	# skip_database_creation = false

	## Name of existing retention policy to write to. Empty string writes to
	## the default retention policy. Only takes effect when using HTTP.
	# retention_policy = ""

	## Write consistency (clusters only), can be: "any", "one", "quorum", "all".
	## Only takes effect when using HTTP.
	# write_consistency = "any"

	## Timeout for HTTP messages.
	# timeout = "5s"

	## HTTP Basic Auth
	# username = "telegraf"
	# password = "metricsmetricsmetricsmetrics"

	## HTTP User-Agent
	# user_agent = "telegraf"

	## UDP payload size is the maximum packet size to send.
	# udp_payload = 512

	## Optional TLS Config for use on HTTP connections.
	# tls_ca = "/etc/telegraf/ca.pem"
	# tls_cert = "/etc/telegraf/cert.pem"
	# tls_key = "/etc/telegraf/key.pem"
	## Use TLS but skip chain & host verification
	# insecure_skip_verify = false

	## HTTP Proxy override, if unset values the standard proxy environment
	## variables are consulted to determine which proxy, if any, should be used.
	# http_proxy = "http://corporate.proxy:3128"

	## Additional HTTP headers
	# http_headers = {"X-Special-Header" = "Special-Value"}

	## HTTP Content-Encoding for write request body, can be set to "gzip" to
	## compress body or "identity" to apply no encoding.
	# content_encoding = "identity"

	## When true, Telegraf will output unsigned integers as unsigned values,
	## i.e.: "42u". You will need a version of InfluxDB supporting unsigned
	## integer values. Enabling this option will result in field type errors if
	## existing data has been written.
	# influx_uint_support = false



	# # Read metrics from MQTT topic(s)
	[[inputs.mqtt_consumer]]
	# ## MQTT broker URLs to be used. The format should be scheme://host:port,
	# ## schema can be tcp, ssl, or ws.
	servers = ["tcp://mosquitto:1883"]
	#
	# ## MQTT QoS, must be 0, 1, or 2
	# qos = 0
	# ## Connection timeout for initial connection in seconds
	# connection_timeout = "30s"
	#
	## Topics to subscribe to
	topics = [
	"/#",
	]
	#
	# # if true, messages that can't be delivered while the subscriber is offline
	# # will be delivered when it comes back (such as on service restart).
	# # NOTE: if true, client_id MUST be set
	# persistent_session = false
	# # If empty, a random client ID will be generated.
	# client_id = ""
	#
	# ## username and password to connect MQTT server.
	# # username = "telegraf"
	# # password = "metricsmetricsmetricsmetrics"
	#
	# ## Optional TLS Config
	# # tls_ca = "/etc/telegraf/ca.pem"
	# # tls_cert = "/etc/telegraf/cert.pem"
	# # tls_key = "/etc/telegraf/key.pem"
	# ## Use TLS but skip chain & host verification
	# # insecure_skip_verify = false
	#
	# ## Data format to consume.
	# ## Each data format has its own unique set of configuration options, read
	# ## more about them here:
	# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
	# data_format = "influx"