-
Install docker, docker-machine,docker-compose
-
Create a manager node and 3 worker nodes
-
ssh into manager node and create swarm :
$ docker-machine ssh manager $ docker swarm init --advertise-addr
docker-machine ip manager
-
Add worker nodes to the swarm :
$ docker-machine ssh agent1 docker swarm join --token
docker swarm join-token -q worker
docker-machine ip manager
:2377 $ docker-machine ssh agent2 docker swarm join --tokendocker swarm join-token -q worker
docker-machine ip manager
:2377 -
Copy the docker-compose file to install Influxdb, Grafana and cAdvisor :
$ vi docker-compose.yml
-
run docker-compose.yml:
$ docker stack deploy -c docker-stack.yml monitor
-
create database named cadvisor in influxdb to store the metrics :
$ docker exec
docker ps | grep -i influx | awk '{print $1}'
influx -execute 'CREATE DATABASE cadvisor' -
goto browser : http://
docker-machine ip manager
username: admin password: admin -
create a datasource in Grafana from above dashboard.json.
Created
September 17, 2017 09:22
-
-
Save ratulbasak/46d3de3c22d3c4a1a451f5a14d6d8029 to your computer and use it in GitHub Desktop.
monitoring docker swarm using influxdb, grafana, cadvisor
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"__inputs": [ | |
{ | |
"name": "DS_INFLUX", | |
"label": "influx", | |
"description": "", | |
"type": "datasource", | |
"pluginId": "influxdb", | |
"pluginName": "InfluxDB" | |
} | |
], | |
"__requires": [ | |
{ | |
"type": "grafana", | |
"id": "grafana", | |
"name": "Grafana", | |
"version": "4.2.0" | |
}, | |
{ | |
"type": "panel", | |
"id": "graph", | |
"name": "Graph", | |
"version": "" | |
}, | |
{ | |
"type": "datasource", | |
"id": "influxdb", | |
"name": "InfluxDB", | |
"version": "1.0.0" | |
} | |
], | |
"annotations": { | |
"list": [] | |
}, | |
"editable": true, | |
"gnetId": null, | |
"graphTooltip": 0, | |
"hideControls": false, | |
"id": 2, | |
"links": [], | |
"refresh": "5s", | |
"rows": [ | |
{ | |
"collapse": false, | |
"height": 250, | |
"panels": [ | |
{ | |
"aliasColors": {}, | |
"bars": false, | |
"dashLength": 10, | |
"dashes": false, | |
"datasource": "influx", | |
"fill": 1, | |
"id": 1, | |
"interval": "", | |
"legend": { | |
"alignAsTable": true, | |
"avg": true, | |
"current": true, | |
"max": true, | |
"min": true, | |
"rightSide": false, | |
"show": true, | |
"total": true, | |
"values": true | |
}, | |
"lines": true, | |
"linewidth": 1, | |
"links": [], | |
"nullPointMode": "connected", | |
"percentage": false, | |
"pointradius": 5, | |
"points": false, | |
"renderer": "flot", | |
"seriesOverrides": [], | |
"spaceLength": 10, | |
"span": 6, | |
"stack": false, | |
"steppedLine": false, | |
"targets": [ | |
{ | |
"alias": "Memory {host: $tag_machine, container: $tag_container_name}", | |
"dsType": "influxdb", | |
"groupBy": [ | |
{ | |
"params": [ | |
"$__interval" | |
], | |
"type": "time" | |
}, | |
{ | |
"params": [ | |
"machine" | |
], | |
"type": "tag" | |
}, | |
{ | |
"params": [ | |
"container_name" | |
], | |
"type": "tag" | |
} | |
], | |
"measurement": "memory_usage", | |
"orderByTime": "ASC", | |
"policy": "default", | |
"query": "SELECT \"value\" FROM \"memory_usage\" WHERE \"container_name\" =~ /^$container$/ AND \"machine\" =~ /^$host$/ AND $timeFilter", | |
"rawQuery": false, | |
"refId": "A", | |
"resultFormat": "time_series", | |
"select": [ | |
[ | |
{ | |
"params": [ | |
"value" | |
], | |
"type": "field" | |
}, | |
{ | |
"params": [], | |
"type": "mean" | |
} | |
] | |
], | |
"tags": [ | |
{ | |
"key": "container_name", | |
"operator": "=~", | |
"value": "/^$container$*/" | |
}, | |
{ | |
"condition": "AND", | |
"key": "machine", | |
"operator": "=~", | |
"value": "/^$host$/" | |
} | |
] | |
} | |
], | |
"thresholds": [], | |
"timeFrom": null, | |
"timeShift": null, | |
"title": "Memory", | |
"tooltip": { | |
"shared": true, | |
"sort": 0, | |
"value_type": "individual" | |
}, | |
"type": "graph", | |
"xaxis": { | |
"buckets": null, | |
"mode": "time", | |
"name": null, | |
"show": true, | |
"values": [] | |
}, | |
"yaxes": [ | |
{ | |
"format": "decbytes", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
}, | |
{ | |
"format": "short", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
} | |
] | |
}, | |
{ | |
"aliasColors": {}, | |
"bars": false, | |
"dashLength": 10, | |
"dashes": false, | |
"datasource": "influx", | |
"fill": 1, | |
"id": 2, | |
"legend": { | |
"alignAsTable": true, | |
"avg": true, | |
"current": true, | |
"max": true, | |
"min": true, | |
"show": true, | |
"total": true, | |
"values": true | |
}, | |
"lines": true, | |
"linewidth": 1, | |
"links": [], | |
"nullPointMode": "connected", | |
"percentage": false, | |
"pointradius": 5, | |
"points": false, | |
"renderer": "flot", | |
"seriesOverrides": [], | |
"spaceLength": 10, | |
"span": 6, | |
"stack": false, | |
"steppedLine": false, | |
"targets": [ | |
{ | |
"alias": "CPU {host: $tag_machine, container: $tag_container_name}", | |
"dsType": "influxdb", | |
"groupBy": [ | |
{ | |
"params": [ | |
"$interval" | |
], | |
"type": "time" | |
}, | |
{ | |
"params": [ | |
"machine" | |
], | |
"type": "tag" | |
}, | |
{ | |
"params": [ | |
"container_name" | |
], | |
"type": "tag" | |
} | |
], | |
"measurement": "cpu_usage_total", | |
"orderByTime": "ASC", | |
"policy": "default", | |
"refId": "A", | |
"resultFormat": "time_series", | |
"select": [ | |
[ | |
{ | |
"params": [ | |
"value" | |
], | |
"type": "field" | |
}, | |
{ | |
"params": [], | |
"type": "mean" | |
}, | |
{ | |
"params": [ | |
"10s" | |
], | |
"type": "derivative" | |
} | |
] | |
], | |
"tags": [ | |
{ | |
"key": "container_name", | |
"operator": "=~", | |
"value": "/^$container$*/" | |
}, | |
{ | |
"condition": "AND", | |
"key": "machine", | |
"operator": "=~", | |
"value": "/^$host$/" | |
} | |
] | |
} | |
], | |
"thresholds": [], | |
"timeFrom": null, | |
"timeShift": null, | |
"title": "CPU", | |
"tooltip": { | |
"shared": true, | |
"sort": 0, | |
"value_type": "individual" | |
}, | |
"type": "graph", | |
"xaxis": { | |
"buckets": null, | |
"mode": "time", | |
"name": null, | |
"show": true, | |
"values": [] | |
}, | |
"yaxes": [ | |
{ | |
"format": "hertz", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
}, | |
{ | |
"format": "short", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
} | |
] | |
} | |
], | |
"repeat": null, | |
"repeatIteration": null, | |
"repeatRowId": null, | |
"showTitle": false, | |
"title": "Dashboard Row", | |
"titleSize": "h6" | |
}, | |
{ | |
"collapse": false, | |
"height": 250, | |
"panels": [ | |
{ | |
"aliasColors": {}, | |
"bars": false, | |
"dashLength": 10, | |
"dashes": false, | |
"datasource": "influx", | |
"fill": 1, | |
"id": 3, | |
"legend": { | |
"alignAsTable": true, | |
"avg": true, | |
"current": true, | |
"max": true, | |
"min": true, | |
"show": true, | |
"total": true, | |
"values": true | |
}, | |
"lines": true, | |
"linewidth": 1, | |
"links": [], | |
"nullPointMode": "connected", | |
"percentage": false, | |
"pointradius": 5, | |
"points": false, | |
"renderer": "flot", | |
"seriesOverrides": [], | |
"spaceLength": 10, | |
"span": 6, | |
"stack": false, | |
"steppedLine": false, | |
"targets": [ | |
{ | |
"alias": "Usage {host: $tag_machine, container: $tag_container_name}", | |
"dsType": "influxdb", | |
"groupBy": [ | |
{ | |
"params": [ | |
"$interval" | |
], | |
"type": "time" | |
}, | |
{ | |
"params": [ | |
"container_name" | |
], | |
"type": "tag" | |
}, | |
{ | |
"params": [ | |
"machine" | |
], | |
"type": "tag" | |
} | |
], | |
"measurement": "fs_usage", | |
"orderByTime": "ASC", | |
"policy": "default", | |
"refId": "A", | |
"resultFormat": "time_series", | |
"select": [ | |
[ | |
{ | |
"params": [ | |
"value" | |
], | |
"type": "field" | |
}, | |
{ | |
"params": [], | |
"type": "mean" | |
} | |
] | |
], | |
"tags": [ | |
{ | |
"key": "machine", | |
"operator": "=~", | |
"value": "/^$host$/" | |
}, | |
{ | |
"condition": "AND", | |
"key": "container_name", | |
"operator": "=~", | |
"value": "/^$container$*/" | |
} | |
] | |
}, | |
{ | |
"alias": "Limit {host: $tag_machine, container: $tag_container_name}", | |
"dsType": "influxdb", | |
"groupBy": [ | |
{ | |
"params": [ | |
"$interval" | |
], | |
"type": "time" | |
}, | |
{ | |
"params": [ | |
"container_name" | |
], | |
"type": "tag" | |
}, | |
{ | |
"params": [ | |
"machine" | |
], | |
"type": "tag" | |
} | |
], | |
"measurement": "fs_limit", | |
"orderByTime": "ASC", | |
"policy": "default", | |
"refId": "B", | |
"resultFormat": "time_series", | |
"select": [ | |
[ | |
{ | |
"params": [ | |
"value" | |
], | |
"type": "field" | |
}, | |
{ | |
"params": [], | |
"type": "mean" | |
} | |
] | |
], | |
"tags": [ | |
{ | |
"key": "machine", | |
"operator": "=~", | |
"value": "/^$host$/" | |
}, | |
{ | |
"condition": "AND", | |
"key": "container_name", | |
"operator": "=~", | |
"value": "/^$container$*/" | |
} | |
] | |
} | |
], | |
"thresholds": [], | |
"timeFrom": null, | |
"timeShift": null, | |
"title": "File System", | |
"tooltip": { | |
"shared": true, | |
"sort": 0, | |
"value_type": "individual" | |
}, | |
"type": "graph", | |
"xaxis": { | |
"buckets": null, | |
"mode": "time", | |
"name": null, | |
"show": true, | |
"values": [] | |
}, | |
"yaxes": [ | |
{ | |
"format": "decbytes", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
}, | |
{ | |
"format": "short", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
} | |
] | |
}, | |
{ | |
"aliasColors": {}, | |
"bars": false, | |
"dashLength": 10, | |
"dashes": false, | |
"datasource": "influx", | |
"fill": 1, | |
"id": 4, | |
"legend": { | |
"alignAsTable": true, | |
"avg": true, | |
"current": true, | |
"max": true, | |
"min": true, | |
"show": true, | |
"total": true, | |
"values": true | |
}, | |
"lines": true, | |
"linewidth": 1, | |
"links": [], | |
"nullPointMode": "connected", | |
"percentage": false, | |
"pointradius": 5, | |
"points": false, | |
"renderer": "flot", | |
"seriesOverrides": [], | |
"spaceLength": 10, | |
"span": 6, | |
"stack": false, | |
"steppedLine": false, | |
"targets": [ | |
{ | |
"alias": "RX {host: $tag_machine, container: $tag_container_name}", | |
"dsType": "influxdb", | |
"groupBy": [ | |
{ | |
"params": [ | |
"$interval" | |
], | |
"type": "time" | |
}, | |
{ | |
"params": [ | |
"container_name" | |
], | |
"type": "tag" | |
}, | |
{ | |
"params": [ | |
"machine" | |
], | |
"type": "tag" | |
} | |
], | |
"measurement": "rx_bytes", | |
"orderByTime": "ASC", | |
"policy": "default", | |
"refId": "A", | |
"resultFormat": "time_series", | |
"select": [ | |
[ | |
{ | |
"params": [ | |
"value" | |
], | |
"type": "field" | |
}, | |
{ | |
"params": [], | |
"type": "mean" | |
}, | |
{ | |
"params": [ | |
"10s" | |
], | |
"type": "derivative" | |
} | |
] | |
], | |
"tags": [ | |
{ | |
"key": "machine", | |
"operator": "=~", | |
"value": "/^$host$/" | |
}, | |
{ | |
"condition": "AND", | |
"key": "container_name", | |
"operator": "=~", | |
"value": "/^$container$*/" | |
} | |
] | |
}, | |
{ | |
"alias": "TX {host: $tag_machine, container: $tag_container_name}", | |
"dsType": "influxdb", | |
"groupBy": [ | |
{ | |
"params": [ | |
"$interval" | |
], | |
"type": "time" | |
}, | |
{ | |
"params": [ | |
"container_name" | |
], | |
"type": "tag" | |
}, | |
{ | |
"params": [ | |
"machine" | |
], | |
"type": "tag" | |
} | |
], | |
"measurement": "tx_bytes", | |
"orderByTime": "ASC", | |
"policy": "default", | |
"refId": "B", | |
"resultFormat": "time_series", | |
"select": [ | |
[ | |
{ | |
"params": [ | |
"value" | |
], | |
"type": "field" | |
}, | |
{ | |
"params": [], | |
"type": "mean" | |
}, | |
{ | |
"params": [ | |
"10s" | |
], | |
"type": "derivative" | |
} | |
] | |
], | |
"tags": [ | |
{ | |
"key": "machine", | |
"operator": "=~", | |
"value": "/^$host$/" | |
}, | |
{ | |
"condition": "AND", | |
"key": "container_name", | |
"operator": "=~", | |
"value": "/^$container$*/" | |
} | |
] | |
} | |
], | |
"thresholds": [], | |
"timeFrom": null, | |
"timeShift": null, | |
"title": "Network", | |
"tooltip": { | |
"shared": true, | |
"sort": 0, | |
"value_type": "individual" | |
}, | |
"type": "graph", | |
"xaxis": { | |
"buckets": null, | |
"mode": "time", | |
"name": null, | |
"show": true, | |
"values": [] | |
}, | |
"yaxes": [ | |
{ | |
"format": "Bps", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
}, | |
{ | |
"format": "short", | |
"label": null, | |
"logBase": 1, | |
"max": null, | |
"min": null, | |
"show": true | |
} | |
] | |
} | |
], | |
"repeat": null, | |
"repeatIteration": null, | |
"repeatRowId": null, | |
"showTitle": false, | |
"title": "Dashboard Row", | |
"titleSize": "h6" | |
} | |
], | |
"schemaVersion": 14, | |
"style": "dark", | |
"tags": [], | |
"templating": { | |
"list": [ | |
{ | |
"allValue": "", | |
"current": { | |
"text": "backbox.corde.org", | |
"value": "backbox.corde.org" | |
}, | |
"datasource": "influx", | |
"hide": 0, | |
"includeAll": true, | |
"label": "Host", | |
"multi": false, | |
"name": "host", | |
"options": [], | |
"query": "show tag values with key = \"machine\"", | |
"refresh": 1, | |
"regex": "", | |
"sort": 0, | |
"tagValuesQuery": "", | |
"tags": [], | |
"tagsQuery": "", | |
"type": "query", | |
"useTags": false | |
}, | |
{ | |
"allValue": null, | |
"current": { | |
"text": "/", | |
"value": "/" | |
}, | |
"datasource": "influx", | |
"hide": 0, | |
"includeAll": false, | |
"label": "Container", | |
"multi": false, | |
"name": "container", | |
"options": [], | |
"query": "show tag values with key = \"container_name\" WHERE machine =~ /^$host$/", | |
"refresh": 1, | |
"regex": "/([^.]+)/", | |
"sort": 0, | |
"tagValuesQuery": "", | |
"tags": [], | |
"tagsQuery": "", | |
"type": "query", | |
"useTags": false | |
} | |
] | |
}, | |
"time": { | |
"from": "now-6h", | |
"to": "now" | |
}, | |
"timepicker": { | |
"refresh_intervals": [ | |
"5s", | |
"10s", | |
"30s", | |
"1m", | |
"5m", | |
"15m", | |
"30m", | |
"1h", | |
"2h", | |
"1d" | |
], | |
"time_options": [ | |
"5m", | |
"15m", | |
"1h", | |
"6h", | |
"12h", | |
"24h", | |
"2d", | |
"7d", | |
"30d" | |
] | |
}, | |
"timezone": "browser", | |
"title": "cAdvisor Copy", | |
"version": 2 | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
version: '3' | |
services: | |
influx: | |
image: influxdb | |
volumes: | |
- influx:/var/lib/influxdb | |
deploy: | |
replicas: 1 | |
placement: | |
constraints: | |
- node.role == manager | |
grafana: | |
image: grafana/grafana | |
ports: | |
- 0.0.0.0:3000:3000 | |
volumes: | |
- grafana:/var/lib/grafana | |
depends_on: | |
- influx | |
deploy: | |
replicas: 1 | |
placement: | |
constraints: | |
- node.role == manager | |
cadvisor: | |
image: google/cadvisor | |
hostname: '{{.Node.ID}}' | |
command: -logtostderr -docker_only -storage_driver=influxdb -storage_driver_db=cadvisor -storage_driver_host=influx:8086 | |
volumes: | |
- /:/rootfs:ro | |
- /var/run:/var/run:rw | |
- /sys:/sys:ro | |
- /var/lib/docker/:/var/lib/docker:ro | |
depends_on: | |
- influx | |
deploy: | |
mode: global | |
volumes: | |
influx: | |
driver: local | |
grafana: | |
driver: local |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment