Skip to content

Instantly share code, notes, and snippets.

@rmoff rmoff/export.json
Last active Mar 28, 2018

Embed
What would you like to do?
Kafka Connect, Elasticsearch, and Kibana config for Ubiquiti/syslog/KSQL blog
[
{
"_id": "f312aa10-3296-11e8-8031-ed44577da5db",
"_type": "dashboard",
"_source": {
"title": "Ubiquiti Access Point activity, via Apache Kafka and KSQL",
"hits": 0,
"description": "",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":2,\"y\":0,\"w\":10,\"h\":3,\"i\":\"1\"},\"version\":\"6.2.3\",\"type\":\"search\",\"id\":\"65696370-3278-11e8-8031-ed44577da5db\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":2,\"h\":7,\"i\":\"2\"},\"version\":\"6.2.3\",\"type\":\"visualization\",\"id\":\"77be88c0-3278-11e8-8031-ed44577da5db\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":7,\"y\":3,\"w\":5,\"h\":4,\"i\":\"3\"},\"version\":\"6.2.3\",\"type\":\"visualization\",\"id\":\"9d3dc5e0-3294-11e8-8031-ed44577da5db\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}}},{\"panelIndex\":\"4\",\"gridData\":{\"x\":2,\"y\":3,\"w\":5,\"h\":4,\"i\":\"4\"},\"version\":\"6.2.3\",\"type\":\"visualization\",\"id\":\"af9411c0-3296-11e8-8031-ed44577da5db\"}]",
"optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}",
"version": 1,
"timeRestore": false,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "65696370-3278-11e8-8031-ed44577da5db",
"_type": "search",
"_source": {
"title": "User Device / Access Point connections",
"description": "",
"hits": 0,
"columns": [
"AP_NAME",
"USER_DEVICE_NAME",
"DEVICE_TYPE",
"IS_GUEST"
],
"sort": [
"EXTRACT_TS",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"f8ef2c20-3295-11e8-8031-ed44577da5db\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "77be88c0-3278-11e8-8031-ed44577da5db",
"_type": "visualization",
"_source": {
"title": "Activity per Device",
"visState": "{\n \"title\": \"Activity per Device\",\n \"type\": \"metric\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"metric\",\n \"metric\": {\n \"percentageMode\": false,\n \"useRanges\": false,\n \"colorSchema\": \"Green to Red\",\n \"metricColorMode\": \"None\",\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 10000\n }\n ],\n \"labels\": {\n \"show\": true\n },\n \"invertColors\": false,\n \"style\": {\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\",\n \"fontSize\": 60\n }\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"USER_DEVICE_NAME\",\n \"otherBucket\": false,\n \"otherBucketLabel\": \"Other\",\n \"missingBucket\": false,\n \"missingBucketLabel\": \"Missing\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ]\n}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\n \"index\": \"f8ef2c20-3295-11e8-8031-ed44577da5db\",\n \"filter\": [],\n \"query\": {\n \"query\": \"\",\n \"language\": \"lucene\"\n }\n}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "9d3dc5e0-3294-11e8-8031-ed44577da5db",
"_type": "visualization",
"_source": {
"title": "Device activity by Access Point",
"visState": "{\n \"title\": \"Device activity by Access Point\",\n \"type\": \"histogram\",\n \"params\": {\n \"type\": \"histogram\",\n \"grid\": {\n \"categoryLines\": false,\n \"style\": {\n \"color\": \"#eee\"\n }\n },\n \"categoryAxes\": [\n {\n \"id\": \"CategoryAxis-1\",\n \"type\": \"category\",\n \"position\": \"bottom\",\n \"show\": true,\n \"style\": {},\n \"scale\": {\n \"type\": \"linear\"\n },\n \"labels\": {\n \"show\": true,\n \"truncate\": 100\n },\n \"title\": {}\n }\n ],\n \"valueAxes\": [\n {\n \"id\": \"ValueAxis-1\",\n \"name\": \"LeftAxis-1\",\n \"type\": \"value\",\n \"position\": \"left\",\n \"show\": true,\n \"style\": {},\n \"scale\": {\n \"type\": \"linear\",\n \"mode\": \"normal\"\n },\n \"labels\": {\n \"show\": true,\n \"rotate\": 0,\n \"filter\": false,\n \"truncate\": 100\n },\n \"title\": {\n \"text\": \"Count\"\n }\n }\n ],\n \"seriesParams\": [\n {\n \"show\": \"true\",\n \"type\": \"histogram\",\n \"mode\": \"stacked\",\n \"data\": {\n \"label\": \"Count\",\n \"id\": \"1\"\n },\n \"valueAxis\": \"ValueAxis-1\",\n \"drawLinesBetweenPoints\": true,\n \"showCircles\": true\n }\n ],\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"times\": [],\n \"addTimeMarker\": false\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"EXTRACT_TS\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"AP_NAME\",\n \"otherBucket\": false,\n \"otherBucketLabel\": \"Other\",\n \"missingBucket\": false,\n \"missingBucketLabel\": \"Missing\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ]\n}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\n \"index\": \"f8ef2c20-3295-11e8-8031-ed44577da5db\",\n \"filter\": [],\n \"query\": {\n \"query\": \"\",\n \"language\": \"lucene\"\n }\n}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "af9411c0-3296-11e8-8031-ed44577da5db",
"_type": "visualization",
"_source": {
"title": "Device Type connections per Day",
"visState": "{\"title\":\"Device Type connections per Day\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"EXTRACT_TS\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"DEVICE_TYPE\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"f8ef2c20-3295-11e8-8031-ed44577da5db\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "470c0260-3297-11e8-8031-ed44577da5db",
"_type": "visualization",
"_source": {
"title": "Device Types",
"visState": "{\"title\":\"Device Types\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"DEVICE_TYPE\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}]}",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"f8ef2c20-3295-11e8-8031-ed44577da5db\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
},
"_meta": {
"savedObjectVersion": 2
}
}
]
curl -XPUT "http://localhost:9200/_template/kafkaconnect/" -H 'Content-Type: application/json' -d'
{
"index_patterns": "*",
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"dates": {
"match": "EXTRACT_TS",
"mapping": {
"type": "date"
}
}
},
{
"non_analysed_string_template": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
]
}
}
}'
curl -X "POST" "http://localhost:8083/connectors/" \
-H "Content-Type: application/json" \
-d '{
"name": "es_sink_UBNT_AP_USER_DEVICE_CONNECTS",
"config": {
"topics": "'UBNT_AP_USER_DEVICE_CONNECTS'",
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
"key.ignore": "true",
"schema.ignore": "true",
"type.name": "type.name=kafkaconnect",
"topic.index.map": "'UBNT_AP_USER_DEVICE_CONNECTS':'ubnt_ap_user_device_connects'",
"connection.url": "http://localhost:9200",
"transforms": "ExtractTimestamp",
"transforms.ExtractTimestamp.type": "org.apache.kafka.connect.transforms.InsertField$Value",
"transforms.ExtractTimestamp.timestamp.field" : "EXTRACT_TS"
}
}'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.