onweek 2023-05 dashboard with event log visualizations

ow23-05-visualizations.ndjson

{"attributes":{"fieldAttrs":"{\"provider_rule_type\":{\"count\":1},\"event.action\":{\"count\":1}}","fieldFormatMap":"{\"event.duration\":{\"id\":\"duration\",\"params\":{\"parsedUrl\":{\"origin\":\"https://pmuellr-8-7-0.kb.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/management/kibana/dataViews/patterns/kibana-event-log\",\"basePath\":\"\"},\"inputFormat\":\"nanoseconds\",\"outputFormat\":\"asSeconds\",\"outputPrecision\":2,\"includeSpaceWithSuffix\":true,\"showSuffix\":true,\"useShortSuffix\":true}}}","fields":"[]","name":"ow23-05-event-log","runtimeFieldMap":"{\"provider_rule_type\":{\"type\":\"keyword\",\"script\":{\"source\":\"def provider = doc[\\\"event.provider\\\"];\\ndef ruleTyp1 = doc[\\\"rule.category\\\"];\\ndef ruleTyp2 = doc[\\\"kibana.alert.rule.rule_type_id\\\"];\\n\\nif (provider == null || provider.size() == 0) return;\\n\\nif (ruleTyp1 != null && ruleTyp1.size() != 0) {\\n    emit(provider.value + \\\"::\\\" + ruleTyp1.value);\\n    return;\\n} \\n\\nif (ruleTyp2 != null && ruleTyp2.size() != 0) {\\n    emit(provider.value + \\\"::\\\" + ruleTyp2.value);\\n    return;\\n}\\n\\nemit(provider.value + \\\"::unknown\\\");\"}}}","sourceFilters":"[]","timeFieldName":"@timestamp","title":".kibana-event-log-*","typeMeta":"{}"},"coreMigrationVersion":"8.7.0","created_at":"2023-05-12T00:56:35.827Z","id":"62afc0ae-b2bd-4353-8b81-6c6adafa768c","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2023-05-12T01:34:11.747Z","version":"WzExNTc1NjA2LDNd"}
{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"da4b6745-4621-4f1f-a338-973834a6a1d2":{"columnOrder":["188958cf-8a2a-444e-a661-225267631aa7","5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01","bcee3a1e-63d0-4f34-9fe3-058f3a1be00b"],"columns":{"188958cf-8a2a-444e-a661-225267631aa7":{"dataType":"date","isBucketed":true,"label":"@timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"@timestamp"},"5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01":{"dataType":"string","isBucketed":true,"label":"Top 10 values of provider_rule_type","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"fallback":false,"type":"alphabetical"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"provider_rule_type"},"bcee3a1e-63d0-4f34-9fe3-058f3a1be00b":{"dataType":"number","filter":{"language":"kuery","query":"event.action : \"execute\" "},"isBucketed":false,"label":"Sum of event.duration","operationType":"sum","params":{"emptyAsNull":false},"scale":"ratio","sourceField":"event.duration"}},"incompleteColumns":{},"sampling":1}}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"None","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["bcee3a1e-63d0-4f34-9fe3-058f3a1be00b"],"layerId":"da4b6745-4621-4f1f-a338-973834a6a1d2","layerType":"data","position":"top","seriesType":"bar_stacked","showGridlines":false,"splitAccessor":"5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01","xAccessor":"188958cf-8a2a-444e-a661-225267631aa7"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"bar_stacked","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide"}},"title":"task duration summed by type, in time buckets","visualizationType":"lnsXY"},"coreMigrationVersion":"8.7.0","created_at":"2023-05-12T13:13:58.720Z","id":"0d3c6690-f0c1-11ed-9343-fb97730af649","migrationVersion":{"lens":"8.6.0"},"references":[{"id":"62afc0ae-b2bd-4353-8b81-6c6adafa768c","name":"indexpattern-datasource-layer-da4b6745-4621-4f1f-a338-973834a6a1d2","type":"index-pattern"}],"type":"lens","updated_at":"2023-05-12T13:13:58.720Z","version":"WzExNzI4OTExLDNd"}
{"attributes":{"description":"use es-apm-sys-sim to drive the index threshold rule, which will cause actions, which will starve the other tasks","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"03b3a744-ad1c-4a2b-a8ef-ff7cb9065980":{"columnOrder":["a74361f6-ae45-4edf-aeb6-18234a6237d6","8af51e78-59e3-4efa-aa57-334403e16a99","6e54d3cd-4793-45bc-85c9-8e8c18ea8eb2"],"columns":{"6e54d3cd-4793-45bc-85c9-8e8c18ea8eb2":{"dataType":"number","isBucketed":false,"label":"Count of records","operationType":"count","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"___records___"},"8af51e78-59e3-4efa-aa57-334403e16a99":{"dataType":"date","isBucketed":true,"label":"@timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"@timestamp"},"a74361f6-ae45-4edf-aeb6-18234a6237d6":{"dataType":"string","isBucketed":true,"label":"Top 10 values of provider_rule_type","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"fallback":false,"type":"alphabetical"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"provider_rule_type"}},"incompleteColumns":{},"sampling":1}}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"bf653943-03b1-43f9-8b05-31980012ba53","key":"event.action","negate":false,"params":{"query":"execute"},"type":"phrase"},"query":{"match_phrase":{"event.action":"execute"}}}],"internalReferences":[],"query":{"language":"kuery","query":"event.action:execute"},"visualization":{"gridConfig":{"isCellLabelVisible":false,"isXAxisLabelVisible":true,"isXAxisTitleVisible":false,"isYAxisLabelVisible":true,"isYAxisTitleVisible":false,"type":"heatmap_grid"},"layerId":"03b3a744-ad1c-4a2b-a8ef-ff7cb9065980","layerType":"data","legend":{"isVisible":true,"position":"right","type":"heatmap_legend"},"shape":"heatmap","valueAccessor":"6e54d3cd-4793-45bc-85c9-8e8c18ea8eb2","xAccessor":"8af51e78-59e3-4efa-aa57-334403e16a99","yAccessor":"a74361f6-ae45-4edf-aeb6-18234a6237d6"}},"title":"live tasks running by type","visualizationType":"lnsHeatmap"},"coreMigrationVersion":"8.7.0","created_at":"2023-05-12T12:24:32.809Z","id":"ff9b9b30-f0b8-11ed-9343-fb97730af649","migrationVersion":{"lens":"8.6.0"},"references":[{"id":"62afc0ae-b2bd-4353-8b81-6c6adafa768c","name":"indexpattern-datasource-layer-03b3a744-ad1c-4a2b-a8ef-ff7cb9065980","type":"index-pattern"},{"id":"62afc0ae-b2bd-4353-8b81-6c6adafa768c","name":"bf653943-03b1-43f9-8b05-31980012ba53","type":"index-pattern"}],"type":"lens","updated_at":"2023-05-12T12:24:32.809Z","version":"WzExNzE3MjM4LDNd"}
{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"da4b6745-4621-4f1f-a338-973834a6a1d2":{"columnOrder":["5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01","188958cf-8a2a-444e-a661-225267631aa7","bcee3a1e-63d0-4f34-9fe3-058f3a1be00b"],"columns":{"188958cf-8a2a-444e-a661-225267631aa7":{"dataType":"date","isBucketed":true,"label":"@timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"@timestamp"},"5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01":{"dataType":"string","isBucketed":true,"label":"Top 10 values of provider_rule_type","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"fallback":false,"type":"alphabetical"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"provider_rule_type"},"bcee3a1e-63d0-4f34-9fe3-058f3a1be00b":{"dataType":"number","filter":{"language":"kuery","query":"event.action : \"execute\" "},"isBucketed":false,"label":"Median of event.duration","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"event.duration"}},"incompleteColumns":{},"sampling":1}}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"None","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["bcee3a1e-63d0-4f34-9fe3-058f3a1be00b"],"layerId":"da4b6745-4621-4f1f-a338-973834a6a1d2","layerType":"data","position":"top","seriesType":"bar","showGridlines":false,"splitAccessor":"5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01","xAccessor":"188958cf-8a2a-444e-a661-225267631aa7"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"bar","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide"}},"title":"task duration by type, in time buckets","visualizationType":"lnsXY"},"coreMigrationVersion":"8.7.0","created_at":"2023-05-12T12:30:06.447Z","id":"ba11cff0-f0c0-11ed-9343-fb97730af649","migrationVersion":{"lens":"8.6.0"},"references":[{"id":"62afc0ae-b2bd-4353-8b81-6c6adafa768c","name":"indexpattern-datasource-layer-da4b6745-4621-4f1f-a338-973834a6a1d2","type":"index-pattern"}],"type":"lens","updated_at":"2023-05-12T12:30:06.447Z","version":"WzExNzE4NTY5LDNd"}
{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.7.0\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"d5cc9c2c-7697-46a5-8842-5f718dc234dc\"},\"panelIndex\":\"d5cc9c2c-7697-46a5-8842-5f718dc234dc\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_d5cc9c2c-7697-46a5-8842-5f718dc234dc\"},{\"version\":\"8.7.0\",\"type\":\"lens\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":7,\"i\":\"e8f347c6-6b91-4d40-907c-3a5fcd04f560\"},\"panelIndex\":\"e8f347c6-6b91-4d40-907c-3a5fcd04f560\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"62afc0ae-b2bd-4353-8b81-6c6adafa768c\",\"name\":\"indexpattern-datasource-layer-da4b6745-4621-4f1f-a338-973834a6a1d2\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar\",\"layers\":[{\"layerId\":\"da4b6745-4621-4f1f-a338-973834a6a1d2\",\"accessors\":[\"bcee3a1e-63d0-4f34-9fe3-058f3a1be00b\"],\"position\":\"top\",\"seriesType\":\"bar\",\"showGridlines\":false,\"layerType\":\"data\",\"splitAccessor\":\"5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01\",\"xAccessor\":\"188958cf-8a2a-444e-a661-225267631aa7\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"da4b6745-4621-4f1f-a338-973834a6a1d2\":{\"columns\":{\"5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01\":{\"label\":\"Top 10 values of provider_rule_type\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"provider_rule_type\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":false},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false}},\"188958cf-8a2a-444e-a661-225267631aa7\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"bcee3a1e-63d0-4f34-9fe3-058f3a1be00b\":{\"label\":\"Median of event.duration\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"event.duration\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"event.action : \\\"execute\\\" \",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"5ad0e9e3-6bc9-4aa1-baa5-9cfbb88fed01\",\"188958cf-8a2a-444e-a661-225267631aa7\",\"bcee3a1e-63d0-4f34-9fe3-058f3a1be00b\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"task duration by task type, per minute\",\"panelRefName\":\"panel_e8f347c6-6b91-4d40-907c-3a5fcd04f560\"},{\"version\":\"8.7.0\",\"type\":\"lens\",\"gridData\":{\"x\":24,\"y\":7,\"w\":24,\"h\":8,\"i\":\"3b72f8f0-0497-4d0c-82ae-7f52f1c6e782\"},\"panelIndex\":\"3b72f8f0-0497-4d0c-82ae-7f52f1c6e782\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3b72f8f0-0497-4d0c-82ae-7f52f1c6e782\"}]","timeRestore":false,"title":"ow23-05 visualizations","version":1},"coreMigrationVersion":"8.7.0","created_at":"2023-05-12T12:36:12.381Z","id":"0a7f6310-f0b9-11ed-9343-fb97730af649","migrationVersion":{"dashboard":"8.7.0"},"references":[{"id":"ff9b9b30-f0b8-11ed-9343-fb97730af649","name":"d5cc9c2c-7697-46a5-8842-5f718dc234dc:panel_d5cc9c2c-7697-46a5-8842-5f718dc234dc","type":"lens"},{"id":"ba11cff0-f0c0-11ed-9343-fb97730af649","name":"e8f347c6-6b91-4d40-907c-3a5fcd04f560:panel_e8f347c6-6b91-4d40-907c-3a5fcd04f560","type":"lens"},{"id":"62afc0ae-b2bd-4353-8b81-6c6adafa768c","name":"e8f347c6-6b91-4d40-907c-3a5fcd04f560:indexpattern-datasource-layer-da4b6745-4621-4f1f-a338-973834a6a1d2","type":"index-pattern"},{"id":"0d3c6690-f0c1-11ed-9343-fb97730af649","name":"3b72f8f0-0497-4d0c-82ae-7f52f1c6e782:panel_3b72f8f0-0497-4d0c-82ae-7f52f1c6e782","type":"lens"}],"type":"dashboard","updated_at":"2023-05-12T12:36:12.381Z","version":"WzExNzIwMDkzLDNd"}
{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":5,"missingRefCount":0,"missingReferences":[]}

a dashboard showing some event log visualizations

This screencap shows what happens when an alert goes off and generates a lot of actions. You can see the actions:.index-threshold row in the left graph go from zero to busy to zero, and how that affects the other scheduled tasks - starves them.

To import into Kibana

• export KBN_URL as appropriate, eg export KBN_URL=http://elastic:changeme@localhost:5601
• run the following commands:

This will curl the .ndjson file in this Gist, directly to a saved objects import.

DOCS=https://gist.githubusercontent.com/pmuellr/2440eebfef1793e2499a37edec99fa47/raw/24acecc3384cabfc1d6e71964180a738dd40b4d8/ow23-05-visualizations.ndjson
curl -L $DOCS | curl $KBN_URL/api/saved_objects/_import "-Ffile=@-;filename=ow23-05-visualizations.ndjson" -H "kbn-xsrf: foo"