0-readme.md

Our current Temporal dashboards, which uses Atlas Stack Language and Lumen.

There are two dashboards here:

1. temporal-dash.json is our Temporal service dashboard. Each Temporal service is broken down into its own section with its own views.
2. temporal-service-client.json is a reusable component that Temporal workers can include in their application dashboards which gives insight into the SDK.
3. temporal-service-go.json reusable component for the Temporal services themselves to expose Go metrics.
4. temporal-internal-client.json reusable component for each Temporal service's clients to other Temporal services. Just an overview.

temporal-dash.json

{
    "content": {
        "toolbar": {
            "controls": [
                {
                    "options": {
                        "default": {
                            "value": "main"
                        },
                        "label": "stack",
                        "name": "nf.stack"
                    },
                    "source": {
                        "options": {
                            "data": [
                                "main",
                                "prestaging",
                                "test"
                            ]
                        },
                        "type": "static"
                    },
                    "type": "list"
                },
                {
                    "options": {
                        "default": "prod.us-west-2.mgmt",
                        "validTargets": [
                            "test.us-west-2.mgmt",
                            "prod.us-west-2.mgmt",
                            "prod.us-east-1.mgmt",
                            "test.us-east-1",
                            "prod.us-east-1"
                        ]
                    },
                    "type": "atlas-target"
                },
                {
                    "options": {},
                    "type": "start-end-select"
                },
                {
                    "import": "atlas-tag",
                    "vars": {
                        "query": "aws.dbname,^(temporal-*{{nf.stack}}.*).*,:re",
                        "tag": "aws.dbname"
                    }
                }
            ]
        },
        "views": [
            {
                "rows": [
                    {
                        "cells": [
                            {
                                "extend": "atlas",
                                "title": "Sync Match Ratio (by Namespace)",
                                "vars": {
                                    "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_poll_success_sync_per_tl,:eq,:and,:sum,(,namespace,),:by,nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_poll_success_per_tl,:eq,:and,:sum,(,namespace,),:by,:div"
                                }
                            }
                        ]
                    },
                    {
                        "cells": [
                            {
                                "extend": "atlas",
                                "title": "Workflows (by Namespace, Result)",
                                "vars": {
                                    "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,(,temporal_workflow_success,temporal_workflow_failed,temporal_workflow_timeout,temporal_worklflow_cancel,temporal_workflow_terminate,),:in,:and,:sum,(,name,namespace,),:by,total,:sort,desc,:order,$(namespace) - $(name),:legend"
                                }
                            }
                        ]
                    }
                ],
                "title": "Overview"
            },
            {
                "subViews": [
                    {
                        "extend": "temporal-service-common",
                        "title": "Overview",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-frontend"
                        }
                    },
                    {
                        "extend": "temporal-service-go",
                        "title": "Go",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-frontend"
                        }
                    },
                    {
                        "extend": "temporal-internal-client",
                        "title": "Client: History",
                        "vars": {
                            "client": "history",
                            "cluster": "temporal-{{nf.stack}}-frontend"
                        }
                    },
                    {
                        "extend": "temporal-internal-client",
                        "title": "Client: Matching",
                        "vars": {
                            "client": "matching",
                            "cluster": "temporal-{{nf.stack}}-frontend"
                        }
                    }
                ],
                "title": "Frontend"
            },
            {
                "subViews": [
                    {
                        "extend": "temporal-service-common",
                        "title": "Overview",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-matching"
                        }
                    },
                    {
                        "extend": "temporal-service-go",
                        "title": "Go",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-matching"
                        }
                    },
                    {
                        "rows": [
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Polls",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_poll_success,:eq,:and,:sum,success,:legend,nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_poll_timeouts,:eq,:and,:sum,timeout,:legend,nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_poll_success_sync,:eq,:and,:sum,sync matches,:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Throttled Requests",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_sync_throttle_count,:eq,:and,:sum"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Sync Match Latency (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_syncmatch_latency,:eq,:and,:dist-avg,(,operation,),:by,$(operation),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Async Match Latency (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_asyncmatch_latency,:eq,:and,:dist-avg,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Forwarded Cells",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_forwarded,:eq,:and,:sum"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Local to Local Match",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_local_to_local_matches,:eq,:and,:sum"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Local to Remote Match",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_local_to_remote_matches,:eq,:and,:sum"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Local to Local Match (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_local_to_local_matches,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Local to Remote Match (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_local_to_remote_matches,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Lease Request vs Failures",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_lease_requests,:eq,:and,:sum,request,:legend,nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_lease_failures,:eq,:and,:sum,failures,:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Expired Tasks",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_tasks_expired,:eq,:and,:sum"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Throttle",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_sync_throttle_count,:eq,:and,:sum,sync throttle,:legend,nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_buffer_throttle_count,:eq,:and,:sum,buffer throttle,:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Error Breakdown",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-matching,:eq,name,temporal_condition_failed_errors,:eq,:and,:sum"
                                        }
                                    }
                                ]
                            }
                        ],
                        "title": "Details"
                    },
                    {
                        "extend": "temporal-internal-client",
                        "title": "Client: History",
                        "vars": {
                            "client": "history",
                            "cluster": "temporal-{{nf.stack}}-matching"
                        }
                    }
                ],
                "title": "Matching"
            },
            {
                "subViews": [
                    {
                        "extend": "temporal-service-common",
                        "title": "Overview",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-history"
                        }
                    },
                    {
                        "extend": "temporal-service-go",
                        "title": "Go",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-history"
                        }
                    },
                    {
                        "rows": [
                            {
                                "cells": [
                                    {
                                        "options": {
                                            "align": "center",
                                            "content": "**Transfer Task Processing**"
                                        },
                                        "type": "text"
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Transfer Task Requests vs Errors",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_requests,:eq,:and,operation,TransferActive.*,:re,:and,:sum,0075dc,:color,requests,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_errors,:eq,:and,operation,TransferActive.*,:re,:and,:sum,ff0000,:color,1,:axis,errors,:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Active Transfer Errors Break Down",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_errors_.*,:re,:and,operation,TransferActive.*,:re,:and,:sum,(,name,),:by,$(name),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Transfer Task (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_requests,:eq,:and,operation,TransferActive.*,:re,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Active Transfer Task Errors (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_errors,:eq,:and,operation,TransferActive.*,:re,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Transfer Task Processing Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_latency_processing,:eq,:and,operation,TransferActive.*,:re,:and,:sum,(,name,),:by,$(name),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Active Transfer Task Queue Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_latency_queue,:eq,:and,operation,TransferActive.*,:re,:and,:sum,(,name,),:by,$(name),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Transfer Task Attempts",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_attempt,:eq,:and,operation,TransferActive.*,:re,:and,:dist-avg,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "options": {
                                            "align": "center",
                                            "content": "**Timer Tasks Processing**"
                                        },
                                        "type": "text"
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Timer Task Requests vs Errors",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_requests,:eq,:and,operation,TimerActive.*,:re,:and,:sum,0075dc,:color,requests,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_errors,:eq,:and,operation,TimerActive.*,:re,:and,:sum,ff0000,:color,1,:axis,errors,:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Active Timer Errors Break Down",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_errors_.*,:re,:and,operation,TimerActive.*,:re,:and,:sum,(,name,),:by,$(name),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Timer Task (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_requests,:eq,:and,operation,TimerActive.*,:re,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Active Timer Task Errors (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_errors,:eq,:and,operation,TimerActive.*,:re,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Timer Task Processing Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_latency_processing,:eq,:and,operation,TimerActive.*,:re,:and,:sum,(,name,),:by,$(name),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Active Timer Task Queue Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_latency_queue,:eq,:and,operation,TimerActive.*,:re,:and,:sum,(,name,),:by,$(name),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Active Timer Task Attempts",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_task_attempt,:eq,:and,operation,TimerActive.*,:re,:and,:dist-avg,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "options": {
                                            "align": "center",
                                            "content": "**Mutable Cache State**"
                                        },
                                        "type": "text"
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Mutable State Cache Requests (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_requests,:eq,:and,cache_type,mutablestate,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Mutable State Cache Misses (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_miss,:eq,:and,cache_type,mutablestate,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Mutable State Cache",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_requests,:eq,:and,cache_type,mutablestate,:eq,:and,operation,HistoryCacheGetCurrentExecution,:eq,:not,:and,:sum,cache requests,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_errors,:eq,:and,cache_type,mutablestate,:eq,:and,operation,HistoryCacheGetCurrentExecution,:eq,:not,:and,:sum,cache errors,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_miss,:eq,:and,cache_type,mutablestate,:eq,:and,operation,HistoryCacheGetCurrentExecution,:eq,:not,:and,:sum,cache miss,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_acquire_lock_failed,:eq,:and,cache_type,mutablestate,:eq,:and,operation,HistoryCacheGetCurrentExecution,:eq,:not,:and,:sum,acquire lock failed,:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Mutable State Cache Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_latency,:eq,:and,cache_type,mutablestate,:eq,:and,operation,HistoryCacheGetCurrentExecution,:eq,:not,:and,:dist-avg"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "options": {
                                            "align": "center",
                                            "content": "**Events Cache** (TODO: Further breakdowns)"
                                        },
                                        "type": "text"
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Events Cache Requests (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_requests,:eq,:and,cache_type,events,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Events Cache Misses (by Operation)",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_miss,:eq,:and,cache_type,events,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Events Cache",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_requests,:eq,:and,cache_type,events,:eq,:and,:sum,cache requests,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_errors,:eq,:and,cache_type,events,:eq,:and,:sum,cache errors,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_miss,:eq,:and,cache_type,events,:eq,:and,:sum,cache miss,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_acquire_lock_failed,:eq,:and,cache_type,events,:eq,:and,:sum,acquire lock failed,:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Events Cache Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cache_latency,:eq,:and,cache_type,events,:eq,:and,:dist-avg"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "options": {
                                            "align": "center",
                                            "content": "**Workflow**"
                                        },
                                        "type": "text"
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Workflow Task Break Down",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_schedule_activity_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,schedule activity,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_complete_workflow_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,complete workflow,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_fail_workflow_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,fail workflow,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cancel_workflow_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,cancel workflow,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_start_timer_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,start timer,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cancel_activity_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,cancel activity,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cancel_timer_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,cancel timer,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_record_marker_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,record marker,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_cancel_external_workflow_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,cancel external workflow,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_continue_as_new_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,continue as new,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_signal_external_workflow_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,signal external workflow,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_upsert_workflow_search_attributes_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,upsert searchable attributes,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_child_workflow_command,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,child workflow,:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Sticky vs Non-sticky",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_service_requests,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,commands complete,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_complete_workflow_task_sticky_enabled_count,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,sticky tasks,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_complete_workflow_task_sticky_disabled_count,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum,non-sticky tasks,:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Failed Workflow Tasks",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_failed_workflow_tasks,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Empty Worfklow Tasks",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_empty_completion_commands,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Multiple Complete Workflows",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_multiple_completion_commands,:eq,:and,operation,RespondWorkflowTaskCompleted,:eq,:and,:sum"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "options": {
                                            "align": "center",
                                            "content": "**Mutable State Insights**"
                                        },
                                        "type": "text"
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Stale Mutable State",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_stale_mutable_state,:eq,:and,:sum,$(operation),:legend"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "options": {
                                            "align": "center",
                                            "content": "**Shard Controller**"
                                        },
                                        "type": "text"
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Acquire Shards Count",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_acquire_shards_count,:eq,:and,:sum"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Acquire Shards Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_acquire_shards_latency,:eq,:and,:dist-avg"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Shards Closed",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_shard_closed_count,:eq,:and,:sum"
                                        }
                                    }
                                ]
                            },
                            {
                                "cells": [
                                    {
                                        "extend": "atlas",
                                        "title": "Shard Item Counts",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_sharditem_created_count,:eq,:and,:sum,created,:legend,nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_sharditem_removed_count,:eq,:and,:sum,removed,:legend"
                                        }
                                    },
                                    {
                                        "extend": "atlas",
                                        "title": "Shard Item Latency",
                                        "vars": {
                                            "q": "nf.cluster,temporal-{{nf.stack}}-history,:eq,name,temporal_sharditem_acquisition_latency,:eq,:and,:dist-avg"
                                        }
                                    }
                                ]
                            }
                        ],
                        "title": "Details"
                    },
                    {
                        "extend": "temporal-internal-client",
                        "title": "Client: Matching",
                        "vars": {
                            "client": "matching",
                            "cluster": "temporal-{{nf.stack}}-history"
                        }
                    }
                ],
                "title": "History"
            },
            {
                "subViews": [
                    {
                        "extend": "temporal-service-common",
                        "title": "Overview",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-worker"
                        }
                    },
                    {
                        "extend": "temporal-service-go",
                        "title": "Go",
                        "vars": {
                            "cluster": "temporal-{{nf.stack}}-worker"
                        }
                    }
                ],
                "title": "Worker"
            },
            {
                "subViews": [
                    {
                        "repeat": {
                            "config": {
                                "extend": "CloudWatchRDS",
                                "title": "{{aws.dbname}}"
                            },
                            "values": "{{aws.dbname_values}}",
                            "varName": "aws.dbname"
                        }
                    }
                ],
                "title": "CloudwatchRDS"
            },
            {
                "rows": [
                    {
                        "cells": [
                            {
                                "options": {
                                    "content": "- [us-west-2](https://lumen-v2.prod.netflix.net/show/Elasticsearch?v.target=s~prod.us-west-2&v.nf.app=s~es_temporal_{{nf.stack}})\n- [us-east-1](https://lumen-v2.prod.netflix.net/show/Elasticsearch?v.target=s~prod.us-east-1&v.nf.app=s~es_temporal_{{nf.stack}})"
                                },
                                "type": "text"
                            }
                        ]
                    }
                ],
                "title": "ElasticSearch"
            },
            {
                "subViews": [
                    {
                        "repeat": {
                            "config": {
                                "extend": "temporal-service-client",
                                "title": "{{cluster}}",
                                "vars": {
                                    "cluster": "{{cluster}}"
                                }
                            },
                            "values": [
                                "clouddriver-{{nf.stack}}-temporal",
                                "orca-{{nf.stack}}-temporal",
                                "chex-{{nf.stack}}"
                            ],
                            "varName": "cluster"
                        }
                    }
                ],
                "title": "Spinnaker Clients"
            },
            {
                "subViews": [
                    {
                        "repeat": {
                            "config": {
                                "extend": "temporal-service-client",
                                "title": "{{cluster}}",
                                "vars": {
                                    "cluster": "{{cluster}}"
                                }
                            },
                            "values": [
                                "billingfraudcancellation",
                                "spaascontroller-dev"
                            ],
                            "varName": "cluster"
                        }
                    }
                ],
                "title": "External Clients"
            }
        ]
    },
    "title": "Temporal"
}

temporal-internal-client.json

{
    "rows": [
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Requests vs Errors",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_client_requests,:eq,:and,service_role,{{client}},:eq,:and,:sum,0075dc,:color,requests,:legend,nf.cluster,{{cluster}},:eq,name,temporal_client_errors,:eq,:and,service_role,{{client}},:eq,:and,:sum,ff0000,:color,errors,:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Requests (by Operation)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_client_requests,:eq,:and,service_role,{{client}},:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Errors (by Operation)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_client_errors,:eq,:and,service_role,{{client}},:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Request Latency (by Operation)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_client_latency,:eq,:and,service_role,{{client}},:eq,:and,:dist-avg,(,operation,),:by,$(operation),:legend",
                        "scale": "log"
                    }
                }
            ]
        }
    ],
    "title": "Rename me"
}

temporal-service-client.json

{
    "rows": [
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Requests vs Errors",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_request,:eq,:and,:sum,nf.cluster,{{cluster}},:eq,name,temporal_request_failure,:eq,:and,:sum,ff0000,:color,1,:axis"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Long Requests vs Errors",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_long_request,:eq,:and,:sum,nf.cluster,{{cluster}},:eq,name,temporal_long_request_failure,:eq,:and,:sum,ff0000,:color,1,:axis"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Requests (by Operation)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_request,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Request Latency (by Operation)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_request_latency,:eq,:and,:dist-avg,(,operation,),:by,$(operation),:legend",
                        "scale": "log"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Long Requests (by Operation)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_long_request,:eq,:and,:sum,(,operation,),:by,$(operation),:legend"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Long Request Latency (by Operation)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_long_request_latency,:eq,:and,:dist-avg,(,operation,),:by,$(operation),:legend",
                        "scale": "log"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "options": {
                        "align": "center",
                        "content": "**Workflows**"
                    },
                    "type": "text"
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Active Workflow Threads (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_active_thread_count,:eq,:and,:sum,(,workflow_type,),:by,$(workflow_type),:legend"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Completed (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_completed,:eq,:and,:sum,(,workflow_type,),:by,nf.cluster,{{cluster}},:eq,name,temporal_workflow_failed,:eq,:and,:sum,(,workflow_type,),:by,ff0000,:color,1,:axis"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "End-to-end Latency (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_endtoend_latency,:eq,:and,:dist-avg,(,workflow_type,),:by,$(workflow_type),:legend",
                        "scale": "log"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "options": {
                        "align": "center",
                        "content": "**Workflow Tasks**"
                    },
                    "type": "text"
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Task Execution Latency (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_execution_latency,:eq,:and,:dist-avg,(,workflow_type,),:by,$(workflow_type),:legend"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Task Execution Total Latency (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_execution_total_latency,:eq,:and,:dist-avg,(,workflow_type,),:by,$(workflow_type),:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Task Replay Latency (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_replay_latency,:eq,:and,:dist-avg,(,workflow_type,),:by,$(workflow_type),:legend",
                        "scale": "log"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Task Schedule to Start Latency (by TaskQueue)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_schedule_to_start_latency,:eq,:and,:dist-avg,(,task_queue,),:by,$(task_queue),:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "options": {
                        "align": "center",
                        "content": "**Activities**"
                    },
                    "type": "text"
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Activity Poll No Task (by TaskQueue)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_activity_poll_no_task,:eq,:and,:sum,(,task_queue,),:by"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Activity Execution Latency (by ActivityType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_activity_execution_latency,:eq,:and,:dist-avg,(,activity_type,),:by,$(activity_type),:legend"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Activity End-to-end Latency (by ActivityType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_activity_endtoend_latency,:eq,:and,:dist-avg,(,activity_type,),:by,$(activity_type),:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Activity Schedule to Start Latency (by ActivityType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_activity_schedule_to_start_latency,:eq,:and,:dist-avg,(,activity_type,),:by,$(activity_type),:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "options": {
                        "align": "center",
                        "content": "**Task Queues**"
                    },
                    "type": "text"
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Workflow Task Polling Success vs Empty",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_queue_poll_succeed,:eq,:and,:sum,success,:legend,nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_queue_poll_empty,:eq,:and,:sum,1,:axis,empty,:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Task Poll Success (by TaskQueue)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_queue_poll_succeed,:eq,:and,:sum,(,task_queue,),:by,$(task_queue),:legend"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Task Poll Empty (by TaskQueue)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_workflow_task_queue_poll_empty,:eq,:and,:sum,(,task_queue,),:by,$(task_queue),:legend"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "options": {
                        "align": "center",
                        "content": "**Cache**"
                    },
                    "type": "text"
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Sticky Queue Cache Hit (by TaskQueue)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_sticky_cache_hit,:eq,:and,:sum,(,task_queue,),:by"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Sticky Queue Cache Hit (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_sticky_cache_hit,:eq,:and,:sum,(,workflow_type,),:by"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Sticky Queue Forced Eviction (by TaskQueue)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_sticky_cache_total_forced_eviction,:eq,:and,:sum,(,task_queue,),:by"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Sticky Queue Forced Eviction (by WorkflowType)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_sticky_cache_total_forced_eviction,:eq,:and,:sum,(,workflow_type,),:by"
                    }
                }
            ]
        }
    ],
    "title": "{{cluster}} temporal service client"
}

temporal-service-go.json

{
    "rows": [
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Num goroutines",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_num_goroutines,:eq,:and,:sum,(,nf.node,),:by"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "gomaxprocs",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_gomaxprocs,:eq,:and,:sum,(,nf.node,),:by"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "options": {
                        "align": "center",
                        "content": "**Memory**"
                    },
                    "type": "text"
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "Allocated",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_memory_allocated,:eq,:and,:sum,(,nf.node,),:by"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "Heap",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_memory_heap,:eq,:and,:sum,nf.cluster,{{cluster}},:eq,name,temporal_memory_heapidle,:eq,:and,:sum,nf.cluster,{{cluster}},:eq,name,temporal_memory_heapinuse,:eq,:and,:sum,(,nf.node,),:by"
                    }
                }
            ]
        },
        {
            "cells": [
                {
                    "options": {
                        "align": "center",
                        "content": "**GC**"
                    },
                    "type": "text"
                }
            ]
        },
        {
            "cells": [
                {
                    "extend": "atlas",
                    "title": "GC num collections",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_memory_num_gc,:eq,:and,:sum,(,nf.node,),:by,:stack"
                    }
                },
                {
                    "extend": "atlas",
                    "title": "GC Pause (ms)",
                    "vars": {
                        "q": "nf.cluster,{{cluster}},:eq,name,temporal_memory_gc_pause_ms,:eq,:and,:dist-avg,(,nf.node,),:by,:stack"
                    }
                }
            ]
        }
    ],
    "title": "Go Metrics"
}