Trace agent monitoring

agent-dashboard.json

{"title":"Trace Agent Monitoring (Datadog's Dashboard)","description":"","widgets":[{"id":1534558304603436,"definition":{"type":"group","layout_type":"ordered","widgets":[{"id":1003543843659836,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.receiver.payload_accepted{$host,$env}.as_count()","display_type":"bars","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Payloads accepted"}},{"id":3157905885946094,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.receiver.traces_bytes{$host,$env} by {host}.as_count()","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Received: bytes"}},{"id":7115587156480682,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.trace_writer.bytes{$host,$env} by {host}.as_count()","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Written: bytes"}},{"id":5655774243827630,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.receiver.spans_received{$host,$env}.as_count()","display_type":"bars","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Received: spans"}},{"id":6040322183970256,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.trace_writer.spans{$host,$env}.as_count()","display_type":"bars","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Written: spans"}},{"id":758316317574066,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.receiver.traces_received{$host,$env}.as_count()","display_type":"bars","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Received: traces"}},{"id":8734966132804986,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.trace_writer.traces{$host,$env}.as_count()","display_type":"bars","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Written: traces"}}],"title":"Load"}},{"id":6551042109585256,"definition":{"type":"group","layout_type":"ordered","widgets":[{"id":6101292230200876,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.heap_alloc{$host,$env} by {host}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Heap allocations"}},{"id":4049106969384328,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.cpu_percent{$host,$env} by {host}/100","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"CPU cores"}}],"title":"Resource usage"}},{"id":6798412738232492,"definition":{"type":"group","layout_type":"ordered","widgets":[{"id":4924109004684112,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.receiver.payload_refused{$host,$env} by {host}.as_count()","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Payloads refused"}},{"id":4925007664066574,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.receiver.traces_dropped{$host,$env}.as_count()","display_type":"bars","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Error normalizing: traces dropped"}},{"id":7107582827733872,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.receiver.spans_dropped{$host,$env}.as_count()","display_type":"bars","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Error normalizing: spans dropped"}},{"id":7638958344104050,"definition":{"type":"timeseries","requests":[{"q":"sum:datadog.trace_agent.presampler_rate{$host,$env} by {host}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Presampler rate"}}],"title":"Rate limiting and loss"}},{"id":3655287520061242,"definition":{"type":"group","layout_type":"ordered","widgets":[{"id":7778498994136742,"definition":{"type":"timeseries","requests":[{"q":"avg:datadog.trace_agent.internal.normalize_ms.avg{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}},{"q":"max:datadog.trace_agent.internal.normalize_ms.max{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Normalization (Avg & Max)"}},{"id":698281756010030,"definition":{"type":"timeseries","requests":[{"q":"avg:datadog.trace_agent.internal.process_trace_ms.avg{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}},{"q":"max:datadog.trace_agent.internal.process_trace_ms.max{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Process Trace (Avg & Max)"}},{"id":5022162017269942,"definition":{"type":"timeseries","requests":[{"q":"avg:datadog.trace_agent.internal.concentrator_ms.avg{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}},{"q":"max:datadog.trace_agent.internal.concentrator_ms.max{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Concentrator (Avg & Max)"}},{"id":1291787453378676,"definition":{"type":"timeseries","requests":[{"q":"avg:datadog.trace_agent.internal.sample_ms.avg{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}},{"q":"max:datadog.trace_agent.internal.sample_ms.max{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Sampler (Avg & Max)"}},{"id":3603713768376558,"definition":{"type":"timeseries","requests":[{"q":"avg:datadog.trace_agent.trace_writer.encode_ms.avg{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}},{"q":"max:datadog.trace_agent.trace_writer.encode_ms.max{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"datadog.trace_agent.trace_writer.encode_ms.avg, datadog.trace_agent.trace_wri..."}},{"id":2946375113483772,"definition":{"type":"timeseries","requests":[{"q":"max:datadog.trace_agent.trace_writer.flush_duration{$host,$env}","display_type":"line","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"}}],"title":"Flush"}}],"title":"Pipeline durations (ms)"}}],"template_variables":[{"name":"host","default":"*","prefix":"host"},{"name":"env","default":"*","prefix":"env"}],"layout_type":"ordered","is_read_only":false,"notify_list":[],"dashboard_id":"q2m-3cz-2hq"}