aws cloudwatch sagemaker inference dashboard for endpoints

sagemaker_inference_infra_metrics.json

{
    "widgets": [
        {
            "height": 6,
            "width": 24,
            "y": 0,
            "x": 0,
            "type": "metric",
            "properties": {
                "view": "timeSeries",
                "stacked": false,
                "metrics": [
                    [ "/aws/sagemaker/Endpoints", "CPUUtilization", "EndpointName", "<endpoint_name>", "VariantName", "<variant_name>" ],
                    [ "/aws/sagemaker/Endpoints", "MemoryUtilization", "EndpointName", "<endpoint_name>", "VariantName", "<variant_name>" ],
                    [ "/aws/sagemaker/Endpoints", "DiskUtilization", "EndpointName", "<endpoint_name>", "VariantName", "<variant_name>" ]
                ],
                "region": "ap-southeast-1",
                "title": "CPU,Memory,Disk Utilization per Second",
                "period": 60,
                "stat": "Average",
                "yAxis": {
                    "right": {
                        "label": ""
                    },
                    "left": {
                        "label": "CPU,Memory,Disk Utilization per Second",
                        "showUnits": false
                    }
                }
            }
        },
        {
            "height": 6,
            "width": 24,
            "y": 6,
            "x": 0,
            "type": "metric",
            "properties": {
                "view": "timeSeries",
                "stacked": false,
                "metrics": [
                    [ { "expression": "m1 / PERIOD(m1)", "label": "Invocations per Second", "id": "e1" } ],
                    [ "AWS/SageMaker", "Invocations", "EndpointName", "<endpoint_name>", "VariantName", "<variant_name>", { "region": "ap-southeast-1", "id": "m1", "visible": false } ]
                ],
                "region": "ap-southeast-1",
                "title": "Invocations per Second",
                "period": 60,
                "stat": "Sum",
                "yAxis": {
                    "right": {
                        "label": ""
                    },
                    "left": {
                        "label": "Invocations per Second",
                        "showUnits": false
                    }
                }
            }
        },
        {
            "height": 6,
            "width": 24,
            "y": 12,
            "x": 0,
            "type": "metric",
            "properties": {
                "view": "timeSeries",
                "stacked": false,
                "metrics": [
                    [ "AWS/SageMaker", "ModelLatency", "EndpointName", "<endpoint_name>", "VariantName", "<variant_name>" ],
                    [ "AWS/SageMaker", "OverheadLatency", "EndpointName", "<endpoint_name>", "VariantName", "<variant_name>" ]
                ],
                "region": "ap-southeast-1",
                "title": "Model,Overhead Latency",
                "period": 60,
                "stat": "p95",
                "yAxis": {
                    "right": {
                        "label": ""
                    },
                    "left": {
                        "label": "Model,Overhead Latency",
                        "showUnits": false
                    }
                }
            }
        },
        {
            "height": 6,
            "width": 24,
            "y": 18,
            "x": 0,
            "type": "metric",
            "properties": {
                "view": "timeSeries",
                "stacked": false,
                "metrics": [
                    [ "AWS/SageMaker", "Invocation5XXErrors", "EndpointName", "<endpoint_name>", "VariantName", "<variant_name>" ]
                ],
                "region": "ap-southeast-1",
                "title": "Invocation 5XX Errors",
                "period": 60,
                "stat": "Sum",
                "yAxis": {
                    "right": {
                        "label": ""
                    },
                    "left": {
                        "label": "Invocation 5XX Errors",
                        "showUnits": false
                    }
                }
            }
        }
    ]
}