Skip to content

Instantly share code, notes, and snippets.

@hagen1778
Created January 30, 2023 14:12
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save hagen1778/d560ee21be02dd9a3a4cb9d664428660 to your computer and use it in GitHub Desktop.
Code generated by Sloth for vmalert's replay blogpost
---
# Code generated by Sloth (v0.11.0): https://github.com/slok/sloth.
# DO NOT EDIT.
groups:
- name: sloth-slo-sli-recordings-sandbox-vmcluster-requests-availability
rules:
- record: slo:sli_error:ratio_rate5m
expr: |
(sum(rate(vm_http_request_errors_total{job="vmselect-benchmark-vm-cluster"}[5m])))
/
(sum(rate(vm_http_requests_total{job="vmselect-benchmark-vm-cluster"}[5m])))
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 5m
- record: slo:sli_error:ratio_rate30m
expr: |
(sum(rate(vm_http_request_errors_total{job="vmselect-benchmark-vm-cluster"}[30m])))
/
(sum(rate(vm_http_requests_total{job="vmselect-benchmark-vm-cluster"}[30m])))
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 30m
- record: slo:sli_error:ratio_rate1h
expr: |
(sum(rate(vm_http_request_errors_total{job="vmselect-benchmark-vm-cluster"}[1h])))
/
(sum(rate(vm_http_requests_total{job="vmselect-benchmark-vm-cluster"}[1h])))
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 1h
- record: slo:sli_error:ratio_rate2h
expr: |
(sum(rate(vm_http_request_errors_total{job="vmselect-benchmark-vm-cluster"}[2h])))
/
(sum(rate(vm_http_requests_total{job="vmselect-benchmark-vm-cluster"}[2h])))
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 2h
- record: slo:sli_error:ratio_rate6h
expr: |
(sum(rate(vm_http_request_errors_total{job="vmselect-benchmark-vm-cluster"}[6h])))
/
(sum(rate(vm_http_requests_total{job="vmselect-benchmark-vm-cluster"}[6h])))
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 6h
- record: slo:sli_error:ratio_rate1d
expr: |
(sum(rate(vm_http_request_errors_total{job="vmselect-benchmark-vm-cluster"}[1d])))
/
(sum(rate(vm_http_requests_total{job="vmselect-benchmark-vm-cluster"}[1d])))
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 1d
- record: slo:sli_error:ratio_rate3d
expr: |
(sum(rate(vm_http_request_errors_total{job="vmselect-benchmark-vm-cluster"}[3d])))
/
(sum(rate(vm_http_requests_total{job="vmselect-benchmark-vm-cluster"}[3d])))
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 3d
- record: slo:sli_error:ratio_rate30d
expr: |
sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"}[30d])
/ ignoring (sloth_window)
count_over_time(slo:sli_error:ratio_rate5m{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"}[30d])
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_window: 30d
- name: sloth-slo-meta-recordings-sandbox-vmcluster-requests-availability
rules:
- record: slo:objective:ratio
expr: vector(0.9990000000000001)
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
- record: slo:error_budget:ratio
expr: vector(1-0.9990000000000001)
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
- record: slo:time_period:days
expr: vector(30)
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
- record: slo:current_burn_rate:ratio
expr: |
slo:sli_error:ratio_rate5m{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"}
/ on(sloth_id, sloth_slo, sloth_service) group_left
slo:error_budget:ratio{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"}
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
- record: slo:period_burn_rate:ratio
expr: |
slo:sli_error:ratio_rate30d{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"}
/ on(sloth_id, sloth_slo, sloth_service) group_left
slo:error_budget:ratio{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"}
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
- record: slo:period_error_budget_remaining:ratio
expr: 1 - slo:period_burn_rate:ratio{sloth_id="sandbox-vmcluster-requests-availability",
sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"}
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
- record: sloth_slo_info
expr: vector(1)
labels:
sloth_id: sandbox-vmcluster-requests-availability
sloth_mode: cli-gen-prom
sloth_objective: "99.9"
sloth_service: sandbox-vmcluster
sloth_slo: requests-availability
sloth_spec: prometheus/v1
sloth_version: v0.11.0
- name: sloth-slo-alerts-sandbox-vmcluster-requests-availability
rules:
- alert: VMHighErrorRate
expr: |
(
max(slo:sli_error:ratio_rate5m{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
and
max(slo:sli_error:ratio_rate1h{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (14.4 * 0.0009999999999999432)) without (sloth_window)
)
or
(
max(slo:sli_error:ratio_rate30m{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (6 * 0.0009999999999999432)) without (sloth_window)
and
max(slo:sli_error:ratio_rate6h{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (6 * 0.0009999999999999432)) without (sloth_window)
)
labels:
sloth_severity: page
annotations:
summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn
rate is over expected.'
title: (page) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
burn rate is too fast.
- alert: VMHighErrorRate
expr: |
(
max(slo:sli_error:ratio_rate2h{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (3 * 0.0009999999999999432)) without (sloth_window)
and
max(slo:sli_error:ratio_rate1d{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (3 * 0.0009999999999999432)) without (sloth_window)
)
or
(
max(slo:sli_error:ratio_rate6h{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (1 * 0.0009999999999999432)) without (sloth_window)
and
max(slo:sli_error:ratio_rate3d{sloth_id="sandbox-vmcluster-requests-availability", sloth_service="sandbox-vmcluster", sloth_slo="requests-availability"} > (1 * 0.0009999999999999432)) without (sloth_window)
)
labels:
sloth_severity: ticket
annotations:
summary: '{{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget burn
rate is over expected.'
title: (ticket) {{$labels.sloth_service}} {{$labels.sloth_slo}} SLO error budget
burn rate is too fast.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment