rules/alerts/slo.yml

# Multi-window multi-burn-rate alerts from the SRE workbook.
# Availability objective 99.9% => budget 0.001.
# Fast page: 14.4x on (1h and 5m), or 6x on (6h and 30m).
# Slow ticket: 3x on (1d and 2h), or 1x on (3d and 6h).
groups:
  - name: slo.alerts
    rules:
      - alert: OrdersApiErrorBudgetBurnFast
        expr: |
          (
            slo:sli_error:ratio_rate1h{service="orders-api",slo="availability"} > (14.4 * 0.001)
            and
            slo:sli_error:ratio_rate5m{service="orders-api",slo="availability"} > (14.4 * 0.001)
          )
          or
          (
            slo:sli_error:ratio_rate6h{service="orders-api",slo="availability"} > (6 * 0.001)
            and
            slo:sli_error:ratio_rate30m{service="orders-api",slo="availability"} > (6 * 0.001)
          )
        for: 2m
        labels:
          severity: critical
          team: orders
          service: orders-api
          slo: availability
        annotations:
          summary: Orders API availability error budget is burning fast
          description: The orders-api availability SLI is above the fast multi-window burn threshold (14.4x over 5m and 1h, or 6x over 30m and 6h). Page the orders on-call.
          runbook_url: https://runbooks.larkspur.example/OrdersApiErrorBudgetBurnFast
      - alert: OrdersApiErrorBudgetBurnSlow
        expr: |
          (
            slo:sli_error:ratio_rate1d{service="orders-api",slo="availability"} > (3 * 0.001)
            and
            slo:sli_error:ratio_rate2h{service="orders-api",slo="availability"} > (3 * 0.001)
          )
          or
          (
            slo:sli_error:ratio_rate3d{service="orders-api",slo="availability"} > (1 * 0.001)
            and
            slo:sli_error:ratio_rate6h{service="orders-api",slo="availability"} > (1 * 0.001)
          )
        for: 2m
        labels:
          severity: warning
          team: orders
          service: orders-api
          slo: availability
        annotations:
          summary: Orders API availability error budget is burning slowly
          description: The orders-api availability SLI is above the slow multi-window burn threshold (3x over 2h and 1d, or 1x over 6h and 3d). Open a ticket for the orders team.
          runbook_url: https://runbooks.larkspur.example/OrdersApiErrorBudgetBurnSlow
      - alert: OrdersApiLatencySLOBurn
        expr: |
          (
            slo:sli_latency_error:ratio_rate1h{service="orders-api",slo="latency"} > (14.4 * 0.01)
            and
            slo:sli_latency_error:ratio_rate5m{service="orders-api",slo="latency"} > (14.4 * 0.01)
          )
          or
          (
            slo:sli_latency_error:ratio_rate6h{service="orders-api",slo="latency"} > (6 * 0.01)
            and
            slo:sli_latency_error:ratio_rate30m{service="orders-api",slo="latency"} > (6 * 0.01)
          )
        for: 2m
        labels:
          severity: warning
          team: orders
          service: orders-api
          slo: latency
        annotations:
          summary: Orders API latency SLI is burning its error budget
          description: The share of orders-api requests slower than 300ms is above the fast latency burn threshold (14.4x over 5m and 1h, or 6x over 30m and 6h). The latency objective is 99 percent.
          runbook_url: https://runbooks.larkspur.example/OrdersApiLatencySLOBurn