rules/alerts/service.yml

groups:
  - name: service.alerts
    rules:
      - alert: ServiceHighErrorRate
        expr: |
          service:http_requests:rate5m > 0
          and
          (
            service:http_requests_errors:rate5m
            /
            service:http_requests:rate5m
          ) > 0.05
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: HTTP 5xx ratio is above 5 percent
          description: More than 5 percent of HTTP requests for a service have been 5xx over 5 minutes, and the condition has held for 10 minutes.
          runbook_url: https://runbooks.larkspur.example/ServiceHighErrorRate
      - alert: ServiceLatencyP95High
        expr: |
          max by (service) (service_route:http_request_duration_seconds:p95_5m)
          > on (service)
          service:http_latency_p95_threshold_seconds:constant
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: HTTP p95 latency is above the service threshold
          description: The worst route p95 for a service is above the constant recorded in service:http_latency_p95_threshold_seconds:constant.
          runbook_url: https://runbooks.larkspur.example/ServiceLatencyP95High
      - alert: PaymentsQueueBacklog
        expr: |
          service:payments_queue_oldest_age_seconds:max > 300
          and
          max by (service) (deriv(payments_queue_depth[10m])) > 0
        for: 5m
        labels:
          severity: warning
          team: payments
        annotations:
          summary: Payments queue oldest message is over 5 minutes and depth is rising
          description: The oldest payments message is older than 300 seconds and payments_queue_depth is increasing. Consumers are not keeping up.
          runbook_url: https://runbooks.larkspur.example/PaymentsQueueBacklog