Operations

Alerting


1
groups:
2
- name: orcher
3
interval: 30s
4
rules:
5
# High workflow failure rate
6
- alert: HighWorkflowFailureRate
7
expr: |
8
sum(rate(orcher_workflows_failed_total[5m]))
9
/
10
(sum(rate(orcher_workflows_completed_total[5m])) + sum(rate(orcher_workflows_failed_total[5m])))
11
> 0.05
12
for: 5m
13
labels:
14
severity: warning
15
annotations:
16
summary: "Workflow failure rate above 5%"
17
18
# Queue depth growing
19
- alert: QueueDepthHigh
20
expr: orcher_queue_depth > 1000
21
for: 5m
22
labels:
23
severity: warning
24
annotations:
25
summary: "Queue {{ $labels.queue }} has {{ $value }} pending items"
26
27
# Database pool near exhaustion
28
- alert: DBPoolExhausted
29
expr: orcher_db_pool_utilization_ratio > 0.90
30
for: 5m
31
labels:
32
severity: critical
33
annotations:
34
summary: "Database pool at {{ $value | humanizePercentage }} utilization"
35
36
# Journal cache hit ratio low
37
- alert: JournalCacheHitRateLow
38
expr: orcher_journal_cache_hit_ratio < 0.80
39
for: 15m
40
labels:
41
severity: warning
42
annotations:
43
summary: "Journal cache hit ratio is {{ $value | humanizePercentage }}"
44
45
# DLQ growing
46
- alert: DLQGrowing
47
expr: orcher_dlq_current_depth > 100
48
for: 10m
49
labels:
50
severity: warning
51
annotations:
52
summary: "Dead letter queue has {{ $value }} entries"
53
54
# PostgreSQL fallback active (Redis down)
55
- alert: RedisUnavailable
56
expr: rate(orcher_postgres_fallback_total[5m]) > 0
57
for: 5m
58
labels:
59
severity: warning
60
annotations:
61
summary: "Workers falling back to PostgreSQL polling (Redis may be down)"

Next Steps