Guides

Dead Letter Queue


Dead Letter Queue

When a task exhausts all its retries and still fails, you need a way to capture these persistent failures for manual investigation rather than silently dropping them. A dead letter queue (DLQ) pattern routes failed jobs to a separate storage location along with error context, retry history, and timestamps. Operations teams can then inspect, fix, and replay these jobs. This pattern is essential for any production workflow where data loss is unacceptable.

1
use orcher::prelude::*;
2
use anyhow::Result;
3
4
#[workflow(name = "dlq_workflow", version = "1.0.0")]
5
pub async fn dlq_workflow(
6
ctx: WorkflowContext,
7
job: Job,
8
) -> Result<JobResult> {
9
// Try to process with retry
10
let result = ctx
11
.execute_task(process_job, job.clone())
12
.await;
13
14
match result {
15
Ok(output) => {
16
Ok(JobResult {
17
status: "completed".to_string(),
18
output: Some(output),
19
})
20
}
21
Err(e) if job.retry_count >= job.max_retries => {
22
// Exceeded max retries - move to DLQ
23
tracing::error!(
24
"Job {} exceeded max retries, moving to DLQ",
25
job.id
26
);
27
28
ctx.execute_task(move_to_dlq, (job, e.to_string()))
29
.await?;
30
31
Ok(JobResult {
32
status: "moved_to_dlq".to_string(),
33
output: None,
34
})
35
}
36
Err(e) => {
37
// Will retry
38
Err(e)
39
}
40
}
41
}
42
43
#[task(name = "move_to_dlq")]
44
pub async fn move_to_dlq(
45
_ctx: TaskContext,
46
input: (Job, String),
47
) -> Result<()> {
48
let (job, error) = input;
49
50
// Store in DLQ for manual inspection
51
let dlq_entry = DLQEntry {
52
job_id: job.id,
53
payload: job.payload,
54
error_message: error,
55
retry_count: job.retry_count,
56
failed_at: chrono::Utc::now(),
57
};
58
59
// Save to database or queue
60
dlq_storage::save(dlq_entry).await?;
61
62
// Alert operations team
63
alerting::send_alert("Job moved to DLQ", &job.id).await?;
64
65
Ok(())
66
}

Next Steps