Guides

Map-Reduce Pattern


Map-Reduce Pattern

The map-reduce pattern splits a large dataset into smaller batches, processes each batch as a separate child workflow (the "map" phase), and then aggregates the results in the parent (the "reduce" phase). This distributes work across multiple workers for parallel processing while maintaining a single parent that tracks overall progress and handles partial failures.

1,000 Records
large dataset
Split into Batches
chunk_size = 100
map — parallel child workflows
1–100
child wf
101–200
child wf
201–300
child wf
...
901–1k
child wf
reduce — aggregate in parent
Merge Results
combine · deduplicate · summarize
Final Result
1
use orcher::prelude::*;
2
use anyhow::Result;
3
4
// Child workflow - processes a batch of records
5
#[workflow(name = "process_batch", version = "1.0.0")]
6
pub async fn process_batch(
7
ctx: WorkflowContext,
8
batch: Vec<Record>,
9
) -> Result<BatchResult> {
10
let mut processed = 0;
11
let mut failed = 0;
12
13
for record in batch {
14
match ctx.execute_task(process_record, record).await {
15
Ok(_) => processed += 1,
16
Err(_) => failed += 1,
17
}
18
}
19
20
Ok(BatchResult { processed, failed })
21
}
22
23
// Parent workflow - map-reduce coordinator
24
#[workflow(name = "process_dataset", version = "1.0.0")]
25
pub async fn process_dataset(
26
ctx: WorkflowContext,
27
records: Vec<Record>,
28
) -> Result<DatasetResult> {
29
const BATCH_SIZE: usize = 100;
30
31
// Map: Split into batches and spawn child workflows
32
let batches: Vec<Vec<Record>> = records
33
.chunks(BATCH_SIZE)
34
.map(|chunk| chunk.to_vec())
35
.collect();
36
37
println!("Processing {} batches", batches.len());
38
39
let mut handles = Vec::new();
40
for (i, batch) in batches.into_iter().enumerate() {
41
let handle = ctx
42
.start_child_workflow("process_batch", batch)
43
.await?;
44
45
handles.push((i, handle));
46
}
47
48
// Reduce: Collect results from all batches
49
let mut total_processed = 0;
50
let mut total_failed = 0;
51
52
for (batch_num, handle) in handles {
53
match handle.result::<BatchResult>().await {
54
Ok(result) => {
55
println!("Batch {} complete: {} processed, {} failed",
56
batch_num, result.processed, result.failed);
57
total_processed += result.processed;
58
total_failed += result.failed;
59
}
60
Err(e) => {
61
eprintln!("Batch {} failed: {}", batch_num, e);
62
total_failed += BATCH_SIZE;
63
}
64
}
65
}
66
67
Ok(DatasetResult {
68
total: records.len(),
69
processed: total_processed,
70
failed: total_failed,
71
})
72
}

Next Steps