Local Development

Testing Workflows Locally


Testing workflows is critical for ensuring reliability in production. Orcher provides tools for unit testing with mocks, integration testing against a local orchestrator, and simulating crash recovery scenarios.

Choose Your SDK

Prerequisites: • Rust 1.70 or later • Test framework: Built-in cargo test • Mock library: mockito or wiremock • Local Orcher server for integration tests


Testing Philosophy

When testing workflows, you should focus on:

  1. Unit Tests - Test workflow logic in isolation with mocked dependencies
  2. Integration Tests - Test against a real orchestrator with real services
  3. Crash Recovery Tests - Verify workflows resume correctly after failures
  4. End-to-End Tests - Test complete user scenarios

Why test workflows?

• Catch bugs before production • Verify durability guarantees • Ensure idempotency of operations • Document expected behavior • Enable confident refactoring


Unit Testing Workflows

Unit tests verify workflow logic without running the orchestrator.

Basic Unit Test

1
#[cfg(test)]
2
mod tests {
3
use super::*;
4
use orcher::testing::TestEnv;
5
6
#[tokio::test]
7
async fn test_payment_workflow_success() {
8
// Create the in-process test environment
9
let mut env = TestEnv::new();
10
11
// Register the workflow under test
12
env.register_workflow("payment_workflow", payment_workflow);
13
14
// Mock each task the workflow calls
15
env.mock_task("charge_card").returns(PaymentResult {
16
charge_id: "ch_123".to_string(),
17
amount: 99.99,
18
});
19
env.mock_task("reserve_inventory").returns(InventoryResult {
20
reservation_id: "inv_456".to_string(),
21
});
22
env.mock_task("send_confirmation").returns(NotificationResult {
23
message_id: "msg_789".to_string(),
24
});
25
26
// Execute the workflow
27
let input = PaymentInput {
28
order_id: "ord_123".to_string(),
29
amount: 99.99,
30
card_token: "tok_visa".to_string(),
31
customer_email: "test@example.com".to_string(),
32
};
33
34
let output: PaymentOutput = env
35
.execute_workflow("payment_workflow", input)
36
.await
37
.unwrap();
38
39
// Assert success
40
assert_eq!(output.charge_id, "ch_123");
41
env.assert_task_called("charge_card", 1);
42
}
43
}

Testing Error Scenarios

1
#[tokio::test]
2
async fn test_payment_workflow_payment_failure() {
3
let mut env = TestEnv::new();
4
env.register_workflow("payment_workflow", payment_workflow);
5
6
// Mock a payment failure
7
env.mock_task("charge_card").fails_with("Card declined");
8
9
let input = PaymentInput {
10
order_id: "ord_123".to_string(),
11
amount: 99.99,
12
card_token: "tok_declined".to_string(),
13
customer_email: "test@example.com".to_string(),
14
};
15
16
let result: orcher::Result<PaymentOutput> =
17
env.execute_workflow("payment_workflow", input).await;
18
19
// Assert failure
20
assert!(result.is_err());
21
assert!(result.unwrap_err().to_string().contains("Card declined"));
22
}

Testing with Fixtures

1
// tests/fixtures/mod.rs
2
pub struct PaymentFixtures;
3
4
impl PaymentFixtures {
5
pub fn valid_input() -> PaymentInput {
6
PaymentInput {
7
order_id: "ord_123".to_string(),
8
amount: 99.99,
9
card_token: "tok_visa".to_string(),
10
customer_email: "test@example.com".to_string(),
11
}
12
}
13
14
pub fn declined_card_input() -> PaymentInput {
15
PaymentInput {
16
order_id: "ord_456".to_string(),
17
amount: 199.99,
18
card_token: "tok_declined".to_string(),
19
customer_email: "test2@example.com".to_string(),
20
}
21
}
22
}
23
24
// Use in tests
25
#[tokio::test]
26
async fn test_with_fixtures() {
27
let input = PaymentFixtures::valid_input();
28
// ... test logic
29
}

Integration Testing

Integration tests run against a real Orcher orchestrator with actual services.

Setup Test Environment

1
// tests/integration_test.rs
2
use orcher::{Client, Worker, WorkerBuilder};
3
use testcontainers::{clients, images};
4
5
struct TestEnv {
6
client: Client,
7
_worker: Worker,
8
_orchestrator: Container,
9
}
10
11
async fn setup_test_env() -> TestEnv {
12
// Start Orcher orchestrator container
13
let docker = clients::Cli::default();
14
let orchestrator = docker.run(images::generic::GenericImage::new(
15
"orcherapp/orcher",
16
"latest"
17
));
18
19
let orchestrator_url = format!(
20
"http://localhost:{}",
21
orchestrator.get_host_port_ipv4(8080)
22
);
23
24
// Start worker with test workflows.
25
// Workflows and tasks are auto-registered from the #[workflow]/#[task] macros.
26
let worker = WorkerBuilder::new()
27
.server_url(&orchestrator_url)
28
.task_queue("default")
29
.build()
30
.await
31
.unwrap();
32
33
tokio::spawn(async move {
34
worker.run().await
35
});
36
37
// Create client
38
let client = Client::connect(&orchestrator_url).await.unwrap();
39
40
// Wait for worker to be ready
41
tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
42
43
TestEnv {
44
client,
45
_worker: worker,
46
_orchestrator: orchestrator,
47
}
48
}
49
50
#[tokio::test]
51
async fn test_payment_workflow_integration() {
52
let env = setup_test_env().await;
53
54
// Start workflow
55
let handle = env.client
56
.start_workflow("payment_workflow", PaymentInput {
57
order_id: "ord_integration_123".to_string(),
58
amount: 99.99,
59
card_token: "tok_visa".to_string(),
60
customer_email: "integration@example.com".to_string(),
61
})
62
.await
63
.unwrap();
64
65
// Wait for result
66
let result: PaymentOutput = handle.result().await.unwrap();
67
68
// Assert
69
assert_eq!(result.status, "completed");
70
assert!(result.charge_id.starts_with("ch_"));
71
}

Testing with Mock Services

1
use wiremock::{MockServer, Mock, ResponseTemplate};
2
use wiremock::matchers::{method, path};
3
4
#[tokio::test]
5
async fn test_with_mock_external_api() {
6
// Start mock HTTP server
7
let mock_server = MockServer::start().await;
8
9
// Mock Stripe API
10
Mock::given(method("POST"))
11
.and(path("/v1/charges"))
12
.respond_with(ResponseTemplate::new(200)
13
.set_body_json(json!({
14
"id": "ch_mock_123",
15
"amount": 9999,
16
"currency": "usd",
17
"status": "succeeded"
18
})))
19
.mount(&mock_server)
20
.await;
21
22
// Configure workflow to use mock server
23
std::env::set_var("STRIPE_API_URL", mock_server.uri());
24
25
let env = setup_test_env().await;
26
27
// Run workflow
28
let handle = env.client
29
.start_workflow("payment_workflow", PaymentInput {
30
order_id: "ord_mock_123".to_string(),
31
amount: 99.99,
32
card_token: "tok_visa".to_string(),
33
customer_email: "mock@example.com".to_string(),
34
})
35
.await
36
.unwrap();
37
38
let result: PaymentOutput = handle.result().await.unwrap();
39
assert_eq!(result.charge_id, "ch_mock_123");
40
}

Crash Recovery Testing

Test that workflows resume correctly after crashes.

Simulated Crash Test

1
#[tokio::test]
2
async fn test_workflow_survives_crash() {
3
let env = setup_test_env().await;
4
5
// Start workflow
6
let handle = env.client
7
.start_workflow("payment_workflow", PaymentInput {
8
order_id: "ord_crash_test".to_string(),
9
amount: 99.99,
10
card_token: "tok_visa".to_string(),
11
customer_email: "crash@example.com".to_string(),
12
})
13
.await
14
.unwrap();
15
16
// Wait for first step to complete
17
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
18
19
// Simulate crash by killing worker (shutdown() is synchronous)
20
env._worker.shutdown();
21
22
// Wait a moment
23
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
24
25
// Restart worker (workflows/tasks are auto-registered from the macros)
26
let new_worker = WorkerBuilder::new()
27
.server_url(&env.orchestrator_url)
28
.task_queue("default")
29
.build()
30
.await
31
.unwrap();
32
33
tokio::spawn(async move {
34
new_worker.run().await
35
});
36
37
// Wait for workflow to resume and complete
38
let result: PaymentOutput = handle.result().await.unwrap();
39
40
// Verify workflow completed successfully
41
assert_eq!(result.status, "completed");
42
}

Verify No Duplicate Operations

1
use std::sync::{Arc, Mutex};
2
3
#[tokio::test]
4
async fn test_no_duplicate_charges() {
5
// Track API calls
6
let call_count = Arc::new(Mutex::new(0));
7
let call_count_clone = call_count.clone();
8
9
// Mock server that counts calls
10
let mock_server = MockServer::start().await;
11
Mock::given(method("POST"))
12
.and(path("/v1/charges"))
13
.respond_with(move |_req: &wiremock::Request| {
14
let mut count = call_count_clone.lock().unwrap();
15
*count += 1;
16
17
ResponseTemplate::new(200).set_body_json(json!({
18
"id": "ch_123",
19
"amount": 9999,
20
"status": "succeeded"
21
}))
22
})
23
.mount(&mock_server)
24
.await;
25
26
std::env::set_var("STRIPE_API_URL", mock_server.uri());
27
28
let env = setup_test_env().await;
29
30
// Start workflow
31
let handle = env.client
32
.start_workflow("payment_workflow", PaymentInput {
33
order_id: "ord_no_dup".to_string(),
34
amount: 99.99,
35
card_token: "tok_visa".to_string(),
36
customer_email: "nodup@example.com".to_string(),
37
})
38
.await
39
.unwrap();
40
41
// Simulate crash and restart (shutdown() is synchronous)
42
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
43
env._worker.shutdown();
44
45
let new_worker = restart_worker(&env).await;
46
47
// Wait for completion
48
let _result: PaymentOutput = handle.result().await.unwrap();
49
50
// Verify API was called exactly once
51
let final_count = *call_count.lock().unwrap();
52
assert_eq!(final_count, 1, "Stripe API should be called exactly once");
53
}

Testing Replay from Journal

1
#[tokio::test]
2
async fn test_replay_from_journal() {
3
let env = setup_test_env().await;
4
5
// Start workflow and wait for first step
6
let handle = env.client
7
.start_workflow("multi_step_workflow", MultiStepInput {
8
steps: vec!["step1", "step2", "step3"],
9
})
10
.await
11
.unwrap();
12
13
// Wait for step 1 to complete
14
tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
15
16
// Check journal has step 1
17
let journal = env.client
18
.get_workflow_journal(handle.id())
19
.await
20
.unwrap();
21
assert_eq!(journal.entries.len(), 1);
22
assert_eq!(journal.entries[0].step_name, "step1");
23
24
// Kill worker (shutdown() is synchronous)
25
env._worker.shutdown();
26
27
// Restart worker
28
let _new_worker = restart_worker(&env).await;
29
30
// Wait for completion
31
let result: MultiStepOutput = handle.result().await.unwrap();
32
33
// Verify all steps completed
34
assert_eq!(result.completed_steps.len(), 3);
35
36
// Check journal has all steps
37
let final_journal = env.client
38
.get_workflow_journal(handle.id())
39
.await
40
.unwrap();
41
assert_eq!(final_journal.entries.len(), 3);
42
}

Testing with Examples

Use Orcher's example projects as test templates:

Running Example Tests

The Rust SDK's examples are built and tested against the SDK on every change:

1
git clone https://github.com/orcher-io/sdk-rust
2
cd sdk-rust
3
4
# Run the durable_execution example's tests, which need no engine
5
cargo test -p orcher-example-durable-execution
6
7
# Run it against a local engine: a worker and a client in one process
8
cargo run -p orcher-example-durable-execution -- --e2e

Adapting Examples to Your Needs

examples/durable_execution/src/workflows/payment.rs is a template for a workflow that calls tasks. Each step is a #[task], and the workflow calls it with ctx.execute_task:

1
use orcher_sdk::prelude::*;
2
3
#[workflow(name = "my_workflow")]
4
async fn my_workflow(ctx: WorkflowContext, input: MyInput) -> Result<MyOutput> {
5
let step1: Step1 = ctx.execute_task(my_step_1, input).await?;
6
let step2: Step2 = ctx.execute_task(my_step_2, step1.clone()).await?;
7
Ok(MyOutput { step1, step2 })
8
}

Its test runs the workflow in memory with each task mocked, using TestWorkflowExecutor; see Testing.


Testing Best Practices

1. Use Descriptive Test Names

1
// Bad
2
#[tokio::test]
3
async fn test1() { ... }
4
5
// Good
6
#[tokio::test]
7
async fn test_payment_workflow_succeeds_with_valid_card() { ... }
8
9
#[tokio::test]
10
async fn test_payment_workflow_fails_with_declined_card() { ... }
11
12
#[tokio::test]
13
async fn test_payment_workflow_resumes_after_crash() { ... }

2. Test Edge Cases

1
#[tokio::test]
2
async fn test_workflow_with_empty_input() { ... }
3
4
#[tokio::test]
5
async fn test_workflow_with_maximum_input_size() { ... }
6
7
#[tokio::test]
8
async fn test_workflow_with_invalid_data() { ... }
9
10
#[tokio::test]
11
async fn test_workflow_timeout() { ... }
12
13
#[tokio::test]
14
async fn test_workflow_cancellation() { ... }

3. Isolate Tests

1
// Use separate database per test
2
async fn setup_isolated_env(test_name: &str) -> TestEnv {
3
let db_name = format!("test_db_{}", test_name);
4
5
// Create isolated database
6
create_test_database(&db_name).await;
7
8
// Start orchestrator with isolated DB
9
start_orchestrator_with_db(&db_name).await
10
}
11
12
#[tokio::test]
13
async fn test_isolated() {
14
let env = setup_isolated_env("test_isolated").await;
15
// Test runs in complete isolation
16
}

4. Clean Up Resources

1
#[tokio::test]
2
async fn test_with_cleanup() {
3
let env = setup_test_env().await;
4
5
// Test logic
6
let result = run_test(&env).await;
7
8
// Always cleanup
9
cleanup(&env).await;
10
11
assert!(result.is_ok());
12
}
13
14
// Or use RAII pattern
15
impl Drop for TestEnv {
16
fn drop(&mut self) {
17
// Cleanup automatically
18
self.cleanup_sync();
19
}
20
}

5. Mock External Dependencies

1
// Don't make real API calls in tests
2
#[tokio::test]
3
async fn test_with_real_stripe() { // Bad
4
// This calls real Stripe API
5
}
6
7
#[tokio::test]
8
async fn test_with_mock_stripe() { // Good
9
let mock_server = MockServer::start().await;
10
// Mock Stripe responses
11
}

Continuous Integration

GitHub Actions Example

1
name: Test Workflows
2
3
on: [push, pull_request]
4
5
jobs:
6
test:
7
runs-on: ubuntu-latest
8
9
services:
10
postgres:
11
image: postgres:14
12
env:
13
POSTGRES_PASSWORD: postgres
14
ports:
15
- 5432:5432
16
17
steps:
18
- uses: actions/checkout@v3
19
20
- name: Install Rust
21
uses: actions-rs/toolchain@v1
22
with:
23
toolchain: stable
24
25
- name: Start Orcher Server
26
run: |
27
docker run -d --name orcher \
28
-p 8080:8080 -p 9090:9090 \
29
--add-host=host.docker.internal:host-gateway \
30
orcherapp/orcher:latest
31
32
# Wait for server
33
timeout 30 sh -c 'until curl -s http://localhost:8080/health; do sleep 1; done'
34
35
- name: Run Unit Tests
36
run: cargo test --lib
37
38
- name: Run Integration Tests
39
run: cargo test --test '*'
40
env:
41
ORCHER_ORCHESTRATOR_URL: http://localhost:9090
42
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/orcher_test
43
44
- name: Run Crash Recovery Tests
45
run: cargo test crash_recovery

Performance Testing

Load Testing Workflows

1
#[tokio::test]
2
async fn load_test_workflow() {
3
let env = setup_test_env().await;
4
5
let start = std::time::Instant::now();
6
let num_workflows = 1000;
7
8
// Start many workflows concurrently
9
let handles: Vec<_> = (0..num_workflows)
10
.map(|i| {
11
let client = env.client.clone();
12
tokio::spawn(async move {
13
client
14
.start_workflow("payment_workflow", PaymentInput {
15
order_id: format!("ord_load_{}", i),
16
amount: 99.99,
17
card_token: "tok_visa".to_string(),
18
customer_email: format!("test{}@example.com", i),
19
})
20
.await
21
})
22
})
23
.collect();
24
25
// Wait for all to start
26
for handle in handles {
27
handle.await.unwrap().unwrap();
28
}
29
30
let elapsed = start.elapsed();
31
let throughput = num_workflows as f64 / elapsed.as_secs_f64();
32
33
println!("Started {} workflows in {:?}", num_workflows, elapsed);
34
println!("Throughput: {:.2} workflows/sec", throughput);
35
36
// Assert reasonable performance
37
assert!(throughput > 100.0, "Throughput too low: {}", throughput);
38
}

Debugging Failed Tests

Enable Detailed Logging

1
# Run tests with full debug logs
2
RUST_LOG=orcher=debug,my_workflow=trace cargo test -- --nocapture
3
4
# Save logs to file
5
RUST_LOG=debug cargo test 2>&1 | tee test.log

Inspect Test Database

1
# Connect to test database
2
psql postgresql://postgres:postgres@localhost:5432/orcher_test
3
4
# View workflow executions
5
SELECT * FROM workflow_executions WHERE workflow_name = 'payment_workflow';
6
7
# View journal entries
8
SELECT * FROM execution_log WHERE workflow_id = 'wf_abc123';
9
10
# View task executions
11
SELECT * FROM task_executions WHERE workflow_id = 'wf_abc123';

Use Test Helpers

1
// Create test helpers for debugging
2
pub mod test_helpers {
3
pub async fn print_workflow_state(client: &Client, workflow_id: &str) {
4
let state = client.get_workflow_state(workflow_id).await.unwrap();
5
println!("Workflow State: {:#?}", state);
6
7
let journal = client.get_workflow_journal(workflow_id).await.unwrap();
8
println!("Journal Entries: {:#?}", journal);
9
}
10
}
11
12
// Use in tests
13
#[tokio::test]
14
async fn test_with_debugging() {
15
let env = setup_test_env().await;
16
let handle = start_workflow(&env).await;
17
18
// Debug workflow state
19
test_helpers::print_workflow_state(&env.client, handle.id()).await;
20
21
let result = handle.result().await;
22
assert!(result.is_ok());
23
}

Next Steps


Learn More

• Workflow Development - Write workflows with best practices