Skip to content

Commit 9b6b529

Browse files
ariesdevildrmingdrmer
authored andcommitted
feat: initial support multi-raft
1 parent 7dabf9e commit 9b6b529

File tree

20 files changed

+1406
-9
lines changed

20 files changed

+1406
-9
lines changed

.github/workflows/ci.yaml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,7 @@ jobs:
374374
- 'raft-kv-memstore-opendal-snapshot-data'
375375
- 'raft-kv-memstore-singlethreaded'
376376
- 'raft-kv-rocksdb'
377+
- 'multi-raft-kv'
377378

378379
steps:
379380
- uses: actions/checkout@v4
@@ -401,13 +402,17 @@ jobs:
401402

402403
- name: Test demo script of examples/${{ matrix.example }}
403404
# The script is not meant for testing. Just to ensure it works but do not
404-
# rely on it.
405+
# rely on it. Skip if test-cluster.sh doesn't exist.
405406
if: ${{ matrix.toolchain == 'stable' }}
406407

407408
shell: bash
408409
run: |
409410
cd examples/${{ matrix.example }}
410-
./test-cluster.sh
411+
if [ -f test-cluster.sh ]; then
412+
./test-cluster.sh
413+
else
414+
echo "No test-cluster.sh found, skipping"
415+
fi
411416
412417
- name: Format
413418
# clippy/format produces different result with stable and nightly. Only with nightly.

Cargo.toml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ exclude = [
7777
"examples/raft-kv-memstore-opendal-snapshot-data",
7878
"examples/raft-kv-rocksdb",
7979

80+
"examples/multi-raft-kv",
81+
8082
"rt-monoio",
81-
"rt-compio"
83+
"rt-compio",
84+
"multiraft"
8285
]

Makefile

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ test:
2525
cargo test --features serde
2626
# only crate `tests` has single-term-leader feature
2727
cargo test --features single-term-leader -p tests
28+
# multiraft crate tests
29+
cargo test --manifest-path multiraft/Cargo.toml
2830
$(MAKE) test-examples
2931

3032
check-parallel:
@@ -40,6 +42,7 @@ test-examples:
4042
cargo test --manifest-path examples/raft-kv-memstore-singlethreaded/Cargo.toml
4143
cargo test --manifest-path examples/raft-kv-rocksdb/Cargo.toml
4244
cargo test --manifest-path examples/rocksstore/Cargo.toml
45+
cargo test --manifest-path examples/multi-raft-kv/Cargo.toml
4346

4447
bench:
4548
cargo bench --features bench
@@ -75,19 +78,27 @@ guide:
7578

7679
lint:
7780
cargo fmt
81+
cargo fmt --manifest-path multiraft/Cargo.toml
82+
cargo fmt --manifest-path rt-compio/Cargo.toml
83+
cargo fmt --manifest-path rt-monoio/Cargo.toml
7884
cargo fmt --manifest-path examples/mem-log/Cargo.toml
7985
cargo fmt --manifest-path examples/raft-kv-memstore-network-v2/Cargo.toml
8086
cargo fmt --manifest-path examples/raft-kv-memstore-opendal-snapshot-data/Cargo.toml
8187
cargo fmt --manifest-path examples/raft-kv-memstore-singlethreaded/Cargo.toml
8288
cargo fmt --manifest-path examples/raft-kv-memstore/Cargo.toml
8389
cargo fmt --manifest-path examples/raft-kv-rocksdb/Cargo.toml
90+
cargo fmt --manifest-path examples/multi-raft-kv/Cargo.toml
8491
cargo clippy --no-deps --all-targets -- -D warnings
85-
cargo clippy --no-deps --manifest-path examples/mem-log/Cargo.toml --all-targets -- -D warnings
92+
cargo clippy --no-deps --manifest-path multiraft/Cargo.toml --all-targets -- -D warnings
93+
cargo clippy --no-deps --manifest-path rt-compio/Cargo.toml --all-targets -- -D warnings
94+
cargo clippy --no-deps --manifest-path rt-monoio/Cargo.toml --all-targets -- -D warnings
95+
cargo clippy --no-deps --manifest-path examples/mem-log/Cargo.toml --all-targets -- -D warnings
8696
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore-network-v2/Cargo.toml --all-targets -- -D warnings
8797
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore-opendal-snapshot-data/Cargo.toml --all-targets -- -D warnings
8898
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore-singlethreaded/Cargo.toml --all-targets -- -D warnings
8999
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore/Cargo.toml --all-targets -- -D warnings
90100
cargo clippy --no-deps --manifest-path examples/raft-kv-rocksdb/Cargo.toml --all-targets -- -D warnings
101+
cargo clippy --no-deps --manifest-path examples/multi-raft-kv/Cargo.toml --all-targets -- -D warnings
91102
# Bug: clippy --all-targets reports false warning about unused dep in
92103
# `[dev-dependencies]`:
93104
# https://github.com/rust-lang/rust/issues/72686#issuecomment-635539688

examples/mem-log/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ license = "MIT OR Apache-2.0"
1515
repository = "https://github.com/databendlabs/openraft"
1616

1717
[dependencies]
18-
openraft = { path = "../../openraft", features = ["type-alias"] }
18+
openraft = { path = "../../openraft", default-features = false, features = ["type-alias", "tokio-rt"] }
1919

2020
tokio = { version = "1.0", default-features = false, features = ["sync"] }
2121

examples/multi-raft-kv/Cargo.toml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[package]
2+
name = "multi-raft-kv"
3+
version = "0.1.0"
4+
readme = "README.md"
5+
6+
edition = "2021"
7+
authors = [
8+
"AriesDevil <[email protected]>",
9+
]
10+
categories = ["algorithms", "asynchronous", "data-structures"]
11+
description = "An example Multi-Raft distributed key-value store with 3 groups built upon `openraft`."
12+
homepage = "https://github.com/databendlabs/openraft"
13+
keywords = ["raft", "consensus", "multi-raft"]
14+
license = "MIT OR Apache-2.0"
15+
repository = "https://github.com/databendlabs/openraft"
16+
17+
[dependencies]
18+
mem-log = { path = "../mem-log", features = [] }
19+
openraft = { path = "../../openraft", default-features = false, features = ["serde", "type-alias"] }
20+
openraft-multi = { path = "../../multiraft" }
21+
22+
futures = { version = "0.3" }
23+
serde = { version = "1", features = ["derive"] }
24+
serde_json = { version = "1" }
25+
tokio = { version = "1", default-features = false, features = ["sync"] }
26+
tracing = { version = "0.1" }
27+
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
28+
29+
30+
[features]
31+
32+
[package.metadata.docs.rs]
33+
all-features = true
34+

examples/multi-raft-kv/README.md

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Multi-Raft KV Store Example
2+
3+
This example demonstrates how to use OpenRaft's Multi-Raft support to run multiple independent Raft consensus groups within a single process.
4+
5+
## Overview
6+
7+
The example creates a distributed key-value store with **3 Raft groups**:
8+
- **users** - Stores user data
9+
- **orders** - Stores order data
10+
- **products** - Stores product data
11+
12+
Each group runs its own independent Raft consensus, but they share the same network infrastructure.
13+
14+
## Architecture
15+
16+
```
17+
+-----------------------------------------------------------------------+
18+
| Node 1 |
19+
| +-------------------+ +-------------------+ +-------------------+ |
20+
| | Group "users" | | Group "orders" | | Group "products" | |
21+
| | (Raft Instance) | | (Raft Instance) | | (Raft Instance) | |
22+
| +-------------------+ +-------------------+ +-------------------+ |
23+
| | | | |
24+
| +----------------------+----------------------+ |
25+
| | |
26+
| +--------+--------+ |
27+
| | Router | |
28+
| | (shared network)| |
29+
| +-----------------+ |
30+
+------------------------------------------------------------------------+
31+
|
32+
Network Connection
33+
|
34+
+------------------------------------------------------------------------+
35+
| Node 2 |
36+
| +-------------------+ +-------------------+ +-------------------+ |
37+
| | Group "users" | | Group "orders" | | Group "products" | |
38+
| | (Raft Instance) | | (Raft Instance) | | (Raft Instance) | |
39+
| +-------------------+ +-------------------+ +-------------------+ |
40+
+------------------------------------------------------------------------+
41+
```
42+
43+
## Key Concepts
44+
45+
### GroupId
46+
A string identifier that uniquely identifies each Raft group (e.g., "users", "orders", "products").
47+
48+
### Shared Network
49+
Multiple Raft groups share the same network infrastructure (`Router`), reducing connection overhead. Messages are routed to the correct group using the `group_id`.
50+
51+
### Independent Consensus
52+
Each group runs its own Raft consensus independently:
53+
- Separate log storage
54+
- Separate state machine
55+
- Separate leader election
56+
- Separate membership
57+
58+
## Running the Test
59+
60+
```bash
61+
# Run the integration test
62+
cargo test -p multi-raft-kv test_multi_raft_cluster -- --nocapture
63+
64+
# With debug logging
65+
RUST_LOG=debug cargo test -p multi-raft-kv test_multi_raft_cluster -- --nocapture
66+
```
67+
68+
## Code Structure
69+
70+
```
71+
multi-raft-kv/
72+
├── Cargo.toml
73+
├── README.md
74+
├── src/
75+
│ ├── lib.rs # Type definitions and group constants
76+
│ ├── app.rs # Application handler for each group
77+
│ ├── api.rs # API handlers (read, write, raft operations)
78+
│ ├── network.rs # Network implementation with group routing
79+
│ ├── router.rs # Message router for (node_id, group_id)
80+
│ └── store.rs # State machine storage
81+
└── tests/
82+
└── cluster/
83+
├── main.rs
84+
└── test_cluster.rs # Integration tests
85+
```

examples/multi-raft-kv/src/api.rs

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
use std::collections::BTreeMap;
2+
use std::collections::BTreeSet;
3+
4+
use openraft::raft::TransferLeaderRequest;
5+
use openraft::BasicNode;
6+
use openraft::ReadPolicy;
7+
8+
use crate::app::GroupApp;
9+
use crate::decode;
10+
use crate::encode;
11+
use crate::typ::*;
12+
use crate::NodeId;
13+
14+
/// Write a key-value pair to the group's state machine
15+
pub async fn write(app: &mut GroupApp, req: String) -> String {
16+
let res = app.raft.client_write(decode(&req)).await;
17+
encode(res)
18+
}
19+
20+
/// Read a value from the group's state machine using linearizable read
21+
pub async fn read(app: &mut GroupApp, req: String) -> String {
22+
let key: String = decode(&req);
23+
24+
let ret = app.raft.get_read_linearizer(ReadPolicy::ReadIndex).await;
25+
26+
let res = match ret {
27+
Ok(linearizer) => {
28+
linearizer.await_ready(&app.raft).await.unwrap();
29+
30+
let state_machine = app.state_machine.state_machine.lock().await;
31+
let value = state_machine.data.get(&key).cloned();
32+
33+
let res: Result<String, RaftError<LinearizableReadError>> = Ok(value.unwrap_or_default());
34+
res
35+
}
36+
Err(e) => Err(e),
37+
};
38+
encode(res)
39+
}
40+
41+
// ============================================================================
42+
// Raft Protocol API
43+
// ============================================================================
44+
45+
/// Handle vote request
46+
pub async fn vote(app: &mut GroupApp, req: String) -> String {
47+
let res = app.raft.vote(decode(&req)).await;
48+
encode(res)
49+
}
50+
51+
/// Handle append entries request
52+
pub async fn append(app: &mut GroupApp, req: String) -> String {
53+
let res = app.raft.append_entries(decode(&req)).await;
54+
encode(res)
55+
}
56+
57+
/// Receive a snapshot and install it
58+
pub async fn snapshot(app: &mut GroupApp, req: String) -> String {
59+
let (vote, snapshot_meta, snapshot_data): (Vote, SnapshotMeta, SnapshotData) = decode(&req);
60+
let snapshot = Snapshot {
61+
meta: snapshot_meta,
62+
snapshot: snapshot_data,
63+
};
64+
let res = app.raft.install_full_snapshot(vote, snapshot).await.map_err(RaftError::<Infallible>::Fatal);
65+
encode(res)
66+
}
67+
68+
/// Handle transfer leader request
69+
pub async fn transfer_leader(app: &mut GroupApp, req: String) -> String {
70+
let transfer_req: TransferLeaderRequest<crate::TypeConfig> = decode(&req);
71+
let res = app.raft.handle_transfer_leader(transfer_req).await;
72+
encode(res)
73+
}
74+
75+
// ============================================================================
76+
// Management API
77+
// ============================================================================
78+
79+
/// Add a node as **Learner** to this group.
80+
///
81+
/// This should be done before adding a node as a member into the cluster
82+
/// (by calling `change-membership`)
83+
pub async fn add_learner(app: &mut GroupApp, req: String) -> String {
84+
let node_id: NodeId = decode(&req);
85+
let node = BasicNode { addr: "".to_string() };
86+
let res = app.raft.add_learner(node_id, node, true).await;
87+
encode(res)
88+
}
89+
90+
/// Changes specified learners to members, or remove members from this group.
91+
pub async fn change_membership(app: &mut GroupApp, req: String) -> String {
92+
let node_ids: BTreeSet<NodeId> = decode(&req);
93+
let res = app.raft.change_membership(node_ids, false).await;
94+
encode(res)
95+
}
96+
97+
/// Initialize a single-node cluster for this group.
98+
pub async fn init(app: &mut GroupApp) -> String {
99+
let mut nodes = BTreeMap::new();
100+
nodes.insert(app.node_id, BasicNode { addr: "".to_string() });
101+
let res = app.raft.initialize(nodes).await;
102+
encode(res)
103+
}
104+
105+
/// Get the latest metrics of this Raft group
106+
pub async fn metrics(app: &mut GroupApp) -> String {
107+
let metrics = app.raft.metrics().borrow().clone();
108+
109+
let res: Result<RaftMetrics, Infallible> = Ok(metrics);
110+
encode(res)
111+
}

0 commit comments

Comments
 (0)