Adding a working bft-json-crdt implementation for the PoC
This commit is contained in:
Dave Hrycyszyn
42
Cargo.lock
generated
42
Cargo.lock
generated
@@ -299,6 +299,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2 0.10.8",
|
"sha2 0.10.8",
|
||||||
|
"time 0.1.45",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1995,7 +1996,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_with_macros",
|
"serde_with_macros",
|
||||||
"time",
|
"time 0.3.36",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2266,6 +2267,17 @@ dependencies = [
|
|||||||
"num_cpus",
|
"num_cpus",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "time"
|
||||||
|
version = "0.1.45"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "time"
|
name = "time"
|
||||||
version = "0.3.36"
|
version = "0.3.36"
|
||||||
@@ -2474,6 +2486,12 @@ version = "0.9.0+wasi-snapshot-preview1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasi"
|
||||||
|
version = "0.10.0+wasi-snapshot-preview1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasi"
|
name = "wasi"
|
||||||
version = "0.11.0+wasi-snapshot-preview1"
|
version = "0.11.0+wasi-snapshot-preview1"
|
||||||
@@ -2554,6 +2572,28 @@ dependencies = [
|
|||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi"
|
||||||
|
version = "0.3.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||||
|
dependencies = [
|
||||||
|
"winapi-i686-pc-windows-gnu",
|
||||||
|
"winapi-x86_64-pc-windows-gnu",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi-i686-pc-windows-gnu"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi-x86_64-pc-windows-gnu"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-core"
|
name = "windows-core"
|
||||||
version = "0.52.0"
|
version = "0.52.0"
|
||||||
|
|||||||
2
crates/bft-json-crdt/.gitattributes
vendored
Normal file
2
crates/bft-json-crdt/.gitattributes
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
*.js linguist-generated
|
||||||
|
*.json linguist-generated
|
||||||
1
crates/bft-json-crdt/.gitignore
vendored
Normal file
1
crates/bft-json-crdt/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
target
|
||||||
1924
crates/bft-json-crdt/Cargo.lock
generated
Normal file
1924
crates/bft-json-crdt/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
30
crates/bft-json-crdt/Cargo.toml
Normal file
30
crates/bft-json-crdt/Cargo.toml
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
[package]
|
||||||
|
name = "bft-json-crdt"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
crate-type = ["lib"]
|
||||||
|
|
||||||
|
[features]
|
||||||
|
default = ["bft", "logging-list", "logging-json"]
|
||||||
|
logging-list = ["logging-base"]
|
||||||
|
logging-json = ["logging-base"]
|
||||||
|
logging-base = []
|
||||||
|
bft = []
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
bft-crdt-derive = { path = "bft-crdt-derive" }
|
||||||
|
colored = "2.0.0"
|
||||||
|
fastcrypto = "0.1.8"
|
||||||
|
itertools = "0.10.5"
|
||||||
|
rand = "0.8.5"
|
||||||
|
random_color = "0.6.1"
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0.85"
|
||||||
|
sha2 = "0.10.6"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0.85"
|
||||||
|
time = "0.1"
|
||||||
21
crates/bft-json-crdt/LICENSE.txt
Normal file
21
crates/bft-json-crdt/LICENSE.txt
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2022 Jacky Zhao
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
58
crates/bft-json-crdt/README.md
Normal file
58
crates/bft-json-crdt/README.md
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
# Byzantine Fault Tolerant CRDTs
|
||||||
|
|
||||||
|
This work is mainly inspired by implementing Martin Kleppmann's 2022 paper on *Making CRDTs Byzantine Fault Tolerant*[^2]
|
||||||
|
on top of a simplified [Automerge](https://automerge.org/) implementation.
|
||||||
|
|
||||||
|
The goal is to show a working prototype that demonstrated in simple code the ideas behind
|
||||||
|
1. An Automerge-like CRDT
|
||||||
|
2. How a primitive list CRDT can be composed to create complex CRDTs like JSON
|
||||||
|
2. How to add Byzantine Fault Tolerance to arbitrary CRDTs
|
||||||
|
|
||||||
|
Unlike most other CRDT implementations, I leave out many performance optimizations that would make the basic algorithm harder to understand.
|
||||||
|
|
||||||
|
Check out the [accompanying blog post for this project!](https://jzhao.xyz/posts/bft-json-crdt)
|
||||||
|
|
||||||
|
## Benchmarks
|
||||||
|
Altough this implementation does not optimize for performance, it still nonetheless performs quite well.
|
||||||
|
|
||||||
|
Benchmarking happened on a 2019 Macbook Pro with a 2.6GHz i7.
|
||||||
|
Numbers are compared to Automerge which report their performance benchmarks [here](https://github.com/automerge/automerge-perf)
|
||||||
|
|
||||||
|
| # Ops | Raw String (JS) | Ours (basic) | Ours (BFT) | Automerge (JS) | Automerge (Rust) |
|
||||||
|
|--|--|--|--|--|--|
|
||||||
|
|10k | n/a | 0.081s | 1.793s | 1.6s | 0.047s |
|
||||||
|
|100k | n/a | 9.321s | 38.842s | 43.0s | 0.597s |
|
||||||
|
|All (259k)| 0.61s | 88.610s | 334.960s | Out of Memory| 1.780s |
|
||||||
|
|Memory | 0.1MB | 27.6MB | 59.5MB | 880MB | 232.5MB |
|
||||||
|
|
||||||
|
## Flamegraph
|
||||||
|
To get some flamegraphs of the time graph on MacOS, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo cargo flamegraph --dev --root --bench speed
|
||||||
|
```
|
||||||
|
|
||||||
|
## Further Work
|
||||||
|
This is mostly a learning/instructional project but there are a few places where performance improvements are obvious:
|
||||||
|
|
||||||
|
1. This is backed by `std::Vec` which isn't great for random insert. Replace with a B-tree or something that provides better insert and find performance
|
||||||
|
1. [Diamond Types](https://github.com/josephg/diamond-types) and [Automerge (Rust)](https://github.com/automerge/automerge-rs) use a B-tree
|
||||||
|
2. Yjs is backed by a doubly linked-list and caches last ~5-10 accessed locations (assumes that most edits happen sequentially; seeks are rare)
|
||||||
|
3. (funnily enough, main peformance hit is dominated by find and not insert, see [this flamegraph](./flamegraphs/flamegraph_unoptimized.svg))
|
||||||
|
2. Avoid calling `find` so many times. A few Automerge optimizations that were not implemented
|
||||||
|
1. Use an index hint (especially for local inserts)
|
||||||
|
2. Skipping the second `find` operation in `integrate` if sequence number is already larger
|
||||||
|
3. Improve storage requirement. As of now, a single `Op` weighs in at *over* 168 bytes. This doesn't even fit in a single cache line!
|
||||||
|
4. Implement 'transactions' for a group of changes that should be considered atomic.
|
||||||
|
1. This would also speed up Ed25519 signature verification time by batching.
|
||||||
|
2. For example, a peer might create an atomic 'transaction' that contains a bunch of changes.
|
||||||
|
5. Currently, each character is a single op. Similar to Yjs, we can combine runs of characters into larger entities like what André, Luc, et al.[^1] suggest
|
||||||
|
6. Implement proper persistence using SQLLite or something similar
|
||||||
|
7. Compile the project to WASM and implement a transport layer so it can be used in browser. Something similar to [Yjs' WebRTC Connector](https://github.com/yjs/y-webrtc) could work.
|
||||||
|
|
||||||
|
[^1]: André, Luc, et al. "Supporting adaptable granularity of changes for massive-scale collaborative editing." 9th IEEE International Conference on Collaborative Computing: Networking, Applications and Worksharing. IEEE, 2013.
|
||||||
|
[^2]: Kleppmann, Martin. "Making CRDTs Byzantine Fault Tolerant." Proceedings of the 9th Workshop on Principles and Practice of Consistency for Distributed Data. 2022.
|
||||||
|
|
||||||
|
## Acknowledgements
|
||||||
|
Thank you to [Nalin Bhardwaj](https://nibnalin.me/) for helping me with my cryptography questions and [Martin Kleppmann](https://martin.kleppmann.com/)
|
||||||
|
for his teaching materials and lectures which taught me a significant portion of what I've learned about distributed systems and CRDTs.
|
||||||
57
crates/bft-json-crdt/benches/speed.rs
Normal file
57
crates/bft-json-crdt/benches/speed.rs
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
#![feature(test)]
|
||||||
|
|
||||||
|
extern crate test;
|
||||||
|
use bft_json_crdt::{keypair::make_author, list_crdt::ListCrdt, op::Op, op::ROOT_ID, json_crdt::Value};
|
||||||
|
use rand::seq::SliceRandom;
|
||||||
|
use test::Bencher;
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_insert_1_000_root(b: &mut Bencher) {
|
||||||
|
b.iter(|| {
|
||||||
|
let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
|
||||||
|
for i in 0..1_000 {
|
||||||
|
list.insert(ROOT_ID, i);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_insert_1_000_linear(b: &mut Bencher) {
|
||||||
|
b.iter(|| {
|
||||||
|
let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
|
||||||
|
let mut prev = ROOT_ID;
|
||||||
|
for i in 0..1_000 {
|
||||||
|
let op = list.insert(prev, i);
|
||||||
|
prev = op.id;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[bench]
|
||||||
|
fn bench_insert_many_agents_conflicts(b: &mut Bencher) {
|
||||||
|
b.iter(|| {
|
||||||
|
const N: u8 = 50;
|
||||||
|
let mut rng = rand::thread_rng();
|
||||||
|
let mut crdts: Vec<ListCrdt<i64>> = Vec::with_capacity(N as usize);
|
||||||
|
let mut logs: Vec<Op<Value>> = Vec::new();
|
||||||
|
for i in 0..N {
|
||||||
|
let list = ListCrdt::new(make_author(i), vec![]);
|
||||||
|
crdts.push(list);
|
||||||
|
for _ in 0..5 {
|
||||||
|
let op = crdts[i as usize].insert(ROOT_ID, i as i32);
|
||||||
|
logs.push(op);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logs.shuffle(&mut rng);
|
||||||
|
for op in logs {
|
||||||
|
for c in &mut crdts {
|
||||||
|
if op.author() != c.our_id {
|
||||||
|
c.apply(op.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(crdts.windows(2).all(|w| w[0].view() == w[1].view()));
|
||||||
|
})
|
||||||
|
}
|
||||||
100
crates/bft-json-crdt/bft-crdt-derive/Cargo.lock
generated
Normal file
100
crates/bft-json-crdt/bft-crdt-derive/Cargo.lock
generated
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 3
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bft-crdt-derive"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro-crate",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell"
|
||||||
|
version = "1.16.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-crate"
|
||||||
|
version = "1.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9"
|
||||||
|
dependencies = [
|
||||||
|
"once_cell",
|
||||||
|
"thiserror",
|
||||||
|
"toml",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.47"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.147"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "1.0.103"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror"
|
||||||
|
version = "1.0.37"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
|
||||||
|
dependencies = [
|
||||||
|
"thiserror-impl",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror-impl"
|
||||||
|
version = "1.0.37"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "toml"
|
||||||
|
version = "0.5.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
|
||||||
14
crates/bft-json-crdt/bft-crdt-derive/Cargo.toml
Normal file
14
crates/bft-json-crdt/bft-crdt-derive/Cargo.toml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
[package]
|
||||||
|
name = "bft-crdt-derive"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
publish = false
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
proc-macro = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
proc-macro2 = "1.0.47"
|
||||||
|
proc-macro-crate = "1.2.1"
|
||||||
|
quote = "1.0.21"
|
||||||
|
syn = { version = "1.0.103", features = ["full"] }
|
||||||
189
crates/bft-json-crdt/bft-crdt-derive/src/lib.rs
Normal file
189
crates/bft-json-crdt/bft-crdt-derive/src/lib.rs
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
use proc_macro::TokenStream as OgTokenStream;
|
||||||
|
use proc_macro2::{Ident, Span, TokenStream};
|
||||||
|
use proc_macro_crate::{crate_name, FoundCrate};
|
||||||
|
use quote::{quote, quote_spanned, ToTokens};
|
||||||
|
use syn::{
|
||||||
|
parse::{self, Parser},
|
||||||
|
parse_macro_input,
|
||||||
|
spanned::Spanned,
|
||||||
|
Data, DeriveInput, Field, Fields, ItemStruct, LitStr, Type
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Helper to get tokenstream representing the parent crate
|
||||||
|
fn get_crate_name() -> TokenStream {
|
||||||
|
let cr8 = crate_name("bft-json-crdt")
|
||||||
|
.unwrap_or(FoundCrate::Itself);
|
||||||
|
match cr8 {
|
||||||
|
FoundCrate::Itself => quote! { ::bft_json_crdt },
|
||||||
|
FoundCrate::Name(name) => {
|
||||||
|
let ident = Ident::new(&name, Span::call_site());
|
||||||
|
quote! { ::#ident }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Proc macro to insert a keypair and path field on a given struct
|
||||||
|
#[proc_macro_attribute]
|
||||||
|
pub fn add_crdt_fields(args: OgTokenStream, input: OgTokenStream) -> OgTokenStream {
|
||||||
|
let mut input = parse_macro_input!(input as ItemStruct);
|
||||||
|
let crate_name = get_crate_name();
|
||||||
|
let _ = parse_macro_input!(args as parse::Nothing);
|
||||||
|
|
||||||
|
if let syn::Fields::Named(ref mut fields) = input.fields {
|
||||||
|
fields.named.push(
|
||||||
|
Field::parse_named
|
||||||
|
.parse2(quote! { path: Vec<#crate_name::op::PathSegment> })
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
fields.named.push(
|
||||||
|
Field::parse_named
|
||||||
|
.parse2(quote! { id: #crate_name::keypair::AuthorId })
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return quote! {
|
||||||
|
#input
|
||||||
|
}
|
||||||
|
.into();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Proc macro to automatically derive the CRDTNode trait
|
||||||
|
#[proc_macro_derive(CrdtNode)]
|
||||||
|
pub fn derive_json_crdt(input: OgTokenStream) -> OgTokenStream {
|
||||||
|
// parse the input tokens into a syntax tree
|
||||||
|
let input = parse_macro_input!(input as DeriveInput);
|
||||||
|
let crate_name = get_crate_name();
|
||||||
|
|
||||||
|
// used in the quasi-quotation below as `#name`
|
||||||
|
let ident = input.ident;
|
||||||
|
let ident_str = LitStr::new(&*ident.to_string(), ident.span());
|
||||||
|
|
||||||
|
let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
|
||||||
|
match input.data {
|
||||||
|
Data::Struct(data) => match &data.fields {
|
||||||
|
Fields::Named(fields) => {
|
||||||
|
let mut field_impls = vec![];
|
||||||
|
let mut ident_literals = vec![];
|
||||||
|
let mut ident_strings = vec![];
|
||||||
|
let mut tys = vec![];
|
||||||
|
// parse all named fields
|
||||||
|
for field in &fields.named {
|
||||||
|
let ident = field.ident.as_ref().expect("Failed to get struct field identifier");
|
||||||
|
if ident != "path" && ident != "id" {
|
||||||
|
let ty = match &field.ty {
|
||||||
|
Type::Path(t) => t.to_token_stream(),
|
||||||
|
_ => return quote_spanned! { field.span() => compile_error!("Field should be a primitive or struct which implements CRDTNode") }.into(),
|
||||||
|
};
|
||||||
|
let str_literal = LitStr::new(&*ident.to_string(), ident.span());
|
||||||
|
ident_strings.push(str_literal.clone());
|
||||||
|
ident_literals.push(ident.clone());
|
||||||
|
tys.push(ty.clone());
|
||||||
|
field_impls.push(quote! {
|
||||||
|
#ident: <#ty as CrdtNode>::new(
|
||||||
|
id,
|
||||||
|
#crate_name::op::join_path(path.clone(), #crate_name::op::PathSegment::Field(#str_literal.to_string()))
|
||||||
|
)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let expanded = quote! {
|
||||||
|
impl #impl_generics #crate_name::json_crdt::CrdtNodeFromValue for #ident #ty_generics #where_clause {
|
||||||
|
fn node_from(value: #crate_name::json_crdt::Value, id: #crate_name::keypair::AuthorId, path: Vec<#crate_name::op::PathSegment>) -> Result<Self, String> {
|
||||||
|
if let #crate_name::json_crdt::Value::Object(mut obj) = value {
|
||||||
|
Ok(#ident {
|
||||||
|
path: path.clone(),
|
||||||
|
id,
|
||||||
|
#(#ident_literals: obj.remove(#ident_strings)
|
||||||
|
.unwrap()
|
||||||
|
.into_node(
|
||||||
|
id,
|
||||||
|
#crate_name::op::join_path(path.clone(), #crate_name::op::PathSegment::Field(#ident_strings.to_string()))
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
),*
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
Err(format!("failed to convert {:?} -> {}<T>", value, #ident_str.to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl #impl_generics std::fmt::Debug for #ident #ty_generics #where_clause {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let mut fields = Vec::new();
|
||||||
|
#(fields.push(format!("{}", #ident_strings.to_string()));)*
|
||||||
|
write!(f, "{{ {:?} }}", fields.join(", "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl #impl_generics #crate_name::json_crdt::CrdtNode for #ident #ty_generics #where_clause {
|
||||||
|
fn apply(&mut self, op: #crate_name::op::Op<#crate_name::json_crdt::Value>) -> #crate_name::json_crdt::OpState {
|
||||||
|
let path = op.path.clone();
|
||||||
|
let author = op.id.clone();
|
||||||
|
if !#crate_name::op::ensure_subpath(&self.path, &op.path) {
|
||||||
|
#crate_name::debug::debug_path_mismatch(self.path.to_owned(), op.path);
|
||||||
|
return #crate_name::json_crdt::OpState::ErrPathMismatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.path.len() == op.path.len() {
|
||||||
|
return #crate_name::json_crdt::OpState::ErrApplyOnStruct;
|
||||||
|
} else {
|
||||||
|
let idx = self.path.len();
|
||||||
|
if let #crate_name::op::PathSegment::Field(path_seg) = &op.path[idx] {
|
||||||
|
match &path_seg[..] {
|
||||||
|
#(#ident_strings => {
|
||||||
|
return self.#ident_literals.apply(op.into());
|
||||||
|
}),*
|
||||||
|
_ => {},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
return #crate_name::json_crdt::OpState::ErrPathMismatch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn view(&self) -> #crate_name::json_crdt::Value {
|
||||||
|
let mut view_map = std::collections::HashMap::new();
|
||||||
|
#(view_map.insert(#ident_strings.to_string(), self.#ident_literals.view().into());)*
|
||||||
|
#crate_name::json_crdt::Value::Object(view_map)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new(id: #crate_name::keypair::AuthorId, path: Vec<#crate_name::op::PathSegment>) -> Self {
|
||||||
|
Self {
|
||||||
|
path: path.clone(),
|
||||||
|
id,
|
||||||
|
#(#field_impls),*
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl #crate_name::debug::DebugView for #ident {
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
fn debug_view(&self, indent: usize) -> String {
|
||||||
|
let inner_spacing = " ".repeat(indent + 2);
|
||||||
|
let path_str = #crate_name::op::print_path(self.path.clone());
|
||||||
|
let mut inner = vec![];
|
||||||
|
#(inner.push(format!("{}\"{}\": {}", inner_spacing, #ident_strings, self.#ident_literals.debug_view(indent + 4)));)*
|
||||||
|
let inner_str = inner.join("\n");
|
||||||
|
format!("{} @ /{}\n{}", #ident_str, path_str, inner_str)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "logging-base"))]
|
||||||
|
fn debug_view(&self, _indent: usize) -> String {
|
||||||
|
"".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Hand the output tokens back to the compiler
|
||||||
|
expanded.into()
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return quote_spanned! { ident.span() => compile_error!("Cannot derive CRDT on tuple or unit structs"); }
|
||||||
|
.into()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
_ => return quote_spanned! { ident.span() => compile_error!("Cannot derive CRDT on enums or unions"); }.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
415
crates/bft-json-crdt/flamegraphs/flamegraph_bft.svg
Normal file
415
crates/bft-json-crdt/flamegraphs/flamegraph_bft.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 502 KiB |
415
crates/bft-json-crdt/flamegraphs/flamegraph_hashes.svg
Normal file
415
crates/bft-json-crdt/flamegraphs/flamegraph_hashes.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 300 KiB |
415
crates/bft-json-crdt/flamegraphs/flamegraph_unoptimized.svg
Normal file
415
crates/bft-json-crdt/flamegraphs/flamegraph_unoptimized.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 80 KiB |
317
crates/bft-json-crdt/src/debug.rs
Normal file
317
crates/bft-json-crdt/src/debug.rs
Normal file
@@ -0,0 +1,317 @@
|
|||||||
|
use crate::{
|
||||||
|
json_crdt::{BaseCrdt, CrdtNode, SignedOp},
|
||||||
|
keypair::SignedDigest,
|
||||||
|
list_crdt::ListCrdt,
|
||||||
|
op::{Op, OpId, PathSegment},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
use {
|
||||||
|
crate::{
|
||||||
|
keypair::{lsb_32, AuthorId},
|
||||||
|
op::{print_hex, print_path, ROOT_ID},
|
||||||
|
},
|
||||||
|
colored::Colorize,
|
||||||
|
random_color::{Luminosity, RandomColor},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "logging-list")]
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::fmt::Display;
|
||||||
|
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
fn author_to_hex(author: AuthorId) -> String {
|
||||||
|
format!("{:#010x}", lsb_32(author))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
fn display_op_id<T: CrdtNode>(op: &Op<T>) -> String {
|
||||||
|
let [r, g, b] = RandomColor::new()
|
||||||
|
.luminosity(Luminosity::Light)
|
||||||
|
.seed(lsb_32(op.author))
|
||||||
|
.to_rgb_array();
|
||||||
|
format!(
|
||||||
|
"[{},{}]",
|
||||||
|
author_to_hex(op.author).bold().truecolor(r, g, b),
|
||||||
|
op.seq.to_string().yellow()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn debug_type_mismatch(_msg: String) {
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
{
|
||||||
|
println!(" {}\n {_msg}", "type mismatch! ignoring this node".red(),);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn debug_path_mismatch(_our_path: Vec<PathSegment>, _op_path: Vec<PathSegment>) {
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
{
|
||||||
|
println!(
|
||||||
|
" {}\n current path: {}\n op path: {}",
|
||||||
|
"path mismatch!".red(),
|
||||||
|
print_path(_our_path),
|
||||||
|
print_path(_op_path),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn debug_op_on_primitive(_op_path: Vec<PathSegment>) {
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
{
|
||||||
|
println!(
|
||||||
|
" {} this is an error, ignoring op.\n op path: {}",
|
||||||
|
"trying to apply() on a primitive!".red(),
|
||||||
|
print_path(_op_path),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
fn display_author(author: AuthorId) -> String {
|
||||||
|
let [r, g, b] = RandomColor::new()
|
||||||
|
.luminosity(Luminosity::Light)
|
||||||
|
.seed(lsb_32(author))
|
||||||
|
.to_rgb_array();
|
||||||
|
format!(" {} ", author_to_hex(author))
|
||||||
|
.black()
|
||||||
|
.on_truecolor(r, g, b)
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait DebugView {
|
||||||
|
fn debug_view(&self, indent: usize) -> String;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: CrdtNode + DebugView> BaseCrdt<T> {
|
||||||
|
pub fn debug_view(&self) {
|
||||||
|
#[cfg(feature = "logging-json")]
|
||||||
|
println!("document is now:\n{}", self.doc.debug_view(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn log_try_apply(&self, _op: &SignedOp) {
|
||||||
|
#[cfg(feature = "logging-json")]
|
||||||
|
println!(
|
||||||
|
"{} trying to apply operation {} from {}",
|
||||||
|
display_author(self.id),
|
||||||
|
&print_hex(&_op.signed_digest)[..6],
|
||||||
|
display_author(_op.inner.author())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn debug_digest_failure(&self, _op: SignedOp) {
|
||||||
|
#[cfg(feature = "logging-json")]
|
||||||
|
println!(
|
||||||
|
" {} cannot confirm signed_digest from {}",
|
||||||
|
"digest failure!".red(),
|
||||||
|
display_author(_op.author())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn log_missing_causal_dep(&self, _missing: &SignedDigest) {
|
||||||
|
#[cfg(feature = "logging-json")]
|
||||||
|
println!(
|
||||||
|
" {} haven't received op with digest {}",
|
||||||
|
"missing causal dependency".red(),
|
||||||
|
print_hex(_missing)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn log_actually_apply(&self, _op: &SignedOp) {
|
||||||
|
#[cfg(feature = "logging-json")]
|
||||||
|
{
|
||||||
|
println!(
|
||||||
|
" applying op to path: /{}",
|
||||||
|
print_path(_op.inner.path.clone())
|
||||||
|
);
|
||||||
|
println!("{}", _op.inner.debug_view(2));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Op<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
pub fn debug_hash_failure(&self) {
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
{
|
||||||
|
println!(" {}", "hash failure!".red());
|
||||||
|
println!(" expected: {}", print_hex(&self.id));
|
||||||
|
println!(" computed: {}", print_hex(&self.hash_to_id()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> DebugView for T
|
||||||
|
where
|
||||||
|
T: Display,
|
||||||
|
{
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
fn debug_view(&self, _indent: usize) -> String {
|
||||||
|
self.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "logging-base"))]
|
||||||
|
fn debug_view(&self, _indent: usize) -> String {
|
||||||
|
"".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> DebugView for Op<T>
|
||||||
|
where
|
||||||
|
T: DebugView + CrdtNode,
|
||||||
|
{
|
||||||
|
#[cfg(not(feature = "logging-base"))]
|
||||||
|
fn debug_view(&self, _indent: usize) -> String {
|
||||||
|
"".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "logging-json")]
|
||||||
|
fn debug_view(&self, indent: usize) -> String {
|
||||||
|
let op_id = display_op_id(self);
|
||||||
|
let content = if self.id == ROOT_ID && self.content.is_none() {
|
||||||
|
"root".blue().bold().to_string()
|
||||||
|
} else {
|
||||||
|
self.content
|
||||||
|
.as_ref()
|
||||||
|
.map_or("[empty]".to_string(), |c| c.debug_view(indent + 2))
|
||||||
|
};
|
||||||
|
let content_str = if self.is_deleted && self.id != ROOT_ID {
|
||||||
|
content.red().strikethrough().to_string()
|
||||||
|
} else {
|
||||||
|
content
|
||||||
|
};
|
||||||
|
|
||||||
|
format!("{op_id} {content_str}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
pub fn log_ops(&self, highlight: Option<OpId>) {
|
||||||
|
#[cfg(feature = "logging-list")]
|
||||||
|
{
|
||||||
|
let mut lines = Vec::<String>::new();
|
||||||
|
|
||||||
|
// do in-order traversal
|
||||||
|
let res: Vec<&Op<T>> = self.ops.iter().collect();
|
||||||
|
if res.is_empty() {
|
||||||
|
println!("[empty]");
|
||||||
|
}
|
||||||
|
|
||||||
|
// figure out parent-child hierarchies from origins
|
||||||
|
let mut parent_child_map: HashMap<OpId, Vec<OpId>> = HashMap::new();
|
||||||
|
for op in &res {
|
||||||
|
let children = parent_child_map.entry(op.origin).or_default();
|
||||||
|
children.push(op.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
let is_last = |op: &Op<T>| -> bool {
|
||||||
|
if op.id == ROOT_ID {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if let Some(children) = parent_child_map.get(&op.origin) {
|
||||||
|
return *children.last().unwrap() == op.id;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
};
|
||||||
|
|
||||||
|
// make stack of origins
|
||||||
|
let mut stack: Vec<(OpId, &str)> = Vec::new();
|
||||||
|
stack.push((ROOT_ID, ""));
|
||||||
|
let mut prev = None;
|
||||||
|
for op in &res {
|
||||||
|
let origin_idx = self.find_idx(op.origin).unwrap();
|
||||||
|
let origin = &res[origin_idx];
|
||||||
|
let origin_id = origin.id;
|
||||||
|
if let Some(prev) = prev {
|
||||||
|
if origin_id == prev {
|
||||||
|
// went down one layer, add to stack
|
||||||
|
let stack_prefix_char = if is_last(origin) { " " } else { "│ " };
|
||||||
|
stack.push((prev, stack_prefix_char));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pop back up until we reach the right origin
|
||||||
|
while stack.last().unwrap().0 != origin_id {
|
||||||
|
stack.pop();
|
||||||
|
}
|
||||||
|
|
||||||
|
let cur_char = if is_last(op) { "╰─" } else { "├─" };
|
||||||
|
let prefixes = stack.iter().map(|s| s.1).collect::<Vec<_>>().join("");
|
||||||
|
let highlight_text = if highlight.is_some() && highlight.unwrap() == op.id {
|
||||||
|
if op.is_deleted {
|
||||||
|
"<- deleted".bold().red()
|
||||||
|
} else {
|
||||||
|
"<- inserted".bold().green()
|
||||||
|
}
|
||||||
|
.to_string()
|
||||||
|
} else {
|
||||||
|
"".to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
let content = if op.id == ROOT_ID {
|
||||||
|
"root".blue().bold().to_string()
|
||||||
|
} else {
|
||||||
|
op.content
|
||||||
|
.as_ref()
|
||||||
|
.map_or("[empty]".to_string(), |c| c.hash())
|
||||||
|
};
|
||||||
|
if op.is_deleted && op.id != ROOT_ID {
|
||||||
|
lines.push(format!(
|
||||||
|
"{}{}{} {} {}",
|
||||||
|
prefixes,
|
||||||
|
cur_char,
|
||||||
|
display_op_id(op),
|
||||||
|
content.strikethrough().red(),
|
||||||
|
highlight_text
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
lines.push(format!(
|
||||||
|
"{}{}{} {} {}",
|
||||||
|
prefixes,
|
||||||
|
cur_char,
|
||||||
|
display_op_id(op),
|
||||||
|
content,
|
||||||
|
highlight_text
|
||||||
|
));
|
||||||
|
}
|
||||||
|
prev = Some(op.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// full string
|
||||||
|
let flat = self.iter().map(|t| t.hash()).collect::<Vec<_>>().join("");
|
||||||
|
lines.push(format!("Flattened result: {}", flat));
|
||||||
|
println!("{}", lines.join("\n"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn log_apply(&self, op: &Op<T>) {
|
||||||
|
#[cfg(feature = "logging-list")]
|
||||||
|
{
|
||||||
|
if op.is_deleted {
|
||||||
|
println!(
|
||||||
|
"{} Performing a delete of {}@{}",
|
||||||
|
display_author(self.our_id),
|
||||||
|
display_op_id(op),
|
||||||
|
op.sequence_num(),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(content) = op.content.as_ref() {
|
||||||
|
println!(
|
||||||
|
"{} Performing an insert of {}@{}: '{}' after {}",
|
||||||
|
display_author(self.our_id),
|
||||||
|
display_op_id(op),
|
||||||
|
op.sequence_num(),
|
||||||
|
content.hash(),
|
||||||
|
display_op_id(op)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
868
crates/bft-json-crdt/src/json_crdt.rs
Normal file
868
crates/bft-json-crdt/src/json_crdt.rs
Normal file
@@ -0,0 +1,868 @@
|
|||||||
|
use std::{
|
||||||
|
collections::{HashMap, HashSet},
|
||||||
|
fmt::Display,
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
debug::{debug_op_on_primitive, DebugView},
|
||||||
|
keypair::{sha256, sign, AuthorId, SignedDigest},
|
||||||
|
list_crdt::ListCrdt,
|
||||||
|
lww_crdt::LwwRegisterCrdt,
|
||||||
|
op::{print_hex, print_path, Hashable, Op, OpId, PathSegment},
|
||||||
|
};
|
||||||
|
pub use bft_crdt_derive::*;
|
||||||
|
use fastcrypto::traits::VerifyingKey;
|
||||||
|
use fastcrypto::{
|
||||||
|
ed25519::{Ed25519KeyPair, Ed25519PublicKey, Ed25519Signature},
|
||||||
|
traits::{KeyPair, ToFromBytes},
|
||||||
|
// Verifier,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Anything that can be nested in a JSON CRDT
|
||||||
|
pub trait CrdtNode: CrdtNodeFromValue + Hashable + Clone {
|
||||||
|
/// Create a new CRDT of this type
|
||||||
|
fn new(id: AuthorId, path: Vec<PathSegment>) -> Self;
|
||||||
|
/// Apply an operation to this CRDT, forwarding if necessary
|
||||||
|
fn apply(&mut self, op: Op<Value>) -> OpState;
|
||||||
|
/// Get a JSON representation of the value in this node
|
||||||
|
fn view(&self) -> Value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Enum representing possible outcomes of applying an operation to a CRDT
|
||||||
|
#[derive(Debug, PartialEq)]
|
||||||
|
pub enum OpState {
|
||||||
|
/// Operation applied successfully
|
||||||
|
Ok,
|
||||||
|
/// Tried to apply an operation to a non-CRDT primative (i.e. f64, bool, etc.)
|
||||||
|
/// If you would like a mutable primitive, wrap it in a [`LWWRegisterCRDT`]
|
||||||
|
ErrApplyOnPrimitive,
|
||||||
|
/// Tried to apply an operation to a static struct CRDT
|
||||||
|
/// If you would like a mutable object, use a [`Value`]
|
||||||
|
ErrApplyOnStruct,
|
||||||
|
/// Tried to apply an operation that contains content of the wrong type.
|
||||||
|
/// In other words, the content cannot be coerced to the CRDT at the path specified.
|
||||||
|
ErrMismatchedType,
|
||||||
|
/// The signed digest of the message did not match the claimed author of the message.
|
||||||
|
/// This can happen if the message was tampered with during delivery
|
||||||
|
ErrDigestMismatch,
|
||||||
|
/// The hash of the message did not match the contents of the mesage.
|
||||||
|
/// This can happen if the author tried to perform an equivocation attack by creating an
|
||||||
|
/// operation and modifying it has already been created
|
||||||
|
ErrHashMismatch,
|
||||||
|
/// Tried to apply an operation to a non-existent path. The author may have forgotten to attach
|
||||||
|
/// a causal dependency
|
||||||
|
ErrPathMismatch,
|
||||||
|
/// Trying to modify/delete the sentinel (zero-th) node element that is used for book-keeping
|
||||||
|
ErrListApplyToEmpty,
|
||||||
|
/// We have not received all of the causal dependencies of this operation. It has been queued
|
||||||
|
/// up and will be executed when its causal dependencies have been delivered
|
||||||
|
MissingCausalDependencies,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The following types can be used as a 'terminal' type in CRDTs
|
||||||
|
pub trait MarkPrimitive: Into<Value> + Default {}
|
||||||
|
impl MarkPrimitive for bool {}
|
||||||
|
impl MarkPrimitive for i32 {}
|
||||||
|
impl MarkPrimitive for i64 {}
|
||||||
|
impl MarkPrimitive for f64 {}
|
||||||
|
impl MarkPrimitive for char {}
|
||||||
|
impl MarkPrimitive for String {}
|
||||||
|
impl MarkPrimitive for Value {}
|
||||||
|
|
||||||
|
/// Implement CrdtNode for non-CRDTs
|
||||||
|
/// This is a stub implementation so most functions don't do anything/log an error
|
||||||
|
impl<T> CrdtNode for T
|
||||||
|
where
|
||||||
|
T: CrdtNodeFromValue + MarkPrimitive + Hashable + Clone,
|
||||||
|
{
|
||||||
|
fn apply(&mut self, _op: Op<Value>) -> OpState {
|
||||||
|
OpState::ErrApplyOnPrimitive
|
||||||
|
}
|
||||||
|
|
||||||
|
fn view(&self) -> Value {
|
||||||
|
self.to_owned().into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new(_id: AuthorId, _path: Vec<PathSegment>) -> Self {
|
||||||
|
debug_op_on_primitive(_path);
|
||||||
|
Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The base struct for a JSON CRDT. Allows for declaring causal
|
||||||
|
/// dependencies across fields. It only accepts messages of [`SignedOp`] for BFT.
|
||||||
|
pub struct BaseCrdt<T: CrdtNode> {
|
||||||
|
/// Public key of this CRDT
|
||||||
|
pub id: AuthorId,
|
||||||
|
|
||||||
|
/// Internal base CRDT
|
||||||
|
pub doc: T,
|
||||||
|
|
||||||
|
/// In a real world scenario, this would be a proper hashgraph that allows for
|
||||||
|
/// efficient reconciliation of missing dependencies. We naively keep a hashset
|
||||||
|
/// of messages we've seen (represented by their [`SignedDigest`]).
|
||||||
|
received: HashSet<SignedDigest>,
|
||||||
|
message_q: HashMap<SignedDigest, Vec<SignedOp>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An [`Op<Value>`] with a few bits of extra metadata
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct SignedOp {
|
||||||
|
// Note that this can be different from the author of the inner op as the inner op could have been created
|
||||||
|
// by a different person
|
||||||
|
author: AuthorId,
|
||||||
|
/// Signed hash using priv key of author. Effectively [`OpID`] Use this as the ID to figure out what has been delivered already
|
||||||
|
pub signed_digest: SignedDigest,
|
||||||
|
pub inner: Op<Value>,
|
||||||
|
/// List of causal dependencies
|
||||||
|
pub depends_on: Vec<SignedDigest>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SignedOp {
|
||||||
|
pub fn id(&self) -> OpId {
|
||||||
|
self.inner.id
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn author(&self) -> AuthorId {
|
||||||
|
self.author
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates a digest of the following fields. Any changes in the fields will change the signed digest
|
||||||
|
/// - id (hash of the following)
|
||||||
|
/// - origin
|
||||||
|
/// - author
|
||||||
|
/// - seq
|
||||||
|
/// - is_deleted
|
||||||
|
/// - path
|
||||||
|
/// - dependencies
|
||||||
|
fn digest(&self) -> [u8; 32] {
|
||||||
|
let path_string = print_path(self.inner.path.clone());
|
||||||
|
let dependency_string = self
|
||||||
|
.depends_on
|
||||||
|
.iter()
|
||||||
|
.map(print_hex)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("");
|
||||||
|
let fmt_str = format!("{:?},{path_string},{dependency_string}", self.id());
|
||||||
|
sha256(fmt_str)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sign this digest with the given keypair. Shouldn't need to be called manually,
|
||||||
|
/// just use [`SignedOp::from_op`] instead
|
||||||
|
fn sign_digest(&mut self, keypair: &Ed25519KeyPair) {
|
||||||
|
self.signed_digest = sign(keypair, &self.digest()).sig.to_bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ensure digest was actually signed by the author it claims to be signed by
|
||||||
|
pub fn is_valid_digest(&self) -> bool {
|
||||||
|
let digest = Ed25519Signature::from_bytes(&self.signed_digest);
|
||||||
|
let pubkey = Ed25519PublicKey::from_bytes(&self.author());
|
||||||
|
match (digest, pubkey) {
|
||||||
|
(Ok(digest), Ok(pubkey)) => pubkey.verify(&self.digest(), &digest).is_ok(),
|
||||||
|
(_, _) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sign a normal op and add all the needed metadata
|
||||||
|
pub fn from_op<T: CrdtNode>(
|
||||||
|
value: Op<T>,
|
||||||
|
keypair: &Ed25519KeyPair,
|
||||||
|
depends_on: Vec<SignedDigest>,
|
||||||
|
) -> Self {
|
||||||
|
let author = keypair.public().0.to_bytes();
|
||||||
|
let mut new = Self {
|
||||||
|
inner: Op {
|
||||||
|
content: value.content.map(|c| c.view()),
|
||||||
|
origin: value.origin,
|
||||||
|
author: value.author,
|
||||||
|
seq: value.seq,
|
||||||
|
path: value.path,
|
||||||
|
is_deleted: value.is_deleted,
|
||||||
|
id: value.id,
|
||||||
|
},
|
||||||
|
author,
|
||||||
|
signed_digest: [0u8; 64],
|
||||||
|
depends_on,
|
||||||
|
};
|
||||||
|
new.sign_digest(keypair);
|
||||||
|
new
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: CrdtNode + DebugView> BaseCrdt<T> {
|
||||||
|
/// Crease a new BaseCRDT of the given type. Multiple BaseCRDTs
|
||||||
|
/// can be created from a single keypair but you are responsible for
|
||||||
|
/// routing messages to the right BaseCRDT. Usually you should just make a single
|
||||||
|
/// struct that contains all the state you need
|
||||||
|
pub fn new(keypair: &Ed25519KeyPair) -> Self {
|
||||||
|
let id = keypair.public().0.to_bytes();
|
||||||
|
Self {
|
||||||
|
id,
|
||||||
|
doc: T::new(id, vec![]),
|
||||||
|
received: HashSet::new(),
|
||||||
|
message_q: HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply a signed operation to this BaseCRDT, verifying integrity and routing to the right
|
||||||
|
/// nested CRDT
|
||||||
|
pub fn apply(&mut self, op: SignedOp) -> OpState {
|
||||||
|
self.log_try_apply(&op);
|
||||||
|
|
||||||
|
#[cfg(feature = "bft")]
|
||||||
|
if !op.is_valid_digest() {
|
||||||
|
self.debug_digest_failure(op);
|
||||||
|
return OpState::ErrDigestMismatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
let op_id = op.signed_digest;
|
||||||
|
if !op.depends_on.is_empty() {
|
||||||
|
for origin in &op.depends_on {
|
||||||
|
if !self.received.contains(origin) {
|
||||||
|
self.log_missing_causal_dep(origin);
|
||||||
|
self.message_q.entry(*origin).or_default().push(op);
|
||||||
|
return OpState::MissingCausalDependencies;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// apply
|
||||||
|
self.log_actually_apply(&op);
|
||||||
|
let status = self.doc.apply(op.inner);
|
||||||
|
self.debug_view();
|
||||||
|
self.received.insert(op_id);
|
||||||
|
|
||||||
|
// apply all of its causal dependents if there are any
|
||||||
|
let dependent_queue = self.message_q.remove(&op_id);
|
||||||
|
if let Some(mut q) = dependent_queue {
|
||||||
|
for dependent in q.drain(..) {
|
||||||
|
self.apply(dependent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
status
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An enum representing a JSON value
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub enum Value {
|
||||||
|
Null,
|
||||||
|
Bool(bool),
|
||||||
|
Number(f64),
|
||||||
|
String(String),
|
||||||
|
Array(Vec<Value>),
|
||||||
|
Object(HashMap<String, Value>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Display for Value {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"{}",
|
||||||
|
match self {
|
||||||
|
Value::Null => "null".to_string(),
|
||||||
|
Value::Bool(b) => b.to_string(),
|
||||||
|
Value::Number(n) => n.to_string(),
|
||||||
|
Value::String(s) => format!("\"{s}\""),
|
||||||
|
Value::Array(arr) => {
|
||||||
|
if arr.len() > 1 {
|
||||||
|
format!(
|
||||||
|
"[\n{}\n]",
|
||||||
|
arr.iter()
|
||||||
|
.map(|x| format!(" {x}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(",\n")
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"[ {} ]",
|
||||||
|
arr.iter()
|
||||||
|
.map(|x| x.to_string())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Object(obj) => format!(
|
||||||
|
"{{ {} }}",
|
||||||
|
obj.iter()
|
||||||
|
.map(|(k, v)| format!(" \"{k}\": {v}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(",\n")
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for Value {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::Null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Allow easy conversion to and from serde's JSON format. This allows us to use the [`json!`]
|
||||||
|
/// macro
|
||||||
|
impl From<Value> for serde_json::Value {
|
||||||
|
fn from(value: Value) -> Self {
|
||||||
|
match value {
|
||||||
|
Value::Null => serde_json::Value::Null,
|
||||||
|
Value::Bool(x) => serde_json::Value::Bool(x),
|
||||||
|
Value::Number(x) => serde_json::Value::Number(serde_json::Number::from_f64(x).unwrap()),
|
||||||
|
Value::String(x) => serde_json::Value::String(x),
|
||||||
|
Value::Array(x) => {
|
||||||
|
serde_json::Value::Array(x.iter().map(|a| a.clone().into()).collect())
|
||||||
|
}
|
||||||
|
Value::Object(x) => serde_json::Value::Object(
|
||||||
|
x.iter()
|
||||||
|
.map(|(k, v)| (k.clone(), v.clone().into()))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<serde_json::Value> for Value {
|
||||||
|
fn from(value: serde_json::Value) -> Self {
|
||||||
|
match value {
|
||||||
|
serde_json::Value::Null => Value::Null,
|
||||||
|
serde_json::Value::Bool(x) => Value::Bool(x),
|
||||||
|
serde_json::Value::Number(x) => Value::Number(x.as_f64().unwrap()),
|
||||||
|
serde_json::Value::String(x) => Value::String(x),
|
||||||
|
serde_json::Value::Array(x) => {
|
||||||
|
Value::Array(x.iter().map(|a| a.clone().into()).collect())
|
||||||
|
}
|
||||||
|
serde_json::Value::Object(x) => Value::Object(
|
||||||
|
x.iter()
|
||||||
|
.map(|(k, v)| (k.clone(), v.clone().into()))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Value {
|
||||||
|
pub fn into_json(self) -> serde_json::Value {
|
||||||
|
self.into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Conversions from primitive types to [`Value`]
|
||||||
|
impl From<bool> for Value {
|
||||||
|
fn from(val: bool) -> Self {
|
||||||
|
Value::Bool(val)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<i64> for Value {
|
||||||
|
fn from(val: i64) -> Self {
|
||||||
|
Value::Number(val as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<i32> for Value {
|
||||||
|
fn from(val: i32) -> Self {
|
||||||
|
Value::Number(val as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<f64> for Value {
|
||||||
|
fn from(val: f64) -> Self {
|
||||||
|
Value::Number(val)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<String> for Value {
|
||||||
|
fn from(val: String) -> Self {
|
||||||
|
Value::String(val)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<char> for Value {
|
||||||
|
fn from(val: char) -> Self {
|
||||||
|
Value::String(val.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> From<Option<T>> for Value
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn from(val: Option<T>) -> Self {
|
||||||
|
match val {
|
||||||
|
Some(x) => x.view(),
|
||||||
|
None => Value::Null,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> From<Vec<T>> for Value
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn from(value: Vec<T>) -> Self {
|
||||||
|
Value::Array(value.iter().map(|x| x.view()).collect())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fallibly create a CRDT Node from a JSON Value
|
||||||
|
pub trait CrdtNodeFromValue: Sized {
|
||||||
|
fn node_from(value: Value, id: AuthorId, path: Vec<PathSegment>) -> Result<Self, String>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fallibly cast a JSON Value into a CRDT Node
|
||||||
|
pub trait IntoCrdtNode<T>: Sized {
|
||||||
|
fn into_node(self, id: AuthorId, path: Vec<PathSegment>) -> Result<T, String>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`CrdtNodeFromValue`] implies [`IntoCRDTNode<T>`]
|
||||||
|
impl<T> IntoCrdtNode<T> for Value
|
||||||
|
where
|
||||||
|
T: CrdtNodeFromValue,
|
||||||
|
{
|
||||||
|
fn into_node(self, id: AuthorId, path: Vec<PathSegment>) -> Result<T, String> {
|
||||||
|
T::node_from(self, id, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trivial conversion from Value to Value as CrdtNodeFromValue
|
||||||
|
impl CrdtNodeFromValue for Value {
|
||||||
|
fn node_from(value: Value, _id: AuthorId, _path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Conversions from primitives to CRDTs
|
||||||
|
impl CrdtNodeFromValue for bool {
|
||||||
|
fn node_from(value: Value, _id: AuthorId, _path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
if let Value::Bool(x) = value {
|
||||||
|
Ok(x)
|
||||||
|
} else {
|
||||||
|
Err(format!("failed to convert {value:?} -> bool"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrdtNodeFromValue for f64 {
|
||||||
|
fn node_from(value: Value, _id: AuthorId, _path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
if let Value::Number(x) = value {
|
||||||
|
Ok(x)
|
||||||
|
} else {
|
||||||
|
Err(format!("failed to convert {value:?} -> f64"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrdtNodeFromValue for i64 {
|
||||||
|
fn node_from(value: Value, _id: AuthorId, _path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
if let Value::Number(x) = value {
|
||||||
|
Ok(x as i64)
|
||||||
|
} else {
|
||||||
|
Err(format!("failed to convert {value:?} -> f64"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrdtNodeFromValue for String {
|
||||||
|
fn node_from(value: Value, _id: AuthorId, _path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
if let Value::String(x) = value {
|
||||||
|
Ok(x)
|
||||||
|
} else {
|
||||||
|
Err(format!("failed to convert {value:?} -> String"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrdtNodeFromValue for char {
|
||||||
|
fn node_from(value: Value, _id: AuthorId, _path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
if let Value::String(x) = value.clone() {
|
||||||
|
x.chars().next().ok_or(format!(
|
||||||
|
"failed to convert {value:?} -> char: found a zero-length string"
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
Err(format!("failed to convert {value:?} -> char"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> CrdtNodeFromValue for LwwRegisterCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn node_from(value: Value, id: AuthorId, path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
let mut crdt = LwwRegisterCrdt::new(id, path);
|
||||||
|
crdt.set(value);
|
||||||
|
Ok(crdt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> CrdtNodeFromValue for ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn node_from(value: Value, id: AuthorId, path: Vec<PathSegment>) -> Result<Self, String> {
|
||||||
|
if let Value::Array(arr) = value {
|
||||||
|
let mut crdt = ListCrdt::new(id, path);
|
||||||
|
let result: Result<(), String> =
|
||||||
|
arr.into_iter().enumerate().try_for_each(|(i, val)| {
|
||||||
|
crdt.insert_idx(i, val);
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
result?;
|
||||||
|
Ok(crdt)
|
||||||
|
} else {
|
||||||
|
Err(format!("failed to convert {value:?} -> ListCRDT<T>"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod test {
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
json_crdt::{add_crdt_fields, BaseCrdt, CrdtNode, IntoCrdtNode, OpState, Value},
|
||||||
|
keypair::make_keypair,
|
||||||
|
list_crdt::ListCrdt,
|
||||||
|
lww_crdt::LwwRegisterCrdt,
|
||||||
|
op::{print_path, ROOT_ID},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_derive_basic() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Player {
|
||||||
|
x: LwwRegisterCrdt<f64>,
|
||||||
|
y: LwwRegisterCrdt<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let keypair = make_keypair();
|
||||||
|
let crdt = BaseCrdt::<Player>::new(&keypair);
|
||||||
|
assert_eq!(print_path(crdt.doc.x.path), "x");
|
||||||
|
assert_eq!(print_path(crdt.doc.y.path), "y");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_derive_nested() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Position {
|
||||||
|
x: LwwRegisterCrdt<f64>,
|
||||||
|
y: LwwRegisterCrdt<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Player {
|
||||||
|
pos: Position,
|
||||||
|
balance: LwwRegisterCrdt<f64>,
|
||||||
|
messages: ListCrdt<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let keypair = make_keypair();
|
||||||
|
let crdt = BaseCrdt::<Player>::new(&keypair);
|
||||||
|
assert_eq!(print_path(crdt.doc.pos.x.path), "pos.x");
|
||||||
|
assert_eq!(print_path(crdt.doc.pos.y.path), "pos.y");
|
||||||
|
assert_eq!(print_path(crdt.doc.balance.path), "balance");
|
||||||
|
assert_eq!(print_path(crdt.doc.messages.path), "messages");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_lww_ops() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Test {
|
||||||
|
a: LwwRegisterCrdt<f64>,
|
||||||
|
b: LwwRegisterCrdt<bool>,
|
||||||
|
c: LwwRegisterCrdt<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let kp1 = make_keypair();
|
||||||
|
let kp2 = make_keypair();
|
||||||
|
let mut base1 = BaseCrdt::<Test>::new(&kp1);
|
||||||
|
let mut base2 = BaseCrdt::<Test>::new(&kp2);
|
||||||
|
|
||||||
|
let _1_a_1 = base1.doc.a.set(3.0).sign(&kp1);
|
||||||
|
let _1_b_1 = base1.doc.b.set(true).sign(&kp1);
|
||||||
|
let _2_a_1 = base2.doc.a.set(1.5).sign(&kp2);
|
||||||
|
let _2_a_2 = base2.doc.a.set(2.13).sign(&kp2);
|
||||||
|
let _2_c_1 = base2.doc.c.set("abc".to_string()).sign(&kp2);
|
||||||
|
|
||||||
|
assert_eq!(base1.doc.a.view(), json!(3.0).into());
|
||||||
|
assert_eq!(base2.doc.a.view(), json!(2.13).into());
|
||||||
|
assert_eq!(base1.doc.b.view(), json!(true).into());
|
||||||
|
assert_eq!(base2.doc.c.view(), json!("abc").into());
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"a": 3.0,
|
||||||
|
"b": true,
|
||||||
|
"c": null,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
base2.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"a": 2.13,
|
||||||
|
"b": null,
|
||||||
|
"c": "abc",
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(base2.apply(_1_a_1), OpState::Ok);
|
||||||
|
assert_eq!(base2.apply(_1_b_1), OpState::Ok);
|
||||||
|
assert_eq!(base1.apply(_2_a_1), OpState::Ok);
|
||||||
|
assert_eq!(base1.apply(_2_a_2), OpState::Ok);
|
||||||
|
assert_eq!(base1.apply(_2_c_1), OpState::Ok);
|
||||||
|
|
||||||
|
assert_eq!(base1.doc.view().into_json(), base2.doc.view().into_json());
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"a": 2.13,
|
||||||
|
"b": true,
|
||||||
|
"c": "abc"
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_vec_and_map_ops() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Test {
|
||||||
|
a: ListCrdt<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let kp1 = make_keypair();
|
||||||
|
let kp2 = make_keypair();
|
||||||
|
let mut base1 = BaseCrdt::<Test>::new(&kp1);
|
||||||
|
let mut base2 = BaseCrdt::<Test>::new(&kp2);
|
||||||
|
|
||||||
|
let _1a = base1.doc.a.insert(ROOT_ID, "a".to_string()).sign(&kp1);
|
||||||
|
let _1b = base1.doc.a.insert(_1a.id(), "b".to_string()).sign(&kp1);
|
||||||
|
let _2c = base2.doc.a.insert(ROOT_ID, "c".to_string()).sign(&kp2);
|
||||||
|
let _2d = base2.doc.a.insert(_1b.id(), "d".to_string()).sign(&kp2);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"a": ["a", "b"],
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
// as _1b hasn't been delivered to base2 yet
|
||||||
|
assert_eq!(
|
||||||
|
base2.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"a": ["c"],
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(base2.apply(_1b), OpState::MissingCausalDependencies);
|
||||||
|
assert_eq!(base2.apply(_1a), OpState::Ok);
|
||||||
|
assert_eq!(base1.apply(_2d), OpState::Ok);
|
||||||
|
assert_eq!(base1.apply(_2c), OpState::Ok);
|
||||||
|
assert_eq!(base1.doc.view().into_json(), base2.doc.view().into_json());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_causal_field_dependency() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Item {
|
||||||
|
name: LwwRegisterCrdt<String>,
|
||||||
|
soulbound: LwwRegisterCrdt<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Player {
|
||||||
|
inventory: ListCrdt<Item>,
|
||||||
|
balance: LwwRegisterCrdt<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let kp1 = make_keypair();
|
||||||
|
let kp2 = make_keypair();
|
||||||
|
let mut base1 = BaseCrdt::<Player>::new(&kp1);
|
||||||
|
let mut base2 = BaseCrdt::<Player>::new(&kp2);
|
||||||
|
|
||||||
|
// require balance update to happen before inventory update
|
||||||
|
let _add_money = base1.doc.balance.set(5000.0).sign(&kp1);
|
||||||
|
let _spend_money = base1
|
||||||
|
.doc
|
||||||
|
.balance
|
||||||
|
.set(3000.0)
|
||||||
|
.sign_with_dependencies(&kp1, vec![&_add_money]);
|
||||||
|
|
||||||
|
let sword: Value = json!({
|
||||||
|
"name": "Sword",
|
||||||
|
"soulbound": true,
|
||||||
|
})
|
||||||
|
.into();
|
||||||
|
let _new_inventory_item = base1
|
||||||
|
.doc
|
||||||
|
.inventory
|
||||||
|
.insert_idx(0, sword)
|
||||||
|
.sign_with_dependencies(&kp1, vec![&_spend_money]);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"balance": 3000.0,
|
||||||
|
"inventory": [
|
||||||
|
{
|
||||||
|
"name": "Sword",
|
||||||
|
"soulbound": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
// do it completely out of order
|
||||||
|
assert_eq!(
|
||||||
|
base2.apply(_new_inventory_item),
|
||||||
|
OpState::MissingCausalDependencies
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
base2.apply(_spend_money),
|
||||||
|
OpState::MissingCausalDependencies
|
||||||
|
);
|
||||||
|
assert_eq!(base2.apply(_add_money), OpState::Ok);
|
||||||
|
assert_eq!(base1.doc.view().into_json(), base2.doc.view().into_json());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_2d_grid() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Game {
|
||||||
|
grid: ListCrdt<ListCrdt<LwwRegisterCrdt<bool>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let kp1 = make_keypair();
|
||||||
|
let kp2 = make_keypair();
|
||||||
|
let mut base1 = BaseCrdt::<Game>::new(&kp1);
|
||||||
|
let mut base2 = BaseCrdt::<Game>::new(&kp2);
|
||||||
|
|
||||||
|
// init a 2d grid
|
||||||
|
let row0: Value = json!([true, false]).into();
|
||||||
|
let row1: Value = json!([false, true]).into();
|
||||||
|
let construct1 = base1.doc.grid.insert_idx(0, row0).sign(&kp1);
|
||||||
|
let construct2 = base1.doc.grid.insert_idx(1, row1).sign(&kp1);
|
||||||
|
|
||||||
|
assert_eq!(base2.apply(construct1), OpState::Ok);
|
||||||
|
assert_eq!(base2.apply(construct2.clone()), OpState::Ok);
|
||||||
|
|
||||||
|
assert_eq!(base1.doc.view().into_json(), base2.doc.view().into_json());
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"grid": [[true, false], [false, true]]
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
let set1 = base1.doc.grid[0][0].set(false).sign(&kp1);
|
||||||
|
let set2 = base2.doc.grid[1][1].set(false).sign(&kp2);
|
||||||
|
assert_eq!(base1.apply(set2), OpState::Ok);
|
||||||
|
assert_eq!(base2.apply(set1), OpState::Ok);
|
||||||
|
|
||||||
|
assert_eq!(base1.doc.view().into_json(), base2.doc.view().into_json());
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"grid": [[false, false], [false, false]]
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
let topright = base1.doc.grid[0].id_at(1).unwrap();
|
||||||
|
base1.doc.grid[0].delete(topright);
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"grid": [[false], [false, false]]
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
base1.doc.grid.delete(construct2.id());
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"grid": [[false]]
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_arb_json() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Test {
|
||||||
|
reg: LwwRegisterCrdt<Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let kp1 = make_keypair();
|
||||||
|
let mut base1 = BaseCrdt::<Test>::new(&kp1);
|
||||||
|
|
||||||
|
let base_val: Value = json!({
|
||||||
|
"a": true,
|
||||||
|
"b": "asdf",
|
||||||
|
"c": {
|
||||||
|
"d": [],
|
||||||
|
"e": [ false ]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.into();
|
||||||
|
base1.doc.reg.set(base_val).sign(&kp1);
|
||||||
|
assert_eq!(
|
||||||
|
base1.doc.view().into_json(),
|
||||||
|
json!({
|
||||||
|
"reg": {
|
||||||
|
"a": true,
|
||||||
|
"b": "asdf",
|
||||||
|
"c": {
|
||||||
|
"d": [],
|
||||||
|
"e": [ false ]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_wrong_json_types() {
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Nested {
|
||||||
|
list: ListCrdt<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Test {
|
||||||
|
reg: LwwRegisterCrdt<bool>,
|
||||||
|
strct: ListCrdt<Nested>,
|
||||||
|
}
|
||||||
|
|
||||||
|
let key = make_keypair();
|
||||||
|
let mut crdt = BaseCrdt::<Test>::new(&key);
|
||||||
|
|
||||||
|
// wrong type should not go through
|
||||||
|
crdt.doc.reg.set(32);
|
||||||
|
assert_eq!(crdt.doc.reg.view(), json!(null).into());
|
||||||
|
crdt.doc.reg.set(true);
|
||||||
|
assert_eq!(crdt.doc.reg.view(), json!(true).into());
|
||||||
|
|
||||||
|
// set nested
|
||||||
|
let mut list_view: Value = crdt.doc.strct.view().into();
|
||||||
|
assert_eq!(list_view, json!([]).into());
|
||||||
|
|
||||||
|
// only keeps actual numbers
|
||||||
|
let list: Value = json!({"list": [0, 123, -0.45, "char", []]}).into();
|
||||||
|
crdt.doc.strct.insert_idx(0, list);
|
||||||
|
list_view = crdt.doc.strct.view().into();
|
||||||
|
assert_eq!(list_view, json!([{ "list": [0, 123, -0.45]}]).into());
|
||||||
|
}
|
||||||
|
}
|
||||||
57
crates/bft-json-crdt/src/keypair.rs
Normal file
57
crates/bft-json-crdt/src/keypair.rs
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
use fastcrypto::traits::VerifyingKey;
|
||||||
|
pub use fastcrypto::{
|
||||||
|
ed25519::{
|
||||||
|
Ed25519KeyPair, Ed25519PublicKey, Ed25519Signature, ED25519_PUBLIC_KEY_LENGTH,
|
||||||
|
ED25519_SIGNATURE_LENGTH,
|
||||||
|
},
|
||||||
|
traits::{KeyPair, Signer},
|
||||||
|
// Verifier,
|
||||||
|
};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
/// Represents the ID of a unique node. An Ed25519 public key
|
||||||
|
pub type AuthorId = [u8; ED25519_PUBLIC_KEY_LENGTH];
|
||||||
|
|
||||||
|
/// A signed message
|
||||||
|
pub type SignedDigest = [u8; ED25519_SIGNATURE_LENGTH];
|
||||||
|
|
||||||
|
/// Create a fake public key from a u8
|
||||||
|
pub fn make_author(n: u8) -> AuthorId {
|
||||||
|
let mut id = [0u8; ED25519_PUBLIC_KEY_LENGTH];
|
||||||
|
id[0] = n;
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get the least significant 32 bits of a public key
|
||||||
|
pub fn lsb_32(pubkey: AuthorId) -> u32 {
|
||||||
|
((pubkey[0] as u32) << 24)
|
||||||
|
+ ((pubkey[1] as u32) << 16)
|
||||||
|
+ ((pubkey[2] as u32) << 8)
|
||||||
|
+ (pubkey[3] as u32)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// SHA256 hash of a string
|
||||||
|
pub fn sha256(input: String) -> [u8; 32] {
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(input.as_bytes());
|
||||||
|
let result = hasher.finalize();
|
||||||
|
let mut bytes = [0u8; 32];
|
||||||
|
bytes.copy_from_slice(&result[..]);
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generate a random Ed25519 keypair from OS rng
|
||||||
|
pub fn make_keypair() -> Ed25519KeyPair {
|
||||||
|
let mut csprng = rand::thread_rng();
|
||||||
|
Ed25519KeyPair::generate(&mut csprng)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sign a byte array
|
||||||
|
pub fn sign(keypair: &Ed25519KeyPair, message: &[u8]) -> Ed25519Signature {
|
||||||
|
keypair.sign(message)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify a byte array was signed by the given pubkey
|
||||||
|
pub fn verify(pubkey: Ed25519PublicKey, message: &[u8], signature: Ed25519Signature) -> bool {
|
||||||
|
pubkey.verify(message, &signature).is_ok()
|
||||||
|
}
|
||||||
8
crates/bft-json-crdt/src/lib.rs
Normal file
8
crates/bft-json-crdt/src/lib.rs
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
pub mod debug;
|
||||||
|
pub mod json_crdt;
|
||||||
|
pub mod keypair;
|
||||||
|
pub mod list_crdt;
|
||||||
|
pub mod lww_crdt;
|
||||||
|
pub mod op;
|
||||||
|
|
||||||
|
extern crate self as bft_json_crdt;
|
||||||
441
crates/bft-json-crdt/src/list_crdt.rs
Normal file
441
crates/bft-json-crdt/src/list_crdt.rs
Normal file
@@ -0,0 +1,441 @@
|
|||||||
|
use crate::{
|
||||||
|
debug::debug_path_mismatch,
|
||||||
|
json_crdt::{CrdtNode, OpState, Value},
|
||||||
|
keypair::AuthorId,
|
||||||
|
op::*,
|
||||||
|
};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::{
|
||||||
|
cmp::{max, Ordering},
|
||||||
|
collections::HashMap,
|
||||||
|
fmt::Debug,
|
||||||
|
ops::{Index, IndexMut},
|
||||||
|
};
|
||||||
|
|
||||||
|
/// An RGA-like list CRDT that can store a CRDT-like datatype
|
||||||
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
/// Public key for this node
|
||||||
|
pub our_id: AuthorId,
|
||||||
|
/// Path to this CRDT
|
||||||
|
pub path: Vec<PathSegment>,
|
||||||
|
/// List of all the operations we know of
|
||||||
|
pub ops: Vec<Op<T>>,
|
||||||
|
/// Queue of messages where K is the ID of the message yet to arrive
|
||||||
|
/// and V is the list of operations depending on it
|
||||||
|
message_q: HashMap<OpId, Vec<Op<T>>>,
|
||||||
|
/// The sequence number of this node
|
||||||
|
our_seq: SequenceNumber,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
/// Create a new List CRDT with the given [`AuthorID`] (it should be unique)
|
||||||
|
pub fn new(id: AuthorId, path: Vec<PathSegment>) -> ListCrdt<T> {
|
||||||
|
let ops = vec![Op::make_root()];
|
||||||
|
ListCrdt {
|
||||||
|
our_id: id,
|
||||||
|
path,
|
||||||
|
ops,
|
||||||
|
message_q: HashMap::new(),
|
||||||
|
our_seq: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Locally insert some content causally after the given operation
|
||||||
|
pub fn insert<U: Into<Value>>(&mut self, after: OpId, content: U) -> Op<Value> {
|
||||||
|
let mut op = Op::new(
|
||||||
|
after,
|
||||||
|
self.our_id,
|
||||||
|
self.our_seq + 1,
|
||||||
|
false,
|
||||||
|
Some(content.into()),
|
||||||
|
self.path.to_owned(),
|
||||||
|
);
|
||||||
|
|
||||||
|
// we need to know the op ID before setting the path as [`PathSegment::Index`] requires an
|
||||||
|
// [`OpID`]
|
||||||
|
let new_path = join_path(self.path.to_owned(), PathSegment::Index(op.id));
|
||||||
|
op.path = new_path;
|
||||||
|
self.apply(op.clone());
|
||||||
|
op
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shorthand function to insert at index locally. Indexing ignores deleted items
|
||||||
|
pub fn insert_idx<U: Into<Value> + Clone>(&mut self, idx: usize, content: U) -> Op<Value> {
|
||||||
|
let mut i = 0;
|
||||||
|
for op in &self.ops {
|
||||||
|
if !op.is_deleted {
|
||||||
|
if idx == i {
|
||||||
|
return self.insert(op.id, content);
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic!("index {idx} out of range (length of {i})")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shorthand to figure out the OpID of something with a given index.
|
||||||
|
/// Useful for declaring a causal dependency if you didn't create the original
|
||||||
|
pub fn id_at(&self, idx: usize) -> Option<OpId> {
|
||||||
|
let mut i = 0;
|
||||||
|
for op in &self.ops {
|
||||||
|
if !op.is_deleted {
|
||||||
|
if idx == i {
|
||||||
|
return Some(op.id);
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark a node as deleted. If the node doesn't exist, it will be stuck
|
||||||
|
/// waiting for that node to be created.
|
||||||
|
pub fn delete(&mut self, id: OpId) -> Op<Value> {
|
||||||
|
let op = Op::new(
|
||||||
|
id,
|
||||||
|
self.our_id,
|
||||||
|
self.our_seq + 1,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
join_path(self.path.to_owned(), PathSegment::Index(id)),
|
||||||
|
);
|
||||||
|
self.apply(op.clone());
|
||||||
|
op
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Find the idx of an operation with the given [`OpID`]
|
||||||
|
pub fn find_idx(&self, id: OpId) -> Option<usize> {
|
||||||
|
self.ops.iter().position(|op| op.id == id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply an operation (both local and remote) to this local list CRDT.
|
||||||
|
/// Forwards it to a nested CRDT if necessary.
|
||||||
|
pub fn apply(&mut self, op: Op<Value>) -> OpState {
|
||||||
|
if !op.is_valid_hash() {
|
||||||
|
return OpState::ErrHashMismatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ensure_subpath(&self.path, &op.path) {
|
||||||
|
return OpState::ErrPathMismatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
// haven't reached end yet, navigate to inner CRDT
|
||||||
|
if op.path.len() - 1 > self.path.len() {
|
||||||
|
if let Some(PathSegment::Index(op_id)) = op.path.get(self.path.len()) {
|
||||||
|
let op_id = op_id.to_owned();
|
||||||
|
if let Some(idx) = self.find_idx(op_id) {
|
||||||
|
if self.ops[idx].content.is_none() {
|
||||||
|
return OpState::ErrListApplyToEmpty;
|
||||||
|
} else {
|
||||||
|
return self.ops[idx].content.as_mut().unwrap().apply(op);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
debug_path_mismatch(
|
||||||
|
join_path(self.path.to_owned(), PathSegment::Index(op_id)),
|
||||||
|
op.path,
|
||||||
|
);
|
||||||
|
return OpState::ErrPathMismatch;
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
debug_path_mismatch(self.path.to_owned(), op.path);
|
||||||
|
return OpState::ErrPathMismatch;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// otherwise, this is just a direct replacement
|
||||||
|
self.integrate(op.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Main CRDT logic of integrating an op properly into our local log
|
||||||
|
/// without causing conflicts. This is basically a really fancy
|
||||||
|
/// insertion sort.
|
||||||
|
///
|
||||||
|
/// Effectively, we
|
||||||
|
/// 1) find the parent item
|
||||||
|
/// 2) find the right spot to insert before the next node
|
||||||
|
fn integrate(&mut self, new_op: Op<T>) -> OpState {
|
||||||
|
let op_id = new_op.id;
|
||||||
|
let seq = new_op.sequence_num();
|
||||||
|
let origin_id = self.find_idx(new_op.origin);
|
||||||
|
|
||||||
|
if origin_id.is_none() {
|
||||||
|
self.message_q
|
||||||
|
.entry(new_op.origin)
|
||||||
|
.or_default()
|
||||||
|
.push(new_op);
|
||||||
|
return OpState::MissingCausalDependencies;
|
||||||
|
}
|
||||||
|
|
||||||
|
let new_op_parent_idx = origin_id.unwrap();
|
||||||
|
|
||||||
|
// if its a delete operation, we don't need to do much
|
||||||
|
self.log_apply(&new_op);
|
||||||
|
if new_op.is_deleted {
|
||||||
|
let op = &mut self.ops[new_op_parent_idx];
|
||||||
|
op.is_deleted = true;
|
||||||
|
return OpState::Ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
// otherwise, we are in an insert case
|
||||||
|
// start looking from right after parent
|
||||||
|
// stop when we reach end of document
|
||||||
|
let mut i = new_op_parent_idx + 1;
|
||||||
|
while i < self.ops.len() {
|
||||||
|
let op = &self.ops[i];
|
||||||
|
let op_parent_idx = self.find_idx(op.origin).unwrap();
|
||||||
|
|
||||||
|
// idempotency
|
||||||
|
if op.id == new_op.id {
|
||||||
|
return OpState::Ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
// first, lets compare causal origins
|
||||||
|
match new_op_parent_idx.cmp(&op_parent_idx) {
|
||||||
|
Ordering::Greater => break,
|
||||||
|
Ordering::Equal => {
|
||||||
|
// our parents our equal, we are siblings
|
||||||
|
// siblings are sorted first by sequence number then by author id
|
||||||
|
match new_op.sequence_num().cmp(&op.sequence_num()) {
|
||||||
|
Ordering::Greater => break,
|
||||||
|
Ordering::Equal => {
|
||||||
|
// conflict, resolve arbitrarily but deterministically
|
||||||
|
// tie-break on author id as that is unique
|
||||||
|
if new_op.author() > op.author() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ordering::Less => (),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ordering::Less => (),
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// insert at i
|
||||||
|
self.ops.insert(i, new_op);
|
||||||
|
self.our_seq = max(self.our_seq, seq);
|
||||||
|
self.log_ops(Some(op_id));
|
||||||
|
|
||||||
|
// apply all of its causal dependents if there are any
|
||||||
|
let dependent_queue = self.message_q.remove(&op_id);
|
||||||
|
if let Some(mut q) = dependent_queue {
|
||||||
|
for dependent in q.drain(..) {
|
||||||
|
self.integrate(dependent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
OpState::Ok
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make an iterator out of list CRDT contents, ignoring deleted items and empty content
|
||||||
|
pub fn iter(&self) -> impl Iterator<Item = &T> {
|
||||||
|
self.ops
|
||||||
|
.iter()
|
||||||
|
.filter(|op| !op.is_deleted && op.content.is_some())
|
||||||
|
.map(|op| op.content.as_ref().unwrap())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convenience function to get a vector of visible list elements
|
||||||
|
pub fn view(&self) -> Vec<T> {
|
||||||
|
self.iter().map(|i| i.to_owned()).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Debug for ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"[{}]",
|
||||||
|
self.ops
|
||||||
|
.iter()
|
||||||
|
.map(|op| format!("{:?}", op.id))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Allows us to index into a List CRDT like we would with an array
|
||||||
|
impl<T> Index<usize> for ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
type Output = T;
|
||||||
|
fn index(&self, idx: usize) -> &Self::Output {
|
||||||
|
let mut i = 0;
|
||||||
|
for op in &self.ops {
|
||||||
|
if !op.is_deleted && op.content.is_some() {
|
||||||
|
if idx == i {
|
||||||
|
return op.content.as_ref().unwrap();
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic!("index {idx} out of range (length of {i})")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Allows us to mutably index into a List CRDT like we would with an array
|
||||||
|
impl<T> IndexMut<usize> for ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn index_mut(&mut self, idx: usize) -> &mut Self::Output {
|
||||||
|
let mut i = 0;
|
||||||
|
for op in &mut self.ops {
|
||||||
|
if !op.is_deleted && op.content.is_some() {
|
||||||
|
if idx == i {
|
||||||
|
return op.content.as_mut().unwrap();
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic!("index {idx} out of range (length of {i})")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> CrdtNode for ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn apply(&mut self, op: Op<Value>) -> OpState {
|
||||||
|
self.apply(op.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn view(&self) -> Value {
|
||||||
|
self.view().into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new(id: AuthorId, path: Vec<PathSegment>) -> Self {
|
||||||
|
Self::new(id, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
use crate::debug::DebugView;
|
||||||
|
#[cfg(feature = "logging-base")]
|
||||||
|
impl<T> DebugView for ListCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode + DebugView,
|
||||||
|
{
|
||||||
|
fn debug_view(&self, indent: usize) -> String {
|
||||||
|
let spacing = " ".repeat(indent);
|
||||||
|
let path_str = print_path(self.path.clone());
|
||||||
|
let inner = self
|
||||||
|
.ops
|
||||||
|
.iter()
|
||||||
|
.map(|op| {
|
||||||
|
format!(
|
||||||
|
"{spacing}{}: {}",
|
||||||
|
&print_hex(&op.id)[..6],
|
||||||
|
op.debug_view(indent)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
format!("List CRDT @ /{path_str}\n{inner}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod test {
|
||||||
|
use crate::{json_crdt::OpState, keypair::make_author, list_crdt::ListCrdt, op::ROOT_ID};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_simple() {
|
||||||
|
let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
|
||||||
|
let _one = list.insert(ROOT_ID, 1);
|
||||||
|
let _two = list.insert(_one.id, 2);
|
||||||
|
let _three = list.insert(_two.id, 3);
|
||||||
|
let _four = list.insert(_one.id, 4);
|
||||||
|
assert_eq!(list.view(), vec![1, 4, 2, 3]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_idempotence() {
|
||||||
|
let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
|
||||||
|
let op = list.insert(ROOT_ID, 1);
|
||||||
|
for _ in 1..10 {
|
||||||
|
assert_eq!(list.apply(op.clone()), OpState::Ok);
|
||||||
|
}
|
||||||
|
assert_eq!(list.view(), vec![1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_delete() {
|
||||||
|
let mut list = ListCrdt::<char>::new(make_author(1), vec![]);
|
||||||
|
let _one = list.insert(ROOT_ID, 'a');
|
||||||
|
let _two = list.insert(_one.id, 'b');
|
||||||
|
let _three = list.insert(ROOT_ID, 'c');
|
||||||
|
list.delete(_one.id);
|
||||||
|
list.delete(_two.id);
|
||||||
|
assert_eq!(list.view(), vec!['c']);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_interweave_chars() {
|
||||||
|
let mut list = ListCrdt::<char>::new(make_author(1), vec![]);
|
||||||
|
let _one = list.insert(ROOT_ID, 'a');
|
||||||
|
let _two = list.insert(_one.id, 'b');
|
||||||
|
let _three = list.insert(ROOT_ID, 'c');
|
||||||
|
assert_eq!(list.view(), vec!['c', 'a', 'b']);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_conflicting_agents() {
|
||||||
|
let mut list1 = ListCrdt::<char>::new(make_author(1), vec![]);
|
||||||
|
let mut list2 = ListCrdt::new(make_author(2), vec![]);
|
||||||
|
let _1_a = list1.insert(ROOT_ID, 'a');
|
||||||
|
assert_eq!(list2.apply(_1_a.clone()), OpState::Ok);
|
||||||
|
let _2_b = list2.insert(_1_a.id, 'b');
|
||||||
|
assert_eq!(list1.apply(_2_b.clone()), OpState::Ok);
|
||||||
|
|
||||||
|
let _2_d = list2.insert(ROOT_ID, 'd');
|
||||||
|
let _2_y = list2.insert(_2_b.id, 'y');
|
||||||
|
let _1_x = list1.insert(_2_b.id, 'x');
|
||||||
|
|
||||||
|
// create artificial delay, then apply out of order
|
||||||
|
assert_eq!(list2.apply(_1_x), OpState::Ok);
|
||||||
|
assert_eq!(list1.apply(_2_y), OpState::Ok);
|
||||||
|
assert_eq!(list1.apply(_2_d), OpState::Ok);
|
||||||
|
|
||||||
|
assert_eq!(list1.view(), vec!['d', 'a', 'b', 'y', 'x']);
|
||||||
|
assert_eq!(list1.view(), list2.view());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_delete_multiple_agent() {
|
||||||
|
let mut list1 = ListCrdt::<char>::new(make_author(1), vec![]);
|
||||||
|
let mut list2 = ListCrdt::new(make_author(2), vec![]);
|
||||||
|
let _1_a = list1.insert(ROOT_ID, 'a');
|
||||||
|
assert_eq!(list2.apply(_1_a.clone()), OpState::Ok);
|
||||||
|
let _2_b = list2.insert(_1_a.id, 'b');
|
||||||
|
let del_1_a = list1.delete(_1_a.id);
|
||||||
|
assert_eq!(list1.apply(_2_b), OpState::Ok);
|
||||||
|
assert_eq!(list2.apply(del_1_a), OpState::Ok);
|
||||||
|
|
||||||
|
assert_eq!(list1.view(), vec!['b']);
|
||||||
|
assert_eq!(list1.view(), list2.view());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_nested() {
|
||||||
|
let mut list1 = ListCrdt::<char>::new(make_author(1), vec![]);
|
||||||
|
let _c = list1.insert(ROOT_ID, 'c');
|
||||||
|
let _a = list1.insert(ROOT_ID, 'a');
|
||||||
|
let _d = list1.insert(_c.id, 'd');
|
||||||
|
let _b = list1.insert(_a.id, 'b');
|
||||||
|
|
||||||
|
assert_eq!(list1.view(), vec!['a', 'b', 'c', 'd']);
|
||||||
|
}
|
||||||
|
}
|
||||||
192
crates/bft-json-crdt/src/lww_crdt.rs
Normal file
192
crates/bft-json-crdt/src/lww_crdt.rs
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
use crate::debug::DebugView;
|
||||||
|
use crate::json_crdt::{CrdtNode, OpState, Value};
|
||||||
|
use crate::op::{join_path, print_path, Op, PathSegment, SequenceNumber};
|
||||||
|
use std::cmp::{max, Ordering};
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use crate::keypair::AuthorId;
|
||||||
|
|
||||||
|
/// A simple delete-wins, last-writer-wins (LWW) register CRDT.
|
||||||
|
/// Basically only for adding support for primitives within a more complex CRDT
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct LwwRegisterCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
/// Public key for this node
|
||||||
|
pub our_id: AuthorId,
|
||||||
|
/// Path to this CRDT
|
||||||
|
pub path: Vec<PathSegment>,
|
||||||
|
/// Internal value of this CRDT. We wrap it in an Op to retain the author/sequence metadata
|
||||||
|
value: Op<T>,
|
||||||
|
/// The sequence number of this node
|
||||||
|
our_seq: SequenceNumber,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> LwwRegisterCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
/// Create a new register CRDT with the given [`AuthorID`] (it should be unique)
|
||||||
|
pub fn new(id: AuthorId, path: Vec<PathSegment>) -> LwwRegisterCrdt<T> {
|
||||||
|
LwwRegisterCrdt {
|
||||||
|
our_id: id,
|
||||||
|
path,
|
||||||
|
value: Op::make_root(),
|
||||||
|
our_seq: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sets the current value of the register
|
||||||
|
pub fn set<U: Into<Value>>(&mut self, content: U) -> Op<Value> {
|
||||||
|
let mut op = Op::new(
|
||||||
|
self.value.id,
|
||||||
|
self.our_id,
|
||||||
|
self.our_seq + 1,
|
||||||
|
false,
|
||||||
|
Some(content.into()),
|
||||||
|
self.path.to_owned(),
|
||||||
|
);
|
||||||
|
|
||||||
|
// we need to know the op ID before setting the path as [`PathSegment::Index`] requires an
|
||||||
|
// [`OpID`]
|
||||||
|
let new_path = join_path(self.path.to_owned(), PathSegment::Index(op.id));
|
||||||
|
op.path = new_path;
|
||||||
|
self.apply(op.clone());
|
||||||
|
op
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply an operation (both local and remote) to this local register CRDT.
|
||||||
|
pub fn apply(&mut self, op: Op<Value>) -> OpState {
|
||||||
|
if !op.is_valid_hash() {
|
||||||
|
return OpState::ErrHashMismatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
let op: Op<T> = op.into();
|
||||||
|
let seq = op.sequence_num();
|
||||||
|
|
||||||
|
// take most recent update by sequence number
|
||||||
|
match seq.cmp(&self.our_seq) {
|
||||||
|
Ordering::Greater => {
|
||||||
|
self.value = Op {
|
||||||
|
id: self.value.id,
|
||||||
|
..op
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Ordering::Equal => {
|
||||||
|
// if we are equal, tie break on author
|
||||||
|
if op.author() < self.value.author() {
|
||||||
|
// we want to keep id constant so replace everything but id
|
||||||
|
self.value = Op {
|
||||||
|
id: self.value.id,
|
||||||
|
..op
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ordering::Less => {} // LWW, ignore if its outdate
|
||||||
|
};
|
||||||
|
|
||||||
|
// update bookkeeping
|
||||||
|
self.our_seq = max(self.our_seq, seq);
|
||||||
|
OpState::Ok
|
||||||
|
}
|
||||||
|
|
||||||
|
fn view(&self) -> Option<T> {
|
||||||
|
self.value.content.to_owned()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> CrdtNode for LwwRegisterCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn apply(&mut self, op: Op<Value>) -> OpState {
|
||||||
|
self.apply(op.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn view(&self) -> Value {
|
||||||
|
self.view().into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new(id: AuthorId, path: Vec<PathSegment>) -> Self {
|
||||||
|
Self::new(id, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> DebugView for LwwRegisterCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode + DebugView,
|
||||||
|
{
|
||||||
|
fn debug_view(&self, indent: usize) -> String {
|
||||||
|
let spacing = " ".repeat(indent);
|
||||||
|
let path_str = print_path(self.path.clone());
|
||||||
|
let inner = self.value.debug_view(indent + 2);
|
||||||
|
format!("LWW Register CRDT @ /{path_str}\n{spacing}{inner}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Debug for LwwRegisterCrdt<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(f, "{:?}", self.value.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod test {
|
||||||
|
use super::LwwRegisterCrdt;
|
||||||
|
use crate::{json_crdt::OpState, keypair::make_author};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_lww_simple() {
|
||||||
|
let mut register = LwwRegisterCrdt::new(make_author(1), vec![]);
|
||||||
|
assert_eq!(register.view(), None);
|
||||||
|
register.set(1);
|
||||||
|
assert_eq!(register.view(), Some(1));
|
||||||
|
register.set(99);
|
||||||
|
assert_eq!(register.view(), Some(99));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_lww_multiple_writer() {
|
||||||
|
let mut register1 = LwwRegisterCrdt::new(make_author(1), vec![]);
|
||||||
|
let mut register2 = LwwRegisterCrdt::new(make_author(2), vec![]);
|
||||||
|
let _a = register1.set('a');
|
||||||
|
let _b = register1.set('b');
|
||||||
|
let _c = register2.set('c');
|
||||||
|
assert_eq!(register2.view(), Some('c'));
|
||||||
|
assert_eq!(register1.apply(_c), OpState::Ok);
|
||||||
|
assert_eq!(register2.apply(_b), OpState::Ok);
|
||||||
|
assert_eq!(register2.apply(_a), OpState::Ok);
|
||||||
|
assert_eq!(register1.view(), Some('b'));
|
||||||
|
assert_eq!(register2.view(), Some('b'));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_lww_idempotence() {
|
||||||
|
let mut register = LwwRegisterCrdt::new(make_author(1), vec![]);
|
||||||
|
let op = register.set(1);
|
||||||
|
for _ in 1..10 {
|
||||||
|
assert_eq!(register.apply(op.clone()), OpState::Ok);
|
||||||
|
}
|
||||||
|
assert_eq!(register.view(), Some(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_lww_consistent_tiebreak() {
|
||||||
|
let mut register1 = LwwRegisterCrdt::new(make_author(1), vec![]);
|
||||||
|
let mut register2 = LwwRegisterCrdt::new(make_author(2), vec![]);
|
||||||
|
let _a = register1.set('a');
|
||||||
|
let _b = register2.set('b');
|
||||||
|
assert_eq!(register1.apply(_b), OpState::Ok);
|
||||||
|
assert_eq!(register2.apply(_a), OpState::Ok);
|
||||||
|
let _c = register1.set('c');
|
||||||
|
let _d = register2.set('d');
|
||||||
|
assert_eq!(register2.apply(_c), OpState::Ok);
|
||||||
|
assert_eq!(register1.apply(_d), OpState::Ok);
|
||||||
|
assert_eq!(register1.view(), register2.view());
|
||||||
|
assert_eq!(register1.view(), Some('c'));
|
||||||
|
}
|
||||||
|
}
|
||||||
237
crates/bft-json-crdt/src/op.rs
Normal file
237
crates/bft-json-crdt/src/op.rs
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
use crate::debug::{debug_path_mismatch, debug_type_mismatch};
|
||||||
|
use crate::json_crdt::{CrdtNode, CrdtNodeFromValue, IntoCrdtNode, SignedOp, Value};
|
||||||
|
use crate::keypair::{sha256, AuthorId};
|
||||||
|
use fastcrypto::ed25519::Ed25519KeyPair;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
/// A lamport clock timestamp. Used to track document versions
|
||||||
|
pub type SequenceNumber = u64;
|
||||||
|
|
||||||
|
/// A unique ID for a single [`Op<T>`]
|
||||||
|
pub type OpId = [u8; 32];
|
||||||
|
|
||||||
|
/// The root/sentinel op
|
||||||
|
pub const ROOT_ID: OpId = [0u8; 32];
|
||||||
|
|
||||||
|
/// Part of a path to get to a specific CRDT in a nested CRDT
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub enum PathSegment {
|
||||||
|
Field(String),
|
||||||
|
Index(OpId),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Format a byte array as a hex string
|
||||||
|
pub fn print_hex<const N: usize>(bytes: &[u8; N]) -> String {
|
||||||
|
bytes
|
||||||
|
.iter()
|
||||||
|
.map(|byte| format!("{byte:02x}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pretty print a path
|
||||||
|
pub fn print_path(path: Vec<PathSegment>) -> String {
|
||||||
|
path.iter()
|
||||||
|
.map(|p| match p {
|
||||||
|
PathSegment::Field(s) => s.to_string(),
|
||||||
|
PathSegment::Index(i) => print_hex(i)[..6].to_string(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(".")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ensure our_path is a subpath of op_path. Note that two identical paths are considered subpaths
|
||||||
|
/// of each other.
|
||||||
|
pub fn ensure_subpath(our_path: &Vec<PathSegment>, op_path: &Vec<PathSegment>) -> bool {
|
||||||
|
// if our_path is longer, it cannot be a subpath
|
||||||
|
if our_path.len() > op_path.len() {
|
||||||
|
debug_path_mismatch(our_path.to_owned(), op_path.to_owned());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// iterate to end of our_path, ensuring each element is the same
|
||||||
|
for i in 0..our_path.len() {
|
||||||
|
let ours = our_path.get(i);
|
||||||
|
let theirs = op_path.get(i);
|
||||||
|
if ours != theirs {
|
||||||
|
debug_path_mismatch(our_path.to_owned(), op_path.to_owned());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper to easily append a [`PathSegment`] to a path
|
||||||
|
pub fn join_path(path: Vec<PathSegment>, segment: PathSegment) -> Vec<PathSegment> {
|
||||||
|
let mut p = path;
|
||||||
|
p.push(segment);
|
||||||
|
p
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse out the field from a [`PathSegment`]
|
||||||
|
pub fn parse_field(path: Vec<PathSegment>) -> Option<String> {
|
||||||
|
path.last().and_then(|segment| {
|
||||||
|
if let PathSegment::Field(key) = segment {
|
||||||
|
Some(key.to_string())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Represents a single node in a CRDT
|
||||||
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
pub struct Op<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
pub origin: OpId,
|
||||||
|
pub author: AuthorId, // pub key of author
|
||||||
|
pub seq: SequenceNumber,
|
||||||
|
pub content: Option<T>,
|
||||||
|
pub path: Vec<PathSegment>, // path to get to target CRDT
|
||||||
|
pub is_deleted: bool,
|
||||||
|
pub id: OpId, // hash of the operation
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Something can be turned into a string. This allows us to use [`content`] as in
|
||||||
|
/// input into the SHA256 hash
|
||||||
|
pub trait Hashable {
|
||||||
|
fn hash(&self) -> String;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Anything that implements Debug is trivially hashable
|
||||||
|
impl<T> Hashable for T
|
||||||
|
where
|
||||||
|
T: Debug,
|
||||||
|
{
|
||||||
|
fn hash(&self) -> String {
|
||||||
|
format!("{self:?}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Conversion from Op<Value> -> Op<T> given that T is a CRDT that can be created from a JSON value
|
||||||
|
impl Op<Value> {
|
||||||
|
pub fn into<T: CrdtNodeFromValue + CrdtNode>(self) -> Op<T> {
|
||||||
|
let content = if let Some(inner_content) = self.content {
|
||||||
|
match inner_content.into_node(self.id, self.path.clone()) {
|
||||||
|
Ok(node) => Some(node),
|
||||||
|
Err(msg) => {
|
||||||
|
debug_type_mismatch(msg);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
Op {
|
||||||
|
content,
|
||||||
|
origin: self.origin,
|
||||||
|
author: self.author,
|
||||||
|
seq: self.seq,
|
||||||
|
path: self.path,
|
||||||
|
is_deleted: self.is_deleted,
|
||||||
|
id: self.id,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Op<T>
|
||||||
|
where
|
||||||
|
T: CrdtNode,
|
||||||
|
{
|
||||||
|
pub fn sign(self, keypair: &Ed25519KeyPair) -> SignedOp {
|
||||||
|
SignedOp::from_op(self, keypair, vec![])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sign_with_dependencies(
|
||||||
|
self,
|
||||||
|
keypair: &Ed25519KeyPair,
|
||||||
|
dependencies: Vec<&SignedOp>,
|
||||||
|
) -> SignedOp {
|
||||||
|
SignedOp::from_op(
|
||||||
|
self,
|
||||||
|
keypair,
|
||||||
|
dependencies
|
||||||
|
.iter()
|
||||||
|
.map(|dep| dep.signed_digest)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn author(&self) -> AuthorId {
|
||||||
|
self.author
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sequence_num(&self) -> SequenceNumber {
|
||||||
|
self.seq
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new(
|
||||||
|
origin: OpId,
|
||||||
|
author: AuthorId,
|
||||||
|
seq: SequenceNumber,
|
||||||
|
is_deleted: bool,
|
||||||
|
content: Option<T>,
|
||||||
|
path: Vec<PathSegment>,
|
||||||
|
) -> Op<T> {
|
||||||
|
let mut op = Self {
|
||||||
|
origin,
|
||||||
|
id: ROOT_ID,
|
||||||
|
author,
|
||||||
|
seq,
|
||||||
|
is_deleted,
|
||||||
|
content,
|
||||||
|
path,
|
||||||
|
};
|
||||||
|
op.id = op.hash_to_id();
|
||||||
|
op
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generate OpID by hashing our contents. Hash includes
|
||||||
|
/// - content
|
||||||
|
/// - origin
|
||||||
|
/// - author
|
||||||
|
/// - seq
|
||||||
|
/// - is_deleted
|
||||||
|
pub fn hash_to_id(&self) -> OpId {
|
||||||
|
let content_str = match self.content.as_ref() {
|
||||||
|
Some(content) => content.hash(),
|
||||||
|
None => "".to_string(),
|
||||||
|
};
|
||||||
|
let fmt_str = format!(
|
||||||
|
"{:?},{:?},{:?},{:?},{content_str}",
|
||||||
|
self.origin, self.author, self.seq, self.is_deleted,
|
||||||
|
);
|
||||||
|
sha256(fmt_str)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rehashes the contents to make sure it matches the ID
|
||||||
|
pub fn is_valid_hash(&self) -> bool {
|
||||||
|
// make sure content is only none for deletion events
|
||||||
|
if self.content.is_none() && !self.is_deleted {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// try to avoid expensive sig check if early fail
|
||||||
|
let res = self.hash_to_id() == self.id;
|
||||||
|
if !res {
|
||||||
|
self.debug_hash_failure();
|
||||||
|
}
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Special constructor for defining the sentinel root node
|
||||||
|
pub fn make_root() -> Op<T> {
|
||||||
|
Self {
|
||||||
|
origin: ROOT_ID,
|
||||||
|
id: ROOT_ID,
|
||||||
|
author: [0u8; 32],
|
||||||
|
seq: 0,
|
||||||
|
is_deleted: false,
|
||||||
|
content: None,
|
||||||
|
path: vec![],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
134
crates/bft-json-crdt/tests/byzantine.rs
Normal file
134
crates/bft-json-crdt/tests/byzantine.rs
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
use bft_json_crdt::{
|
||||||
|
json_crdt::{add_crdt_fields, BaseCrdt, CrdtNode, IntoCrdtNode, OpState},
|
||||||
|
keypair::make_keypair,
|
||||||
|
list_crdt::ListCrdt,
|
||||||
|
lww_crdt::LwwRegisterCrdt,
|
||||||
|
op::{Op, PathSegment, ROOT_ID},
|
||||||
|
};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
// What is potentially Byzantine behaviour?
|
||||||
|
// 1. send valid updates
|
||||||
|
// 2. send a mix of valid and invalid updates
|
||||||
|
// a) messages with duplicate ID (attempt to overwrite old entries)
|
||||||
|
// b) send incorrect sequence number to multiple nodes (which could lead to divergent state) -- this is called equivocation
|
||||||
|
// c) ‘forge’ updates from another author (could happen when forwarding valid messages from peers)
|
||||||
|
// 3. send malformed updates (e.g. missing fields)
|
||||||
|
// this we don't test as we assume transport layer only allows valid messages
|
||||||
|
// 4. overwhelm message queue by sending many updates far into the future
|
||||||
|
// also untestested! currently we keep an unbounded message queue
|
||||||
|
// 5. block actual messages from honest actors (eclipse attack)
|
||||||
|
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct ListExample {
|
||||||
|
list: ListCrdt<char>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// case 2a + 2b
|
||||||
|
#[test]
|
||||||
|
fn test_equivocation() {
|
||||||
|
let key = make_keypair();
|
||||||
|
let testkey = make_keypair();
|
||||||
|
let mut crdt = BaseCrdt::<ListExample>::new(&key);
|
||||||
|
let mut testcrdt = BaseCrdt::<ListExample>::new(&testkey);
|
||||||
|
let _a = crdt.doc.list.insert(ROOT_ID, 'a').sign(&key);
|
||||||
|
let _b = crdt.doc.list.insert(_a.id(), 'b').sign(&key);
|
||||||
|
|
||||||
|
// make a fake operation with same id as _b but different content
|
||||||
|
let mut fake_op = _b.clone();
|
||||||
|
fake_op.inner.content = Some('c'.into());
|
||||||
|
|
||||||
|
// also try modifying the sequence number
|
||||||
|
let mut fake_op_seq = _b.clone();
|
||||||
|
fake_op_seq.inner.seq = 99;
|
||||||
|
fake_op_seq.inner.is_deleted = true;
|
||||||
|
|
||||||
|
assert_eq!(crdt.apply(fake_op.clone()), OpState::ErrHashMismatch);
|
||||||
|
assert_eq!(crdt.apply(fake_op_seq.clone()), OpState::ErrHashMismatch);
|
||||||
|
|
||||||
|
assert_eq!(testcrdt.apply(fake_op_seq), OpState::ErrHashMismatch);
|
||||||
|
assert_eq!(testcrdt.apply(fake_op), OpState::ErrHashMismatch);
|
||||||
|
assert_eq!(testcrdt.apply(_a), OpState::Ok);
|
||||||
|
assert_eq!(testcrdt.apply(_b), OpState::Ok);
|
||||||
|
|
||||||
|
// make sure it doesnt accept either of the fake operations
|
||||||
|
assert_eq!(crdt.doc.list.view(), vec!['a', 'b']);
|
||||||
|
assert_eq!(crdt.doc.list.view(), testcrdt.doc.list.view());
|
||||||
|
}
|
||||||
|
|
||||||
|
// case 2c
|
||||||
|
#[test]
|
||||||
|
fn test_forge_update() {
|
||||||
|
let key = make_keypair();
|
||||||
|
let testkey = make_keypair();
|
||||||
|
let mut crdt = BaseCrdt::<ListExample>::new(&key);
|
||||||
|
let mut testcrdt = BaseCrdt::<ListExample>::new(&testkey);
|
||||||
|
let _a = crdt.doc.list.insert(ROOT_ID, 'a').sign(&key);
|
||||||
|
|
||||||
|
let fake_key = make_keypair(); // generate a new keypair as we dont have privkey of list.our_id
|
||||||
|
let mut op = Op {
|
||||||
|
origin: _a.inner.id,
|
||||||
|
author: crdt.doc.id, // pretend to be the owner of list
|
||||||
|
content: Some('b'),
|
||||||
|
path: vec![PathSegment::Field("list".to_string())],
|
||||||
|
seq: 1,
|
||||||
|
is_deleted: false,
|
||||||
|
id: ROOT_ID, // placeholder, to be generated
|
||||||
|
};
|
||||||
|
|
||||||
|
// this is a completely valid hash and digest, just signed by the wrong person
|
||||||
|
// as keypair.public != list.public
|
||||||
|
op.id = op.hash_to_id();
|
||||||
|
let signed = op.sign(&fake_key);
|
||||||
|
|
||||||
|
assert_eq!(crdt.apply(signed.clone()), OpState::ErrHashMismatch);
|
||||||
|
assert_eq!(testcrdt.apply(signed), OpState::ErrHashMismatch);
|
||||||
|
assert_eq!(testcrdt.apply(_a), OpState::Ok);
|
||||||
|
|
||||||
|
// make sure it doesnt accept fake operation
|
||||||
|
assert_eq!(crdt.doc.list.view(), vec!['a']);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Nested {
|
||||||
|
a: Nested2,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode)]
|
||||||
|
struct Nested2 {
|
||||||
|
b: LwwRegisterCrdt<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_path_update() {
|
||||||
|
let key = make_keypair();
|
||||||
|
let testkey = make_keypair();
|
||||||
|
let mut crdt = BaseCrdt::<Nested>::new(&key);
|
||||||
|
let mut testcrdt = BaseCrdt::<Nested>::new(&testkey);
|
||||||
|
let mut _true = crdt.doc.a.b.set(true);
|
||||||
|
_true.path = vec![PathSegment::Field("x".to_string())];
|
||||||
|
let mut _false = crdt.doc.a.b.set(false);
|
||||||
|
_false.path = vec![
|
||||||
|
PathSegment::Field("a".to_string()),
|
||||||
|
PathSegment::Index(_false.id),
|
||||||
|
];
|
||||||
|
|
||||||
|
let signedtrue = _true.sign(&key);
|
||||||
|
let signedfalse = _false.sign(&key);
|
||||||
|
let mut signedfalsefakepath = signedfalse.clone();
|
||||||
|
signedfalsefakepath.inner.path = vec![
|
||||||
|
PathSegment::Field("a".to_string()),
|
||||||
|
PathSegment::Field("b".to_string()),
|
||||||
|
];
|
||||||
|
|
||||||
|
assert_eq!(testcrdt.apply(signedtrue), OpState::ErrPathMismatch);
|
||||||
|
assert_eq!(testcrdt.apply(signedfalse), OpState::ErrPathMismatch);
|
||||||
|
assert_eq!(testcrdt.apply(signedfalsefakepath), OpState::ErrDigestMismatch);
|
||||||
|
|
||||||
|
// make sure it doesnt accept fake operation
|
||||||
|
assert_eq!(crdt.doc.a.b.view(), json!(false).into());
|
||||||
|
assert_eq!(testcrdt.doc.a.b.view(), json!(null).into());
|
||||||
|
}
|
||||||
91
crates/bft-json-crdt/tests/commutative.rs
Normal file
91
crates/bft-json-crdt/tests/commutative.rs
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
use bft_json_crdt::{
|
||||||
|
keypair::make_author,
|
||||||
|
list_crdt::ListCrdt,
|
||||||
|
op::{Op, OpId, ROOT_ID}, json_crdt::{CrdtNode, Value},
|
||||||
|
};
|
||||||
|
use rand::{rngs::ThreadRng, seq::SliceRandom, Rng};
|
||||||
|
|
||||||
|
fn random_op<T: CrdtNode>(arr: &Vec<Op<T>>, rng: &mut ThreadRng) -> OpId {
|
||||||
|
arr.choose(rng).map(|op| op.id).unwrap_or(ROOT_ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
const TEST_N: usize = 100;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_list_fuzz_commutative() {
|
||||||
|
let mut rng = rand::thread_rng();
|
||||||
|
let mut op_log = Vec::<Op<Value>>::new();
|
||||||
|
let mut op_log1 = Vec::<Op<Value>>::new();
|
||||||
|
let mut op_log2 = Vec::<Op<Value>>::new();
|
||||||
|
let mut l1 = ListCrdt::<char>::new(make_author(1), vec![]);
|
||||||
|
let mut l2 = ListCrdt::<char>::new(make_author(2), vec![]);
|
||||||
|
let mut chk = ListCrdt::<char>::new(make_author(3), vec![]);
|
||||||
|
for _ in 0..TEST_N {
|
||||||
|
let letter1: char = rng.gen_range(b'a'..=b'z') as char;
|
||||||
|
let letter2: char = rng.gen_range(b'a'..=b'z') as char;
|
||||||
|
let op1 = if rng.gen_bool(4.0 / 5.0) {
|
||||||
|
l1.insert(random_op(&op_log1, &mut rng), letter1)
|
||||||
|
} else {
|
||||||
|
l1.delete(random_op(&op_log1, &mut rng))
|
||||||
|
};
|
||||||
|
let op2 = if rng.gen_bool(4.0 / 5.0) {
|
||||||
|
l2.insert(random_op(&op_log2, &mut rng), letter2)
|
||||||
|
} else {
|
||||||
|
l2.delete(random_op(&op_log2, &mut rng))
|
||||||
|
};
|
||||||
|
op_log1.push(op1.clone());
|
||||||
|
op_log2.push(op2.clone());
|
||||||
|
op_log.push(op1.clone());
|
||||||
|
op_log.push(op2.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
// shuffle ops
|
||||||
|
op_log1.shuffle(&mut rng);
|
||||||
|
op_log2.shuffle(&mut rng);
|
||||||
|
|
||||||
|
// apply to each other
|
||||||
|
for op in op_log1 {
|
||||||
|
l2.apply(op.clone());
|
||||||
|
chk.apply(op.into());
|
||||||
|
}
|
||||||
|
for op in op_log2 {
|
||||||
|
l1.apply(op.clone());
|
||||||
|
chk.apply(op);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure all equal
|
||||||
|
let l1_doc = l1.view();
|
||||||
|
let l2_doc = l2.view();
|
||||||
|
let chk_doc = chk.view();
|
||||||
|
assert_eq!(l1_doc, l2_doc);
|
||||||
|
assert_eq!(l1_doc, chk_doc);
|
||||||
|
assert_eq!(l2_doc, chk_doc);
|
||||||
|
|
||||||
|
// now, allow cross mixing between both
|
||||||
|
let mut op_log1 = Vec::<Op<Value>>::new();
|
||||||
|
let mut op_log2 = Vec::<Op<Value>>::new();
|
||||||
|
for _ in 0..TEST_N {
|
||||||
|
let letter1: char = rng.gen_range(b'a'..=b'z') as char;
|
||||||
|
let letter2: char = rng.gen_range(b'a'..=b'z') as char;
|
||||||
|
let op1 = l1.insert(random_op(&op_log, &mut rng), letter1);
|
||||||
|
let op2 = l2.insert(random_op(&op_log, &mut rng), letter2);
|
||||||
|
op_log1.push(op1);
|
||||||
|
op_log2.push(op2);
|
||||||
|
}
|
||||||
|
|
||||||
|
for op in op_log1 {
|
||||||
|
l2.apply(op.clone());
|
||||||
|
chk.apply(op);
|
||||||
|
}
|
||||||
|
for op in op_log2 {
|
||||||
|
l1.apply(op.clone());
|
||||||
|
chk.apply(op);
|
||||||
|
}
|
||||||
|
|
||||||
|
let l1_doc = l1.view();
|
||||||
|
let l2_doc = l2.view();
|
||||||
|
let chk_doc = chk.view();
|
||||||
|
assert_eq!(l1_doc, l2_doc);
|
||||||
|
assert_eq!(l1_doc, chk_doc);
|
||||||
|
assert_eq!(l2_doc, chk_doc);
|
||||||
|
}
|
||||||
259883
crates/bft-json-crdt/tests/editing-trace.js
generated
Normal file
259883
crates/bft-json-crdt/tests/editing-trace.js
generated
Normal file
File diff suppressed because one or more lines are too long
1
crates/bft-json-crdt/tests/edits.json
generated
Normal file
1
crates/bft-json-crdt/tests/edits.json
generated
Normal file
File diff suppressed because one or more lines are too long
76
crates/bft-json-crdt/tests/kleppmann_trace.rs
Normal file
76
crates/bft-json-crdt/tests/kleppmann_trace.rs
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
use bft_json_crdt::keypair::make_author;
|
||||||
|
use bft_json_crdt::list_crdt::ListCrdt;
|
||||||
|
use bft_json_crdt::op::{OpId, ROOT_ID};
|
||||||
|
use std::{fs::File, io::Read};
|
||||||
|
use time::PreciseTime;
|
||||||
|
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, Clone)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct Edit {
|
||||||
|
pos: usize,
|
||||||
|
delete: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
content: Option<char>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct Trace {
|
||||||
|
final_text: String,
|
||||||
|
edits: Vec<Edit>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_trace() -> Trace {
|
||||||
|
let fp = "./tests/edits.json";
|
||||||
|
match File::open(fp) {
|
||||||
|
Err(e) => panic!("Open edits.json failed: {:?}", e.kind()),
|
||||||
|
Ok(mut file) => {
|
||||||
|
let mut content: String = String::new();
|
||||||
|
file.read_to_string(&mut content)
|
||||||
|
.expect("Problem reading file");
|
||||||
|
serde_json::from_str(&content).expect("JSON was not well-formatted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Really large test to run Martin Kleppmann's
|
||||||
|
/// editing trace over his paper
|
||||||
|
/// Data source: https://github.com/automerge/automerge-perf
|
||||||
|
#[test]
|
||||||
|
fn test_editing_trace() {
|
||||||
|
let t = get_trace();
|
||||||
|
let mut list = ListCrdt::<char>::new(make_author(1), vec![]);
|
||||||
|
let mut ops: Vec<OpId> = Vec::new();
|
||||||
|
ops.push(ROOT_ID);
|
||||||
|
let start = PreciseTime::now();
|
||||||
|
let edits = t.edits;
|
||||||
|
for (i, op) in edits.into_iter().enumerate() {
|
||||||
|
let origin = ops[op.pos];
|
||||||
|
if op.delete {
|
||||||
|
let delete_op = list.delete(origin);
|
||||||
|
ops.push(delete_op.id);
|
||||||
|
} else {
|
||||||
|
let new_op = list.insert(origin, op.content.unwrap());
|
||||||
|
ops.push(new_op.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
match i {
|
||||||
|
10_000 | 100_000 => {
|
||||||
|
let end = PreciseTime::now();
|
||||||
|
let runtime_sec = start.to(end);
|
||||||
|
println!("took {runtime_sec:?} to run {i} ops");
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let end = PreciseTime::now();
|
||||||
|
let runtime_sec = start.to(end);
|
||||||
|
println!("took {runtime_sec:?} to finish");
|
||||||
|
let result = list.iter().collect::<String>();
|
||||||
|
let expected = t.final_text;
|
||||||
|
assert_eq!(result.len(), expected.len());
|
||||||
|
assert_eq!(result, expected);
|
||||||
|
}
|
||||||
@@ -9,8 +9,8 @@ edition = "2021"
|
|||||||
clap = { version = "4.5.4", features = ["derive"] }
|
clap = { version = "4.5.4", features = ["derive"] }
|
||||||
tokio = { version = "1.37.0", features = ["time"] }
|
tokio = { version = "1.37.0", features = ["time"] }
|
||||||
websockets = "0.3.0"
|
websockets = "0.3.0"
|
||||||
bft-json-crdt = { path = "../../bft-json-crdt" }
|
bft-json-crdt = { path = "../crates/bft-json-crdt" }
|
||||||
bft-crdt-derive = { path = "../../bft-json-crdt/bft-crdt-derive" }
|
bft-crdt-derive = { path = "../crates/bft-json-crdt/bft-crdt-derive" }
|
||||||
# serde_cbor = "0.11.2" # move to this once we need to pack things in CBOR
|
# serde_cbor = "0.11.2" # move to this once we need to pack things in CBOR
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0.117"
|
serde_json = "1.0.117"
|
||||||
|
|||||||
Reference in New Issue
Block a user