Minor readme edit

I'll keep it around for a while in case I run into trouble with esplora/mutiny

.idea/.gitignore

@@ -0,0 +1,5 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/

.idea/side-crdt.iml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="EMPTY_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/crates/bft-json-crdt/benches" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/crates/bft-json-crdt/bft-crdt-derive/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/crates/bft-json-crdt/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/crates/bft-json-crdt/tests" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/crdt-node/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/crdt-node/tests" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/crdt-relayer/src" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

Cargo.toml

@@ -1,4 +1,4 @@
 [workspace]
 
-members = ["side-node", "side-watcher"]
+members = ["crdt-node", "crdt-relayer"]
 resolver = "2"

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Side Protocol
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,101 @@
+# BFT-CRDT PoC
+
+This is a proof of concept implementation of a [BFT-CRDT](https://jzhao.xyz/posts/bft-json-crdt) blockchain-like system. It is willfully, wildly insecure as a blockchain right now. Think of it as an experiment which is strictly for fun and poking at ideas.
+
+This code is based on the ideas of [Martin Kleppmann](https://martin.kleppmann.com/papers/bft-crdt-papoc22.pdf) and the ideas and code of [Jacky Zhao](https://jzhao.xyz/). Have a read, they are both excellent writers and have some of the most interesting computing ideas I've run across in quite a while.
+
+It is not clear what this thing is for, yet. It's not a blockchain. It makes a kind of secure DAG. It uses BFT-CRDTs to make a Sybil-proof and secure information transmission system for messages, with eventual consistency guarantees.
+
+The idea that it could be possible to set up a secure Sybil-proof system, negating the energy burn required for proof of work, the financially exclusionary proof of stake, or the meat space hassle of a proof of personhood ceremony, is too attractive to ignore. At least, if you're interested in cool P2P systems.
+
+## Prerequisites
+
+Install a recent version of Rust.
+
+## Running in development
+
+Run the watcher first:
+
+```bash
+cd crdt-relayer
+cargo watch -x run
+```
+
+To init a Side node:
+
+```bash
+cd crdt-node
+cargo run -- init node1
+cargo run -- init node2
+cargo run -- init node3
+cargo run -- init node4
+```
+
+To start a node with a cargo watch for development purposes (from the `crdt-node` dir), open up a few terminals and run:
+
+```bash
+cargo watch -x "run -- run -- node1"
+cargo watch -x "run -- run -- node2"
+cargo watch -x "run -- run -- node3"
+cargo watch -x "run -- run -- node4"
+```
+You can then type directly into each of the Crdt Node consoles. Messages will be relayed to each Crdt Node, and the transaction history will end up being the same on all nodes.
+
+## Discussion
+
+What we have here is a very simple system comprised of two parts: the Crdt Node, and the Crdt Relayer.
+
+### Crdt Node(s)
+
+The Crdt Nodes make up a system of BFT-CRDT-producing nodes that can make a sort of wildly insecure blockchain. Currently, they can reliably send transactions to each other in a secure way, such that all nodes they communicate with can tell whether received transactions are obeying the rules of the system.
+
+The Crdt Node does not download any chain state, and if one goes off-line it will miss transactions. This is expected at the moment and fairly easy to fix, with a bit of work.
+
+### Crdt Relayer
+
+The Crdt Relayer replicates transactions between nodes using a websocket. We aim to eliminate this component from the architecture, but for the moment it simplifies networking and consensus agreement while we experiment with higher-value concepts.
+
+Later, we will aim to remove the Crdt Relayer from the architecture, by (a) moving to pure P2P transactions between Crdt Nodes, and (b) doing leader election of a Crdt Node to reach agreement on the submitted block.
+
+## Bitcoin integration
+
+There is a Bitcoin client integrated into the node, which can do simple coin transfers using esplora and the Mutinynet server's Signet (30 second blocktime).
+
+The client's demo driver can be run by doing:
+
+```
+cargo run -- init dave
+cargo run -- init sammy
+cargo run -- btc
+```
+
+You'll need to have funded the "dave" address prior to running the `btc` command - otherwise the transfer will fail gracefully.
+
+I was using this primarily as a way to experiment with constructing and broadcasting Bitcoin transactions, with the hope that it would be possible to move on to more advanced constructions (e.g. state channels). However, now that I look at all the options, it seems that multi-party state channels in Bitcoin are (probably) impossible to construct.
+
+There is a second, unused Bitcoin client in place which uses Blockstream's Electrum server, but this didn't seem to be working properly with respect to Signet Bitcoin network during my testing, so I went with the esplora / Mutiny version instead.
+
+## Possible uses
+
+### DKG
+
+It strikes me that there are many, many systems which rely on a trusted setup, and which might be able to use Distributed Key Generation (DKG) instead. SNARK systems for instance all have this problem. Could BFT-CRDTs help here?
+
+It is not necessarily the case that e.g. signer participants and Cosmos validators are the same entities. Being able to quickly spin up a blockchain and use it to sign (potentially temporary or ephemeral) keyshare data might be pretty useful.
+
+### Cross chain transfers
+
+Might the ability to be part of multiple consensus groups at once provide new opportunities for cross-chain transfers?
+
+## Next dev tasks:
+
+- [ ] we don't need a relayer, the first crdt node can act as a leader until people decide they don't want to trust it any more
+- [ ] the leader node can have a timer in it for block creation
+- [ ] code up the ability to switch leaders (can be a human decision at first, later an (optional) automated choice)
+- [ ] pick a commit and reveal scheme to remove MEV. One thing to investigate is [single-use seals](https://docs.rgb.info/distributed-computing-concepts/single-use-seals)
+- [ ] enable Crdt Nodes should download current P2P chain/dag state so that they start - out with a consistent copy of transaction data, and also do catch-up after going off-line
+- [ ] remove the proc macro code from bft-json-crdt, it's not really needed in this implementation
+- [ ] add smart contract execution engine (CosmWasm would be a good first choice)
+- [ ] enable Crdt Nodes to download contract code for a given contract
+- [ ] enable Crdt Nodes to download current contract state for a given contract
+- [ ] switch to full P2P messaging instead of websockets

crates/bft-json-crdt/Cargo.toml

@@ -17,15 +17,20 @@ bft = []
 bft-crdt-derive = { path = "bft-crdt-derive" }
 colored = "2.0.0"
 fastcrypto = "0.1.8"
-itertools = "0.10.5"
+indexmap = { version = "2.2.6", features = ["serde"] }
 rand = "0.8.5"
 random_color = "0.6.1"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.85"
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
 serde_with = "3.8.1"
 sha2 = "0.10.6"
 
 [dev-dependencies]
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.85"
+criterion = { version = "0.4", features = ["html_reports"] }
 time = "0.1"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
+
+[[bench]]
+name = "speed"
+harness = false

crates/bft-json-crdt/benches/speed.rs

@@ -1,59 +1,67 @@
-#![feature(test)]
-
-extern crate test;
 use bft_json_crdt::{
     json_crdt::JsonValue, keypair::make_author, list_crdt::ListCrdt, op::Op, op::ROOT_ID,
 };
+use criterion::{criterion_group, criterion_main, Criterion};
 use rand::seq::SliceRandom;
-use test::Bencher;
 
-#[bench]
-fn bench_insert_1_000_root(b: &mut Bencher) {
-    b.iter(|| {
-        let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
-        for i in 0..1_000 {
-            list.insert(ROOT_ID, i);
-        }
-    })
-}
-
-#[bench]
-fn bench_insert_1_000_linear(b: &mut Bencher) {
-    b.iter(|| {
-        let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
-        let mut prev = ROOT_ID;
-        for i in 0..1_000 {
-            let op = list.insert(prev, i);
-            prev = op.id;
-        }
-    })
-}
-
-#[bench]
-fn bench_insert_many_agents_conflicts(b: &mut Bencher) {
-    b.iter(|| {
-        const N: u8 = 50;
-        let mut rng = rand::thread_rng();
-        let mut crdts: Vec<ListCrdt<i64>> = Vec::with_capacity(N as usize);
-        let mut logs: Vec<Op<JsonValue>> = Vec::new();
-        for i in 0..N {
-            let list = ListCrdt::new(make_author(i), vec![]);
-            crdts.push(list);
-            for _ in 0..5 {
-                let op = crdts[i as usize].insert(ROOT_ID, i as i32);
-                logs.push(op);
+fn bench_insert_100_root(c: &mut Criterion) {
+    c.bench_function("bench insert 100 root", |b| {
+        b.iter(|| {
+            let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
+            for i in 0..100 {
+                list.insert(ROOT_ID, i);
             }
-        }
+        })
+    });
+}
 
-        logs.shuffle(&mut rng);
-        for op in logs {
-            for c in &mut crdts {
-                if op.author() != c.our_id {
-                    c.apply(op.clone());
+fn bench_insert_100_linear(c: &mut Criterion) {
+    c.bench_function("bench insert 100 linear", |b| {
+        b.iter(|| {
+            let mut list = ListCrdt::<i64>::new(make_author(1), vec![]);
+            let mut prev = ROOT_ID;
+            for i in 0..100 {
+                let op = list.insert(prev, i);
+                prev = op.id;
+            }
+        })
+    });
+}
+
+fn bench_insert_many_agents_conflicts(c: &mut Criterion) {
+    c.bench_function("bench insert many agents conflicts", |b| {
+        b.iter(|| {
+            const N: u8 = 10;
+            let mut rng = rand::thread_rng();
+            let mut crdts: Vec<ListCrdt<i64>> = Vec::with_capacity(N as usize);
+            let mut logs: Vec<Op<JsonValue>> = Vec::new();
+            for i in 0..N {
+                let list = ListCrdt::new(make_author(i), vec![]);
+                crdts.push(list);
+                for _ in 0..5 {
+                    let op = crdts[i as usize].insert(ROOT_ID, i as i32);
+                    logs.push(op);
                 }
             }
-        }
 
-        assert!(crdts.windows(2).all(|w| w[0].view() == w[1].view()));
-    })
+            logs.shuffle(&mut rng);
+            for op in logs {
+                for c in &mut crdts {
+                    if op.author() != c.our_id {
+                        c.apply(op.clone());
+                    }
+                }
+            }
+
+            assert!(crdts.windows(2).all(|w| w[0].view() == w[1].view()));
+        })
+    });
 }
+
+criterion_group!(
+    benches,
+    bench_insert_100_root,
+    bench_insert_100_linear,
+    bench_insert_many_agents_conflicts
+);
+criterion_main!(benches);

crates/bft-json-crdt/bft-crdt-derive/Cargo.toml

@@ -8,6 +8,7 @@ publish = false
 proc-macro = true
 
 [dependencies]
+indexmap = { version = "2.2.6", features = ["serde"] }
 proc-macro2 = "1.0.47"
 proc-macro-crate = "1.2.1"
 quote = "1.0.21"

crates/bft-json-crdt/bft-crdt-derive/src/lib.rs

@@ -6,13 +6,12 @@ use syn::{
     parse::{self, Parser},
     parse_macro_input,
     spanned::Spanned,
-    Data, DeriveInput, Field, Fields, ItemStruct, LitStr, Type
+    Data, DeriveInput, Field, Fields, ItemStruct, LitStr, Type,
 };
 
 /// Helper to get tokenstream representing the parent crate
 fn get_crate_name() -> TokenStream {
-    let cr8 = crate_name("bft-json-crdt")
-        .unwrap_or(FoundCrate::Itself);
+    let cr8 = crate_name("bft-json-bft-crdt").unwrap_or(FoundCrate::Itself);
     match cr8 {
         FoundCrate::Itself => quote! { ::bft_json_crdt },
         FoundCrate::Name(name) => {
@@ -106,17 +105,23 @@ pub fn derive_json_crdt(input: OgTokenStream) -> OgTokenStream {
                                 })
                             } else {
                                 Err(format!("failed to convert {:?} -> {}<T>", value, #ident_str.to_string()))
-                            }  
+                            }
                         }
                     }
 
-                    impl #impl_generics std::fmt::Debug for #ident #ty_generics #where_clause {
-                        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                            let mut fields = Vec::new();
-                            #(fields.push(format!("{}", #ident_strings.to_string()));)*
-                            write!(f, "{{ {:?} }}", fields.join(", "))
-                        }
-                    } 
+                    // I'm pulling this out so that we can see actual CRD content in debug output.
+                    //
+                    // The plan is to mostly get rid of the macros anyway, so it's a reasonable first step.
+                    // It could (alternately) be just as good to keep the macros and change this function to
+                    // output actual field content instead of just field names.
+                    //
+                    // impl #impl_generics std::fmt::Debug for #ident #ty_generics #where_clause {
+                    //     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                    //         let mut fields = Vec::new();
+                    //         #(fields.push(format!("{}", #ident_strings.to_string()));)*
+                    //         write!(f, "{{ {:?} }}", fields.join(", "))
+                    //     }
+                    // }
 
                     impl #impl_generics #crate_name::json_crdt::CrdtNode for #ident #ty_generics #where_clause {
                         fn apply(&mut self, op: #crate_name::op::Op<#crate_name::json_crdt::JsonValue>) -> #crate_name::json_crdt::OpState {
@@ -128,7 +133,7 @@ pub fn derive_json_crdt(input: OgTokenStream) -> OgTokenStream {
                             }
 
                             if self.path.len() == op.path.len() {
-                                return #crate_name::json_crdt::OpState::ErrApplyOnStruct; 
+                                return #crate_name::json_crdt::OpState::ErrApplyOnStruct;
                             } else {
                                 let idx = self.path.len();
                                 if let #crate_name::op::PathSegment::Field(path_seg) = &op.path[idx] {
@@ -139,12 +144,12 @@ pub fn derive_json_crdt(input: OgTokenStream) -> OgTokenStream {
                                         _ => {},
                                     };
                                 };
-                                return #crate_name::json_crdt::OpState::ErrPathMismatch 
+                                return #crate_name::json_crdt::OpState::ErrPathMismatch
                             }
                         }
 
                         fn view(&self) -> #crate_name::json_crdt::JsonValue {
-                            let mut view_map = std::collections::HashMap::new();
+                            let mut view_map = indexmap::IndexMap::new();
                             #(view_map.insert(#ident_strings.to_string(), self.#ident_literals.view().into());)*
                             #crate_name::json_crdt::JsonValue::Object(view_map)
                         }
@@ -173,7 +178,7 @@ pub fn derive_json_crdt(input: OgTokenStream) -> OgTokenStream {
                         fn debug_view(&self, _indent: usize) -> String {
                             "".to_string()
                         }
-                    } 
+                    }
                 };
 
                 // Hand the output tokens back to the compiler

crates/bft-json-crdt/src/json_crdt.rs

@@ -17,6 +17,13 @@ use fastcrypto::{
     traits::{KeyPair, ToFromBytes},
     // Verifier,
 };
+// TODO: serde's json object serialization and deserialization (correctly) do not define anything
+// object field order in JSON objects. However, the hash check impl in bft-json-bft-crdt does take order
+// into account. This is going to cause problems later for non-Rust implementations, BFT hash checking
+// currently depends on JSON serialization/deserialization object order. This shouldn't be the case
+// but I've hacked in an IndexMap for the moment to get the PoC working. To see the problem, replace this with
+// a std HashMap, everything will screw up (annoyingly, only *most* of the time).
+use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, Bytes};
 
@@ -212,7 +219,7 @@ impl<T: CrdtNode + DebugView> BaseCrdt<T> {
     /// Apply a signed operation to this BaseCRDT, verifying integrity and routing to the right
     /// nested CRDT
     pub fn apply(&mut self, op: SignedOp) -> OpState {
-        self.log_try_apply(&op);
+        // self.log_try_apply(&op);
 
         #[cfg(feature = "bft")]
         if !op.is_valid_digest() {
@@ -232,9 +239,9 @@ impl<T: CrdtNode + DebugView> BaseCrdt<T> {
         }
 
         // apply
-        self.log_actually_apply(&op);
+        // self.log_actually_apply(&op);
         let status = self.doc.apply(op.inner);
-        self.debug_view();
+        // self.debug_view();
         self.received.insert(op_id);
 
         // apply all of its causal dependents if there are any
@@ -256,7 +263,7 @@ pub enum JsonValue {
     Number(f64),
     String(String),
     Array(Vec<JsonValue>),
-    Object(HashMap<String, JsonValue>),
+    Object(IndexMap<String, JsonValue>),
 }
 
 impl Display for JsonValue {
@@ -542,7 +549,7 @@ mod test {
     #[test]
     fn test_derive_basic() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Player {
             x: LwwRegisterCrdt<f64>,
             y: LwwRegisterCrdt<f64>,
@@ -557,14 +564,14 @@ mod test {
     #[test]
     fn test_derive_nested() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Position {
             x: LwwRegisterCrdt<f64>,
             y: LwwRegisterCrdt<f64>,
         }
 
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Player {
             pos: Position,
             balance: LwwRegisterCrdt<f64>,
@@ -582,7 +589,7 @@ mod test {
     #[test]
     fn test_lww_ops() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Test {
             a: LwwRegisterCrdt<f64>,
             b: LwwRegisterCrdt<bool>,
@@ -642,7 +649,7 @@ mod test {
     #[test]
     fn test_vec_and_map_ops() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Test {
             a: ListCrdt<String>,
         }
@@ -682,14 +689,14 @@ mod test {
     #[test]
     fn test_causal_field_dependency() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Item {
             name: LwwRegisterCrdt<String>,
             soulbound: LwwRegisterCrdt<bool>,
         }
 
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Player {
             inventory: ListCrdt<Item>,
             balance: LwwRegisterCrdt<f64>,
@@ -748,7 +755,7 @@ mod test {
     #[test]
     fn test_2d_grid() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Game {
             grid: ListCrdt<ListCrdt<LwwRegisterCrdt<bool>>>,
         }
@@ -809,7 +816,7 @@ mod test {
     #[test]
     fn test_arb_json() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Test {
             reg: LwwRegisterCrdt<JsonValue>,
         }
@@ -845,13 +852,13 @@ mod test {
     #[test]
     fn test_wrong_json_types() {
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Nested {
             list: ListCrdt<f64>,
         }
 
         #[add_crdt_fields]
-        #[derive(Clone, CrdtNode)]
+        #[derive(Clone, CrdtNode, Debug)]
         struct Test {
             reg: LwwRegisterCrdt<bool>,
             strct: ListCrdt<Nested>,

crates/bft-json-crdt/tests/byzantine.rs

@@ -20,7 +20,7 @@ use serde_json::json;
 // 5. block actual messages from honest actors (eclipse attack)
 
 #[add_crdt_fields]
-#[derive(Clone, CrdtNode)]
+#[derive(Clone, CrdtNode, Debug)]
 struct ListExample {
     list: ListCrdt<char>,
 }
@@ -91,13 +91,13 @@ fn test_forge_update() {
 }
 
 #[add_crdt_fields]
-#[derive(Clone, CrdtNode)]
+#[derive(Clone, CrdtNode, Debug)]
 struct Nested {
     a: Nested2,
 }
 
 #[add_crdt_fields]
-#[derive(Clone, CrdtNode)]
+#[derive(Clone, CrdtNode, Debug)]
 struct Nested2 {
     b: LwwRegisterCrdt<bool>,
 }

crates/bft-json-crdt/tests/editing-trace.js

@@ -259796,7 +259796,7 @@ function insertAt(idx, elt) {
       const pos = new_log.findIndex(log => log[0] === parent_id)
       new_log.push([ID_COUNTER, pos, 0, elt])
       crdt.splice(raw_i + 1, 0, { deleted: false, content: elt, id: ID_COUNTER })
-      // console.log(`insert at ${idx} translated as op [${ID_COUNTER}, ${pos}, ${0}, ${escape(elt)}] found at ${raw_i + 1}::`, crdt[raw_i + 1])
+      // console.log(`insert at ${idx} translated as op [${ID_COUNTER}, ${pos}, ${0}, ${escape(elt)}] found at ${raw_i + 1}::`, bft-crdt[raw_i + 1])
       return
     }
 
@@ -259816,7 +259816,7 @@ function deleteAt(idx) {
       const pos = new_log.findIndex(log => log[0] === our_id)
       new_log.push([ID_COUNTER, pos, 1]);
       crdt[raw_i].deleted = true
-      // console.log(`delete at ${idx} translated as op [${ID_COUNTER}, ${pos}, ${1}] found at ${raw_i} with our_id ${our_id}::`, crdt[raw_i])
+      // console.log(`delete at ${idx} translated as op [${ID_COUNTER}, ${pos}, ${1}] found at ${raw_i} with our_id ${our_id}::`, bft-crdt[raw_i])
       return
     }
 
@@ -259853,7 +259853,7 @@ function rawJSString(edits) {
 //     deleteAt(edit[0])
 //   }
 // }
-// console.log(crdt)
+// console.log(bft-crdt)
 // rawJSString(mock_edits)
 // console.log(new_log)
 // const subset = edits.slice(0, 50000) 

crdt-node/Cargo.toml

@@ -1,31 +1,40 @@
 [package]
-name = "side-node"
+name = "crdt-node"
 version = "0.1.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1.0.86"
 async-trait = "0.1.52"
-base64 = "0.21.7"
+bdk = { version = "0.29.0", default-feature = false, features = ["all-keys"] }
+bdk_esplora = "0.15.0"
+bdk_sqlite = "0.2.0"
+bdk_wallet = { version = "1.0.0-alpha.13", features = ["all-keys"] }
 bft-json-crdt = { path = "../crates/bft-json-crdt" }
 bft-crdt-derive = { path = "../crates/bft-json-crdt/bft-crdt-derive" }
+bitcoin = { version = "0.32.2", features = ["rand"] }
 clap = { version = "4.5.4", features = ["derive"] }
 dirs = "5.0.1"
+electrum-client = "0.20"
 ezsockets = { version = "*", features = ["client"] }
 fastcrypto = "0.1.8"
+indexmap = { version = "2.2.6", features = ["serde"] }
+reqwest = { version = "*", features = ["blocking"] }
 # serde_cbor = "0.11.2" # move to this once we need to pack things in CBOR
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.117"
-serde_with = "3.8.1"
 sha256 = "1.5.0"
 tokio = { version = "1.37.0", features = ["full"] }
-tracing = "0.1.32"
-tracing-subscriber = "0.3.9"
 toml = "0.8.14"
-url = "2.2.2"
+tracing = "0.1.32"
+# tracing-subscriber = "0.3.9"
 
 
+[dev-dependencies]
+uuid = { version = "1.8.0", features = ["v4"] }
+
 [features]
 default = ["bft", "logging-list", "logging-json"]
 logging-list = ["logging-base"]

crdt-node/src/bft_crdt/keys.rs

@@ -8,7 +8,7 @@ use bft_json_crdt::keypair::{make_keypair, Ed25519KeyPair};
 use fastcrypto::traits::EncodeDecodeBase64;
 
 /// Writes a new Ed25519 keypair to the file at key_path.
-pub(crate) fn write(key_path: PathBuf) -> Result<(), std::io::Error> {
+pub(crate) fn write(key_path: &PathBuf) -> Result<(), std::io::Error> {
     let keys = make_keypair();
 
     let mut file = File::create(key_path)?;
@@ -17,10 +17,10 @@ pub(crate) fn write(key_path: PathBuf) -> Result<(), std::io::Error> {
     Ok(())
 }
 
-pub(crate) fn load_from_file(side_dir: PathBuf) -> Ed25519KeyPair {
+pub(crate) fn load_from_file(side_dir: &PathBuf) -> Ed25519KeyPair {
     let key_path = crate::utils::side_paths(side_dir.clone()).0;
 
-    let data = fs::read_to_string(key_path).expect("couldn't read key file");
+    let data = fs::read_to_string(key_path).expect("couldn't read bft-bft-crdt key file");
     println!("data: {:?}", data);
 
     Ed25519KeyPair::decode_base64(&data).expect("couldn't load keypair from file")

crdt-node/src/bft_crdt/mod.rs

@@ -0,0 +1,31 @@
+use bft_crdt_derive::add_crdt_fields;
+use bft_json_crdt::{
+    json_crdt::{CrdtNode, IntoCrdtNode},
+    list_crdt::ListCrdt,
+};
+use serde::{Deserialize, Serialize};
+
+pub mod keys;
+pub mod stdin;
+pub mod websocket;
+
+#[add_crdt_fields]
+#[derive(Clone, CrdtNode, Serialize, Deserialize, Debug)]
+pub struct TransactionList {
+    pub list: ListCrdt<Transaction>,
+}
+
+impl TransactionList {
+    pub fn view_sha(&self) -> String {
+        sha256::digest(serde_json::to_string(&self.list.view()).unwrap().as_bytes()).to_string()
+    }
+}
+
+/// A fake Transaction struct we can use as a simulated payload
+#[add_crdt_fields]
+#[derive(Clone, CrdtNode, Serialize, Deserialize, PartialEq, Debug)]
+pub struct Transaction {
+    from: String,
+    to: String,
+    amount: f64,
+}

crdt-node/src/bft_crdt/stdin.rs

@@ -5,7 +5,6 @@ pub(crate) fn input(stdin_sender: std::sync::mpsc::Sender<String>) {
     let stdin = std::io::stdin();
     let lines = stdin.lock().lines();
     for line in lines {
-        println!("We're in stdin_input");
         let line = line.unwrap();
         stdin_sender.send(line).unwrap();
     }

crdt-node/src/bft_crdt/websocket.rs

@@ -3,20 +3,19 @@ use bft_json_crdt::json_crdt::SignedOp;
 use ezsockets::ClientConfig;
 use tokio::sync::mpsc;
 
-pub(crate) struct WebSocketClient {
+use crate::utils;
+
+pub struct Client {
     incoming_sender: mpsc::Sender<SignedOp>,
-    handle: ezsockets::Client<WebSocketClient>,
+    handle: ezsockets::Client<Client>,
 }
 
-impl WebSocketClient {
+impl Client {
     /// Start the websocket client
-    pub(crate) async fn new(
-        incoming_sender: mpsc::Sender<SignedOp>,
-    ) -> ezsockets::Client<WebSocketClient> {
-        tracing_subscriber::fmt::init();
+    pub async fn new(incoming_sender: mpsc::Sender<SignedOp>) -> ezsockets::Client<Client> {
         let config = ClientConfig::new("ws://localhost:8080/websocket");
         let (handle, future) = ezsockets::connect(
-            |client| WebSocketClient {
+            |client| Client {
                 incoming_sender,
                 handle: client,
             },
@@ -31,26 +30,36 @@ impl WebSocketClient {
 }
 
 #[async_trait]
-impl ezsockets::ClientExt for WebSocketClient {
+impl ezsockets::ClientExt for Client {
     // Right now we're only using the Call type for sending signed ops
     // change this to an enum if we need to send other types of calls, and
     // match on it.
     type Call = String;
 
+    /// When we receive a text message, apply the bft-crdt operation contained in it to our
+    /// local bft-crdt.
     async fn on_text(&mut self, text: String) -> Result<(), ezsockets::Error> {
-        tracing::info!("received text: {text:?}");
+        let string_sha = utils::shassy(text.clone());
+        println!("received text, sha: {string_sha}");
         let incoming: bft_json_crdt::json_crdt::SignedOp = serde_json::from_str(&text).unwrap();
+        let object_sha = utils::shappy(incoming.clone());
+        println!("deserialized: {}", object_sha);
+        if string_sha != object_sha {
+            panic!("sha mismatch: {string_sha} != {object_sha}, bft-bft-crdt has failed");
+        }
         self.incoming_sender.send(incoming).await?;
         Ok(())
     }
 
+    /// When we receive a binary message, log the bytes. Currently unused.
     async fn on_binary(&mut self, bytes: Vec<u8>) -> Result<(), ezsockets::Error> {
         tracing::info!("received bytes: {bytes:?}");
         Ok(())
     }
 
+    /// Call this with the `Call` type to send application data to the websocket client
+    /// (and from there, to the server).
     async fn on_call(&mut self, call: Self::Call) -> Result<(), ezsockets::Error> {
-        tracing::info!("sending signed op: {call:?}");
         self.handle.text(call)?;
         Ok(())
     }

crdt-node/src/bitcoin/clients/electrum.rs

@@ -0,0 +1,101 @@
+use crate::{bitcoin, utils};
+use bdk::bitcoin::psbt::PartiallySignedTransaction;
+use bdk::bitcoin::Network;
+use bdk::database::MemoryDatabase;
+use bdk::keys::ExtendedKey;
+use bdk::template::Bip84;
+use bdk::wallet::AddressIndex::{self, New};
+use bdk::wallet::AddressInfo;
+use bdk::{blockchain::ElectrumBlockchain, electrum_client, SyncOptions};
+use bdk::{FeeRate, KeychainKind, SignOptions, TransactionDetails, Wallet};
+
+/// DEPRECATED
+///
+/// This is a bdk example that uses the Electrum client to interact with the Bitcoin network.
+/// Electrum is a light client that connects to a server to get information about the Bitcoin network.
+/// The BDK itself does not have the ability to connect to e.g. esplora servers. As the Blockstream Electrum Signet
+/// server does not appear to be picking up transactions properly at the moment, I've shifted over to using
+/// the (more complex) `bdk_wallet` crate and the esplora client there (see the other bitcoin client).
+///
+/// Note:the types below are all completely different than the types in `bdk_wallet`.
+pub async fn run() -> Result<(), anyhow::Error> {
+    let dave = utils::home(&"dave".to_string());
+    let sammy = utils::home(&"sammy".to_string());
+    let dave_key = bitcoin::keys::load_from_file(&dave).unwrap();
+    let sammy_key = bitcoin::keys::load_from_file(&sammy).unwrap();
+
+    let dave_wallet = create_wallet(dave_key)?;
+    let sammy_wallet = create_wallet(sammy_key)?;
+
+    let dave_address = dave_wallet.get_address(AddressIndex::Peek(0))?.to_string();
+    let sammy_address = sammy_wallet.get_address(AddressIndex::Peek(0))?.to_string();
+
+    println!("Dave's address: {}", dave_address);
+    println!("Sammy's address: {}", sammy_address);
+
+    let blockchain = ElectrumBlockchain::from(electrum_client::Client::new(
+        "ssl://electrum.blockstream.info:60002",
+    )?);
+
+    println!("Syncing...");
+    dave_wallet.sync(&blockchain, SyncOptions::default())?;
+
+    display_balance(&dave_wallet);
+    display_balance(&sammy_wallet);
+
+    let (mut psbt, details) =
+        build_sending_tx(&dave_wallet, sammy_wallet.get_address(New)?).expect("psbt build error");
+
+    println!("About to sign the transaction: {:?}", details);
+
+    dave_wallet.sign(&mut psbt, SignOptions::default())?;
+    let _signed_tx = psbt.extract_tx();
+
+    // println!("Broadcasting...");
+    // blockchain.broadcast(&signed_tx).expect("broadcast error");
+    // println!("Transaction ID: {:?}", signed_tx.txid());
+    Ok(())
+}
+
+/// Create a BDK wallet using BIP 84 descriptor ("m/84h/1h/0h/0" and "m/84h/1h/0h/1")
+pub fn create_wallet(xkey: ExtendedKey) -> anyhow::Result<Wallet<MemoryDatabase>> {
+    let xprv = xkey
+        .into_xprv(Network::Testnet)
+        .expect("couldn't turn xkey into xprv");
+
+    let external_descriptor = Bip84(xprv, KeychainKind::External);
+    let internal_descriptor = Some(Bip84(xprv, KeychainKind::Internal));
+
+    let wallet = Wallet::new(
+        external_descriptor,
+        internal_descriptor,
+        Network::Testnet,
+        MemoryDatabase::default(),
+    )?;
+
+    Ok(wallet)
+}
+
+fn display_balance(wallet: &Wallet<MemoryDatabase>) {
+    println!(
+        "Wallet balance for {} after syncing: {:?} sats on network {}",
+        wallet
+            .get_address(bdk::wallet::AddressIndex::Peek(0))
+            .expect("couldn't get address"),
+        wallet.get_balance().expect("couldn't show balance"),
+        wallet.network(),
+    );
+}
+
+fn build_sending_tx(
+    wallet: &Wallet<MemoryDatabase>,
+    recipient: AddressInfo,
+) -> anyhow::Result<(PartiallySignedTransaction, TransactionDetails), anyhow::Error> {
+    let mut builder = wallet.build_tx();
+    builder
+        .add_recipient(recipient.script_pubkey(), 1000)
+        .enable_rbf()
+        .do_not_spend_change()
+        .fee_rate(FeeRate::from_sat_per_vb(7.0));
+    Ok(builder.finish()?)
+}

crdt-node/src/bitcoin/clients/esplora.rs

@@ -0,0 +1,188 @@
+use std::{collections::BTreeSet, fs, io::Write};
+
+use bdk::keys::bip39::Mnemonic;
+use bdk_esplora::{
+    esplora_client::{self, AsyncClient},
+    EsploraAsyncExt,
+};
+use bdk_wallet::{
+    bitcoin::{Address, Amount, Network, Script},
+    chain::ConfirmationTimeHeightAnchor,
+    keys::{DerivableKey, ExtendedKey},
+    wallet::AddressInfo,
+    KeychainKind, SignOptions, Wallet,
+};
+
+use bdk_sqlite::{rusqlite::Connection, Store};
+
+use crate::utils;
+
+const STOP_GAP: usize = 50;
+const PARALLEL_REQUESTS: usize = 5;
+
+/// A wallet that uses the Esplora client to interact with the Bitcoin network.
+pub struct EsploraWallet {
+    client: AsyncClient,
+    db: Store<KeychainKind, ConfirmationTimeHeightAnchor>,
+    name: String,
+    wallet: Wallet,
+}
+
+impl EsploraWallet {
+    /// Builds and signs a send transaction to send coins between addresses.
+    ///
+    /// Does NOT send it, you must call `broadcast` to do that.
+    ///
+    /// We could split the creation and signing easily if needed.
+    pub(crate) fn build_and_sign_send_tx(
+        &mut self,
+        recipient: Address,
+        amount: Amount,
+    ) -> Result<bitcoin::Transaction, anyhow::Error> {
+        let mut tx_builder = self.wallet.build_tx();
+        tx_builder
+            .add_recipient(recipient.script_pubkey(), amount)
+            .enable_rbf();
+        let mut psbt = tx_builder.finish()?;
+        let finalized = self.wallet.sign(&mut psbt, SignOptions::default())?;
+        assert!(finalized);
+        let tx = psbt.extract_tx()?;
+        Ok(tx)
+    }
+
+    /// Syncs the wallet with the latest state of the Bitcoin blockchain
+    pub(crate) async fn sync(&mut self) -> Result<(), anyhow::Error> {
+        print!("Syncing...");
+
+        fn generate_inspect(
+            kind: KeychainKind,
+        ) -> impl FnMut(u32, &Script) + Send + Sync + 'static {
+            let mut once = Some(());
+            let mut stdout = std::io::stdout();
+            move |spk_i, _| {
+                match once.take() {
+                    Some(_) => print!("\nScanning keychain [{:?}]", kind),
+                    None => print!(" {:<3}", spk_i),
+                };
+                stdout.flush().expect("must flush");
+            }
+        }
+        let request = self
+            .wallet
+            .start_full_scan()
+            .inspect_spks_for_all_keychains({
+                let mut once = BTreeSet::<KeychainKind>::new();
+                move |keychain, spk_i, _| {
+                    match once.insert(keychain) {
+                        true => print!("\nScanning keychain [{:?}]", keychain),
+                        false => print!(" {:<3}", spk_i),
+                    }
+                    std::io::stdout().flush().expect("must flush")
+                }
+            })
+            .inspect_spks_for_keychain(
+                KeychainKind::External,
+                generate_inspect(KeychainKind::External),
+            )
+            .inspect_spks_for_keychain(
+                KeychainKind::Internal,
+                generate_inspect(KeychainKind::Internal),
+            );
+        let mut update = self
+            .client
+            .full_scan(request, STOP_GAP, PARALLEL_REQUESTS)
+            .await?;
+        let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
+        let _ = update.graph_update.update_last_seen_unconfirmed(now);
+        self.wallet.apply_update(update)?;
+        self.persist_local()?;
+        println!("Sync complete for {}", self.name);
+        Ok(())
+    }
+
+    fn persist_local(&mut self) -> Result<(), anyhow::Error> {
+        Ok(if let Some(changeset) = self.wallet.take_staged() {
+            self.db.write(&changeset)?;
+        })
+    }
+
+    /// Gets the next unused address from the wallet.
+    pub(crate) fn next_unused_address(&mut self) -> Result<AddressInfo, anyhow::Error> {
+        let address = self.wallet.next_unused_address(KeychainKind::External);
+        self.persist_local()?;
+        println!(
+            "Generated address: https://mutinynet.com/address/{}",
+            address
+        );
+        Ok(address)
+    }
+
+    /// Returns the balance of the wallet.
+    pub(crate) fn balance(&self) -> bdk_wallet::wallet::Balance {
+        self.wallet.balance()
+    }
+
+    /// Broadcasts a signed transaction to the network.
+    pub(crate) async fn broadcast(
+        &self,
+        tx: &bitcoin::Transaction,
+    ) -> Result<(), esplora_client::Error> {
+        println!(
+            "{} broadcasting tx https://mutinynet.com/tx/{}",
+            self.name,
+            tx.compute_txid()
+        );
+        self.client.broadcast(tx).await
+    }
+}
+
+/// Creates a Bitcoin descriptor wallet with the mnemonic in the given user directory.
+pub(crate) fn create_wallet(name: &str, network: Network) -> anyhow::Result<EsploraWallet> {
+    let keys_dir = utils::home(name);
+
+    let mnemonic_path = crate::utils::side_paths(keys_dir).1; // TODO: this tuple stinks
+    let mnemonic_words = fs::read_to_string(mnemonic_path).expect("couldn't read bitcoin key file");
+
+    println!("Creating wallet from mnemonic: {mnemonic_words}");
+    let mnemonic = Mnemonic::parse(mnemonic_words).unwrap();
+
+    // Generate the extended key
+    let xkey: ExtendedKey = mnemonic
+        .into_extended_key()
+        .expect("couldn't turn mnemonic into xkey");
+
+    let xprv = xkey
+        .into_xprv(Network::Signet)
+        .expect("problem converting xkey to xprv")
+        .to_string();
+
+    println!("Setting up esplora database for {name}");
+
+    let db_path = format!("/tmp/{name}-bdk-esplora-async-example.sqlite");
+    let conn = Connection::open(db_path)?;
+    let mut db = Store::new(conn)?;
+    let external_descriptor = format!("wpkh({xprv}/84'/1'/0'/0/*)");
+    let internal_descriptor = format!("wpkh({xprv}/84'/1'/0'/1/*)");
+    let changeset = db.read().expect("couldn't read esplora database");
+
+    let wallet = Wallet::new_or_load(
+        &external_descriptor,
+        &internal_descriptor,
+        changeset,
+        network,
+    )
+    .expect("problem setting up wallet");
+
+    let client = esplora_client::Builder::new("https://mutinynet.com/api")
+        .build_async()
+        .expect("couldn't build esplora client");
+
+    let esplora = EsploraWallet {
+        name: name.to_string(),
+        wallet,
+        db,
+        client,
+    };
+
+    Ok(esplora)
+}

crdt-node/src/bitcoin/clients/mod.rs

@@ -0,0 +1,2 @@
+pub mod electrum;
+pub mod esplora;

crdt-node/src/bitcoin/driver.rs

@@ -0,0 +1,60 @@
+use bdk_wallet::bitcoin::{Amount, Network};
+
+use crate::bitcoin::clients;
+
+/// Demonstrates the use of bdk with the Esplora client.
+///
+/// This is more complex than the bare `bdk` crate, but the esplora client works.
+///
+/// Also, it very handily works with the mutinynet.com esplora server, which is configured
+/// with 30 second block times.
+pub(crate) async fn run() -> Result<(), anyhow::Error> {
+    simple_transfer().await
+}
+
+async fn simple_transfer() -> Result<(), anyhow::Error> {
+    let mut dave = clients::esplora::create_wallet("dave", Network::Signet)?;
+    let mut sammy = clients::esplora::create_wallet("sammy", Network::Signet)?;
+
+    let _next_address = dave.next_unused_address()?;
+
+    let dave_balance = dave.balance();
+    println!(
+        "Dave wallet balance before syncing: {} sats",
+        dave_balance.total()
+    );
+
+    dave.sync().await?;
+
+    let dave_balance = dave.balance();
+    println!("Wallet balance after syncing: {} sats", dave_balance);
+
+    let sammy_address = sammy.next_unused_address()?.address;
+    println!("Sammy's address: {}", sammy_address);
+
+    let sammy_balance = sammy.balance();
+    println!(
+        "Sammy wallet balance before syncing: {} sats",
+        sammy_balance
+    );
+
+    sammy.sync().await?;
+
+    let sammy_balance = sammy.balance();
+    println!("Sammy wallet balance after syncing: {} sats", sammy_balance);
+
+    let send_amount = Amount::from_sat(500);
+
+    if dave_balance.total() < send_amount {
+        println!(
+            "Please send at least {} sats to the receiving address",
+            send_amount
+        );
+        std::process::exit(0);
+    }
+
+    let tx = dave.build_and_sign_send_tx(sammy_address, send_amount)?;
+    dave.broadcast(&tx).await?;
+
+    Ok(())
+}

crdt-node/src/bitcoin/keys.rs

@@ -0,0 +1,50 @@
+use bdk::{
+    keys::{
+        bip39::{Language, Mnemonic, WordCount},
+        DerivableKey, ExtendedKey, GeneratableKey, GeneratedKey,
+    },
+    miniscript,
+};
+
+use std::{
+    fs::{self, File},
+    io::Write,
+    path::PathBuf,
+};
+
+pub fn make_mnemonic() -> String {
+    let mnemonic: GeneratedKey<_, miniscript::Segwitv0> =
+        Mnemonic::generate((WordCount::Words12, Language::English)).unwrap();
+    mnemonic.to_string()
+}
+
+/// Write the mnemonic to a file in the node's side directory
+///
+/// TODO: obviously spitting the mnemonic out to the console is not for production
+pub(crate) fn write(mnemonic_path: &PathBuf) -> Result<(), std::io::Error> {
+    let mnemonic = make_mnemonic();
+    let mut file = File::create(mnemonic_path)?;
+    println!("mnemonic: {mnemonic}");
+    file.write(mnemonic.as_bytes())?;
+
+    Ok(())
+}
+
+/// Creates Signet Bitcoin descriptors from a mnemonic
+pub fn get(mnemonic_words: String) -> anyhow::Result<ExtendedKey> {
+    let mnemonic = Mnemonic::parse(mnemonic_words).unwrap();
+
+    // Generate the extended key
+    let xkey: ExtendedKey = mnemonic
+        .into_extended_key()
+        .expect("couldn't turn mnemonic into xkey");
+
+    Ok(xkey)
+}
+
+pub(crate) fn load_from_file(side_dir: &PathBuf) -> anyhow::Result<ExtendedKey> {
+    let mnemonic_path = crate::utils::side_paths(side_dir.clone()).1; // TODO: this tuple stinks
+    let mnemonic_words = fs::read_to_string(mnemonic_path).expect("couldn't read bitcoin key file");
+    println!("Creating wallet from mnemonic: {mnemonic_words}");
+    get(mnemonic_words)
+}

crdt-node/src/bitcoin/mod.rs

@@ -0,0 +1,3 @@
+pub mod clients;
+pub mod driver;
+pub mod keys;

crdt-node/src/cli/mod.rs

@@ -17,6 +17,9 @@ pub(crate) struct Args {
 
 #[derive(Subcommand)]
 pub(crate) enum Commands {
+    /// Placeholder for future BTC commands
+    Btc {},
+
     /// runs the Side Node
     Run { name: String },
 

crdt-node/src/init/mod.rs

@@ -0,0 +1,120 @@
+use std::path::PathBuf;
+
+use config::SideNodeConfig;
+
+use crate::{bft_crdt, bitcoin, utils};
+
+pub(crate) mod config;
+
+pub(crate) fn init(home: PathBuf, config: SideNodeConfig) -> Result<(), std::io::Error> {
+    ensure_side_directory_exists(&home)?;
+    let (bft_crdt_key_path, bitcoin_key_path, config_path) = utils::side_paths(home.clone());
+
+    println!("Writing bft bft-crdt key to: {:?}", bft_crdt_key_path);
+    bft_crdt::keys::write(&bft_crdt_key_path)?;
+
+    println!("Writing bitcoin key to: {:?}", bitcoin_key_path);
+    bitcoin::keys::write(&bitcoin_key_path)?;
+
+    println!("Writing config to: {:?}", config_path);
+    config::write_toml(&config, &config_path).expect("unable to write config file");
+
+    Ok(())
+}
+
+/// Ensures that the directory at side_dir exists, so we have a place
+/// to store our key files and config file.
+fn ensure_side_directory_exists(side_dir: &PathBuf) -> Result<(), std::io::Error> {
+    if side_dir.exists() {
+        return Ok(());
+    }
+    println!(
+        "Config directory doesn't exist, creating at: {:?}",
+        side_dir
+    );
+    std::fs::create_dir_all(side_dir)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{fs, path::Path, str::FromStr};
+
+    use fastcrypto::{
+        ed25519::Ed25519KeyPair,
+        traits::{EncodeDecodeBase64, KeyPair, ToFromBytes},
+    };
+
+    use super::*;
+
+    /// Generates a SideNodeConfig with a unique name for each test.
+    /// This is necessary because the tests run in parallel and we
+    /// don't want them to interfere with each other - without a unique
+    /// name, the tests would all try to write to the same directory and we
+    /// get test indeterminacy
+    fn side_node_config() -> (SideNodeConfig, String) {
+        let name = format!("test-{}", uuid::Uuid::new_v4()).to_string();
+        (SideNodeConfig { name: name.clone() }, name)
+    }
+
+    #[test]
+    fn creates_bitcoin_keys() {
+        let (config, name) = side_node_config();
+        let side_dir = format!("/tmp/side/{name}");
+
+        let mut bitcoin_keys_path = PathBuf::new();
+        bitcoin_keys_path.push(side_dir.clone());
+        bitcoin_keys_path.push(utils::BITCOIN_KEY_FILE);
+
+        let _ = init(PathBuf::from_str(&side_dir).unwrap(), config);
+        assert!(bitcoin_keys_path.exists());
+
+        // check that the pem is readable
+        // let data = fs::read_to_string(bitcoin_keys_path).expect("couldn't read key file");
+        // let keys = Ed25519KeyPair::decode_base64(&data).expect("couldn't load keypair from file");
+        // assert_eq!(keys.public().as_bytes().len(), 32);
+    }
+
+    #[test]
+    fn creates_side_node_directory() {
+        let (config, name) = side_node_config();
+        let side_dir = format!("/tmp/side/{name}");
+
+        let mut test_home = PathBuf::new();
+
+        test_home.push(side_dir);
+        let node_dir = Path::new(&test_home).parent().unwrap().to_str().unwrap();
+        let _ = init(test_home.clone(), config);
+        assert!(std::path::Path::new(node_dir).exists());
+    }
+
+    #[test]
+    fn creates_bft_crdt_key_file() {
+        let (config, name) = side_node_config();
+        let side_dir = format!("/tmp/side/{name}");
+
+        let mut key_file_path = PathBuf::new();
+        key_file_path.push(side_dir.clone());
+        key_file_path.push(utils::BFT_CRDT_KEY_FILE);
+
+        let _ = init(PathBuf::from_str(&side_dir).unwrap(), config);
+        assert!(key_file_path.exists());
+
+        // check that the pem is readable
+        let data = fs::read_to_string(key_file_path).expect("couldn't read key file");
+        let keys = Ed25519KeyPair::decode_base64(&data).expect("couldn't load keypair from file");
+        assert_eq!(keys.public().as_bytes().len(), 32);
+    }
+
+    #[test]
+    fn creates_config_file() {
+        let (config, name) = side_node_config();
+        let side_dir = format!("/tmp/side/{name}");
+
+        let mut config_file_path = PathBuf::new();
+        config_file_path.push(side_dir.clone());
+        config_file_path.push(utils::CONFIG_FILE);
+
+        let _ = init(PathBuf::from_str(&side_dir).unwrap(), config);
+        assert!(config_file_path.exists());
+    }
+}

crdt-node/src/lib.rs

@@ -1,21 +1,19 @@
+use bft_crdt::websocket;
+use bft_crdt::TransactionList;
 use bft_json_crdt::json_crdt::{BaseCrdt, SignedOp};
 use cli::{parse_args, Commands};
-use crdt::TransactionList;
 use node::SideNode;
 use tokio::{sync::mpsc, task};
-use websocket::WebSocketClient;
 
+pub mod bft_crdt;
+pub mod bitcoin;
 pub(crate) mod cli;
-pub(crate) mod crdt;
 pub(crate) mod init;
-pub(crate) mod keys;
-pub(crate) mod node;
-pub(crate) mod stdin;
-pub(crate) mod utils;
-pub(crate) mod websocket;
+pub mod node;
+pub mod utils;
 
 #[tokio::main]
-async fn main() {
+pub async fn run() {
     let args = parse_args();
 
     match &args.command {
@@ -30,27 +28,39 @@ async fn main() {
             let mut node = setup(name).await;
             node.start().await;
         }
+        Some(Commands::Btc {}) => {
+            let _ = bitcoin::driver::run().await;
+        }
         None => println!("No command provided. Exiting. See --help for more information."),
     }
 }
 
-/// Wire everything up outside the application so we can test more easily later
+/// Wire everything up outside the application so that we can test more easily later
 async fn setup(name: &String) -> SideNode {
-    // First, load up the keys and create a bft-crdt
+    // First, load up the keys and create a bft-bft-crdt
     let side_dir = utils::home(name);
-    let keys = keys::load_from_file(side_dir);
-    let crdt = BaseCrdt::<TransactionList>::new(&keys);
+    let bft_crdt_keys = bft_crdt::keys::load_from_file(&side_dir);
+    let keys = bitcoin::keys::load_from_file(&side_dir).unwrap();
+    let bitcoin_wallet = bitcoin::clients::electrum::create_wallet(keys).unwrap();
+    let crdt = BaseCrdt::<TransactionList>::new(&bft_crdt_keys);
 
     // Channels for internal communication, and a tokio task for stdin input
     let (incoming_sender, incoming_receiver) = mpsc::channel::<SignedOp>(32);
     let (stdin_sender, stdin_receiver) = std::sync::mpsc::channel();
     task::spawn(async move {
-        stdin::input(stdin_sender);
+        bft_crdt::stdin::input(stdin_sender);
     });
 
     // Finally, create the node and return it
-    let handle = WebSocketClient::new(incoming_sender).await;
-    let node = SideNode::new(crdt, keys, incoming_receiver, stdin_receiver, handle);
+    let handle = websocket::Client::new(incoming_sender).await;
+    let node = SideNode::new(
+        crdt,
+        bft_crdt_keys,
+        bitcoin_wallet,
+        incoming_receiver,
+        stdin_receiver,
+        handle,
+    );
     println!("Node setup complete.");
     node
 }

crdt-node/src/main.rs

@@ -0,0 +1,5 @@
+use crdt_node;
+
+fn main() {
+    crdt_node::run();
+}

crdt-node/src/node.rs

@@ -1,28 +1,32 @@
+use bdk::database::MemoryDatabase;
 use bft_json_crdt::json_crdt::{BaseCrdt, SignedOp};
 use fastcrypto::ed25519::Ed25519KeyPair;
 use tokio::sync::mpsc;
 
-use crate::{crdt::TransactionList, utils, websocket::WebSocketClient};
+use crate::{bft_crdt::websocket::Client, bft_crdt::TransactionList, utils};
 
-pub(crate) struct SideNode {
+pub struct SideNode {
     crdt: BaseCrdt<TransactionList>,
-    keys: fastcrypto::ed25519::Ed25519KeyPair,
+    bft_crdt_keys: fastcrypto::ed25519::Ed25519KeyPair,
+    _bitcoin_wallet: bdk::Wallet<MemoryDatabase>, // currently not read anywhere
     incoming_receiver: mpsc::Receiver<SignedOp>,
     stdin_receiver: std::sync::mpsc::Receiver<String>,
-    handle: ezsockets::Client<WebSocketClient>,
+    handle: ezsockets::Client<Client>,
 }
 
 impl SideNode {
-    pub(crate) fn new(
+    pub fn new(
         crdt: BaseCrdt<TransactionList>,
-        keys: Ed25519KeyPair,
+        bft_crdt_keys: Ed25519KeyPair,
+        bitcoin_wallet: bdk::Wallet<MemoryDatabase>,
         incoming_receiver: mpsc::Receiver<SignedOp>,
         stdin_receiver: std::sync::mpsc::Receiver<String>,
-        handle: ezsockets::Client<WebSocketClient>,
+        handle: ezsockets::Client<Client>,
     ) -> Self {
         let node = Self {
             crdt,
-            keys,
+            bft_crdt_keys,
+            _bitcoin_wallet: bitcoin_wallet,
             incoming_receiver,
             stdin_receiver,
             handle,
@@ -36,17 +40,18 @@ impl SideNode {
         loop {
             match self.stdin_receiver.try_recv() {
                 Ok(stdin) => {
-                    println!("Received stdin input: {:?}", stdin);
-                    let transaction = utils::fake_transaction(stdin);
+                    let transaction = utils::fake_generic_transaction_json(stdin);
                     let json = serde_json::to_value(transaction).unwrap();
                     let signed_op = self.add_transaction_local(json);
+                    println!("STDIN: {}", utils::shappy(signed_op.clone()));
                     self.send_to_network(signed_op).await;
                 }
                 Err(_) => {} // ignore empty channel errors in this PoC
             }
             match self.incoming_receiver.try_recv() {
                 Ok(incoming) => {
-                    self.handle_incoming(&incoming);
+                    println!("INCOMING: {}", utils::shappy(incoming.clone()));
+                    self.handle_incoming(incoming);
                 }
                 Err(_) => {} // ignore empty channel errors in this PoC
             }
@@ -54,17 +59,16 @@ impl SideNode {
     }
 
     async fn send_to_network(&self, signed_op: SignedOp) {
-        println!("sending to network: {:?}", signed_op);
         let to_send = serde_json::to_string(&signed_op).unwrap();
         self.handle.call(to_send).unwrap();
     }
 
-    fn handle_incoming(&mut self, incoming: &SignedOp) {
-        println!("WINNNINGINGINGINGINGIGNIGN");
-        self.crdt.apply(incoming.clone());
+    pub fn handle_incoming(&mut self, incoming: SignedOp) {
+        self.crdt.apply(incoming);
+        // self.trace_crdt();
     }
 
-    pub(crate) fn add_transaction_local(
+    pub fn add_transaction_local(
         &mut self,
         transaction: serde_json::Value,
     ) -> bft_json_crdt::json_crdt::SignedOp {
@@ -80,12 +84,17 @@ impl SideNode {
             .doc
             .list
             .insert(last.id, transaction)
-            .sign(&self.keys);
+            .sign(&self.bft_crdt_keys);
+        // self.trace_crdt();
         signed_op
     }
 
     /// Print the current state of the CRDT, can be used to debug
-    pub(crate) fn _trace_crdt(&self) {
-        println!("{:?}", self.crdt.doc.list);
+    pub fn trace_crdt(&self) {
+        println!("{:?}", self.crdt.doc.view_sha());
+    }
+
+    pub fn current_sha(&self) -> String {
+        self.crdt.doc.view_sha()
     }
 }

crdt-node/src/utils.rs

@@ -0,0 +1,48 @@
+use bft_json_crdt::json_crdt::SignedOp;
+use serde_json::{json, Value};
+use std::path::PathBuf;
+
+pub(crate) const BITCOIN_KEY_FILE: &str = "bitcoin_keys.pem";
+pub(crate) const BFT_CRDT_KEY_FILE: &str = "keys.pem";
+pub(crate) const CONFIG_FILE: &str = "config.toml";
+
+/// Returns the path to the key file and config for this host OS.
+pub(crate) fn side_paths(prefix: PathBuf) -> (PathBuf, PathBuf, PathBuf) {
+    let mut bft_crdt_key_path = prefix.clone();
+    bft_crdt_key_path.push(BFT_CRDT_KEY_FILE);
+
+    let mut bitcoin_key_path = prefix.clone();
+    bitcoin_key_path.push(BITCOIN_KEY_FILE);
+
+    let mut config_path = prefix.clone();
+    config_path.push(CONFIG_FILE);
+
+    (bft_crdt_key_path, bitcoin_key_path, config_path)
+}
+
+/// Returns the path to the home directory for this host OS and the given node name
+pub(crate) fn home(name: &str) -> std::path::PathBuf {
+    let mut path = dirs::home_dir().unwrap();
+    path.push(".side");
+    path.push(name);
+    path
+}
+
+/// Generate a fake transaction with customizable from_pubkey String
+pub fn fake_generic_transaction_json(from: String) -> Value {
+    json!({
+        "from": from,
+        "to": "Bob",
+        "amount": 1
+    })
+}
+
+pub fn shappy(op: SignedOp) -> String {
+    let b = serde_json::to_string(&op).unwrap().into_bytes();
+    sha256::digest(b).to_string()
+}
+
+pub fn shassy(text: String) -> String {
+    let b = text.into_bytes();
+    sha256::digest(b).to_string()
+}

crdt-node/tests/crdt.rs

@@ -0,0 +1,54 @@
+use bft_json_crdt::json_crdt::BaseCrdt;
+use bft_json_crdt::keypair::make_keypair;
+use bft_json_crdt::op::ROOT_ID;
+use side_node::bft_crdt::TransactionList;
+
+// case 1 - send valid updates
+#[test]
+fn test_valid_updates() {
+    // Insert to bft-crdt.doc on local node, test applying the same operation to a remote node
+    // and check that the view is the same
+    let keypair1 = make_keypair();
+    let mut crdt1 = BaseCrdt::<TransactionList>::new(&keypair1);
+
+    let val_a = side_node::utils::fake_generic_transaction_json(String::from("a"));
+    let val_b = side_node::utils::fake_generic_transaction_json(String::from("b"));
+    let val_c = side_node::utils::fake_generic_transaction_json(String::from("c"));
+
+    let _a = crdt1
+        .doc
+        .list
+        .insert(ROOT_ID, val_a.clone())
+        .sign(&keypair1);
+    let _b = crdt1
+        .doc
+        .list
+        .insert(_a.id(), val_b.clone())
+        .sign(&keypair1);
+    let _c = crdt1
+        .doc
+        .list
+        .insert(_b.id(), val_c.clone())
+        .sign(&keypair1);
+
+    let keypair2 = make_keypair();
+    let mut crdt2 = BaseCrdt::<TransactionList>::new(&keypair2);
+    crdt2.apply(_a.clone());
+    crdt2.apply(_b);
+    crdt2.apply(_c.clone());
+
+    assert_eq!(
+        crdt2.doc.list.view(),
+        crdt1.doc.list.view(),
+        "views should be equal"
+    );
+
+    crdt2.apply(_a.clone());
+    crdt2.apply(_a);
+
+    assert_eq!(
+        crdt1.doc.list.view(),
+        crdt2.doc.list.view(),
+        "views are still equal after repeated applies"
+    );
+}

crdt-node/tests/side_node.rs

@@ -0,0 +1,60 @@
+use bft_json_crdt::{
+    json_crdt::{BaseCrdt, SignedOp},
+    keypair::make_keypair,
+};
+use side_node::{
+    bft_crdt::websocket::Client, bft_crdt::TransactionList, bitcoin, node::SideNode, utils,
+};
+use tokio::sync::mpsc;
+
+#[tokio::test]
+async fn test_distribute_via_websockets() {
+    let mut node1 = setup("alice").await;
+    let mut node2 = setup("bob").await;
+
+    assert_eq!(node1.current_sha(), node2.current_sha());
+
+    let transaction = utils::fake_generic_transaction_json("from_alice".to_string());
+    let signed_op = node1.add_transaction_local(transaction);
+    node2.handle_incoming(signed_op);
+
+    assert_eq!(node1.current_sha(), node2.current_sha());
+
+    let transaction = utils::fake_generic_transaction_json("from_alice2".to_string());
+    let signed_op = node1.add_transaction_local(transaction);
+    node2.handle_incoming(signed_op);
+
+    assert_eq!(node1.current_sha(), node2.current_sha());
+
+    let transaction = utils::fake_generic_transaction_json("from_alice3".to_string());
+    let signed_op = node1.add_transaction_local(transaction);
+    node2.handle_incoming(signed_op);
+
+    assert_eq!(node1.current_sha(), node2.current_sha());
+}
+
+/// Wire everything up, ignoring things we are not using in the test
+async fn setup(_: &str) -> SideNode {
+    // First, load up the keys and create a bft-bft-crdt
+    let bft_crdt_keys = make_keypair();
+    let mnemonic_words = bitcoin::keys::make_mnemonic();
+    let keys = bitcoin::keys::get(mnemonic_words).unwrap();
+    let bitcoin_wallet = bitcoin::clients::electrum::create_wallet(keys).unwrap();
+    let crdt = BaseCrdt::<TransactionList>::new(&bft_crdt_keys);
+
+    // Channels for internal communication, and a tokio task for stdin input
+    let (incoming_sender, incoming_receiver) = mpsc::channel::<SignedOp>(32);
+    let (_, stdin_receiver) = std::sync::mpsc::channel();
+
+    // Finally, create the node and return it
+    let handle = Client::new(incoming_sender).await;
+    let node = SideNode::new(
+        crdt,
+        bft_crdt_keys,
+        bitcoin_wallet,
+        incoming_receiver,
+        stdin_receiver,
+        handle,
+    );
+    node
+}

crdt-relayer/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "side-watcher"
+name = "crdt-relayer"
 version = "0.1.0"
 edition = "2021"
 
@@ -8,6 +8,7 @@ edition = "2021"
 [dependencies]
 async-trait = "0.1.52"
 ezsockets = { version = "*", features = ["tungstenite"] }
+sha256 = "1.5.0"
 tokio = { version = "1.17.0", features = ["full"] }
 tracing = "0.1.32"
 tracing-subscriber = "0.3.9"

crdt-relayer/src/main.rs

@@ -85,12 +85,13 @@ impl ezsockets::ServerExt for ChatServer {
                     .unzip();
 
                 tracing::info!(
-                    "sending {text} to [{sessions}] at `{room}`",
+                    "sending {hash} to [{sessions}] at `{room}`",
                     sessions = ids
                         .iter()
                         .map(|id| id.to_string())
                         .collect::<Vec<_>>()
-                        .join(",")
+                        .join(","),
+                    hash = shappy(text.clone())
                 );
                 for session in sessions {
                     session.text(text.clone()).unwrap();
@@ -144,7 +145,7 @@ impl ezsockets::SessionExt for SessionActor {
     }
 
     async fn on_text(&mut self, text: String) -> Result<(), Error> {
-        tracing::info!("received: {text}");
+        tracing::info!("received: {}", shappy(text.clone()));
         if text.starts_with('/') {
             let mut args = text.split_whitespace();
             let command = args.next().unwrap();
@@ -183,6 +184,11 @@ impl ezsockets::SessionExt for SessionActor {
     }
 }
 
+fn shappy(text: String) -> String {
+    let b = text.into_bytes();
+    sha256::digest(b).to_string()
+}
+
 #[tokio::main]
 async fn main() {
     tracing_subscriber::fmt::init();

side-node/Cargo.lock

@@ -1,7 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "side-node"
-version = "0.1.0"

side-node/src/crdt.rs

@@ -1,34 +0,0 @@
-use bft_crdt_derive::add_crdt_fields;
-use bft_json_crdt::{
-    json_crdt::{CrdtNode, IntoCrdtNode},
-    list_crdt::ListCrdt,
-};
-use serde::{Deserialize, Serialize};
-
-#[add_crdt_fields]
-#[derive(Clone, CrdtNode, Serialize, Deserialize)]
-pub(crate) struct TransactionList {
-    pub(crate) list: ListCrdt<Transaction>,
-}
-
-/// A fake Transaction struct we can use as a simulated payload
-#[add_crdt_fields]
-#[derive(Clone, CrdtNode, Serialize, Deserialize)]
-pub(crate) struct Transaction {
-    from: String,
-    to: String,
-    amount: f64,
-}
-
-// impl TransactionList {
-//     fn create(&mut self, keys: &Ed25519KeyPair) -> bft_json_crdt::json_crdt::SignedOp {
-//         // generate a placeholder transaction
-//         let transaction = _fake_transaction(keys.public().to_string());
-
-//         // next job is to keep adding to this guy
-//         let last: &Op<Transaction>;
-//         last = self.list.ops.last().expect("couldn't find last op");
-//         let signed_op = self.list.insert(last.id, transaction.clone()).sign(&keys);
-//         signed_op
-//     }
-// }

side-node/src/init/mod.rs

@@ -1,92 +0,0 @@
-use std::path::PathBuf;
-
-use config::SideNodeConfig;
-
-use crate::{keys, utils};
-
-pub(crate) mod config;
-
-pub(crate) fn init(home: PathBuf, config: SideNodeConfig) -> Result<(), std::io::Error> {
-    ensure_side_directory_exists(&home)?;
-    let (key_path, config_path) = utils::side_paths(home.clone());
-
-    println!("Writing key to: {:?}", key_path);
-    keys::write(key_path)?;
-
-    println!("Writing config to: {:?}", config_path);
-    config::write_toml(&config, &config_path).expect("unable to write config file");
-
-    Ok(())
-}
-
-/// Ensures that the directory at side_dir exists, so we have a place
-/// to store our key file and config file.
-fn ensure_side_directory_exists(side_dir: &PathBuf) -> Result<(), std::io::Error> {
-    if side_dir.exists() {
-        return Ok(());
-    }
-    println!(
-        "Config directory doesn't exist, creating at: {:?}",
-        side_dir
-    );
-    std::fs::create_dir_all(side_dir)
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{fs, path::Path};
-
-    use fastcrypto::{
-        ed25519::Ed25519KeyPair,
-        traits::{EncodeDecodeBase64, KeyPair, ToFromBytes},
-    };
-
-    use super::*;
-
-    fn default_side_node_config() -> SideNodeConfig {
-        SideNodeConfig {
-            name: "alice".to_string(),
-        }
-    }
-
-    #[test]
-    fn creates_side_node_directory() {
-        let mut test_home = PathBuf::new();
-        let side_dir = "/tmp/side";
-
-        // clean up any previous test runs
-        fs::remove_dir_all(side_dir).expect("couldn't remove side directory during test");
-
-        test_home.push(side_dir);
-        let node_dir = Path::new(&test_home).parent().unwrap().to_str().unwrap();
-        let _ = init(test_home.clone(), default_side_node_config());
-        assert!(std::path::Path::new(node_dir).exists());
-    }
-
-    #[test]
-    fn creates_key_file() {
-        let mut file_path = PathBuf::new();
-        file_path.push("/tmp/side");
-        let side_dir = file_path.clone();
-        file_path.push(utils::KEY_FILE);
-
-        let _ = init(side_dir.clone(), default_side_node_config());
-        assert!(file_path.exists());
-
-        // check that the pem is readable
-        let data = fs::read_to_string(file_path).expect("couldn't read key file");
-        let keys = Ed25519KeyPair::decode_base64(&data).expect("couldn't load keypair from file");
-        assert_eq!(keys.public().as_bytes().len(), 32);
-    }
-
-    #[test]
-    fn creates_config_file() {
-        let mut file_path = PathBuf::new();
-        file_path.push("/tmp/side");
-        let side_dir = file_path.clone();
-        file_path.push(utils::CONFIG_FILE);
-
-        let _ = init(side_dir.clone(), default_side_node_config());
-        assert!(file_path.exists());
-    }
-}

side-node/src/utils.rs

@@ -1,33 +0,0 @@
-use std::path::PathBuf;
-
-use serde_json::{json, Value};
-
-pub(crate) const KEY_FILE: &str = "keys.pem";
-pub(crate) const CONFIG_FILE: &str = "config.toml";
-
-/// Returns the path to the key file and config for this host OS.
-pub(crate) fn side_paths(prefix: PathBuf) -> (PathBuf, PathBuf) {
-    let mut key_path = prefix.clone();
-    key_path.push(KEY_FILE);
-
-    let mut config_path = prefix.clone();
-    config_path.push(CONFIG_FILE);
-
-    (key_path, config_path)
-}
-
-pub(crate) fn home(name: &String) -> std::path::PathBuf {
-    let mut path = dirs::home_dir().unwrap();
-    path.push(".side");
-    path.push(name);
-    path
-}
-
-/// Generate a fake transaction with customizable from_pubkey String
-pub(crate) fn fake_transaction(from_pubkey: String) -> Value {
-    json!({
-        "from": from_pubkey,
-        "to": "Bob",
-        "amount": 1
-    })
-}