From 33406135e9a76daee4e6677cec62076ccbc286b0 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 26 Jul 2023 20:57:21 +0200 Subject: [PATCH 001/172] feat: initial prototype of integrating iroh-sync --- Cargo.lock | 124 ++-- Cargo.toml | 1 + iroh-bytes/src/protocol.rs | 4 +- iroh-sync/Cargo.toml | 27 + iroh-sync/LICENSE-APACHE | 201 ++++++ iroh-sync/LICENSE-MIT | 25 + iroh-sync/README.md | 19 + iroh-sync/src/lib.rs | 2 + iroh-sync/src/ranger.rs | 1246 ++++++++++++++++++++++++++++++++++++ iroh-sync/src/sync.rs | 789 +++++++++++++++++++++++ iroh/Cargo.toml | 4 +- iroh/src/lib.rs | 1 + iroh/src/sync.rs | 189 ++++++ 13 files changed, 2591 insertions(+), 41 deletions(-) create mode 100644 iroh-sync/Cargo.toml create mode 100644 iroh-sync/LICENSE-APACHE create mode 100644 iroh-sync/LICENSE-MIT create mode 100644 iroh-sync/README.md create mode 100644 iroh-sync/src/lib.rs create mode 100644 iroh-sync/src/ranger.rs create mode 100644 iroh-sync/src/sync.rs create mode 100644 iroh/src/sync.rs diff --git a/Cargo.lock b/Cargo.lock index 885024000c..1537539433 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,15 +182,6 @@ dependencies = [ "syn 2.0.27", ] -[[package]] -name = "atomic-polyfill" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ff7eb3f316534d83a8a2c3d1674ace8a5a71198eba31e2e2b597833f699b28" -dependencies = [ - "critical-section", -] - [[package]] name = "atomic-waker" version = "1.1.1" @@ -614,10 +605,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" [[package]] -name = "critical-section" -version = "1.1.1" +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset 0.9.0", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] [[package]] name = "crossbeam-utils" @@ -1321,15 +1364,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hash32" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" -dependencies = [ - "byteorder", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1345,20 +1379,6 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" -[[package]] -name = "heapless" -version = "0.7.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" -dependencies = [ - "atomic-polyfill", - "hash32", - "rustc_version", - "serde", - "spin 0.9.8", - "stable_deref_trait", -] - [[package]] name = "heck" version = "0.4.1" @@ -1655,6 +1675,7 @@ dependencies = [ "iroh-io", "iroh-metrics", "iroh-net", + "iroh-sync", "multibase", "nix", "num_cpus", @@ -1664,6 +1685,7 @@ dependencies = [ "quic-rpc", "quinn", "rand", + "range-collections", "regex", "serde", "tempfile", @@ -1856,6 +1878,26 @@ dependencies = [ "zeroize", ] +[[package]] +name = "iroh-sync" +version = "0.1.0" +dependencies = [ + "anyhow", + "blake3", + "bytes", + "crossbeam", + "ed25519-dalek", + "hex", + "iroh-bytes", + "once_cell", + "parking_lot", + "rand", + "rand_core", + "serde", + "tokio", + "url", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -1997,6 +2039,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2123,7 +2174,7 @@ dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", - "memoffset", + "memoffset 0.7.1", "pin-utils", "static_assertions", ] @@ -2631,7 +2682,6 @@ checksum = "c9ee729232311d3cd113749948b689627618133b1c5012b77342c1950b25eaeb" dependencies = [ "cobs", "const_format", - "heapless", "postcard-derive", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 12f45b8100..156cba8b9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "iroh-bytes", "iroh-gossip", "iroh-metrics", + "iroh-sync", ] [profile.release] diff --git a/iroh-bytes/src/protocol.rs b/iroh-bytes/src/protocol.rs index 8ace63a8c8..5b1f8444d1 100644 --- a/iroh-bytes/src/protocol.rs +++ b/iroh-bytes/src/protocol.rs @@ -172,7 +172,7 @@ impl GetRequest { } /// Write the given data to the provider sink, with a unsigned varint length prefix. -pub(crate) async fn write_lp(writer: &mut W, data: &[u8]) -> Result<()> { +pub async fn write_lp(writer: &mut W, data: &[u8]) -> Result<()> { ensure!( data.len() < MAX_MESSAGE_SIZE, "sending message is too large" @@ -193,7 +193,7 @@ pub(crate) async fn write_lp(writer: &mut W, data: &[u8]) /// /// The message as raw bytes. If the end of the stream is reached and there is no partial /// message, returns `None`. -pub(crate) async fn read_lp( +pub async fn read_lp( mut reader: impl AsyncRead + Unpin, buffer: &mut BytesMut, ) -> Result> { diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml new file mode 100644 index 0000000000..b7aa4ff340 --- /dev/null +++ b/iroh-sync/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "iroh-sync" +version = "0.1.0" +edition = "2021" +readme = "README.md" +description = "Iroh sync" +license = "MIT/Apache-2.0" +authors = ["n0 team"] +repository = "https://github.com/n0-computer/iroh" + +[dependencies] +anyhow = "1.0.71" +blake3 = "1.3.3" +crossbeam = "0.8.2" +ed25519-dalek = { version = "2.0.0-rc.2", features = ["serde", "rand_core"] } +iroh-bytes = { version = "0.5.0", path = "../iroh-bytes" } +once_cell = "1.18.0" +rand = "0.8.5" +rand_core = "0.6.4" +serde = { version = "1.0.164", features = ["derive"] } +url = "2.4.0" +bytes = "1.4.0" +parking_lot = "0.12.1" +hex = "0.4" + +[dev-dependencies] +tokio = { version = "1.28.2", features = ["sync", "macros"] } diff --git a/iroh-sync/LICENSE-APACHE b/iroh-sync/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/iroh-sync/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/iroh-sync/LICENSE-MIT b/iroh-sync/LICENSE-MIT new file mode 100644 index 0000000000..dfd85baf84 --- /dev/null +++ b/iroh-sync/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2023 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/iroh-sync/README.md b/iroh-sync/README.md new file mode 100644 index 0000000000..7c79e368f2 --- /dev/null +++ b/iroh-sync/README.md @@ -0,0 +1,19 @@ +# iroh-sync + + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this project by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs new file mode 100644 index 0000000000..a37ead1b6f --- /dev/null +++ b/iroh-sync/src/lib.rs @@ -0,0 +1,2 @@ +pub mod ranger; +pub mod sync; diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs new file mode 100644 index 0000000000..159d99d8cf --- /dev/null +++ b/iroh-sync/src/ranger.rs @@ -0,0 +1,1246 @@ +//! Implementation of Set Reconcilliation based on +//! "Range-Based Set Reconciliation" by Aljoscha Meyer. +//! + +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::marker::PhantomData; + +use serde::{Deserialize, Serialize}; + +/// Stores a range. +/// +/// There are three possibilities +/// - x, x: All elements in a set, denoted with +/// - [x, y): x < y: Includes x, but not y +/// - S \ [y, x) y < x: Includes x, but not y. +/// This means that ranges are "wrap around" conceptually. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Default)] +pub struct Range { + x: K, + y: K, +} + +impl Range { + pub fn x(&self) -> &K { + &self.x + } + + pub fn y(&self) -> &K { + &self.y + } + + pub fn new(x: K, y: K) -> Self { + Range { x, y } + } + + pub fn map(self, f: impl FnOnce(K, K) -> (X, X)) -> Range { + let (x, y) = f(self.x, self.y); + Range { x, y } + } +} + +impl From<(K, K)> for Range { + fn from((x, y): (K, K)) -> Self { + Range { x, y } + } +} + +pub trait RangeKey: Sized + Ord + Debug { + /// Is this key inside the range? + fn contains(&self, range: &Range) -> bool { + contains(self, range) + } +} + +/// Default implementation of `contains` for `Ord` types. +pub fn contains(t: &T, range: &Range) -> bool { + match range.x().cmp(range.y()) { + Ordering::Equal => true, + Ordering::Less => range.x() <= t && t < range.y(), + Ordering::Greater => range.x() <= t || t < range.y(), + } +} + +impl RangeKey for &str {} +impl RangeKey for &[u8] {} + +#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] +pub struct Fingerprint(pub [u8; 32]); + +impl Debug for Fingerprint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Fp({})", blake3::Hash::from(self.0).to_hex()) + } +} + +impl Fingerprint { + /// The fingerprint of the empty set + pub fn empty() -> Self { + Fingerprint::new(&[][..]) + } + + pub fn new(val: T) -> Self { + val.as_fingerprint() + } +} + +pub trait AsFingerprint { + fn as_fingerprint(&self) -> Fingerprint; +} + +impl> AsFingerprint for T { + fn as_fingerprint(&self) -> Fingerprint { + Fingerprint(blake3::hash(self.as_ref()).into()) + } +} + +impl std::ops::BitXorAssign for Fingerprint { + fn bitxor_assign(&mut self, rhs: Self) { + for (a, b) in self.0.iter_mut().zip(rhs.0.iter()) { + *a ^= b; + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RangeFingerprint { + #[serde(bound( + serialize = "Range: Serialize", + deserialize = "Range: Deserialize<'de>" + ))] + pub range: Range, + /// The fingerprint of `range`. + pub fingerprint: Fingerprint, +} + +/// Transfers items inside a range to the other participant. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RangeItem { + /// The range out of which the elements are. + #[serde(bound( + serialize = "Range: Serialize", + deserialize = "Range: Deserialize<'de>" + ))] + pub range: Range, + #[serde(bound( + serialize = "K: Serialize, V: Serialize", + deserialize = "K: Deserialize<'de>, V: Deserialize<'de>" + ))] + pub values: Vec<(K, V)>, + /// If false, requests to send local items in the range. + /// Otherwise not. + pub have_local: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum MessagePart { + #[serde(bound( + serialize = "RangeFingerprint: Serialize", + deserialize = "RangeFingerprint: Deserialize<'de>" + ))] + RangeFingerprint(RangeFingerprint), + #[serde(bound( + serialize = "RangeItem: Serialize", + deserialize = "RangeItem: Deserialize<'de>" + ))] + RangeItem(RangeItem), +} + +impl MessagePart { + pub fn is_range_fingerprint(&self) -> bool { + matches!(self, MessagePart::RangeFingerprint(_)) + } + + pub fn is_range_item(&self) -> bool { + matches!(self, MessagePart::RangeItem(_)) + } + + pub fn values(&self) -> Option<&[(K, V)]> { + match self { + MessagePart::RangeFingerprint(_) => None, + MessagePart::RangeItem(RangeItem { values, .. }) => Some(&values), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Message { + #[serde(bound( + serialize = "MessagePart: Serialize", + deserialize = "MessagePart: Deserialize<'de>" + ))] + parts: Vec>, +} + +impl Message +where + K: RangeKey + Clone + Default + AsFingerprint, +{ + /// Construct the initial message. + fn init>(store: &S, limit: Option<&Range>) -> Self { + let x = store.get_first().clone(); + let range = Range::new(x.clone(), x); + let fingerprint = store.get_fingerprint(&range, limit); + let part = MessagePart::RangeFingerprint(RangeFingerprint { range, fingerprint }); + Message { parts: vec![part] } + } + + pub fn parts(&self) -> &[MessagePart] { + &self.parts + } +} + +pub trait Store: Sized + Default +where + K: RangeKey + Clone + Default + AsFingerprint, +{ + /// Get a the first key (or the default if none is available). + fn get_first(&self) -> K; + fn get(&self, key: &K) -> Option<&V>; + fn len(&self) -> usize; + fn is_empty(&self) -> bool; + /// Calculate the fingerprint of the given range. + fn get_fingerprint(&self, range: &Range, limit: Option<&Range>) -> Fingerprint; + + /// Insert the given key value pair. + fn put(&mut self, k: K, v: V); + + type RangeIterator<'a>: Iterator + where + Self: 'a, + K: 'a, + V: 'a; + + /// Returns all items in the given range + fn get_range<'a>(&'a self, range: Range, limit: Option>) + -> Self::RangeIterator<'a>; + fn remove(&mut self, key: &K) -> Option; + + type AllIterator<'a>: Iterator + where + Self: 'a, + K: 'a, + V: 'a; + fn all(&self) -> Self::AllIterator<'_>; +} + +#[derive(Debug)] +pub struct SimpleStore { + data: BTreeMap, +} + +impl Default for SimpleStore { + fn default() -> Self { + SimpleStore { + data: BTreeMap::default(), + } + } +} + +impl Store for SimpleStore +where + K: RangeKey + Clone + Default + AsFingerprint, +{ + fn get_first(&self) -> K { + if let Some((k, _)) = self.data.first_key_value() { + k.clone() + } else { + Default::default() + } + } + + fn get(&self, key: &K) -> Option<&V> { + self.data.get(key) + } + + fn len(&self) -> usize { + self.data.len() + } + + fn is_empty(&self) -> bool { + self.data.is_empty() + } + + /// Calculate the fingerprint of the given range. + fn get_fingerprint(&self, range: &Range, limit: Option<&Range>) -> Fingerprint { + let elements = self.get_range(range.clone(), limit.cloned()); + let mut fp = Fingerprint::empty(); + for el in elements { + fp ^= el.0.as_fingerprint(); + } + + fp + } + + /// Insert the given key value pair. + fn put(&mut self, k: K, v: V) { + self.data.insert(k, v); + } + + type RangeIterator<'a> = SimpleRangeIterator<'a, K, V> + where K: 'a, V: 'a; + /// Returns all items in the given range + fn get_range<'a>( + &'a self, + range: Range, + limit: Option>, + ) -> Self::RangeIterator<'a> { + // TODO: this is not very efficient, optimize depending on data structure + let iter = self.data.iter(); + + SimpleRangeIterator { iter, range, limit } + } + + fn remove(&mut self, key: &K) -> Option { + self.data.remove(key) + } + + type AllIterator<'a> = std::collections::btree_map::Iter<'a, K, V> + where K: 'a, + V: 'a; + + fn all(&self) -> Self::AllIterator<'_> { + self.data.iter() + } +} + +#[derive(Debug)] +pub struct SimpleRangeIterator<'a, K: 'a, V: 'a> { + iter: std::collections::btree_map::Iter<'a, K, V>, + range: Range, + limit: Option>, +} + +impl<'a, K, V> Iterator for SimpleRangeIterator<'a, K, V> +where + K: RangeKey, +{ + type Item = (&'a K, &'a V); + + fn next(&mut self) -> Option { + let mut next = self.iter.next()?; + + let filter = |x: &K| { + let r = x.contains(&self.range); + if let Some(ref limit) = self.limit { + r && x.contains(limit) + } else { + r + } + }; + + loop { + if filter(&next.0) { + return Some(next); + } + + next = self.iter.next()?; + } + } +} + +#[derive(Debug)] +pub struct Peer = SimpleStore> +where + K: RangeKey + Clone + Default + AsFingerprint, +{ + store: S, + /// Up to how many values to send immediately, before sending only a fingerprint. + max_set_size: usize, + /// `k` in the protocol, how many splits to generate. at least 2 + split_factor: usize, + limit: Option>, + + _phantom: PhantomData, // why??? +} + +impl Default for Peer +where + K: RangeKey + Clone + Default + AsFingerprint, + S: Store + Default, +{ + fn default() -> Self { + Peer { + store: S::default(), + max_set_size: 1, + split_factor: 2, + limit: None, + _phantom: Default::default(), + } + } +} + +impl Peer +where + K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, + V: Clone + Debug, + S: Store + Default, +{ + pub fn with_limit(limit: Range) -> Self { + Peer { + store: S::default(), + max_set_size: 1, + split_factor: 2, + limit: Some(limit), + _phantom: Default::default(), + } + } +} +impl Peer +where + K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, + V: Clone + Debug, + S: Store, +{ + /// Generates the initial message. + pub fn initial_message(&self) -> Message { + Message::init(&self.store, self.limit.as_ref()) + } + + /// Processes an incoming message and produces a response. + /// If terminated, returns `None` + pub fn process_message(&mut self, message: Message) -> Option> { + let mut out = Vec::new(); + + // TODO: can these allocs be avoided? + let mut items = Vec::new(); + let mut fingerprints = Vec::new(); + for part in message.parts { + match part { + MessagePart::RangeItem(item) => { + items.push(item); + } + MessagePart::RangeFingerprint(fp) => { + fingerprints.push(fp); + } + } + } + + // Process item messages + for RangeItem { + range, + values, + have_local, + } in items + { + let diff: Option> = if have_local { + None + } else { + Some( + self.store + .get_range(range.clone(), self.limit.clone()) + .into_iter() + .filter(|(k, _)| values.iter().find(|(vk, _)| &vk == k).is_none()) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ) + }; + + // Store incoming values + for (k, v) in values { + self.store.put(k, v); + } + + if let Some(diff) = diff { + if !diff.is_empty() { + out.push(MessagePart::RangeItem(RangeItem { + range, + values: diff, + have_local: true, + })); + } + } + } + + // Process fingerprint messages + for RangeFingerprint { range, fingerprint } in fingerprints { + let local_fingerprint = self.store.get_fingerprint(&range, self.limit.as_ref()); + + // Case1 Match, nothing to do + if local_fingerprint == fingerprint { + continue; + } + + // Case2 Recursion Anchor + let local_values: Vec<_> = self + .store + .get_range(range.clone(), self.limit.clone()) + .collect(); + if local_values.len() <= 1 || fingerprint == Fingerprint::empty() { + let values = local_values + .into_iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + out.push(MessagePart::RangeItem(RangeItem { + range, + values, + have_local: false, + })); + } else { + // Case3 Recurse + // Create partition + // m0 = x < m1 < .. < mk = y, with k>= 2 + // such that [ml, ml+1) is nonempty + let mut ranges = Vec::with_capacity(self.split_factor); + let chunk_len = div_ceil(local_values.len(), self.split_factor); + + // Select the first index, for which the key is larger than the x of the range. + let mut start_index = local_values + .iter() + .position(|(k, _)| range.x() <= k) + .unwrap_or(0); + let max_len = local_values.len(); + for i in 0..self.split_factor { + let s_index = start_index; + let start = (s_index * chunk_len) % max_len; + let e_index = s_index + 1; + let end = (e_index * chunk_len) % max_len; + + let (x, y) = if i == 0 { + // first + (range.x(), local_values[end].0) + } else if i == self.split_factor - 1 { + // last + (local_values[start].0, range.y()) + } else { + // regular + (local_values[start].0, local_values[end].0) + }; + let range = Range::new(x.clone(), y.clone()); + ranges.push(range); + start_index += 1; + } + + for range in ranges.into_iter() { + let chunk: Vec<_> = self + .store + .get_range(range.clone(), self.limit.clone()) + .collect(); + // Add either the fingerprint or the item set + let fingerprint = self.store.get_fingerprint(&range, self.limit.as_ref()); + if chunk.len() > self.max_set_size { + out.push(MessagePart::RangeFingerprint(RangeFingerprint { + range, + fingerprint, + })); + } else { + let values = chunk + .into_iter() + .map(|(k, v)| { + let k: K = k.clone(); + let v: V = v.clone(); + (k, v) + }) + .collect(); + out.push(MessagePart::RangeItem(RangeItem { + range, + values, + have_local: false, + })); + } + } + } + } + + // If we have any parts, return a message + if !out.is_empty() { + Some(Message { parts: out }) + } else { + None + } + } + + /// Insert a key value pair. + pub fn put(&mut self, k: K, v: V) { + self.store.put(k, v); + } + + pub fn get(&self, k: &K) -> Option<&V> { + self.store.get(k) + } + + /// Remove the given key. + pub fn remove(&mut self, k: &K) -> Option { + self.store.remove(k) + } + + /// List all existing key value pairs. + pub fn all(&self) -> impl Iterator { + self.store.all() + } + + /// Returns a refernce to the underlying store. + pub fn store(&self) -> &S { + &self.store + } +} + +/// Sadly https://doc.rust-lang.org/std/primitive.usize.html#method.div_ceil is still unstable.. +fn div_ceil(a: usize, b: usize) -> usize { + debug_assert!(a != 0); + debug_assert!(b != 0); + + a / b + (a % b != 0) as usize +} + +#[cfg(test)] +mod tests { + use std::fmt::Debug; + + use super::*; + + #[test] + fn test_paper_1() { + let alice_set = [("ape", 1), ("eel", 1), ("fox", 1), ("gnu", 1)]; + let bob_set = [ + ("bee", 1), + ("cat", 1), + ("doe", 1), + ("eel", 1), + ("fox", 1), + ("hog", 1), + ]; + + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); + + // Initial message + assert_eq!(res.alice_to_bob[0].parts.len(), 1); + assert!(res.alice_to_bob[0].parts[0].is_range_fingerprint()); + + // Response from Bob - recurse once + assert_eq!(res.bob_to_alice[0].parts.len(), 2); + assert!(res.bob_to_alice[0].parts[0].is_range_fingerprint()); + assert!(res.bob_to_alice[0].parts[1].is_range_fingerprint()); + + // Last response from Alice + assert_eq!(res.alice_to_bob[1].parts.len(), 3); + assert!(res.alice_to_bob[1].parts[0].is_range_item()); + assert!(res.alice_to_bob[1].parts[1].is_range_fingerprint()); + assert!(res.alice_to_bob[1].parts[2].is_range_item()); + + // Last response from Bob + assert_eq!(res.bob_to_alice[1].parts.len(), 2); + assert!(res.bob_to_alice[1].parts[0].is_range_item()); + assert!(res.bob_to_alice[1].parts[1].is_range_item()); + } + + #[test] + fn test_paper_2() { + let alice_set = [ + ("ape", 1), + ("bee", 1), + ("cat", 1), + ("doe", 1), + ("eel", 1), + ("fox", 1), // the only value being sent + ("gnu", 1), + ("hog", 1), + ]; + let bob_set = [ + ("ape", 1), + ("bee", 1), + ("cat", 1), + ("doe", 1), + ("eel", 1), + ("gnu", 1), + ("hog", 1), + ]; + + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 3, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); + } + + #[test] + fn test_paper_3() { + let alice_set = [ + ("ape", 1), + ("bee", 1), + ("cat", 1), + ("doe", 1), + ("eel", 1), + ("fox", 1), + ("gnu", 1), + ("hog", 1), + ]; + let bob_set = [("ape", 1), ("cat", 1), ("eel", 1), ("gnu", 1)]; + + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 3, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); + } + + #[test] + fn test_limits() { + let alice_set = [("ape", 1), ("bee", 1), ("cat", 1)]; + let bob_set = [("ape", 1), ("cat", 1), ("doe", 1)]; + + // No Limit + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 3, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); + + // With Limit: just ape + let limit = ("ape", "bee").into(); + let res = sync(Some(limit), &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 1, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 0, "B -> A message count"); + + // With Limit: just bee, cat + let limit = ("bee", "doe").into(); + let res = sync(Some(limit), &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); + } + + #[test] + fn test_prefixes_simple() { + let alice_set = [("/foo/bar", 1), ("/foo/baz", 1), ("/foo/cat", 1)]; + let bob_set = [("/foo/bar", 1), ("/alice/bar", 1), ("/alice/baz", 1)]; + + // No Limit + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); + + // With Limit: just /alice + let limit = ("/alice", "/b").into(); + let res = sync(Some(limit), &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 1, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); + } + + #[test] + fn test_prefixes_empty_alice() { + let alice_set = []; + let bob_set = [("/foo/bar", 1), ("/alice/bar", 1), ("/alice/baz", 1)]; + + // No Limit + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 1, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); + + // With Limit: just /alice + let limit = ("/alice", "/b").into(); + let res = sync(Some(limit), &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 1, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); + } + + #[test] + fn test_prefixes_empty_bob() { + let alice_set = [("/foo/bar", 1), ("/foo/baz", 1), ("/foo/cat", 1)]; + let bob_set = []; + + // No Limit + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); + + // With Limit: just /alice + let limit = ("/alice", "/b").into(); + let res = sync(Some(limit), &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 1, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 0, "B -> A message count"); + } + + #[test] + fn test_multikey() { + #[derive(Default, Clone, PartialEq, Eq, PartialOrd, Ord)] + struct Multikey { + author: [u8; 4], + key: Vec, + } + + impl RangeKey for Multikey { + fn contains(&self, range: &Range) -> bool { + let author = range.x().author.cmp(&range.y().author); + let key = range.x().key.cmp(&range.y().key); + + match (author, key) { + (Ordering::Equal, Ordering::Equal) => { + // All + true + } + (Ordering::Equal, Ordering::Less) => { + // Regular, based on key + range.x().key <= self.key && self.key < range.y().key + } + (Ordering::Equal, Ordering::Greater) => { + // Reverse, based on key + range.x().key <= self.key || self.key < range.y().key + } + (Ordering::Less, Ordering::Equal) => { + // Regular, based on author + range.x().author <= self.author && self.author < range.y().author + } + (Ordering::Greater, Ordering::Equal) => { + // Reverse, based on key + range.x().author <= self.author || self.author < range.y().author + } + (Ordering::Less, Ordering::Less) => { + // Regular, key and author + range.x().key <= self.key + && self.key < range.y().key + && range.x().author <= self.author + && self.author < range.y().author + } + (Ordering::Greater, Ordering::Greater) => { + // Reverse, key and author + (range.x().key <= self.key || self.key < range.y().key) + && (range.x().author <= self.author || self.author < range.y().author) + } + (Ordering::Less, Ordering::Greater) => { + // Regular author, Reverse key + (range.x().key <= self.key || self.key < range.y().key) + && (range.x().author <= self.author && self.author < range.y().author) + } + (Ordering::Greater, Ordering::Less) => { + // Regular key, Reverse author + (range.x().key <= self.key && self.key < range.y().key) + && (range.x().author <= self.author || self.author < range.y().author) + } + } + } + } + + impl Debug for Multikey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let key = if let Ok(key) = std::str::from_utf8(&self.key) { + key.to_string() + } else { + hex::encode(&self.key) + }; + f.debug_struct("Multikey") + .field("author", &hex::encode(&self.author)) + .field("key", &key) + .finish() + } + } + impl AsFingerprint for Multikey { + fn as_fingerprint(&self) -> Fingerprint { + let mut hasher = blake3::Hasher::new(); + hasher.update(&self.author); + hasher.update(&self.key); + Fingerprint(hasher.finalize().into()) + } + } + + impl Multikey { + fn new(author: [u8; 4], key: impl AsRef<[u8]>) -> Self { + Multikey { + author, + key: key.as_ref().to_vec(), + } + } + } + let author_a = [1u8; 4]; + let author_b = [2u8; 4]; + let alice_set = [ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_b, "bee"), 1), + (Multikey::new(author_a, "doe"), 1), + ]; + let bob_set = [ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_a, "cat"), 1), + (Multikey::new(author_b, "cat"), 1), + ]; + + // No limit + let res = sync(None, &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 2, "B -> A message count"); + res.assert_alice_set( + "no limit", + &[ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_b, "bee"), 1), + (Multikey::new(author_a, "doe"), 1), + (Multikey::new(author_a, "cat"), 1), + (Multikey::new(author_b, "cat"), 1), + ], + ); + + res.assert_bob_set( + "no limit", + &[ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_b, "bee"), 1), + (Multikey::new(author_a, "doe"), 1), + (Multikey::new(author_a, "cat"), 1), + (Multikey::new(author_b, "cat"), 1), + ], + ); + + // Only author_a + let limit = Range::new(Multikey::new(author_a, ""), Multikey::new(author_b, "")); + let res = sync(Some(limit), &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 2, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); + res.assert_alice_set( + "only author_a", + &[ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_b, "bee"), 1), + (Multikey::new(author_a, "doe"), 1), + (Multikey::new(author_a, "cat"), 1), + ], + ); + + res.assert_bob_set( + "only author_a", + &[ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_a, "cat"), 1), + (Multikey::new(author_b, "cat"), 1), + (Multikey::new(author_a, "doe"), 1), + ], + ); + + // All authors, but only cat + let limit = Range::new( + Multikey::new(author_a, "cat"), + Multikey::new(author_a, "doe"), + ); + let res = sync(Some(limit), &alice_set, &bob_set); + assert_eq!(res.alice_to_bob.len(), 1, "A -> B message count"); + assert_eq!(res.bob_to_alice.len(), 1, "B -> A message count"); + + res.assert_alice_set( + "only cat", + &[ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_b, "bee"), 1), + (Multikey::new(author_a, "doe"), 1), + (Multikey::new(author_a, "cat"), 1), + (Multikey::new(author_b, "cat"), 1), + ], + ); + + res.assert_bob_set( + "only cat", + &[ + (Multikey::new(author_a, "ape"), 1), + (Multikey::new(author_a, "bee"), 1), + (Multikey::new(author_a, "cat"), 1), + (Multikey::new(author_b, "cat"), 1), + ], + ); + } + + struct SyncResult + where + K: RangeKey + Clone + Default + AsFingerprint, + { + alice: Peer, + bob: Peer, + alice_to_bob: Vec>, + bob_to_alice: Vec>, + } + + impl SyncResult + where + K: RangeKey + Clone + Default + AsFingerprint + Debug, + V: Debug, + { + fn print_messages(&self) { + let len = std::cmp::max(self.alice_to_bob.len(), self.bob_to_alice.len()); + for i in 0..len { + if let Some(msg) = self.alice_to_bob.get(i) { + println!("A -> B:"); + print_message(msg); + } + if let Some(msg) = self.bob_to_alice.get(i) { + println!("B -> A:"); + print_message(msg); + } + } + } + } + + impl SyncResult + where + K: Debug + RangeKey + Clone + Default + AsFingerprint, + V: Debug + Clone + PartialEq, + { + fn assert_alice_set(&self, ctx: &str, expected: &[(K, V)]) { + dbg!(self.alice.all().collect::>()); + for (k, v) in expected { + assert_eq!( + self.alice.store.get(k), + Some(v), + "{}: (alice) missing key {:?}", + ctx, + k + ); + } + assert_eq!(expected.len(), self.alice.store.len(), "{}: (alice)", ctx); + } + + fn assert_bob_set(&self, ctx: &str, expected: &[(K, V)]) { + dbg!(self.bob.all().collect::>()); + + for (k, v) in expected { + assert_eq!( + self.bob.store.get(k), + Some(v), + "{}: (bob) missing key {:?}", + ctx, + k + ); + } + assert_eq!(expected.len(), self.bob.store.len(), "{}: (bob)", ctx); + } + } + + fn print_message(msg: &Message) + where + K: Debug, + V: Debug, + { + for part in &msg.parts { + match part { + MessagePart::RangeFingerprint(RangeFingerprint { range, fingerprint }) => { + println!( + " RangeFingerprint({:?}, {:?}, {:?})", + range.x(), + range.y(), + fingerprint + ); + } + MessagePart::RangeItem(RangeItem { + range, + values, + have_local, + }) => { + println!( + " RangeItem({:?} | {:?}) (local?: {})\n {:?}", + range.x(), + range.y(), + have_local, + values, + ); + } + } + } + } + + fn sync( + limit: Option>, + alice_set: &[(K, V)], + bob_set: &[(K, V)], + ) -> SyncResult + where + K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, + V: Clone + Debug + PartialEq, + { + println!("Using Limit: {:?}", limit); + let mut expected_set_alice = BTreeMap::new(); + let mut expected_set_bob = BTreeMap::new(); + + let mut alice = if let Some(limit) = limit.clone() { + Peer::::with_limit(limit) + } else { + Peer::::default() + }; + for (k, v) in alice_set { + alice.put(k.clone(), v.clone()); + + let include = if let Some(ref limit) = limit { + k.contains(limit) + } else { + true + }; + if include { + expected_set_bob.insert(k.clone(), v.clone()); + } + // alices things are always in alices store + expected_set_alice.insert(k.clone(), v.clone()); + } + + let mut bob = if let Some(limit) = limit.clone() { + Peer::::with_limit(limit) + } else { + Peer::::default() + }; + for (k, v) in bob_set { + bob.put(k.clone(), v.clone()); + let include = if let Some(ref limit) = limit { + k.contains(limit) + } else { + true + }; + if include { + expected_set_alice.insert(k.clone(), v.clone()); + } + // bobs things are always in bobs store + expected_set_bob.insert(k.clone(), v.clone()); + } + + let mut alice_to_bob = Vec::new(); + let mut bob_to_alice = Vec::new(); + let initial_message = alice.initial_message(); + + let mut next_to_bob = Some(initial_message); + let mut rounds = 0; + while let Some(msg) = next_to_bob.take() { + assert!(rounds < 100, "too many rounds"); + rounds += 1; + alice_to_bob.push(msg.clone()); + + if let Some(msg) = bob.process_message(msg) { + bob_to_alice.push(msg.clone()); + next_to_bob = alice.process_message(msg); + } + } + let res = SyncResult { + alice, + bob, + alice_to_bob, + bob_to_alice, + }; + res.print_messages(); + + let alice_now: Vec<_> = res.alice.all().collect(); + assert_eq!( + expected_set_alice.iter().collect::>(), + alice_now, + "alice" + ); + + let bob_now: Vec<_> = res.bob.all().collect(); + assert_eq!(expected_set_bob.iter().collect::>(), bob_now, "bob"); + + // Check that values were never sent twice + let mut alice_sent = BTreeMap::new(); + for msg in &res.alice_to_bob { + for part in &msg.parts { + if let Some(values) = part.values() { + for (key, value) in values { + assert!( + alice_sent.insert(key.clone(), value.clone()).is_none(), + "alice: duplicate {:?} - {:?}", + key, + value + ); + } + } + } + } + + let mut bob_sent = BTreeMap::new(); + for msg in &res.bob_to_alice { + for part in &msg.parts { + if let Some(values) = part.values() { + for (key, value) in values { + assert!( + bob_sent.insert(key.clone(), value.clone()).is_none(), + "bob: duplicate {:?} - {:?}", + key, + value + ); + } + } + } + } + + res + } + + #[test] + fn store_get_range() { + let mut store = SimpleStore::<&'static str, usize>::default(); + let set = [ + ("bee", 1), + ("cat", 1), + ("doe", 1), + ("eel", 1), + ("fox", 1), + ("hog", 1), + ]; + for (k, v) in &set { + store.put(*k, *v); + } + + let all: Vec<_> = store + .get_range(Range::new("", ""), None) + .into_iter() + .map(|(k, v)| (*k, *v)) + .collect(); + assert_eq!(&all, &set[..]); + + let regular: Vec<_> = store + .get_range(("bee", "eel").into(), None) + .into_iter() + .map(|(k, v)| (*k, *v)) + .collect(); + assert_eq!(®ular, &set[..3]); + + // empty start + let regular: Vec<_> = store + .get_range(("", "eel").into(), None) + .into_iter() + .map(|(k, v)| (*k, *v)) + .collect(); + assert_eq!(®ular, &set[..3]); + + let regular: Vec<_> = store + .get_range(("cat", "hog").into(), None) + .into_iter() + .map(|(k, v)| (*k, *v)) + .collect(); + assert_eq!(®ular, &set[1..5]); + + let excluded: Vec<_> = store + .get_range(("fox", "bee").into(), None) + .into_iter() + .map(|(k, v)| (*k, *v)) + .collect(); + + assert_eq!(excluded[0].0, "fox"); + assert_eq!(excluded[1].0, "hog"); + assert_eq!(excluded.len(), 2); + + let excluded: Vec<_> = store + .get_range(("fox", "doe").into(), None) + .into_iter() + .map(|(k, v)| (*k, *v)) + .collect(); + + assert_eq!(excluded.len(), 4); + assert_eq!(excluded[0].0, "bee"); + assert_eq!(excluded[1].0, "cat"); + assert_eq!(excluded[2].0, "fox"); + assert_eq!(excluded[3].0, "hog"); + + // Limit + let all: Vec<_> = store + .get_range(("", "").into(), Some(("bee", "doe").into())) + .into_iter() + .map(|(k, v)| (*k, *v)) + .collect(); + assert_eq!(&all, &set[..2]); + } + + #[test] + fn test_div_ceil() { + assert_eq!(div_ceil(1, 1), 1 / 1); + assert_eq!(div_ceil(2, 1), 2 / 1); + assert_eq!(div_ceil(4, 2), 4 / 2); + + assert_eq!(div_ceil(3, 2), 2); + assert_eq!(div_ceil(5, 3), 2); + } +} diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs new file mode 100644 index 0000000000..f4dcece817 --- /dev/null +++ b/iroh-sync/src/sync.rs @@ -0,0 +1,789 @@ +// Names and concepts are roughly based on Willows design at the moment: +// +// https://hackmd.io/DTtck8QOQm6tZaQBBtTf7w +// +// This is going to change! + +use std::{ + cmp::Ordering, + collections::{BTreeMap, HashMap}, + fmt::{Debug, Display}, + str::FromStr, + sync::Arc, + time::SystemTime, +}; + +use parking_lot::RwLock; + +use bytes::Bytes; +use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey}; +use iroh_bytes::Hash; +use rand_core::CryptoRngCore; +use serde::{Deserialize, Serialize}; + +use crate::ranger::{AsFingerprint, Fingerprint, Peer, Range, RangeKey}; + +pub type ProtocolMessage = crate::ranger::Message; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Author { + priv_key: SigningKey, + id: AuthorId, +} + +impl Display for Author { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Author({})", hex::encode(self.priv_key.to_bytes())) + } +} + +impl Author { + pub fn new(rng: &mut R) -> Self { + let priv_key = SigningKey::generate(rng); + let id = AuthorId(priv_key.verifying_key()); + + Author { priv_key, id } + } + + pub fn id(&self) -> &AuthorId { + &self.id + } + + pub fn sign(&self, msg: &[u8]) -> Signature { + self.priv_key.sign(msg) + } + + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.id.verify(msg, signature) + } +} + +#[derive(Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct AuthorId(VerifyingKey); + +impl Debug for AuthorId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "AuthorId({})", hex::encode(self.0.as_bytes())) + } +} + +impl AuthorId { + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.0.verify_strict(msg, signature) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + self.0.as_bytes() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Namespace { + priv_key: SigningKey, + id: NamespaceId, +} + +impl Display for Namespace { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Namespace({})", hex::encode(self.priv_key.to_bytes())) + } +} + +impl FromStr for Namespace { + type Err = (); + + fn from_str(s: &str) -> Result { + let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; + let priv_key = SigningKey::from_bytes(&priv_key); + let id = NamespaceId(priv_key.verifying_key()); + Ok(Namespace { priv_key, id }) + } +} + +impl FromStr for Author { + type Err = (); + + fn from_str(s: &str) -> Result { + let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; + let priv_key = SigningKey::from_bytes(&priv_key); + let id = AuthorId(priv_key.verifying_key()); + Ok(Author { priv_key, id }) + } +} + +impl Namespace { + pub fn new(rng: &mut R) -> Self { + let priv_key = SigningKey::generate(rng); + let id = NamespaceId(priv_key.verifying_key()); + + Namespace { priv_key, id } + } + + pub fn id(&self) -> &NamespaceId { + &self.id + } + + pub fn sign(&self, msg: &[u8]) -> Signature { + self.priv_key.sign(msg) + } + + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.id.verify(msg, signature) + } +} + +#[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct NamespaceId(VerifyingKey); + +impl Display for NamespaceId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "NamespaceId({})", hex::encode(self.0.as_bytes())) + } +} + +impl Debug for NamespaceId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "NamespaceId({})", hex::encode(self.0.as_bytes())) + } +} + +impl NamespaceId { + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.0.verify_strict(msg, signature) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + self.0.as_bytes() + } +} + +/// Manages the replicas and authors for an instance. +#[derive(Debug, Clone, Default)] +pub struct ReplicaStore { + replicas: Arc>>, + authors: Arc>>, +} + +impl ReplicaStore { + pub fn get_replica(&self, namespace: &NamespaceId) -> Option { + let replicas = &*self.replicas.read(); + replicas.get(namespace).cloned() + } + + pub fn get_author(&self, author: &AuthorId) -> Option { + let authors = &*self.authors.read(); + authors.get(author).cloned() + } + + pub fn new_author(&self, rng: &mut R) -> Author { + let author = Author::new(rng); + self.authors.write().insert(*author.id(), author.clone()); + author + } + + pub fn new_replica(&self, namespace: Namespace) -> Replica { + let replica = Replica::new(namespace); + self.replicas + .write() + .insert(replica.namespace(), replica.clone()); + replica + } +} + +#[derive(Debug, Clone)] +pub struct Replica { + inner: Arc>, +} + +#[derive(Debug)] +struct InnerReplica { + namespace: Namespace, + peer: Peer, + content: HashMap, +} + +#[derive(Default, Debug, Clone)] +pub struct Store { + /// Stores records by identifier + timestamp + records: BTreeMap>, +} + +impl Store { + pub fn latest(&self) -> impl Iterator { + self.records.iter().filter_map(|(k, values)| { + let (_, v) = values.last_key_value()?; + Some((k, v)) + }) + } +} + +impl crate::ranger::Store for Store { + /// Get a the first key (or the default if none is available). + fn get_first(&self) -> RecordIdentifier { + self.records + .first_key_value() + .map(|(k, _)| k.clone()) + .unwrap_or_default() + } + + fn get(&self, key: &RecordIdentifier) -> Option<&SignedEntry> { + self.records + .get(key) + .and_then(|values| values.last_key_value()) + .map(|(_, v)| v) + } + + fn len(&self) -> usize { + self.records.len() + } + + fn is_empty(&self) -> bool { + self.records.is_empty() + } + + fn get_fingerprint( + &self, + range: &Range, + limit: Option<&Range>, + ) -> Fingerprint { + let elements = self.get_range(range.clone(), limit.cloned()); + let mut fp = Fingerprint::empty(); + for el in elements { + fp ^= el.0.as_fingerprint(); + } + + fp + } + + fn put(&mut self, k: RecordIdentifier, v: SignedEntry) { + // TODO: propagate error/not insertion? + if v.verify().is_ok() { + let timestamp = v.entry().record().timestamp(); + // TODO: verify timestamp is "reasonable" + + self.records.entry(k).or_default().insert(timestamp, v); + } + } + + type RangeIterator<'a> = RangeIterator<'a>; + fn get_range<'a>( + &'a self, + range: Range, + limit: Option>, + ) -> Self::RangeIterator<'a> { + RangeIterator { + iter: self.records.iter(), + range: Some(range), + limit, + } + } + + fn remove(&mut self, key: &RecordIdentifier) -> Option { + self.records + .remove(key) + .and_then(|mut v| v.last_entry().map(|e| e.remove_entry().1)) + } + + type AllIterator<'a> = RangeIterator<'a>; + + fn all(&self) -> Self::AllIterator<'_> { + RangeIterator { + iter: self.records.iter(), + range: None, + limit: None, + } + } +} + +#[derive(Debug)] +pub struct RangeIterator<'a> { + iter: std::collections::btree_map::Iter<'a, RecordIdentifier, BTreeMap>, + range: Option>, + limit: Option>, +} + +impl<'a> RangeIterator<'a> { + fn matches(&self, x: &RecordIdentifier) -> bool { + let range = self.range.as_ref().map(|r| x.contains(r)).unwrap_or(true); + let limit = self.limit.as_ref().map(|r| x.contains(r)).unwrap_or(true); + range && limit + } +} + +impl<'a> Iterator for RangeIterator<'a> { + type Item = (&'a RecordIdentifier, &'a SignedEntry); + + fn next(&mut self) -> Option { + let mut next = self.iter.next()?; + loop { + if self.matches(&next.0) { + let (k, values) = next; + let (_, v) = values.last_key_value()?; + return Some((k, v)); + } + + next = self.iter.next()?; + } + } +} + +impl Replica { + pub fn new(namespace: Namespace) -> Self { + Replica { + inner: Arc::new(RwLock::new(InnerReplica { + namespace, + peer: Peer::default(), + content: HashMap::default(), + })), + } + } + + pub fn get_content(&self, hash: &Hash) -> Option { + self.inner.read().content.get(hash).cloned() + } + + // TODO: not horrible + pub fn all(&self) -> Vec<(RecordIdentifier, SignedEntry)> { + self.inner + .read() + .peer + .all() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() + } + + /// Inserts a new record at the given key. + pub fn insert(&self, key: impl AsRef<[u8]>, author: &Author, data: impl Into) { + let mut inner = self.inner.write(); + + let id = RecordIdentifier::new(key, inner.namespace.id(), author.id()); + let data: Bytes = data.into(); + let record = Record::from_data(&data, inner.namespace.id()); + + // Store content + inner.content.insert(*record.content_hash(), data); + + // Store signed entries + let entry = Entry::new(id.clone(), record); + let signed_entry = entry.sign(&inner.namespace, author); + inner.peer.put(id, signed_entry); + } + + /// Gets all entries matching this key and author. + pub fn get_latest(&self, key: impl AsRef<[u8]>, author: &AuthorId) -> Option { + let inner = self.inner.read(); + inner + .peer + .get(&RecordIdentifier::new(key, &inner.namespace.id(), author)) + .cloned() + } + + /// Returns all versions of the matching documents. + pub fn get_all<'a, 'b: 'a>( + &'a self, + key: impl AsRef<[u8]> + 'b, + author: &AuthorId, + ) -> GetAllIter<'a> { + let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); + let record_id = RecordIdentifier::new(key, guard.namespace.id(), author); + GetAllIter { + records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { + &inner.peer.store().records + }), + record_id, + index: 0, + } + } + + pub fn sync_initial_message(&self) -> crate::ranger::Message { + self.inner.read().peer.initial_message() + } + + pub fn sync_process_message( + &self, + message: crate::ranger::Message, + ) -> Option> { + self.inner.write().peer.process_message(message) + } + + pub fn namespace(&self) -> NamespaceId { + *self.inner.read().namespace.id() + } +} + +#[derive(Debug)] +pub struct GetAllIter<'a> { + // Oh my god, rust why u do this to me? + records: parking_lot::lock_api::MappedRwLockReadGuard< + 'a, + parking_lot::RawRwLock, + BTreeMap>, + >, + record_id: RecordIdentifier, + /// Current iteration index. + index: usize, +} + +impl<'a> Iterator for GetAllIter<'a> { + type Item = SignedEntry; + + fn next(&mut self) -> Option { + let values = self.records.get(&self.record_id)?; + + let (_, res) = values.iter().nth(self.index)?; + self.index += 1; + Some(res.clone()) // :( I give up + } +} + +/// A signed entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SignedEntry { + signature: EntrySignature, + entry: Entry, +} + +impl SignedEntry { + pub fn from_entry(entry: Entry, namespace: &Namespace, author: &Author) -> Self { + let signature = EntrySignature::from_entry(&entry, namespace, author); + SignedEntry { signature, entry } + } + + pub fn verify(&self) -> Result<(), SignatureError> { + self.signature + .verify(&self.entry, &self.entry.id.namespace, &self.entry.id.author) + } + + pub fn signature(&self) -> &EntrySignature { + &self.signature + } + + pub fn entry(&self) -> &Entry { + &self.entry + } +} + +/// Signature over an entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EntrySignature { + author_signature: Signature, + namespace_signature: Signature, +} + +impl EntrySignature { + pub fn from_entry(entry: &Entry, namespace: &Namespace, author: &Author) -> Self { + // TODO: this should probably include a namespace prefix + // namespace in the cryptographic sense. + let bytes = entry.to_vec(); + let namespace_signature = namespace.sign(&bytes); + let author_signature = author.sign(&bytes); + + EntrySignature { + author_signature, + namespace_signature, + } + } + + pub fn verify( + &self, + entry: &Entry, + namespace: &NamespaceId, + author: &AuthorId, + ) -> Result<(), SignatureError> { + let bytes = entry.to_vec(); + namespace.verify(&bytes, &self.namespace_signature)?; + author.verify(&bytes, &self.author_signature)?; + + Ok(()) + } +} + +/// A single entry in a replica. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Entry { + id: RecordIdentifier, + record: Record, +} + +impl Entry { + pub fn new(id: RecordIdentifier, record: Record) -> Self { + Entry { id, record } + } + + pub fn id(&self) -> &RecordIdentifier { + &self.id + } + + pub fn record(&self) -> &Record { + &self.record + } + + /// Serialize this entry into its canonical byte representation used for signing. + pub fn into_vec(&self, out: &mut Vec) { + self.id.as_bytes(out); + self.record.as_bytes(out); + } + + pub fn to_vec(&self) -> Vec { + let mut out = Vec::new(); + self.into_vec(&mut out); + out + } + + pub fn sign(self, namespace: &Namespace, author: &Author) -> SignedEntry { + SignedEntry::from_entry(self, namespace, author) + } +} + +/// The indentifier of a record. +#[derive(Default, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct RecordIdentifier { + /// The key of the record. + key: Vec, + /// The namespace this record belongs to. + namespace: NamespaceId, + /// The author that wrote this record. + author: AuthorId, +} + +impl AsFingerprint for RecordIdentifier { + fn as_fingerprint(&self) -> crate::ranger::Fingerprint { + let mut hasher = blake3::Hasher::new(); + hasher.update(self.namespace.as_bytes()); + hasher.update(self.author.as_bytes()); + hasher.update(&self.key); + Fingerprint(hasher.finalize().into()) + } +} + +impl PartialOrd for NamespaceId { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for NamespaceId { + fn cmp(&self, other: &Self) -> Ordering { + self.0.as_bytes().cmp(other.0.as_bytes()) + } +} + +impl PartialOrd for AuthorId { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for AuthorId { + fn cmp(&self, other: &Self) -> Ordering { + self.0.as_bytes().cmp(other.0.as_bytes()) + } +} + +impl RangeKey for RecordIdentifier { + fn contains(&self, range: &crate::ranger::Range) -> bool { + // For now we just do key inclusion and check if namespace and author match + if self.namespace != range.x().namespace || self.namespace != range.y().namespace { + return false; + } + if self.author != range.x().author || self.author != range.y().author { + return false; + } + + let mapped_range = range.clone().map(|x, y| (x.key, y.key)); + crate::ranger::contains(&self.key, &mapped_range) + } +} + +impl RecordIdentifier { + pub fn new(key: impl AsRef<[u8]>, namespace: &NamespaceId, author: &AuthorId) -> Self { + RecordIdentifier { + key: key.as_ref().to_vec(), + namespace: *namespace, + author: *author, + } + } + + pub fn as_bytes(&self, out: &mut Vec) { + out.extend_from_slice(self.namespace.as_bytes()); + out.extend_from_slice(self.author.as_bytes()); + out.extend_from_slice(&self.key); + } + + pub fn key(&self) -> &[u8] { + &self.key + } + + pub fn namespace(&self) -> &NamespaceId { + &self.namespace + } + + pub fn author(&self) -> &AuthorId { + &self.author + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Record { + /// Record creation timestamp. Counted as micros since the Unix epoch. + timestamp: u64, + /// Length of the data referenced by `hash`. + len: u64, + hash: Hash, +} + +impl Record { + pub fn new(timestamp: u64, len: u64, hash: Hash) -> Self { + Record { + timestamp, + len, + hash, + } + } + + pub fn timestamp(&self) -> u64 { + self.timestamp + } + + pub fn content_len(&self) -> u64 { + self.len + } + + pub fn content_hash(&self) -> &Hash { + &self.hash + } + + pub fn from_data(data: impl AsRef<[u8]>, namespace: &NamespaceId) -> Self { + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("time drift") + .as_micros() as u64; + let data = data.as_ref(); + let len = data.len() as u64; + // Salted hash + // TODO: do we actually want this? + // TODO: this should probably use a namespace prefix if used + let mut hasher = blake3::Hasher::new(); + hasher.update(namespace.as_bytes()); + hasher.update(data); + let hash = hasher.finalize(); + + Self::new(timestamp, len, hash.into()) + } + + pub fn as_bytes(&self, out: &mut Vec) { + out.extend_from_slice(&self.timestamp.to_be_bytes()); + out.extend_from_slice(&self.len.to_be_bytes()); + out.extend_from_slice(self.hash.as_ref()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basics() { + let mut rng = rand::thread_rng(); + let alice = Author::new(&mut rng); + let myspace = Namespace::new(&mut rng); + + let record_id = RecordIdentifier::new("/my/key", myspace.id(), alice.id()); + let record = Record::from_data(b"this is my cool data", myspace.id()); + let entry = Entry::new(record_id, record); + let signed_entry = entry.sign(&myspace, &alice); + signed_entry.verify().expect("failed to verify"); + + let my_replica = Replica::new(myspace); + for i in 0..10 { + my_replica.insert(format!("/{i}"), &alice, format!("{i}: hello from alice")); + } + + for i in 0..10 { + let res = my_replica.get_latest(format!("/{i}"), alice.id()).unwrap(); + let len = format!("{i}: hello from alice").as_bytes().len() as u64; + assert_eq!(res.entry().record().content_len(), len); + res.verify().expect("invalid signature"); + } + + // Test multiple records for the same key + my_replica.insert("/cool/path", &alice, "round 1"); + let entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); + let content = my_replica + .get_content(entry.entry().record().content_hash()) + .unwrap(); + assert_eq!(&content[..], b"round 1"); + + // Second + + my_replica.insert("/cool/path", &alice, "round 2"); + let entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); + let content = my_replica + .get_content(entry.entry().record().content_hash()) + .unwrap(); + assert_eq!(&content[..], b"round 2"); + + // Get All + let entries: Vec<_> = my_replica.get_all("/cool/path", alice.id()).collect(); + assert_eq!(entries.len(), 2); + let content = my_replica + .get_content(entries[0].entry().record().content_hash()) + .unwrap(); + assert_eq!(&content[..], b"round 1"); + let content = my_replica + .get_content(entries[1].entry().record().content_hash()) + .unwrap(); + assert_eq!(&content[..], b"round 2"); + } + + #[test] + fn test_replica_sync() { + let alice_set = ["ape", "eel", "fox", "gnu"]; + let bob_set = ["bee", "cat", "doe", "eel", "fox", "hog"]; + + let mut rng = rand::thread_rng(); + let author = Author::new(&mut rng); + let myspace = Namespace::new(&mut rng); + let mut alice = Replica::new(myspace.clone()); + for el in &alice_set { + alice.insert(el, &author, el.as_bytes()); + } + + let mut bob = Replica::new(myspace); + for el in &bob_set { + bob.insert(el, &author, el.as_bytes()); + } + + sync(&author, &mut alice, &mut bob, &alice_set, &bob_set); + } + + fn sync( + author: &Author, + alice: &mut Replica, + bob: &mut Replica, + alice_set: &[&str], + bob_set: &[&str], + ) { + // Sync alice - bob + let mut next_to_bob = Some(alice.sync_initial_message()); + let mut rounds = 0; + while let Some(msg) = next_to_bob.take() { + assert!(rounds < 100, "too many rounds"); + rounds += 1; + if let Some(msg) = bob.sync_process_message(msg) { + next_to_bob = alice.sync_process_message(msg); + } + } + + // Check result + for el in alice_set { + alice.get_latest(el, author.id()).unwrap(); + bob.get_latest(el, author.id()).unwrap(); + } + + for el in bob_set { + alice.get_latest(el, author.id()).unwrap(); + bob.get_latest(el, author.id()).unwrap(); + } + } +} diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index dc7ea98b02..37f83a2585 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -26,9 +26,11 @@ iroh-metrics = { version = "0.5.0", path = "../iroh-metrics", optional = true } iroh-net = { version = "0.5.1", path = "../iroh-net" } num_cpus = { version = "1.15.0" } portable-atomic = "1" +iroh-sync = { path = "../iroh-sync" } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quic-rpc = { version = "0.6", default-features = false, features = ["flume-transport"] } quinn = "0.10" +range-collections = { version = "0.4.0" } rand = "0.8" serde = { version = "1", features = ["derive"] } thiserror = "1" @@ -65,9 +67,7 @@ bytes = "1" duct = "0.13.6" genawaiter = { version = "0.99", features = ["futures03"] } nix = "0.26.2" -postcard = "1" proptest = "1.2.0" -rand = "0.8" regex = { version = "1.7.1", features = ["std"] } tempfile = "3.4" testdir = "0.8" diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index b8a363921c..c1c5e6c1d3 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -11,6 +11,7 @@ pub mod dial; pub mod get; pub mod node; pub mod rpc_protocol; +pub mod sync; pub mod util; /// Expose metrics module diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs new file mode 100644 index 0000000000..751bc13efe --- /dev/null +++ b/iroh/src/sync.rs @@ -0,0 +1,189 @@ +//! Implementation of the iroh-sync protocol + +use anyhow::{bail, ensure, Result}; +use bytes::BytesMut; +use iroh_sync::sync::{NamespaceId, Replica, ReplicaStore}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncWrite}; + +/// The ALPN identifier for the iroh-sync protocol +pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; + +/// Sync Protocol +/// +/// - Init message: signals which namespace is being synced +/// - N Sync messages +/// +/// On any error and on success the substream is closed. +#[derive(Debug, Clone, Serialize, Deserialize)] +enum Message { + Init { + /// Namespace to sync + namespace: NamespaceId, + /// Initial message + message: iroh_sync::sync::ProtocolMessage, + }, + Sync(iroh_sync::sync::ProtocolMessage), +} + +/// Runs the initiator side of the sync protocol. +pub async fn run_alice( + writer: &mut W, + reader: &mut R, + alice: &Replica, +) -> Result<()> { + let mut buffer = BytesMut::with_capacity(1024); + + // Init message + + let init_message = Message::Init { + namespace: alice.namespace(), + message: alice.sync_initial_message(), + }; + let msg_bytes = postcard::to_stdvec(&init_message)?; + iroh_bytes::protocol::write_lp(writer, &msg_bytes).await?; + + // Sync message loop + + while let Some(read) = iroh_bytes::protocol::read_lp(&mut *reader, &mut buffer).await? { + println!("read {}", read.len()); + let msg = postcard::from_bytes(&read)?; + match msg { + Message::Init { .. } => { + bail!("unexpected message: init"); + } + Message::Sync(msg) => { + if let Some(msg) = alice.sync_process_message(msg) { + send_sync_message(writer, msg).await?; + } else { + break; + } + } + } + } + + Ok(()) +} + +/// Handle an iroh-sync connection and sync all shared documents in the replica store. +pub async fn handle_connection( + connecting: quinn::Connecting, + replica_store: ReplicaStore, +) -> Result<()> { + let connection = connecting.await?; + let (mut send_stream, mut recv_stream) = connection.accept_bi().await?; + + run_bob(&mut send_stream, &mut recv_stream, replica_store).await?; + send_stream.finish().await?; + + println!("done"); + + Ok(()) +} + +/// Runs the receiver side of the sync protocol. +pub async fn run_bob( + writer: &mut W, + reader: &mut R, + replica_store: ReplicaStore, +) -> Result<()> { + let mut buffer = BytesMut::with_capacity(1024); + + let mut replica = None; + while let Some(read) = iroh_bytes::protocol::read_lp(&mut *reader, &mut buffer).await? { + println!("read {}", read.len()); + let msg = postcard::from_bytes(&read)?; + + match msg { + Message::Init { namespace, message } => { + ensure!(replica.is_none(), "double init message"); + + match replica_store.get_replica(&namespace) { + Some(r) => { + println!("starting sync for {}", namespace); + if let Some(msg) = r.sync_process_message(message) { + send_sync_message(writer, msg).await?; + } else { + break; + } + replica = Some(r); + } + None => { + // TODO: this should be possible. + bail!("unable to synchronize unknown namespace: {}", namespace); + } + } + } + Message::Sync(msg) => match replica { + Some(ref replica) => { + if let Some(msg) = replica.sync_process_message(msg) { + send_sync_message(writer, msg).await?; + } else { + break; + } + } + None => { + bail!("unexpected sync message without init"); + } + }, + } + } + + Ok(()) +} + +async fn send_sync_message( + stream: &mut W, + msg: iroh_sync::sync::ProtocolMessage, +) -> Result<()> { + let msg_bytes = postcard::to_stdvec(&Message::Sync(msg))?; + iroh_bytes::protocol::write_lp(stream, &msg_bytes).await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use iroh_sync::sync::Namespace; + + use super::*; + + #[tokio::test] + async fn test_sync_simple() -> Result<()> { + let mut rng = rand::thread_rng(); + + let replica_store = ReplicaStore::default(); + // For now uses same author on both sides. + let author = replica_store.new_author(&mut rng); + let namespace = Namespace::new(&mut rng); + let bob_replica = replica_store.new_replica(namespace.clone()); + bob_replica.insert("hello alice", &author, "from bob"); + + let alice_replica = Replica::new(namespace.clone()); + alice_replica.insert("hello bob", &author, "from alice"); + + assert_eq!(bob_replica.all().len(), 1); + assert_eq!(alice_replica.all().len(), 1); + + let (alice, bob) = tokio::io::duplex(64); + + let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); + let replica = alice_replica.clone(); + let alice_task = tokio::task::spawn(async move { + run_alice(&mut alice_writer, &mut alice_reader, &replica).await + }); + + let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); + let bob_replica_store = replica_store.clone(); + let bob_task = tokio::task::spawn(async move { + run_bob(&mut bob_writer, &mut bob_reader, bob_replica_store).await + }); + + alice_task.await??; + bob_task.await??; + + assert_eq!(bob_replica.all().len(), 2); + assert_eq!(alice_replica.all().len(), 2); + + Ok(()) + } +} From 867d74183c6902ad3912ff58d65c52df453faac6 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 3 Jul 2023 10:38:42 +0200 Subject: [PATCH 002/172] sync: start impl of multikey --- iroh-sync/src/sync.rs | 78 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index f4dcece817..8a5278b6a8 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -582,16 +582,15 @@ impl Ord for AuthorId { impl RangeKey for RecordIdentifier { fn contains(&self, range: &crate::ranger::Range) -> bool { - // For now we just do key inclusion and check if namespace and author match - if self.namespace != range.x().namespace || self.namespace != range.y().namespace { - return false; - } - if self.author != range.x().author || self.author != range.y().author { - return false; - } + use crate::ranger::contains; + + let key_range = range.clone().map(|x, y| (x.key, y.key)); + let namespace_range = range.clone().map(|x, y| (x.namespace, y.namespace)); + let author_range = range.clone().map(|x, y| (x.author, y.author)); - let mapped_range = range.clone().map(|x, y| (x.key, y.key)); - crate::ranger::contains(&self.key, &mapped_range) + contains(&self.key, &key_range) + && contains(&self.namespace, &namespace_range) + && contains(&self.author, &author_range) } } @@ -736,6 +735,67 @@ mod tests { assert_eq!(&content[..], b"round 2"); } + #[test] + fn test_multikey() { + let mut rng = rand::thread_rng(); + + let k = vec!["a", "c", "z"]; + + let mut n: Vec<_> = (0..3).map(|_| Namespace::new(&mut rng)).collect(); + n.sort_by_key(|n| *n.id()); + + let mut a: Vec<_> = (0..3).map(|_| Author::new(&mut rng)).collect(); + a.sort_by_key(|a| *a.id()); + + // Just key + { + let ri0 = RecordIdentifier::new(k[0], n[0].id(), a[0].id()); + let ri1 = RecordIdentifier::new(k[1], n[0].id(), a[0].id()); + let ri2 = RecordIdentifier::new(k[2], n[0].id(), a[0].id()); + + let range = Range::new(ri0.clone(), ri2.clone()); + assert!(ri0.contains(&range), "start"); + assert!(ri1.contains(&range), "inside"); + assert!(!ri2.contains(&range), "end"); + } + + // Just namespace + { + let ri0 = RecordIdentifier::new(k[0], n[0].id(), a[0].id()); + let ri1 = RecordIdentifier::new(k[0], n[1].id(), a[0].id()); + let ri2 = RecordIdentifier::new(k[0], n[2].id(), a[0].id()); + + let range = Range::new(ri0.clone(), ri2.clone()); + assert!(ri0.contains(&range), "start"); + assert!(ri1.contains(&range), "inside"); + assert!(!ri2.contains(&range), "end"); + } + + // Just author + { + let ri0 = RecordIdentifier::new(k[0], n[0].id(), a[0].id()); + let ri1 = RecordIdentifier::new(k[0], n[0].id(), a[1].id()); + let ri2 = RecordIdentifier::new(k[0], n[0].id(), a[2].id()); + + let range = Range::new(ri0.clone(), ri2.clone()); + assert!(ri0.contains(&range), "start"); + assert!(ri1.contains(&range), "inside"); + assert!(!ri2.contains(&range), "end"); + } + + // Just key and namespace + { + let ri0 = RecordIdentifier::new(k[0], n[0].id(), a[0].id()); + let ri1 = RecordIdentifier::new(k[1], n[1].id(), a[0].id()); + let ri2 = RecordIdentifier::new(k[2], n[2].id(), a[0].id()); + + let range = Range::new(ri0.clone(), ri2.clone()); + assert!(ri0.contains(&range), "start"); + assert!(ri1.contains(&range), "inside"); + assert!(!ri2.contains(&range), "end"); + } + } + #[test] fn test_replica_sync() { let alice_set = ["ape", "eel", "fox", "gnu"]; From f5c51c0920e0999d03ccf0ebc4987e606cb7722a Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 6 Jul 2023 19:27:50 +0200 Subject: [PATCH 003/172] feat: integrate iroh-sync and iroh-gossip, add example --- Cargo.lock | 4 + iroh-sync/Cargo.toml | 1 + iroh-sync/src/sync.rs | 71 ++++++++- iroh/Cargo.toml | 10 ++ iroh/examples/sync.rs | 349 ++++++++++++++++++++++++++++++++++++++++++ iroh/src/lib.rs | 1 + iroh/src/sync.rs | 36 ++++- iroh/src/sync/live.rs | 282 ++++++++++++++++++++++++++++++++++ 8 files changed, 747 insertions(+), 7 deletions(-) create mode 100644 iroh/examples/sync.rs create mode 100644 iroh/src/sync/live.rs diff --git a/Cargo.lock b/Cargo.lock index 1537539433..bb565f78b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1666,12 +1666,14 @@ dependencies = [ "derive_more", "dirs-next", "duct", + "ed25519-dalek", "flume", "futures", "genawaiter", "hex", "indicatif", "iroh-bytes", + "iroh-gossip", "iroh-io", "iroh-metrics", "iroh-net", @@ -1679,6 +1681,7 @@ dependencies = [ "multibase", "nix", "num_cpus", + "once_cell", "portable-atomic", "postcard", "proptest", @@ -1886,6 +1889,7 @@ dependencies = [ "blake3", "bytes", "crossbeam", + "derive_more", "ed25519-dalek", "hex", "iroh-bytes", diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index b7aa4ff340..0ed3ee73f7 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -12,6 +12,7 @@ repository = "https://github.com/n0-computer/iroh" anyhow = "1.0.71" blake3 = "1.3.3" crossbeam = "0.8.2" +derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } ed25519-dalek = { version = "2.0.0-rc.2", features = ["serde", "rand_core"] } iroh-bytes = { version = "0.5.0", path = "../iroh-bytes" } once_cell = "1.18.0" diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 8a5278b6a8..34d7d9bd50 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -45,6 +45,10 @@ impl Author { Author { priv_key, id } } + pub fn from_bytes(bytes: &[u8; 32]) -> Self { + SigningKey::from_bytes(&bytes).into() + } + pub fn id(&self) -> &AuthorId { &self.id } @@ -67,6 +71,12 @@ impl Debug for AuthorId { } } +impl Display for AuthorId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", hex::encode(self.0.as_bytes())) + } +} + impl AuthorId { pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.0.verify_strict(msg, signature) @@ -111,6 +121,20 @@ impl FromStr for Author { } } +impl From for Author { + fn from(priv_key: SigningKey) -> Self { + let id = AuthorId(priv_key.verifying_key()); + Self { priv_key, id } + } +} + +impl From for Namespace { + fn from(priv_key: SigningKey) -> Self { + let id = NamespaceId(priv_key.verifying_key()); + Self { priv_key, id } + } +} + impl Namespace { pub fn new(rng: &mut R) -> Self { let priv_key = SigningKey::generate(rng); @@ -119,6 +143,10 @@ impl Namespace { Namespace { priv_key, id } } + pub fn from_bytes(bytes: &[u8; 32]) -> Self { + SigningKey::from_bytes(bytes).into() + } + pub fn id(&self) -> &NamespaceId { &self.id } @@ -190,16 +218,30 @@ impl ReplicaStore { } } +/// TODO: Would potentially nice to pass a `&SignedEntry` reference, however that would make +/// everything `!Send`. +/// TODO: Not sure if the `Sync` requirement will be a problem for implementers. It comes from +/// [parking_lot::RwLock] requiring `Sync`. +pub type OnInsertCallback = Box; + #[derive(Debug, Clone)] +pub enum InsertOrigin { + Local, + Sync, +} + +#[derive(derive_more::Debug, Clone)] pub struct Replica { inner: Arc>, } -#[derive(Debug)] +#[derive(derive_more::Debug)] struct InnerReplica { namespace: Namespace, peer: Peer, content: HashMap, + #[debug("on_insert: [Box; {}]", "self.on_insert.len()")] + on_insert: Vec, } #[derive(Default, Debug, Clone)] @@ -334,10 +376,16 @@ impl Replica { namespace, peer: Peer::default(), content: HashMap::default(), + on_insert: Default::default(), })), } } + pub fn on_insert(&self, callback: OnInsertCallback) { + let mut inner = self.inner.write(); + inner.on_insert.push(callback); + } + pub fn get_content(&self, hash: &Hash) -> Option { self.inner.read().content.get(hash).cloned() } @@ -366,7 +414,26 @@ impl Replica { // Store signed entries let entry = Entry::new(id.clone(), record); let signed_entry = entry.sign(&inner.namespace, author); - inner.peer.put(id, signed_entry); + inner.peer.put(id, signed_entry.clone()); + for cb in &inner.on_insert { + cb(InsertOrigin::Local, signed_entry.clone()) + } + } + + pub fn id(&self, key: impl AsRef<[u8]>, author: &Author) -> RecordIdentifier { + let inner = self.inner.read(); + let id = RecordIdentifier::new(key, inner.namespace.id(), author.id()); + id + } + + pub fn insert_remote_entry(&self, entry: SignedEntry) -> anyhow::Result<()> { + entry.verify()?; + let mut inner = self.inner.write(); + inner.peer.put(entry.entry.id.clone(), entry.clone()); + for cb in &inner.on_insert { + cb(InsertOrigin::Sync, entry.clone()) + } + Ok(()) } /// Gets all entries matching this key and author. diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 37f83a2585..75f3c13f31 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -27,6 +27,7 @@ iroh-net = { version = "0.5.1", path = "../iroh-net" } num_cpus = { version = "1.15.0" } portable-atomic = "1" iroh-sync = { path = "../iroh-sync" } +iroh-gossip = { path = "../iroh-gossip" } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quic-rpc = { version = "0.6", default-features = false, features = ["flume-transport"] } quinn = "0.10" @@ -52,6 +53,10 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = tr data-encoding = "2.4.0" url = { version = "2.4", features = ["serde"] } +# Examples +once_cell = { version = "1.18.0", optional = true } +ed25519-dalek = { version = "=2.0.0-rc.3", features = ["serde", "rand_core"], optional = true } + [features] default = ["cli", "metrics"] cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection"] @@ -60,6 +65,7 @@ mem-db = [] flat-db = [] iroh-collection = [] test = [] +example-sync = ["cli", "ed25519-dalek", "once_cell"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } @@ -85,3 +91,7 @@ required-features = ["mem-db", "iroh-collection"] [[example]] name = "hello-world" required-features = ["mem-db"] + +[[example]] +name = "sync" +required-features = ["example-sync"] diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs new file mode 100644 index 0000000000..0b8acf5774 --- /dev/null +++ b/iroh/examples/sync.rs @@ -0,0 +1,349 @@ +//! Live edit a p2p document +//! +//! By default a new peer id is created when starting the example. To reuse your identity, +//! set the `--private-key` CLI flag with the private key printed on a previous invocation. +//! +//! You can use this with a local DERP server. To do so, run +//! `cargo run --bin derper -- --dev` +//! and then set the `-d http://localhost:3340` flag on this example. + +use std::{fmt, str::FromStr}; + +use anyhow::bail; +use clap::Parser; +use ed25519_dalek::SigningKey; +use iroh::sync::{LiveSync, PeerSource, SYNC_ALPN}; +use iroh_gossip::{ + net::{GossipHandle, GOSSIP_ALPN}, + proto::TopicId, +}; +use iroh_net::{ + defaults::{default_derp_map, DEFAULT_DERP_STUN_PORT}, + derp::{DerpMap, UseIpv4, UseIpv6}, + magic_endpoint::get_alpn, + tls::Keypair, + MagicEndpoint, +}; +use iroh_sync::sync::{Author, Namespace, Replica, ReplicaStore, SignedEntry}; +use once_cell::sync::OnceCell; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +use url::Url; + +#[derive(Parser, Debug)] +struct Args { + /// Private key to derive our peer id from + #[clap(long)] + private_key: Option, + /// Set a custom DERP server. By default, the DERP server hosted by n0 will be used. + #[clap(short, long)] + derp: Option, + /// Disable DERP completeley + #[clap(long)] + no_derp: bool, + /// Set your nickname + #[clap(short, long)] + name: Option, + /// Set the bind port for our socket. By default, a random port will be used. + #[clap(short, long, default_value = "0")] + bind_port: u16, + #[clap(subcommand)] + command: Command, +} + +#[derive(Parser, Debug)] +enum Command { + Open { doc_name: String }, + Join { ticket: String }, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt::init(); + let args = Args::parse(); + run(args).await +} + +async fn run(args: Args) -> anyhow::Result<()> { + // parse or generate our keypair + let keypair = match args.private_key { + None => Keypair::generate(), + Some(key) => parse_keypair(&key)?, + }; + println!("> our private key: {}", fmt_secret(&keypair)); + + // configure our derp map + let derp_map = match (args.no_derp, args.derp) { + (false, None) => Some(default_derp_map()), + (false, Some(url)) => Some(derp_map_from_url(url)?), + (true, None) => None, + (true, Some(_)) => bail!("You cannot set --no-derp and --derp at the same time"), + }; + println!("> using DERP servers: {}", fmt_derp_map(&derp_map)); + + // init a cell that will hold our gossip handle to be used in endpoint callbacks + let gossip_cell: OnceCell = OnceCell::new(); + // init a channel that will emit once the initial endpoints of our local node are discovered + let (initial_endpoints_tx, mut initial_endpoints_rx) = mpsc::channel(1); + + // build our magic endpoint + let gossip_cell_clone = gossip_cell.clone(); + let endpoint = MagicEndpoint::builder() + .keypair(keypair.clone()) + .alpns(vec![GOSSIP_ALPN.to_vec(), SYNC_ALPN.to_vec()]) + .derp_map(derp_map) + .on_endpoints(Box::new(move |endpoints| { + // send our updated endpoints to the gossip protocol to be sent as PeerData to peers + if let Some(gossip) = gossip_cell_clone.get() { + gossip.update_endpoints(endpoints).ok(); + } + // trigger oneshot on the first endpoint update + initial_endpoints_tx.try_send(endpoints.to_vec()).ok(); + })) + .bind(args.bind_port) + .await?; + println!("> our peer id: {}", endpoint.peer_id()); + + // wait for a first endpoint update so that we know about at least one of our addrs + let initial_endpoints = initial_endpoints_rx.recv().await.unwrap(); + // println!("> our endpoints: {initial_endpoints:?}"); + + let (topic, peers) = match &args.command { + Command::Open { doc_name } => { + let topic: TopicId = blake3::hash(doc_name.as_bytes()).into(); + println!( + "> opening document {doc_name} as namespace {} and waiting for peers to join us...", + fmt_hash(topic.as_bytes()) + ); + (topic, vec![]) + } + Command::Join { ticket } => { + let Ticket { topic, peers } = Ticket::from_str(ticket)?; + println!("> joining topic {topic} and connecting to {peers:?}",); + (topic, peers) + } + }; + + let our_ticket = { + // add our local endpoints to the ticket and print it for others to join + let addrs = initial_endpoints.iter().map(|ep| ep.addr).collect(); + let mut peers = peers.clone(); + peers.push(PeerSource { + peer_id: endpoint.peer_id(), + addrs, + derp_region: endpoint.my_derp().await, + }); + Ticket { peers, topic } + }; + println!("> ticket to join us: {our_ticket}"); + + // create the gossip protocol + let gossip = { + let gossip = GossipHandle::from_endpoint(endpoint.clone(), Default::default()); + // insert the gossip handle into the gossip cell to be used in the endpoint callbacks above + gossip_cell.set(gossip.clone()).unwrap(); + // pass our initial peer println to the gossip protocol + gossip.update_endpoints(&initial_endpoints)?; + gossip + }; + + // create the sync doc and store + let (store, author, doc) = create_document(topic, &keypair)?; + + // spawn our endpoint loop that forwards incoming connections + tokio::spawn(endpoint_loop( + endpoint.clone(), + gossip.clone(), + store.clone(), + )); + + // spawn an input thread that reads stdin + // not using tokio here because they recommend this for "technical reasons" + let (line_tx, mut line_rx) = tokio::sync::mpsc::channel::(1); + std::thread::spawn(move || input_loop(line_tx)); + + // create the live syncer + let mut sync_handle = LiveSync::spawn(endpoint.clone(), gossip.clone(), doc.clone(), peers); + + // do some logging + doc.on_insert(Box::new(move |origin, entry| { + println!("> insert from {origin:?}: {}", fmt_entry(&entry)); + })); + + // process stdin lines + println!("> read to accept commands: set | get | ls | exit"); + while let Some(text) = line_rx.recv().await { + let mut parts = text.split(' '); + match [parts.next(), parts.next(), parts.next()] { + [Some("set"), Some(key), Some(value)] => { + let key = key.to_string(); + let value = value.to_string(); + doc.insert(&key, &author, value); + } + [Some("get"), Some(key), None] => { + // TODO: we need a way to get all filtered by key from all authors + let mut entries = doc + .all() + .into_iter() + .filter_map(|(id, entry)| (id.key() == key.as_bytes()).then(|| entry)); + while let Some(entry) = entries.next() { + println!("{} -> {}", fmt_entry(&entry), fmt_content(&doc, &entry)); + } + } + [Some("ls"), None, None] => { + let all = doc.all(); + println!("> {} entries", all.len()); + for (_id, entry) in all { + println!("{} -> {}", fmt_entry(&entry), fmt_content(&doc, &entry)); + } + } + [Some("exit"), None, None] => { + let res = sync_handle.cancel().await?; + println!("syncer closed with {res:?}"); + break; + } + _ => println!("> invalid command"), + } + } + + Ok(()) +} + +fn create_document( + topic: TopicId, + keypair: &Keypair, +) -> anyhow::Result<(ReplicaStore, Author, Replica)> { + let author = Author::from(keypair.secret().clone()); + let namespace = Namespace::from_bytes(topic.as_bytes()); + let store = ReplicaStore::default(); + let doc = store.new_replica(namespace); + Ok((store, author, doc)) +} + +async fn endpoint_loop( + endpoint: MagicEndpoint, + gossip: GossipHandle, + replica_store: ReplicaStore, +) -> anyhow::Result<()> { + while let Some(mut conn) = endpoint.accept().await { + let alpn = get_alpn(&mut conn).await?; + println!("> incoming connection with alpn {alpn}"); + // let (peer_id, alpn, conn) = accept_conn(conn).await?; + let res = match alpn.as_bytes() { + GOSSIP_ALPN => gossip.handle_connection(conn.await?).await, + SYNC_ALPN => iroh::sync::handle_connection(conn, replica_store.clone()).await, + _ => Err(anyhow::anyhow!( + "ignoring connection: unsupported ALPN protocol" + )), + }; + if let Err(err) = res { + tracing::error!("connection for {alpn} errored: {err:?}"); + } + } + Ok(()) +} + +fn input_loop(line_tx: tokio::sync::mpsc::Sender) -> anyhow::Result<()> { + let mut buffer = String::new(); + let stdin = std::io::stdin(); // We get `Stdin` here. + loop { + stdin.read_line(&mut buffer)?; + line_tx.blocking_send(buffer.trim().to_string())?; + buffer.clear(); + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Ticket { + topic: TopicId, + peers: Vec, +} +impl Ticket { + /// Deserializes from bytes. + fn from_bytes(bytes: &[u8]) -> anyhow::Result { + postcard::from_bytes(bytes).map_err(Into::into) + } + /// Serializes to bytes. + pub fn to_bytes(&self) -> Vec { + postcard::to_stdvec(self).expect("postcard::to_stdvec is infallible") + } +} + +/// Serializes to base32. +impl fmt::Display for Ticket { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let encoded = self.to_bytes(); + let mut text = data_encoding::BASE32_NOPAD.encode(&encoded); + text.make_ascii_lowercase(); + write!(f, "{text}") + } +} + +/// Deserializes from base32. +impl FromStr for Ticket { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let bytes = data_encoding::BASE32_NOPAD.decode(s.to_ascii_uppercase().as_bytes())?; + let slf = Self::from_bytes(&bytes)?; + Ok(slf) + } +} + +// helpers + +fn fmt_entry(entry: &SignedEntry) -> String { + let id = entry.entry().id(); + let key = std::str::from_utf8(id.key()).unwrap_or(""); + let hash = entry.entry().record().content_hash(); + let author = fmt_hash(id.author().as_bytes()); + let fmt_hash = fmt_hash(hash.as_bytes()); + format!("@{author}: {key} = {fmt_hash}") +} +fn fmt_content(doc: &Replica, entry: &SignedEntry) -> String { + let hash = entry.entry().record().content_hash(); + let content = doc.get_content(hash); + let content = content + .map(|content| String::from_utf8(content.into()).unwrap()) + .unwrap_or_else(|| "".into()); + content +} +fn fmt_hash(hash: &[u8]) -> String { + let mut text = data_encoding::BASE32_NOPAD.encode(hash); + text.make_ascii_lowercase(); + format!("{}…{}", &text[..5], &text[(text.len() - 2)..]) +} +fn fmt_secret(keypair: &Keypair) -> String { + let mut text = data_encoding::BASE32_NOPAD.encode(&keypair.secret().to_bytes()); + text.make_ascii_lowercase(); + text +} +fn parse_keypair(secret: &str) -> anyhow::Result { + let bytes: [u8; 32] = data_encoding::BASE32_NOPAD + .decode(secret.to_ascii_uppercase().as_bytes())? + .try_into() + .map_err(|_| anyhow::anyhow!("Invalid secret"))?; + let key = SigningKey::from_bytes(&bytes); + Ok(key.into()) +} +fn fmt_derp_map(derp_map: &Option) -> String { + match derp_map { + None => "None".to_string(), + Some(map) => { + let regions = map.regions.iter().map(|(id, region)| { + let nodes = region.nodes.iter().map(|node| node.url.to_string()); + (*id, nodes.collect::>()) + }); + format!("{:?}", regions.collect::>()) + } + } +} +fn derp_map_from_url(url: Url) -> anyhow::Result { + Ok(DerpMap::default_from_node( + url, + DEFAULT_DERP_STUN_PORT, + UseIpv4::TryDns, + UseIpv6::TryDns, + 0 + )) +} diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index c1c5e6c1d3..55afd7c449 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -11,6 +11,7 @@ pub mod dial; pub mod get; pub mod node; pub mod rpc_protocol; +#[allow(missing_docs)] pub mod sync; pub mod util; diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index 751bc13efe..68de0fe252 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -1,14 +1,21 @@ //! Implementation of the iroh-sync protocol -use anyhow::{bail, ensure, Result}; +use std::net::SocketAddr; + +use anyhow::{bail, ensure, Context, Result}; use bytes::BytesMut; +use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::sync::{NamespaceId, Replica, ReplicaStore}; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; +use tracing::debug; /// The ALPN identifier for the iroh-sync protocol pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; +mod live; +pub use live::*; + /// Sync Protocol /// /// - Init message: signals which namespace is being synced @@ -26,6 +33,24 @@ enum Message { Sync(iroh_sync::sync::ProtocolMessage), } +pub async fn connect_and_sync( + endpoint: &MagicEndpoint, + doc: &Replica, + peer_id: PeerId, + derp_region: Option, + addrs: &[SocketAddr], +) -> anyhow::Result<()> { + debug!("sync with peer {}: start", peer_id); + let connection = endpoint + .connect(peer_id, SYNC_ALPN, derp_region, addrs) + .await + .context("dial_and_sync")?; + let (mut send_stream, mut recv_stream) = connection.open_bi().await?; + let res = run_alice(&mut send_stream, &mut recv_stream, &doc).await; + debug!("sync with peer {}: finish {:?}", peer_id, res); + res +} + /// Runs the initiator side of the sync protocol. pub async fn run_alice( writer: &mut W, @@ -46,7 +71,7 @@ pub async fn run_alice( // Sync message loop while let Some(read) = iroh_bytes::protocol::read_lp(&mut *reader, &mut buffer).await? { - println!("read {}", read.len()); + debug!("read {}", read.len()); let msg = postcard::from_bytes(&read)?; match msg { Message::Init { .. } => { @@ -71,12 +96,13 @@ pub async fn handle_connection( replica_store: ReplicaStore, ) -> Result<()> { let connection = connecting.await?; + debug!("> connection established!"); let (mut send_stream, mut recv_stream) = connection.accept_bi().await?; run_bob(&mut send_stream, &mut recv_stream, replica_store).await?; send_stream.finish().await?; - println!("done"); + debug!("done"); Ok(()) } @@ -91,7 +117,7 @@ pub async fn run_bob( let mut replica = None; while let Some(read) = iroh_bytes::protocol::read_lp(&mut *reader, &mut buffer).await? { - println!("read {}", read.len()); + debug!("read {}", read.len()); let msg = postcard::from_bytes(&read)?; match msg { @@ -100,7 +126,7 @@ pub async fn run_bob( match replica_store.get_replica(&namespace) { Some(r) => { - println!("starting sync for {}", namespace); + debug!("starting sync for {}", namespace); if let Some(msg) = r.sync_process_message(message) { send_sync_message(writer, msg).await?; } else { diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs new file mode 100644 index 0000000000..1697a87819 --- /dev/null +++ b/iroh/src/sync/live.rs @@ -0,0 +1,282 @@ +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; + +use crate::sync::connect_and_sync; +use anyhow::{anyhow, Context}; +use futures::{ + future::{BoxFuture, Shared}, + stream::FuturesUnordered, + FutureExt, TryFutureExt, +}; +use iroh_gossip::{ + net::{Event, GossipHandle}, + proto::TopicId, +}; +use iroh_net::{tls::PeerId, MagicEndpoint}; +use iroh_sync::sync::{InsertOrigin, Replica, SignedEntry}; +use serde::{Deserialize, Serialize}; +use tokio::{ + sync::{broadcast, mpsc}, + task::JoinError, +}; +use tokio_stream::StreamExt; +use tracing::error; + +const CHANNEL_CAP: usize = 8; + +/// The address to connect to a peer +/// TODO: Move into iroh-net +/// TODO: Make an enum and support DNS resolution +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PeerSource { + pub peer_id: PeerId, + pub addrs: Vec, + pub derp_region: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Op { + Put(SignedEntry), +} + +#[derive(Debug)] +enum SyncState { + Running, + Finished, + Failed(anyhow::Error), +} + +#[derive(Debug)] +pub enum ToActor { + Shutdown, +} + +/// Handle to a running live sync actor +#[derive(Debug, Clone)] +pub struct LiveSync { + to_actor_tx: mpsc::Sender, + task: Shared>>>, +} + +impl LiveSync { + pub fn spawn( + endpoint: MagicEndpoint, + gossip: GossipHandle, + doc: Replica, + initial_peers: Vec, + ) -> Self { + let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); + let mut actor = Actor::new(endpoint, gossip, doc, initial_peers, to_actor_rx); + let task = tokio::spawn(async move { actor.run().await }); + let handle = LiveSync { + to_actor_tx, + task: task.map_err(Arc::new).boxed().shared(), + }; + handle + } + + /// Cancel the live sync. + pub async fn cancel(&mut self) -> anyhow::Result<()> { + self.to_actor_tx.send(ToActor::Shutdown).await?; + self.task.clone().await?; + Ok(()) + } +} + +// TODO: Right now works with a single doc. Can quite easily be extended to work on a set of +// replicas. Then the handle above could have a +// `join_doc(doc: Replica, initial_peers: Vec, + gossip_stream: GossipStream, + to_actor_rx: mpsc::Receiver, + insert_entry_rx: mpsc::UnboundedReceiver, + sync_state: HashMap, + gossip: GossipHandle, + running_sync_tasks: FuturesUnordered)>>, +} + +impl Actor { + pub fn new( + endpoint: MagicEndpoint, + gossip: GossipHandle, + replica: Replica, + initial_peers: Vec, + to_actor_rx: mpsc::Receiver, + ) -> Self { + // TODO: instead of an unbounded channel, we'd want a FIFO ring buffer likely + // (we have to send from the blocking Replica::on_insert callback, so we need a channel + // with nonblocking sending, so either unbounded or ringbuffer like) + let (insert_tx, insert_rx) = mpsc::unbounded_channel(); + // let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); + // setup replica insert notifications. + replica.on_insert(Box::new(move |origin, entry| { + // only care for local inserts, otherwise we'd do endless gossip loops + if let InsertOrigin::Local = origin { + insert_tx.send(entry.clone()).ok(); + } + })); + + // setup a gossip subscripion + let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id.clone()).collect(); + let topic: TopicId = replica.namespace().as_bytes().into(); + let gossip_subscription = GossipStream::new(gossip.clone(), topic, peer_ids); + + Self { + gossip, + replica, + endpoint, + gossip_stream: gossip_subscription, + insert_entry_rx: insert_rx, + to_actor_rx, + sync_state: Default::default(), + running_sync_tasks: Default::default(), + initial_peers, + } + } + pub async fn run(&mut self) { + if let Err(err) = self.run_inner().await { + error!("live sync failed: {err:?}"); + } + } + + async fn run_inner(&mut self) -> anyhow::Result<()> { + // add addresses of initial peers to our endpoint address book + for peer in &self.initial_peers { + self.endpoint + .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) + .await?; + } + // trigger initial sync with initial peers + for peer in self.initial_peers.clone().iter().map(|p| p.peer_id) { + self.sync_with_peer(peer); + } + loop { + tokio::select! { + biased; + msg = self.to_actor_rx.recv() => { + match msg { + // received shutdown signal, or livesync handle was dropped: break loop and + // exit + Some(ToActor::Shutdown) | None => break, + } + } + // new gossip message + event = self.gossip_stream.next() => { + if let Err(err) = self.on_gossip_event(event?) { + error!("Failed to process gossip event: {err:?}"); + } + }, + entry = self.insert_entry_rx.recv() => { + let entry = entry.ok_or_else(|| anyhow!("insert_rx returned None"))?; + self.on_insert_entry(entry).await?; + } + Some(res) = self.running_sync_tasks.next() => { + let (peer, res) = res.context("task sync_with_peer paniced")?; + self.on_sync_finished(peer, res); + + } + } + } + Ok(()) + } + + fn sync_with_peer(&mut self, peer: PeerId) { + // Check if we synced and only start sync if not yet synced + // sync_with_peer is triggered on NeighborUp events, so might trigger repeatedly for the + // same peers. + // TODO: Track finished time and potentially re-run sync + if let Some(_state) = self.sync_state.get(&peer) { + return; + }; + self.sync_state.insert(peer, SyncState::Running); + let task = { + let endpoint = self.endpoint.clone(); + let replica = self.replica.clone(); + tokio::spawn(async move { + println!("> connect and sync with {peer}"); + // TODO: Make sure that the peer is dialable. + let res = connect_and_sync(&endpoint, &replica, peer, None, &[]).await; + println!("> sync with {peer} done: {res:?}"); + (peer, res) + }) + }; + self.running_sync_tasks.push(task); + } + + fn on_sync_finished(&mut self, peer: PeerId, res: anyhow::Result<()>) { + let state = match res { + Ok(_) => SyncState::Finished, + Err(err) => SyncState::Failed(err), + }; + self.sync_state.insert(peer, state); + } + + fn on_gossip_event(&mut self, event: Event) -> anyhow::Result<()> { + match event { + // We received a gossip message. Try to insert it into our replica. + Event::Received(data) => { + let op: Op = postcard::from_bytes(&data)?; + match op { + Op::Put(entry) => { + self.replica.insert_remote_entry(entry)?; + } + } + } + // A new neighbor appeared in the gossip swarm. Try to sync with it directly. + // [Self::sync_with_peer] will check to not resync with peers synced previously in the + // same session. TODO: Maybe this is too broad and leads to too many sync requests. + Event::NeighborUp(peer) => { + self.sync_with_peer(peer); + } + _ => {} + } + Ok(()) + } + + /// A new entry was inserted locally. Broadcast a gossip message. + async fn on_insert_entry(&mut self, entry: SignedEntry) -> anyhow::Result<()> { + let op = Op::Put(entry); + let topic: TopicId = self.replica.namespace().as_bytes().into(); + self.gossip + .broadcast(topic, postcard::to_stdvec(&op)?.into()) + .await?; + Ok(()) + } +} + +// TODO: If this is the API surface we want move to iroh-gossip/src/net and make this be +// GossipHandle::subscribe +#[derive(Debug)] +pub enum GossipStream { + Joining(GossipHandle, TopicId, Vec), + Running(broadcast::Receiver), +} + +impl GossipStream { + pub fn new(gossip: GossipHandle, topic: TopicId, peers: Vec) -> Self { + Self::Joining(gossip, topic, peers) + } + pub async fn next(&mut self) -> anyhow::Result { + loop { + match self { + Self::Joining(gossip, topic, peers) => { + // TODO: avoid the clone + gossip.join(*topic, peers.clone()).await?; + let sub = gossip.subscribe(*topic).await?; + *self = Self::Running(sub); + } + Self::Running(sub) => { + let ret = sub.recv().await.map_err(|e| e.into()); + return ret; + } + } + } + } +} From 230a362ffcca5bda0ffad09d73f14ca05c3ddc79 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 7 Jul 2023 12:11:54 +0200 Subject: [PATCH 004/172] feat: make the live sync handler work with many docs --- iroh/examples/sync.rs | 133 ++++++++++++++--------- iroh/src/sync/live.rs | 247 +++++++++++++++++++++--------------------- 2 files changed, 203 insertions(+), 177 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 0b8acf5774..b603940761 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -9,7 +9,7 @@ use std::{fmt, str::FromStr}; -use anyhow::bail; +use anyhow::{anyhow, bail}; use clap::Parser; use ed25519_dalek::SigningKey; use iroh::sync::{LiveSync, PeerSource, SYNC_ALPN}; @@ -81,32 +81,45 @@ async fn run(args: Args) -> anyhow::Result<()> { }; println!("> using DERP servers: {}", fmt_derp_map(&derp_map)); - // init a cell that will hold our gossip handle to be used in endpoint callbacks - let gossip_cell: OnceCell = OnceCell::new(); - // init a channel that will emit once the initial endpoints of our local node are discovered - let (initial_endpoints_tx, mut initial_endpoints_rx) = mpsc::channel(1); - // build our magic endpoint - let gossip_cell_clone = gossip_cell.clone(); - let endpoint = MagicEndpoint::builder() - .keypair(keypair.clone()) - .alpns(vec![GOSSIP_ALPN.to_vec(), SYNC_ALPN.to_vec()]) - .derp_map(derp_map) - .on_endpoints(Box::new(move |endpoints| { - // send our updated endpoints to the gossip protocol to be sent as PeerData to peers - if let Some(gossip) = gossip_cell_clone.get() { - gossip.update_endpoints(endpoints).ok(); - } - // trigger oneshot on the first endpoint update - initial_endpoints_tx.try_send(endpoints.to_vec()).ok(); - })) - .bind(args.bind_port) - .await?; - println!("> our peer id: {}", endpoint.peer_id()); + let (endpoint, gossip, initial_endpoints) = { + // init a cell that will hold our gossip handle to be used in endpoint callbacks + let gossip_cell: OnceCell = OnceCell::new(); + // init a channel that will emit once the initial endpoints of our local node are discovered + let (initial_endpoints_tx, mut initial_endpoints_rx) = mpsc::channel(1); - // wait for a first endpoint update so that we know about at least one of our addrs - let initial_endpoints = initial_endpoints_rx.recv().await.unwrap(); - // println!("> our endpoints: {initial_endpoints:?}"); + let endpoint = MagicEndpoint::builder() + .keypair(keypair.clone()) + .alpns(vec![GOSSIP_ALPN.to_vec(), SYNC_ALPN.to_vec()]) + .derp_map(derp_map) + .on_endpoints({ + let gossip_cell = gossip_cell.clone(); + Box::new(move |endpoints| { + // send our updated endpoints to the gossip protocol to be sent as PeerData to peers + if let Some(gossip) = gossip_cell.get() { + gossip.update_endpoints(endpoints).ok(); + } + // trigger oneshot on the first endpoint update + initial_endpoints_tx.try_send(endpoints.to_vec()).ok(); + }) + }) + .bind(args.bind_port) + .await?; + + // create the gossip protocol + let gossip = { + let gossip = GossipHandle::from_endpoint(endpoint.clone(), Default::default()); + // insert the gossip handle into the gossip cell to be used in the endpoint callbacks above + gossip_cell.set(gossip.clone()).unwrap(); + gossip + }; + // wait for a first endpoint update so that we know about at least one of our addrs + let initial_endpoints = initial_endpoints_rx.recv().await.unwrap(); + // pass our initial endpoints to the gossip protocol + gossip.update_endpoints(&initial_endpoints)?; + (endpoint, gossip, initial_endpoints) + }; + println!("> our peer id: {}", endpoint.peer_id()); let (topic, peers) = match &args.command { Command::Open { doc_name } => { @@ -124,6 +137,7 @@ async fn run(args: Args) -> anyhow::Result<()> { } }; + // println!("> our endpoints: {initial_endpoints:?}"); let our_ticket = { // add our local endpoints to the ticket and print it for others to join let addrs = initial_endpoints.iter().map(|ep| ep.addr).collect(); @@ -137,16 +151,6 @@ async fn run(args: Args) -> anyhow::Result<()> { }; println!("> ticket to join us: {our_ticket}"); - // create the gossip protocol - let gossip = { - let gossip = GossipHandle::from_endpoint(endpoint.clone(), Default::default()); - // insert the gossip handle into the gossip cell to be used in the endpoint callbacks above - gossip_cell.set(gossip.clone()).unwrap(); - // pass our initial peer println to the gossip protocol - gossip.update_endpoints(&initial_endpoints)?; - gossip - }; - // create the sync doc and store let (store, author, doc) = create_document(topic, &keypair)?; @@ -163,7 +167,8 @@ async fn run(args: Args) -> anyhow::Result<()> { std::thread::spawn(move || input_loop(line_tx)); // create the live syncer - let mut sync_handle = LiveSync::spawn(endpoint.clone(), gossip.clone(), doc.clone(), peers); + let sync_handle = LiveSync::spawn(endpoint.clone(), gossip.clone()); + sync_handle.sync_doc(doc.clone(), peers.clone()).await?; // do some logging doc.on_insert(Box::new(move |origin, entry| { @@ -173,15 +178,18 @@ async fn run(args: Args) -> anyhow::Result<()> { // process stdin lines println!("> read to accept commands: set | get | ls | exit"); while let Some(text) = line_rx.recv().await { - let mut parts = text.split(' '); - match [parts.next(), parts.next(), parts.next()] { - [Some("set"), Some(key), Some(value)] => { - let key = key.to_string(); - let value = value.to_string(); + let cmd = match Cmd::from_str(&text) { + Ok(cmd) => cmd, + Err(err) => { + println!("> failed to parse command: {}", err); + continue; + } + }; + match cmd { + Cmd::Set { key, value } => { doc.insert(&key, &author, value); } - [Some("get"), Some(key), None] => { - // TODO: we need a way to get all filtered by key from all authors + Cmd::Get { key } => { let mut entries = doc .all() .into_iter() @@ -190,25 +198,48 @@ async fn run(args: Args) -> anyhow::Result<()> { println!("{} -> {}", fmt_entry(&entry), fmt_content(&doc, &entry)); } } - [Some("ls"), None, None] => { + Cmd::Ls => { let all = doc.all(); println!("> {} entries", all.len()); for (_id, entry) in all { println!("{} -> {}", fmt_entry(&entry), fmt_content(&doc, &entry)); } } - [Some("exit"), None, None] => { + Cmd::Exit => { let res = sync_handle.cancel().await?; println!("syncer closed with {res:?}"); break; } - _ => println!("> invalid command"), } } Ok(()) } +pub enum Cmd { + Set { key: String, value: String }, + Get { key: String }, + Ls, + Exit, +} +impl FromStr for Cmd { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let mut parts = s.split(' '); + match [parts.next(), parts.next(), parts.next()] { + [Some("set"), Some(key), Some(value)] => Ok(Self::Set { + key: key.into(), + value: value.into(), + }), + [Some("get"), Some(key), None] => Ok(Self::Get { key: key.into() }), + [Some("ls"), None, None] => Ok(Self::Ls), + [Some("exit"), None, None] => Ok(Self::Exit), + _ => Err(anyhow!("invalid command")), + } + } +} + fn create_document( topic: TopicId, keypair: &Keypair, @@ -237,7 +268,7 @@ async fn endpoint_loop( )), }; if let Err(err) = res { - tracing::error!("connection for {alpn} errored: {err:?}"); + println!("> connection for {alpn} closed, reason: {err}"); } } Ok(()) @@ -295,16 +326,16 @@ impl FromStr for Ticket { fn fmt_entry(entry: &SignedEntry) -> String { let id = entry.entry().id(); let key = std::str::from_utf8(id.key()).unwrap_or(""); - let hash = entry.entry().record().content_hash(); let author = fmt_hash(id.author().as_bytes()); - let fmt_hash = fmt_hash(hash.as_bytes()); - format!("@{author}: {key} = {fmt_hash}") + let hash = entry.entry().record().content_hash(); + let hash = fmt_hash(hash.as_bytes()); + format!("@{author}: {key} = {hash}") } fn fmt_content(doc: &Replica, entry: &SignedEntry) -> String { let hash = entry.entry().record().content_hash(); let content = doc.get_content(hash); let content = content - .map(|content| String::from_utf8(content.into()).unwrap()) + .map(|content| String::from_utf8(content.into()).unwrap_or_else(|_| "".into())) .unwrap_or_else(|| "".into()); content } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 1697a87819..40f2d68b56 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -1,10 +1,10 @@ use std::{collections::HashMap, net::SocketAddr, sync::Arc}; use crate::sync::connect_and_sync; -use anyhow::{anyhow, Context}; +use anyhow::{anyhow, Result}; use futures::{ future::{BoxFuture, Shared}, - stream::FuturesUnordered, + stream::{BoxStream, FuturesUnordered, StreamExt}, FutureExt, TryFutureExt, }; use iroh_gossip::{ @@ -14,11 +14,7 @@ use iroh_gossip::{ use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::sync::{InsertOrigin, Replica, SignedEntry}; use serde::{Deserialize, Serialize}; -use tokio::{ - sync::{broadcast, mpsc}, - task::JoinError, -}; -use tokio_stream::StreamExt; +use tokio::{sync::mpsc, task::JoinError}; use tracing::error; const CHANNEL_CAP: usize = 8; @@ -47,6 +43,10 @@ enum SyncState { #[derive(Debug)] pub enum ToActor { + SyncDoc { + doc: Replica, + initial_peers: Vec, + }, Shutdown, } @@ -58,15 +58,14 @@ pub struct LiveSync { } impl LiveSync { - pub fn spawn( - endpoint: MagicEndpoint, - gossip: GossipHandle, - doc: Replica, - initial_peers: Vec, - ) -> Self { + pub fn spawn(endpoint: MagicEndpoint, gossip: GossipHandle) -> Self { let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); - let mut actor = Actor::new(endpoint, gossip, doc, initial_peers, to_actor_rx); - let task = tokio::spawn(async move { actor.run().await }); + let mut actor = Actor::new(endpoint, gossip, to_actor_rx); + let task = tokio::spawn(async move { + if let Err(err) = actor.run().await { + error!("live sync failed: {err:?}"); + } + }); let handle = LiveSync { to_actor_tx, task: task.map_err(Arc::new).boxed().shared(), @@ -75,208 +74,204 @@ impl LiveSync { } /// Cancel the live sync. - pub async fn cancel(&mut self) -> anyhow::Result<()> { + pub async fn cancel(&self) -> Result<()> { self.to_actor_tx.send(ToActor::Shutdown).await?; self.task.clone().await?; Ok(()) } + + pub async fn sync_doc(&self, doc: Replica, initial_peers: Vec) -> Result<()> { + self.to_actor_tx + .send(ToActor::SyncDoc { doc, initial_peers }) + .await?; + Ok(()) + } } -// TODO: Right now works with a single doc. Can quite easily be extended to work on a set of -// replicas. Then the handle above could have a -// `join_doc(doc: Replica, initial_peers: Vec, - gossip_stream: GossipStream, - to_actor_rx: mpsc::Receiver, - insert_entry_rx: mpsc::UnboundedReceiver, - sync_state: HashMap, gossip: GossipHandle, - running_sync_tasks: FuturesUnordered)>>, + + docs: HashMap, + subscription: BoxStream<'static, Result<(TopicId, Event)>>, + sync_state: HashMap<(TopicId, PeerId), SyncState>, + + to_actor_rx: mpsc::Receiver, + insert_entry_tx: mpsc::UnboundedSender<(TopicId, SignedEntry)>, + insert_entry_rx: mpsc::UnboundedReceiver<(TopicId, SignedEntry)>, + + pending_syncs: FuturesUnordered)>>, + pending_joins: FuturesUnordered)>>, } impl Actor { pub fn new( endpoint: MagicEndpoint, gossip: GossipHandle, - replica: Replica, - initial_peers: Vec, to_actor_rx: mpsc::Receiver, ) -> Self { // TODO: instead of an unbounded channel, we'd want a FIFO ring buffer likely // (we have to send from the blocking Replica::on_insert callback, so we need a channel // with nonblocking sending, so either unbounded or ringbuffer like) let (insert_tx, insert_rx) = mpsc::unbounded_channel(); - // let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); - // setup replica insert notifications. - replica.on_insert(Box::new(move |origin, entry| { - // only care for local inserts, otherwise we'd do endless gossip loops - if let InsertOrigin::Local = origin { - insert_tx.send(entry.clone()).ok(); - } - })); - - // setup a gossip subscripion - let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id.clone()).collect(); - let topic: TopicId = replica.namespace().as_bytes().into(); - let gossip_subscription = GossipStream::new(gossip.clone(), topic, peer_ids); + let sub = gossip.clone().subscribe_all().boxed(); Self { gossip, - replica, + // replica, endpoint, - gossip_stream: gossip_subscription, + // gossip_stream: gossip_subscription, insert_entry_rx: insert_rx, + insert_entry_tx: insert_tx, to_actor_rx, sync_state: Default::default(), - running_sync_tasks: Default::default(), - initial_peers, - } - } - pub async fn run(&mut self) { - if let Err(err) = self.run_inner().await { - error!("live sync failed: {err:?}"); + pending_syncs: Default::default(), + // initial_peers, + pending_joins: Default::default(), + docs: Default::default(), + subscription: sub, } } - async fn run_inner(&mut self) -> anyhow::Result<()> { - // add addresses of initial peers to our endpoint address book - for peer in &self.initial_peers { - self.endpoint - .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) - .await?; - } - // trigger initial sync with initial peers - for peer in self.initial_peers.clone().iter().map(|p| p.peer_id) { - self.sync_with_peer(peer); - } + async fn run(&mut self) -> Result<()> { loop { tokio::select! { biased; msg = self.to_actor_rx.recv() => { match msg { - // received shutdown signal, or livesync handle was dropped: break loop and - // exit + // received shutdown signal, or livesync handle was dropped: + // break loop and exit Some(ToActor::Shutdown) | None => break, + Some(ToActor::SyncDoc { doc, initial_peers }) => self.insert_doc(doc, initial_peers).await?, } } // new gossip message - event = self.gossip_stream.next() => { - if let Err(err) = self.on_gossip_event(event?) { + Some(event) = self.subscription.next() => { + let (topic, event) = event?; + if let Err(err) = self.on_gossip_event(topic, event) { error!("Failed to process gossip event: {err:?}"); } }, entry = self.insert_entry_rx.recv() => { - let entry = entry.ok_or_else(|| anyhow!("insert_rx returned None"))?; - self.on_insert_entry(entry).await?; + let (topic, entry) = entry.ok_or_else(|| anyhow!("insert_rx returned None"))?; + self.on_insert_entry(topic, entry).await?; } - Some(res) = self.running_sync_tasks.next() => { - let (peer, res) = res.context("task sync_with_peer paniced")?; - self.on_sync_finished(peer, res); + Some((topic, peer, res)) = self.pending_syncs.next() => { + // let (topic, peer, res) = res.context("task sync_with_peer paniced")?; + self.on_sync_finished(topic, peer, res); } + Some((topic, res)) = self.pending_joins.next() => { + if let Err(err) = res { + error!("failed to join {topic:?}: {err:?}"); + } + // TODO: maintain some join state + } } } Ok(()) } - fn sync_with_peer(&mut self, peer: PeerId) { + fn sync_with_peer(&mut self, topic: TopicId, peer: PeerId) { + let Some(doc) = self.docs.get(&topic) else { + return; + }; // Check if we synced and only start sync if not yet synced // sync_with_peer is triggered on NeighborUp events, so might trigger repeatedly for the // same peers. // TODO: Track finished time and potentially re-run sync - if let Some(_state) = self.sync_state.get(&peer) { + if let Some(_state) = self.sync_state.get(&(topic, peer)) { return; }; - self.sync_state.insert(peer, SyncState::Running); + // TODO: fixme (doc_id, peer) + self.sync_state.insert((topic, peer), SyncState::Running); let task = { let endpoint = self.endpoint.clone(); - let replica = self.replica.clone(); - tokio::spawn(async move { + let doc = doc.clone(); + async move { println!("> connect and sync with {peer}"); // TODO: Make sure that the peer is dialable. - let res = connect_and_sync(&endpoint, &replica, peer, None, &[]).await; + let res = connect_and_sync(&endpoint, &doc, peer, None, &[]).await; println!("> sync with {peer} done: {res:?}"); - (peer, res) - }) + (topic, peer, res) + } + .boxed() }; - self.running_sync_tasks.push(task); + self.pending_syncs.push(task); } - fn on_sync_finished(&mut self, peer: PeerId, res: anyhow::Result<()>) { + async fn insert_doc(&mut self, doc: Replica, initial_peers: Vec) -> Result<()> { + let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id.clone()).collect(); + let topic: TopicId = doc.namespace().as_bytes().into(); + // join gossip for the topic to receive and send message + // let gossip = self.gossip.clone(); + self.pending_joins.push({ + let peer_ids = peer_ids.clone(); + let gossip = self.gossip.clone(); + async move { + let res = gossip.join(topic, peer_ids).await; + (topic, res) + } + .boxed() + }); + // setup replica insert notifications. + let insert_entry_tx = self.insert_entry_tx.clone(); + doc.on_insert(Box::new(move |origin, entry| { + // only care for local inserts, otherwise we'd do endless gossip loops + if let InsertOrigin::Local = origin { + insert_entry_tx.send((topic, entry.clone())).ok(); + } + })); + self.docs.insert(topic, doc); + // add addresses of initial peers to our endpoint address book + for peer in &initial_peers { + self.endpoint + .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) + .await?; + } + // trigger initial sync with initial peers + for peer in peer_ids { + self.sync_with_peer(topic, peer); + } + Ok(()) + } + + fn on_sync_finished(&mut self, topic: TopicId, peer: PeerId, res: Result<()>) { let state = match res { Ok(_) => SyncState::Finished, Err(err) => SyncState::Failed(err), }; - self.sync_state.insert(peer, state); + self.sync_state.insert((topic, peer), state); } - fn on_gossip_event(&mut self, event: Event) -> anyhow::Result<()> { + fn on_gossip_event(&mut self, topic: TopicId, event: Event) -> Result<()> { + let Some(doc) = self.docs.get(&topic) else { + return Err(anyhow!("Missing doc for {topic:?}")); + }; match event { // We received a gossip message. Try to insert it into our replica. Event::Received(data) => { let op: Op = postcard::from_bytes(&data)?; match op { - Op::Put(entry) => { - self.replica.insert_remote_entry(entry)?; - } + Op::Put(entry) => doc.insert_remote_entry(entry)?, } } // A new neighbor appeared in the gossip swarm. Try to sync with it directly. // [Self::sync_with_peer] will check to not resync with peers synced previously in the // same session. TODO: Maybe this is too broad and leads to too many sync requests. - Event::NeighborUp(peer) => { - self.sync_with_peer(peer); - } + Event::NeighborUp(peer) => self.sync_with_peer(topic, peer), _ => {} } Ok(()) } /// A new entry was inserted locally. Broadcast a gossip message. - async fn on_insert_entry(&mut self, entry: SignedEntry) -> anyhow::Result<()> { + async fn on_insert_entry(&mut self, topic: TopicId, entry: SignedEntry) -> Result<()> { let op = Op::Put(entry); - let topic: TopicId = self.replica.namespace().as_bytes().into(); - self.gossip - .broadcast(topic, postcard::to_stdvec(&op)?.into()) - .await?; + let message = postcard::to_stdvec(&op)?.into(); + self.gossip.broadcast(topic, message).await?; Ok(()) } } - -// TODO: If this is the API surface we want move to iroh-gossip/src/net and make this be -// GossipHandle::subscribe -#[derive(Debug)] -pub enum GossipStream { - Joining(GossipHandle, TopicId, Vec), - Running(broadcast::Receiver), -} - -impl GossipStream { - pub fn new(gossip: GossipHandle, topic: TopicId, peers: Vec) -> Self { - Self::Joining(gossip, topic, peers) - } - pub async fn next(&mut self) -> anyhow::Result { - loop { - match self { - Self::Joining(gossip, topic, peers) => { - // TODO: avoid the clone - gossip.join(*topic, peers.clone()).await?; - let sub = gossip.subscribe(*topic).await?; - *self = Self::Running(sub); - } - Self::Running(sub) => { - let ret = sub.recv().await.map_err(|e| e.into()); - return ret; - } - } - } - } -} From 350a4867f4cfc746a828d33e0d0d55f326a99bfb Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 7 Jul 2023 12:25:14 +0200 Subject: [PATCH 005/172] chore: cleanup and clippy --- iroh-sync/src/ranger.rs | 14 +++++++------- iroh-sync/src/sync.rs | 6 +++--- iroh/src/sync.rs | 2 +- iroh/src/sync/live.rs | 13 +++++-------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index 159d99d8cf..7317d2a321 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -160,7 +160,7 @@ impl MessagePart { pub fn values(&self) -> Option<&[(K, V)]> { match self { MessagePart::RangeFingerprint(_) => None, - MessagePart::RangeItem(RangeItem { values, .. }) => Some(&values), + MessagePart::RangeItem(RangeItem { values, .. }) => Some(values), } } } @@ -180,7 +180,7 @@ where { /// Construct the initial message. fn init>(store: &S, limit: Option<&Range>) -> Self { - let x = store.get_first().clone(); + let x = store.get_first(); let range = Range::new(x.clone(), x); let fingerprint = store.get_fingerprint(&range, limit); let part = MessagePart::RangeFingerprint(RangeFingerprint { range, fingerprint }); @@ -332,7 +332,7 @@ where }; loop { - if filter(&next.0) { + if filter(next.0) { return Some(next); } @@ -432,7 +432,7 @@ where self.store .get_range(range.clone(), self.limit.clone()) .into_iter() - .filter(|(k, _)| values.iter().find(|(vk, _)| &vk == k).is_none()) + .filter(|(k, _)| !values.iter().any(|(vk, _)| &vk == k)) .map(|(k, v)| (k.clone(), v.clone())) .collect(), ) @@ -816,7 +816,7 @@ mod tests { hex::encode(&self.key) }; f.debug_struct("Multikey") - .field("author", &hex::encode(&self.author)) + .field("author", &hex::encode(self.author)) .field("key", &key) .finish() } @@ -1236,8 +1236,8 @@ mod tests { #[test] fn test_div_ceil() { - assert_eq!(div_ceil(1, 1), 1 / 1); - assert_eq!(div_ceil(2, 1), 2 / 1); + assert_eq!(div_ceil(1, 1), 1); + assert_eq!(div_ceil(2, 1), 2); assert_eq!(div_ceil(4, 2), 4 / 2); assert_eq!(div_ceil(3, 2), 2); diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 34d7d9bd50..adf75507e9 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -46,7 +46,7 @@ impl Author { } pub fn from_bytes(bytes: &[u8; 32]) -> Self { - SigningKey::from_bytes(&bytes).into() + SigningKey::from_bytes(bytes).into() } pub fn id(&self) -> &AuthorId { @@ -358,7 +358,7 @@ impl<'a> Iterator for RangeIterator<'a> { fn next(&mut self) -> Option { let mut next = self.iter.next()?; loop { - if self.matches(&next.0) { + if self.matches(next.0) { let (k, values) = next; let (_, v) = values.last_key_value()?; return Some((k, v)); @@ -441,7 +441,7 @@ impl Replica { let inner = self.inner.read(); inner .peer - .get(&RecordIdentifier::new(key, &inner.namespace.id(), author)) + .get(&RecordIdentifier::new(key, inner.namespace.id(), author)) .cloned() } diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index 68de0fe252..235e190915 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -46,7 +46,7 @@ pub async fn connect_and_sync( .await .context("dial_and_sync")?; let (mut send_stream, mut recv_stream) = connection.open_bi().await?; - let res = run_alice(&mut send_stream, &mut recv_stream, &doc).await; + let res = run_alice(&mut send_stream, &mut recv_stream, doc).await; debug!("sync with peer {}: finish {:?}", peer_id, res); res } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 40f2d68b56..e11948d3aa 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -15,7 +15,7 @@ use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::sync::{InsertOrigin, Replica, SignedEntry}; use serde::{Deserialize, Serialize}; use tokio::{sync::mpsc, task::JoinError}; -use tracing::error; +use tracing::{debug, error}; const CHANNEL_CAP: usize = 8; @@ -120,15 +120,12 @@ impl Actor { Self { gossip, - // replica, endpoint, - // gossip_stream: gossip_subscription, insert_entry_rx: insert_rx, insert_entry_tx: insert_tx, to_actor_rx, sync_state: Default::default(), pending_syncs: Default::default(), - // initial_peers, pending_joins: Default::default(), docs: Default::default(), subscription: sub, @@ -191,10 +188,10 @@ impl Actor { let endpoint = self.endpoint.clone(); let doc = doc.clone(); async move { - println!("> connect and sync with {peer}"); + debug!("sync with {peer}"); // TODO: Make sure that the peer is dialable. let res = connect_and_sync(&endpoint, &doc, peer, None, &[]).await; - println!("> sync with {peer} done: {res:?}"); + debug!("> synced with {peer}: {res:?}"); (topic, peer, res) } .boxed() @@ -203,7 +200,7 @@ impl Actor { } async fn insert_doc(&mut self, doc: Replica, initial_peers: Vec) -> Result<()> { - let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id.clone()).collect(); + let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id).collect(); let topic: TopicId = doc.namespace().as_bytes().into(); // join gossip for the topic to receive and send message // let gossip = self.gossip.clone(); @@ -221,7 +218,7 @@ impl Actor { doc.on_insert(Box::new(move |origin, entry| { // only care for local inserts, otherwise we'd do endless gossip loops if let InsertOrigin::Local = origin { - insert_entry_tx.send((topic, entry.clone())).ok(); + insert_entry_tx.send((topic, entry)).ok(); } })); self.docs.insert(topic, doc); From e8e9158655b021863341a2dc243b2d9aeb8d44f1 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 12 Jul 2023 16:14:48 +0200 Subject: [PATCH 006/172] chore: remove old code and add docs --- iroh/src/sync.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index 235e190915..72b5c02ade 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -33,6 +33,7 @@ enum Message { Sync(iroh_sync::sync::ProtocolMessage), } +/// Connect to a peer and sync a replica pub async fn connect_and_sync( endpoint: &MagicEndpoint, doc: &Replica, From 9a9266ef587f6bf8e315c28be0a9e4e26b46777b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 12 Jul 2023 14:03:45 +0200 Subject: [PATCH 007/172] feat: WIP integration of sync and bytes * removes content support from iroh-sync * adds a quick-and-dirty writable database to iroh-bytes (will be replaced with a better generic writable database soon) * adds a `Downloader` to queue get requests for individual hashes from individual peers * adds a `BlobStore` that combines the writable db with the downloader * adds a `Doc` abstraction that combines an iroh-sync `Replica` with a `BlobStore` to download content from peers on-demand * updates the sync repl example to plug it all together * also adds very basic persistence to `Replica` (encode to byte string) and uses this in the repl example --- Cargo.lock | 1 + iroh-bytes/src/lib.rs | 1 + iroh-bytes/src/writable.rs | 178 ++++++++++++++++++++ iroh-net/src/tls.rs | 12 ++ iroh-sync/Cargo.toml | 1 + iroh-sync/src/ranger.rs | 9 +- iroh-sync/src/sync.rs | 80 ++++++--- iroh/examples/sync.rs | 289 +++++++++++++++++++++++++-------- iroh/src/sync.rs | 2 + iroh/src/sync/content.rs | 324 +++++++++++++++++++++++++++++++++++++ 10 files changed, 805 insertions(+), 92 deletions(-) create mode 100644 iroh-bytes/src/writable.rs create mode 100644 iroh/src/sync/content.rs diff --git a/Cargo.lock b/Cargo.lock index bb565f78b0..89bec29a07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1895,6 +1895,7 @@ dependencies = [ "iroh-bytes", "once_cell", "parking_lot", + "postcard", "rand", "rand_core", "serde", diff --git a/iroh-bytes/src/lib.rs b/iroh-bytes/src/lib.rs index ea257d1d11..369c3dee7c 100644 --- a/iroh-bytes/src/lib.rs +++ b/iroh-bytes/src/lib.rs @@ -9,6 +9,7 @@ pub mod get; pub mod protocol; pub mod provider; pub mod util; +pub mod writable; #[cfg(test)] pub(crate) mod test_utils; diff --git a/iroh-bytes/src/writable.rs b/iroh-bytes/src/writable.rs new file mode 100644 index 0000000000..fe492e47b4 --- /dev/null +++ b/iroh-bytes/src/writable.rs @@ -0,0 +1,178 @@ +#![allow(missing_docs)] +//! Quick-and-dirty writable database +//! +//! I wrote this while diving into iroh-bytes, wildly copying code around. This will be solved much +//! nicer with the upcoming generic writable database branch by @rklaehn. + +use std::{collections::HashMap, io, path::PathBuf, sync::Arc}; + +use anyhow::Context; +use bytes::Bytes; +use iroh_io::{AsyncSliceWriter, File}; +use range_collections::RangeSet2; + +use crate::{ + get::fsm, + protocol::{GetRequest, RangeSpecSeq, Request}, + provider::{create_collection, DataSource, Database, DbEntry, FNAME_PATHS}, + Hash, +}; + +/// A blob database into which new blobs can be inserted. +/// +/// Blobs can be inserted either from bytes or by downloading from open connections to peers. +/// New blobs will be saved as files with a filename based on their hash. +/// +/// TODO: Replace with the generic writable database. +#[derive(Debug, Clone)] +pub struct WritableFileDatabase { + db: Database, + storage: Arc, +} + +impl WritableFileDatabase { + pub async fn new(data_path: PathBuf) -> anyhow::Result { + let storage = Arc::new(StoragePaths::new(data_path).await?); + let db = if storage.db_path.join(FNAME_PATHS).exists() { + Database::load(&storage.db_path).await.with_context(|| { + format!( + "Failed to load iroh database from {}", + storage.db_path.display() + ) + })? + } else { + Database::default() + }; + Ok(Self { db, storage }) + } + + pub fn db(&self) -> &Database { + &self.db + } + + pub async fn save(&self) -> io::Result<()> { + self.db.save(&self.storage.db_path).await + } + + pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64)> { + let (hash, size, entry) = self.storage.put_bytes(data).await?; + self.db.union_with(HashMap::from_iter([(hash, entry)])); + Ok((hash, size)) + } + + pub async fn put_from_temp_file(&self, temp_path: &PathBuf) -> anyhow::Result<(Hash, u64)> { + let (hash, size, entry) = self.storage.move_to_blobs(&temp_path).await?; + self.db.union_with(HashMap::from_iter([(hash, entry)])); + Ok((hash, size)) + } + + pub async fn get_size(&self, hash: &Hash) -> Option { + Some(self.db.get(&hash)?.size().await) + } + + pub fn has(&self, hash: &Hash) -> bool { + self.db.to_inner().contains_key(hash) + } + pub async fn download_single( + &self, + conn: quinn::Connection, + hash: Hash, + ) -> anyhow::Result> { + // 1. Download to temp file + let temp_path = { + let temp_path = self.storage.temp_path(); + let request = + Request::Get(GetRequest::new(hash, RangeSpecSeq::new([RangeSet2::all()]))); + let response = fsm::start(conn, request); + let connected = response.next().await?; + + let fsm::ConnectedNext::StartRoot(curr) = connected.next().await? else { + return Ok(None) + }; + let header = curr.next(); + + let path = temp_path.clone(); + let mut data_file = File::create(move || { + std::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&path) + }) + .await?; + + let (curr, _size) = header.next().await?; + let _curr = curr.write_all(&mut data_file).await?; + // Flush the data file first, it is the only thing that matters at this point + data_file.sync().await?; + temp_path + }; + + // 2. Insert into database + let (hash, size, entry) = self.storage.move_to_blobs(&temp_path).await?; + let entries = HashMap::from_iter([(hash, entry)]); + self.db.union_with(entries); + Ok(Some((hash, size))) + } +} + +#[derive(Debug)] +pub struct StoragePaths { + blob_path: PathBuf, + temp_path: PathBuf, + db_path: PathBuf, +} + +impl StoragePaths { + pub async fn new(data_path: PathBuf) -> anyhow::Result { + let blob_path = data_path.join("blobs"); + let temp_path = data_path.join("temp"); + let db_path = data_path.join("db"); + tokio::fs::create_dir_all(&blob_path).await?; + tokio::fs::create_dir_all(&temp_path).await?; + tokio::fs::create_dir_all(&db_path).await?; + Ok(Self { + blob_path, + temp_path, + db_path, + }) + } + + pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64, DbEntry)> { + let temp_path = self.temp_path(); + tokio::fs::write(&temp_path, &data).await?; + let (hash, size, entry) = self.move_to_blobs(&temp_path).await?; + Ok((hash, size, entry)) + } + + async fn move_to_blobs(&self, path: &PathBuf) -> anyhow::Result<(Hash, u64, DbEntry)> { + let datasource = DataSource::new(path.clone()); + // TODO: this needlessly creates a collection, but that's what's pub atm in iroh-bytes + let (db, _collection_hash) = create_collection(vec![datasource]).await?; + // the actual blob is the first entry in the external entries in the created collection + let (hash, _path, _len) = db.external().next().unwrap(); + let Some(DbEntry::External { outboard, size, .. }) = db.get(&hash) else { + unreachable!("just inserted"); + }; + + let final_path = prepare_hash_dir(&self.blob_path, &hash).await?; + tokio::fs::rename(&path, &final_path).await?; + let entry = DbEntry::External { + outboard, + path: final_path, + size, + }; + Ok((hash, size, entry)) + } + + fn temp_path(&self) -> PathBuf { + let name = hex::encode(rand::random::().to_be_bytes()); + self.temp_path.join(name) + } +} + +async fn prepare_hash_dir(path: &PathBuf, hash: &Hash) -> anyhow::Result { + let hash = hex::encode(hash.as_ref()); + let path = path.join(&hash[0..2]).join(&hash[2..4]).join(&hash[4..]); + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + Ok(path) +} diff --git a/iroh-net/src/tls.rs b/iroh-net/src/tls.rs index 4f92c9106a..07dbeda035 100644 --- a/iroh-net/src/tls.rs +++ b/iroh-net/src/tls.rs @@ -106,6 +106,18 @@ impl PeerId { pub fn as_bytes(&self) -> &[u8; 32] { self.0.as_bytes() } + + /// Try to create a peer id from a byte array. + /// + /// # Warning + /// + /// The caller is responsible for ensuring that the bytes passed into this + /// method actually represent a `curve25519_dalek::curve::CompressedEdwardsY` + /// and that said compressed point is actually a point on the curve. + pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { + let key = PublicKey::from_bytes(bytes)?; + Ok(PeerId(key)) + } } impl From for PeerId { diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 0ed3ee73f7..0d0c9b891a 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -16,6 +16,7 @@ derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from" ed25519-dalek = { version = "2.0.0-rc.2", features = ["serde", "rand_core"] } iroh-bytes = { version = "0.5.0", path = "../iroh-bytes" } once_cell = "1.18.0" +postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } rand = "0.8.5" rand_core = "0.6.4" serde = { version = "1.0.164", features = ["derive"] } diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index 7317d2a321..d5310839a6 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -401,11 +401,12 @@ where /// Processes an incoming message and produces a response. /// If terminated, returns `None` - pub fn process_message(&mut self, message: Message) -> Option> { + pub fn process_message(&mut self, message: Message) -> (Vec, Option>) { let mut out = Vec::new(); // TODO: can these allocs be avoided? let mut items = Vec::new(); + let mut inserted = Vec::new(); let mut fingerprints = Vec::new(); for part in message.parts { match part { @@ -431,7 +432,6 @@ where Some( self.store .get_range(range.clone(), self.limit.clone()) - .into_iter() .filter(|(k, _)| !values.iter().any(|(vk, _)| &vk == k)) .map(|(k, v)| (k.clone(), v.clone())) .collect(), @@ -440,6 +440,7 @@ where // Store incoming values for (k, v) in values { + inserted.push(k.clone()); self.store.put(k, v); } @@ -546,9 +547,9 @@ where // If we have any parts, return a message if !out.is_empty() { - Some(Message { parts: out }) + (inserted, Some(Message { parts: out })) } else { - None + (inserted, None) } } diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index adf75507e9..623bd81af7 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -216,6 +216,14 @@ impl ReplicaStore { .insert(replica.namespace(), replica.clone()); replica } + + pub fn open_replica(&self, bytes: &[u8]) -> anyhow::Result { + let replica = Replica::from_bytes(bytes)?; + self.replicas + .write() + .insert(replica.namespace(), replica.clone()); + Ok(replica) + } } /// TODO: Would potentially nice to pass a `&SignedEntry` reference, however that would make @@ -239,7 +247,6 @@ pub struct Replica { struct InnerReplica { namespace: Namespace, peer: Peer, - content: HashMap, #[debug("on_insert: [Box; {}]", "self.on_insert.len()")] on_insert: Vec, } @@ -337,6 +344,12 @@ impl crate::ranger::Store for Store { } } +#[derive(Debug, Serialize, Deserialize)] +struct ReplicaData { + entries: Vec, + namespace: Namespace, +} + #[derive(Debug)] pub struct RangeIterator<'a> { iter: std::collections::btree_map::Iter<'a, RecordIdentifier, BTreeMap>, @@ -375,7 +388,6 @@ impl Replica { inner: Arc::new(RwLock::new(InnerReplica { namespace, peer: Peer::default(), - content: HashMap::default(), on_insert: Default::default(), })), } @@ -386,10 +398,6 @@ impl Replica { inner.on_insert.push(callback); } - pub fn get_content(&self, hash: &Hash) -> Option { - self.inner.read().content.get(hash).cloned() - } - // TODO: not horrible pub fn all(&self) -> Vec<(RecordIdentifier, SignedEntry)> { self.inner @@ -400,23 +408,45 @@ impl Replica { .collect() } + // TODO: not horrible + pub fn all_for_key(&self, key: impl AsRef<[u8]>) -> Vec<(RecordIdentifier, SignedEntry)> { + self.all() + .into_iter() + .filter(|(id, _entry)| id.key() == key.as_ref()) + .collect() + } + + pub fn to_bytes(&self) -> anyhow::Result { + let entries = self.all().into_iter().map(|(_id, entry)| entry).collect(); + let data = ReplicaData { + entries, + namespace: self.inner.read().namespace.clone(), + }; + let bytes = postcard::to_stdvec(&data)?; + Ok(bytes.into()) + } + pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { + let data: ReplicaData = postcard::from_bytes(bytes)?; + let replica = Self::new(data.namespace); + for entry in data.entries { + replica.insert_remote_entry(entry)?; + } + Ok(replica) + } + /// Inserts a new record at the given key. - pub fn insert(&self, key: impl AsRef<[u8]>, author: &Author, data: impl Into) { + pub fn insert(&self, key: impl AsRef<[u8]>, author: &Author, hash: Hash, len: u64) { let mut inner = self.inner.write(); let id = RecordIdentifier::new(key, inner.namespace.id(), author.id()); - let data: Bytes = data.into(); - let record = Record::from_data(&data, inner.namespace.id()); - - // Store content - inner.content.insert(*record.content_hash(), data); + let record = Record::from_hash(hash, len); // Store signed entries let entry = Entry::new(id.clone(), record); let signed_entry = entry.sign(&inner.namespace, author); inner.peer.put(id, signed_entry.clone()); for cb in &inner.on_insert { - cb(InsertOrigin::Local, signed_entry.clone()) + cb(InsertOrigin::Local, signed_entry.clone()); } } @@ -470,7 +500,15 @@ impl Replica { &self, message: crate::ranger::Message, ) -> Option> { - self.inner.write().peer.process_message(message) + let (inserted_keys, reply) = self.inner.write().peer.process_message(message); + let inner = self.inner.read(); + for key in inserted_keys { + let entry = inner.peer.get(&key).unwrap(); + for cb in &inner.on_insert { + cb(InsertOrigin::Sync, entry.clone()) + } + } + reply } pub fn namespace(&self) -> NamespaceId { @@ -719,22 +757,24 @@ impl Record { &self.hash } - pub fn from_data(data: impl AsRef<[u8]>, namespace: &NamespaceId) -> Self { + pub fn from_hash(hash: Hash, len: u64) -> Self { let timestamp = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .expect("time drift") .as_micros() as u64; - let data = data.as_ref(); - let len = data.len() as u64; + Self::new(timestamp, len, hash) + } + + // TODO: remove + pub fn from_data(data: impl AsRef<[u8]>, namespace: &NamespaceId) -> Self { // Salted hash // TODO: do we actually want this? // TODO: this should probably use a namespace prefix if used let mut hasher = blake3::Hasher::new(); hasher.update(namespace.as_bytes()); - hasher.update(data); + hasher.update(data.as_ref()); let hash = hasher.finalize(); - - Self::new(timestamp, len, hash.into()) + Self::from_hash(hash.into(), data.as_ref().len() as u64) } pub fn as_bytes(&self, out: &mut Vec) { diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index b603940761..3c0a332d8d 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -7,12 +7,15 @@ //! `cargo run --bin derper -- --dev` //! and then set the `-d http://localhost:3340` flag on this example. -use std::{fmt, str::FromStr}; +use std::{fmt, path::PathBuf, str::FromStr, sync::Arc}; use anyhow::{anyhow, bail}; +use bytes::Bytes; use clap::Parser; use ed25519_dalek::SigningKey; -use iroh::sync::{LiveSync, PeerSource, SYNC_ALPN}; +use futures::{future::BoxFuture, FutureExt}; +use iroh::sync::{BlobStore, Doc, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; +use iroh_bytes::provider::Database; use iroh_gossip::{ net::{GossipHandle, GOSSIP_ALPN}, proto::TopicId, @@ -24,7 +27,7 @@ use iroh_net::{ tls::Keypair, MagicEndpoint, }; -use iroh_sync::sync::{Author, Namespace, Replica, ReplicaStore, SignedEntry}; +use iroh_sync::sync::{Author, Namespace, NamespaceId, Replica, ReplicaStore, SignedEntry}; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; @@ -35,6 +38,9 @@ struct Args { /// Private key to derive our peer id from #[clap(long)] private_key: Option, + /// Path to a data directory where blobs will be persisted + #[clap(short, long)] + storage_path: Option, /// Set a custom DERP server. By default, the DERP server hosted by n0 will be used. #[clap(short, long)] derp: Option, @@ -90,7 +96,11 @@ async fn run(args: Args) -> anyhow::Result<()> { let endpoint = MagicEndpoint::builder() .keypair(keypair.clone()) - .alpns(vec![GOSSIP_ALPN.to_vec(), SYNC_ALPN.to_vec()]) + .alpns(vec![ + GOSSIP_ALPN.to_vec(), + SYNC_ALPN.to_vec(), + iroh_bytes::protocol::ALPN.to_vec(), + ]) .derp_map(derp_map) .on_endpoints({ let gossip_cell = gossip_cell.clone(); @@ -151,68 +161,94 @@ async fn run(args: Args) -> anyhow::Result<()> { }; println!("> ticket to join us: {our_ticket}"); + // unwrap our storage path or default to temp + let storage_path = args.storage_path.unwrap_or_else(|| { + let dir = format!("/tmp/iroh-example-sync-{}", endpoint.peer_id()); + let dir = PathBuf::from(dir); + if !dir.exists() { + std::fs::create_dir(&dir).expect("failed to create temp dir"); + } + dir + }); + println!("> persisting data in {storage_path:?}"); + + // create a runtime + // we need this because some things need to spawn !Send futures + let rt = create_rt()?; // create the sync doc and store - let (store, author, doc) = create_document(topic, &keypair)?; + // we need to pass the runtime because a !Send task is spawned for + // the downloader in the blob store + let blobs = BlobStore::new(rt.clone(), storage_path.clone(), endpoint.clone()).await?; + let (store, author, doc) = + create_or_open_document(&storage_path, blobs.clone(), topic, &keypair).await?; + // construct the state that is passed to the endpoint loop and from there cloned + // into to the connection handler task for incoming connections. + let state = Arc::new(State { + gossip: gossip.clone(), + replica_store: store.clone(), + db: blobs.db().clone(), + rt, + }); // spawn our endpoint loop that forwards incoming connections - tokio::spawn(endpoint_loop( - endpoint.clone(), - gossip.clone(), - store.clone(), - )); - - // spawn an input thread that reads stdin - // not using tokio here because they recommend this for "technical reasons" - let (line_tx, mut line_rx) = tokio::sync::mpsc::channel::(1); - std::thread::spawn(move || input_loop(line_tx)); + tokio::spawn(endpoint_loop(endpoint.clone(), state)); // create the live syncer let sync_handle = LiveSync::spawn(endpoint.clone(), gossip.clone()); - sync_handle.sync_doc(doc.clone(), peers.clone()).await?; + sync_handle + .sync_doc(doc.replica().clone(), peers.clone()) + .await?; - // do some logging - doc.on_insert(Box::new(move |origin, entry| { - println!("> insert from {origin:?}: {}", fmt_entry(&entry)); - })); + // spawn an input thread that reads stdin and parses each line as a `Cmd` command + // not using tokio here because they recommend this for "technical reasons" + let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::channel::(1); + std::thread::spawn(move || input_loop(cmd_tx)); + + // process commands in a loop + println!("> ready to accept commands: set | get | ls | exit"); + loop { + let cmd = tokio::select! { + Some(cmd) = cmd_rx.recv() => cmd, + _ = tokio::signal::ctrl_c() => Cmd::Exit - // process stdin lines - println!("> read to accept commands: set | get | ls | exit"); - while let Some(text) = line_rx.recv().await { - let cmd = match Cmd::from_str(&text) { - Ok(cmd) => cmd, - Err(err) => { - println!("> failed to parse command: {}", err); - continue; - } }; match cmd { Cmd::Set { key, value } => { - doc.insert(&key, &author, value); + doc.insert(&key, &author, value.into_bytes().into()).await?; } Cmd::Get { key } => { - let mut entries = doc - .all() - .into_iter() - .filter_map(|(id, entry)| (id.key() == key.as_bytes()).then(|| entry)); - while let Some(entry) = entries.next() { - println!("{} -> {}", fmt_entry(&entry), fmt_content(&doc, &entry)); + let entries = doc.replica().all_for_key(key.as_bytes()); + for (_id, entry) in entries { + let content = fmt_content(&doc, &entry).await?; + println!("{} -> {content}", fmt_entry(&entry),); } } Cmd::Ls => { - let all = doc.all(); + let all = doc.replica().all(); println!("> {} entries", all.len()); for (_id, entry) in all { - println!("{} -> {}", fmt_entry(&entry), fmt_content(&doc, &entry)); + println!( + "{} -> {}", + fmt_entry(&entry), + fmt_content(&doc, &entry).await? + ); } } Cmd::Exit => { - let res = sync_handle.cancel().await?; - println!("syncer closed with {res:?}"); break; } } } + let res = sync_handle.cancel().await; + if let Err(err) = res { + println!("> syncer closed with error: {err:?}"); + } + + println!("> persisting document and blob database at {storage_path:?}"); + blobs.save().await?; + save_document(&storage_path, doc.replica()).await?; + Ok(()) } @@ -240,46 +276,161 @@ impl FromStr for Cmd { } } -fn create_document( +async fn create_or_open_document( + storage_path: &PathBuf, + blobs: BlobStore, topic: TopicId, keypair: &Keypair, -) -> anyhow::Result<(ReplicaStore, Author, Replica)> { +) -> anyhow::Result<(ReplicaStore, Author, Doc)> { let author = Author::from(keypair.secret().clone()); let namespace = Namespace::from_bytes(topic.as_bytes()); let store = ReplicaStore::default(); - let doc = store.new_replica(namespace); + + let replica_path = replica_path(storage_path, namespace.id()); + let replica = if replica_path.exists() { + let bytes = tokio::fs::read(replica_path).await?; + store.open_replica(&bytes)? + } else { + store.new_replica(namespace) + }; + + // do some logging + replica.on_insert(Box::new(move |origin, entry| { + println!("> insert from {origin:?}: {}", fmt_entry(&entry)); + })); + + let doc = Doc::new(replica, blobs, DownloadMode::Always); Ok((store, author, doc)) } -async fn endpoint_loop( - endpoint: MagicEndpoint, +async fn save_document(base_path: &PathBuf, replica: &Replica) -> anyhow::Result<()> { + let replica_path = replica_path(base_path, &replica.namespace()); + tokio::fs::create_dir_all(replica_path.parent().unwrap()).await?; + let bytes = replica.to_bytes()?; + tokio::fs::write(replica_path, bytes).await?; + Ok(()) +} + +fn replica_path(storage_path: &PathBuf, namespace: &NamespaceId) -> PathBuf { + storage_path + .join("docs") + .join(hex::encode(namespace.as_bytes())) +} + +#[derive(Debug)] +struct State { + rt: iroh_bytes::runtime::Handle, gossip: GossipHandle, replica_store: ReplicaStore, + db: Database, +} + +async fn endpoint_loop(endpoint: MagicEndpoint, state: Arc) -> anyhow::Result<()> { + while let Some(conn) = endpoint.accept().await { + // spawn a new task for each incoming connection. + let state = state.clone(); + tokio::spawn(async move { + if let Err(err) = handle_connection(conn, state).await { + println!("> connection closed, reason: {err}"); + } + }); + } + Ok(()) +} + +async fn handle_connection(mut conn: quinn::Connecting, state: Arc) -> anyhow::Result<()> { + let alpn = get_alpn(&mut conn).await?; + println!("> incoming connection with alpn {alpn}"); + match alpn.as_bytes() { + GOSSIP_ALPN => state.gossip.handle_connection(conn.await?).await, + SYNC_ALPN => iroh::sync::handle_connection(conn, state.replica_store.clone()).await, + alpn if alpn == iroh_bytes::protocol::ALPN => { + handle_iroh_byes_connection(conn, state).await + } + _ => bail!("ignoring connection: unsupported ALPN protocol"), + } +} + +async fn handle_iroh_byes_connection( + conn: quinn::Connecting, + state: Arc, ) -> anyhow::Result<()> { - while let Some(mut conn) = endpoint.accept().await { - let alpn = get_alpn(&mut conn).await?; - println!("> incoming connection with alpn {alpn}"); - // let (peer_id, alpn, conn) = accept_conn(conn).await?; - let res = match alpn.as_bytes() { - GOSSIP_ALPN => gossip.handle_connection(conn.await?).await, - SYNC_ALPN => iroh::sync::handle_connection(conn, replica_store.clone()).await, - _ => Err(anyhow::anyhow!( - "ignoring connection: unsupported ALPN protocol" - )), - }; - if let Err(err) = res { - println!("> connection for {alpn} closed, reason: {err}"); + use iroh_bytes::{ + protocol::{GetRequest, RequestToken}, + provider::{ + CustomGetHandler, EventSender, IrohCollectionParser, RequestAuthorizationHandler, + }, + }; + iroh_bytes::provider::handle_connection( + conn, + state.db.clone(), + NoopEventSender, + IrohCollectionParser, + Arc::new(NoopCustomGetHandler), + Arc::new(NoopRequestAuthorizationHandler), + state.rt.clone(), + ) + .await; + + #[derive(Debug, Clone)] + struct NoopEventSender; + impl EventSender for NoopEventSender { + fn send(&self, _event: iroh_bytes::provider::Event) -> Option { + None + } + } + #[derive(Debug)] + struct NoopCustomGetHandler; + impl CustomGetHandler for NoopCustomGetHandler { + fn handle( + &self, + _token: Option, + _request: Bytes, + ) -> BoxFuture<'static, anyhow::Result> { + async move { Err(anyhow::anyhow!("no custom get handler defined")) }.boxed() + } + } + #[derive(Debug)] + struct NoopRequestAuthorizationHandler; + impl RequestAuthorizationHandler for NoopRequestAuthorizationHandler { + fn authorize( + &self, + token: Option, + _request: &iroh_bytes::protocol::Request, + ) -> BoxFuture<'static, anyhow::Result<()>> { + async move { + if let Some(token) = token { + anyhow::bail!( + "no authorization handler defined, but token was provided: {:?}", + token + ); + } + Ok(()) + } + .boxed() } } Ok(()) } -fn input_loop(line_tx: tokio::sync::mpsc::Sender) -> anyhow::Result<()> { +fn create_rt() -> anyhow::Result { + let rt = iroh::bytes::runtime::Handle::from_currrent(num_cpus::get())?; + Ok(rt) +} + +fn input_loop(line_tx: tokio::sync::mpsc::Sender) -> anyhow::Result<()> { let mut buffer = String::new(); - let stdin = std::io::stdin(); // We get `Stdin` here. + let stdin = std::io::stdin(); loop { stdin.read_line(&mut buffer)?; - line_tx.blocking_send(buffer.trim().to_string())?; + let cmd = match Cmd::from_str(buffer.trim()) { + Ok(cmd) => cmd, + Err(err) => { + println!("> failed to parse command: {}", err); + continue; + } + }; + line_tx.blocking_send(cmd)?; buffer.clear(); } } @@ -331,13 +482,15 @@ fn fmt_entry(entry: &SignedEntry) -> String { let hash = fmt_hash(hash.as_bytes()); format!("@{author}: {key} = {hash}") } -fn fmt_content(doc: &Replica, entry: &SignedEntry) -> String { - let hash = entry.entry().record().content_hash(); - let content = doc.get_content(hash); - let content = content - .map(|content| String::from_utf8(content.into()).unwrap_or_else(|_| "".into())) - .unwrap_or_else(|| "".into()); - content +async fn fmt_content(doc: &Doc, entry: &SignedEntry) -> anyhow::Result { + let content = match doc.get_content(entry).await { + None => "".to_string(), + Some(content) => match String::from_utf8(content.into()) { + Ok(str) => str, + Err(_err) => "".to_string(), + }, + }; + Ok(content) } fn fmt_hash(hash: &[u8]) -> String { let mut text = data_encoding::BASE32_NOPAD.encode(hash); diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index 72b5c02ade..cf9f8e0fd0 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -13,7 +13,9 @@ use tracing::debug; /// The ALPN identifier for the iroh-sync protocol pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; +mod content; mod live; +pub use content::*; pub use live::*; /// Sync Protocol diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs new file mode 100644 index 0000000000..5a4f1d4489 --- /dev/null +++ b/iroh/src/sync/content.rs @@ -0,0 +1,324 @@ +use std::{ + collections::{HashMap, HashSet, VecDeque}, + io, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use bytes::Bytes; +use futures::{ + future::{BoxFuture, LocalBoxFuture, Shared}, + stream::FuturesUnordered, + FutureExt, +}; +use iroh_bytes::{provider::Database, util::Hash, writable::WritableFileDatabase}; +use iroh_gossip::net::util::Dialer; +use iroh_io::AsyncSliceReaderExt; +use iroh_net::{tls::PeerId, MagicEndpoint}; +use iroh_sync::sync::{Author, InsertOrigin, Replica, SignedEntry}; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::StreamExt; +use tracing::{debug, error, warn}; + +#[derive(Debug, Copy, Clone)] +pub enum DownloadMode { + Always, + Manual, +} + +/// A replica with a [`BlobStore`] for contents. +/// +/// This will also download missing content from peers. +/// +/// TODO: Currently content is only downloaded from the author of a entry. +/// We want to try other peers if the author is offline (or always). +/// We'll need some heuristics which peers to try. +#[derive(Clone)] +pub struct Doc { + replica: Replica, + blobs: BlobStore, +} + +impl Doc { + pub fn new(replica: Replica, blobs: BlobStore, download_mode: DownloadMode) -> Self { + let doc = Self { replica, blobs }; + if let DownloadMode::Always = download_mode { + let doc2 = doc.clone(); + doc.replica.on_insert(Box::new(move |origin, entry| { + if matches!(origin, InsertOrigin::Sync) { + doc2.download_content_fron_author(&entry); + } + })); + } + doc + } + + pub fn replica(&self) -> &Replica { + &self.replica + } + + pub async fn insert( + &self, + key: impl AsRef<[u8]>, + author: &Author, + content: Bytes, + ) -> anyhow::Result<()> { + let (hash, len) = self.blobs.put_bytes(content).await?; + self.replica.insert(key, author, hash, len); + Ok(()) + } + + pub fn download_content_fron_author(&self, entry: &SignedEntry) { + let hash = *entry.entry().record().content_hash(); + let peer_id = PeerId::from_bytes(entry.entry().id().author().as_bytes()) + .expect("failed to convert author to peer id"); + self.blobs.start_download(hash, peer_id); + } + + pub async fn get_content(&self, entry: &SignedEntry) -> Option { + let hash = entry.entry().record().content_hash(); + let bytes = self.blobs.get_bytes(hash).await.ok().flatten(); + bytes + } +} + +/// A blob database that can download missing blobs from peers. +/// +/// Blobs can be inserted either from bytes or by downloading from peers. +/// Downloads can be started and will be tracked in the blobstore. +/// New blobs will be saved as files with a filename based on their hash. +/// +/// TODO: This is similar to what is used in the iroh provider. +/// Unify once we know how the APIs should look like. +#[derive(Debug, Clone)] +pub struct BlobStore { + db: WritableFileDatabase, + downloader: Downloader, +} +impl BlobStore { + pub async fn new( + rt: iroh_bytes::runtime::Handle, + data_path: PathBuf, + endpoint: MagicEndpoint, + ) -> anyhow::Result { + let db = WritableFileDatabase::new(data_path).await?; + let downloader = Downloader::new(rt, endpoint, db.clone()); + Ok(Self { db, downloader }) + } + + pub async fn save(&self) -> io::Result<()> { + self.db.save().await + } + + pub fn db(&self) -> &Database { + &self.db.db() + } + + pub fn start_download(&self, hash: Hash, peer: PeerId) { + if !self.db.has(&hash) { + self.downloader.start_download(hash, peer); + } + } + + pub async fn get_bytes(&self, hash: &Hash) -> anyhow::Result> { + self.downloader.wait_for_download(hash).await; + let Some(entry) = self.db().get(hash) else { + return Ok(None) + }; + let bytes = entry.data_reader().await?.read_to_end().await?; + Ok(Some(bytes)) + } + + pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64)> { + self.db.put_bytes(data).await + } +} + +pub type DownloadReply = oneshot::Sender>; +pub type DownloadFuture = Shared>>; + +#[derive(Debug)] +pub struct DownloadRequest { + hash: Hash, + peer: PeerId, + reply: DownloadReply, +} + +/// A download queue +/// +/// Spawns a background task that handles connecting to peers and performing get requests. +/// +/// TODO: Queued downloads are pushed into an unbounded channel. Maybe make it bounded instead. +/// We want the start_download() method to be sync though because it is used +/// from sync on_insert callbacks on the replicas. +/// TODO: Move to iroh-bytes or replace with corresponding feature from iroh-bytes once available +#[derive(Debug, Clone)] +pub struct Downloader { + pending_downloads: Arc>>, + to_actor_tx: mpsc::UnboundedSender, +} + +impl Downloader { + pub fn new( + rt: iroh_bytes::runtime::Handle, + endpoint: MagicEndpoint, + blobs: WritableFileDatabase, + ) -> Self { + let (tx, rx) = mpsc::unbounded_channel(); + // spawn the actor on a local pool + // the local pool is required because WritableFileDatabase::download_single + // returns a future that is !Send + rt.local_pool().spawn_pinned(move || async move { + let mut actor = DownloadActor::new(endpoint, blobs, rx); + if let Err(err) = actor.run().await { + error!("download actor failed with error {err:?}"); + } + }); + Self { + pending_downloads: Arc::new(Mutex::new(HashMap::new())), + to_actor_tx: tx, + } + } + + pub fn wait_for_download(&self, hash: &Hash) -> DownloadFuture { + match self.pending_downloads.lock().unwrap().get(hash) { + Some(fut) => fut.clone(), + None => futures::future::ready(None).boxed().shared(), + } + } + + pub fn start_download(&self, hash: Hash, peer: PeerId) { + let (reply, reply_rx) = oneshot::channel(); + let req = DownloadRequest { hash, peer, reply }; + let pending_downloads = self.pending_downloads.clone(); + let fut = async move { + let res = reply_rx.await; + pending_downloads.lock().unwrap().remove(&hash); + res.ok().flatten() + }; + self.pending_downloads + .lock() + .unwrap() + .insert(hash, fut.boxed().shared()); + if let Err(err) = self.to_actor_tx.send(req) { + warn!("download actor dropped: {err}"); + } + } +} + +pub struct DownloadActor { + dialer: Dialer, + db: WritableFileDatabase, + conns: HashMap, + replies: HashMap>, + peer_hashes: HashMap>, + hash_peers: HashMap>, + pending_downloads: FuturesUnordered< + LocalBoxFuture<'static, (PeerId, Hash, anyhow::Result>)>, + >, + rx: mpsc::UnboundedReceiver, +} +impl DownloadActor { + fn new( + endpoint: MagicEndpoint, + db: WritableFileDatabase, + rx: mpsc::UnboundedReceiver, + ) -> Self { + Self { + rx, + db, + dialer: Dialer::new(endpoint), + replies: Default::default(), + conns: Default::default(), + pending_downloads: Default::default(), + peer_hashes: Default::default(), + hash_peers: Default::default(), + } + } + pub async fn run(&mut self) -> anyhow::Result<()> { + loop { + tokio::select! { + req = self.rx.recv() => match req { + None => return Ok(()), + Some(req) => self.on_download_request(req).await + }, + (peer, conn) = self.dialer.next() => match conn { + Ok(conn) => { + debug!("connection to {peer} established"); + self.conns.insert(peer, conn); + self.on_peer_ready(peer); + }, + Err(err) => self.on_peer_fail(&peer, err), + }, + Some((peer, hash, res)) = self.pending_downloads.next() => match res { + Ok(Some((hash, size))) => { + self.reply(hash, Some((hash, size))); + self.on_peer_ready(peer); + } + Ok(None) => { + self.on_not_found(&peer, hash); + self.on_peer_ready(peer); + } + Err(err) => self.on_peer_fail(&peer, err), + } + } + } + } + + fn reply(&mut self, hash: Hash, res: Option<(Hash, u64)>) { + for reply in self.replies.remove(&hash).into_iter().flatten() { + reply.send(res.clone()).ok(); + } + } + + fn on_peer_fail(&mut self, peer: &PeerId, err: anyhow::Error) { + warn!("download from {peer} failed: {err}"); + for hash in self.peer_hashes.remove(&peer).into_iter().flatten() { + self.on_not_found(peer, hash); + } + self.conns.remove(&peer); + } + + fn on_not_found(&mut self, peer: &PeerId, hash: Hash) { + if let Some(peers) = self.hash_peers.get_mut(&hash) { + peers.remove(&peer); + if peers.is_empty() { + self.reply(hash, None); + self.hash_peers.remove(&hash); + } + } + } + + fn on_peer_ready(&mut self, peer: PeerId) { + if let Some(hash) = self + .peer_hashes + .get_mut(&peer) + .map(|hashes| hashes.pop_front()) + .flatten() + { + let conn = self.conns.get(&peer).unwrap().clone(); + let blobs = self.db.clone(); + let fut = async move { (peer, hash, blobs.download_single(conn, hash).await) }; + self.pending_downloads.push(fut.boxed_local()); + } else { + self.conns.remove(&peer); + self.peer_hashes.remove(&peer); + } + } + + async fn on_download_request(&mut self, req: DownloadRequest) { + let DownloadRequest { peer, hash, reply } = req; + if self.db.has(&hash) { + let size = self.db.get_size(&hash).await.unwrap(); + reply.send(Some((hash, size))).ok(); + return; + } + debug!("queue download {hash} from {peer}"); + self.replies.entry(hash).or_default().push_back(reply); + self.hash_peers.entry(hash).or_default().insert(peer); + self.peer_hashes.entry(peer).or_default().push_back(hash); + if self.conns.get(&peer).is_none() && !self.dialer.is_pending(&peer) { + self.dialer.queue_dial(peer, &iroh_bytes::protocol::ALPN); + } + } +} From b1868ad758634f1e1b3f6fe2db1663c7a31e0d53 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 13 Jul 2023 17:24:09 +0200 Subject: [PATCH 008/172] feat: proper REPL for sync example, and docs store * make the REPL in the sync example work properly with rustyline for editing and reading input, shell-style argument parsing and clap for parsing commands * add a docs store for opening and closing docs * add author to doc struct --- Cargo.lock | 146 ++++++++++ iroh-bytes/src/writable.rs | 22 ++ iroh-sync/src/sync.rs | 15 ++ iroh/Cargo.toml | 5 +- iroh/examples/sync.rs | 536 +++++++++++++++++++++---------------- iroh/src/sync/content.rs | 128 ++++++++- iroh/src/sync/live.rs | 16 +- 7 files changed, 618 insertions(+), 250 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89bec29a07..54246da656 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,6 +490,17 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "clipboard-win" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" +dependencies = [ + "error-code", + "str-buf", + "winapi", +] + [[package]] name = "cobs" version = "0.2.3" @@ -885,6 +896,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -895,6 +915,18 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1031,6 +1063,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enum-as-inner" version = "0.5.1" @@ -1098,6 +1136,16 @@ dependencies = [ "libc", ] +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1110,6 +1158,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "fd-lock" +version = "3.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "ff" version = "0.13.0" @@ -1421,6 +1480,15 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3688e69b38018fec1557254f64c8dc2cc8ec502890182f395dbb0aa997aa5735" +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -1690,7 +1758,10 @@ dependencies = [ "rand", "range-collections", "regex", + "rustyline", "serde", + "shell-words", + "shellexpand", "tempfile", "testdir", "thiserror", @@ -2170,6 +2241,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.26.2" @@ -2396,6 +2476,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "os_info" version = "3.7.0" @@ -2987,6 +3073,16 @@ dependencies = [ "pest_derive", ] +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.5" @@ -3387,6 +3483,29 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "rustyline" +version = "12.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9" +dependencies = [ + "bitflags 2.3.3", + "cfg-if", + "clipboard-win", + "fd-lock", + "home", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "scopeguard", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi", +] + [[package]] name = "ryu" version = "1.0.15" @@ -3596,6 +3715,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + +[[package]] +name = "shellexpand" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -3731,6 +3865,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "str-buf" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" + [[package]] name = "strsim" version = "0.10.0" @@ -4331,6 +4471,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + [[package]] name = "unicode-width" version = "0.1.10" diff --git a/iroh-bytes/src/writable.rs b/iroh-bytes/src/writable.rs index fe492e47b4..d1666831da 100644 --- a/iroh-bytes/src/writable.rs +++ b/iroh-bytes/src/writable.rs @@ -10,6 +10,7 @@ use anyhow::Context; use bytes::Bytes; use iroh_io::{AsyncSliceWriter, File}; use range_collections::RangeSet2; +use tokio::io::AsyncRead; use crate::{ get::fsm, @@ -60,6 +61,12 @@ impl WritableFileDatabase { Ok((hash, size)) } + pub async fn put_reader(&self, data: impl AsyncRead + Unpin) -> anyhow::Result<(Hash, u64)> { + let (hash, size, entry) = self.storage.put_reader(data).await?; + self.db.union_with(HashMap::from_iter([(hash, entry)])); + Ok((hash, size)) + } + pub async fn put_from_temp_file(&self, temp_path: &PathBuf) -> anyhow::Result<(Hash, u64)> { let (hash, size, entry) = self.storage.move_to_blobs(&temp_path).await?; self.db.union_with(HashMap::from_iter([(hash, entry)])); @@ -144,6 +151,21 @@ impl StoragePaths { Ok((hash, size, entry)) } + pub async fn put_reader( + &self, + mut reader: impl AsyncRead + Unpin, + ) -> anyhow::Result<(Hash, u64, DbEntry)> { + let temp_path = self.temp_path(); + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&temp_path) + .await?; + tokio::io::copy(&mut reader, &mut file).await?; + let (hash, size, entry) = self.move_to_blobs(&temp_path).await?; + Ok((hash, size, entry)) + } + async fn move_to_blobs(&self, path: &PathBuf) -> anyhow::Result<(Hash, u64, DbEntry)> { let datasource = DataSource::new(path.clone()); // TODO: this needlessly creates a collection, but that's what's pub atm in iroh-bytes diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 623bd81af7..a6f05391bb 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -416,6 +416,17 @@ impl Replica { .collect() } + // TODO: not horrible + pub fn all_with_key_prefix( + &self, + prefix: impl AsRef<[u8]>, + ) -> Vec<(RecordIdentifier, SignedEntry)> { + self.all() + .into_iter() + .filter(|(id, _entry)| id.key().starts_with(prefix.as_ref())) + .collect() + } + pub fn to_bytes(&self) -> anyhow::Result { let entries = self.all().into_iter().map(|(_id, entry)| entry).collect(); let data = ReplicaData { @@ -566,6 +577,10 @@ impl SignedEntry { pub fn entry(&self) -> &Entry { &self.entry } + + pub fn content_hash(&self) -> &Hash { + self.entry().record().content_hash() + } } /// Signature over an entry. diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 75f3c13f31..d0b45dbe23 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -56,6 +56,9 @@ url = { version = "2.4", features = ["serde"] } # Examples once_cell = { version = "1.18.0", optional = true } ed25519-dalek = { version = "=2.0.0-rc.3", features = ["serde", "rand_core"], optional = true } +shell-words = { version = "1.1.0", optional = true } +shellexpand = { version = "3.1.0", optional = true } +rustyline = { version = "12.0.0", optional = true } [features] default = ["cli", "metrics"] @@ -65,7 +68,7 @@ mem-db = [] flat-db = [] iroh-collection = [] test = [] -example-sync = ["cli", "ed25519-dalek", "once_cell"] +example-sync = ["cli", "ed25519-dalek", "once_cell", "shell-words", "shellexpand", "rustyline"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 3c0a332d8d..3ec5ec6ddb 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -9,13 +9,11 @@ use std::{fmt, path::PathBuf, str::FromStr, sync::Arc}; -use anyhow::{anyhow, bail}; -use bytes::Bytes; -use clap::Parser; +use anyhow::bail; +use clap::{CommandFactory, FromArgMatches, Parser}; use ed25519_dalek::SigningKey; -use futures::{future::BoxFuture, FutureExt}; -use iroh::sync::{BlobStore, Doc, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; -use iroh_bytes::provider::Database; +use indicatif::HumanBytes; +use iroh::sync::{BlobStore, Doc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; use iroh_gossip::{ net::{GossipHandle, GOSSIP_ALPN}, proto::TopicId, @@ -27,12 +25,17 @@ use iroh_net::{ tls::Keypair, MagicEndpoint, }; -use iroh_sync::sync::{Author, Namespace, NamespaceId, Replica, ReplicaStore, SignedEntry}; +use iroh_sync::sync::{Author, Namespace, SignedEntry}; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; +use tracing_subscriber::{EnvFilter, Registry}; use url::Url; +use iroh_bytes_handlers::IrohBytesHandlers; + +const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; + #[derive(Parser, Debug)] struct Args { /// Private key to derive our peer id from @@ -65,12 +68,12 @@ enum Command { #[tokio::main] async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt::init(); let args = Args::parse(); run(args).await } async fn run(args: Args) -> anyhow::Result<()> { + let log_filter = init_logging(); // parse or generate our keypair let keypair = match args.private_key { None => Keypair::generate(), @@ -87,13 +90,13 @@ async fn run(args: Args) -> anyhow::Result<()> { }; println!("> using DERP servers: {}", fmt_derp_map(&derp_map)); - // build our magic endpoint + // build our magic endpoint and the gossip protocol let (endpoint, gossip, initial_endpoints) = { // init a cell that will hold our gossip handle to be used in endpoint callbacks let gossip_cell: OnceCell = OnceCell::new(); // init a channel that will emit once the initial endpoints of our local node are discovered let (initial_endpoints_tx, mut initial_endpoints_rx) = mpsc::channel(1); - + // build the magic endpoint let endpoint = MagicEndpoint::builder() .keypair(keypair.clone()) .alpns(vec![ @@ -116,16 +119,14 @@ async fn run(args: Args) -> anyhow::Result<()> { .bind(args.bind_port) .await?; - // create the gossip protocol - let gossip = { - let gossip = GossipHandle::from_endpoint(endpoint.clone(), Default::default()); - // insert the gossip handle into the gossip cell to be used in the endpoint callbacks above - gossip_cell.set(gossip.clone()).unwrap(); - gossip - }; + // initialize the gossip protocol + let gossip = GossipHandle::from_endpoint(endpoint.clone(), Default::default()); + // insert into the gossip cell to be used in the endpoint callbacks above + gossip_cell.set(gossip.clone()).unwrap(); + // wait for a first endpoint update so that we know about at least one of our addrs let initial_endpoints = initial_endpoints_rx.recv().await.unwrap(); - // pass our initial endpoints to the gossip protocol + // pass our initial endpoints to the gossip protocol so that they can be announced to peers gossip.update_endpoints(&initial_endpoints)?; (endpoint, gossip, initial_endpoints) }; @@ -147,7 +148,6 @@ async fn run(args: Args) -> anyhow::Result<()> { } }; - // println!("> our endpoints: {initial_endpoints:?}"); let our_ticket = { // add our local endpoints to the ticket and print it for others to join let addrs = initial_endpoints.iter().map(|ep| ep.addr).collect(); @@ -163,171 +163,183 @@ async fn run(args: Args) -> anyhow::Result<()> { // unwrap our storage path or default to temp let storage_path = args.storage_path.unwrap_or_else(|| { - let dir = format!("/tmp/iroh-example-sync-{}", endpoint.peer_id()); - let dir = PathBuf::from(dir); + let name = format!("iroh-sync-{}", endpoint.peer_id()); + let dir = std::env::temp_dir().join(name); if !dir.exists() { std::fs::create_dir(&dir).expect("failed to create temp dir"); } dir }); - println!("> persisting data in {storage_path:?}"); - - // create a runtime - // we need this because some things need to spawn !Send futures - let rt = create_rt()?; - // create the sync doc and store - // we need to pass the runtime because a !Send task is spawned for - // the downloader in the blob store - let blobs = BlobStore::new(rt.clone(), storage_path.clone(), endpoint.clone()).await?; - let (store, author, doc) = - create_or_open_document(&storage_path, blobs.clone(), topic, &keypair).await?; + println!("> storage directory: {storage_path:?}"); + + // create a runtime that can spawn tasks on a local-thread executors (to support !Send futures) + let rt = iroh::bytes::runtime::Handle::from_currrent(num_cpus::get())?; + + // create a blob store (with a iroh-bytes database inside) + let blobs = BlobStore::new(rt.clone(), storage_path.join("blobs"), endpoint.clone()).await?; + + // create a doc store for the iroh-sync docs + let author = Author::from(keypair.secret().clone()); + let docs = DocStore::new(blobs.clone(), author, storage_path.join("docs")); + + // create the live syncer + let live_sync = LiveSync::spawn(endpoint.clone(), gossip.clone()); // construct the state that is passed to the endpoint loop and from there cloned // into to the connection handler task for incoming connections. let state = Arc::new(State { gossip: gossip.clone(), - replica_store: store.clone(), - db: blobs.db().clone(), - rt, + docs: docs.clone(), + bytes: IrohBytesHandlers::new(rt.clone(), blobs.db().clone()), }); + // spawn our endpoint loop that forwards incoming connections tokio::spawn(endpoint_loop(endpoint.clone(), state)); - // create the live syncer - let sync_handle = LiveSync::spawn(endpoint.clone(), gossip.clone()); - sync_handle - .sync_doc(doc.replica().clone(), peers.clone()) - .await?; - - // spawn an input thread that reads stdin and parses each line as a `Cmd` command - // not using tokio here because they recommend this for "technical reasons" - let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::channel::(1); - std::thread::spawn(move || input_loop(cmd_tx)); + // open our document and add to the live syncer + let namespace = Namespace::from_bytes(topic.as_bytes()); + println!("> opening doc {}", fmt_hash(namespace.id().as_bytes())); + let doc = docs.create_or_open(namespace, DownloadMode::Always).await?; + live_sync.add(doc.replica().clone(), peers.clone()).await?; + // spawn an repl thread that reads stdin and parses each line as a `Cmd` command + let (cmd_tx, mut cmd_rx) = mpsc::channel(1); + std::thread::spawn(move || repl_loop(cmd_tx).expect("input loop crashed")); // process commands in a loop - println!("> ready to accept commands: set | get | ls | exit"); + println!("> ready to accept commands"); + println!("> type `help` for a list of commands"); loop { - let cmd = tokio::select! { - Some(cmd) = cmd_rx.recv() => cmd, - _ = tokio::signal::ctrl_c() => Cmd::Exit - + // wait for a command from the input repl thread + let Some((cmd, to_repl_tx)) = cmd_rx.recv().await else { + break; }; - match cmd { - Cmd::Set { key, value } => { - doc.insert(&key, &author, value.into_bytes().into()).await?; - } - Cmd::Get { key } => { - let entries = doc.replica().all_for_key(key.as_bytes()); - for (_id, entry) in entries { - let content = fmt_content(&doc, &entry).await?; - println!("{} -> {content}", fmt_entry(&entry),); - } - } - Cmd::Ls => { - let all = doc.replica().all(); - println!("> {} entries", all.len()); - for (_id, entry) in all { - println!( - "{} -> {}", - fmt_entry(&entry), - fmt_content(&doc, &entry).await? - ); - } - } - Cmd::Exit => { - break; - } + // exit command: break early + if let Cmd::Exit = cmd { + to_repl_tx.send(ToRepl::Exit).ok(); + break; } + + // handle the command, but select against Ctrl-C signal so that commands can be aborted + tokio::select! { + biased; + _ = tokio::signal::ctrl_c() => { + println!("> aborted"); + } + res = handle_command(cmd, &doc, &log_filter) => if let Err(err) = res { + println!("> error: {err}"); + }, + }; + // notify to the repl that we want to get the next command + to_repl_tx.send(ToRepl::Continue).ok(); } - let res = sync_handle.cancel().await; - if let Err(err) = res { + // exit: cancel the sync and store blob database and document + if let Err(err) = live_sync.cancel().await { println!("> syncer closed with error: {err:?}"); } - println!("> persisting document and blob database at {storage_path:?}"); blobs.save().await?; - save_document(&storage_path, doc.replica()).await?; + docs.save(&doc).await?; Ok(()) } +async fn handle_command(cmd: Cmd, doc: &Doc, log_filter: &LogLevelReload) -> anyhow::Result<()> { + match cmd { + Cmd::Set { key, value } => { + doc.insert_bytes(&key, value.into_bytes().into()).await?; + } + Cmd::Get { key, print_content } => { + let entries = doc.replica().all_for_key(key.as_bytes()); + for (_id, entry) in entries { + println!("{}", fmt_entry(&entry)); + if print_content { + println!("{}", fmt_content(&doc, &entry).await); + } + } + } + Cmd::Ls { prefix } => { + let entries = match prefix { + None => doc.replica().all(), + Some(prefix) => doc.replica().all_with_key_prefix(prefix.as_bytes()), + }; + println!("> {} entries", entries.len()); + for (_id, entry) in entries { + println!("{}", fmt_entry(&entry),); + } + } + Cmd::Log { directive } => { + let next_filter = EnvFilter::from_str(&directive)?; + log_filter.modify(|layer| *layer = next_filter)?; + } + Cmd::Exit => {} + } + Ok(()) +} + +#[derive(Parser)] pub enum Cmd { - Set { key: String, value: String }, - Get { key: String }, - Ls, + /// Set an entry + Set { + /// Key to the entry (parsed as UTF-8 string). + key: String, + /// Content to store for this entry (parsed as UTF-8 string) + value: String, + }, + /// Get entries by key + /// + /// Shows the author, content hash and content length for all entries for this key. + Get { + /// Key to the entry (parsed as UTF-8 string). + key: String, + /// Print the value (but only if it is valid UTF-8 and smaller than 1MB) + #[clap(short = 'c', long)] + print_content: bool, + }, + /// List entries + Ls { + /// Optionally list only entries whose key starts with PREFIX. + prefix: Option, + }, + /// Change the log level + Log { + /// The log level or log filtering directive + /// + /// Valid log levels are: "trace", "debug", "info", "warn", "error" + /// + /// You can also set one or more filtering directives to enable more fine-grained log + /// filtering. The supported filtering directives and their semantics are documented here: + /// https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives + /// + /// To disable logging completely, set to the empty string (via empty double quotes: ""). + #[clap(verbatim_doc_comment)] + directive: String, + }, + /// Quit Exit, } impl FromStr for Cmd { type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let mut parts = s.split(' '); - match [parts.next(), parts.next(), parts.next()] { - [Some("set"), Some(key), Some(value)] => Ok(Self::Set { - key: key.into(), - value: value.into(), - }), - [Some("get"), Some(key), None] => Ok(Self::Get { key: key.into() }), - [Some("ls"), None, None] => Ok(Self::Ls), - [Some("exit"), None, None] => Ok(Self::Exit), - _ => Err(anyhow!("invalid command")), - } + let args = shell_words::split(s)?; + let matches = Cmd::command() + .multicall(true) + .subcommand_required(true) + .try_get_matches_from(args)?; + let cmd = Cmd::from_arg_matches(&matches)?; + Ok(cmd) } } -async fn create_or_open_document( - storage_path: &PathBuf, - blobs: BlobStore, - topic: TopicId, - keypair: &Keypair, -) -> anyhow::Result<(ReplicaStore, Author, Doc)> { - let author = Author::from(keypair.secret().clone()); - let namespace = Namespace::from_bytes(topic.as_bytes()); - let store = ReplicaStore::default(); - - let replica_path = replica_path(storage_path, namespace.id()); - let replica = if replica_path.exists() { - let bytes = tokio::fs::read(replica_path).await?; - store.open_replica(&bytes)? - } else { - store.new_replica(namespace) - }; - - // do some logging - replica.on_insert(Box::new(move |origin, entry| { - println!("> insert from {origin:?}: {}", fmt_entry(&entry)); - })); - - let doc = Doc::new(replica, blobs, DownloadMode::Always); - Ok((store, author, doc)) -} - -async fn save_document(base_path: &PathBuf, replica: &Replica) -> anyhow::Result<()> { - let replica_path = replica_path(base_path, &replica.namespace()); - tokio::fs::create_dir_all(replica_path.parent().unwrap()).await?; - let bytes = replica.to_bytes()?; - tokio::fs::write(replica_path, bytes).await?; - Ok(()) -} - -fn replica_path(storage_path: &PathBuf, namespace: &NamespaceId) -> PathBuf { - storage_path - .join("docs") - .join(hex::encode(namespace.as_bytes())) -} - #[derive(Debug)] struct State { - rt: iroh_bytes::runtime::Handle, gossip: GossipHandle, - replica_store: ReplicaStore, - db: Database, + docs: DocStore, + bytes: IrohBytesHandlers, } async fn endpoint_loop(endpoint: MagicEndpoint, state: Arc) -> anyhow::Result<()> { while let Some(conn) = endpoint.accept().await { - // spawn a new task for each incoming connection. let state = state.clone(); tokio::spawn(async move { if let Err(err) = handle_connection(conn, state).await { @@ -343,96 +355,50 @@ async fn handle_connection(mut conn: quinn::Connecting, state: Arc) -> an println!("> incoming connection with alpn {alpn}"); match alpn.as_bytes() { GOSSIP_ALPN => state.gossip.handle_connection(conn.await?).await, - SYNC_ALPN => iroh::sync::handle_connection(conn, state.replica_store.clone()).await, - alpn if alpn == iroh_bytes::protocol::ALPN => { - handle_iroh_byes_connection(conn, state).await - } + SYNC_ALPN => state.docs.handle_connection(conn).await, + alpn if alpn == iroh_bytes::protocol::ALPN => state.bytes.handle_connection(conn).await, _ => bail!("ignoring connection: unsupported ALPN protocol"), } } -async fn handle_iroh_byes_connection( - conn: quinn::Connecting, - state: Arc, -) -> anyhow::Result<()> { - use iroh_bytes::{ - protocol::{GetRequest, RequestToken}, - provider::{ - CustomGetHandler, EventSender, IrohCollectionParser, RequestAuthorizationHandler, - }, - }; - iroh_bytes::provider::handle_connection( - conn, - state.db.clone(), - NoopEventSender, - IrohCollectionParser, - Arc::new(NoopCustomGetHandler), - Arc::new(NoopRequestAuthorizationHandler), - state.rt.clone(), - ) - .await; - - #[derive(Debug, Clone)] - struct NoopEventSender; - impl EventSender for NoopEventSender { - fn send(&self, _event: iroh_bytes::provider::Event) -> Option { - None - } - } - #[derive(Debug)] - struct NoopCustomGetHandler; - impl CustomGetHandler for NoopCustomGetHandler { - fn handle( - &self, - _token: Option, - _request: Bytes, - ) -> BoxFuture<'static, anyhow::Result> { - async move { Err(anyhow::anyhow!("no custom get handler defined")) }.boxed() - } - } - #[derive(Debug)] - struct NoopRequestAuthorizationHandler; - impl RequestAuthorizationHandler for NoopRequestAuthorizationHandler { - fn authorize( - &self, - token: Option, - _request: &iroh_bytes::protocol::Request, - ) -> BoxFuture<'static, anyhow::Result<()>> { - async move { - if let Some(token) = token { - anyhow::bail!( - "no authorization handler defined, but token was provided: {:?}", - token - ); - } - Ok(()) - } - .boxed() - } - } - Ok(()) -} - -fn create_rt() -> anyhow::Result { - let rt = iroh::bytes::runtime::Handle::from_currrent(num_cpus::get())?; - Ok(rt) +#[derive(Debug)] +enum ToRepl { + Continue, + Exit, } -fn input_loop(line_tx: tokio::sync::mpsc::Sender) -> anyhow::Result<()> { - let mut buffer = String::new(); - let stdin = std::io::stdin(); +fn repl_loop(cmd_tx: mpsc::Sender<(Cmd, oneshot::Sender)>) -> anyhow::Result<()> { + use rustyline::{error::ReadlineError, Config, DefaultEditor}; + let mut rl = DefaultEditor::with_config(Config::builder().check_cursor_position(true).build())?; loop { - stdin.read_line(&mut buffer)?; - let cmd = match Cmd::from_str(buffer.trim()) { - Ok(cmd) => cmd, - Err(err) => { - println!("> failed to parse command: {}", err); - continue; + // prepare a channel to receive a signal from the main thread when a command completed + let (to_repl_tx, to_repl_rx) = oneshot::channel(); + let readline = rl.readline(">> "); + match readline { + Ok(line) if line.is_empty() => continue, + Ok(line) => { + rl.add_history_entry(line.as_str())?; + match Cmd::from_str(&line) { + Ok(cmd) => cmd_tx.blocking_send((cmd, to_repl_tx))?, + Err(err) => { + println!("{err}"); + continue; + } + }; } - }; - line_tx.blocking_send(cmd)?; - buffer.clear(); + Err(ReadlineError::Interrupted | ReadlineError::Eof) => { + cmd_tx.blocking_send((Cmd::Exit, to_repl_tx))?; + } + Err(ReadlineError::WindowResized) => continue, + Err(err) => return Err(err.into()), + } + // wait for reply from main thread + match to_repl_rx.blocking_recv()? { + ToRepl::Continue => continue, + ToRepl::Exit => break, + } } + Ok(()) } #[derive(Debug, Serialize, Deserialize)] @@ -472,6 +438,19 @@ impl FromStr for Ticket { } } +type LogLevelReload = tracing_subscriber::reload::Handle; +fn init_logging() -> LogLevelReload { + use tracing_subscriber::{filter, fmt, prelude::*, reload}; + let filter = filter::EnvFilter::from_default_env(); + let (filter, reload_handle) = reload::Layer::new(filter); + tracing_subscriber::registry() + .with(filter) + .with(fmt::Layer::default()) + .init(); + reload_handle +} + + // helpers fn fmt_entry(entry: &SignedEntry) -> String { @@ -480,20 +459,25 @@ fn fmt_entry(entry: &SignedEntry) -> String { let author = fmt_hash(id.author().as_bytes()); let hash = entry.entry().record().content_hash(); let hash = fmt_hash(hash.as_bytes()); - format!("@{author}: {key} = {hash}") + let len = HumanBytes(entry.entry().record().content_len()); + format!("@{author}: {key} = {hash} ({len})",) } -async fn fmt_content(doc: &Doc, entry: &SignedEntry) -> anyhow::Result { - let content = match doc.get_content(entry).await { - None => "".to_string(), - Some(content) => match String::from_utf8(content.into()) { - Ok(str) => str, - Err(_err) => "".to_string(), - }, - }; - Ok(content) +async fn fmt_content(doc: &Doc, entry: &SignedEntry) -> String { + let len = entry.entry().record().content_len(); + if len > MAX_DISPLAY_CONTENT_LEN { + format!("<{}>", HumanBytes(len)) + } else { + match doc.get_content_bytes(entry).await { + None => "".to_string(), + Some(content) => match String::from_utf8(content.into()) { + Ok(str) => str, + Err(_err) => format!("", HumanBytes(len)), + }, + } + } } -fn fmt_hash(hash: &[u8]) -> String { - let mut text = data_encoding::BASE32_NOPAD.encode(hash); +fn fmt_hash(hash: impl AsRef<[u8]>) -> String { + let mut text = data_encoding::BASE32_NOPAD.encode(hash.as_ref()); text.make_ascii_lowercase(); format!("{}…{}", &text[..5], &text[(text.len() - 2)..]) } @@ -531,3 +515,89 @@ fn derp_map_from_url(url: Url) -> anyhow::Result { 0 )) } + +/// handlers for iroh_bytes connections +mod iroh_bytes_handlers { + use std::sync::Arc; + + use bytes::Bytes; + use futures::{future::BoxFuture, FutureExt}; + use iroh_bytes::{ + protocol::{GetRequest, RequestToken}, + provider::{ + CustomGetHandler, Database, EventSender, IrohCollectionParser, + RequestAuthorizationHandler, + }, + }; + #[derive(Debug, Clone)] + pub struct IrohBytesHandlers { + db: Database, + rt: iroh_bytes::runtime::Handle, + event_sender: NoopEventSender, + get_handler: Arc, + auth_handler: Arc, + } + impl IrohBytesHandlers { + pub fn new(rt: iroh_bytes::runtime::Handle, db: Database) -> Self { + Self { + db, + rt, + event_sender: NoopEventSender, + get_handler: Arc::new(NoopCustomGetHandler), + auth_handler: Arc::new(NoopRequestAuthorizationHandler), + } + } + pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { + iroh_bytes::provider::handle_connection( + conn, + self.db.clone(), + self.event_sender.clone(), + IrohCollectionParser, + self.get_handler.clone(), + self.auth_handler.clone(), + self.rt.clone(), + ) + .await; + Ok(()) + } + } + + #[derive(Debug, Clone)] + struct NoopEventSender; + impl EventSender for NoopEventSender { + fn send(&self, _event: iroh_bytes::provider::Event) -> Option { + None + } + } + #[derive(Debug)] + struct NoopCustomGetHandler; + impl CustomGetHandler for NoopCustomGetHandler { + fn handle( + &self, + _token: Option, + _request: Bytes, + ) -> BoxFuture<'static, anyhow::Result> { + async move { Err(anyhow::anyhow!("no custom get handler defined")) }.boxed() + } + } + #[derive(Debug)] + struct NoopRequestAuthorizationHandler; + impl RequestAuthorizationHandler for NoopRequestAuthorizationHandler { + fn authorize( + &self, + token: Option, + _request: &iroh_bytes::protocol::Request, + ) -> BoxFuture<'static, anyhow::Result<()>> { + async move { + if let Some(token) = token { + anyhow::bail!( + "no authorization handler defined, but token was provided: {:?}", + token + ); + } + Ok(()) + } + .boxed() + } + } +} diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 5a4f1d4489..f061808609 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -13,10 +13,15 @@ use futures::{ }; use iroh_bytes::{provider::Database, util::Hash, writable::WritableFileDatabase}; use iroh_gossip::net::util::Dialer; -use iroh_io::AsyncSliceReaderExt; +use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; use iroh_net::{tls::PeerId, MagicEndpoint}; -use iroh_sync::sync::{Author, InsertOrigin, Replica, SignedEntry}; -use tokio::sync::{mpsc, oneshot}; +use iroh_sync::sync::{ + Author, InsertOrigin, Namespace, NamespaceId, Replica, ReplicaStore, SignedEntry, +}; +use tokio::{ + io::AsyncRead, + sync::{mpsc, oneshot}, +}; use tokio_stream::StreamExt; use tracing::{debug, error, warn}; @@ -26,6 +31,63 @@ pub enum DownloadMode { Manual, } +#[derive(Debug, Clone)] +pub struct DocStore { + replicas: ReplicaStore, + blobs: BlobStore, + local_author: Arc, + storage_path: PathBuf, +} + +impl DocStore { + pub fn new(blobs: BlobStore, author: Author, storage_path: PathBuf) -> Self { + Self { + replicas: ReplicaStore::default(), + local_author: Arc::new(author), + storage_path, + blobs, + } + } + + pub async fn create_or_open( + &self, + namespace: Namespace, + download_mode: DownloadMode, + ) -> anyhow::Result { + let path = self.replica_path(namespace.id()); + let replica = if path.exists() { + let bytes = tokio::fs::read(path).await?; + self.replicas.open_replica(&bytes)? + } else { + self.replicas.new_replica(namespace) + }; + + let doc = Doc::new( + replica, + self.blobs.clone(), + self.local_author.clone(), + download_mode, + ); + Ok(doc) + } + + pub async fn save(&self, doc: &Doc) -> anyhow::Result<()> { + let replica_path = self.replica_path(&doc.replica().namespace()); + tokio::fs::create_dir_all(replica_path.parent().unwrap()).await?; + let bytes = doc.replica().to_bytes()?; + tokio::fs::write(replica_path, bytes).await?; + Ok(()) + } + + fn replica_path(&self, namespace: &NamespaceId) -> PathBuf { + self.storage_path.join(hex::encode(namespace.as_bytes())) + } + + pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { + crate::sync::handle_connection(conn, self.replicas.clone()).await + } +} + /// A replica with a [`BlobStore`] for contents. /// /// This will also download missing content from peers. @@ -33,15 +95,25 @@ pub enum DownloadMode { /// TODO: Currently content is only downloaded from the author of a entry. /// We want to try other peers if the author is offline (or always). /// We'll need some heuristics which peers to try. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Doc { replica: Replica, blobs: BlobStore, + local_author: Arc, } impl Doc { - pub fn new(replica: Replica, blobs: BlobStore, download_mode: DownloadMode) -> Self { - let doc = Self { replica, blobs }; + pub fn new( + replica: Replica, + blobs: BlobStore, + local_author: Arc, + download_mode: DownloadMode, + ) -> Self { + let doc = Self { + replica, + blobs, + local_author, + }; if let DownloadMode::Always = download_mode { let doc2 = doc.clone(); doc.replica.on_insert(Box::new(move |origin, entry| { @@ -57,15 +129,28 @@ impl Doc { &self.replica } - pub async fn insert( + pub fn local_author(&self) -> &Author { + &self.local_author + } + + pub async fn insert_bytes( &self, key: impl AsRef<[u8]>, - author: &Author, content: Bytes, - ) -> anyhow::Result<()> { + ) -> anyhow::Result<(Hash, u64)> { let (hash, len) = self.blobs.put_bytes(content).await?; - self.replica.insert(key, author, hash, len); - Ok(()) + self.replica.insert(key, &self.local_author, hash, len); + Ok((hash, len)) + } + + pub async fn insert_reader( + &self, + key: impl AsRef<[u8]>, + content: impl AsyncRead + Unpin, + ) -> anyhow::Result<(Hash, u64)> { + let (hash, len) = self.blobs.put_reader(content).await?; + self.replica.insert(key, &self.local_author, hash, len); + Ok((hash, len)) } pub fn download_content_fron_author(&self, entry: &SignedEntry) { @@ -75,11 +160,16 @@ impl Doc { self.blobs.start_download(hash, peer_id); } - pub async fn get_content(&self, entry: &SignedEntry) -> Option { + pub async fn get_content_bytes(&self, entry: &SignedEntry) -> Option { let hash = entry.entry().record().content_hash(); let bytes = self.blobs.get_bytes(hash).await.ok().flatten(); bytes } + pub async fn get_content_reader(&self, entry: &SignedEntry) -> Option { + let hash = entry.entry().record().content_hash(); + let bytes = self.blobs.get_reader(hash).await.ok().flatten(); + bytes + } } /// A blob database that can download missing blobs from peers. @@ -129,9 +219,22 @@ impl BlobStore { Ok(Some(bytes)) } + pub async fn get_reader(&self, hash: &Hash) -> anyhow::Result> { + self.downloader.wait_for_download(hash).await; + let Some(entry) = self.db().get(hash) else { + return Ok(None) + }; + let reader = entry.data_reader().await?; + Ok(Some(reader)) + } + pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64)> { self.db.put_bytes(data).await } + + pub async fn put_reader(&self, data: impl AsyncRead + Unpin) -> anyhow::Result<(Hash, u64)> { + self.db.put_reader(data).await + } } pub type DownloadReply = oneshot::Sender>; @@ -206,6 +309,7 @@ impl Downloader { } } +#[derive(Debug)] pub struct DownloadActor { dialer: Dialer, db: WritableFileDatabase, diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index e11948d3aa..e8ba2583f7 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -80,7 +80,7 @@ impl LiveSync { Ok(()) } - pub async fn sync_doc(&self, doc: Replica, initial_peers: Vec) -> Result<()> { + pub async fn add(&self, doc: Replica, initial_peers: Vec) -> Result<()> { self.to_actor_tx .send(ToActor::SyncDoc { doc, initial_peers }) .await?; @@ -201,9 +201,16 @@ impl Actor { async fn insert_doc(&mut self, doc: Replica, initial_peers: Vec) -> Result<()> { let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id).collect(); - let topic: TopicId = doc.namespace().as_bytes().into(); + + // add addresses of initial peers to our endpoint address book + for peer in &initial_peers { + self.endpoint + .add_known_addrs(peer.peer_id, &peer.addrs) + .await?; + } + // join gossip for the topic to receive and send message - // let gossip = self.gossip.clone(); + let topic: TopicId = doc.namespace().as_bytes().into(); self.pending_joins.push({ let peer_ids = peer_ids.clone(); let gossip = self.gossip.clone(); @@ -213,6 +220,7 @@ impl Actor { } .boxed() }); + // setup replica insert notifications. let insert_entry_tx = self.insert_entry_tx.clone(); doc.on_insert(Box::new(move |origin, entry| { @@ -227,7 +235,7 @@ impl Actor { self.endpoint .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) .await?; - } + // trigger initial sync with initial peers for peer in peer_ids { self.sync_with_peer(topic, peer); From 607fd818bb2bd09a3dca70f2099349021a371f8f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 13 Jul 2023 17:42:04 +0200 Subject: [PATCH 009/172] feat(example-sync): add ticket command --- iroh/examples/sync.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 3ec5ec6ddb..ef6b917ca8 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -225,7 +225,7 @@ async fn run(args: Args) -> anyhow::Result<()> { _ = tokio::signal::ctrl_c() => { println!("> aborted"); } - res = handle_command(cmd, &doc, &log_filter) => if let Err(err) = res { + res = handle_command(cmd, &doc, &our_ticket, &log_filter) => if let Err(err) = res { println!("> error: {err}"); }, }; @@ -244,7 +244,7 @@ async fn run(args: Args) -> anyhow::Result<()> { Ok(()) } -async fn handle_command(cmd: Cmd, doc: &Doc, log_filter: &LogLevelReload) -> anyhow::Result<()> { +async fn handle_command(cmd: Cmd, doc: &Doc, ticket: &Ticket, log_filter: &LogLevelReload) -> anyhow::Result<()> { match cmd { Cmd::Set { key, value } => { doc.insert_bytes(&key, value.into_bytes().into()).await?; @@ -268,6 +268,9 @@ async fn handle_command(cmd: Cmd, doc: &Doc, log_filter: &LogLevelReload) -> any println!("{}", fmt_entry(&entry),); } } + Cmd::Ticket => { + println!("Ticket: {ticket}"); + } Cmd::Log { directive } => { let next_filter = EnvFilter::from_str(&directive)?; log_filter.modify(|layer| *layer = next_filter)?; @@ -301,6 +304,8 @@ pub enum Cmd { /// Optionally list only entries whose key starts with PREFIX. prefix: Option, }, + /// Print the ticket with which other peers can join our document. + Ticket, /// Change the log level Log { /// The log level or log filtering directive @@ -450,7 +455,6 @@ fn init_logging() -> LogLevelReload { reload_handle } - // helpers fn fmt_entry(entry: &SignedEntry) -> String { From a0f96db9053fd6e68bb3127772a2ba9a5760366e Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 26 Jul 2023 23:34:31 +0200 Subject: [PATCH 010/172] fix: adapt to changes on main after rebase --- iroh-bytes/src/lib.rs | 1 - iroh/examples/sync.rs | 27 +++++++++++-------- .../src/database/flat}/writable.rs | 5 ++-- iroh/src/sync/content.rs | 8 +++--- iroh/src/sync/live.rs | 3 ++- 5 files changed, 26 insertions(+), 18 deletions(-) rename {iroh-bytes/src => iroh/src/database/flat}/writable.rs (98%) diff --git a/iroh-bytes/src/lib.rs b/iroh-bytes/src/lib.rs index 369c3dee7c..ea257d1d11 100644 --- a/iroh-bytes/src/lib.rs +++ b/iroh-bytes/src/lib.rs @@ -9,7 +9,6 @@ pub mod get; pub mod protocol; pub mod provider; pub mod util; -pub mod writable; #[cfg(test)] pub(crate) mod test_utils; diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index ef6b917ca8..147280bb82 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -173,7 +173,7 @@ async fn run(args: Args) -> anyhow::Result<()> { println!("> storage directory: {storage_path:?}"); // create a runtime that can spawn tasks on a local-thread executors (to support !Send futures) - let rt = iroh::bytes::runtime::Handle::from_currrent(num_cpus::get())?; + let rt = iroh_bytes::util::runtime::Handle::from_currrent(num_cpus::get())?; // create a blob store (with a iroh-bytes database inside) let blobs = BlobStore::new(rt.clone(), storage_path.join("blobs"), endpoint.clone()).await?; @@ -244,7 +244,12 @@ async fn run(args: Args) -> anyhow::Result<()> { Ok(()) } -async fn handle_command(cmd: Cmd, doc: &Doc, ticket: &Ticket, log_filter: &LogLevelReload) -> anyhow::Result<()> { +async fn handle_command( + cmd: Cmd, + doc: &Doc, + ticket: &Ticket, + log_filter: &LogLevelReload, +) -> anyhow::Result<()> { match cmd { Cmd::Set { key, value } => { doc.insert_bytes(&key, value.into_bytes().into()).await?; @@ -516,7 +521,7 @@ fn derp_map_from_url(url: Url) -> anyhow::Result { DEFAULT_DERP_STUN_PORT, UseIpv4::TryDns, UseIpv6::TryDns, - 0 + 0, )) } @@ -528,21 +533,21 @@ mod iroh_bytes_handlers { use futures::{future::BoxFuture, FutureExt}; use iroh_bytes::{ protocol::{GetRequest, RequestToken}, - provider::{ - CustomGetHandler, Database, EventSender, IrohCollectionParser, - RequestAuthorizationHandler, - }, + provider::{CustomGetHandler, EventSender, RequestAuthorizationHandler}, }; + + use iroh::{collection::IrohCollectionParser, database::flat::Database}; + #[derive(Debug, Clone)] pub struct IrohBytesHandlers { db: Database, - rt: iroh_bytes::runtime::Handle, + rt: iroh_bytes::util::runtime::Handle, event_sender: NoopEventSender, get_handler: Arc, auth_handler: Arc, } impl IrohBytesHandlers { - pub fn new(rt: iroh_bytes::runtime::Handle, db: Database) -> Self { + pub fn new(rt: iroh_bytes::util::runtime::Handle, db: Database) -> Self { Self { db, rt, @@ -569,8 +574,8 @@ mod iroh_bytes_handlers { #[derive(Debug, Clone)] struct NoopEventSender; impl EventSender for NoopEventSender { - fn send(&self, _event: iroh_bytes::provider::Event) -> Option { - None + fn send(&self, _event: iroh_bytes::provider::Event) -> BoxFuture<()> { + async {}.boxed() } } #[derive(Debug)] diff --git a/iroh-bytes/src/writable.rs b/iroh/src/database/flat/writable.rs similarity index 98% rename from iroh-bytes/src/writable.rs rename to iroh/src/database/flat/writable.rs index d1666831da..aa9bb36a6e 100644 --- a/iroh-bytes/src/writable.rs +++ b/iroh/src/database/flat/writable.rs @@ -12,13 +12,14 @@ use iroh_io::{AsyncSliceWriter, File}; use range_collections::RangeSet2; use tokio::io::AsyncRead; -use crate::{ +use iroh_bytes::{ get::fsm, protocol::{GetRequest, RangeSpecSeq, Request}, - provider::{create_collection, DataSource, Database, DbEntry, FNAME_PATHS}, Hash, }; +use crate::database::flat::{create_collection, DataSource, Database, DbEntry, FNAME_PATHS}; + /// A blob database into which new blobs can be inserted. /// /// Blobs can be inserted either from bytes or by downloading from open connections to peers. diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index f061808609..3f396e37c6 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -11,7 +11,7 @@ use futures::{ stream::FuturesUnordered, FutureExt, }; -use iroh_bytes::{provider::Database, util::Hash, writable::WritableFileDatabase}; +use iroh_bytes::util::Hash; use iroh_gossip::net::util::Dialer; use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; use iroh_net::{tls::PeerId, MagicEndpoint}; @@ -25,6 +25,8 @@ use tokio::{ use tokio_stream::StreamExt; use tracing::{debug, error, warn}; +use crate::database::flat::{writable::WritableFileDatabase, Database}; + #[derive(Debug, Copy, Clone)] pub enum DownloadMode { Always, @@ -187,7 +189,7 @@ pub struct BlobStore { } impl BlobStore { pub async fn new( - rt: iroh_bytes::runtime::Handle, + rt: iroh_bytes::util::runtime::Handle, data_path: PathBuf, endpoint: MagicEndpoint, ) -> anyhow::Result { @@ -263,7 +265,7 @@ pub struct Downloader { impl Downloader { pub fn new( - rt: iroh_bytes::runtime::Handle, + rt: iroh_bytes::util::runtime::Handle, endpoint: MagicEndpoint, blobs: WritableFileDatabase, ) -> Self { diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index e8ba2583f7..c2c961e660 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -205,7 +205,7 @@ impl Actor { // add addresses of initial peers to our endpoint address book for peer in &initial_peers { self.endpoint - .add_known_addrs(peer.peer_id, &peer.addrs) + .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) .await?; } @@ -235,6 +235,7 @@ impl Actor { self.endpoint .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) .await?; + } // trigger initial sync with initial peers for peer in peer_ids { From e0e062c4a8e2cc15f56303d60e905e231398ee65 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 26 Jul 2023 21:00:09 +0200 Subject: [PATCH 011/172] example(sync): add watch command --- iroh/examples/sync.rs | 34 +++++++++++++++++++++++++++++++++- iroh/src/sync/content.rs | 7 ++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 147280bb82..26a44bf032 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -208,6 +208,8 @@ async fn run(args: Args) -> anyhow::Result<()> { // process commands in a loop println!("> ready to accept commands"); println!("> type `help` for a list of commands"); + + let mut current_watch = Arc::new(std::sync::Mutex::new(None)); loop { // wait for a command from the input repl thread let Some((cmd, to_repl_tx)) = cmd_rx.recv().await else { @@ -225,7 +227,7 @@ async fn run(args: Args) -> anyhow::Result<()> { _ = tokio::signal::ctrl_c() => { println!("> aborted"); } - res = handle_command(cmd, &doc, &our_ticket, &log_filter) => if let Err(err) = res { + res = handle_command(cmd, &doc, &our_ticket, &log_filter, ¤t_watch) => if let Err(err) = res { println!("> error: {err}"); }, }; @@ -249,6 +251,7 @@ async fn handle_command( doc: &Doc, ticket: &Ticket, log_filter: &LogLevelReload, + current_watch: &Arc>>, ) -> anyhow::Result<()> { match cmd { Cmd::Set { key, value } => { @@ -263,6 +266,28 @@ async fn handle_command( } } } + Cmd::Watch { key } => { + println!("watching key: '{key}'"); + current_watch.lock().unwrap().replace(key); + let watch = current_watch.clone(); + doc.on_insert(Box::new(move |origin, entry| { + let matcher = watch.lock().unwrap(); + if let Some(matcher) = &*matcher { + let key = entry.entry().id().key(); + if key.starts_with(matcher.as_bytes()) { + println!("change: {}", fmt_entry(&entry)); + } + } + })); + } + Cmd::WatchCancel => match current_watch.lock().unwrap().take() { + Some(key) => { + println!("canceled watching key: '{key}'"); + } + None => { + println!("no watch active"); + } + }, Cmd::Ls { prefix } => { let entries = match prefix { None => doc.replica().all(), @@ -325,6 +350,13 @@ pub enum Cmd { #[clap(verbatim_doc_comment)] directive: String, }, + /// Watch for changes. + Watch { + /// The key to watch. + key: String, + }, + /// Cancels any running watch command. + WatchCancel, /// Quit Exit, } diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 3f396e37c6..3d48424749 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -16,7 +16,8 @@ use iroh_gossip::net::util::Dialer; use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::sync::{ - Author, InsertOrigin, Namespace, NamespaceId, Replica, ReplicaStore, SignedEntry, + Author, InsertOrigin, Namespace, NamespaceId, OnInsertCallback, Replica, ReplicaStore, + SignedEntry, }; use tokio::{ io::AsyncRead, @@ -127,6 +128,10 @@ impl Doc { doc } + pub fn on_insert(&self, callback: OnInsertCallback) { + self.replica.on_insert(callback); + } + pub fn replica(&self) -> &Replica { &self.replica } From 6347541d1662ddd9822e94406164905585b1affd Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 26 Jul 2023 21:06:17 +0200 Subject: [PATCH 012/172] fixup --- iroh/examples/sync.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 26a44bf032..87ebdc2f4d 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -209,7 +209,19 @@ async fn run(args: Args) -> anyhow::Result<()> { println!("> ready to accept commands"); println!("> type `help` for a list of commands"); - let mut current_watch = Arc::new(std::sync::Mutex::new(None)); + let mut current_watch: Arc>> = + Arc::new(std::sync::Mutex::new(None)); + let watch = current_watch.clone(); + doc.on_insert(Box::new(move |origin, entry| { + let matcher = watch.lock().unwrap(); + if let Some(matcher) = &*matcher { + let key = entry.entry().id().key(); + if key.starts_with(matcher.as_bytes()) { + println!("change: {}", fmt_entry(&entry)); + } + } + })); + loop { // wait for a command from the input repl thread let Some((cmd, to_repl_tx)) = cmd_rx.recv().await else { @@ -269,16 +281,6 @@ async fn handle_command( Cmd::Watch { key } => { println!("watching key: '{key}'"); current_watch.lock().unwrap().replace(key); - let watch = current_watch.clone(); - doc.on_insert(Box::new(move |origin, entry| { - let matcher = watch.lock().unwrap(); - if let Some(matcher) = &*matcher { - let key = entry.entry().id().key(); - if key.starts_with(matcher.as_bytes()) { - println!("change: {}", fmt_entry(&entry)); - } - } - })); } Cmd::WatchCancel => match current_watch.lock().unwrap().take() { Some(key) => { From 82d4589cacf1440f6dd5584054c2b5b722da2e1a Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 26 Jul 2023 23:44:54 +0200 Subject: [PATCH 013/172] chore: unused variables --- iroh/examples/sync.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 87ebdc2f4d..5df2db1772 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -209,10 +209,10 @@ async fn run(args: Args) -> anyhow::Result<()> { println!("> ready to accept commands"); println!("> type `help` for a list of commands"); - let mut current_watch: Arc>> = + let current_watch: Arc>> = Arc::new(std::sync::Mutex::new(None)); let watch = current_watch.clone(); - doc.on_insert(Box::new(move |origin, entry| { + doc.on_insert(Box::new(move |_origin, entry| { let matcher = watch.lock().unwrap(); if let Some(matcher) = &*matcher { let key = entry.entry().id().key(); From 94163c7f3d4c9e8c6b2fbcc40911fb30a7e2540a Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 11:39:27 +0200 Subject: [PATCH 014/172] fix tests and warnings --- iroh-sync/src/ranger.rs | 16 ++++--- iroh-sync/src/sync.rs | 94 ++++++++++++++++++++++------------------- iroh/src/sync.rs | 4 +- 3 files changed, 62 insertions(+), 52 deletions(-) diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index d5310839a6..665751d248 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -401,12 +401,14 @@ where /// Processes an incoming message and produces a response. /// If terminated, returns `None` - pub fn process_message(&mut self, message: Message) -> (Vec, Option>) { + pub fn process_message(&mut self, message: Message, cb: F) -> Option> + where + F: Fn(K, V), + { let mut out = Vec::new(); // TODO: can these allocs be avoided? let mut items = Vec::new(); - let mut inserted = Vec::new(); let mut fingerprints = Vec::new(); for part in message.parts { match part { @@ -440,7 +442,7 @@ where // Store incoming values for (k, v) in values { - inserted.push(k.clone()); + cb(k.clone(), v.clone()); self.store.put(k, v); } @@ -547,9 +549,9 @@ where // If we have any parts, return a message if !out.is_empty() { - (inserted, Some(Message { parts: out })) + Some(Message { parts: out }) } else { - (inserted, None) + None } } @@ -1101,9 +1103,9 @@ mod tests { rounds += 1; alice_to_bob.push(msg.clone()); - if let Some(msg) = bob.process_message(msg) { + if let Some(msg) = bob.process_message(msg, |_, _| {}) { bob_to_alice.push(msg.clone()); - next_to_bob = alice.process_message(msg); + next_to_bob = alice.process_message(msg, |_, _| {}); } } let res = SyncResult { diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index a6f05391bb..7cfc9ce60b 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -241,14 +241,14 @@ pub enum InsertOrigin { #[derive(derive_more::Debug, Clone)] pub struct Replica { inner: Arc>, + #[debug("on_insert: [Box; {}]", "self.on_insert.len()")] + on_insert: Arc>>, } #[derive(derive_more::Debug)] struct InnerReplica { namespace: Namespace, peer: Peer, - #[debug("on_insert: [Box; {}]", "self.on_insert.len()")] - on_insert: Vec, } #[derive(Default, Debug, Clone)] @@ -388,14 +388,14 @@ impl Replica { inner: Arc::new(RwLock::new(InnerReplica { namespace, peer: Peer::default(), - on_insert: Default::default(), })), + on_insert: Default::default(), } } pub fn on_insert(&self, callback: OnInsertCallback) { - let mut inner = self.inner.write(); - inner.on_insert.push(callback); + let mut on_insert = self.on_insert.write(); + on_insert.push(callback); } // TODO: not horrible @@ -454,13 +454,31 @@ impl Replica { // Store signed entries let entry = Entry::new(id.clone(), record); - let signed_entry = entry.sign(&inner.namespace, author); - inner.peer.put(id, signed_entry.clone()); - for cb in &inner.on_insert { + let signed_entry = entry.sign(&inner.namespace, author).clone(); + inner.peer.put(id.clone(), signed_entry.clone()); + drop(inner); + let on_insert = self.on_insert.read(); + for cb in &*on_insert { cb(InsertOrigin::Local, signed_entry.clone()); } } + /// Hashes the given data and inserts it. + /// This does not store the content, just the record of it. + /// + /// Returns the calculated hash. + pub fn hash_and_insert( + &self, + key: impl AsRef<[u8]>, + author: &Author, + data: impl AsRef<[u8]>, + ) -> Hash { + let len = data.as_ref().len() as u64; + let hash = Hash::new(data); + self.insert(key, author, hash, len); + hash + } + pub fn id(&self, key: impl AsRef<[u8]>, author: &Author) -> RecordIdentifier { let inner = self.inner.read(); let id = RecordIdentifier::new(key, inner.namespace.id(), author.id()); @@ -470,9 +488,12 @@ impl Replica { pub fn insert_remote_entry(&self, entry: SignedEntry) -> anyhow::Result<()> { entry.verify()?; let mut inner = self.inner.write(); - inner.peer.put(entry.entry.id.clone(), entry.clone()); - for cb in &inner.on_insert { - cb(InsertOrigin::Sync, entry.clone()) + let id = entry.entry.id.clone(); + inner.peer.put(id.clone(), entry.clone()); + drop(inner); + let on_insert = self.on_insert.read(); + for cb in &*on_insert { + cb(InsertOrigin::Sync, entry.clone()); } Ok(()) } @@ -511,14 +532,17 @@ impl Replica { &self, message: crate::ranger::Message, ) -> Option> { - let (inserted_keys, reply) = self.inner.write().peer.process_message(message); - let inner = self.inner.read(); - for key in inserted_keys { - let entry = inner.peer.get(&key).unwrap(); - for cb in &inner.on_insert { - cb(InsertOrigin::Sync, entry.clone()) - } - } + let reply = self + .inner + .write() + .peer + .process_message(message, |_key, entry| { + let on_insert = self.on_insert.read(); + for cb in &*on_insert { + cb(InsertOrigin::Sync, entry.clone()); + } + }); + reply } @@ -817,7 +841,7 @@ mod tests { let my_replica = Replica::new(myspace); for i in 0..10 { - my_replica.insert(format!("/{i}"), &alice, format!("{i}: hello from alice")); + my_replica.hash_and_insert(format!("/{i}"), &alice, format!("{i}: hello from alice")); } for i in 0..10 { @@ -828,33 +852,16 @@ mod tests { } // Test multiple records for the same key - my_replica.insert("/cool/path", &alice, "round 1"); - let entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); - let content = my_replica - .get_content(entry.entry().record().content_hash()) - .unwrap(); - assert_eq!(&content[..], b"round 1"); + my_replica.hash_and_insert("/cool/path", &alice, "round 1"); + let _entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); // Second - - my_replica.insert("/cool/path", &alice, "round 2"); - let entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); - let content = my_replica - .get_content(entry.entry().record().content_hash()) - .unwrap(); - assert_eq!(&content[..], b"round 2"); + my_replica.hash_and_insert("/cool/path", &alice, "round 2"); + let _entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); // Get All let entries: Vec<_> = my_replica.get_all("/cool/path", alice.id()).collect(); assert_eq!(entries.len(), 2); - let content = my_replica - .get_content(entries[0].entry().record().content_hash()) - .unwrap(); - assert_eq!(&content[..], b"round 1"); - let content = my_replica - .get_content(entries[1].entry().record().content_hash()) - .unwrap(); - assert_eq!(&content[..], b"round 2"); } #[test] @@ -928,12 +935,12 @@ mod tests { let myspace = Namespace::new(&mut rng); let mut alice = Replica::new(myspace.clone()); for el in &alice_set { - alice.insert(el, &author, el.as_bytes()); + alice.hash_and_insert(el, &author, el.as_bytes()); } let mut bob = Replica::new(myspace); for el in &bob_set { - bob.insert(el, &author, el.as_bytes()); + bob.hash_and_insert(el, &author, el.as_bytes()); } sync(&author, &mut alice, &mut bob, &alice_set, &bob_set); @@ -952,6 +959,7 @@ mod tests { while let Some(msg) = next_to_bob.take() { assert!(rounds < 100, "too many rounds"); rounds += 1; + println!("round {}", rounds); if let Some(msg) = bob.sync_process_message(msg) { next_to_bob = alice.sync_process_message(msg); } diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index cf9f8e0fd0..bc396b58a7 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -185,10 +185,10 @@ mod tests { let author = replica_store.new_author(&mut rng); let namespace = Namespace::new(&mut rng); let bob_replica = replica_store.new_replica(namespace.clone()); - bob_replica.insert("hello alice", &author, "from bob"); + bob_replica.hash_and_insert("hello alice", &author, "from bob"); let alice_replica = Replica::new(namespace.clone()); - alice_replica.insert("hello bob", &author, "from alice"); + alice_replica.hash_and_insert("hello bob", &author, "from alice"); assert_eq!(bob_replica.all().len(), 1); assert_eq!(alice_replica.all().len(), 1); From 7240fc58926582454441835b38351d5fbe929d7c Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 11:47:06 +0200 Subject: [PATCH 015/172] clippy cleanups --- iroh-sync/src/ranger.rs | 16 ++-------------- iroh-sync/src/sync.rs | 12 ++++++------ iroh/examples/sync.rs | 2 +- iroh/src/database/flat/writable.rs | 13 +++++++++---- iroh/src/sync/content.rs | 20 +++++++++----------- 5 files changed, 27 insertions(+), 36 deletions(-) diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index 665751d248..ea80f49f9e 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -214,8 +214,7 @@ where V: 'a; /// Returns all items in the given range - fn get_range<'a>(&'a self, range: Range, limit: Option>) - -> Self::RangeIterator<'a>; + fn get_range(&self, range: Range, limit: Option>) -> Self::RangeIterator<'_>; fn remove(&mut self, key: &K) -> Option; type AllIterator<'a>: Iterator @@ -282,11 +281,7 @@ where type RangeIterator<'a> = SimpleRangeIterator<'a, K, V> where K: 'a, V: 'a; /// Returns all items in the given range - fn get_range<'a>( - &'a self, - range: Range, - limit: Option>, - ) -> Self::RangeIterator<'a> { + fn get_range(&self, range: Range, limit: Option>) -> Self::RangeIterator<'_> { // TODO: this is not very efficient, optimize depending on data structure let iter = self.data.iter(); @@ -1179,14 +1174,12 @@ mod tests { let all: Vec<_> = store .get_range(Range::new("", ""), None) - .into_iter() .map(|(k, v)| (*k, *v)) .collect(); assert_eq!(&all, &set[..]); let regular: Vec<_> = store .get_range(("bee", "eel").into(), None) - .into_iter() .map(|(k, v)| (*k, *v)) .collect(); assert_eq!(®ular, &set[..3]); @@ -1194,21 +1187,18 @@ mod tests { // empty start let regular: Vec<_> = store .get_range(("", "eel").into(), None) - .into_iter() .map(|(k, v)| (*k, *v)) .collect(); assert_eq!(®ular, &set[..3]); let regular: Vec<_> = store .get_range(("cat", "hog").into(), None) - .into_iter() .map(|(k, v)| (*k, *v)) .collect(); assert_eq!(®ular, &set[1..5]); let excluded: Vec<_> = store .get_range(("fox", "bee").into(), None) - .into_iter() .map(|(k, v)| (*k, *v)) .collect(); @@ -1218,7 +1208,6 @@ mod tests { let excluded: Vec<_> = store .get_range(("fox", "doe").into(), None) - .into_iter() .map(|(k, v)| (*k, *v)) .collect(); @@ -1231,7 +1220,6 @@ mod tests { // Limit let all: Vec<_> = store .get_range(("", "").into(), Some(("bee", "doe").into())) - .into_iter() .map(|(k, v)| (*k, *v)) .collect(); assert_eq!(&all, &set[..2]); diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 7cfc9ce60b..baed4e678d 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -315,11 +315,11 @@ impl crate::ranger::Store for Store { } type RangeIterator<'a> = RangeIterator<'a>; - fn get_range<'a>( - &'a self, + fn get_range( + &self, range: Range, limit: Option>, - ) -> Self::RangeIterator<'a> { + ) -> Self::RangeIterator<'_> { RangeIterator { iter: self.records.iter(), range: Some(range), @@ -454,8 +454,8 @@ impl Replica { // Store signed entries let entry = Entry::new(id.clone(), record); - let signed_entry = entry.sign(&inner.namespace, author).clone(); - inner.peer.put(id.clone(), signed_entry.clone()); + let signed_entry = entry.sign(&inner.namespace, author); + inner.peer.put(id, signed_entry.clone()); drop(inner); let on_insert = self.on_insert.read(); for cb in &*on_insert { @@ -489,7 +489,7 @@ impl Replica { entry.verify()?; let mut inner = self.inner.write(); let id = entry.entry.id.clone(); - inner.peer.put(id.clone(), entry.clone()); + inner.peer.put(id, entry.clone()); drop(inner); let on_insert = self.on_insert.read(); for cb in &*on_insert { diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 5df2db1772..25a8e839c0 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -274,7 +274,7 @@ async fn handle_command( for (_id, entry) in entries { println!("{}", fmt_entry(&entry)); if print_content { - println!("{}", fmt_content(&doc, &entry).await); + println!("{}", fmt_content(doc, &entry).await); } } } diff --git a/iroh/src/database/flat/writable.rs b/iroh/src/database/flat/writable.rs index aa9bb36a6e..8f933813ee 100644 --- a/iroh/src/database/flat/writable.rs +++ b/iroh/src/database/flat/writable.rs @@ -4,7 +4,12 @@ //! I wrote this while diving into iroh-bytes, wildly copying code around. This will be solved much //! nicer with the upcoming generic writable database branch by @rklaehn. -use std::{collections::HashMap, io, path::PathBuf, sync::Arc}; +use std::{ + collections::HashMap, + io, + path::{Path, PathBuf}, + sync::Arc, +}; use anyhow::Context; use bytes::Bytes; @@ -69,13 +74,13 @@ impl WritableFileDatabase { } pub async fn put_from_temp_file(&self, temp_path: &PathBuf) -> anyhow::Result<(Hash, u64)> { - let (hash, size, entry) = self.storage.move_to_blobs(&temp_path).await?; + let (hash, size, entry) = self.storage.move_to_blobs(temp_path).await?; self.db.union_with(HashMap::from_iter([(hash, entry)])); Ok((hash, size)) } pub async fn get_size(&self, hash: &Hash) -> Option { - Some(self.db.get(&hash)?.size().await) + Some(self.db.get(hash)?.size().await) } pub fn has(&self, hash: &Hash) -> bool { @@ -193,7 +198,7 @@ impl StoragePaths { } } -async fn prepare_hash_dir(path: &PathBuf, hash: &Hash) -> anyhow::Result { +async fn prepare_hash_dir(path: &Path, hash: &Hash) -> anyhow::Result { let hash = hex::encode(hash.as_ref()); let path = path.join(&hash[0..2]).join(&hash[2..4]).join(&hash[4..]); tokio::fs::create_dir_all(path.parent().unwrap()).await?; diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 3d48424749..de43f69dfc 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -169,13 +169,12 @@ impl Doc { pub async fn get_content_bytes(&self, entry: &SignedEntry) -> Option { let hash = entry.entry().record().content_hash(); - let bytes = self.blobs.get_bytes(hash).await.ok().flatten(); - bytes + self.blobs.get_bytes(hash).await.ok().flatten() } + pub async fn get_content_reader(&self, entry: &SignedEntry) -> Option { let hash = entry.entry().record().content_hash(); - let bytes = self.blobs.get_reader(hash).await.ok().flatten(); - bytes + self.blobs.get_reader(hash).await.ok().flatten() } } @@ -208,7 +207,7 @@ impl BlobStore { } pub fn db(&self) -> &Database { - &self.db.db() + self.db.db() } pub fn start_download(&self, hash: Hash, peer: PeerId) { @@ -378,21 +377,21 @@ impl DownloadActor { fn reply(&mut self, hash: Hash, res: Option<(Hash, u64)>) { for reply in self.replies.remove(&hash).into_iter().flatten() { - reply.send(res.clone()).ok(); + reply.send(res).ok(); } } fn on_peer_fail(&mut self, peer: &PeerId, err: anyhow::Error) { warn!("download from {peer} failed: {err}"); - for hash in self.peer_hashes.remove(&peer).into_iter().flatten() { + for hash in self.peer_hashes.remove(peer).into_iter().flatten() { self.on_not_found(peer, hash); } - self.conns.remove(&peer); + self.conns.remove(peer); } fn on_not_found(&mut self, peer: &PeerId, hash: Hash) { if let Some(peers) = self.hash_peers.get_mut(&hash) { - peers.remove(&peer); + peers.remove(peer); if peers.is_empty() { self.reply(hash, None); self.hash_peers.remove(&hash); @@ -404,8 +403,7 @@ impl DownloadActor { if let Some(hash) = self .peer_hashes .get_mut(&peer) - .map(|hashes| hashes.pop_front()) - .flatten() + .and_then(|hashes| hashes.pop_front()) { let conn = self.conns.get(&peer).unwrap().clone(); let blobs = self.db.clone(); From 069a53b27a6e2b514e7b7c2929f8f883036a54f9 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 27 Jul 2023 12:35:25 +0200 Subject: [PATCH 016/172] feat(iroh): metrics for iroh-sync --- iroh/examples/sync.rs | 40 +++++++++++++++++++++++++++++++++++ iroh/src/sync.rs | 2 ++ iroh/src/sync/content.rs | 43 +++++++++++++++++++++++++++++++++++--- iroh/src/sync/live.rs | 21 ++++++++++++++++++- iroh/src/sync/metrics.rs | 45 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 iroh/src/sync/metrics.rs diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 25a8e839c0..ec780768e2 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -18,6 +18,10 @@ use iroh_gossip::{ net::{GossipHandle, GOSSIP_ALPN}, proto::TopicId, }; +use iroh_metrics::{ + core::{Counter, Metric}, + struct_iterable::Iterable, +}; use iroh_net::{ defaults::{default_derp_map, DEFAULT_DERP_STUN_PORT}, derp::{DerpMap, UseIpv4, UseIpv6}, @@ -73,7 +77,15 @@ async fn main() -> anyhow::Result<()> { } async fn run(args: Args) -> anyhow::Result<()> { + // setup logging let log_filter = init_logging(); + + // init metrics + iroh_metrics::core::Core::init(|reg, metrics| { + metrics.insert(iroh::sync::metrics::Metrics::new(reg)); + metrics.insert(iroh_gossip::metrics::Metrics::new(reg)); + }); + // parse or generate our keypair let keypair = match args.private_key { None => Keypair::generate(), @@ -307,6 +319,7 @@ async fn handle_command( let next_filter = EnvFilter::from_str(&directive)?; log_filter.modify(|layer| *layer = next_filter)?; } + Cmd::Stats => get_stats(), Cmd::Exit => {} } Ok(()) @@ -359,6 +372,8 @@ pub enum Cmd { }, /// Cancels any running watch command. WatchCancel, + /// Show stats about the current session + Stats, /// Quit Exit, } @@ -445,6 +460,31 @@ fn repl_loop(cmd_tx: mpsc::Sender<(Cmd, oneshot::Sender)>) -> anyhow::Re Ok(()) } +fn get_stats() { + let core = iroh_metrics::core::Core::get().expect("Metrics core not initialized"); + println!("# sync"); + let metrics = core + .get_collector::() + .unwrap(); + fmt_metrics(metrics); + println!("# gossip"); + let metrics = core + .get_collector::() + .unwrap(); + fmt_metrics(metrics); +} + +fn fmt_metrics(metrics: &impl Iterable) { + for (name, counter) in metrics.iter() { + if let Some(counter) = counter.downcast_ref::() { + let value = counter.get(); + println!("{name:23} : {value:>6} ({})", counter.description); + } else { + println!("{name:23} : unsupported metric kind"); + } + } +} + #[derive(Debug, Serialize, Deserialize)] struct Ticket { topic: TopicId, diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index bc396b58a7..d5b92516b8 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -15,6 +15,8 @@ pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; mod content; mod live; +pub mod metrics; + pub use content::*; pub use live::*; diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index de43f69dfc..f5942eb193 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -3,6 +3,7 @@ use std::{ io, path::PathBuf, sync::{Arc, Mutex}, + time::Instant, }; use bytes::Bytes; @@ -14,6 +15,7 @@ use futures::{ use iroh_bytes::util::Hash; use iroh_gossip::net::util::Dialer; use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; +use iroh_metrics::{inc, inc_by}; use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::sync::{ Author, InsertOrigin, Namespace, NamespaceId, OnInsertCallback, Replica, ReplicaStore, @@ -26,6 +28,7 @@ use tokio::{ use tokio_stream::StreamExt; use tracing::{debug, error, warn}; +use super::metrics::Metrics; use crate::database::flat::{writable::WritableFileDatabase, Database}; #[derive(Debug, Copy, Clone)] @@ -117,14 +120,33 @@ impl Doc { blobs, local_author, }; + + // If download mode is set to always download: + // setup on_insert callback to trigger download on remote insert if let DownloadMode::Always = download_mode { - let doc2 = doc.clone(); + let doc_clone = doc.clone(); doc.replica.on_insert(Box::new(move |origin, entry| { if matches!(origin, InsertOrigin::Sync) { - doc2.download_content_fron_author(&entry); + doc_clone.download_content_fron_author(&entry); } })); } + + // Collect metrics + doc.replica.on_insert(Box::new(move |origin, entry| { + let size = entry.entry().record().content_len(); + match origin { + InsertOrigin::Local => { + inc!(Metrics, new_entries_local); + inc_by!(Metrics, new_entries_local_size, size); + } + InsertOrigin::Sync => { + inc!(Metrics, new_entries_remote); + inc_by!(Metrics, new_entries_remote_size, size); + } + } + })); + doc } @@ -407,7 +429,22 @@ impl DownloadActor { { let conn = self.conns.get(&peer).unwrap().clone(); let blobs = self.db.clone(); - let fut = async move { (peer, hash, blobs.download_single(conn, hash).await) }; + let fut = async move { + let start = Instant::now(); + let res = blobs.download_single(conn, hash).await; + // record metrics + let elapsed = start.elapsed().as_millis(); + match &res { + Ok(Some((_hash, len))) => { + inc!(Metrics, downloads_success); + inc_by!(Metrics, download_bytes_total, *len); + inc_by!(Metrics, download_time_total, elapsed as u64); + } + Ok(None) => inc!(Metrics, downloads_notfound), + Err(_) => inc!(Metrics, downloads_error), + } + (peer, hash, res) + }; self.pending_downloads.push(fut.boxed_local()); } else { self.conns.remove(&peer); diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index c2c961e660..2d68c96e44 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -11,12 +11,15 @@ use iroh_gossip::{ net::{Event, GossipHandle}, proto::TopicId, }; +use iroh_metrics::inc; use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::sync::{InsertOrigin, Replica, SignedEntry}; use serde::{Deserialize, Serialize}; use tokio::{sync::mpsc, task::JoinError}; use tracing::{debug, error}; +use super::metrics::Metrics; + const CHANNEL_CAP: usize = 8; /// The address to connect to a peer @@ -140,7 +143,10 @@ impl Actor { match msg { // received shutdown signal, or livesync handle was dropped: // break loop and exit - Some(ToActor::Shutdown) | None => break, + Some(ToActor::Shutdown) | None => { + self.on_shutdown().await?; + break; + } Some(ToActor::SyncDoc { doc, initial_peers }) => self.insert_doc(doc, initial_peers).await?, } } @@ -192,6 +198,11 @@ impl Actor { // TODO: Make sure that the peer is dialable. let res = connect_and_sync(&endpoint, &doc, peer, None, &[]).await; debug!("> synced with {peer}: {res:?}"); + // collect metrics + match &res { + Ok(_) => inc!(Metrics, initial_sync_success), + Err(_) => inc!(Metrics, initial_sync_failed), + } (topic, peer, res) } .boxed() @@ -199,6 +210,14 @@ impl Actor { self.pending_syncs.push(task); } + async fn on_shutdown(&mut self) -> anyhow::Result<()> { + for (topic, _doc) in self.docs.drain() { + // TODO: Remove the on_insert callbacks + self.gossip.quit(topic).await?; + } + Ok(()) + } + async fn insert_doc(&mut self, doc: Replica, initial_peers: Vec) -> Result<()> { let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id).collect(); diff --git a/iroh/src/sync/metrics.rs b/iroh/src/sync/metrics.rs new file mode 100644 index 0000000000..257f2afa07 --- /dev/null +++ b/iroh/src/sync/metrics.rs @@ -0,0 +1,45 @@ +use iroh_metrics::{ + core::{Counter, Metric}, + struct_iterable::Iterable, +}; + +/// Metrics for iroh-sync +#[allow(missing_docs)] +#[derive(Debug, Clone, Iterable)] +pub struct Metrics { + pub new_entries_local: Counter, + pub new_entries_remote: Counter, + pub new_entries_local_size: Counter, + pub new_entries_remote_size: Counter, + pub download_bytes_total: Counter, + pub download_time_total: Counter, + pub downloads_success: Counter, + pub downloads_error: Counter, + pub downloads_notfound: Counter, + pub initial_sync_success: Counter, + pub initial_sync_failed: Counter, +} + +impl Default for Metrics { + fn default() -> Self { + Self { + new_entries_local: Counter::new("Number of document entries added locally"), + new_entries_remote: Counter::new("Number of document entries added by peers"), + new_entries_local_size: Counter::new("Total size of entry contents added locally"), + new_entries_remote_size: Counter::new("Total size of entry contents added by peers"), + download_bytes_total: Counter::new("Total number of content bytes downloaded"), + download_time_total: Counter::new("Total time in ms spent downloading content bytes"), + downloads_success: Counter::new("Total number of successfull downloads"), + downloads_error: Counter::new("Total number of downloads failed with error"), + downloads_notfound: Counter::new("Total number of downloads failed with not found"), + initial_sync_success: Counter::new("Number of successfull initial syncs "), + initial_sync_failed: Counter::new("Number of failed initial syncs"), + } + } +} + +impl Metric for Metrics { + fn name() -> &'static str { + "iroh-sync" + } +} From ce9d809cb9d45155e6dc1befa5292394ea678c5a Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 13:01:45 +0200 Subject: [PATCH 017/172] fix: remove usage of unbounded channels uses flume channels to allow for combined sync and async usage --- iroh-sync/src/sync.rs | 1 + iroh/src/sync/content.rs | 23 ++++++++++------------- iroh/src/sync/live.rs | 13 +++++-------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index baed4e678d..499da06612 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -436,6 +436,7 @@ impl Replica { let bytes = postcard::to_stdvec(&data)?; Ok(bytes.into()) } + pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { let data: ReplicaData = postcard::from_bytes(bytes)?; let replica = Self::new(data.namespace); diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index f5942eb193..54e616c414 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -21,10 +21,7 @@ use iroh_sync::sync::{ Author, InsertOrigin, Namespace, NamespaceId, OnInsertCallback, Replica, ReplicaStore, SignedEntry, }; -use tokio::{ - io::AsyncRead, - sync::{mpsc, oneshot}, -}; +use tokio::{io::AsyncRead, sync::oneshot}; use tokio_stream::StreamExt; use tracing::{debug, error, warn}; @@ -127,7 +124,7 @@ impl Doc { let doc_clone = doc.clone(); doc.replica.on_insert(Box::new(move |origin, entry| { if matches!(origin, InsertOrigin::Sync) { - doc_clone.download_content_fron_author(&entry); + doc_clone.download_content_from_author(&entry); } })); } @@ -182,7 +179,7 @@ impl Doc { Ok((hash, len)) } - pub fn download_content_fron_author(&self, entry: &SignedEntry) { + pub fn download_content_from_author(&self, entry: &SignedEntry) { let hash = *entry.entry().record().content_hash(); let peer_id = PeerId::from_bytes(entry.entry().id().author().as_bytes()) .expect("failed to convert author to peer id"); @@ -286,7 +283,7 @@ pub struct DownloadRequest { #[derive(Debug, Clone)] pub struct Downloader { pending_downloads: Arc>>, - to_actor_tx: mpsc::UnboundedSender, + to_actor_tx: flume::Sender, } impl Downloader { @@ -295,7 +292,7 @@ impl Downloader { endpoint: MagicEndpoint, blobs: WritableFileDatabase, ) -> Self { - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = flume::bounded(64); // spawn the actor on a local pool // the local pool is required because WritableFileDatabase::download_single // returns a future that is !Send @@ -348,13 +345,13 @@ pub struct DownloadActor { pending_downloads: FuturesUnordered< LocalBoxFuture<'static, (PeerId, Hash, anyhow::Result>)>, >, - rx: mpsc::UnboundedReceiver, + rx: flume::Receiver, } impl DownloadActor { fn new( endpoint: MagicEndpoint, db: WritableFileDatabase, - rx: mpsc::UnboundedReceiver, + rx: flume::Receiver, ) -> Self { Self { rx, @@ -370,9 +367,9 @@ impl DownloadActor { pub async fn run(&mut self) -> anyhow::Result<()> { loop { tokio::select! { - req = self.rx.recv() => match req { - None => return Ok(()), - Some(req) => self.on_download_request(req).await + req = self.rx.recv_async() => match req { + Err(_) => return Ok(()), + Ok(req) => self.on_download_request(req).await }, (peer, conn) = self.dialer.next() => match conn { Ok(conn) => { diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 2d68c96e44..6fa2c71b85 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -102,8 +102,8 @@ struct Actor { sync_state: HashMap<(TopicId, PeerId), SyncState>, to_actor_rx: mpsc::Receiver, - insert_entry_tx: mpsc::UnboundedSender<(TopicId, SignedEntry)>, - insert_entry_rx: mpsc::UnboundedReceiver<(TopicId, SignedEntry)>, + insert_entry_tx: flume::Sender<(TopicId, SignedEntry)>, + insert_entry_rx: flume::Receiver<(TopicId, SignedEntry)>, pending_syncs: FuturesUnordered)>>, pending_joins: FuturesUnordered)>>, @@ -115,10 +115,7 @@ impl Actor { gossip: GossipHandle, to_actor_rx: mpsc::Receiver, ) -> Self { - // TODO: instead of an unbounded channel, we'd want a FIFO ring buffer likely - // (we have to send from the blocking Replica::on_insert callback, so we need a channel - // with nonblocking sending, so either unbounded or ringbuffer like) - let (insert_tx, insert_rx) = mpsc::unbounded_channel(); + let (insert_tx, insert_rx) = flume::bounded(64); let sub = gossip.clone().subscribe_all().boxed(); Self { @@ -157,8 +154,8 @@ impl Actor { error!("Failed to process gossip event: {err:?}"); } }, - entry = self.insert_entry_rx.recv() => { - let (topic, entry) = entry.ok_or_else(|| anyhow!("insert_rx returned None"))?; + entry = self.insert_entry_rx.recv_async() => { + let (topic, entry) = entry?; self.on_insert_entry(topic, entry).await?; } Some((topic, peer, res)) = self.pending_syncs.next() => { From 2c9aa58ddeba1bc92ee5a99dd7b64bdc99aa2443 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 14:20:56 +0200 Subject: [PATCH 018/172] add todo --- iroh/src/sync/content.rs | 1 + iroh/src/sync/live.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 54e616c414..197e9c2d2f 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -328,6 +328,7 @@ impl Downloader { .lock() .unwrap() .insert(hash, fut.boxed().shared()); + // TODO: this is potentially blocking inside an async call. figure out a better solution if let Err(err) = self.to_actor_tx.send(req) { warn!("download actor dropped: {err}"); } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 6fa2c71b85..8be595b5f1 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -242,6 +242,7 @@ impl Actor { doc.on_insert(Box::new(move |origin, entry| { // only care for local inserts, otherwise we'd do endless gossip loops if let InsertOrigin::Local = origin { + // TODO: this is potentially blocking inside an async call. figure out a better solution insert_entry_tx.send((topic, entry)).ok(); } })); From 5a627b9bc705cefa3bfd9674ca1582432974ca16 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 14:08:59 +0200 Subject: [PATCH 019/172] sync: implement more extensive fetch methods --- iroh-sync/src/sync.rs | 244 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 225 insertions(+), 19 deletions(-) diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 499da06612..186ea38537 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -500,7 +500,11 @@ impl Replica { } /// Gets all entries matching this key and author. - pub fn get_latest(&self, key: impl AsRef<[u8]>, author: &AuthorId) -> Option { + pub fn get_latest_by_key_and_author( + &self, + key: impl AsRef<[u8]>, + author: &AuthorId, + ) -> Option { let inner = self.inner.read(); inner .peer @@ -508,19 +512,83 @@ impl Replica { .cloned() } - /// Returns all versions of the matching documents. - pub fn get_all<'a, 'b: 'a>( + /// Returns the latest version of the matching documents by key. + pub fn get_latest_by_key(&self, key: impl AsRef<[u8]>) -> GetLatestIter<'_> { + let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); + let key = key.as_ref().to_vec(); + let namespace = *guard.namespace.id(); + let filter = GetFilter::Key { namespace, key }; + + GetLatestIter { + records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { + &inner.peer.store().records + }), + filter, + index: 0, + } + } + + /// Returns the latest versions of all documents. + pub fn get_latest(&self) -> GetLatestIter<'_> { + let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); + let namespace = *guard.namespace.id(); + let filter = GetFilter::All { namespace }; + + GetLatestIter { + records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { + &inner.peer.store().records + }), + filter, + index: 0, + } + } + + /// Returns all versions of the matching documents by author. + pub fn get_all_by_key_and_author<'a, 'b: 'a>( &'a self, key: impl AsRef<[u8]> + 'b, author: &AuthorId, ) -> GetAllIter<'a> { let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); let record_id = RecordIdentifier::new(key, guard.namespace.id(), author); + let filter = GetFilter::KeyAuthor(record_id); + GetAllIter { records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { &inner.peer.store().records }), - record_id, + filter, + index: 0, + } + } + + /// Returns all versions of the matching documents by key. + pub fn get_all_by_key(&self, key: impl AsRef<[u8]>) -> GetAllIter<'_> { + let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); + let key = key.as_ref().to_vec(); + let namespace = *guard.namespace.id(); + let filter = GetFilter::Key { namespace, key }; + + GetAllIter { + records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { + &inner.peer.store().records + }), + filter, + index: 0, + } + } + + /// Returns all versions of all documents. + pub fn get_all(&self) -> GetAllIter<'_> { + let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); + let namespace = *guard.namespace.id(); + let filter = GetFilter::All { namespace }; + + GetAllIter { + records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { + &inner.peer.store().records + }), + filter, index: 0, } } @@ -553,30 +621,114 @@ impl Replica { } #[derive(Debug)] -pub struct GetAllIter<'a> { +pub enum GetFilter { + /// All entries. + All { namespace: NamespaceId }, + /// Filter by key and author. + KeyAuthor(RecordIdentifier), + /// Filter by key only. + Key { + namespace: NamespaceId, + key: Vec, + }, +} + +#[derive(Debug)] +pub struct GetLatestIter<'a> { // Oh my god, rust why u do this to me? records: parking_lot::lock_api::MappedRwLockReadGuard< 'a, parking_lot::RawRwLock, BTreeMap>, >, - record_id: RecordIdentifier, + filter: GetFilter, /// Current iteration index. index: usize, } -impl<'a> Iterator for GetAllIter<'a> { +impl<'a> Iterator for GetLatestIter<'a> { type Item = SignedEntry; fn next(&mut self) -> Option { - let values = self.records.get(&self.record_id)?; - - let (_, res) = values.iter().nth(self.index)?; + let res = match self.filter { + GetFilter::All { namespace } => { + let (_, res) = self + .records + .iter() + .filter(|(k, _)| k.namespace() == &namespace) + .filter_map(|(_key, value)| value.last_key_value()) + .nth(self.index)?; + res + } + GetFilter::KeyAuthor(ref record_id) => { + let values = self.records.get(record_id)?; + let (_, res) = values.iter().nth(self.index)?; + res + } + GetFilter::Key { namespace, ref key } => { + let (_, res) = self + .records + .iter() + .filter(|(k, _)| k.key() == key && k.namespace() == &namespace) + .filter_map(|(_key, value)| value.last_key_value()) + .nth(self.index)?; + res + } + }; self.index += 1; Some(res.clone()) // :( I give up } } +#[derive(Debug)] +pub struct GetAllIter<'a> { + // Oh my god, rust why u do this to me? + records: parking_lot::lock_api::MappedRwLockReadGuard< + 'a, + parking_lot::RawRwLock, + BTreeMap>, + >, + filter: GetFilter, + /// Current iteration index. + index: usize, +} + +impl<'a> Iterator for GetAllIter<'a> { + type Item = (RecordIdentifier, u64, SignedEntry); + + fn next(&mut self) -> Option { + let res = match self.filter { + GetFilter::All { namespace } => self + .records + .iter() + .filter(|(k, _)| k.namespace() == &namespace) + .flat_map(|(key, value)| { + value + .iter() + .map(|(t, value)| (key.clone(), *t, value.clone())) + }) + .nth(self.index)?, + GetFilter::KeyAuthor(ref record_id) => { + let values = self.records.get(record_id)?; + let (t, value) = values.iter().nth(self.index)?; + (record_id.clone(), *t, value.clone()) + } + GetFilter::Key { namespace, ref key } => self + .records + .iter() + .filter(|(k, _)| k.key() == key && k.namespace() == &namespace) + .flat_map(|(key, value)| { + value + .iter() + .map(|(t, value)| (key.clone(), *t, value.clone())) + }) + .nth(self.index)?, + }; + self.index += 1; + Some(res) + } +} + /// A signed entry. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SignedEntry { @@ -832,6 +984,7 @@ mod tests { fn test_basics() { let mut rng = rand::thread_rng(); let alice = Author::new(&mut rng); + let bob = Author::new(&mut rng); let myspace = Namespace::new(&mut rng); let record_id = RecordIdentifier::new("/my/key", myspace.id(), alice.id()); @@ -846,7 +999,9 @@ mod tests { } for i in 0..10 { - let res = my_replica.get_latest(format!("/{i}"), alice.id()).unwrap(); + let res = my_replica + .get_latest_by_key_and_author(format!("/{i}"), alice.id()) + .unwrap(); let len = format!("{i}: hello from alice").as_bytes().len() as u64; assert_eq!(res.entry().record().content_len(), len); res.verify().expect("invalid signature"); @@ -854,15 +1009,66 @@ mod tests { // Test multiple records for the same key my_replica.hash_and_insert("/cool/path", &alice, "round 1"); - let _entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); - + let _entry = my_replica + .get_latest_by_key_and_author("/cool/path", alice.id()) + .unwrap(); // Second my_replica.hash_and_insert("/cool/path", &alice, "round 2"); - let _entry = my_replica.get_latest("/cool/path", alice.id()).unwrap(); + let _entry = my_replica + .get_latest_by_key_and_author("/cool/path", alice.id()) + .unwrap(); + + // Get All by author + let entries: Vec<_> = my_replica + .get_all_by_key_and_author("/cool/path", alice.id()) + .collect(); + assert_eq!(entries.len(), 2); + + // Get All by key + let entries: Vec<_> = my_replica.get_all_by_key(b"/cool/path").collect(); + assert_eq!(entries.len(), 2); + + // Get latest by key + let entries: Vec<_> = my_replica.get_latest_by_key(b"/cool/path").collect(); + assert_eq!(entries.len(), 1); // Get All - let entries: Vec<_> = my_replica.get_all("/cool/path", alice.id()).collect(); + let entries: Vec<_> = my_replica.get_all().collect(); + assert_eq!(entries.len(), 12); + + // Get All latest + let entries: Vec<_> = my_replica.get_latest().collect(); + assert_eq!(entries.len(), 11); + + // insert record from different author + let _entry = my_replica.hash_and_insert("/cool/path", &bob, "bob round 1"); + + // Get All by author + let entries: Vec<_> = my_replica + .get_all_by_key_and_author("/cool/path", alice.id()) + .collect(); assert_eq!(entries.len(), 2); + + let entries: Vec<_> = my_replica + .get_all_by_key_and_author("/cool/path", bob.id()) + .collect(); + assert_eq!(entries.len(), 1); + + // Get All by key + let entries: Vec<_> = my_replica.get_all_by_key(b"/cool/path").collect(); + assert_eq!(entries.len(), 3); + + // Get latest by key + let entries: Vec<_> = my_replica.get_latest_by_key(b"/cool/path").collect(); + assert_eq!(entries.len(), 2); + + // Get All + let entries: Vec<_> = my_replica.get_all().collect(); + assert_eq!(entries.len(), 13); + + // Get All latest + let entries: Vec<_> = my_replica.get_latest().collect(); + assert_eq!(entries.len(), 12); } #[test] @@ -968,13 +1174,13 @@ mod tests { // Check result for el in alice_set { - alice.get_latest(el, author.id()).unwrap(); - bob.get_latest(el, author.id()).unwrap(); + alice.get_latest_by_key_and_author(el, author.id()).unwrap(); + bob.get_latest_by_key_and_author(el, author.id()).unwrap(); } for el in bob_set { - alice.get_latest(el, author.id()).unwrap(); - bob.get_latest(el, author.id()).unwrap(); + alice.get_latest_by_key_and_author(el, author.id()).unwrap(); + bob.get_latest_by_key_and_author(el, author.id()).unwrap(); } } } From 5899d0e1bdeed6012ad6dc16e3890e7f438d0f1b Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 14:19:09 +0200 Subject: [PATCH 020/172] add prefix methods --- iroh-sync/src/sync.rs | 82 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 186ea38537..2faf711ec5 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -528,6 +528,22 @@ impl Replica { } } + /// Returns the latest version of the matching documents by prefix. + pub fn get_latest_by_prefix(&self, prefix: impl AsRef<[u8]>) -> GetLatestIter<'_> { + let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); + let prefix = prefix.as_ref().to_vec(); + let namespace = *guard.namespace.id(); + let filter = GetFilter::Prefix { namespace, prefix }; + + GetLatestIter { + records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { + &inner.peer.store().records + }), + filter, + index: 0, + } + } + /// Returns the latest versions of all documents. pub fn get_latest(&self) -> GetLatestIter<'_> { let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); @@ -578,6 +594,22 @@ impl Replica { } } + /// Returns all versions of the matching documents by prefix. + pub fn get_all_by_prefix(&self, prefix: impl AsRef<[u8]>) -> GetAllIter<'_> { + let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); + let prefix = prefix.as_ref().to_vec(); + let namespace = *guard.namespace.id(); + let filter = GetFilter::Prefix { namespace, prefix }; + + GetAllIter { + records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { + &inner.peer.store().records + }), + filter, + index: 0, + } + } + /// Returns all versions of all documents. pub fn get_all(&self) -> GetAllIter<'_> { let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); @@ -631,6 +663,11 @@ pub enum GetFilter { namespace: NamespaceId, key: Vec, }, + /// Filter by prefix only. + Prefix { + namespace: NamespaceId, + prefix: Vec, + }, } #[derive(Debug)] @@ -658,12 +695,12 @@ impl<'a> Iterator for GetLatestIter<'a> { .filter(|(k, _)| k.namespace() == &namespace) .filter_map(|(_key, value)| value.last_key_value()) .nth(self.index)?; - res + res.clone() } GetFilter::KeyAuthor(ref record_id) => { let values = self.records.get(record_id)?; let (_, res) = values.iter().nth(self.index)?; - res + res.clone() } GetFilter::Key { namespace, ref key } => { let (_, res) = self @@ -672,11 +709,23 @@ impl<'a> Iterator for GetLatestIter<'a> { .filter(|(k, _)| k.key() == key && k.namespace() == &namespace) .filter_map(|(_key, value)| value.last_key_value()) .nth(self.index)?; - res + res.clone() + } + GetFilter::Prefix { + namespace, + ref prefix, + } => { + let (_, res) = self + .records + .iter() + .filter(|(k, _)| k.key().starts_with(prefix) && k.namespace() == &namespace) + .filter_map(|(_key, value)| value.last_key_value()) + .nth(self.index)?; + res.clone() } }; self.index += 1; - Some(res.clone()) // :( I give up + Some(res) } } @@ -723,6 +772,19 @@ impl<'a> Iterator for GetAllIter<'a> { .map(|(t, value)| (key.clone(), *t, value.clone())) }) .nth(self.index)?, + GetFilter::Prefix { + namespace, + ref prefix, + } => self + .records + .iter() + .filter(|(k, _)| k.key().starts_with(prefix) && k.namespace() == &namespace) + .flat_map(|(key, value)| { + value + .iter() + .map(|(t, value)| (key.clone(), *t, value.clone())) + }) + .nth(self.index)?, }; self.index += 1; Some(res) @@ -1032,6 +1094,10 @@ mod tests { let entries: Vec<_> = my_replica.get_latest_by_key(b"/cool/path").collect(); assert_eq!(entries.len(), 1); + // Get latest by prefix + let entries: Vec<_> = my_replica.get_latest_by_prefix(b"/cool").collect(); + assert_eq!(entries.len(), 1); + // Get All let entries: Vec<_> = my_replica.get_all().collect(); assert_eq!(entries.len(), 12); @@ -1062,6 +1128,14 @@ mod tests { let entries: Vec<_> = my_replica.get_latest_by_key(b"/cool/path").collect(); assert_eq!(entries.len(), 2); + // Get latest by prefix + let entries: Vec<_> = my_replica.get_latest_by_prefix(b"/cool").collect(); + assert_eq!(entries.len(), 2); + + // Get all by prefix + let entries: Vec<_> = my_replica.get_all_by_prefix(b"/cool").collect(); + assert_eq!(entries.len(), 3); + // Get All let entries: Vec<_> = my_replica.get_all().collect(); assert_eq!(entries.len(), 13); From 43eff2549bf9328e9f86e46eee320f3be59f7a30 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 27 Jul 2023 14:32:18 +0200 Subject: [PATCH 021/172] feat: enable metrics server on sync (#1308) --- iroh/examples/sync.rs | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index ec780768e2..cf8d18fdff 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -6,8 +6,7 @@ //! You can use this with a local DERP server. To do so, run //! `cargo run --bin derper -- --dev` //! and then set the `-d http://localhost:3340` flag on this example. - -use std::{fmt, path::PathBuf, str::FromStr, sync::Arc}; +use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr, sync::Arc}; use anyhow::bail; use clap::{CommandFactory, FromArgMatches, Parser}; @@ -60,6 +59,9 @@ struct Args { /// Set the bind port for our socket. By default, a random port will be used. #[clap(short, long, default_value = "0")] bind_port: u16, + /// Bind address on which to serve Prometheus metrics + #[clap(long)] + metrics_addr: Option, #[clap(subcommand)] command: Command, } @@ -76,16 +78,32 @@ async fn main() -> anyhow::Result<()> { run(args).await } -async fn run(args: Args) -> anyhow::Result<()> { - // setup logging - let log_filter = init_logging(); - - // init metrics +pub fn init_metrics_collection( + metrics_addr: Option, +) -> Option> { iroh_metrics::core::Core::init(|reg, metrics| { metrics.insert(iroh::sync::metrics::Metrics::new(reg)); metrics.insert(iroh_gossip::metrics::Metrics::new(reg)); }); + // doesn't start the server if the address is None + if let Some(metrics_addr) = metrics_addr { + return Some(tokio::spawn(async move { + if let Err(e) = iroh_metrics::metrics::start_metrics_server(metrics_addr).await { + eprintln!("Failed to start metrics server: {e}"); + } + })); + } + tracing::info!("Metrics server not started, no address provided"); + None +} + +async fn run(args: Args) -> anyhow::Result<()> { + // setup logging + let log_filter = init_logging(); + + let metrics_fut = init_metrics_collection(args.metrics_addr); + // parse or generate our keypair let keypair = match args.private_key { None => Keypair::generate(), @@ -267,6 +285,11 @@ async fn run(args: Args) -> anyhow::Result<()> { blobs.save().await?; docs.save(&doc).await?; + if let Some(metrics_fut) = metrics_fut { + metrics_fut.abort(); + drop(metrics_fut); + } + Ok(()) } From 33883f870936578c43d5a86dd5691e95417b7aec Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 14:35:00 +0200 Subject: [PATCH 022/172] chore: update deny.toml --- deny.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/deny.toml b/deny.toml index 1488465e15..517d074102 100644 --- a/deny.toml +++ b/deny.toml @@ -6,6 +6,7 @@ allow = [ "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", + "BSL-1.0", # BOSL license "ISC", "MIT", "OpenSSL", From b487671ef84b950348ec0deb0a9c3dbadec7c49a Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 14:41:41 +0200 Subject: [PATCH 023/172] fix clippy and feature selection --- iroh/Cargo.toml | 13 +++++++++---- iroh/examples/sync.rs | 2 +- iroh/src/lib.rs | 1 + iroh/src/sync/content.rs | 7 ++++--- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index d0b45dbe23..b2b8aad58a 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -26,7 +26,7 @@ iroh-metrics = { version = "0.5.0", path = "../iroh-metrics", optional = true } iroh-net = { version = "0.5.1", path = "../iroh-net" } num_cpus = { version = "1.15.0" } portable-atomic = "1" -iroh-sync = { path = "../iroh-sync" } +iroh-sync = { path = "../iroh-sync" } iroh-gossip = { path = "../iroh-gossip" } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quic-rpc = { version = "0.6", default-features = false, features = ["flume-transport"] } @@ -61,14 +61,15 @@ shellexpand = { version = "3.1.0", optional = true } rustyline = { version = "12.0.0", optional = true } [features] -default = ["cli", "metrics"] +default = ["cli", "metrics", "sync"] +sync = ["metrics"] cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection"] -metrics = ["iroh-metrics"] +metrics = ["iroh-metrics", "flat-db", "mem-db", "iroh-collection"] mem-db = [] flat-db = [] iroh-collection = [] test = [] -example-sync = ["cli", "ed25519-dalek", "once_cell", "shell-words", "shellexpand", "rustyline"] +example-sync = ["cli", "ed25519-dalek", "once_cell", "shell-words", "shellexpand", "sync", "rustyline"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } @@ -91,6 +92,10 @@ required-features = ["cli"] name = "collection" required-features = ["mem-db", "iroh-collection"] +[[example]] +name = "dump-blob-stream" +required-features = ["mem-db", "iroh-collection"] + [[example]] name = "hello-world" required-features = ["mem-db"] diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index cf8d18fdff..058e796995 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -348,7 +348,7 @@ async fn handle_command( Ok(()) } -#[derive(Parser)] +#[derive(Parser, Debug)] pub enum Cmd { /// Set an entry Set { diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index 55afd7c449..a0d58246e4 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -12,6 +12,7 @@ pub mod get; pub mod node; pub mod rpc_protocol; #[allow(missing_docs)] +#[cfg(feature = "sync")] pub mod sync; pub mod util; diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 197e9c2d2f..39bf4a793c 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -335,6 +335,9 @@ impl Downloader { } } +type PendingDownloadsFutures = + FuturesUnordered>)>>; + #[derive(Debug)] pub struct DownloadActor { dialer: Dialer, @@ -343,9 +346,7 @@ pub struct DownloadActor { replies: HashMap>, peer_hashes: HashMap>, hash_peers: HashMap>, - pending_downloads: FuturesUnordered< - LocalBoxFuture<'static, (PeerId, Hash, anyhow::Result>)>, - >, + pending_downloads: PendingDownloadsFutures, rx: flume::Receiver, } impl DownloadActor { From e56e7aa2e1fc5a11612356664dda2dc2cb41ef40 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 27 Jul 2023 14:46:28 +0200 Subject: [PATCH 024/172] doc: more docs fixes --- iroh-sync/src/ranger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index ea80f49f9e..e12db74498 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -575,7 +575,7 @@ where } } -/// Sadly https://doc.rust-lang.org/std/primitive.usize.html#method.div_ceil is still unstable.. +/// Sadly is still unstable.. fn div_ceil(a: usize, b: usize) -> usize { debug_assert!(a != 0); debug_assert!(b != 0); From b4358dd9316a4a8ffa3092dfa1f2c3f9c5ed2f5b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 27 Jul 2023 14:52:55 +0200 Subject: [PATCH 025/172] feat: file system export/import in sync repl --- iroh/examples/sync.rs | 173 ++++++++++++++++++++++++++++++++++++++- iroh/src/sync/content.rs | 11 ++- 2 files changed, 179 insertions(+), 5 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 058e796995..7e76a0817b 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -6,9 +6,10 @@ //! You can use this with a local DERP server. To do so, run //! `cargo run --bin derper -- --dev` //! and then set the `-d http://localhost:3340` flag on this example. -use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr, sync::Arc}; -use anyhow::bail; +use std::{collections::HashSet, fmt, net::SocketAddr, path::PathBuf, str::FromStr, sync::Arc}; + +use anyhow::{anyhow, bail}; use clap::{CommandFactory, FromArgMatches, Parser}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; @@ -31,7 +32,11 @@ use iroh_net::{ use iroh_sync::sync::{Author, Namespace, SignedEntry}; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; -use tokio::sync::{mpsc, oneshot}; +use tokio::{ + io::AsyncWriteExt, + sync::{mpsc, oneshot}, +}; +use tracing::warn; use tracing_subscriber::{EnvFilter, Registry}; use url::Url; @@ -343,11 +348,106 @@ async fn handle_command( log_filter.modify(|layer| *layer = next_filter)?; } Cmd::Stats => get_stats(), + Cmd::Fs(cmd) => handle_fs_command(cmd, doc).await?, Cmd::Exit => {} } Ok(()) } +async fn handle_fs_command(cmd: FsCmd, doc: &Doc) -> anyhow::Result<()> { + match cmd { + FsCmd::ImportFile { file_path, key } => { + let file_path = canonicalize_path(&file_path)?.canonicalize()?; + let (hash, len) = doc.insert_from_file(&key, &file_path).await?; + println!( + "> imported {file_path:?}: {} ({})", + fmt_hash(hash), + HumanBytes(len) + ); + } + FsCmd::ImportDir { + dir_path, + mut key_prefix, + } => { + if key_prefix.ends_with("/") { + key_prefix.pop(); + } + let root = canonicalize_path(&dir_path)?.canonicalize()?; + let files = walkdir::WalkDir::new(&root).into_iter(); + // TODO: parallelize + for file in files { + let file = file?; + if file.file_type().is_file() { + let relative = file.path().strip_prefix(&root)?.to_string_lossy(); + if relative.is_empty() { + warn!("invalid file path: {:?}", file.path()); + continue; + } + let key = format!("{key_prefix}/{relative}"); + let (hash, len) = doc.insert_from_file(key, file.path()).await?; + println!( + "> imported {relative}: {} ({})", + fmt_hash(hash), + HumanBytes(len) + ); + } + } + } + FsCmd::ExportDir { + mut key_prefix, + dir_path, + } => { + if !key_prefix.ends_with("/") { + key_prefix.push('/'); + } + let root = canonicalize_path(&dir_path)?; + println!("> exporting {key_prefix} to {root:?}"); + let entries = doc.replica().get_latest_by_prefix(key_prefix.as_bytes()); + let mut checked_dirs = HashSet::new(); + for entry in entries { + let key = entry.entry().id().key(); + let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; + let len = entry.entry().record().content_len(); + if let Some(mut reader) = doc.get_content_reader(&entry).await { + let path = root.join(&relative); + let parent = path.parent().unwrap(); + if !checked_dirs.contains(parent) { + tokio::fs::create_dir_all(&parent).await?; + checked_dirs.insert(parent.to_owned()); + } + let mut file = tokio::fs::File::create(&path).await?; + copy(&mut reader, &mut file).await?; + println!( + "> exported {} to {path:?} ({})", + fmt_hash(entry.content_hash()), + HumanBytes(len) + ); + } + } + } + FsCmd::ExportFile { key, file_path } => { + let path = canonicalize_path(&file_path)?; + // TODO: Fix + let entry = doc.replica().get_latest_by_key(&key).next(); + if let Some(entry) = entry { + println!("> exporting {key} to {path:?}"); + let parent = path.parent().ok_or_else(|| anyhow!("Invalid path"))?; + tokio::fs::create_dir_all(&parent).await?; + let mut file = tokio::fs::File::create(&path).await?; + let mut reader = doc + .get_content_reader(&entry) + .await + .ok_or_else(|| anyhow!(format!("content for {key} is not available")))?; + copy(&mut reader, &mut file).await?; + } else { + println!("> key not found, abort"); + } + } + } + + Ok(()) +} + #[derive(Parser, Debug)] pub enum Cmd { /// Set an entry @@ -367,11 +467,16 @@ pub enum Cmd { #[clap(short = 'c', long)] print_content: bool, }, - /// List entries + /// List entries. Ls { /// Optionally list only entries whose key starts with PREFIX. prefix: Option, }, + + /// Import from and export to the local file system. + #[clap(subcommand)] + Fs(FsCmd), + /// Print the ticket with which other peers can join our document. Ticket, /// Change the log level @@ -400,6 +505,39 @@ pub enum Cmd { /// Quit Exit, } + +#[derive(Parser, Debug)] +pub enum FsCmd { + /// Import a file system directory into the document. + ImportDir { + /// The file system path to import recursively + dir_path: String, + /// The key prefix to apply to the document keys + key_prefix: String, + }, + /// Import a file into the document. + ImportFile { + /// The path to the file + file_path: String, + /// The key in the document + key: String, + }, + /// Export a part of the document into a file system directory + ExportDir { + /// The key prefix to filter on + key_prefix: String, + /// The file system path to export to + dir_path: String, + }, + /// Import a file into the document. + ExportFile { + /// The key in the document + key: String, + /// The path to the file + file_path: String, + }, +} + impl FromStr for Cmd { type Err = anyhow::Error; fn from_str(s: &str) -> Result { @@ -622,6 +760,33 @@ fn derp_map_from_url(url: Url) -> anyhow::Result { )) } +fn canonicalize_path(path: &str) -> anyhow::Result { + let path = PathBuf::from(shellexpand::tilde(&path).to_string()); + Ok(path) +} + +/// Copy from a [`iroh_io::AsyncSliceReader`] into a [`tokio::io::AsyncWrite`] +/// +/// TODO: move to iroh-io or iroh-bytes +async fn copy( + mut reader: impl iroh_io::AsyncSliceReader, + mut writer: impl tokio::io::AsyncWrite + Unpin, +) -> anyhow::Result<()> { + // this is the max chunk size. + // will only allocate this much if the resource behind the reader is at least this big. + let chunk_size = 1024 * 16; + let mut pos = 0u64; + loop { + let chunk = reader.read_at(pos, chunk_size).await?; + if chunk.is_empty() { + break; + } + writer.write_all(&chunk).await?; + pos += chunk.len() as u64; + } + Ok(()) +} + /// handlers for iroh_bytes connections mod iroh_bytes_handlers { use std::sync::Arc; diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 39bf4a793c..c68d06272c 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -1,7 +1,7 @@ use std::{ collections::{HashMap, HashSet, VecDeque}, io, - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, Mutex}, time::Instant, }; @@ -179,6 +179,15 @@ impl Doc { Ok((hash, len)) } + pub async fn insert_from_file( + &self, + key: impl AsRef<[u8]>, + file_path: impl AsRef, + ) -> anyhow::Result<(Hash, u64)> { + let reader = tokio::fs::File::open(&file_path).await?; + self.insert_reader(&key, reader).await + } + pub fn download_content_from_author(&self, entry: &SignedEntry) { let hash = *entry.entry().record().content_hash(); let peer_id = PeerId::from_bytes(entry.entry().id().author().as_bytes()) From c700721b91120448bcc151cba040afc73b0bc87a Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 27 Jul 2023 15:04:13 +0200 Subject: [PATCH 026/172] feat: hammer sync example --- iroh/examples/sync.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 7e76a0817b..93acabd2cd 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -39,7 +39,6 @@ use tokio::{ use tracing::warn; use tracing_subscriber::{EnvFilter, Registry}; use url::Url; - use iroh_bytes_handlers::IrohBytesHandlers; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; @@ -349,6 +348,21 @@ async fn handle_command( } Cmd::Stats => get_stats(), Cmd::Fs(cmd) => handle_fs_command(cmd, doc).await?, + Cmd::Hammer { prefix, count, size}=> { + println!( + "> hammering with prefix {prefix} for {count} messages of size {size} bytes", + prefix = prefix, + count = count, + size = size, + ); + let mut bytes = vec![0; size]; + bytes.fill(97); + for i in 0..count { + let value = String::from_utf8(bytes.clone())?; + let key = format!("{}/{}", prefix, i); + doc.insert_bytes(key, value.into_bytes().into()).await?; + } + } Cmd::Exit => {} } Ok(()) @@ -502,6 +516,15 @@ pub enum Cmd { WatchCancel, /// Show stats about the current session Stats, + /// Stress test with the hammer + Hammer { + /// The key prefix + prefix: String, + /// The number of entries to create + count: usize, + /// The size of each entry in Bytes + size: usize, + }, /// Quit Exit, } From 6bdfaf642dd95004639e7b30691c87109210e96e Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 27 Jul 2023 15:17:26 +0200 Subject: [PATCH 027/172] chore: fmt --- iroh/examples/sync.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 93acabd2cd..1dff08cd7a 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -14,6 +14,7 @@ use clap::{CommandFactory, FromArgMatches, Parser}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; use iroh::sync::{BlobStore, Doc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; +use iroh_bytes_handlers::IrohBytesHandlers; use iroh_gossip::{ net::{GossipHandle, GOSSIP_ALPN}, proto::TopicId, @@ -39,7 +40,6 @@ use tokio::{ use tracing::warn; use tracing_subscriber::{EnvFilter, Registry}; use url::Url; -use iroh_bytes_handlers::IrohBytesHandlers; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; @@ -348,7 +348,11 @@ async fn handle_command( } Cmd::Stats => get_stats(), Cmd::Fs(cmd) => handle_fs_command(cmd, doc).await?, - Cmd::Hammer { prefix, count, size}=> { + Cmd::Hammer { + prefix, + count, + size, + } => { println!( "> hammering with prefix {prefix} for {count} messages of size {size} bytes", prefix = prefix, From 27e0923cbaf59623a206cdc008541c23d5c90d4d Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 27 Jul 2023 15:58:28 +0200 Subject: [PATCH 028/172] allow specifying number of threads and print out some basic stats --- iroh/examples/sync.rs | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 1dff08cd7a..e20c92eb12 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -14,7 +14,6 @@ use clap::{CommandFactory, FromArgMatches, Parser}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; use iroh::sync::{BlobStore, Doc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; -use iroh_bytes_handlers::IrohBytesHandlers; use iroh_gossip::{ net::{GossipHandle, GOSSIP_ALPN}, proto::TopicId, @@ -41,6 +40,8 @@ use tracing::warn; use tracing_subscriber::{EnvFilter, Registry}; use url::Url; +use iroh_bytes_handlers::IrohBytesHandlers; + const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; #[derive(Parser, Debug)] @@ -350,22 +351,47 @@ async fn handle_command( Cmd::Fs(cmd) => handle_fs_command(cmd, doc).await?, Cmd::Hammer { prefix, + threads, count, size, } => { println!( - "> hammering with prefix {prefix} for {count} messages of size {size} bytes", + "> Hammering with prefix {prefix} for {threads} x {count} messages of size {size} bytes", prefix = prefix, + threads = threads, count = count, size = size, ); let mut bytes = vec![0; size]; bytes.fill(97); - for i in 0..count { - let value = String::from_utf8(bytes.clone())?; - let key = format!("{}/{}", prefix, i); - doc.insert_bytes(key, value.into_bytes().into()).await?; + let mut handles = Vec::new(); + let start = std::time::Instant::now(); + for t in 0..threads { + let p = prefix.clone(); + let t_doc = doc.clone(); + let b = bytes.clone(); + let h = tokio::spawn(async move { + for i in 0..count { + let value = String::from_utf8(b.clone()).unwrap(); + let key = format!("{}/{}/{}", p, t, i); + t_doc + .insert_bytes(key, value.into_bytes().into()) + .await + .unwrap(); + } + }); + handles.push(h); } + + let _result = futures::future::join_all(handles).await; + + let diff = start.elapsed().as_secs_f64(); + println!( + "> Hammering done in {:.2}s for {} messages with total of {} bytes", + diff, + threads * count, + threads * count * size + ); } Cmd::Exit => {} } @@ -524,6 +550,8 @@ pub enum Cmd { Hammer { /// The key prefix prefix: String, + /// The number of threads to use (each thread will create it's own replica) + threads: usize, /// The number of entries to create count: usize, /// The size of each entry in Bytes From 6745dfa4c69d0a4c8c9296fd41f0fa3645e873ea Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 27 Jul 2023 22:06:54 +0200 Subject: [PATCH 029/172] mad clippy is mad --- iroh/examples/sync.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index e20c92eb12..72314be6d9 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -413,7 +413,7 @@ async fn handle_fs_command(cmd: FsCmd, doc: &Doc) -> anyhow::Result<()> { dir_path, mut key_prefix, } => { - if key_prefix.ends_with("/") { + if key_prefix.ends_with('/') { key_prefix.pop(); } let root = canonicalize_path(&dir_path)?.canonicalize()?; @@ -441,7 +441,7 @@ async fn handle_fs_command(cmd: FsCmd, doc: &Doc) -> anyhow::Result<()> { mut key_prefix, dir_path, } => { - if !key_prefix.ends_with("/") { + if !key_prefix.ends_with('/') { key_prefix.push('/'); } let root = canonicalize_path(&dir_path)?; From aeda86864ded006d610f85971cc95fd80ad2e263 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 27 Jul 2023 22:53:06 +0200 Subject: [PATCH 030/172] extend hammer with get/set modes --- iroh/examples/sync.rs | 107 +++++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 22 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 72314be6d9..29e676d316 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -11,6 +11,7 @@ use std::{collections::HashSet, fmt, net::SocketAddr, path::PathBuf, str::FromSt use anyhow::{anyhow, bail}; use clap::{CommandFactory, FromArgMatches, Parser}; +use core::fmt::{Display, Formatter}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; use iroh::sync::{BlobStore, Doc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; @@ -354,43 +355,66 @@ async fn handle_command( threads, count, size, + mode, } => { println!( - "> Hammering with prefix {prefix} for {threads} x {count} messages of size {size} bytes", + "> Hammering with prefix \"{prefix}\" for {threads} x {count} messages of size {size} bytes in {mode} mode", prefix = prefix, threads = threads, count = count, size = size, + mode = mode, ); - let mut bytes = vec![0; size]; - bytes.fill(97); - let mut handles = Vec::new(); let start = std::time::Instant::now(); - for t in 0..threads { - let p = prefix.clone(); - let t_doc = doc.clone(); - let b = bytes.clone(); - let h = tokio::spawn(async move { - for i in 0..count { - let value = String::from_utf8(b.clone()).unwrap(); - let key = format!("{}/{}/{}", p, t, i); - t_doc - .insert_bytes(key, value.into_bytes().into()) - .await - .unwrap(); + let mut handles = Vec::new(); + match mode { + HammerMode::Set => { + let mut bytes = vec![0; size]; + bytes.fill(97); + for t in 0..threads { + let p = prefix.clone(); + let t_doc = doc.clone(); + let b = bytes.clone(); + let h = tokio::spawn(async move { + for i in 0..count { + let value = String::from_utf8(b.clone()).unwrap(); + let key = format!("{}/{}/{}", p, t, i); + t_doc + .insert_bytes(key, value.into_bytes().into()) + .await + .unwrap(); + } + }); + handles.push(h); + } + } + HammerMode::Get => { + for t in 0..threads { + let p = prefix.clone(); + let t_doc = doc.clone(); + let h = tokio::spawn(async move { + for i in 0..count { + let key = format!("{}/{}/{}", p, t, i); + let entries = t_doc.replica().all_for_key(key.as_bytes()); + for (_id, entry) in entries { + let _content = fmt_content(&t_doc, &entry).await; + } + } + }); + handles.push(h); } - }); - handles.push(h); + } } let _result = futures::future::join_all(handles).await; let diff = start.elapsed().as_secs_f64(); + let total_count = threads as u64 * count as u64; println!( - "> Hammering done in {:.2}s for {} messages with total of {} bytes", + "> Hammering done in {:.2}s for {} messages with total of {}", diff, - threads * count, - threads * count * size + total_count, + HumanBytes(total_count * size as u64), ); } Cmd::Exit => {} @@ -546,21 +570,60 @@ pub enum Cmd { WatchCancel, /// Show stats about the current session Stats, - /// Stress test with the hammer + /// Hammer time - stress test with the hammer Hammer { /// The key prefix prefix: String, /// The number of threads to use (each thread will create it's own replica) + #[clap(long, short, default_value = "2")] threads: usize, /// The number of entries to create + #[clap(long, short, default_value = "1000")] count: usize, /// The size of each entry in Bytes + #[clap(long, short, default_value = "1024")] size: usize, + /// Select the hammer mode (set or get) + #[clap(long, short, default_value = "set")] + mode: HammerMode, }, /// Quit Exit, } +#[derive(Clone, Debug, Parser)] +pub enum HammerMode { + Set, + Get, +} + +impl FromStr for HammerMode { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "set" => Ok(HammerMode::Set), + "get" => Ok(HammerMode::Get), + _ => Err(anyhow!("Invalid hammer mode")), + } + } +} + +impl HammerMode { + pub fn to_string(&self) -> &'static str { + match self { + HammerMode::Set => "set", + HammerMode::Get => "get", + } + } +} + +impl Display for HammerMode { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.to_string()) + } +} + #[derive(Parser, Debug)] pub enum FsCmd { /// Import a file system directory into the document. From 4de81b6b7537364d9e4dd437cd6c4a73f31c8026 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 28 Jul 2023 15:19:00 +0200 Subject: [PATCH 031/172] refactor: use clap::ValueEnum --- iroh/examples/sync.rs | 42 ++++++------------------------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 29e676d316..474ab699f2 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -11,7 +11,6 @@ use std::{collections::HashSet, fmt, net::SocketAddr, path::PathBuf, str::FromSt use anyhow::{anyhow, bail}; use clap::{CommandFactory, FromArgMatches, Parser}; -use core::fmt::{Display, Formatter}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; use iroh::sync::{BlobStore, Doc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; @@ -359,11 +358,7 @@ async fn handle_command( } => { println!( "> Hammering with prefix \"{prefix}\" for {threads} x {count} messages of size {size} bytes in {mode} mode", - prefix = prefix, - threads = threads, - count = count, - size = size, - mode = mode, + mode = format!("{mode:?}").to_lowercase() ); let start = std::time::Instant::now(); let mut handles = Vec::new(); @@ -583,47 +578,22 @@ pub enum Cmd { /// The size of each entry in Bytes #[clap(long, short, default_value = "1024")] size: usize, - /// Select the hammer mode (set or get) - #[clap(long, short, default_value = "set")] + /// Select the hammer mode + #[clap(long, short, value_enum, default_value = "set")] mode: HammerMode, }, /// Quit Exit, } -#[derive(Clone, Debug, Parser)] +#[derive(Clone, Debug, clap::ValueEnum)] pub enum HammerMode { + /// Set mode (create entries) Set, + /// Get mode (read entries) Get, } -impl FromStr for HammerMode { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "set" => Ok(HammerMode::Set), - "get" => Ok(HammerMode::Get), - _ => Err(anyhow!("Invalid hammer mode")), - } - } -} - -impl HammerMode { - pub fn to_string(&self) -> &'static str { - match self { - HammerMode::Set => "set", - HammerMode::Get => "get", - } - } -} - -impl Display for HammerMode { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.to_string()) - } -} - #[derive(Parser, Debug)] pub enum FsCmd { /// Import a file system directory into the document. From 494748bc7872e04aef45dbd35078a3d2191e3e18 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 28 Jul 2023 15:31:12 +0200 Subject: [PATCH 032/172] refactor: check results, and clearer variable names --- iroh/examples/sync.rs | 58 ++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 474ab699f2..f7bbe356a8 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -7,7 +7,10 @@ //! `cargo run --bin derper -- --dev` //! and then set the `-d http://localhost:3340` flag on this example. -use std::{collections::HashSet, fmt, net::SocketAddr, path::PathBuf, str::FromStr, sync::Arc}; +use std::{ + collections::HashSet, fmt, net::SocketAddr, path::PathBuf, str::FromStr, sync::Arc, + time::Instant, +}; use anyhow::{anyhow, bail}; use clap::{CommandFactory, FromArgMatches, Parser}; @@ -35,6 +38,7 @@ use serde::{Deserialize, Serialize}; use tokio::{ io::AsyncWriteExt, sync::{mpsc, oneshot}, + task::JoinHandle, }; use tracing::warn; use tracing_subscriber::{EnvFilter, Registry}; @@ -360,56 +364,58 @@ async fn handle_command( "> Hammering with prefix \"{prefix}\" for {threads} x {count} messages of size {size} bytes in {mode} mode", mode = format!("{mode:?}").to_lowercase() ); - let start = std::time::Instant::now(); - let mut handles = Vec::new(); + let start = Instant::now(); + let mut handles: Vec>> = Vec::new(); match mode { HammerMode::Set => { let mut bytes = vec![0; size]; + // TODO: Add a flag to fill content differently per entry to be able to + // test downloading too bytes.fill(97); for t in 0..threads { - let p = prefix.clone(); - let t_doc = doc.clone(); - let b = bytes.clone(); - let h = tokio::spawn(async move { + let prefix = prefix.clone(); + let doc = doc.clone(); + let bytes = bytes.clone(); + let handle = tokio::spawn(async move { for i in 0..count { - let value = String::from_utf8(b.clone()).unwrap(); - let key = format!("{}/{}/{}", p, t, i); - t_doc - .insert_bytes(key, value.into_bytes().into()) - .await - .unwrap(); + let value = String::from_utf8(bytes.clone()).unwrap(); + let key = format!("{}/{}/{}", prefix, t, i); + doc.insert_bytes(key, value.into_bytes().into()).await?; } + Ok(()) }); - handles.push(h); + handles.push(handle); } } HammerMode::Get => { for t in 0..threads { - let p = prefix.clone(); - let t_doc = doc.clone(); - let h = tokio::spawn(async move { + let prefix = prefix.clone(); + let doc = doc.clone(); + let handle = tokio::spawn(async move { for i in 0..count { - let key = format!("{}/{}/{}", p, t, i); - let entries = t_doc.replica().all_for_key(key.as_bytes()); + let key = format!("{}/{}/{}", prefix, t, i); + let entries = doc.replica().all_for_key(key.as_bytes()); for (_id, entry) in entries { - let _content = fmt_content(&t_doc, &entry).await; + let _content = fmt_content(&doc, &entry).await; } } + Ok(()) }); - handles.push(h); + handles.push(handle); } } } - let _result = futures::future::join_all(handles).await; + for result in futures::future::join_all(handles).await { + // Check that no errors ocurred + result??; + } let diff = start.elapsed().as_secs_f64(); let total_count = threads as u64 * count as u64; println!( - "> Hammering done in {:.2}s for {} messages with total of {}", - diff, - total_count, - HumanBytes(total_count * size as u64), + "> Hammering done in {diff:.2}s for {total_count} messages with total of {size}", + size = HumanBytes(total_count * size as u64), ); } Cmd::Exit => {} From 193a33a64c213fa663d8852cc4b3f2035e2f7160 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 28 Jul 2023 15:38:44 +0200 Subject: [PATCH 033/172] refactor: count actual rows, and make mode an argument --- iroh/examples/sync.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index f7bbe356a8..48afe81265 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -365,7 +365,7 @@ async fn handle_command( mode = format!("{mode:?}").to_lowercase() ); let start = Instant::now(); - let mut handles: Vec>> = Vec::new(); + let mut handles: Vec>> = Vec::new(); match mode { HammerMode::Set => { let mut bytes = vec![0; size]; @@ -382,7 +382,7 @@ async fn handle_command( let key = format!("{}/{}/{}", prefix, t, i); doc.insert_bytes(key, value.into_bytes().into()).await?; } - Ok(()) + Ok(count) }); handles.push(handle); } @@ -392,30 +392,32 @@ async fn handle_command( let prefix = prefix.clone(); let doc = doc.clone(); let handle = tokio::spawn(async move { + let mut read = 0; for i in 0..count { let key = format!("{}/{}/{}", prefix, t, i); let entries = doc.replica().all_for_key(key.as_bytes()); for (_id, entry) in entries { let _content = fmt_content(&doc, &entry).await; + read += 1; } } - Ok(()) + Ok(read) }); handles.push(handle); } } } + let mut total_count = 0; for result in futures::future::join_all(handles).await { - // Check that no errors ocurred - result??; + // Check that no errors ocurred and count rows inserted/read + total_count += result??; } let diff = start.elapsed().as_secs_f64(); - let total_count = threads as u64 * count as u64; println!( "> Hammering done in {diff:.2}s for {total_count} messages with total of {size}", - size = HumanBytes(total_count * size as u64), + size = HumanBytes(total_count as u64 * size as u64), ); } Cmd::Exit => {} @@ -573,6 +575,9 @@ pub enum Cmd { Stats, /// Hammer time - stress test with the hammer Hammer { + /// The hammer mode + #[clap(value_enum)] + mode: HammerMode, /// The key prefix prefix: String, /// The number of threads to use (each thread will create it's own replica) @@ -584,9 +589,6 @@ pub enum Cmd { /// The size of each entry in Bytes #[clap(long, short, default_value = "1024")] size: usize, - /// Select the hammer mode - #[clap(long, short, value_enum, default_value = "set")] - mode: HammerMode, }, /// Quit Exit, @@ -594,9 +596,9 @@ pub enum Cmd { #[derive(Clone, Debug, clap::ValueEnum)] pub enum HammerMode { - /// Set mode (create entries) + /// Create entries Set, - /// Get mode (read entries) + /// Read entries Get, } From b0a7ab07a0f2dd31b15af9b92f61b105081aa6ba Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 28 Jul 2023 19:07:30 +0200 Subject: [PATCH 034/172] fix: rebase fix --- iroh/src/sync/live.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 8be595b5f1..46159cd4bc 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -275,7 +275,7 @@ impl Actor { }; match event { // We received a gossip message. Try to insert it into our replica. - Event::Received(data) => { + Event::Received(data, _prev_peer) => { let op: Op = postcard::from_bytes(&data)?; match op { Op::Put(entry) => doc.insert_remote_entry(entry)?, From fb693b1641e4a31e1632d7004e581671a0066160 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Thu, 3 Aug 2023 22:22:45 +0200 Subject: [PATCH 035/172] feat(iroh-sync): implement file system backed for documents (#1315) * start refactoring store into its own module * implement more details * works again * draft fs db and integrate error handling * fill out more of the implemenation * lifetime sadness * self referential fight: Rust 0 - Dig 1 * basic tests and range fixes * introduce Store trait and update tests to test against both impls * implement remove * integrate new storage into the example * implement iterators * fixes and more tests * clippy and deny cleanup --- Cargo.lock | 59 +++ deny.toml | 1 + iroh-bytes/src/util.rs | 6 + iroh-sync/Cargo.toml | 9 + iroh-sync/src/lib.rs | 1 + iroh-sync/src/ranger.rs | 300 +++++++----- iroh-sync/src/store.rs | 79 ++++ iroh-sync/src/store/fs.rs | 751 ++++++++++++++++++++++++++++++ iroh-sync/src/store/memory.rs | 502 ++++++++++++++++++++ iroh-sync/src/sync.rs | 833 +++++++++++----------------------- iroh/Cargo.toml | 4 +- iroh/examples/sync.rs | 92 +++- iroh/src/sync.rs | 108 +++-- iroh/src/sync/content.rs | 68 ++- iroh/src/sync/live.rs | 43 +- 15 files changed, 2058 insertions(+), 798 deletions(-) create mode 100644 iroh-sync/src/store.rs create mode 100644 iroh-sync/src/store/fs.rs create mode 100644 iroh-sync/src/store/memory.rs diff --git a/Cargo.lock b/Cargo.lock index 54246da656..4cb8ff9129 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -1965,11 +1971,14 @@ dependencies = [ "hex", "iroh-bytes", "once_cell", + "ouroboros", "parking_lot", "postcard", "rand", "rand_core", + "redb", "serde", + "tempfile", "tokio", "url", ] @@ -2503,6 +2512,30 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ouroboros" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ba07320d39dfea882faa70554b4bd342a5f273ed59ba7c1c6b4c840492c954" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4c6225c69b4ca778c0aea097321a64c421cf4577b331c61b229267edabb6f8" +dependencies = [ + "heck", + "proc-macro-error 1.0.4", + "proc-macro2", + "quote", + "syn 2.0.27", +] + [[package]] name = "overload" version = "0.1.1" @@ -2965,6 +2998,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "pyo3-build-config" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "554db24f0b3c180a9c0b1268f91287ab3f17c162e15b54caaae5a6b3773396b0" +dependencies = [ + "once_cell", + "target-lexicon", +] + [[package]] name = "quanta" version = "0.11.1" @@ -3155,6 +3198,16 @@ dependencies = [ "yasna", ] +[[package]] +name = "redb" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "717a806693d0e1ed6cc55b392066bf13e703dd835acf5c5888c74740f924d355" +dependencies = [ + "libc", + "pyo3-build-config", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -4030,6 +4083,12 @@ dependencies = [ "libc", ] +[[package]] +name = "target-lexicon" +version = "0.12.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2faeef5759ab89935255b1a4cd98e0baf99d1085e37d36599c625dac49ae8e" + [[package]] name = "tempfile" version = "3.7.0" diff --git a/deny.toml b/deny.toml index 517d074102..023b15271a 100644 --- a/deny.toml +++ b/deny.toml @@ -4,6 +4,7 @@ multiple-versions = "allow" [licenses] allow = [ "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", "BSD-2-Clause", "BSD-3-Clause", "BSL-1.0", # BOSL license diff --git a/iroh-bytes/src/util.rs b/iroh-bytes/src/util.rs index 490c9e5a3b..56e83051a4 100644 --- a/iroh-bytes/src/util.rs +++ b/iroh-bytes/src/util.rs @@ -83,6 +83,12 @@ impl From<[u8; 32]> for Hash { } } +impl From<&[u8; 32]> for Hash { + fn from(value: &[u8; 32]) -> Self { + Hash(blake3::Hash::from(*value)) + } +} + impl PartialOrd for Hash { fn partial_cmp(&self, other: &Self) -> Option { Some(self.0.as_bytes().cmp(other.0.as_bytes())) diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 0d0c9b891a..e0132aed13 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -25,5 +25,14 @@ bytes = "1.4.0" parking_lot = "0.12.1" hex = "0.4" +# fs-store +redb = { version = "1.0.5", optional = true } +ouroboros = { version = "0.17", optional = true } + [dev-dependencies] tokio = { version = "1.28.2", features = ["sync", "macros"] } +tempfile = "3.4" + +[features] +default = ["fs-store"] +fs-store = ["redb", "ouroboros"] \ No newline at end of file diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index a37ead1b6f..4c73579e2d 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -1,2 +1,3 @@ pub mod ranger; +pub mod store; pub mod sync; diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index e12db74498..aff7f66578 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -2,10 +2,10 @@ //! "Range-Based Set Reconciliation" by Aljoscha Meyer. //! -use std::cmp::Ordering; use std::collections::BTreeMap; use std::fmt::Debug; use std::marker::PhantomData; +use std::{cmp::Ordering, convert::Infallible}; use serde::{Deserialize, Serialize}; @@ -179,12 +179,12 @@ where K: RangeKey + Clone + Default + AsFingerprint, { /// Construct the initial message. - fn init>(store: &S, limit: Option<&Range>) -> Self { - let x = store.get_first(); + fn init>(store: &S, limit: Option<&Range>) -> Result { + let x = store.get_first()?; let range = Range::new(x.clone(), x); - let fingerprint = store.get_fingerprint(&range, limit); + let fingerprint = store.get_fingerprint(&range, limit)?; let part = MessagePart::RangeFingerprint(RangeFingerprint { range, fingerprint }); - Message { parts: vec![part] } + Ok(Message { parts: vec![part] }) } pub fn parts(&self) -> &[MessagePart] { @@ -192,37 +192,47 @@ where } } -pub trait Store: Sized + Default +pub trait Store: Sized where K: RangeKey + Clone + Default + AsFingerprint, { + type Error: Debug + Send + Sync + Into; + /// Get a the first key (or the default if none is available). - fn get_first(&self) -> K; - fn get(&self, key: &K) -> Option<&V>; - fn len(&self) -> usize; - fn is_empty(&self) -> bool; + fn get_first(&self) -> Result; + fn get(&self, key: &K) -> Result, Self::Error>; + fn len(&self) -> Result; + fn is_empty(&self) -> Result; /// Calculate the fingerprint of the given range. - fn get_fingerprint(&self, range: &Range, limit: Option<&Range>) -> Fingerprint; + fn get_fingerprint( + &self, + range: &Range, + limit: Option<&Range>, + ) -> Result; /// Insert the given key value pair. - fn put(&mut self, k: K, v: V); + fn put(&mut self, k: K, v: V) -> Result<(), Self::Error>; - type RangeIterator<'a>: Iterator + type RangeIterator<'a>: Iterator> where Self: 'a, K: 'a, V: 'a; /// Returns all items in the given range - fn get_range(&self, range: Range, limit: Option>) -> Self::RangeIterator<'_>; - fn remove(&mut self, key: &K) -> Option; + fn get_range( + &self, + range: Range, + limit: Option>, + ) -> Result, Self::Error>; + fn remove(&mut self, key: &K) -> Result, Self::Error>; - type AllIterator<'a>: Iterator + type AllIterator<'a>: Iterator> where Self: 'a, K: 'a, V: 'a; - fn all(&self) -> Self::AllIterator<'_>; + fn all(&self) -> Result, Self::Error>; } #[derive(Debug)] @@ -241,94 +251,119 @@ impl Default for SimpleStore { impl Store for SimpleStore where K: RangeKey + Clone + Default + AsFingerprint, + V: Clone, { - fn get_first(&self) -> K { + type Error = Infallible; + + fn get_first(&self) -> Result { if let Some((k, _)) = self.data.first_key_value() { - k.clone() + Ok(k.clone()) } else { - Default::default() + Ok(Default::default()) } } - fn get(&self, key: &K) -> Option<&V> { - self.data.get(key) + fn get(&self, key: &K) -> Result, Self::Error> { + Ok(self.data.get(key).cloned()) } - fn len(&self) -> usize { - self.data.len() + fn len(&self) -> Result { + Ok(self.data.len()) } - fn is_empty(&self) -> bool { - self.data.is_empty() + fn is_empty(&self) -> Result { + Ok(self.data.is_empty()) } /// Calculate the fingerprint of the given range. - fn get_fingerprint(&self, range: &Range, limit: Option<&Range>) -> Fingerprint { - let elements = self.get_range(range.clone(), limit.cloned()); + fn get_fingerprint( + &self, + range: &Range, + limit: Option<&Range>, + ) -> Result { + let elements = self.get_range(range.clone(), limit.cloned())?; let mut fp = Fingerprint::empty(); for el in elements { + let el = el?; fp ^= el.0.as_fingerprint(); } - fp + Ok(fp) } /// Insert the given key value pair. - fn put(&mut self, k: K, v: V) { + fn put(&mut self, k: K, v: V) -> Result<(), Self::Error> { self.data.insert(k, v); + Ok(()) } type RangeIterator<'a> = SimpleRangeIterator<'a, K, V> where K: 'a, V: 'a; /// Returns all items in the given range - fn get_range(&self, range: Range, limit: Option>) -> Self::RangeIterator<'_> { + fn get_range( + &self, + range: Range, + limit: Option>, + ) -> Result, Self::Error> { // TODO: this is not very efficient, optimize depending on data structure let iter = self.data.iter(); - SimpleRangeIterator { iter, range, limit } + Ok(SimpleRangeIterator { + iter, + range: Some(range), + limit, + }) } - fn remove(&mut self, key: &K) -> Option { - self.data.remove(key) + fn remove(&mut self, key: &K) -> Result, Self::Error> { + // No versions stored + + let res = self.data.remove(key).into_iter().collect(); + Ok(res) } - type AllIterator<'a> = std::collections::btree_map::Iter<'a, K, V> + type AllIterator<'a> = SimpleRangeIterator<'a, K, V> where K: 'a, V: 'a; - fn all(&self) -> Self::AllIterator<'_> { - self.data.iter() + fn all(&self) -> Result, Self::Error> { + let iter = self.data.iter(); + + Ok(SimpleRangeIterator { + iter, + range: None, + limit: None, + }) } } #[derive(Debug)] pub struct SimpleRangeIterator<'a, K: 'a, V: 'a> { iter: std::collections::btree_map::Iter<'a, K, V>, - range: Range, + range: Option>, limit: Option>, } impl<'a, K, V> Iterator for SimpleRangeIterator<'a, K, V> where - K: RangeKey, + K: RangeKey + Clone, + V: Clone, { - type Item = (&'a K, &'a V); + type Item = Result<(K, V), Infallible>; fn next(&mut self) -> Option { let mut next = self.iter.next()?; - let filter = |x: &K| { - let r = x.contains(&self.range); - if let Some(ref limit) = self.limit { - r && x.contains(limit) - } else { - r - } + let filter = |x: &K| match (&self.range, &self.limit) { + (None, None) => true, + (Some(ref range), Some(ref limit)) => x.contains(range) && x.contains(limit), + (Some(ref range), None) => x.contains(range), + (None, Some(ref limit)) => x.contains(limit), }; loop { if filter(next.0) { - return Some(next); + return Some(Ok((next.0.clone(), next.1.clone()))); } next = self.iter.next()?; @@ -389,14 +424,28 @@ where V: Clone + Debug, S: Store, { + pub fn from_store(store: S) -> Self { + Peer { + store, + max_set_size: 1, + split_factor: 2, + limit: None, + _phantom: Default::default(), + } + } + /// Generates the initial message. - pub fn initial_message(&self) -> Message { + pub fn initial_message(&self) -> Result, S::Error> { Message::init(&self.store, self.limit.as_ref()) } /// Processes an incoming message and produces a response. /// If terminated, returns `None` - pub fn process_message(&mut self, message: Message, cb: F) -> Option> + pub fn process_message( + &mut self, + message: Message, + cb: F, + ) -> Result>, S::Error> where F: Fn(K, V), { @@ -428,17 +477,25 @@ where } else { Some( self.store - .get_range(range.clone(), self.limit.clone()) - .filter(|(k, _)| !values.iter().any(|(vk, _)| &vk == k)) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), + .get_range(range.clone(), self.limit.clone())? + .filter_map(|el| match el { + Ok((k, v)) => { + if !values.iter().any(|(vk, _)| vk == &k) { + Some(Ok((k, v))) + } else { + None + } + } + Err(err) => Some(Err(err)), + }) + .collect::>()?, ) }; // Store incoming values for (k, v) in values { cb(k.clone(), v.clone()); - self.store.put(k, v); + self.store.put(k, v)?; } if let Some(diff) = diff { @@ -454,7 +511,7 @@ where // Process fingerprint messages for RangeFingerprint { range, fingerprint } in fingerprints { - let local_fingerprint = self.store.get_fingerprint(&range, self.limit.as_ref()); + let local_fingerprint = self.store.get_fingerprint(&range, self.limit.as_ref())?; // Case1 Match, nothing to do if local_fingerprint == fingerprint { @@ -464,13 +521,10 @@ where // Case2 Recursion Anchor let local_values: Vec<_> = self .store - .get_range(range.clone(), self.limit.clone()) - .collect(); + .get_range(range.clone(), self.limit.clone())? + .collect::>()?; if local_values.len() <= 1 || fingerprint == Fingerprint::empty() { - let values = local_values - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); + let values = local_values.into_iter().map(|(k, v)| (k, v)).collect(); out.push(MessagePart::RangeItem(RangeItem { range, values, @@ -498,13 +552,13 @@ where let (x, y) = if i == 0 { // first - (range.x(), local_values[end].0) + (range.x(), &local_values[end].0) } else if i == self.split_factor - 1 { // last - (local_values[start].0, range.y()) + (&local_values[start].0, range.y()) } else { // regular - (local_values[start].0, local_values[end].0) + (&local_values[start].0, &local_values[end].0) }; let range = Range::new(x.clone(), y.clone()); ranges.push(range); @@ -514,10 +568,10 @@ where for range in ranges.into_iter() { let chunk: Vec<_> = self .store - .get_range(range.clone(), self.limit.clone()) + .get_range(range.clone(), self.limit.clone())? .collect(); // Add either the fingerprint or the item set - let fingerprint = self.store.get_fingerprint(&range, self.limit.as_ref()); + let fingerprint = self.store.get_fingerprint(&range, self.limit.as_ref())?; if chunk.len() > self.max_set_size { out.push(MessagePart::RangeFingerprint(RangeFingerprint { range, @@ -526,12 +580,15 @@ where } else { let values = chunk .into_iter() - .map(|(k, v)| { - let k: K = k.clone(); - let v: V = v.clone(); - (k, v) + .map(|el| match el { + Ok((k, v)) => { + let k: K = k; + let v: V = v; + Ok((k, v)) + } + Err(err) => Err(err), }) - .collect(); + .collect::>()?; out.push(MessagePart::RangeItem(RangeItem { range, values, @@ -544,28 +601,28 @@ where // If we have any parts, return a message if !out.is_empty() { - Some(Message { parts: out }) + Ok(Some(Message { parts: out })) } else { - None + Ok(None) } } /// Insert a key value pair. - pub fn put(&mut self, k: K, v: V) { - self.store.put(k, v); + pub fn put(&mut self, k: K, v: V) -> Result<(), S::Error> { + self.store.put(k, v) } - pub fn get(&self, k: &K) -> Option<&V> { + pub fn get(&self, k: &K) -> Result, S::Error> { self.store.get(k) } /// Remove the given key. - pub fn remove(&mut self, k: &K) -> Option { + pub fn remove(&mut self, k: &K) -> Result, S::Error> { self.store.remove(k) } /// List all existing key value pairs. - pub fn all(&self) -> impl Iterator { + pub fn all(&self) -> Result> + '_, S::Error> { self.store.all() } @@ -941,6 +998,7 @@ mod tests { struct SyncResult where K: RangeKey + Clone + Default + AsFingerprint, + V: Clone, { alice: Peer, bob: Peer, @@ -951,7 +1009,7 @@ mod tests { impl SyncResult where K: RangeKey + Clone + Default + AsFingerprint + Debug, - V: Debug, + V: Clone + Debug, { fn print_messages(&self) { let len = std::cmp::max(self.alice_to_bob.len(), self.bob_to_alice.len()); @@ -974,32 +1032,42 @@ mod tests { V: Debug + Clone + PartialEq, { fn assert_alice_set(&self, ctx: &str, expected: &[(K, V)]) { - dbg!(self.alice.all().collect::>()); + dbg!(self.alice.all().unwrap().collect::>()); for (k, v) in expected { assert_eq!( - self.alice.store.get(k), + self.alice.store.get(k).unwrap().as_ref(), Some(v), "{}: (alice) missing key {:?}", ctx, k ); } - assert_eq!(expected.len(), self.alice.store.len(), "{}: (alice)", ctx); + assert_eq!( + expected.len(), + self.alice.store.len().unwrap(), + "{}: (alice)", + ctx + ); } fn assert_bob_set(&self, ctx: &str, expected: &[(K, V)]) { - dbg!(self.bob.all().collect::>()); + dbg!(self.bob.all().unwrap().collect::>()); for (k, v) in expected { assert_eq!( - self.bob.store.get(k), + self.bob.store.get(k).unwrap().as_ref(), Some(v), "{}: (bob) missing key {:?}", ctx, k ); } - assert_eq!(expected.len(), self.bob.store.len(), "{}: (bob)", ctx); + assert_eq!( + expected.len(), + self.bob.store.len().unwrap(), + "{}: (bob)", + ctx + ); } } @@ -1054,7 +1122,7 @@ mod tests { Peer::::default() }; for (k, v) in alice_set { - alice.put(k.clone(), v.clone()); + alice.put(k.clone(), v.clone()).unwrap(); let include = if let Some(ref limit) = limit { k.contains(limit) @@ -1074,7 +1142,7 @@ mod tests { Peer::::default() }; for (k, v) in bob_set { - bob.put(k.clone(), v.clone()); + bob.put(k.clone(), v.clone()).unwrap(); let include = if let Some(ref limit) = limit { k.contains(limit) } else { @@ -1089,7 +1157,7 @@ mod tests { let mut alice_to_bob = Vec::new(); let mut bob_to_alice = Vec::new(); - let initial_message = alice.initial_message(); + let initial_message = alice.initial_message().unwrap(); let mut next_to_bob = Some(initial_message); let mut rounds = 0; @@ -1098,9 +1166,9 @@ mod tests { rounds += 1; alice_to_bob.push(msg.clone()); - if let Some(msg) = bob.process_message(msg, |_, _| {}) { + if let Some(msg) = bob.process_message(msg, |_, _| {}).unwrap() { bob_to_alice.push(msg.clone()); - next_to_bob = alice.process_message(msg, |_, _| {}); + next_to_bob = alice.process_message(msg, |_, _| {}).unwrap(); } } let res = SyncResult { @@ -1111,15 +1179,19 @@ mod tests { }; res.print_messages(); - let alice_now: Vec<_> = res.alice.all().collect(); + let alice_now: Vec<_> = res.alice.all().unwrap().collect::>().unwrap(); assert_eq!( - expected_set_alice.iter().collect::>(), + expected_set_alice.into_iter().collect::>(), alice_now, "alice" ); - let bob_now: Vec<_> = res.bob.all().collect(); - assert_eq!(expected_set_bob.iter().collect::>(), bob_now, "bob"); + let bob_now: Vec<_> = res.bob.all().unwrap().collect::>().unwrap(); + assert_eq!( + expected_set_bob.into_iter().collect::>(), + bob_now, + "bob" + ); // Check that values were never sent twice let mut alice_sent = BTreeMap::new(); @@ -1169,38 +1241,44 @@ mod tests { ("hog", 1), ]; for (k, v) in &set { - store.put(*k, *v); + store.put(*k, *v).unwrap(); } let all: Vec<_> = store .get_range(Range::new("", ""), None) - .map(|(k, v)| (*k, *v)) - .collect(); + .unwrap() + .collect::>() + .unwrap(); assert_eq!(&all, &set[..]); let regular: Vec<_> = store .get_range(("bee", "eel").into(), None) - .map(|(k, v)| (*k, *v)) - .collect(); + .unwrap() + .collect::>() + .unwrap(); assert_eq!(®ular, &set[..3]); // empty start let regular: Vec<_> = store .get_range(("", "eel").into(), None) - .map(|(k, v)| (*k, *v)) - .collect(); + .unwrap() + .collect::>() + .unwrap(); assert_eq!(®ular, &set[..3]); let regular: Vec<_> = store .get_range(("cat", "hog").into(), None) - .map(|(k, v)| (*k, *v)) - .collect(); + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(®ular, &set[1..5]); let excluded: Vec<_> = store .get_range(("fox", "bee").into(), None) - .map(|(k, v)| (*k, *v)) - .collect(); + .unwrap() + .collect::>() + .unwrap(); assert_eq!(excluded[0].0, "fox"); assert_eq!(excluded[1].0, "hog"); @@ -1208,8 +1286,9 @@ mod tests { let excluded: Vec<_> = store .get_range(("fox", "doe").into(), None) - .map(|(k, v)| (*k, *v)) - .collect(); + .unwrap() + .collect::>() + .unwrap(); assert_eq!(excluded.len(), 4); assert_eq!(excluded[0].0, "bee"); @@ -1220,8 +1299,9 @@ mod tests { // Limit let all: Vec<_> = store .get_range(("", "").into(), Some(("bee", "doe").into())) - .map(|(k, v)| (*k, *v)) - .collect(); + .unwrap() + .collect::>() + .unwrap(); assert_eq!(&all, &set[..2]); } diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs new file mode 100644 index 0000000000..a28b9a904a --- /dev/null +++ b/iroh-sync/src/store.rs @@ -0,0 +1,79 @@ +use anyhow::Result; +use rand_core::CryptoRngCore; + +use crate::{ + ranger, + sync::{Author, AuthorId, Namespace, NamespaceId, RecordIdentifier, Replica, SignedEntry}, +}; + +#[cfg(feature = "fs-store")] +pub mod fs; +pub mod memory; + +/// Abstraction over the different available storage solutions. +pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { + /// The specialized instance scoped to a `Namespace`. + type Instance: ranger::Store + Send + Sync + 'static + Clone; + + type GetLatestIter<'a>: Iterator> + where + Self: 'a; + type GetAllIter<'a>: Iterator> + where + Self: 'a; + + fn get_replica(&self, namespace: &NamespaceId) -> Result>>; + fn get_author(&self, author: &AuthorId) -> Result>; + fn new_author(&self, rng: &mut R) -> Result; + fn new_replica(&self, namespace: Namespace) -> Result>; + + /// Gets all entries matching this key and author. + fn get_latest_by_key_and_author( + &self, + namespace: NamespaceId, + author: AuthorId, + key: impl AsRef<[u8]>, + ) -> Result>; + + /// Returns the latest version of the matching documents by key. + fn get_latest_by_key( + &self, + namespace: NamespaceId, + key: impl AsRef<[u8]>, + ) -> Result>; + + /// Returns the latest version of the matching documents by prefix. + fn get_latest_by_prefix( + &self, + namespace: NamespaceId, + prefix: impl AsRef<[u8]>, + ) -> Result>; + + /// Returns the latest versions of all documents. + fn get_latest(&self, namespace: NamespaceId) -> Result>; + + /// Returns all versions of the matching documents by author. + fn get_all_by_key_and_author<'a, 'b: 'a>( + &'a self, + namespace: NamespaceId, + author: AuthorId, + key: impl AsRef<[u8]> + 'b, + ) -> Result>; + + /// Returns all versions of the matching documents by key. + fn get_all_by_key( + &self, + namespace: NamespaceId, + key: impl AsRef<[u8]>, + ) -> Result>; + + /// Returns all versions of the matching documents by prefix. + fn get_all_by_prefix( + &self, + namespace: NamespaceId, + prefix: impl AsRef<[u8]>, + ) -> Result>; + + /// Returns all versions of all documents. + fn get_all(&self, namespace: NamespaceId) -> Result>; +} diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs new file mode 100644 index 0000000000..e490c177a2 --- /dev/null +++ b/iroh-sync/src/store/fs.rs @@ -0,0 +1,751 @@ +//! On disk storage for replicas. + +use std::{path::Path, sync::Arc}; + +use anyhow::Result; +use ouroboros::self_referencing; +use rand_core::CryptoRngCore; +use redb::{ + AccessGuard, Database, MultimapRange, MultimapTableDefinition, MultimapValue, + ReadOnlyMultimapTable, ReadTransaction, ReadableMultimapTable, ReadableTable, TableDefinition, +}; + +use crate::{ + ranger::{AsFingerprint, Fingerprint, Range, RangeKey}, + store::Store as _, + sync::{ + Author, AuthorId, Entry, EntrySignature, Namespace, NamespaceId, Record, RecordIdentifier, + Replica, SignedEntry, + }, +}; + +use self::ouroboros_impl_range_all_iterator::BorrowedMutFields; + +/// Manages the replicas and authors for an instance. +#[derive(Debug, Clone)] +pub struct Store { + db: Arc, +} + +// Table Definitions + +// Authors +// Table +// Key: [u8; 32] # AuthorId +// Value: #[u8; 32] # Author +const AUTHORS_TABLE: TableDefinition<&[u8; 32], &[u8; 32]> = TableDefinition::new("authors-1"); + +// Namespaces +// Table +// Key: [u8; 32] # NamespaceId +// Value: #[u8; 32] # Namespace +const NAMESPACES_TABLE: TableDefinition<&[u8; 32], &[u8; 32]> = + TableDefinition::new("namespaces-1"); + +// Records +// Multimap +// Key: ([u8; 32], [u8; 32], Vec) # (NamespaceId, AuthorId, Key) +// Values: +// (u64, [u8; 32], [u8; 32], u64, [u8; 32]) +// # (timestamp, signature_namespace, signature_author, len, hash) + +type RecordsId<'a> = (&'a [u8; 32], &'a [u8; 32], &'a [u8]); +type RecordsValue<'a> = (u64, &'a [u8; 64], &'a [u8; 64], u64, &'a [u8; 32]); + +const RECORDS_TABLE: MultimapTableDefinition = + MultimapTableDefinition::new("records-1"); + +impl Store { + pub fn new(path: impl AsRef) -> Result { + let db = Database::create(path)?; + + // Setup all tables + let write_tx = db.begin_write()?; + { + let _table = write_tx.open_multimap_table(RECORDS_TABLE)?; + let _table = write_tx.open_table(NAMESPACES_TABLE)?; + let _table = write_tx.open_table(AUTHORS_TABLE)?; + } + write_tx.commit()?; + + Ok(Store { db: Arc::new(db) }) + } + /// Stores a new namespace + fn insert_namespace(&self, namespace: Namespace) -> Result<()> { + let write_tx = self.db.begin_write()?; + { + let mut namespace_table = write_tx.open_table(NAMESPACES_TABLE)?; + namespace_table.insert(&namespace.id_bytes(), &namespace.to_bytes())?; + } + write_tx.commit()?; + + Ok(()) + } + + fn insert_author(&self, author: Author) -> Result<()> { + let write_tx = self.db.begin_write()?; + { + let mut author_table = write_tx.open_table(AUTHORS_TABLE)?; + author_table.insert(&author.id_bytes(), &author.to_bytes())?; + } + write_tx.commit()?; + + Ok(()) + } +} + +impl super::Store for Store { + type Instance = StoreInstance; + type GetAllIter<'a> = RangeAllIterator<'a>; + type GetLatestIter<'a> = RangeLatestIterator<'a>; + + fn get_replica(&self, namespace_id: &NamespaceId) -> Result>> { + let read_tx = self.db.begin_read()?; + let namespace_table = read_tx.open_table(NAMESPACES_TABLE)?; + let Some(namespace) = namespace_table.get(namespace_id.as_bytes())? else { + return Ok(None); + }; + let namespace = Namespace::from_bytes(namespace.value()); + let replica = Replica::new(namespace, StoreInstance::new(*namespace_id, self.clone())); + Ok(Some(replica)) + } + + fn get_author(&self, author_id: &AuthorId) -> Result> { + let read_tx = self.db.begin_read()?; + let author_table = read_tx.open_table(AUTHORS_TABLE)?; + let Some(author) = author_table.get(author_id.as_bytes())? else { + return Ok(None); + }; + + let author = Author::from_bytes(author.value()); + Ok(Some(author)) + } + + /// Generates a new author, using the passed in randomness. + fn new_author(&self, rng: &mut R) -> Result { + let author = Author::new(rng); + self.insert_author(author.clone())?; + Ok(author) + } + + fn new_replica(&self, namespace: Namespace) -> Result> { + let id = namespace.id(); + self.insert_namespace(namespace.clone())?; + + let replica = Replica::new(namespace, StoreInstance::new(id, self.clone())); + + Ok(replica) + } + + /// Gets all entries matching this key and author. + fn get_latest_by_key_and_author( + &self, + namespace: NamespaceId, + author: AuthorId, + key: impl AsRef<[u8]>, + ) -> Result> { + let read_tx = self.db.begin_read()?; + let record_table = read_tx.open_multimap_table(RECORDS_TABLE)?; + + let db_key = (namespace.as_bytes(), author.as_bytes(), key.as_ref()); + let records = record_table.get(db_key)?; + let Some(record) = records.last() else { + return Ok(None); + }; + let record = record?; + let (timestamp, namespace_sig, author_sig, len, hash) = record.value(); + let record = Record::new(timestamp, len, hash.into()); + let id = RecordIdentifier::new(key, namespace, author); + let entry = Entry::new(id, record); + let entry_signature = EntrySignature::from_parts(namespace_sig, author_sig); + let signed_entry = SignedEntry::new(entry_signature, entry); + + Ok(Some(signed_entry)) + } + + fn get_latest_by_key( + &self, + namespace: NamespaceId, + key: impl AsRef<[u8]>, + ) -> Result> { + let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); + let iter = RangeLatestIterator::try_new( + self.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| record_table.range(start..=end).map_err(anyhow::Error::from), + None, + RangeFilter::Key(key.as_ref().to_vec()), + )?; + + Ok(iter) + } + + fn get_latest_by_prefix( + &self, + namespace: NamespaceId, + prefix: impl AsRef<[u8]>, + ) -> Result> { + let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); + let iter = RangeLatestIterator::try_new( + self.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| record_table.range(start..=end).map_err(anyhow::Error::from), + None, + RangeFilter::Prefix(prefix.as_ref().to_vec()), + )?; + + Ok(iter) + } + + fn get_latest(&self, namespace: NamespaceId) -> Result> { + let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); + let iter = RangeLatestIterator::try_new( + self.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| record_table.range(start..=end).map_err(anyhow::Error::from), + None, + RangeFilter::None, + )?; + + Ok(iter) + } + + fn get_all_by_key_and_author<'a, 'b: 'a>( + &'a self, + namespace: NamespaceId, + author: AuthorId, + key: impl AsRef<[u8]> + 'b, + ) -> Result> { + let start = (namespace.as_bytes(), author.as_bytes(), key.as_ref()); + let end = (namespace.as_bytes(), author.as_bytes(), key.as_ref()); + let iter = RangeAllIterator::try_new( + self.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| { + record_table + .range(start..=end) + .map_err(anyhow::Error::from) + .map(|v| (v, None)) + }, + RangeFilter::None, + )?; + + Ok(iter) + } + + fn get_all_by_key( + &self, + namespace: NamespaceId, + key: impl AsRef<[u8]>, + ) -> Result> { + let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); + let iter = RangeAllIterator::try_new( + self.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| { + record_table + .range(start..=end) + .map_err(anyhow::Error::from) + .map(|v| (v, None)) + }, + RangeFilter::Key(key.as_ref().to_vec()), + )?; + + Ok(iter) + } + + fn get_all_by_prefix( + &self, + namespace: NamespaceId, + prefix: impl AsRef<[u8]>, + ) -> Result> { + let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); + let iter = RangeAllIterator::try_new( + self.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| { + record_table + .range(start..=end) + .map_err(anyhow::Error::from) + .map(|v| (v, None)) + }, + RangeFilter::Prefix(prefix.as_ref().to_vec()), + )?; + + Ok(iter) + } + + fn get_all(&self, namespace: NamespaceId) -> Result> { + let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); + let iter = RangeAllIterator::try_new( + self.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| { + record_table + .range(start..=end) + .map_err(anyhow::Error::from) + .map(|v| (v, None)) + }, + RangeFilter::None, + )?; + + Ok(iter) + } +} + +/// [`Namespace`] specific wrapper around the [`Store`]. +#[derive(Debug, Clone)] +pub struct StoreInstance { + namespace: NamespaceId, + store: Store, +} + +impl StoreInstance { + fn new(namespace: NamespaceId, store: Store) -> Self { + StoreInstance { namespace, store } + } +} + +impl crate::ranger::Store for StoreInstance { + type Error = anyhow::Error; + + /// Get a the first key (or the default if none is available). + fn get_first(&self) -> Result { + let read_tx = self.store.db.begin_read()?; + let record_table = read_tx.open_multimap_table(RECORDS_TABLE)?; + + // TODO: verify this fetches all keys with this namespace + let start = (self.namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (self.namespace.as_bytes(), &[255u8; 32], &[][..]); + let mut records = record_table.range(start..=end)?; + + let Some(record) = records.next() else { + return Ok(RecordIdentifier::default()); + }; + let (compound_key, _) = record?; + let (namespace_id, author_id, key) = compound_key.value(); + + let id = RecordIdentifier::from_parts(key, namespace_id, author_id)?; + Ok(id) + } + + fn get(&self, id: &RecordIdentifier) -> Result> { + self.store + .get_latest_by_key_and_author(id.namespace(), id.author(), id.key()) + } + + fn len(&self) -> Result { + let read_tx = self.store.db.begin_read()?; + let record_table = read_tx.open_multimap_table(RECORDS_TABLE)?; + + // TODO: verify this fetches all keys with this namespace + let start = (self.namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (self.namespace.as_bytes(), &[255u8; 32], &[][..]); + let records = record_table.range(start..=end)?; + Ok(records.count()) + } + + fn is_empty(&self) -> Result { + Ok(self.len()? == 0) + } + + fn get_fingerprint( + &self, + range: &Range, + limit: Option<&Range>, + ) -> Result { + // TODO: optimize? + + let elements = self.get_range(range.clone(), limit.cloned())?; + let mut fp = Fingerprint::empty(); + for el in elements { + let el = el?; + fp ^= el.0.as_fingerprint(); + } + + Ok(fp) + } + + fn put(&mut self, k: RecordIdentifier, v: SignedEntry) -> Result<()> { + // TODO: propagate error/not insertion? + if v.verify().is_ok() { + let timestamp = v.entry().record().timestamp(); + // TODO: verify timestamp is "reasonable" + + let write_tx = self.store.db.begin_write()?; + { + let mut record_table = write_tx.open_multimap_table(RECORDS_TABLE)?; + let key = (k.namespace_bytes(), k.author_bytes(), k.key()); + let record = v.entry().record(); + let value = ( + timestamp, + &v.signature().namespace_signature().to_bytes(), + &v.signature().author_signature().to_bytes(), + record.content_len(), + record.content_hash().as_bytes(), + ); + record_table.insert(key, value)?; + } + write_tx.commit()?; + } + Ok(()) + } + + type RangeIterator<'a> = RangeLatestIterator<'a>; + fn get_range( + &self, + range: Range, + limit: Option>, + ) -> Result> { + // TODO: implement inverted range + let range_start = range.x(); + let range_end = range.y(); + + let start = ( + range_start.namespace_bytes(), + range_start.author_bytes(), + range_start.key(), + ); + let end = ( + range_end.namespace_bytes(), + range_end.author_bytes(), + range_end.key(), + ); + let iter = RangeLatestIterator::try_new( + self.store.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| record_table.range(start..=end).map_err(anyhow::Error::from), + limit, + RangeFilter::None, + )?; + + Ok(iter) + } + + fn remove(&mut self, k: &RecordIdentifier) -> Result> { + let write_tx = self.store.db.begin_write()?; + let res = { + let mut records_table = write_tx.open_multimap_table(RECORDS_TABLE)?; + let key = (k.namespace_bytes(), k.author_bytes(), k.key()); + let records = records_table.remove_all(key)?; + let mut res = Vec::new(); + for record in records.into_iter() { + let record = record?; + let (timestamp, namespace_sig, author_sig, len, hash) = record.value(); + let record = Record::new(timestamp, len, hash.into()); + let entry = Entry::new(k.clone(), record); + let entry_signature = EntrySignature::from_parts(namespace_sig, author_sig); + let signed_entry = SignedEntry::new(entry_signature, entry); + res.push(signed_entry); + } + res + }; + write_tx.commit()?; + Ok(res) + } + + type AllIterator<'a> = RangeLatestIterator<'a>; + + fn all(&self) -> Result> { + let start = (self.namespace.as_bytes(), &[0u8; 32], &[][..]); + let end = (self.namespace.as_bytes(), &[255u8; 32], &[][..]); + let iter = RangeLatestIterator::try_new( + self.store.db.begin_read()?, + |read_tx| { + read_tx + .open_multimap_table(RECORDS_TABLE) + .map_err(anyhow::Error::from) + }, + |record_table| record_table.range(start..=end).map_err(anyhow::Error::from), + None, + RangeFilter::None, + )?; + + Ok(iter) + } +} + +fn matches(limit: &Option>, x: &RecordIdentifier) -> bool { + limit.as_ref().map(|r| x.contains(r)).unwrap_or(true) +} + +#[self_referencing] +pub struct RangeLatestIterator<'a> { + read_tx: ReadTransaction<'a>, + #[borrows(read_tx)] + #[covariant] + record_table: ReadOnlyMultimapTable<'this, RecordsId<'static>, RecordsValue<'static>>, + #[covariant] + #[borrows(record_table)] + records: MultimapRange<'this, RecordsId<'static>, RecordsValue<'static>>, + limit: Option>, + filter: RangeFilter, +} + +impl std::fmt::Debug for RangeLatestIterator<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RangeLatestIterator") + .finish_non_exhaustive() + } +} + +impl Iterator for RangeLatestIterator<'_> { + type Item = Result<(RecordIdentifier, SignedEntry)>; + + fn next(&mut self) -> Option { + self.with_mut(|fields| { + for next in fields.records.by_ref() { + let next = match next { + Ok(next) => next, + Err(err) => return Some(Err(err.into())), + }; + + let (namespace, author, key) = next.0.value(); + let id = match RecordIdentifier::from_parts(key, namespace, author) { + Ok(id) => id, + Err(err) => return Some(Err(err)), + }; + if fields.filter.matches(&id) && matches(fields.limit, &id) { + let last = next.1.last(); + let value = match last? { + Ok(value) => value, + Err(err) => return Some(Err(err.into())), + }; + let (timestamp, namespace_sig, author_sig, len, hash) = value.value(); + let record = Record::new(timestamp, len, hash.into()); + let entry = Entry::new(id.clone(), record); + let entry_signature = EntrySignature::from_parts(namespace_sig, author_sig); + let signed_entry = SignedEntry::new(entry_signature, entry); + + return Some(Ok((id, signed_entry))); + } + } + None + }) + } +} + +#[self_referencing] +pub struct RangeAllIterator<'a> { + read_tx: ReadTransaction<'a>, + #[borrows(read_tx)] + #[covariant] + record_table: ReadOnlyMultimapTable<'this, RecordsId<'static>, RecordsValue<'static>>, + #[covariant] + #[borrows(record_table)] + records: ( + MultimapRange<'this, RecordsId<'static>, RecordsValue<'static>>, + Option<( + AccessGuard<'this, RecordsId<'static>>, + MultimapValue<'this, RecordsValue<'static>>, + RecordIdentifier, + )>, + ), + filter: RangeFilter, +} + +#[derive(Debug)] +enum RangeFilter { + None, + Prefix(Vec), + Key(Vec), +} + +impl RangeFilter { + fn matches(&self, id: &RecordIdentifier) -> bool { + match self { + RangeFilter::None => true, + RangeFilter::Prefix(ref prefix) => id.key().starts_with(prefix), + RangeFilter::Key(ref key) => id.key() == key, + } + } +} + +impl std::fmt::Debug for RangeAllIterator<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RangeAllIterator").finish_non_exhaustive() + } +} + +/// Advance the internal iterator to the next set of multimap values +fn next_iter(fields: &mut BorrowedMutFields) -> Result<()> { + for next_iter in fields.records.0.by_ref() { + let (id_guard, values_guard) = next_iter?; + let (namespace, author, key) = id_guard.value(); + let id = RecordIdentifier::from_parts(key, namespace, author)?; + if fields.filter.matches(&id) { + fields.records.1 = Some((id_guard, values_guard, id)); + return Ok(()); + } + } + Ok(()) +} + +impl Iterator for RangeAllIterator<'_> { + type Item = Result<(RecordIdentifier, SignedEntry)>; + + fn next(&mut self) -> Option { + self.with_mut(|mut fields| { + loop { + if fields.records.1.is_none() { + if let Err(err) = next_iter(&mut fields) { + return Some(Err(err)); + } + } + // If this is None, nothing is available anymore + let (_id_guard, values_guard, id) = fields.records.1.as_mut()?; + + match values_guard.next() { + Some(Ok(value)) => { + let (timestamp, namespace_sig, author_sig, len, hash) = value.value(); + let record = Record::new(timestamp, len, hash.into()); + let entry = Entry::new(id.clone(), record); + let entry_signature = EntrySignature::from_parts(namespace_sig, author_sig); + let signed_entry = SignedEntry::new(entry_signature, entry); + return Some(Ok((id.clone(), signed_entry))); + } + Some(Err(err)) => return Some(Err(err.into())), + None => { + // clear the current + fields.records.1 = None; + } + } + } + }) + } +} + +#[cfg(test)] +mod tests { + use crate::ranger::Store as _; + use crate::store::Store as _; + + use super::*; + + #[test] + fn test_basics() -> Result<()> { + let dbfile = tempfile::NamedTempFile::new()?; + let store = Store::new(dbfile.path())?; + + let author = store.new_author(&mut rand::thread_rng())?; + let namespace = Namespace::new(&mut rand::thread_rng()); + let replica = store.new_replica(namespace.clone())?; + + let replica_back = store.get_replica(&namespace.id())?.unwrap(); + assert_eq!( + replica.namespace().as_bytes(), + replica_back.namespace().as_bytes() + ); + + let author_back = store.get_author(&author.id())?.unwrap(); + assert_eq!(author.to_bytes(), author_back.to_bytes(),); + + let mut wrapper = StoreInstance::new(namespace.id(), store.clone()); + for i in 0..5 { + let id = RecordIdentifier::new(format!("hello-{i}"), namespace.id(), author.id()); + let entry = Entry::new( + id.clone(), + Record::from_data(format!("world-{i}"), namespace.id()), + ); + let entry = SignedEntry::from_entry(entry, &namespace, &author); + wrapper.put(id, entry)?; + } + + // all + let all: Vec<_> = wrapper.all()?.collect(); + assert_eq!(all.len(), 5); + + // add a second version + for i in 0..5 { + let id = RecordIdentifier::new(format!("hello-{i}"), namespace.id(), author.id()); + let entry = Entry::new( + id.clone(), + Record::from_data(format!("world-{i}-2"), namespace.id()), + ); + let entry = SignedEntry::from_entry(entry, &namespace, &author); + wrapper.put(id, entry)?; + } + + // get all + let entries = store.get_all(namespace.id())?.collect::>>()?; + assert_eq!(entries.len(), 10); + + // get all prefix + let entries = store + .get_all_by_prefix(namespace.id(), "hello-")? + .collect::>>()?; + assert_eq!(entries.len(), 10); + + // get latest + let entries = store + .get_latest(namespace.id())? + .collect::>>()?; + assert_eq!(entries.len(), 5); + + // get latest by prefix + let entries = store + .get_latest_by_prefix(namespace.id(), "hello-")? + .collect::>>()?; + assert_eq!(entries.len(), 5); + + // delete and get + for i in 0..5 { + let id = RecordIdentifier::new(format!("hello-{i}"), namespace.id(), author.id()); + let res = wrapper.get(&id)?; + assert!(res.is_some()); + let out = wrapper.remove(&id)?; + assert_eq!(out.len(), 2); + for val in out { + assert_eq!(val.entry().id(), &id); + } + let res = wrapper.get(&id)?; + assert!(res.is_none()); + } + + // get latest + let entries = store + .get_latest(namespace.id())? + .collect::>>()?; + assert_eq!(entries.len(), 0); + + Ok(()) + } +} diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs new file mode 100644 index 0000000000..b10213f9ce --- /dev/null +++ b/iroh-sync/src/store/memory.rs @@ -0,0 +1,502 @@ +//! In memory storage for replicas. + +use std::{ + collections::{BTreeMap, HashMap}, + convert::Infallible, + sync::Arc, +}; + +use anyhow::Result; +use parking_lot::{RwLock, RwLockReadGuard}; +use rand_core::CryptoRngCore; + +use crate::{ + ranger::{AsFingerprint, Fingerprint, Range, RangeKey}, + sync::{Author, AuthorId, Namespace, NamespaceId, RecordIdentifier, Replica, SignedEntry}, +}; + +/// Manages the replicas and authors for an instance. +#[derive(Debug, Clone, Default)] +pub struct Store { + replicas: Arc>>>, + authors: Arc>>, + /// Stores records by namespace -> identifier + timestamp + replica_records: Arc>, +} + +type ReplicaRecordsOwned = + HashMap>>; + +impl super::Store for Store { + type Instance = ReplicaStoreInstance; + type GetLatestIter<'a> = GetLatestIter<'a>; + type GetAllIter<'a> = GetAllIter<'a>; + + fn get_replica(&self, namespace: &NamespaceId) -> Result>> { + let replicas = &*self.replicas.read(); + Ok(replicas.get(namespace).cloned()) + } + + fn get_author(&self, author: &AuthorId) -> Result> { + let authors = &*self.authors.read(); + Ok(authors.get(author).cloned()) + } + + fn new_author(&self, rng: &mut R) -> Result { + let author = Author::new(rng); + self.authors.write().insert(author.id(), author.clone()); + Ok(author) + } + + fn new_replica(&self, namespace: Namespace) -> Result> { + let id = namespace.id(); + let replica = Replica::new(namespace, ReplicaStoreInstance::new(id, self.clone())); + self.replicas + .write() + .insert(replica.namespace(), replica.clone()); + Ok(replica) + } + + fn get_latest_by_key_and_author( + &self, + namespace: NamespaceId, + author: AuthorId, + key: impl AsRef<[u8]>, + ) -> Result> { + let inner = self.replica_records.read(); + + let value = inner + .get(&namespace) + .and_then(|records| records.get(&RecordIdentifier::new(key, namespace, author))) + .and_then(|values| values.last_key_value()); + + Ok(value.map(|(_, v)| v.clone())) + } + + fn get_latest_by_key( + &self, + namespace: NamespaceId, + key: impl AsRef<[u8]>, + ) -> Result> { + let records = self.replica_records.read(); + let key = key.as_ref().to_vec(); + let filter = GetFilter::Key { namespace, key }; + + Ok(GetLatestIter { + records, + filter, + index: 0, + }) + } + + fn get_latest_by_prefix( + &self, + namespace: NamespaceId, + prefix: impl AsRef<[u8]>, + ) -> Result> { + let records = self.replica_records.read(); + let prefix = prefix.as_ref().to_vec(); + let filter = GetFilter::Prefix { namespace, prefix }; + + Ok(GetLatestIter { + records, + filter, + index: 0, + }) + } + + fn get_latest(&self, namespace: NamespaceId) -> Result> { + let records = self.replica_records.read(); + let filter = GetFilter::All { namespace }; + + Ok(GetLatestIter { + records, + filter, + index: 0, + }) + } + + fn get_all_by_key_and_author<'a, 'b: 'a>( + &'a self, + namespace: NamespaceId, + author: AuthorId, + key: impl AsRef<[u8]> + 'b, + ) -> Result> { + let records = self.replica_records.read(); + let record_id = RecordIdentifier::new(key, namespace, author); + let filter = GetFilter::KeyAuthor(record_id); + + Ok(GetAllIter { + records, + filter, + index: 0, + }) + } + + fn get_all_by_key( + &self, + namespace: NamespaceId, + key: impl AsRef<[u8]>, + ) -> Result> { + let records = self.replica_records.read(); + let key = key.as_ref().to_vec(); + let filter = GetFilter::Key { namespace, key }; + + Ok(GetAllIter { + records, + filter, + index: 0, + }) + } + + fn get_all_by_prefix( + &self, + namespace: NamespaceId, + prefix: impl AsRef<[u8]>, + ) -> Result> { + let records = self.replica_records.read(); + let prefix = prefix.as_ref().to_vec(); + let filter = GetFilter::Prefix { namespace, prefix }; + + Ok(GetAllIter { + records, + filter, + index: 0, + }) + } + + fn get_all(&self, namespace: NamespaceId) -> Result> { + let records = self.replica_records.read(); + let filter = GetFilter::All { namespace }; + + Ok(GetAllIter { + records, + filter, + index: 0, + }) + } +} +#[derive(Debug)] +enum GetFilter { + /// All entries. + All { namespace: NamespaceId }, + /// Filter by key and author. + KeyAuthor(RecordIdentifier), + /// Filter by key only. + Key { + namespace: NamespaceId, + key: Vec, + }, + /// Filter by prefix only. + Prefix { + namespace: NamespaceId, + prefix: Vec, + }, +} + +impl GetFilter { + fn namespace(&self) -> NamespaceId { + match self { + GetFilter::All { namespace } => *namespace, + GetFilter::KeyAuthor(ref r) => r.namespace(), + GetFilter::Key { namespace, .. } => *namespace, + GetFilter::Prefix { namespace, .. } => *namespace, + } + } +} + +#[derive(Debug)] +pub struct GetLatestIter<'a> { + records: ReplicaRecords<'a>, + filter: GetFilter, + /// Current iteration index. + index: usize, +} + +impl<'a> Iterator for GetLatestIter<'a> { + type Item = Result<(RecordIdentifier, SignedEntry)>; + + fn next(&mut self) -> Option { + let records = self.records.get(&self.filter.namespace())?; + let res = match self.filter { + GetFilter::All { namespace } => records + .iter() + .filter(|(k, _)| k.namespace() == namespace) + .filter_map(|(key, value)| { + value + .last_key_value() + .map(|(_, v)| (key.clone(), v.clone())) + }) + .nth(self.index)?, + GetFilter::KeyAuthor(ref record_id) => { + let values = records.get(record_id)?; + let (_, res) = values.iter().nth(self.index)?; + (record_id.clone(), res.clone()) + } + GetFilter::Key { namespace, ref key } => records + .iter() + .filter(|(k, _)| k.key() == key && k.namespace() == namespace) + .filter_map(|(key, value)| { + value + .last_key_value() + .map(|(_, v)| (key.clone(), v.clone())) + }) + .nth(self.index)?, + GetFilter::Prefix { + namespace, + ref prefix, + } => records + .iter() + .filter(|(k, _)| k.key().starts_with(prefix) && k.namespace() == namespace) + .filter_map(|(key, value)| { + value + .last_key_value() + .map(|(_, v)| (key.clone(), v.clone())) + }) + .nth(self.index)?, + }; + self.index += 1; + Some(Ok(res)) + } +} + +#[derive(Debug)] +pub struct GetAllIter<'a> { + records: ReplicaRecords<'a>, + filter: GetFilter, + /// Current iteration index. + index: usize, +} + +impl<'a> Iterator for GetAllIter<'a> { + type Item = Result<(RecordIdentifier, SignedEntry)>; + + fn next(&mut self) -> Option { + let records = self.records.get(&self.filter.namespace())?; + let res = match self.filter { + GetFilter::All { namespace } => records + .iter() + .filter(|(k, _)| k.namespace() == namespace) + .flat_map(|(key, value)| { + value.iter().map(|(_, value)| (key.clone(), value.clone())) + }) + .nth(self.index)?, + GetFilter::KeyAuthor(ref record_id) => { + let values = records.get(record_id)?; + let (_, value) = values.iter().nth(self.index)?; + (record_id.clone(), value.clone()) + } + GetFilter::Key { namespace, ref key } => records + .iter() + .filter(|(k, _)| k.key() == key && k.namespace() == namespace) + .flat_map(|(key, value)| { + value.iter().map(|(_, value)| (key.clone(), value.clone())) + }) + .nth(self.index)?, + GetFilter::Prefix { + namespace, + ref prefix, + } => records + .iter() + .filter(|(k, _)| k.key().starts_with(prefix) && k.namespace() == namespace) + .flat_map(|(key, value)| { + value.iter().map(|(_, value)| (key.clone(), value.clone())) + }) + .nth(self.index)?, + }; + self.index += 1; + Some(Ok(res)) + } +} + +#[derive(Debug, Clone)] +pub struct ReplicaStoreInstance { + namespace: NamespaceId, + store: Store, +} + +impl ReplicaStoreInstance { + fn new(namespace: NamespaceId, store: Store) -> Self { + ReplicaStoreInstance { namespace, store } + } + + fn with_records(&self, f: F) -> T + where + F: FnOnce(Option<&BTreeMap>>) -> T, + { + let guard = self.store.replica_records.read(); + let value = guard.get(&self.namespace); + f(value) + } + + fn with_records_mut(&self, f: F) -> T + where + F: FnOnce(Option<&mut BTreeMap>>) -> T, + { + let mut guard = self.store.replica_records.write(); + let value = guard.get_mut(&self.namespace); + f(value) + } + + fn with_records_mut_with_default(&self, f: F) -> T + where + F: FnOnce(&mut BTreeMap>) -> T, + { + let mut guard = self.store.replica_records.write(); + let value = guard.entry(self.namespace).or_default(); + f(value) + } + + fn records_iter(&self) -> RecordsIter<'_> { + RecordsIter { + namespace: self.namespace, + replica_records: self.store.replica_records.read(), + i: 0, + } + } +} + +type ReplicaRecords<'a> = RwLockReadGuard< + 'a, + HashMap>>, +>; + +#[derive(Debug)] +struct RecordsIter<'a> { + namespace: NamespaceId, + replica_records: ReplicaRecords<'a>, + i: usize, +} + +impl Iterator for RecordsIter<'_> { + type Item = (RecordIdentifier, BTreeMap); + + fn next(&mut self) -> Option { + let records = self.replica_records.get(&self.namespace)?; + let (key, value) = records.iter().nth(self.i)?; + self.i += 1; + Some((key.clone(), value.clone())) + } +} + +impl crate::ranger::Store for ReplicaStoreInstance { + type Error = Infallible; + + /// Get a the first key (or the default if none is available). + fn get_first(&self) -> Result { + Ok(self.with_records(|records| { + records + .and_then(|r| r.first_key_value().map(|(k, _)| k.clone())) + .unwrap_or_default() + })) + } + + fn get(&self, key: &RecordIdentifier) -> Result, Self::Error> { + Ok(self.with_records(|records| { + records + .and_then(|r| r.get(key)) + .and_then(|values| values.last_key_value()) + .map(|(_, v)| v.clone()) + })) + } + + fn len(&self) -> Result { + Ok(self.with_records(|records| records.map(|v| v.len()).unwrap_or_default())) + } + + fn is_empty(&self) -> Result { + Ok(self.len()? == 0) + } + + fn get_fingerprint( + &self, + range: &Range, + limit: Option<&Range>, + ) -> Result { + let elements = self.get_range(range.clone(), limit.cloned())?; + let mut fp = Fingerprint::empty(); + for el in elements { + let el = el?; + fp ^= el.0.as_fingerprint(); + } + + Ok(fp) + } + + fn put(&mut self, k: RecordIdentifier, v: SignedEntry) -> Result<(), Self::Error> { + // TODO: propagate error/not insertion? + if v.verify().is_ok() { + let timestamp = v.entry().record().timestamp(); + // TODO: verify timestamp is "reasonable" + + self.with_records_mut_with_default(|records| { + records.entry(k).or_default().insert(timestamp, v); + }); + } + Ok(()) + } + + type RangeIterator<'a> = RangeIterator<'a>; + fn get_range( + &self, + range: Range, + limit: Option>, + ) -> Result, Self::Error> { + Ok(RangeIterator { + iter: self.records_iter(), + range: Some(range), + limit, + }) + } + + fn remove(&mut self, key: &RecordIdentifier) -> Result, Self::Error> { + let res = self.with_records_mut(|records| { + records + .and_then(|records| records.remove(key).map(|v| v.into_values().collect())) + .unwrap_or_default() + }); + Ok(res) + } + + type AllIterator<'a> = RangeIterator<'a>; + + fn all(&self) -> Result, Self::Error> { + Ok(RangeIterator { + iter: self.records_iter(), + range: None, + limit: None, + }) + } +} + +#[derive(Debug)] +pub struct RangeIterator<'a> { + iter: RecordsIter<'a>, + range: Option>, + limit: Option>, +} + +impl RangeIterator<'_> { + fn matches(&self, x: &RecordIdentifier) -> bool { + let range = self.range.as_ref().map(|r| x.contains(r)).unwrap_or(true); + let limit = self.limit.as_ref().map(|r| x.contains(r)).unwrap_or(true); + range && limit + } +} + +impl Iterator for RangeIterator<'_> { + type Item = Result<(RecordIdentifier, SignedEntry), Infallible>; + + fn next(&mut self) -> Option { + let mut next = self.iter.next()?; + loop { + if self.matches(&next.0) { + let (k, mut values) = next; + let (_, v) = values.pop_last()?; + return Some(Ok((k, v))); + } + + next = self.iter.next()?; + } + } +} diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 2faf711ec5..7ac32f7522 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -6,7 +6,6 @@ use std::{ cmp::Ordering, - collections::{BTreeMap, HashMap}, fmt::{Debug, Display}, str::FromStr, sync::Arc, @@ -15,20 +14,18 @@ use std::{ use parking_lot::RwLock; -use bytes::Bytes; use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey}; use iroh_bytes::Hash; use rand_core::CryptoRngCore; use serde::{Deserialize, Serialize}; -use crate::ranger::{AsFingerprint, Fingerprint, Peer, Range, RangeKey}; +use crate::ranger::{self, AsFingerprint, Fingerprint, Peer, RangeKey}; pub type ProtocolMessage = crate::ranger::Message; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Author { priv_key: SigningKey, - id: AuthorId, } impl Display for Author { @@ -40,17 +37,26 @@ impl Display for Author { impl Author { pub fn new(rng: &mut R) -> Self { let priv_key = SigningKey::generate(rng); - let id = AuthorId(priv_key.verifying_key()); - Author { priv_key, id } + Author { priv_key } } pub fn from_bytes(bytes: &[u8; 32]) -> Self { SigningKey::from_bytes(bytes).into() } - pub fn id(&self) -> &AuthorId { - &self.id + /// Returns the Author byte representation. + pub fn to_bytes(&self) -> [u8; 32] { + self.priv_key.to_bytes() + } + + /// Returns the AuthorId byte representation. + pub fn id_bytes(&self) -> [u8; 32] { + self.priv_key.verifying_key().to_bytes() + } + + pub fn id(&self) -> AuthorId { + AuthorId(self.priv_key.verifying_key()) } pub fn sign(&self, msg: &[u8]) -> Signature { @@ -58,7 +64,7 @@ impl Author { } pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.id.verify(msg, signature) + self.priv_key.verify_strict(msg, signature) } } @@ -85,12 +91,15 @@ impl AuthorId { pub fn as_bytes(&self) -> &[u8; 32] { self.0.as_bytes() } + + pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { + Ok(AuthorId(VerifyingKey::from_bytes(bytes)?)) + } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Namespace { priv_key: SigningKey, - id: NamespaceId, } impl Display for Namespace { @@ -105,8 +114,8 @@ impl FromStr for Namespace { fn from_str(s: &str) -> Result { let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; let priv_key = SigningKey::from_bytes(&priv_key); - let id = NamespaceId(priv_key.verifying_key()); - Ok(Namespace { priv_key, id }) + + Ok(Namespace { priv_key }) } } @@ -116,39 +125,46 @@ impl FromStr for Author { fn from_str(s: &str) -> Result { let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; let priv_key = SigningKey::from_bytes(&priv_key); - let id = AuthorId(priv_key.verifying_key()); - Ok(Author { priv_key, id }) + + Ok(Author { priv_key }) } } impl From for Author { fn from(priv_key: SigningKey) -> Self { - let id = AuthorId(priv_key.verifying_key()); - Self { priv_key, id } + Self { priv_key } } } impl From for Namespace { fn from(priv_key: SigningKey) -> Self { - let id = NamespaceId(priv_key.verifying_key()); - Self { priv_key, id } + Self { priv_key } } } impl Namespace { pub fn new(rng: &mut R) -> Self { let priv_key = SigningKey::generate(rng); - let id = NamespaceId(priv_key.verifying_key()); - Namespace { priv_key, id } + Namespace { priv_key } } pub fn from_bytes(bytes: &[u8; 32]) -> Self { SigningKey::from_bytes(bytes).into() } - pub fn id(&self) -> &NamespaceId { - &self.id + /// Returns the Namespace byte representation. + pub fn to_bytes(&self) -> [u8; 32] { + self.priv_key.to_bytes() + } + + /// Returns the NamespaceId byte representation. + pub fn id_bytes(&self) -> [u8; 32] { + self.priv_key.verifying_key().to_bytes() + } + + pub fn id(&self) -> NamespaceId { + NamespaceId(self.priv_key.verifying_key()) } pub fn sign(&self, msg: &[u8]) -> Signature { @@ -156,7 +172,7 @@ impl Namespace { } pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.id.verify(msg, signature) + self.priv_key.verify_strict(msg, signature) } } @@ -183,46 +199,9 @@ impl NamespaceId { pub fn as_bytes(&self) -> &[u8; 32] { self.0.as_bytes() } -} - -/// Manages the replicas and authors for an instance. -#[derive(Debug, Clone, Default)] -pub struct ReplicaStore { - replicas: Arc>>, - authors: Arc>>, -} - -impl ReplicaStore { - pub fn get_replica(&self, namespace: &NamespaceId) -> Option { - let replicas = &*self.replicas.read(); - replicas.get(namespace).cloned() - } - - pub fn get_author(&self, author: &AuthorId) -> Option { - let authors = &*self.authors.read(); - authors.get(author).cloned() - } - pub fn new_author(&self, rng: &mut R) -> Author { - let author = Author::new(rng); - self.authors.write().insert(*author.id(), author.clone()); - author - } - - pub fn new_replica(&self, namespace: Namespace) -> Replica { - let replica = Replica::new(namespace); - self.replicas - .write() - .insert(replica.namespace(), replica.clone()); - replica - } - - pub fn open_replica(&self, bytes: &[u8]) -> anyhow::Result { - let replica = Replica::from_bytes(bytes)?; - self.replicas - .write() - .insert(replica.namespace(), replica.clone()); - Ok(replica) + pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { + Ok(NamespaceId(VerifyingKey::from_bytes(bytes)?)) } } @@ -239,109 +218,16 @@ pub enum InsertOrigin { } #[derive(derive_more::Debug, Clone)] -pub struct Replica { - inner: Arc>, +pub struct Replica> { + inner: Arc>>, #[debug("on_insert: [Box; {}]", "self.on_insert.len()")] on_insert: Arc>>, } #[derive(derive_more::Debug)] -struct InnerReplica { +struct InnerReplica> { namespace: Namespace, - peer: Peer, -} - -#[derive(Default, Debug, Clone)] -pub struct Store { - /// Stores records by identifier + timestamp - records: BTreeMap>, -} - -impl Store { - pub fn latest(&self) -> impl Iterator { - self.records.iter().filter_map(|(k, values)| { - let (_, v) = values.last_key_value()?; - Some((k, v)) - }) - } -} - -impl crate::ranger::Store for Store { - /// Get a the first key (or the default if none is available). - fn get_first(&self) -> RecordIdentifier { - self.records - .first_key_value() - .map(|(k, _)| k.clone()) - .unwrap_or_default() - } - - fn get(&self, key: &RecordIdentifier) -> Option<&SignedEntry> { - self.records - .get(key) - .and_then(|values| values.last_key_value()) - .map(|(_, v)| v) - } - - fn len(&self) -> usize { - self.records.len() - } - - fn is_empty(&self) -> bool { - self.records.is_empty() - } - - fn get_fingerprint( - &self, - range: &Range, - limit: Option<&Range>, - ) -> Fingerprint { - let elements = self.get_range(range.clone(), limit.cloned()); - let mut fp = Fingerprint::empty(); - for el in elements { - fp ^= el.0.as_fingerprint(); - } - - fp - } - - fn put(&mut self, k: RecordIdentifier, v: SignedEntry) { - // TODO: propagate error/not insertion? - if v.verify().is_ok() { - let timestamp = v.entry().record().timestamp(); - // TODO: verify timestamp is "reasonable" - - self.records.entry(k).or_default().insert(timestamp, v); - } - } - - type RangeIterator<'a> = RangeIterator<'a>; - fn get_range( - &self, - range: Range, - limit: Option>, - ) -> Self::RangeIterator<'_> { - RangeIterator { - iter: self.records.iter(), - range: Some(range), - limit, - } - } - - fn remove(&mut self, key: &RecordIdentifier) -> Option { - self.records - .remove(key) - .and_then(|mut v| v.last_entry().map(|e| e.remove_entry().1)) - } - - type AllIterator<'a> = RangeIterator<'a>; - - fn all(&self) -> Self::AllIterator<'_> { - RangeIterator { - iter: self.records.iter(), - range: None, - limit: None, - } - } + peer: Peer, } #[derive(Debug, Serialize, Deserialize)] @@ -350,44 +236,13 @@ struct ReplicaData { namespace: Namespace, } -#[derive(Debug)] -pub struct RangeIterator<'a> { - iter: std::collections::btree_map::Iter<'a, RecordIdentifier, BTreeMap>, - range: Option>, - limit: Option>, -} - -impl<'a> RangeIterator<'a> { - fn matches(&self, x: &RecordIdentifier) -> bool { - let range = self.range.as_ref().map(|r| x.contains(r)).unwrap_or(true); - let limit = self.limit.as_ref().map(|r| x.contains(r)).unwrap_or(true); - range && limit - } -} - -impl<'a> Iterator for RangeIterator<'a> { - type Item = (&'a RecordIdentifier, &'a SignedEntry); - - fn next(&mut self) -> Option { - let mut next = self.iter.next()?; - loop { - if self.matches(next.0) { - let (k, values) = next; - let (_, v) = values.last_key_value()?; - return Some((k, v)); - } - - next = self.iter.next()?; - } - } -} - -impl Replica { - pub fn new(namespace: Namespace) -> Self { +impl> Replica { + // TODO: check that read only replicas are possible + pub fn new(namespace: Namespace, store: S) -> Self { Replica { inner: Arc::new(RwLock::new(InnerReplica { namespace, - peer: Peer::default(), + peer: Peer::from_store(store), })), on_insert: Default::default(), } @@ -398,56 +253,14 @@ impl Replica { on_insert.push(callback); } - // TODO: not horrible - pub fn all(&self) -> Vec<(RecordIdentifier, SignedEntry)> { - self.inner - .read() - .peer - .all() - .map(|(k, v)| (k.clone(), v.clone())) - .collect() - } - - // TODO: not horrible - pub fn all_for_key(&self, key: impl AsRef<[u8]>) -> Vec<(RecordIdentifier, SignedEntry)> { - self.all() - .into_iter() - .filter(|(id, _entry)| id.key() == key.as_ref()) - .collect() - } - - // TODO: not horrible - pub fn all_with_key_prefix( - &self, - prefix: impl AsRef<[u8]>, - ) -> Vec<(RecordIdentifier, SignedEntry)> { - self.all() - .into_iter() - .filter(|(id, _entry)| id.key().starts_with(prefix.as_ref())) - .collect() - } - - pub fn to_bytes(&self) -> anyhow::Result { - let entries = self.all().into_iter().map(|(_id, entry)| entry).collect(); - let data = ReplicaData { - entries, - namespace: self.inner.read().namespace.clone(), - }; - let bytes = postcard::to_stdvec(&data)?; - Ok(bytes.into()) - } - - pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { - let data: ReplicaData = postcard::from_bytes(bytes)?; - let replica = Self::new(data.namespace); - for entry in data.entries { - replica.insert_remote_entry(entry)?; - } - Ok(replica) - } - /// Inserts a new record at the given key. - pub fn insert(&self, key: impl AsRef<[u8]>, author: &Author, hash: Hash, len: u64) { + pub fn insert( + &self, + key: impl AsRef<[u8]>, + author: &Author, + hash: Hash, + len: u64, + ) -> Result<(), S::Error> { let mut inner = self.inner.write(); let id = RecordIdentifier::new(key, inner.namespace.id(), author.id()); @@ -456,12 +269,13 @@ impl Replica { // Store signed entries let entry = Entry::new(id.clone(), record); let signed_entry = entry.sign(&inner.namespace, author); - inner.peer.put(id, signed_entry.clone()); + inner.peer.put(id, signed_entry.clone())?; drop(inner); let on_insert = self.on_insert.read(); for cb in &*on_insert { cb(InsertOrigin::Local, signed_entry.clone()); } + Ok(()) } /// Hashes the given data and inserts it. @@ -473,24 +287,23 @@ impl Replica { key: impl AsRef<[u8]>, author: &Author, data: impl AsRef<[u8]>, - ) -> Hash { + ) -> Result { let len = data.as_ref().len() as u64; let hash = Hash::new(data); - self.insert(key, author, hash, len); - hash + self.insert(key, author, hash, len)?; + Ok(hash) } pub fn id(&self, key: impl AsRef<[u8]>, author: &Author) -> RecordIdentifier { let inner = self.inner.read(); - let id = RecordIdentifier::new(key, inner.namespace.id(), author.id()); - id + RecordIdentifier::new(key, inner.namespace.id(), author.id()) } pub fn insert_remote_entry(&self, entry: SignedEntry) -> anyhow::Result<()> { entry.verify()?; let mut inner = self.inner.write(); let id = entry.entry.id.clone(); - inner.peer.put(id, entry.clone()); + inner.peer.put(id, entry.clone()).map_err(Into::into)?; drop(inner); let on_insert = self.on_insert.read(); for cb in &*on_insert { @@ -499,140 +312,16 @@ impl Replica { Ok(()) } - /// Gets all entries matching this key and author. - pub fn get_latest_by_key_and_author( + pub fn sync_initial_message( &self, - key: impl AsRef<[u8]>, - author: &AuthorId, - ) -> Option { - let inner = self.inner.read(); - inner - .peer - .get(&RecordIdentifier::new(key, inner.namespace.id(), author)) - .cloned() - } - - /// Returns the latest version of the matching documents by key. - pub fn get_latest_by_key(&self, key: impl AsRef<[u8]>) -> GetLatestIter<'_> { - let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); - let key = key.as_ref().to_vec(); - let namespace = *guard.namespace.id(); - let filter = GetFilter::Key { namespace, key }; - - GetLatestIter { - records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { - &inner.peer.store().records - }), - filter, - index: 0, - } - } - - /// Returns the latest version of the matching documents by prefix. - pub fn get_latest_by_prefix(&self, prefix: impl AsRef<[u8]>) -> GetLatestIter<'_> { - let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); - let prefix = prefix.as_ref().to_vec(); - let namespace = *guard.namespace.id(); - let filter = GetFilter::Prefix { namespace, prefix }; - - GetLatestIter { - records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { - &inner.peer.store().records - }), - filter, - index: 0, - } - } - - /// Returns the latest versions of all documents. - pub fn get_latest(&self) -> GetLatestIter<'_> { - let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); - let namespace = *guard.namespace.id(); - let filter = GetFilter::All { namespace }; - - GetLatestIter { - records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { - &inner.peer.store().records - }), - filter, - index: 0, - } - } - - /// Returns all versions of the matching documents by author. - pub fn get_all_by_key_and_author<'a, 'b: 'a>( - &'a self, - key: impl AsRef<[u8]> + 'b, - author: &AuthorId, - ) -> GetAllIter<'a> { - let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); - let record_id = RecordIdentifier::new(key, guard.namespace.id(), author); - let filter = GetFilter::KeyAuthor(record_id); - - GetAllIter { - records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { - &inner.peer.store().records - }), - filter, - index: 0, - } - } - - /// Returns all versions of the matching documents by key. - pub fn get_all_by_key(&self, key: impl AsRef<[u8]>) -> GetAllIter<'_> { - let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); - let key = key.as_ref().to_vec(); - let namespace = *guard.namespace.id(); - let filter = GetFilter::Key { namespace, key }; - - GetAllIter { - records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { - &inner.peer.store().records - }), - filter, - index: 0, - } - } - - /// Returns all versions of the matching documents by prefix. - pub fn get_all_by_prefix(&self, prefix: impl AsRef<[u8]>) -> GetAllIter<'_> { - let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); - let prefix = prefix.as_ref().to_vec(); - let namespace = *guard.namespace.id(); - let filter = GetFilter::Prefix { namespace, prefix }; - - GetAllIter { - records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { - &inner.peer.store().records - }), - filter, - index: 0, - } - } - - /// Returns all versions of all documents. - pub fn get_all(&self) -> GetAllIter<'_> { - let guard: parking_lot::lock_api::RwLockReadGuard<_, _> = self.inner.read(); - let namespace = *guard.namespace.id(); - let filter = GetFilter::All { namespace }; - - GetAllIter { - records: parking_lot::lock_api::RwLockReadGuard::map(guard, move |inner| { - &inner.peer.store().records - }), - filter, - index: 0, - } - } - - pub fn sync_initial_message(&self) -> crate::ranger::Message { + ) -> Result, S::Error> { self.inner.read().peer.initial_message() } pub fn sync_process_message( &self, message: crate::ranger::Message, - ) -> Option> { + ) -> Result>, S::Error> { let reply = self .inner .write() @@ -642,152 +331,13 @@ impl Replica { for cb in &*on_insert { cb(InsertOrigin::Sync, entry.clone()); } - }); + })?; - reply + Ok(reply) } pub fn namespace(&self) -> NamespaceId { - *self.inner.read().namespace.id() - } -} - -#[derive(Debug)] -pub enum GetFilter { - /// All entries. - All { namespace: NamespaceId }, - /// Filter by key and author. - KeyAuthor(RecordIdentifier), - /// Filter by key only. - Key { - namespace: NamespaceId, - key: Vec, - }, - /// Filter by prefix only. - Prefix { - namespace: NamespaceId, - prefix: Vec, - }, -} - -#[derive(Debug)] -pub struct GetLatestIter<'a> { - // Oh my god, rust why u do this to me? - records: parking_lot::lock_api::MappedRwLockReadGuard< - 'a, - parking_lot::RawRwLock, - BTreeMap>, - >, - filter: GetFilter, - /// Current iteration index. - index: usize, -} - -impl<'a> Iterator for GetLatestIter<'a> { - type Item = SignedEntry; - - fn next(&mut self) -> Option { - let res = match self.filter { - GetFilter::All { namespace } => { - let (_, res) = self - .records - .iter() - .filter(|(k, _)| k.namespace() == &namespace) - .filter_map(|(_key, value)| value.last_key_value()) - .nth(self.index)?; - res.clone() - } - GetFilter::KeyAuthor(ref record_id) => { - let values = self.records.get(record_id)?; - let (_, res) = values.iter().nth(self.index)?; - res.clone() - } - GetFilter::Key { namespace, ref key } => { - let (_, res) = self - .records - .iter() - .filter(|(k, _)| k.key() == key && k.namespace() == &namespace) - .filter_map(|(_key, value)| value.last_key_value()) - .nth(self.index)?; - res.clone() - } - GetFilter::Prefix { - namespace, - ref prefix, - } => { - let (_, res) = self - .records - .iter() - .filter(|(k, _)| k.key().starts_with(prefix) && k.namespace() == &namespace) - .filter_map(|(_key, value)| value.last_key_value()) - .nth(self.index)?; - res.clone() - } - }; - self.index += 1; - Some(res) - } -} - -#[derive(Debug)] -pub struct GetAllIter<'a> { - // Oh my god, rust why u do this to me? - records: parking_lot::lock_api::MappedRwLockReadGuard< - 'a, - parking_lot::RawRwLock, - BTreeMap>, - >, - filter: GetFilter, - /// Current iteration index. - index: usize, -} - -impl<'a> Iterator for GetAllIter<'a> { - type Item = (RecordIdentifier, u64, SignedEntry); - - fn next(&mut self) -> Option { - let res = match self.filter { - GetFilter::All { namespace } => self - .records - .iter() - .filter(|(k, _)| k.namespace() == &namespace) - .flat_map(|(key, value)| { - value - .iter() - .map(|(t, value)| (key.clone(), *t, value.clone())) - }) - .nth(self.index)?, - GetFilter::KeyAuthor(ref record_id) => { - let values = self.records.get(record_id)?; - let (t, value) = values.iter().nth(self.index)?; - (record_id.clone(), *t, value.clone()) - } - GetFilter::Key { namespace, ref key } => self - .records - .iter() - .filter(|(k, _)| k.key() == key && k.namespace() == &namespace) - .flat_map(|(key, value)| { - value - .iter() - .map(|(t, value)| (key.clone(), *t, value.clone())) - }) - .nth(self.index)?, - GetFilter::Prefix { - namespace, - ref prefix, - } => self - .records - .iter() - .filter(|(k, _)| k.key().starts_with(prefix) && k.namespace() == &namespace) - .flat_map(|(key, value)| { - value - .iter() - .map(|(t, value)| (key.clone(), *t, value.clone())) - }) - .nth(self.index)?, - }; - self.index += 1; - Some(res) + self.inner.read().namespace.id() } } @@ -799,6 +349,10 @@ pub struct SignedEntry { } impl SignedEntry { + pub fn new(signature: EntrySignature, entry: Entry) -> Self { + SignedEntry { signature, entry } + } + pub fn from_entry(entry: Entry, namespace: &Namespace, author: &Author) -> Self { let signature = EntrySignature::from_entry(&entry, namespace, author); SignedEntry { signature, entry } @@ -855,6 +409,24 @@ impl EntrySignature { Ok(()) } + + pub fn from_parts(namespace_sig: &[u8; 64], author_sig: &[u8; 64]) -> Self { + let namespace_signature = Signature::from_bytes(namespace_sig); + let author_signature = Signature::from_bytes(author_sig); + + EntrySignature { + author_signature, + namespace_signature, + } + } + + pub fn author_signature(&self) -> &Signature { + &self.author_signature + } + + pub fn namespace_signature(&self) -> &Signature { + &self.namespace_signature + } } /// A single entry in a replica. @@ -954,14 +526,22 @@ impl RangeKey for RecordIdentifier { } impl RecordIdentifier { - pub fn new(key: impl AsRef<[u8]>, namespace: &NamespaceId, author: &AuthorId) -> Self { + pub fn new(key: impl AsRef<[u8]>, namespace: NamespaceId, author: AuthorId) -> Self { RecordIdentifier { key: key.as_ref().to_vec(), - namespace: *namespace, - author: *author, + namespace, + author, } } + pub fn from_parts(key: &[u8], namespace: &[u8; 32], author: &[u8; 32]) -> anyhow::Result { + Ok(RecordIdentifier { + key: key.to_vec(), + namespace: NamespaceId::from_bytes(namespace)?, + author: AuthorId::from_bytes(author)?, + }) + } + pub fn as_bytes(&self, out: &mut Vec) { out.extend_from_slice(self.namespace.as_bytes()); out.extend_from_slice(self.author.as_bytes()); @@ -972,12 +552,20 @@ impl RecordIdentifier { &self.key } - pub fn namespace(&self) -> &NamespaceId { - &self.namespace + pub fn namespace(&self) -> NamespaceId { + self.namespace + } + + pub fn namespace_bytes(&self) -> &[u8; 32] { + self.namespace.as_bytes() } - pub fn author(&self) -> &AuthorId { - &self.author + pub fn author(&self) -> AuthorId { + self.author + } + + pub fn author_bytes(&self) -> &[u8; 32] { + self.author.as_bytes() } } @@ -1020,7 +608,7 @@ impl Record { } // TODO: remove - pub fn from_data(data: impl AsRef<[u8]>, namespace: &NamespaceId) -> Self { + pub fn from_data(data: impl AsRef<[u8]>, namespace: NamespaceId) -> Self { // Salted hash // TODO: do we actually want this? // TODO: this should probably use a namespace prefix if used @@ -1040,10 +628,30 @@ impl Record { #[cfg(test)] mod tests { + use anyhow::Result; + + use crate::{ranger::Range, store}; + use super::*; #[test] - fn test_basics() { + fn test_basics_memory() -> Result<()> { + let store = store::memory::Store::default(); + test_basics(store)?; + + Ok(()) + } + + #[cfg(feature = "fs-store")] + #[test] + fn test_basics_fs() -> Result<()> { + let dbfile = tempfile::NamedTempFile::new()?; + let store = store::fs::Store::new(dbfile.path())?; + test_basics(store)?; + Ok(()) + } + + fn test_basics(store: S) -> Result<()> { let mut rng = rand::thread_rng(); let alice = Author::new(&mut rng); let bob = Author::new(&mut rng); @@ -1055,94 +663,126 @@ mod tests { let signed_entry = entry.sign(&myspace, &alice); signed_entry.verify().expect("failed to verify"); - let my_replica = Replica::new(myspace); + let my_replica = store.new_replica(myspace)?; for i in 0..10 { - my_replica.hash_and_insert(format!("/{i}"), &alice, format!("{i}: hello from alice")); + my_replica + .hash_and_insert(format!("/{i}"), &alice, format!("{i}: hello from alice")) + .map_err(Into::into)?; } for i in 0..10 { - let res = my_replica - .get_latest_by_key_and_author(format!("/{i}"), alice.id()) + let res = store + .get_latest_by_key_and_author(my_replica.namespace(), alice.id(), format!("/{i}"))? .unwrap(); let len = format!("{i}: hello from alice").as_bytes().len() as u64; assert_eq!(res.entry().record().content_len(), len); - res.verify().expect("invalid signature"); + res.verify()?; } // Test multiple records for the same key - my_replica.hash_and_insert("/cool/path", &alice, "round 1"); - let _entry = my_replica - .get_latest_by_key_and_author("/cool/path", alice.id()) + my_replica + .hash_and_insert("/cool/path", &alice, "round 1") + .map_err(Into::into)?; + let _entry = store + .get_latest_by_key_and_author(my_replica.namespace(), alice.id(), "/cool/path")? .unwrap(); // Second - my_replica.hash_and_insert("/cool/path", &alice, "round 2"); - let _entry = my_replica - .get_latest_by_key_and_author("/cool/path", alice.id()) + my_replica + .hash_and_insert("/cool/path", &alice, "round 2") + .map_err(Into::into)?; + let _entry = store + .get_latest_by_key_and_author(my_replica.namespace(), alice.id(), "/cool/path")? .unwrap(); // Get All by author - let entries: Vec<_> = my_replica - .get_all_by_key_and_author("/cool/path", alice.id()) - .collect(); + let entries: Vec<_> = store + .get_all_by_key_and_author(my_replica.namespace(), alice.id(), "/cool/path")? + .collect::>()?; assert_eq!(entries.len(), 2); // Get All by key - let entries: Vec<_> = my_replica.get_all_by_key(b"/cool/path").collect(); + let entries: Vec<_> = store + .get_all_by_key(my_replica.namespace(), b"/cool/path")? + .collect::>()?; assert_eq!(entries.len(), 2); // Get latest by key - let entries: Vec<_> = my_replica.get_latest_by_key(b"/cool/path").collect(); + let entries: Vec<_> = store + .get_latest_by_key(my_replica.namespace(), b"/cool/path")? + .collect::>()?; assert_eq!(entries.len(), 1); // Get latest by prefix - let entries: Vec<_> = my_replica.get_latest_by_prefix(b"/cool").collect(); + let entries: Vec<_> = store + .get_latest_by_prefix(my_replica.namespace(), b"/cool")? + .collect::>()?; assert_eq!(entries.len(), 1); // Get All - let entries: Vec<_> = my_replica.get_all().collect(); + let entries: Vec<_> = store + .get_all(my_replica.namespace())? + .collect::>()?; assert_eq!(entries.len(), 12); // Get All latest - let entries: Vec<_> = my_replica.get_latest().collect(); + let entries: Vec<_> = store + .get_latest(my_replica.namespace())? + .collect::>()?; assert_eq!(entries.len(), 11); // insert record from different author - let _entry = my_replica.hash_and_insert("/cool/path", &bob, "bob round 1"); + let _entry = my_replica + .hash_and_insert("/cool/path", &bob, "bob round 1") + .map_err(Into::into)?; // Get All by author - let entries: Vec<_> = my_replica - .get_all_by_key_and_author("/cool/path", alice.id()) - .collect(); + let entries: Vec<_> = store + .get_all_by_key_and_author(my_replica.namespace(), alice.id(), "/cool/path")? + .collect::>()?; assert_eq!(entries.len(), 2); - let entries: Vec<_> = my_replica - .get_all_by_key_and_author("/cool/path", bob.id()) - .collect(); + let entries: Vec<_> = store + .get_all_by_key_and_author(my_replica.namespace(), bob.id(), "/cool/path")? + .collect::>()?; assert_eq!(entries.len(), 1); // Get All by key - let entries: Vec<_> = my_replica.get_all_by_key(b"/cool/path").collect(); + let entries: Vec<_> = store + .get_all_by_key(my_replica.namespace(), b"/cool/path")? + .collect::>()?; assert_eq!(entries.len(), 3); // Get latest by key - let entries: Vec<_> = my_replica.get_latest_by_key(b"/cool/path").collect(); + let entries: Vec<_> = store + .get_latest_by_key(my_replica.namespace(), b"/cool/path")? + .collect::>()?; assert_eq!(entries.len(), 2); // Get latest by prefix - let entries: Vec<_> = my_replica.get_latest_by_prefix(b"/cool").collect(); + let entries: Vec<_> = store + .get_latest_by_prefix(my_replica.namespace(), b"/cool")? + .collect::>()?; assert_eq!(entries.len(), 2); // Get all by prefix - let entries: Vec<_> = my_replica.get_all_by_prefix(b"/cool").collect(); + let entries: Vec<_> = store + .get_all_by_prefix(my_replica.namespace(), b"/cool")? + .collect::>()?; assert_eq!(entries.len(), 3); // Get All - let entries: Vec<_> = my_replica.get_all().collect(); + let entries: Vec<_> = store + .get_all(my_replica.namespace())? + .collect::>()?; assert_eq!(entries.len(), 13); // Get All latest - let entries: Vec<_> = my_replica.get_latest().collect(); + let entries: Vec<_> = store + .get_latest(my_replica.namespace())? + .collect::>()?; assert_eq!(entries.len(), 12); + + Ok(()) } #[test] @@ -1152,10 +792,10 @@ mod tests { let k = vec!["a", "c", "z"]; let mut n: Vec<_> = (0..3).map(|_| Namespace::new(&mut rng)).collect(); - n.sort_by_key(|n| *n.id()); + n.sort_by_key(|n| n.id()); let mut a: Vec<_> = (0..3).map(|_| Author::new(&mut rng)).collect(); - a.sort_by_key(|a| *a.id()); + a.sort_by_key(|a| a.id()); // Just key { @@ -1207,54 +847,89 @@ mod tests { } #[test] - fn test_replica_sync() { + fn test_replica_sync_memory() -> Result<()> { + let alice_store = store::memory::Store::default(); + let bob_store = store::memory::Store::default(); + + test_replica_sync(alice_store, bob_store)?; + Ok(()) + } + + #[cfg(feature = "fs-store")] + #[test] + fn test_replica_sync_fs() -> Result<()> { + let alice_dbfile = tempfile::NamedTempFile::new()?; + let alice_store = store::fs::Store::new(alice_dbfile.path())?; + let bob_dbfile = tempfile::NamedTempFile::new()?; + let bob_store = store::fs::Store::new(bob_dbfile.path())?; + test_replica_sync(alice_store, bob_store)?; + + Ok(()) + } + + fn test_replica_sync(alice_store: S, bob_store: S) -> Result<()> { let alice_set = ["ape", "eel", "fox", "gnu"]; let bob_set = ["bee", "cat", "doe", "eel", "fox", "hog"]; let mut rng = rand::thread_rng(); let author = Author::new(&mut rng); let myspace = Namespace::new(&mut rng); - let mut alice = Replica::new(myspace.clone()); + let alice = alice_store.new_replica(myspace.clone())?; for el in &alice_set { - alice.hash_and_insert(el, &author, el.as_bytes()); + alice + .hash_and_insert(el, &author, el.as_bytes()) + .map_err(Into::into)?; } - let mut bob = Replica::new(myspace); + let bob = bob_store.new_replica(myspace)?; for el in &bob_set { - bob.hash_and_insert(el, &author, el.as_bytes()); + bob.hash_and_insert(el, &author, el.as_bytes()) + .map_err(Into::into)?; } - sync(&author, &mut alice, &mut bob, &alice_set, &bob_set); + sync( + &author, + &alice, + &alice_store, + &bob, + &bob_store, + &alice_set, + &bob_set, + )?; + Ok(()) } - fn sync( + fn sync( author: &Author, - alice: &mut Replica, - bob: &mut Replica, + alice: &Replica, + alice_store: &S, + bob: &Replica, + bob_store: &S, alice_set: &[&str], bob_set: &[&str], - ) { + ) -> Result<()> { // Sync alice - bob - let mut next_to_bob = Some(alice.sync_initial_message()); + let mut next_to_bob = Some(alice.sync_initial_message().map_err(Into::into)?); let mut rounds = 0; while let Some(msg) = next_to_bob.take() { assert!(rounds < 100, "too many rounds"); rounds += 1; println!("round {}", rounds); - if let Some(msg) = bob.sync_process_message(msg) { - next_to_bob = alice.sync_process_message(msg); + if let Some(msg) = bob.sync_process_message(msg).map_err(Into::into)? { + next_to_bob = alice.sync_process_message(msg).map_err(Into::into)?; } } // Check result for el in alice_set { - alice.get_latest_by_key_and_author(el, author.id()).unwrap(); - bob.get_latest_by_key_and_author(el, author.id()).unwrap(); + alice_store.get_latest_by_key_and_author(alice.namespace(), author.id(), el)?; + bob_store.get_latest_by_key_and_author(bob.namespace(), author.id(), el)?; } for el in bob_set { - alice.get_latest_by_key_and_author(el, author.id()).unwrap(); - bob.get_latest_by_key_and_author(el, author.id()).unwrap(); + alice_store.get_latest_by_key_and_author(alice.namespace(), author.id(), el)?; + bob_store.get_latest_by_key_and_author(bob.namespace(), author.id(), el)?; } + Ok(()) } } diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index b2b8aad58a..4f1b902b44 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -26,7 +26,7 @@ iroh-metrics = { version = "0.5.0", path = "../iroh-metrics", optional = true } iroh-net = { version = "0.5.1", path = "../iroh-net" } num_cpus = { version = "1.15.0" } portable-atomic = "1" -iroh-sync = { path = "../iroh-sync" } +iroh-sync = { path = "../iroh-sync" } iroh-gossip = { path = "../iroh-gossip" } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quic-rpc = { version = "0.6", default-features = false, features = ["flume-transport"] } @@ -62,7 +62,7 @@ rustyline = { version = "12.0.0", optional = true } [features] default = ["cli", "metrics", "sync"] -sync = ["metrics"] +sync = ["metrics", "iroh-sync/fs-store"] cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection"] metrics = ["iroh-metrics", "flat-db", "mem-db", "iroh-collection"] mem-db = [] diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 48afe81265..917011efd1 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -16,7 +16,10 @@ use anyhow::{anyhow, bail}; use clap::{CommandFactory, FromArgMatches, Parser}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; -use iroh::sync::{BlobStore, Doc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN}; +use iroh::sync::{ + BlobStore, Doc as SyncDoc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN, +}; +use iroh_bytes::util::runtime; use iroh_gossip::{ net::{GossipHandle, GOSSIP_ALPN}, proto::TopicId, @@ -32,7 +35,10 @@ use iroh_net::{ tls::Keypair, MagicEndpoint, }; -use iroh_sync::sync::{Author, Namespace, SignedEntry}; +use iroh_sync::{ + store::{self, Store as _}, + sync::{Author, Namespace, SignedEntry}, +}; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; use tokio::{ @@ -48,6 +54,8 @@ use iroh_bytes_handlers::IrohBytesHandlers; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; +type Doc = SyncDoc; + #[derive(Parser, Debug)] struct Args { /// Private key to derive our peer id from @@ -219,10 +227,12 @@ async fn run(args: Args) -> anyhow::Result<()> { // create a doc store for the iroh-sync docs let author = Author::from(keypair.secret().clone()); - let docs = DocStore::new(blobs.clone(), author, storage_path.join("docs")); + let docs_path = storage_path.join("docs"); + tokio::fs::create_dir_all(&docs_path).await?; + let docs = DocStore::new(blobs.clone(), author, docs_path)?; // create the live syncer - let live_sync = LiveSync::spawn(endpoint.clone(), gossip.clone()); + let live_sync = LiveSync::::spawn(endpoint.clone(), gossip.clone()); // construct the state that is passed to the endpoint loop and from there cloned // into to the connection handler task for incoming connections. @@ -233,12 +243,12 @@ async fn run(args: Args) -> anyhow::Result<()> { }); // spawn our endpoint loop that forwards incoming connections - tokio::spawn(endpoint_loop(endpoint.clone(), state)); + rt.main().spawn(endpoint_loop(endpoint.clone(), state)); // open our document and add to the live syncer let namespace = Namespace::from_bytes(topic.as_bytes()); println!("> opening doc {}", fmt_hash(namespace.id().as_bytes())); - let doc = docs.create_or_open(namespace, DownloadMode::Always).await?; + let doc: Doc = docs.create_or_open(namespace, DownloadMode::Always).await?; live_sync.add(doc.replica().clone(), peers.clone()).await?; // spawn an repl thread that reads stdin and parses each line as a `Cmd` command @@ -278,7 +288,7 @@ async fn run(args: Args) -> anyhow::Result<()> { _ = tokio::signal::ctrl_c() => { println!("> aborted"); } - res = handle_command(cmd, &doc, &our_ticket, &log_filter, ¤t_watch) => if let Err(err) = res { + res = handle_command(cmd, &rt, docs.store(), &doc, &our_ticket, &log_filter, ¤t_watch) => if let Err(err) = res { println!("> error: {err}"); }, }; @@ -292,7 +302,6 @@ async fn run(args: Args) -> anyhow::Result<()> { } println!("> persisting document and blob database at {storage_path:?}"); blobs.save().await?; - docs.save(&doc).await?; if let Some(metrics_fut) = metrics_fut { metrics_fut.abort(); @@ -304,6 +313,8 @@ async fn run(args: Args) -> anyhow::Result<()> { async fn handle_command( cmd: Cmd, + rt: &runtime::Handle, + store: &store::fs::Store, doc: &Doc, ticket: &Ticket, log_filter: &LogLevelReload, @@ -313,9 +324,18 @@ async fn handle_command( Cmd::Set { key, value } => { doc.insert_bytes(&key, value.into_bytes().into()).await?; } - Cmd::Get { key, print_content } => { - let entries = doc.replica().all_for_key(key.as_bytes()); - for (_id, entry) in entries { + Cmd::Get { + key, + print_content, + prefix, + } => { + let entries = if prefix { + store.get_all_by_prefix(doc.replica().namespace(), key.as_bytes())? + } else { + store.get_all_by_key(doc.replica().namespace(), key.as_bytes())? + }; + for entry in entries { + let (_id, entry) = entry?; println!("{}", fmt_entry(&entry)); if print_content { println!("{}", fmt_content(doc, &entry).await); @@ -336,13 +356,18 @@ async fn handle_command( }, Cmd::Ls { prefix } => { let entries = match prefix { - None => doc.replica().all(), - Some(prefix) => doc.replica().all_with_key_prefix(prefix.as_bytes()), + None => store.get_all(doc.replica().namespace())?, + Some(prefix) => { + store.get_all_by_prefix(doc.replica().namespace(), prefix.as_bytes())? + } }; - println!("> {} entries", entries.len()); - for (_id, entry) in entries { + let mut count = 0; + for entry in entries { + let (_id, entry) = entry?; + count += 1; println!("{}", fmt_entry(&entry),); } + println!("> {} entries", count); } Cmd::Ticket => { println!("Ticket: {ticket}"); @@ -352,7 +377,7 @@ async fn handle_command( log_filter.modify(|layer| *layer = next_filter)?; } Cmd::Stats => get_stats(), - Cmd::Fs(cmd) => handle_fs_command(cmd, doc).await?, + Cmd::Fs(cmd) => handle_fs_command(cmd, store, doc).await?, Cmd::Hammer { prefix, threads, @@ -376,7 +401,7 @@ async fn handle_command( let prefix = prefix.clone(); let doc = doc.clone(); let bytes = bytes.clone(); - let handle = tokio::spawn(async move { + let handle = rt.main().spawn(async move { for i in 0..count { let value = String::from_utf8(bytes.clone()).unwrap(); let key = format!("{}/{}/{}", prefix, t, i); @@ -391,13 +416,16 @@ async fn handle_command( for t in 0..threads { let prefix = prefix.clone(); let doc = doc.clone(); - let handle = tokio::spawn(async move { + let store = store.clone(); + let handle = rt.main().spawn(async move { let mut read = 0; for i in 0..count { let key = format!("{}/{}/{}", prefix, t, i); - let entries = doc.replica().all_for_key(key.as_bytes()); - for (_id, entry) in entries { - let _content = fmt_content(&doc, &entry).await; + let entries = store + .get_all_by_key(doc.replica().namespace(), key.as_bytes())?; + for entry in entries { + let (_id, entry) = entry?; + let _content = fmt_content_simple(&doc, &entry); read += 1; } } @@ -425,7 +453,7 @@ async fn handle_command( Ok(()) } -async fn handle_fs_command(cmd: FsCmd, doc: &Doc) -> anyhow::Result<()> { +async fn handle_fs_command(cmd: FsCmd, store: &store::fs::Store, doc: &Doc) -> anyhow::Result<()> { match cmd { FsCmd::ImportFile { file_path, key } => { let file_path = canonicalize_path(&file_path)?.canonicalize()?; @@ -473,10 +501,12 @@ async fn handle_fs_command(cmd: FsCmd, doc: &Doc) -> anyhow::Result<()> { } let root = canonicalize_path(&dir_path)?; println!("> exporting {key_prefix} to {root:?}"); - let entries = doc.replica().get_latest_by_prefix(key_prefix.as_bytes()); + let entries = + store.get_latest_by_prefix(doc.replica().namespace(), key_prefix.as_bytes())?; let mut checked_dirs = HashSet::new(); for entry in entries { - let key = entry.entry().id().key(); + let (id, entry) = entry?; + let key = id.key(); let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; let len = entry.entry().record().content_len(); if let Some(mut reader) = doc.get_content_reader(&entry).await { @@ -499,8 +529,11 @@ async fn handle_fs_command(cmd: FsCmd, doc: &Doc) -> anyhow::Result<()> { FsCmd::ExportFile { key, file_path } => { let path = canonicalize_path(&file_path)?; // TODO: Fix - let entry = doc.replica().get_latest_by_key(&key).next(); + let entry = store + .get_latest_by_key(doc.replica().namespace(), &key)? + .next(); if let Some(entry) = entry { + let (_, entry) = entry?; println!("> exporting {key} to {path:?}"); let parent = path.parent().ok_or_else(|| anyhow!("Invalid path"))?; tokio::fs::create_dir_all(&parent).await?; @@ -537,6 +570,9 @@ pub enum Cmd { /// Print the value (but only if it is valid UTF-8 and smaller than 1MB) #[clap(short = 'c', long)] print_content: bool, + /// Match the key as prefix, not an exact match. + #[clap(short = 'p', long)] + prefix: bool, }, /// List entries. Ls { @@ -802,6 +838,12 @@ fn fmt_entry(entry: &SignedEntry) -> String { let len = HumanBytes(entry.entry().record().content_len()); format!("@{author}: {key} = {hash} ({len})",) } + +async fn fmt_content_simple(_doc: &Doc, entry: &SignedEntry) -> String { + let len = entry.entry().record().content_len(); + format!("<{}>", HumanBytes(len)) +} + async fn fmt_content(doc: &Doc, entry: &SignedEntry) -> String { let len = entry.entry().record().content_len(); if len > MAX_DISPLAY_CONTENT_LEN { diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index d5b92516b8..bcadeb23f9 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -5,7 +5,10 @@ use std::net::SocketAddr; use anyhow::{bail, ensure, Context, Result}; use bytes::BytesMut; use iroh_net::{tls::PeerId, MagicEndpoint}; -use iroh_sync::sync::{NamespaceId, Replica, ReplicaStore}; +use iroh_sync::{ + store, + sync::{NamespaceId, Replica}, +}; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::debug; @@ -38,9 +41,9 @@ enum Message { } /// Connect to a peer and sync a replica -pub async fn connect_and_sync( +pub async fn connect_and_sync( endpoint: &MagicEndpoint, - doc: &Replica, + doc: &Replica, peer_id: PeerId, derp_region: Option, addrs: &[SocketAddr], @@ -51,16 +54,16 @@ pub async fn connect_and_sync( .await .context("dial_and_sync")?; let (mut send_stream, mut recv_stream) = connection.open_bi().await?; - let res = run_alice(&mut send_stream, &mut recv_stream, doc).await; + let res = run_alice::(&mut send_stream, &mut recv_stream, doc).await; debug!("sync with peer {}: finish {:?}", peer_id, res); res } /// Runs the initiator side of the sync protocol. -pub async fn run_alice( +pub async fn run_alice( writer: &mut W, reader: &mut R, - alice: &Replica, + alice: &Replica, ) -> Result<()> { let mut buffer = BytesMut::with_capacity(1024); @@ -68,7 +71,7 @@ pub async fn run_alice( let init_message = Message::Init { namespace: alice.namespace(), - message: alice.sync_initial_message(), + message: alice.sync_initial_message().map_err(Into::into)?, }; let msg_bytes = postcard::to_stdvec(&init_message)?; iroh_bytes::protocol::write_lp(writer, &msg_bytes).await?; @@ -83,7 +86,7 @@ pub async fn run_alice( bail!("unexpected message: init"); } Message::Sync(msg) => { - if let Some(msg) = alice.sync_process_message(msg) { + if let Some(msg) = alice.sync_process_message(msg).map_err(Into::into)? { send_sync_message(writer, msg).await?; } else { break; @@ -96,9 +99,9 @@ pub async fn run_alice( } /// Handle an iroh-sync connection and sync all shared documents in the replica store. -pub async fn handle_connection( +pub async fn handle_connection( connecting: quinn::Connecting, - replica_store: ReplicaStore, + replica_store: S, ) -> Result<()> { let connection = connecting.await?; debug!("> connection established!"); @@ -113,10 +116,10 @@ pub async fn handle_connection( } /// Runs the receiver side of the sync protocol. -pub async fn run_bob( +pub async fn run_bob( writer: &mut W, reader: &mut R, - replica_store: ReplicaStore, + replica_store: S, ) -> Result<()> { let mut buffer = BytesMut::with_capacity(1024); @@ -129,10 +132,10 @@ pub async fn run_bob( Message::Init { namespace, message } => { ensure!(replica.is_none(), "double init message"); - match replica_store.get_replica(&namespace) { + match replica_store.get_replica(&namespace)? { Some(r) => { debug!("starting sync for {}", namespace); - if let Some(msg) = r.sync_process_message(message) { + if let Some(msg) = r.sync_process_message(message).map_err(Into::into)? { send_sync_message(writer, msg).await?; } else { break; @@ -147,7 +150,7 @@ pub async fn run_bob( } Message::Sync(msg) => match replica { Some(ref replica) => { - if let Some(msg) = replica.sync_process_message(msg) { + if let Some(msg) = replica.sync_process_message(msg).map_err(Into::into)? { send_sync_message(writer, msg).await?; } else { break; @@ -174,7 +177,7 @@ async fn send_sync_message( #[cfg(test)] mod tests { - use iroh_sync::sync::Namespace; + use iroh_sync::{store::Store as _, sync::Namespace}; use super::*; @@ -182,38 +185,83 @@ mod tests { async fn test_sync_simple() -> Result<()> { let mut rng = rand::thread_rng(); - let replica_store = ReplicaStore::default(); + let alice_replica_store = store::memory::Store::default(); // For now uses same author on both sides. - let author = replica_store.new_author(&mut rng); - let namespace = Namespace::new(&mut rng); - let bob_replica = replica_store.new_replica(namespace.clone()); - bob_replica.hash_and_insert("hello alice", &author, "from bob"); + let author = alice_replica_store.new_author(&mut rng).unwrap(); - let alice_replica = Replica::new(namespace.clone()); - alice_replica.hash_and_insert("hello bob", &author, "from alice"); + let namespace = Namespace::new(&mut rng); - assert_eq!(bob_replica.all().len(), 1); - assert_eq!(alice_replica.all().len(), 1); + let alice_replica = alice_replica_store.new_replica(namespace.clone()).unwrap(); + alice_replica + .hash_and_insert("hello bob", &author, "from alice") + .unwrap(); + + let bob_replica_store = store::memory::Store::default(); + let bob_replica = bob_replica_store.new_replica(namespace.clone()).unwrap(); + bob_replica + .hash_and_insert("hello alice", &author, "from bob") + .unwrap(); + + assert_eq!( + bob_replica_store + .get_all(bob_replica.namespace()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 1 + ); + assert_eq!( + alice_replica_store + .get_all(alice_replica.namespace()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 1 + ); let (alice, bob) = tokio::io::duplex(64); let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); let replica = alice_replica.clone(); let alice_task = tokio::task::spawn(async move { - run_alice(&mut alice_writer, &mut alice_reader, &replica).await + run_alice::(&mut alice_writer, &mut alice_reader, &replica) + .await }); let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); - let bob_replica_store = replica_store.clone(); + let bob_replica_store_task = bob_replica_store.clone(); let bob_task = tokio::task::spawn(async move { - run_bob(&mut bob_writer, &mut bob_reader, bob_replica_store).await + run_bob::( + &mut bob_writer, + &mut bob_reader, + bob_replica_store_task, + ) + .await }); alice_task.await??; bob_task.await??; - assert_eq!(bob_replica.all().len(), 2); - assert_eq!(alice_replica.all().len(), 2); + assert_eq!( + bob_replica_store + .get_all(bob_replica.namespace()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 2 + ); + assert_eq!( + alice_replica_store + .get_all(alice_replica.namespace()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 2 + ); Ok(()) } diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index c68d06272c..5193fe7b53 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -6,6 +6,7 @@ use std::{ time::Instant, }; +use anyhow::Result; use bytes::Bytes; use futures::{ future::{BoxFuture, LocalBoxFuture, Shared}, @@ -17,9 +18,9 @@ use iroh_gossip::net::util::Dialer; use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; use iroh_metrics::{inc, inc_by}; use iroh_net::{tls::PeerId, MagicEndpoint}; -use iroh_sync::sync::{ - Author, InsertOrigin, Namespace, NamespaceId, OnInsertCallback, Replica, ReplicaStore, - SignedEntry, +use iroh_sync::{ + store::{self, Store as _}, + sync::{Author, InsertOrigin, Namespace, OnInsertCallback, Replica, SignedEntry}, }; use tokio::{io::AsyncRead, sync::oneshot}; use tokio_stream::StreamExt; @@ -36,33 +37,32 @@ pub enum DownloadMode { #[derive(Debug, Clone)] pub struct DocStore { - replicas: ReplicaStore, + replicas: store::fs::Store, blobs: BlobStore, local_author: Arc, - storage_path: PathBuf, } +const REPLICA_DB_NAME: &str = "replica.db"; + impl DocStore { - pub fn new(blobs: BlobStore, author: Author, storage_path: PathBuf) -> Self { - Self { - replicas: ReplicaStore::default(), + pub fn new(blobs: BlobStore, author: Author, storage_path: PathBuf) -> Result { + let replicas = store::fs::Store::new(storage_path.join(REPLICA_DB_NAME))?; + + Ok(Self { + replicas, local_author: Arc::new(author), - storage_path, blobs, - } + }) } pub async fn create_or_open( &self, namespace: Namespace, download_mode: DownloadMode, - ) -> anyhow::Result { - let path = self.replica_path(namespace.id()); - let replica = if path.exists() { - let bytes = tokio::fs::read(path).await?; - self.replicas.open_replica(&bytes)? - } else { - self.replicas.new_replica(namespace) + ) -> Result> { + let replica = match self.replicas.get_replica(&namespace.id())? { + Some(replica) => replica, + None => self.replicas.new_replica(namespace)?, }; let doc = Doc::new( @@ -74,21 +74,13 @@ impl DocStore { Ok(doc) } - pub async fn save(&self, doc: &Doc) -> anyhow::Result<()> { - let replica_path = self.replica_path(&doc.replica().namespace()); - tokio::fs::create_dir_all(replica_path.parent().unwrap()).await?; - let bytes = doc.replica().to_bytes()?; - tokio::fs::write(replica_path, bytes).await?; - Ok(()) - } - - fn replica_path(&self, namespace: &NamespaceId) -> PathBuf { - self.storage_path.join(hex::encode(namespace.as_bytes())) - } - pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { crate::sync::handle_connection(conn, self.replicas.clone()).await } + + pub fn store(&self) -> &store::fs::Store { + &self.replicas + } } /// A replica with a [`BlobStore`] for contents. @@ -99,15 +91,15 @@ impl DocStore { /// We want to try other peers if the author is offline (or always). /// We'll need some heuristics which peers to try. #[derive(Clone, Debug)] -pub struct Doc { - replica: Replica, +pub struct Doc { + replica: Replica, blobs: BlobStore, local_author: Arc, } -impl Doc { +impl Doc { pub fn new( - replica: Replica, + replica: Replica, blobs: BlobStore, local_author: Arc, download_mode: DownloadMode, @@ -151,7 +143,7 @@ impl Doc { self.replica.on_insert(callback); } - pub fn replica(&self) -> &Replica { + pub fn replica(&self) -> &Replica { &self.replica } @@ -165,7 +157,9 @@ impl Doc { content: Bytes, ) -> anyhow::Result<(Hash, u64)> { let (hash, len) = self.blobs.put_bytes(content).await?; - self.replica.insert(key, &self.local_author, hash, len); + self.replica + .insert(key, &self.local_author, hash, len) + .map_err(Into::into)?; Ok((hash, len)) } @@ -175,7 +169,9 @@ impl Doc { content: impl AsyncRead + Unpin, ) -> anyhow::Result<(Hash, u64)> { let (hash, len) = self.blobs.put_reader(content).await?; - self.replica.insert(key, &self.local_author, hash, len); + self.replica + .insert(key, &self.local_author, hash, len) + .map_err(Into::into)?; Ok((hash, len)) } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 46159cd4bc..3b1ca971a2 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -13,7 +13,10 @@ use iroh_gossip::{ }; use iroh_metrics::inc; use iroh_net::{tls::PeerId, MagicEndpoint}; -use iroh_sync::sync::{InsertOrigin, Replica, SignedEntry}; +use iroh_sync::{ + store, + sync::{InsertOrigin, Replica, SignedEntry}, +}; use serde::{Deserialize, Serialize}; use tokio::{sync::mpsc, task::JoinError}; use tracing::{debug, error}; @@ -45,9 +48,9 @@ enum SyncState { } #[derive(Debug)] -pub enum ToActor { +pub enum ToActor { SyncDoc { - doc: Replica, + doc: Replica, initial_peers: Vec, }, Shutdown, @@ -55,12 +58,12 @@ pub enum ToActor { /// Handle to a running live sync actor #[derive(Debug, Clone)] -pub struct LiveSync { - to_actor_tx: mpsc::Sender, +pub struct LiveSync { + to_actor_tx: mpsc::Sender>, task: Shared>>>, } -impl LiveSync { +impl LiveSync { pub fn spawn(endpoint: MagicEndpoint, gossip: GossipHandle) -> Self { let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); let mut actor = Actor::new(endpoint, gossip, to_actor_rx); @@ -78,14 +81,18 @@ impl LiveSync { /// Cancel the live sync. pub async fn cancel(&self) -> Result<()> { - self.to_actor_tx.send(ToActor::Shutdown).await?; + self.to_actor_tx.send(ToActor::::Shutdown).await?; self.task.clone().await?; Ok(()) } - pub async fn add(&self, doc: Replica, initial_peers: Vec) -> Result<()> { + pub async fn add( + &self, + doc: Replica, + initial_peers: Vec, + ) -> Result<()> { self.to_actor_tx - .send(ToActor::SyncDoc { doc, initial_peers }) + .send(ToActor::::SyncDoc { doc, initial_peers }) .await?; Ok(()) } @@ -93,15 +100,15 @@ impl LiveSync { // TODO: Also add `handle_connection` to the replica and track incoming sync requests here too. // Currently peers might double-sync in both directions. -struct Actor { +struct Actor { endpoint: MagicEndpoint, gossip: GossipHandle, - docs: HashMap, + docs: HashMap>, subscription: BoxStream<'static, Result<(TopicId, Event)>>, sync_state: HashMap<(TopicId, PeerId), SyncState>, - to_actor_rx: mpsc::Receiver, + to_actor_rx: mpsc::Receiver>, insert_entry_tx: flume::Sender<(TopicId, SignedEntry)>, insert_entry_rx: flume::Receiver<(TopicId, SignedEntry)>, @@ -109,11 +116,11 @@ struct Actor { pending_joins: FuturesUnordered)>>, } -impl Actor { +impl Actor { pub fn new( endpoint: MagicEndpoint, gossip: GossipHandle, - to_actor_rx: mpsc::Receiver, + to_actor_rx: mpsc::Receiver>, ) -> Self { let (insert_tx, insert_rx) = flume::bounded(64); let sub = gossip.clone().subscribe_all().boxed(); @@ -193,7 +200,7 @@ impl Actor { async move { debug!("sync with {peer}"); // TODO: Make sure that the peer is dialable. - let res = connect_and_sync(&endpoint, &doc, peer, None, &[]).await; + let res = connect_and_sync::(&endpoint, &doc, peer, None, &[]).await; debug!("> synced with {peer}: {res:?}"); // collect metrics match &res { @@ -215,7 +222,11 @@ impl Actor { Ok(()) } - async fn insert_doc(&mut self, doc: Replica, initial_peers: Vec) -> Result<()> { + async fn insert_doc( + &mut self, + doc: Replica, + initial_peers: Vec, + ) -> Result<()> { let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id).collect(); // add addresses of initial peers to our endpoint address book From 56fb90cb9e5daf855c04eb5fab6bbcba04a7c922 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 12:55:10 +0200 Subject: [PATCH 036/172] adapt after rebase on main --- Cargo.lock | 22 ++++---------------- iroh-bytes/src/util.rs | 6 ++++++ iroh-gossip/Cargo.toml | 4 ++-- iroh-gossip/src/proto/util.rs | 2 +- iroh-sync/Cargo.toml | 6 +++--- iroh/Cargo.toml | 4 ++-- iroh/examples/sync.rs | 38 ++++++++++++----------------------- iroh/src/sync/live.rs | 16 ++++++++------- 8 files changed, 40 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4cb8ff9129..991697a349 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,20 +327,6 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" -[[package]] -name = "blake3" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "digest", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1830,10 +1816,9 @@ dependencies = [ [[package]] name = "iroh-gossip" -version = "0.4.1" +version = "0.5.1" dependencies = [ "anyhow", - "blake3", "bytes", "clap", "data-encoding", @@ -1842,6 +1827,7 @@ dependencies = [ "futures", "genawaiter", "indexmap 2.0.0", + "iroh-blake3", "iroh-metrics", "iroh-net", "once_cell", @@ -1960,15 +1946,15 @@ dependencies = [ [[package]] name = "iroh-sync" -version = "0.1.0" +version = "0.5.1" dependencies = [ "anyhow", - "blake3", "bytes", "crossbeam", "derive_more", "ed25519-dalek", "hex", + "iroh-blake3", "iroh-bytes", "once_cell", "ouroboros", diff --git a/iroh-bytes/src/util.rs b/iroh-bytes/src/util.rs index 56e83051a4..cc89fc47c6 100644 --- a/iroh-bytes/src/util.rs +++ b/iroh-bytes/src/util.rs @@ -83,6 +83,12 @@ impl From<[u8; 32]> for Hash { } } +impl From for [u8; 32]{ + fn from(value: Hash) -> Self { + *value.as_bytes() + } +} + impl From<&[u8; 32]> for Hash { fn from(value: &[u8; 32]) -> Self { Hash(blake3::Hash::from(*value)) diff --git a/iroh-gossip/Cargo.toml b/iroh-gossip/Cargo.toml index bf2254095b..326254a209 100644 --- a/iroh-gossip/Cargo.toml +++ b/iroh-gossip/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-gossip" -version = "0.4.1" +version = "0.5.1" edition = "2021" readme = "README.md" description = "gossip messages over broadcast trees" @@ -11,7 +11,7 @@ repository = "https://github.com/n0-computer/iroh-sync" [dependencies] # proto dependencies (required) anyhow = { version = "1", features = ["backtrace"] } -blake3 = "1.3.3" +blake3 = { package = "iroh-blake3", version = "1.4.3"} bytes = { version = "1.4.0", features = ["serde"] } data-encoding = "2.4.0" derive_more = { version = "1.0.0-beta.1", features = ["add", "debug", "display", "from", "try_into"] } diff --git a/iroh-gossip/src/proto/util.rs b/iroh-gossip/src/proto/util.rs index 64c7a92c9a..03a759ad01 100644 --- a/iroh-gossip/src/proto/util.rs +++ b/iroh-gossip/src/proto/util.rs @@ -56,7 +56,7 @@ macro_rules! idbytes_impls { } } - impl> From for $ty { + impl> std::convert::From for $ty { fn from(value: T) -> Self { Self::from_bytes(value.into()) } diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index e0132aed13..8ab9a1de20 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-sync" -version = "0.1.0" +version = "0.5.1" edition = "2021" readme = "README.md" description = "Iroh sync" @@ -10,7 +10,7 @@ repository = "https://github.com/n0-computer/iroh" [dependencies] anyhow = "1.0.71" -blake3 = "1.3.3" +blake3 = { package = "iroh-blake3", version = "1.4.3"} crossbeam = "0.8.2" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } ed25519-dalek = { version = "2.0.0-rc.2", features = ["serde", "rand_core"] } @@ -35,4 +35,4 @@ tempfile = "3.4" [features] default = ["fs-store"] -fs-store = ["redb", "ouroboros"] \ No newline at end of file +fs-store = ["redb", "ouroboros"] diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 4f1b902b44..f750758471 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -26,8 +26,8 @@ iroh-metrics = { version = "0.5.0", path = "../iroh-metrics", optional = true } iroh-net = { version = "0.5.1", path = "../iroh-net" } num_cpus = { version = "1.15.0" } portable-atomic = "1" -iroh-sync = { path = "../iroh-sync" } -iroh-gossip = { path = "../iroh-gossip" } +iroh-sync = { version = "0.5.1", path = "../iroh-sync" } +iroh-gossip = { version = "0.5.1", path = "../iroh-gossip" } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quic-rpc = { version = "0.6", default-features = false, features = ["flume-transport"] } quinn = "0.10" diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 917011efd1..edffe96fc4 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -21,7 +21,7 @@ use iroh::sync::{ }; use iroh_bytes::util::runtime; use iroh_gossip::{ - net::{GossipHandle, GOSSIP_ALPN}, + net::{Gossip, GOSSIP_ALPN}, proto::TopicId, }; use iroh_metrics::{ @@ -29,8 +29,8 @@ use iroh_metrics::{ struct_iterable::Iterable, }; use iroh_net::{ - defaults::{default_derp_map, DEFAULT_DERP_STUN_PORT}, - derp::{DerpMap, UseIpv4, UseIpv6}, + defaults::{default_derp_map}, + derp::{DerpMap}, magic_endpoint::get_alpn, tls::Keypair, MagicEndpoint, @@ -131,7 +131,7 @@ async fn run(args: Args) -> anyhow::Result<()> { // configure our derp map let derp_map = match (args.no_derp, args.derp) { (false, None) => Some(default_derp_map()), - (false, Some(url)) => Some(derp_map_from_url(url)?), + (false, Some(url)) => Some(DerpMap::from_url(url, 0)), (true, None) => None, (true, Some(_)) => bail!("You cannot set --no-derp and --derp at the same time"), }; @@ -140,7 +140,7 @@ async fn run(args: Args) -> anyhow::Result<()> { // build our magic endpoint and the gossip protocol let (endpoint, gossip, initial_endpoints) = { // init a cell that will hold our gossip handle to be used in endpoint callbacks - let gossip_cell: OnceCell = OnceCell::new(); + let gossip_cell: OnceCell = OnceCell::new(); // init a channel that will emit once the initial endpoints of our local node are discovered let (initial_endpoints_tx, mut initial_endpoints_rx) = mpsc::channel(1); // build the magic endpoint @@ -167,7 +167,7 @@ async fn run(args: Args) -> anyhow::Result<()> { .await?; // initialize the gossip protocol - let gossip = GossipHandle::from_endpoint(endpoint.clone(), Default::default()); + let gossip = Gossip::from_endpoint(endpoint.clone(), Default::default()); // insert into the gossip cell to be used in the endpoint callbacks above gossip_cell.set(gossip.clone()).unwrap(); @@ -181,7 +181,7 @@ async fn run(args: Args) -> anyhow::Result<()> { let (topic, peers) = match &args.command { Command::Open { doc_name } => { - let topic: TopicId = blake3::hash(doc_name.as_bytes()).into(); + let topic: TopicId = iroh_bytes::Hash::new(doc_name.as_bytes()).into(); println!( "> opening document {doc_name} as namespace {} and waiting for peers to join us...", fmt_hash(topic.as_bytes()) @@ -685,7 +685,7 @@ impl FromStr for Cmd { #[derive(Debug)] struct State { - gossip: GossipHandle, + gossip: Gossip, docs: DocStore, bytes: IrohBytesHandlers, } @@ -879,25 +879,13 @@ fn parse_keypair(secret: &str) -> anyhow::Result { fn fmt_derp_map(derp_map: &Option) -> String { match derp_map { None => "None".to_string(), - Some(map) => { - let regions = map.regions.iter().map(|(id, region)| { - let nodes = region.nodes.iter().map(|node| node.url.to_string()); - (*id, nodes.collect::>()) - }); - format!("{:?}", regions.collect::>()) - } + Some(map) => map + .regions() + .flat_map(|region| region.nodes.iter().map(|node| node.url.to_string())) + .collect::>() + .join(", "), } } -fn derp_map_from_url(url: Url) -> anyhow::Result { - Ok(DerpMap::default_from_node( - url, - DEFAULT_DERP_STUN_PORT, - UseIpv4::TryDns, - UseIpv6::TryDns, - 0, - )) -} - fn canonicalize_path(path: &str) -> anyhow::Result { let path = PathBuf::from(shellexpand::tilde(&path).to_string()); Ok(path) diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 3b1ca971a2..7e35e36e1c 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -8,7 +8,7 @@ use futures::{ FutureExt, TryFutureExt, }; use iroh_gossip::{ - net::{Event, GossipHandle}, + net::{Event, Gossip}, proto::TopicId, }; use iroh_metrics::inc; @@ -64,7 +64,7 @@ pub struct LiveSync { } impl LiveSync { - pub fn spawn(endpoint: MagicEndpoint, gossip: GossipHandle) -> Self { + pub fn spawn(endpoint: MagicEndpoint, gossip: Gossip) -> Self { let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); let mut actor = Actor::new(endpoint, gossip, to_actor_rx); let task = tokio::spawn(async move { @@ -102,7 +102,7 @@ impl LiveSync { // Currently peers might double-sync in both directions. struct Actor { endpoint: MagicEndpoint, - gossip: GossipHandle, + gossip: Gossip, docs: HashMap>, subscription: BoxStream<'static, Result<(TopicId, Event)>>, @@ -119,7 +119,7 @@ struct Actor { impl Actor { pub fn new( endpoint: MagicEndpoint, - gossip: GossipHandle, + gossip: Gossip, to_actor_rx: mpsc::Receiver>, ) -> Self { let (insert_tx, insert_rx) = flume::bounded(64); @@ -237,13 +237,15 @@ impl Actor { } // join gossip for the topic to receive and send message - let topic: TopicId = doc.namespace().as_bytes().into(); + let topic = TopicId::from_bytes(*doc.namespace().as_bytes()); self.pending_joins.push({ let peer_ids = peer_ids.clone(); let gossip = self.gossip.clone(); async move { - let res = gossip.join(topic, peer_ids).await; - (topic, res) + match gossip.join(topic, peer_ids).await { + Err(err) => (topic, Err(err)), + Ok(fut) => (topic, fut.await), + } } .boxed() }); From e4fca324a3e071ece2f01721213d80f518fc6fab Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 13:48:51 +0200 Subject: [PATCH 037/172] chore: fmt --- iroh-bytes/src/util.rs | 2 +- iroh/examples/sync.rs | 5 +---- iroh/src/database/flat/writable.rs | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/iroh-bytes/src/util.rs b/iroh-bytes/src/util.rs index cc89fc47c6..40244827fb 100644 --- a/iroh-bytes/src/util.rs +++ b/iroh-bytes/src/util.rs @@ -83,7 +83,7 @@ impl From<[u8; 32]> for Hash { } } -impl From for [u8; 32]{ +impl From for [u8; 32] { fn from(value: Hash) -> Self { *value.as_bytes() } diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index edffe96fc4..d27076a584 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -29,10 +29,7 @@ use iroh_metrics::{ struct_iterable::Iterable, }; use iroh_net::{ - defaults::{default_derp_map}, - derp::{DerpMap}, - magic_endpoint::get_alpn, - tls::Keypair, + defaults::default_derp_map, derp::DerpMap, magic_endpoint::get_alpn, tls::Keypair, MagicEndpoint, }; use iroh_sync::{ diff --git a/iroh/src/database/flat/writable.rs b/iroh/src/database/flat/writable.rs index 8f933813ee..6973d6e4fe 100644 --- a/iroh/src/database/flat/writable.rs +++ b/iroh/src/database/flat/writable.rs @@ -109,7 +109,7 @@ impl WritableFileDatabase { std::fs::OpenOptions::new() .write(true) .create(true) - .open(&path) + .open(path) }) .await?; From b33d3ef0e80ceefd14acd712c08602c3952f3fa6 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 28 Jul 2023 14:09:27 +0200 Subject: [PATCH 038/172] feat: download from peer that informed us about a change --- iroh-net/src/tls.rs | 5 + iroh-sync/src/sync.rs | 20 ++- iroh/src/sync.rs | 26 +++- iroh/src/sync/content.rs | 267 ++++++++++++++++++++++++++++----------- iroh/src/sync/live.rs | 4 +- 5 files changed, 234 insertions(+), 88 deletions(-) diff --git a/iroh-net/src/tls.rs b/iroh-net/src/tls.rs index 07dbeda035..83e1dde697 100644 --- a/iroh-net/src/tls.rs +++ b/iroh-net/src/tls.rs @@ -118,6 +118,11 @@ impl PeerId { let key = PublicKey::from_bytes(bytes)?; Ok(PeerId(key)) } + + /// Get the peer id as a byte array. + pub fn to_bytes(&self) -> [u8; 32] { + self.0.to_bytes() + } } impl From for PeerId { diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 7ac32f7522..b569aea487 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -211,10 +211,13 @@ impl NamespaceId { /// [parking_lot::RwLock] requiring `Sync`. pub type OnInsertCallback = Box; +/// TODO: PeerId is in iroh-net which iroh-sync doesn't depend on. Add iroh-common crate with `PeerId`. +pub type PeerIdBytes = [u8; 32]; + #[derive(Debug, Clone)] pub enum InsertOrigin { Local, - Sync, + Sync(Option), } #[derive(derive_more::Debug, Clone)] @@ -299,7 +302,11 @@ impl> Replica { RecordIdentifier::new(key, inner.namespace.id(), author.id()) } - pub fn insert_remote_entry(&self, entry: SignedEntry) -> anyhow::Result<()> { + pub fn insert_remote_entry( + &self, + entry: SignedEntry, + received_from: Option, + ) -> anyhow::Result<()> { entry.verify()?; let mut inner = self.inner.write(); let id = entry.entry.id.clone(); @@ -307,7 +314,7 @@ impl> Replica { drop(inner); let on_insert = self.on_insert.read(); for cb in &*on_insert { - cb(InsertOrigin::Sync, entry.clone()); + cb(InsertOrigin::Sync(received_from), entry.clone()); } Ok(()) } @@ -321,6 +328,7 @@ impl> Replica { pub fn sync_process_message( &self, message: crate::ranger::Message, + from_peer: Option, ) -> Result>, S::Error> { let reply = self .inner @@ -329,7 +337,7 @@ impl> Replica { .process_message(message, |_key, entry| { let on_insert = self.on_insert.read(); for cb in &*on_insert { - cb(InsertOrigin::Sync, entry.clone()); + cb(InsertOrigin::Sync(from_peer), entry.clone()); } })?; @@ -915,8 +923,8 @@ mod tests { assert!(rounds < 100, "too many rounds"); rounds += 1; println!("round {}", rounds); - if let Some(msg) = bob.sync_process_message(msg).map_err(Into::into)? { - next_to_bob = alice.sync_process_message(msg).map_err(Into::into)?; + if let Some(msg) = bob.sync_process_message(msg, None).map_err(Into::into)? { + next_to_bob = alice.sync_process_message(msg, None).map_err(Into::into); } } diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index bcadeb23f9..f56e4b5459 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -4,7 +4,7 @@ use std::net::SocketAddr; use anyhow::{bail, ensure, Context, Result}; use bytes::BytesMut; -use iroh_net::{tls::PeerId, MagicEndpoint}; +use iroh_net::{tls::PeerId, MagicEndpoint, magic_endpoint::get_peer_id}; use iroh_sync::{ store, sync::{NamespaceId, Replica}, @@ -54,7 +54,7 @@ pub async fn connect_and_sync( .await .context("dial_and_sync")?; let (mut send_stream, mut recv_stream) = connection.open_bi().await?; - let res = run_alice::(&mut send_stream, &mut recv_stream, doc).await; + let res = run_alice::(&mut send_stream, &mut recv_stream, doc, Some(peer_id)).await; debug!("sync with peer {}: finish {:?}", peer_id, res); res } @@ -64,7 +64,9 @@ pub async fn run_alice, + peer: Option, ) -> Result<()> { + let peer = peer.map(|peer| peer.to_bytes()); let mut buffer = BytesMut::with_capacity(1024); // Init message @@ -86,7 +88,7 @@ pub async fn run_alice { - if let Some(msg) = alice.sync_process_message(msg).map_err(Into::into)? { + if let Some(msg) = alice.sync_process_message(msg, peer).map_err(Into::into)? { send_sync_message(writer, msg).await?; } else { break; @@ -105,9 +107,16 @@ pub async fn handle_connection( ) -> Result<()> { let connection = connecting.await?; debug!("> connection established!"); + let peer_id = get_peer_id(&connection).await?; let (mut send_stream, mut recv_stream) = connection.accept_bi().await?; - run_bob(&mut send_stream, &mut recv_stream, replica_store).await?; + run_bob( + &mut send_stream, + &mut recv_stream, + replica_store, + Some(peer_id), + ) + .await?; send_stream.finish().await?; debug!("done"); @@ -120,7 +129,9 @@ pub async fn run_bob, ) -> Result<()> { + let peer = peer.map(|peer| peer.to_bytes()); let mut buffer = BytesMut::with_capacity(1024); let mut replica = None; @@ -135,7 +146,7 @@ pub async fn run_bob { debug!("starting sync for {}", namespace); - if let Some(msg) = r.sync_process_message(message).map_err(Into::into)? { + if let Some(msg) = r.sync_process_message(message, peer).map_err(Into::into)? { send_sync_message(writer, msg).await?; } else { break; @@ -150,7 +161,7 @@ pub async fn run_bob match replica { Some(ref replica) => { - if let Some(msg) = replica.sync_process_message(msg).map_err(Into::into)? { + if let Some(msg) = replica.sync_process_message(msg, peer).map_err(Into::into)? { send_sync_message(writer, msg).await?; } else { break; @@ -226,7 +237,7 @@ mod tests { let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); let replica = alice_replica.clone(); let alice_task = tokio::task::spawn(async move { - run_alice::(&mut alice_writer, &mut alice_reader, &replica) + run_alice::(&mut alice_writer, &mut alice_reader, &replica, None) .await }); @@ -237,6 +248,7 @@ mod tests { &mut bob_writer, &mut bob_reader, bob_replica_store_task, + None ) .await }); diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 5193fe7b53..5f15885ad7 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::{HashMap, VecDeque}, io, path::{Path, PathBuf}, sync::{Arc, Mutex}, @@ -20,7 +20,7 @@ use iroh_metrics::{inc, inc_by}; use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::{ store::{self, Store as _}, - sync::{Author, InsertOrigin, Namespace, OnInsertCallback, Replica, SignedEntry}, + sync::{Author, InsertOrigin, Namespace, OnInsertCallback, PeerIdBytes, Replica, SignedEntry}, }; use tokio::{io::AsyncRead, sync::oneshot}; use tokio_stream::StreamExt; @@ -114,11 +114,13 @@ impl Doc { // setup on_insert callback to trigger download on remote insert if let DownloadMode::Always = download_mode { let doc_clone = doc.clone(); - doc.replica.on_insert(Box::new(move |origin, entry| { - if matches!(origin, InsertOrigin::Sync) { - doc_clone.download_content_from_author(&entry); - } - })); + doc.replica + .on_insert(Box::new(move |origin, entry| match origin { + InsertOrigin::Sync(peer) => { + doc_clone.download_content_from_author_and_other_peer(&entry, peer); + } + InsertOrigin::Local => {} + })); } // Collect metrics @@ -129,7 +131,7 @@ impl Doc { inc!(Metrics, new_entries_local); inc_by!(Metrics, new_entries_local_size, size); } - InsertOrigin::Sync => { + InsertOrigin::Sync(_) => { inc!(Metrics, new_entries_remote); inc_by!(Metrics, new_entries_remote_size, size); } @@ -184,11 +186,26 @@ impl Doc { self.insert_reader(&key, reader).await } - pub fn download_content_from_author(&self, entry: &SignedEntry) { - let hash = *entry.entry().record().content_hash(); - let peer_id = PeerId::from_bytes(entry.entry().id().author().as_bytes()) + pub fn download_content_from_author_and_other_peer( + &self, + entry: &SignedEntry, + other_peer: Option, + ) { + let author_peer_id = PeerId::from_bytes(entry.entry().id().author().as_bytes()) .expect("failed to convert author to peer id"); - self.blobs.start_download(hash, peer_id); + + let mut peers = vec![author_peer_id]; + + if let Some(other_peer) = other_peer { + let other_peer_id = + PeerId::from_bytes(&other_peer).expect("failed to convert author to peer id"); + if other_peer_id != peers[0] { + peers.push(other_peer_id); + } + } + + let hash = *entry.entry().record().content_hash(); + self.blobs.start_download(hash, peers); } pub async fn get_content_bytes(&self, entry: &SignedEntry) -> Option { @@ -234,9 +251,9 @@ impl BlobStore { self.db.db() } - pub fn start_download(&self, hash: Hash, peer: PeerId) { + pub fn start_download(&self, hash: Hash, peers: Vec) { if !self.db.has(&hash) { - self.downloader.start_download(hash, peer); + self.downloader.start_download(hash, peers); } } @@ -273,7 +290,7 @@ pub type DownloadFuture = Shared>>; #[derive(Debug)] pub struct DownloadRequest { hash: Hash, - peer: PeerId, + peers: Vec, reply: DownloadReply, } @@ -320,19 +337,21 @@ impl Downloader { } } - pub fn start_download(&self, hash: Hash, peer: PeerId) { + pub fn start_download(&self, hash: Hash, peers: Vec) { let (reply, reply_rx) = oneshot::channel(); - let req = DownloadRequest { hash, peer, reply }; - let pending_downloads = self.pending_downloads.clone(); - let fut = async move { - let res = reply_rx.await; - pending_downloads.lock().unwrap().remove(&hash); - res.ok().flatten() - }; - self.pending_downloads - .lock() - .unwrap() - .insert(hash, fut.boxed().shared()); + let req = DownloadRequest { hash, peers, reply }; + if self.pending_downloads.lock().unwrap().get(&hash).is_none() { + let pending_downloads = self.pending_downloads.clone(); + let fut = async move { + let res = reply_rx.await; + pending_downloads.lock().unwrap().remove(&hash); + res.ok().flatten() + }; + self.pending_downloads + .lock() + .unwrap() + .insert(hash, fut.boxed().shared()); + } // TODO: this is potentially blocking inside an async call. figure out a better solution if let Err(err) = self.to_actor_tx.send(req) { warn!("download actor dropped: {err}"); @@ -349,9 +368,8 @@ pub struct DownloadActor { db: WritableFileDatabase, conns: HashMap, replies: HashMap>, - peer_hashes: HashMap>, - hash_peers: HashMap>, - pending_downloads: PendingDownloadsFutures, + pending_download_futs: PendingDownloadsFutures, + queue: DownloadQueue, rx: flume::Receiver, } impl DownloadActor { @@ -366,9 +384,8 @@ impl DownloadActor { dialer: Dialer::new(endpoint), replies: Default::default(), conns: Default::default(), - pending_downloads: Default::default(), - peer_hashes: Default::default(), - hash_peers: Default::default(), + pending_download_futs: Default::default(), + queue: Default::default(), } } pub async fn run(&mut self) -> anyhow::Result<()> { @@ -386,8 +403,9 @@ impl DownloadActor { }, Err(err) => self.on_peer_fail(&peer, err), }, - Some((peer, hash, res)) = self.pending_downloads.next() => match res { + Some((peer, hash, res)) = self.pending_download_futs.next() => match res { Ok(Some((hash, size))) => { + self.queue.on_success(hash, peer); self.reply(hash, Some((hash, size))); self.on_peer_ready(peer); } @@ -409,66 +427,169 @@ impl DownloadActor { fn on_peer_fail(&mut self, peer: &PeerId, err: anyhow::Error) { warn!("download from {peer} failed: {err}"); - for hash in self.peer_hashes.remove(peer).into_iter().flatten() { - self.on_not_found(peer, hash); + for hash in self.queue.on_peer_fail(peer) { + self.reply(hash, None); } self.conns.remove(peer); } fn on_not_found(&mut self, peer: &PeerId, hash: Hash) { - if let Some(peers) = self.hash_peers.get_mut(&hash) { - peers.remove(peer); - if peers.is_empty() { - self.reply(hash, None); - self.hash_peers.remove(&hash); - } + self.queue.on_not_found(hash, *peer); + if self.queue.has_no_candidates(&hash) { + self.reply(hash, None); } } fn on_peer_ready(&mut self, peer: PeerId) { - if let Some(hash) = self - .peer_hashes - .get_mut(&peer) - .and_then(|hashes| hashes.pop_front()) - { - let conn = self.conns.get(&peer).unwrap().clone(); - let blobs = self.db.clone(); - let fut = async move { - let start = Instant::now(); - let res = blobs.download_single(conn, hash).await; - // record metrics - let elapsed = start.elapsed().as_millis(); - match &res { - Ok(Some((_hash, len))) => { - inc!(Metrics, downloads_success); - inc_by!(Metrics, download_bytes_total, *len); - inc_by!(Metrics, download_time_total, elapsed as u64); - } - Ok(None) => inc!(Metrics, downloads_notfound), - Err(_) => inc!(Metrics, downloads_error), - } - (peer, hash, res) - }; - self.pending_downloads.push(fut.boxed_local()); + if let Some(hash) = self.queue.try_next_for_peer(peer) { + self.start_download_unchecked(peer, hash); } else { self.conns.remove(&peer); - self.peer_hashes.remove(&peer); } } + fn start_download_unchecked(&mut self, peer: PeerId, hash: Hash) { + let conn = self.conns.get(&peer).unwrap().clone(); + let blobs = self.db.clone(); + let fut = async move { + let start = Instant::now(); + let res = blobs.download_single(conn, hash).await; + // record metrics + let elapsed = start.elapsed().as_millis(); + match &res { + Ok(Some((_hash, len))) => { + inc!(Metrics, downloads_success); + inc_by!(Metrics, download_bytes_total, *len); + inc_by!(Metrics, download_time_total, elapsed as u64); + } + Ok(None) => inc!(Metrics, downloads_notfound), + Err(_) => inc!(Metrics, downloads_error), + } + (peer, hash, res) + }; + self.pending_download_futs.push(fut.boxed_local()); + } + async fn on_download_request(&mut self, req: DownloadRequest) { - let DownloadRequest { peer, hash, reply } = req; + let DownloadRequest { peers, hash, reply } = req; if self.db.has(&hash) { let size = self.db.get_size(&hash).await.unwrap(); reply.send(Some((hash, size))).ok(); return; } - debug!("queue download {hash} from {peer}"); self.replies.entry(hash).or_default().push_back(reply); - self.hash_peers.entry(hash).or_default().insert(peer); - self.peer_hashes.entry(peer).or_default().push_back(hash); - if self.conns.get(&peer).is_none() && !self.dialer.is_pending(&peer) { - self.dialer.queue_dial(peer, &iroh_bytes::protocol::ALPN); + for peer in peers { + self.queue.push_candidate(hash, peer); + // TODO: Don't dial all peers instantly. + if self.conns.get(&peer).is_none() && !self.dialer.is_pending(&peer) { + self.dialer.queue_dial(peer, &iroh_bytes::protocol::ALPN); + } } } } + +#[derive(Debug, Default)] +struct DownloadQueue { + candidates_by_hash: HashMap>, + candidates_by_peer: HashMap>, + running_by_hash: HashMap, + running_by_peer: HashMap, +} + +impl DownloadQueue { + pub fn push_candidate(&mut self, hash: Hash, peer: PeerId) { + self.candidates_by_hash + .entry(hash) + .or_default() + .push_back(peer); + self.candidates_by_peer + .entry(peer) + .or_default() + .push_back(hash); + } + + pub fn try_next_for_peer(&mut self, peer: PeerId) -> Option { + let mut next = None; + for (idx, hash) in self.candidates_by_peer.get(&peer)?.iter().enumerate() { + if !self.running_by_hash.contains_key(hash) { + next = Some((idx, *hash)); + break; + } + } + if let Some((idx, hash)) = next { + self.running_by_hash.insert(hash, peer); + self.running_by_peer.insert(peer, hash); + self.candidates_by_peer.get_mut(&peer).unwrap().remove(idx); + if let Some(peers) = self.candidates_by_hash.get_mut(&hash) { + peers.retain(|p| p != &peer); + } + self.ensure_no_empty(hash, peer); + return Some(hash); + } else { + None + } + } + + pub fn has_no_candidates(&self, hash: &Hash) -> bool { + self.candidates_by_hash.get(hash).is_none() && self.running_by_hash.get(&hash).is_none() + } + + pub fn on_success(&mut self, hash: Hash, peer: PeerId) -> Option<(PeerId, Hash)> { + let peer2 = self.running_by_hash.remove(&hash); + debug_assert_eq!(peer2, Some(peer)); + self.running_by_peer.remove(&peer); + self.try_next_for_peer(peer).map(|hash| (peer, hash)) + } + + pub fn on_peer_fail(&mut self, peer: &PeerId) -> Vec { + let mut failed = vec![]; + for hash in self + .candidates_by_peer + .remove(peer) + .map(|hashes| hashes.into_iter()) + .into_iter() + .flatten() + { + if let Some(peers) = self.candidates_by_hash.get_mut(&hash) { + peers.retain(|p| p != peer); + if peers.is_empty() && self.running_by_hash.get(&hash).is_none() { + failed.push(hash); + } + } + } + if let Some(hash) = self.running_by_peer.remove(&peer) { + self.running_by_hash.remove(&hash); + if self.candidates_by_hash.get(&hash).is_none() { + failed.push(hash); + } + } + failed + } + + pub fn on_not_found(&mut self, hash: Hash, peer: PeerId) { + let peer2 = self.running_by_hash.remove(&hash); + debug_assert_eq!(peer2, Some(peer)); + self.running_by_peer.remove(&peer); + self.ensure_no_empty(hash, peer); + } + + fn ensure_no_empty(&mut self, hash: Hash, peer: PeerId) { + if self + .candidates_by_peer + .get(&peer) + .map_or(false, |hashes| hashes.is_empty()) + { + self.candidates_by_peer.remove(&peer); + } + if self + .candidates_by_hash + .get(&hash) + .map_or(false, |peers| peers.is_empty()) + { + self.candidates_by_hash.remove(&hash); + } + } +} + +#[cfg(test)] +mod test {} diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 7e35e36e1c..320c236385 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -288,10 +288,10 @@ impl Actor { }; match event { // We received a gossip message. Try to insert it into our replica. - Event::Received(data, _prev_peer) => { + Event::Received(data, prev_peer) => { let op: Op = postcard::from_bytes(&data)?; match op { - Op::Put(entry) => doc.insert_remote_entry(entry)?, + Op::Put(entry) => doc.insert_remote_entry(entry, Some(prev_peer.to_bytes()))?, } } // A new neighbor appeared in the gossip swarm. Try to sync with it directly. From cd17db84e1c94a3f1bed816d1254e4be6b38979d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 13:37:54 +0200 Subject: [PATCH 039/172] refactor: move downloader out of sync module --- iroh/src/download.rs | 347 +++++++++++++++++++++++++++++++++++++++ iroh/src/lib.rs | 2 +- iroh/src/sync/content.rs | 335 +------------------------------------ 3 files changed, 357 insertions(+), 327 deletions(-) create mode 100644 iroh/src/download.rs diff --git a/iroh/src/download.rs b/iroh/src/download.rs new file mode 100644 index 0000000000..c43640f38f --- /dev/null +++ b/iroh/src/download.rs @@ -0,0 +1,347 @@ +//! Download queue + +use std::{ + collections::{HashMap, VecDeque}, + sync::{Arc, Mutex}, + time::Instant, +}; + +use futures::{ + future::{BoxFuture, LocalBoxFuture, Shared}, + stream::FuturesUnordered, + FutureExt, +}; +use iroh_bytes::util::Hash; +use iroh_gossip::net::util::Dialer; +use iroh_metrics::{inc, inc_by}; +use iroh_net::{tls::PeerId, MagicEndpoint}; +use tokio::sync::oneshot; +use tokio_stream::StreamExt; +use tracing::{debug, error, warn}; + +// TODO: Move metrics to iroh-bytes metrics +use super::sync::metrics::Metrics; +// TODO: Will be replaced by proper persistent DB once +// https://github.com/n0-computer/iroh/pull/1320 is merged +use crate::database::flat::writable::WritableFileDatabase; + +/// Future for the completion of a download request +pub type DownloadFuture = Shared>>; + +/// A download queue for iroh-bytes +/// +/// Spawns a background task that handles connecting to peers and performing get requests. +/// +/// TODO: Move to iroh-bytes or replace with corresponding feature from iroh-bytes once available +/// TODO: Support retries and backoff - become a proper queue... +/// TODO: Download requests send via synchronous flume::Sender::send. Investigate if we want async +/// here. We currently use [`Downloader::push`] from [`iroh_sync::Replica::on_insert`] callbacks, +/// which are sync, thus we need a sync method on the Downloader to push new download requests. +#[derive(Debug, Clone)] +pub struct Downloader { + pending_downloads: Arc>>, + to_actor_tx: flume::Sender, +} + +impl Downloader { + /// Create a new downloader + pub fn new( + rt: iroh_bytes::util::runtime::Handle, + endpoint: MagicEndpoint, + db: WritableFileDatabase, + ) -> Self { + let (tx, rx) = flume::bounded(64); + // spawn the actor on a local pool + // the local pool is required because WritableFileDatabase::download_single + // returns a future that is !Send + rt.local_pool().spawn_pinned(move || async move { + let mut actor = DownloadActor::new(endpoint, db, rx); + if let Err(err) = actor.run().await { + error!("download actor failed with error {err:?}"); + } + }); + Self { + pending_downloads: Arc::new(Mutex::new(HashMap::new())), + to_actor_tx: tx, + } + } + + /// Add a new download request to the download queue. + /// + /// Note: This method takes only [`PeerId`]s and will attempt to connect to those peers. For + /// this to succeed, you need to add addresses for these peers to the magic endpoint's + /// addressbook yourself. See [`MagicEndpoint::add_known_addrs`]. + pub fn push(&self, hash: Hash, peers: Vec) { + let (reply, reply_rx) = oneshot::channel(); + let req = DownloadRequest { hash, peers, reply }; + + // TODO: this is potentially blocking inside an async call. figure out a better solution + if let Err(err) = self.to_actor_tx.send(req) { + warn!("download actor dropped: {err}"); + } + + if self.pending_downloads.lock().unwrap().get(&hash).is_none() { + let pending_downloads = self.pending_downloads.clone(); + let fut = async move { + let res = reply_rx.await; + pending_downloads.lock().unwrap().remove(&hash); + res.ok().flatten() + }; + self.pending_downloads + .lock() + .unwrap() + .insert(hash, fut.boxed().shared()); + } + } + + /// Returns a future that completes once the blob for `hash` has been downloaded, or all queued + /// requests for that blob have failed. + /// + /// NOTE: This does not start the download itself. Use [`Self::push`] for that. + pub fn finished(&self, hash: &Hash) -> DownloadFuture { + match self.pending_downloads.lock().unwrap().get(hash) { + Some(fut) => fut.clone(), + None => futures::future::ready(None).boxed().shared(), + } + } +} + +type DownloadReply = oneshot::Sender>; +type PendingDownloadsFutures = + FuturesUnordered>)>>; + +#[derive(Debug)] +struct DownloadRequest { + hash: Hash, + peers: Vec, + reply: DownloadReply, +} + +#[derive(Debug)] +struct DownloadActor { + dialer: Dialer, + db: WritableFileDatabase, + conns: HashMap, + replies: HashMap>, + pending_download_futs: PendingDownloadsFutures, + queue: DownloadQueue, + rx: flume::Receiver, +} +impl DownloadActor { + fn new( + endpoint: MagicEndpoint, + db: WritableFileDatabase, + rx: flume::Receiver, + ) -> Self { + Self { + rx, + db, + dialer: Dialer::new(endpoint), + replies: Default::default(), + conns: Default::default(), + pending_download_futs: Default::default(), + queue: Default::default(), + } + } + pub async fn run(&mut self) -> anyhow::Result<()> { + loop { + tokio::select! { + req = self.rx.recv_async() => match req { + Err(_) => return Ok(()), + Ok(req) => self.on_download_request(req).await + }, + (peer, conn) = self.dialer.next() => match conn { + Ok(conn) => { + debug!("connection to {peer} established"); + self.conns.insert(peer, conn); + self.on_peer_ready(peer); + }, + Err(err) => self.on_peer_fail(&peer, err), + }, + Some((peer, hash, res)) = self.pending_download_futs.next() => match res { + Ok(Some((hash, size))) => { + self.queue.on_success(hash, peer); + self.reply(hash, Some((hash, size))); + self.on_peer_ready(peer); + } + Ok(None) => { + self.on_not_found(&peer, hash); + self.on_peer_ready(peer); + } + Err(err) => self.on_peer_fail(&peer, err), + } + } + } + } + + fn reply(&mut self, hash: Hash, res: Option<(Hash, u64)>) { + for reply in self.replies.remove(&hash).into_iter().flatten() { + reply.send(res).ok(); + } + } + + fn on_peer_fail(&mut self, peer: &PeerId, err: anyhow::Error) { + warn!("download from {peer} failed: {err}"); + for hash in self.queue.on_peer_fail(peer) { + self.reply(hash, None); + } + self.conns.remove(peer); + } + + fn on_not_found(&mut self, peer: &PeerId, hash: Hash) { + self.queue.on_not_found(hash, *peer); + if self.queue.has_no_candidates(&hash) { + self.reply(hash, None); + } + } + + fn on_peer_ready(&mut self, peer: PeerId) { + if let Some(hash) = self.queue.try_next_for_peer(peer) { + self.start_download_unchecked(peer, hash); + } else { + self.conns.remove(&peer); + } + } + + fn start_download_unchecked(&mut self, peer: PeerId, hash: Hash) { + let conn = self.conns.get(&peer).unwrap().clone(); + let blobs = self.db.clone(); + let fut = async move { + let start = Instant::now(); + let res = blobs.download_single(conn, hash).await; + // record metrics + let elapsed = start.elapsed().as_millis(); + match &res { + Ok(Some((_hash, len))) => { + inc!(Metrics, downloads_success); + inc_by!(Metrics, download_bytes_total, *len); + inc_by!(Metrics, download_time_total, elapsed as u64); + } + Ok(None) => inc!(Metrics, downloads_notfound), + Err(_) => inc!(Metrics, downloads_error), + } + (peer, hash, res) + }; + self.pending_download_futs.push(fut.boxed_local()); + } + + async fn on_download_request(&mut self, req: DownloadRequest) { + let DownloadRequest { peers, hash, reply } = req; + if self.db.has(&hash) { + let size = self.db.get_size(&hash).await.unwrap(); + reply.send(Some((hash, size))).ok(); + return; + } + self.replies.entry(hash).or_default().push_back(reply); + for peer in peers { + self.queue.push_candidate(hash, peer); + // TODO: Don't dial all peers instantly. + if self.conns.get(&peer).is_none() && !self.dialer.is_pending(&peer) { + self.dialer.queue_dial(peer, &iroh_bytes::protocol::ALPN); + } + } + } +} + +#[derive(Debug, Default)] +struct DownloadQueue { + candidates_by_hash: HashMap>, + candidates_by_peer: HashMap>, + running_by_hash: HashMap, + running_by_peer: HashMap, +} + +impl DownloadQueue { + pub fn push_candidate(&mut self, hash: Hash, peer: PeerId) { + self.candidates_by_hash + .entry(hash) + .or_default() + .push_back(peer); + self.candidates_by_peer + .entry(peer) + .or_default() + .push_back(hash); + } + + pub fn try_next_for_peer(&mut self, peer: PeerId) -> Option { + let mut next = None; + for (idx, hash) in self.candidates_by_peer.get(&peer)?.iter().enumerate() { + if !self.running_by_hash.contains_key(hash) { + next = Some((idx, *hash)); + break; + } + } + if let Some((idx, hash)) = next { + self.running_by_hash.insert(hash, peer); + self.running_by_peer.insert(peer, hash); + self.candidates_by_peer.get_mut(&peer).unwrap().remove(idx); + if let Some(peers) = self.candidates_by_hash.get_mut(&hash) { + peers.retain(|p| p != &peer); + } + self.ensure_no_empty(hash, peer); + return Some(hash); + } else { + None + } + } + + pub fn has_no_candidates(&self, hash: &Hash) -> bool { + self.candidates_by_hash.get(hash).is_none() && self.running_by_hash.get(&hash).is_none() + } + + pub fn on_success(&mut self, hash: Hash, peer: PeerId) -> Option<(PeerId, Hash)> { + let peer2 = self.running_by_hash.remove(&hash); + debug_assert_eq!(peer2, Some(peer)); + self.running_by_peer.remove(&peer); + self.try_next_for_peer(peer).map(|hash| (peer, hash)) + } + + pub fn on_peer_fail(&mut self, peer: &PeerId) -> Vec { + let mut failed = vec![]; + for hash in self + .candidates_by_peer + .remove(peer) + .map(|hashes| hashes.into_iter()) + .into_iter() + .flatten() + { + if let Some(peers) = self.candidates_by_hash.get_mut(&hash) { + peers.retain(|p| p != peer); + if peers.is_empty() && self.running_by_hash.get(&hash).is_none() { + failed.push(hash); + } + } + } + if let Some(hash) = self.running_by_peer.remove(&peer) { + self.running_by_hash.remove(&hash); + if self.candidates_by_hash.get(&hash).is_none() { + failed.push(hash); + } + } + failed + } + + pub fn on_not_found(&mut self, hash: Hash, peer: PeerId) { + let peer2 = self.running_by_hash.remove(&hash); + debug_assert_eq!(peer2, Some(peer)); + self.running_by_peer.remove(&peer); + self.ensure_no_empty(hash, peer); + } + + fn ensure_no_empty(&mut self, hash: Hash, peer: PeerId) { + if self + .candidates_by_peer + .get(&peer) + .map_or(false, |hashes| hashes.is_empty()) + { + self.candidates_by_peer.remove(&peer); + } + if self + .candidates_by_hash + .get(&hash) + .map_or(false, |peers| peers.is_empty()) + { + self.candidates_by_hash.remove(&hash); + } + } +} diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index a0d58246e4..ef50b484cb 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -9,10 +9,10 @@ pub mod baomap; pub mod collection; pub mod dial; pub mod get; +pub mod download; pub mod node; pub mod rpc_protocol; #[allow(missing_docs)] -#[cfg(feature = "sync")] pub mod sync; pub mod util; diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs index 5f15885ad7..7e04323807 100644 --- a/iroh/src/sync/content.rs +++ b/iroh/src/sync/content.rs @@ -1,20 +1,12 @@ use std::{ - collections::{HashMap, VecDeque}, io, path::{Path, PathBuf}, - sync::{Arc, Mutex}, - time::Instant, + sync::Arc, }; use anyhow::Result; use bytes::Bytes; -use futures::{ - future::{BoxFuture, LocalBoxFuture, Shared}, - stream::FuturesUnordered, - FutureExt, -}; use iroh_bytes::util::Hash; -use iroh_gossip::net::util::Dialer; use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; use iroh_metrics::{inc, inc_by}; use iroh_net::{tls::PeerId, MagicEndpoint}; @@ -22,12 +14,13 @@ use iroh_sync::{ store::{self, Store as _}, sync::{Author, InsertOrigin, Namespace, OnInsertCallback, PeerIdBytes, Replica, SignedEntry}, }; -use tokio::{io::AsyncRead, sync::oneshot}; -use tokio_stream::StreamExt; -use tracing::{debug, error, warn}; +use tokio::io::AsyncRead; use super::metrics::Metrics; -use crate::database::flat::{writable::WritableFileDatabase, Database}; +use crate::{ + database::flat::{writable::WritableFileDatabase, Database}, + download::Downloader, +}; #[derive(Debug, Copy, Clone)] pub enum DownloadMode { @@ -253,12 +246,12 @@ impl BlobStore { pub fn start_download(&self, hash: Hash, peers: Vec) { if !self.db.has(&hash) { - self.downloader.start_download(hash, peers); + self.downloader.push(hash, peers); } } pub async fn get_bytes(&self, hash: &Hash) -> anyhow::Result> { - self.downloader.wait_for_download(hash).await; + self.downloader.finished(hash).await; let Some(entry) = self.db().get(hash) else { return Ok(None) }; @@ -267,7 +260,7 @@ impl BlobStore { } pub async fn get_reader(&self, hash: &Hash) -> anyhow::Result> { - self.downloader.wait_for_download(hash).await; + self.downloader.finished(hash).await; let Some(entry) = self.db().get(hash) else { return Ok(None) }; @@ -283,313 +276,3 @@ impl BlobStore { self.db.put_reader(data).await } } - -pub type DownloadReply = oneshot::Sender>; -pub type DownloadFuture = Shared>>; - -#[derive(Debug)] -pub struct DownloadRequest { - hash: Hash, - peers: Vec, - reply: DownloadReply, -} - -/// A download queue -/// -/// Spawns a background task that handles connecting to peers and performing get requests. -/// -/// TODO: Queued downloads are pushed into an unbounded channel. Maybe make it bounded instead. -/// We want the start_download() method to be sync though because it is used -/// from sync on_insert callbacks on the replicas. -/// TODO: Move to iroh-bytes or replace with corresponding feature from iroh-bytes once available -#[derive(Debug, Clone)] -pub struct Downloader { - pending_downloads: Arc>>, - to_actor_tx: flume::Sender, -} - -impl Downloader { - pub fn new( - rt: iroh_bytes::util::runtime::Handle, - endpoint: MagicEndpoint, - blobs: WritableFileDatabase, - ) -> Self { - let (tx, rx) = flume::bounded(64); - // spawn the actor on a local pool - // the local pool is required because WritableFileDatabase::download_single - // returns a future that is !Send - rt.local_pool().spawn_pinned(move || async move { - let mut actor = DownloadActor::new(endpoint, blobs, rx); - if let Err(err) = actor.run().await { - error!("download actor failed with error {err:?}"); - } - }); - Self { - pending_downloads: Arc::new(Mutex::new(HashMap::new())), - to_actor_tx: tx, - } - } - - pub fn wait_for_download(&self, hash: &Hash) -> DownloadFuture { - match self.pending_downloads.lock().unwrap().get(hash) { - Some(fut) => fut.clone(), - None => futures::future::ready(None).boxed().shared(), - } - } - - pub fn start_download(&self, hash: Hash, peers: Vec) { - let (reply, reply_rx) = oneshot::channel(); - let req = DownloadRequest { hash, peers, reply }; - if self.pending_downloads.lock().unwrap().get(&hash).is_none() { - let pending_downloads = self.pending_downloads.clone(); - let fut = async move { - let res = reply_rx.await; - pending_downloads.lock().unwrap().remove(&hash); - res.ok().flatten() - }; - self.pending_downloads - .lock() - .unwrap() - .insert(hash, fut.boxed().shared()); - } - // TODO: this is potentially blocking inside an async call. figure out a better solution - if let Err(err) = self.to_actor_tx.send(req) { - warn!("download actor dropped: {err}"); - } - } -} - -type PendingDownloadsFutures = - FuturesUnordered>)>>; - -#[derive(Debug)] -pub struct DownloadActor { - dialer: Dialer, - db: WritableFileDatabase, - conns: HashMap, - replies: HashMap>, - pending_download_futs: PendingDownloadsFutures, - queue: DownloadQueue, - rx: flume::Receiver, -} -impl DownloadActor { - fn new( - endpoint: MagicEndpoint, - db: WritableFileDatabase, - rx: flume::Receiver, - ) -> Self { - Self { - rx, - db, - dialer: Dialer::new(endpoint), - replies: Default::default(), - conns: Default::default(), - pending_download_futs: Default::default(), - queue: Default::default(), - } - } - pub async fn run(&mut self) -> anyhow::Result<()> { - loop { - tokio::select! { - req = self.rx.recv_async() => match req { - Err(_) => return Ok(()), - Ok(req) => self.on_download_request(req).await - }, - (peer, conn) = self.dialer.next() => match conn { - Ok(conn) => { - debug!("connection to {peer} established"); - self.conns.insert(peer, conn); - self.on_peer_ready(peer); - }, - Err(err) => self.on_peer_fail(&peer, err), - }, - Some((peer, hash, res)) = self.pending_download_futs.next() => match res { - Ok(Some((hash, size))) => { - self.queue.on_success(hash, peer); - self.reply(hash, Some((hash, size))); - self.on_peer_ready(peer); - } - Ok(None) => { - self.on_not_found(&peer, hash); - self.on_peer_ready(peer); - } - Err(err) => self.on_peer_fail(&peer, err), - } - } - } - } - - fn reply(&mut self, hash: Hash, res: Option<(Hash, u64)>) { - for reply in self.replies.remove(&hash).into_iter().flatten() { - reply.send(res).ok(); - } - } - - fn on_peer_fail(&mut self, peer: &PeerId, err: anyhow::Error) { - warn!("download from {peer} failed: {err}"); - for hash in self.queue.on_peer_fail(peer) { - self.reply(hash, None); - } - self.conns.remove(peer); - } - - fn on_not_found(&mut self, peer: &PeerId, hash: Hash) { - self.queue.on_not_found(hash, *peer); - if self.queue.has_no_candidates(&hash) { - self.reply(hash, None); - } - } - - fn on_peer_ready(&mut self, peer: PeerId) { - if let Some(hash) = self.queue.try_next_for_peer(peer) { - self.start_download_unchecked(peer, hash); - } else { - self.conns.remove(&peer); - } - } - - fn start_download_unchecked(&mut self, peer: PeerId, hash: Hash) { - let conn = self.conns.get(&peer).unwrap().clone(); - let blobs = self.db.clone(); - let fut = async move { - let start = Instant::now(); - let res = blobs.download_single(conn, hash).await; - // record metrics - let elapsed = start.elapsed().as_millis(); - match &res { - Ok(Some((_hash, len))) => { - inc!(Metrics, downloads_success); - inc_by!(Metrics, download_bytes_total, *len); - inc_by!(Metrics, download_time_total, elapsed as u64); - } - Ok(None) => inc!(Metrics, downloads_notfound), - Err(_) => inc!(Metrics, downloads_error), - } - (peer, hash, res) - }; - self.pending_download_futs.push(fut.boxed_local()); - } - - async fn on_download_request(&mut self, req: DownloadRequest) { - let DownloadRequest { peers, hash, reply } = req; - if self.db.has(&hash) { - let size = self.db.get_size(&hash).await.unwrap(); - reply.send(Some((hash, size))).ok(); - return; - } - self.replies.entry(hash).or_default().push_back(reply); - for peer in peers { - self.queue.push_candidate(hash, peer); - // TODO: Don't dial all peers instantly. - if self.conns.get(&peer).is_none() && !self.dialer.is_pending(&peer) { - self.dialer.queue_dial(peer, &iroh_bytes::protocol::ALPN); - } - } - } -} - -#[derive(Debug, Default)] -struct DownloadQueue { - candidates_by_hash: HashMap>, - candidates_by_peer: HashMap>, - running_by_hash: HashMap, - running_by_peer: HashMap, -} - -impl DownloadQueue { - pub fn push_candidate(&mut self, hash: Hash, peer: PeerId) { - self.candidates_by_hash - .entry(hash) - .or_default() - .push_back(peer); - self.candidates_by_peer - .entry(peer) - .or_default() - .push_back(hash); - } - - pub fn try_next_for_peer(&mut self, peer: PeerId) -> Option { - let mut next = None; - for (idx, hash) in self.candidates_by_peer.get(&peer)?.iter().enumerate() { - if !self.running_by_hash.contains_key(hash) { - next = Some((idx, *hash)); - break; - } - } - if let Some((idx, hash)) = next { - self.running_by_hash.insert(hash, peer); - self.running_by_peer.insert(peer, hash); - self.candidates_by_peer.get_mut(&peer).unwrap().remove(idx); - if let Some(peers) = self.candidates_by_hash.get_mut(&hash) { - peers.retain(|p| p != &peer); - } - self.ensure_no_empty(hash, peer); - return Some(hash); - } else { - None - } - } - - pub fn has_no_candidates(&self, hash: &Hash) -> bool { - self.candidates_by_hash.get(hash).is_none() && self.running_by_hash.get(&hash).is_none() - } - - pub fn on_success(&mut self, hash: Hash, peer: PeerId) -> Option<(PeerId, Hash)> { - let peer2 = self.running_by_hash.remove(&hash); - debug_assert_eq!(peer2, Some(peer)); - self.running_by_peer.remove(&peer); - self.try_next_for_peer(peer).map(|hash| (peer, hash)) - } - - pub fn on_peer_fail(&mut self, peer: &PeerId) -> Vec { - let mut failed = vec![]; - for hash in self - .candidates_by_peer - .remove(peer) - .map(|hashes| hashes.into_iter()) - .into_iter() - .flatten() - { - if let Some(peers) = self.candidates_by_hash.get_mut(&hash) { - peers.retain(|p| p != peer); - if peers.is_empty() && self.running_by_hash.get(&hash).is_none() { - failed.push(hash); - } - } - } - if let Some(hash) = self.running_by_peer.remove(&peer) { - self.running_by_hash.remove(&hash); - if self.candidates_by_hash.get(&hash).is_none() { - failed.push(hash); - } - } - failed - } - - pub fn on_not_found(&mut self, hash: Hash, peer: PeerId) { - let peer2 = self.running_by_hash.remove(&hash); - debug_assert_eq!(peer2, Some(peer)); - self.running_by_peer.remove(&peer); - self.ensure_no_empty(hash, peer); - } - - fn ensure_no_empty(&mut self, hash: Hash, peer: PeerId) { - if self - .candidates_by_peer - .get(&peer) - .map_or(false, |hashes| hashes.is_empty()) - { - self.candidates_by_peer.remove(&peer); - } - if self - .candidates_by_hash - .get(&hash) - .map_or(false, |peers| peers.is_empty()) - { - self.candidates_by_hash.remove(&hash); - } - } -} - -#[cfg(test)] -mod test {} From 722df3479febf59895c04b76f7c645ffd3352fad Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 15:27:15 +0200 Subject: [PATCH 040/172] fix: rebase --- iroh-sync/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index b569aea487..3cc7767df9 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -924,7 +924,7 @@ mod tests { rounds += 1; println!("round {}", rounds); if let Some(msg) = bob.sync_process_message(msg, None).map_err(Into::into)? { - next_to_bob = alice.sync_process_message(msg, None).map_err(Into::into); + next_to_bob = alice.sync_process_message(msg, None).map_err(Into::into)?; } } From f0d9ef299331ed506ecfcdaab98474c7542e2331 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 16:13:44 +0200 Subject: [PATCH 041/172] fix imports --- iroh/src/download.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/download.rs b/iroh/src/download.rs index c43640f38f..08679aac1b 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -20,7 +20,7 @@ use tokio_stream::StreamExt; use tracing::{debug, error, warn}; // TODO: Move metrics to iroh-bytes metrics -use super::sync::metrics::Metrics; +use crate::sync::metrics::Metrics; // TODO: Will be replaced by proper persistent DB once // https://github.com/n0-computer/iroh/pull/1320 is merged use crate::database::flat::writable::WritableFileDatabase; From 111b117553a7763e5b057ad348e4b0966a37c3ec Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 16:16:26 +0200 Subject: [PATCH 042/172] chore: fmt --- iroh/src/sync.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index f56e4b5459..b710b70da5 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -4,7 +4,7 @@ use std::net::SocketAddr; use anyhow::{bail, ensure, Context, Result}; use bytes::BytesMut; -use iroh_net::{tls::PeerId, MagicEndpoint, magic_endpoint::get_peer_id}; +use iroh_net::{magic_endpoint::get_peer_id, tls::PeerId, MagicEndpoint}; use iroh_sync::{ store, sync::{NamespaceId, Replica}, @@ -146,7 +146,9 @@ pub async fn run_bob { debug!("starting sync for {}", namespace); - if let Some(msg) = r.sync_process_message(message, peer).map_err(Into::into)? { + if let Some(msg) = + r.sync_process_message(message, peer).map_err(Into::into)? + { send_sync_message(writer, msg).await?; } else { break; @@ -161,7 +163,10 @@ pub async fn run_bob match replica { Some(ref replica) => { - if let Some(msg) = replica.sync_process_message(msg, peer).map_err(Into::into)? { + if let Some(msg) = replica + .sync_process_message(msg, peer) + .map_err(Into::into)? + { send_sync_message(writer, msg).await?; } else { break; @@ -237,8 +242,13 @@ mod tests { let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); let replica = alice_replica.clone(); let alice_task = tokio::task::spawn(async move { - run_alice::(&mut alice_writer, &mut alice_reader, &replica, None) - .await + run_alice::( + &mut alice_writer, + &mut alice_reader, + &replica, + None, + ) + .await }); let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); @@ -248,7 +258,7 @@ mod tests { &mut bob_writer, &mut bob_reader, bob_replica_store_task, - None + None, ) .await }); From 59077b1e421f5589b33dc9e971bb83a662fb594f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 16:18:41 +0200 Subject: [PATCH 043/172] fix: metrics --- iroh/src/download.rs | 2 +- iroh/src/metrics.rs | 10 ++++++++++ iroh/src/sync/metrics.rs | 10 ---------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 08679aac1b..aaa12630be 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -20,7 +20,7 @@ use tokio_stream::StreamExt; use tracing::{debug, error, warn}; // TODO: Move metrics to iroh-bytes metrics -use crate::sync::metrics::Metrics; +use crate::metrics::Metrics; // TODO: Will be replaced by proper persistent DB once // https://github.com/n0-computer/iroh/pull/1320 is merged use crate::database::flat::writable::WritableFileDatabase; diff --git a/iroh/src/metrics.rs b/iroh/src/metrics.rs index 3b3b7f46ef..74355f2a09 100644 --- a/iroh/src/metrics.rs +++ b/iroh/src/metrics.rs @@ -10,6 +10,11 @@ pub struct Metrics { pub requests_total: Counter, pub bytes_sent: Counter, pub bytes_received: Counter, + pub download_bytes_total: Counter, + pub download_time_total: Counter, + pub downloads_success: Counter, + pub downloads_error: Counter, + pub downloads_notfound: Counter, } impl Default for Metrics { @@ -18,6 +23,11 @@ impl Default for Metrics { requests_total: Counter::new("Total number of requests received"), bytes_sent: Counter::new("Number of bytes streamed"), bytes_received: Counter::new("Number of bytes received"), + download_bytes_total: Counter::new("Total number of content bytes downloaded"), + download_time_total: Counter::new("Total time in ms spent downloading content bytes"), + downloads_success: Counter::new("Total number of successfull downloads"), + downloads_error: Counter::new("Total number of downloads failed with error"), + downloads_notfound: Counter::new("Total number of downloads failed with not found"), } } } diff --git a/iroh/src/sync/metrics.rs b/iroh/src/sync/metrics.rs index 257f2afa07..37185e6cec 100644 --- a/iroh/src/sync/metrics.rs +++ b/iroh/src/sync/metrics.rs @@ -11,11 +11,6 @@ pub struct Metrics { pub new_entries_remote: Counter, pub new_entries_local_size: Counter, pub new_entries_remote_size: Counter, - pub download_bytes_total: Counter, - pub download_time_total: Counter, - pub downloads_success: Counter, - pub downloads_error: Counter, - pub downloads_notfound: Counter, pub initial_sync_success: Counter, pub initial_sync_failed: Counter, } @@ -27,11 +22,6 @@ impl Default for Metrics { new_entries_remote: Counter::new("Number of document entries added by peers"), new_entries_local_size: Counter::new("Total size of entry contents added locally"), new_entries_remote_size: Counter::new("Total size of entry contents added by peers"), - download_bytes_total: Counter::new("Total number of content bytes downloaded"), - download_time_total: Counter::new("Total time in ms spent downloading content bytes"), - downloads_success: Counter::new("Total number of successfull downloads"), - downloads_error: Counter::new("Total number of downloads failed with error"), - downloads_notfound: Counter::new("Total number of downloads failed with not found"), initial_sync_success: Counter::new("Number of successfull initial syncs "), initial_sync_failed: Counter::new("Number of failed initial syncs"), } From 437c031d390885cce65c2a5a206cddf04aa819dc Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 16:53:46 +0200 Subject: [PATCH 044/172] fix: feature flags --- iroh/src/download.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/iroh/src/download.rs b/iroh/src/download.rs index aaa12630be..681a65b168 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -19,7 +19,7 @@ use tokio::sync::oneshot; use tokio_stream::StreamExt; use tracing::{debug, error, warn}; -// TODO: Move metrics to iroh-bytes metrics +#[cfg(feature = "metrics")] use crate::metrics::Metrics; // TODO: Will be replaced by proper persistent DB once // https://github.com/n0-computer/iroh/pull/1320 is merged @@ -207,9 +207,12 @@ impl DownloadActor { let conn = self.conns.get(&peer).unwrap().clone(); let blobs = self.db.clone(); let fut = async move { +#[cfg(feature = "metrics")] let start = Instant::now(); let res = blobs.download_single(conn, hash).await; // record metrics +#[cfg(feature = "metrics")] + { let elapsed = start.elapsed().as_millis(); match &res { Ok(Some((_hash, len))) => { @@ -220,6 +223,7 @@ impl DownloadActor { Ok(None) => inc!(Metrics, downloads_notfound), Err(_) => inc!(Metrics, downloads_error), } + } (peer, hash, res) }; self.pending_download_futs.push(fut.boxed_local()); From 481b7710ef34bcd2eb6bf367429f15332761d56c Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 7 Aug 2023 16:56:24 +0200 Subject: [PATCH 045/172] chore: fmt --- iroh/src/download.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 681a65b168..60be7b3f92 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -207,22 +207,22 @@ impl DownloadActor { let conn = self.conns.get(&peer).unwrap().clone(); let blobs = self.db.clone(); let fut = async move { -#[cfg(feature = "metrics")] + #[cfg(feature = "metrics")] let start = Instant::now(); let res = blobs.download_single(conn, hash).await; // record metrics -#[cfg(feature = "metrics")] + #[cfg(feature = "metrics")] { - let elapsed = start.elapsed().as_millis(); - match &res { - Ok(Some((_hash, len))) => { - inc!(Metrics, downloads_success); - inc_by!(Metrics, download_bytes_total, *len); - inc_by!(Metrics, download_time_total, elapsed as u64); + let elapsed = start.elapsed().as_millis(); + match &res { + Ok(Some((_hash, len))) => { + inc!(Metrics, downloads_success); + inc_by!(Metrics, download_bytes_total, *len); + inc_by!(Metrics, download_time_total, elapsed as u64); + } + Ok(None) => inc!(Metrics, downloads_notfound), + Err(_) => inc!(Metrics, downloads_error), } - Ok(None) => inc!(Metrics, downloads_notfound), - Err(_) => inc!(Metrics, downloads_error), - } } (peer, hash, res) }; @@ -283,14 +283,14 @@ impl DownloadQueue { peers.retain(|p| p != &peer); } self.ensure_no_empty(hash, peer); - return Some(hash); + Some(hash) } else { None } } pub fn has_no_candidates(&self, hash: &Hash) -> bool { - self.candidates_by_hash.get(hash).is_none() && self.running_by_hash.get(&hash).is_none() + self.candidates_by_hash.get(hash).is_none() && self.running_by_hash.get(hash).is_none() } pub fn on_success(&mut self, hash: Hash, peer: PeerId) -> Option<(PeerId, Hash)> { @@ -316,7 +316,7 @@ impl DownloadQueue { } } } - if let Some(hash) = self.running_by_peer.remove(&peer) { + if let Some(hash) = self.running_by_peer.remove(peer) { self.running_by_hash.remove(&hash); if self.candidates_by_hash.get(&hash).is_none() { failed.push(hash); From 8da512fcceebbb4baeee25d57e07fd0b586724fb Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 10:21:37 +0200 Subject: [PATCH 046/172] rework abstractions for integration in iroh node * removed the `Doc`, `DocStore`, `BlobStore` abstractions, they were not the right way for now * Introduce `SyncEngine` that combines the gossip `LiveSync` with on_insert callbacks to download content, to replace the functionality from the above-mentioned abstractions * Rename `iroh_sync::get_replica` to `iroh_sync::open_replica` and expect them to be singletons per namespace * Various other minor fixes and improvements to logging --- Cargo.lock | 17 ++ iroh-gossip/src/net.rs | 46 +++-- iroh-sync/src/store.rs | 100 ++++++++++- iroh-sync/src/store/fs.rs | 45 ++++- iroh-sync/src/store/memory.rs | 10 +- iroh-sync/src/sync.rs | 67 ++++++- iroh/Cargo.toml | 7 +- iroh/examples/sync.rs | 132 +++++++++----- iroh/src/commands/sync.rs | 227 +++++++++++++++++++++++ iroh/src/database/flat/writable.rs | 32 ++-- iroh/src/download.rs | 2 +- iroh/src/sync.rs | 16 +- iroh/src/sync/content.rs | 278 ----------------------------- iroh/src/sync/engine.rs | 113 ++++++++++++ iroh/src/sync/live.rs | 219 +++++++++++++++++------ iroh/tests/sync.rs | 215 ++++++++++++++++++++++ 16 files changed, 1090 insertions(+), 436 deletions(-) create mode 100644 iroh/src/commands/sync.rs delete mode 100644 iroh/src/sync/content.rs create mode 100644 iroh/src/sync/engine.rs create mode 100644 iroh/tests/sync.rs diff --git a/Cargo.lock b/Cargo.lock index 991697a349..300e865321 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1021,6 +1021,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + [[package]] name = "elliptic-curve" version = "0.13.5" @@ -1738,10 +1744,12 @@ dependencies = [ "iroh-metrics", "iroh-net", "iroh-sync", + "itertools", "multibase", "nix", "num_cpus", "once_cell", + "parking_lot", "portable-atomic", "postcard", "proptest", @@ -1980,6 +1988,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" diff --git a/iroh-gossip/src/net.rs b/iroh-gossip/src/net.rs index 2500710a1d..458bc0bc2f 100644 --- a/iroh-gossip/src/net.rs +++ b/iroh-gossip/src/net.rs @@ -17,7 +17,7 @@ use tokio::{ sync::{broadcast, mpsc, oneshot, watch}, task::JoinHandle, }; -use tracing::{debug, warn}; +use tracing::{debug, trace, warn}; use self::util::{read_message, write_message, Dialer, Timers}; use crate::proto::{self, TopicId}; @@ -25,7 +25,7 @@ use crate::proto::{self, TopicId}; pub mod util; /// ALPN protocol name -pub const GOSSIP_ALPN: &[u8] = b"n0/iroh-gossip/0"; +pub const GOSSIP_ALPN: &[u8] = b"/iroh-gossip/0"; /// Maximum message size is limited to 1024 bytes. pub const MAX_MESSAGE_SIZE: usize = 1024; @@ -261,6 +261,20 @@ struct IrohInfo { derp_region: Option, } +impl IrohInfo { + async fn from_endpoint(endpoint: &MagicEndpoint) -> anyhow::Result { + Ok(Self { + addrs: endpoint + .local_endpoints() + .await? + .iter() + .map(|ep| ep.addr) + .collect(), + derp_region: endpoint.my_derp().await, + }) + } +} + /// Whether a connection is initiated by us (Dial) or by the remote peer (Accept) #[derive(Debug)] enum ConnOrigin { @@ -352,11 +366,7 @@ impl Actor { } }, _ = self.on_endpoints_rx.changed() => { - let endpoints = self.on_endpoints_rx.borrow().clone(); - let info = IrohInfo { - addrs: endpoints.iter().map(|ep| ep.addr).collect(), - derp_region: self.endpoint.my_derp().await - }; + let info = IrohInfo::from_endpoint(&self.endpoint).await?; let peer_data = postcard::to_stdvec(&info)?; self.handle_in_event(InEvent::UpdatePeerData(peer_data.into()), Instant::now()).await?; } @@ -465,13 +475,21 @@ impl Actor { async fn handle_in_event(&mut self, event: InEvent, now: Instant) -> anyhow::Result<()> { let me = *self.state.me(); - debug!(me = ?me, "handle in_event {event:?}"); + if matches!(event, InEvent::TimerExpired(_)) { + trace!(me = ?me, "handle in_event {event:?}"); + } else { + debug!(me = ?me, "handle in_event {event:?}"); + }; if let InEvent::PeerDisconnected(peer) = &event { self.conn_send_tx.remove(peer); } let out = self.state.handle(event, now); for event in out { - debug!(me = ?me, "handle out_event {event:?}"); + if matches!(event, OutEvent::ScheduleTimer(_, _)) { + trace!(me = ?me, "handle out_event {event:?}"); + } else { + debug!(me = ?me, "handle out_event {event:?}"); + }; match event { OutEvent::SendMessage(peer_id, message) => { if let Some(send) = self.conn_send_tx.get(&peer_id) { @@ -514,10 +532,14 @@ impl Actor { OutEvent::PeerData(peer, data) => match postcard::from_bytes::(&data) { Err(err) => warn!("Failed to decode PeerData from {peer}: {err}"), Ok(info) => { - debug!("add known addrs for {peer}: {info:?}..."); - self.endpoint + debug!(me = ?self.endpoint.peer_id(), peer = ?peer, "add known addrs: {info:?}"); + if let Err(err) = self + .endpoint .add_known_addrs(peer, info.derp_region, &info.addrs) - .await?; + .await + { + debug!(me = ?self.endpoint.peer_id(), peer = ?peer, "add known failed: {err:?}"); + } } }, } diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index a28b9a904a..465f341593 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -1,5 +1,6 @@ -use anyhow::Result; +use anyhow::{bail, Result}; use rand_core::CryptoRngCore; +use serde::{Deserialize, Serialize}; use crate::{ ranger, @@ -22,9 +23,21 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { where Self: 'a; - fn get_replica(&self, namespace: &NamespaceId) -> Result>>; + /// Open a replica + /// + /// Store implementers must ensure that only a single instance of [`Replica`] is created per + /// namespace. On subsequent calls, a clone of that singleton instance must be returned. + /// + /// TODO: Add close_replica + fn open_replica(&self, namespace: &NamespaceId) -> Result>>; + + // TODO: return iterator + fn list_replicas(&self) -> Result>; fn get_author(&self, author: &AuthorId) -> Result>; fn new_author(&self, rng: &mut R) -> Result; + + // TODO: return iterator + fn list_authors(&self) -> Result>; fn new_replica(&self, namespace: Namespace) -> Result>; /// Gets all entries matching this key and author. @@ -76,4 +89,87 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// Returns all versions of all documents. fn get_all(&self, namespace: NamespaceId) -> Result>; + + /// Returns an iterator over the entries in a namespace. + fn get(&self, namespace: NamespaceId, filter: GetFilter) -> Result> { + GetIter::new(self, namespace, filter) + } +} + +/// Filter a get query onto a namespace +#[derive(Debug, Serialize, Deserialize)] +pub struct GetFilter { + pub latest: bool, + pub author: Option, + pub key: KeyFilter, +} + +/// Filter the keys in a namespace +#[derive(Debug, Serialize, Deserialize)] +pub enum KeyFilter { + /// No filter, list all entries + All, + /// Filter for entries starting with a prefix + Prefix(Vec), + /// Filter for exact key match + Key(Vec), +} + +/// Iterator over the entries in a namespace +pub enum GetIter<'s, S: Store> { + All(S::GetAllIter<'s>), + Latest(S::GetLatestIter<'s>), + Single(std::option::IntoIter>), +} + +impl<'s, S: Store> Iterator for GetIter<'s, S> { + type Item = anyhow::Result; + + fn next(&mut self) -> Option { + match self { + GetIter::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIter::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIter::Single(iter) => iter.next(), + } + } +} + +impl<'s, S: Store> GetIter<'s, S> { + fn new(store: &'s S, namespace: NamespaceId, filter: GetFilter) -> anyhow::Result { + use KeyFilter::*; + Ok(match filter.latest { + false => match (filter.key, filter.author) { + (All, None) => Self::All(store.get_all(namespace)?), + (Prefix(prefix), None) => Self::All(store.get_all_by_prefix(namespace, &prefix)?), + (Key(key), None) => Self::All(store.get_all_by_key(namespace, key)?), + (Key(key), Some(author)) => { + Self::All(store.get_all_by_key_and_author(namespace, author, key)?) + } + (All, Some(_)) | (Prefix(_), Some(_)) => { + bail!("This filter combination is not yet supported") + } + }, + true => match (filter.key, filter.author) { + (All, None) => Self::Latest(store.get_latest(namespace)?), + (Prefix(prefix), None) => { + Self::Latest(store.get_latest_by_prefix(namespace, &prefix)?) + } + (Key(key), None) => Self::Latest(store.get_latest_by_key(namespace, key)?), + (Key(key), Some(author)) => Self::Single( + store + .get_latest_by_key_and_author(namespace, author, key)? + .map(|entry| Ok(entry)) + .into_iter(), + ), + (All, Some(_)) | (Prefix(_), Some(_)) => { + bail!("This filter combination is not yet supported") + } + }, + }) + } + + /// Returns true if this iterator is known to return only a single result. + pub fn single(&self) -> bool { + matches!(self, Self::Single(_)) + } } diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs index e490c177a2..42f85a0bd7 100644 --- a/iroh-sync/src/store/fs.rs +++ b/iroh-sync/src/store/fs.rs @@ -1,9 +1,10 @@ //! On disk storage for replicas. -use std::{path::Path, sync::Arc}; +use std::{collections::HashMap, path::Path, sync::Arc}; use anyhow::Result; use ouroboros::self_referencing; +use parking_lot::RwLock; use rand_core::CryptoRngCore; use redb::{ AccessGuard, Database, MultimapRange, MultimapTableDefinition, MultimapValue, @@ -25,6 +26,7 @@ use self::ouroboros_impl_range_all_iterator::BorrowedMutFields; #[derive(Debug, Clone)] pub struct Store { db: Arc, + replicas: Arc>>>, } // Table Definitions @@ -68,7 +70,10 @@ impl Store { } write_tx.commit()?; - Ok(Store { db: Arc::new(db) }) + Ok(Store { + db: Arc::new(db), + replicas: Default::default(), + }) } /// Stores a new namespace fn insert_namespace(&self, namespace: Namespace) -> Result<()> { @@ -99,7 +104,11 @@ impl super::Store for Store { type GetAllIter<'a> = RangeAllIterator<'a>; type GetLatestIter<'a> = RangeLatestIterator<'a>; - fn get_replica(&self, namespace_id: &NamespaceId) -> Result>> { + fn open_replica(&self, namespace_id: &NamespaceId) -> Result>> { + if let Some(replica) = self.replicas.read().get(namespace_id) { + return Ok(Some(replica.clone())); + } + let read_tx = self.db.begin_read()?; let namespace_table = read_tx.open_table(NAMESPACES_TABLE)?; let Some(namespace) = namespace_table.get(namespace_id.as_bytes())? else { @@ -107,9 +116,21 @@ impl super::Store for Store { }; let namespace = Namespace::from_bytes(namespace.value()); let replica = Replica::new(namespace, StoreInstance::new(*namespace_id, self.clone())); + self.replicas.write().insert(*namespace_id, replica.clone()); Ok(Some(replica)) } + // TODO: return iterator + fn list_replicas(&self) -> Result> { + let read_tx = self.db.begin_read()?; + let namespace_table = read_tx.open_table(NAMESPACES_TABLE)?; + let namespaces = namespace_table + .iter()? + .filter_map(|entry| entry.ok()) + .map(|(_key, value)| Namespace::from_bytes(value.value()).id()); + Ok(namespaces.collect()) + } + fn get_author(&self, author_id: &AuthorId) -> Result> { let read_tx = self.db.begin_read()?; let author_table = read_tx.open_table(AUTHORS_TABLE)?; @@ -128,12 +149,28 @@ impl super::Store for Store { Ok(author) } + /// Generates a new author, using the passed in randomness. + fn list_authors(&self) -> Result> { + let read_tx = self.db.begin_read()?; + let author_table = read_tx.open_table(AUTHORS_TABLE)?; + + let mut authors = vec![]; + let iter = author_table.iter()?; + for entry in iter { + let (_key, value) = entry?; + let author = Author::from_bytes(value.value()); + authors.push(author); + } + Ok(authors) + } + fn new_replica(&self, namespace: Namespace) -> Result> { let id = namespace.id(); self.insert_namespace(namespace.clone())?; let replica = Replica::new(namespace, StoreInstance::new(id, self.clone())); + self.replicas.write().insert(id, replica.clone()); Ok(replica) } @@ -669,7 +706,7 @@ mod tests { let namespace = Namespace::new(&mut rand::thread_rng()); let replica = store.new_replica(namespace.clone())?; - let replica_back = store.get_replica(&namespace.id())?.unwrap(); + let replica_back = store.open_replica(&namespace.id())?.unwrap(); assert_eq!( replica.namespace().as_bytes(), replica_back.namespace().as_bytes() diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs index b10213f9ce..9dd39ec667 100644 --- a/iroh-sync/src/store/memory.rs +++ b/iroh-sync/src/store/memory.rs @@ -32,11 +32,15 @@ impl super::Store for Store { type GetLatestIter<'a> = GetLatestIter<'a>; type GetAllIter<'a> = GetAllIter<'a>; - fn get_replica(&self, namespace: &NamespaceId) -> Result>> { + fn open_replica(&self, namespace: &NamespaceId) -> Result>> { let replicas = &*self.replicas.read(); Ok(replicas.get(namespace).cloned()) } + fn list_replicas(&self) -> Result> { + Ok(self.replicas.read().keys().cloned().collect()) + } + fn get_author(&self, author: &AuthorId) -> Result> { let authors = &*self.authors.read(); Ok(authors.get(author).cloned()) @@ -48,6 +52,10 @@ impl super::Store for Store { Ok(author) } + fn list_authors(&self) -> Result> { + Ok(self.authors.read().values().cloned().collect()) + } + fn new_replica(&self, namespace: Namespace) -> Result> { let id = namespace.id(); let replica = Replica::new(namespace, ReplicaStoreInstance::new(id, self.clone())); diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 3cc7767df9..84cbd513c6 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -6,9 +6,10 @@ use std::{ cmp::Ordering, + collections::HashMap, fmt::{Debug, Display}, str::FromStr, - sync::Arc, + sync::{atomic::AtomicU64, Arc}, time::SystemTime, }; @@ -130,6 +131,30 @@ impl FromStr for Author { } } +impl FromStr for AuthorId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let pub_key: [u8; 32] = hex::decode(s)? + .try_into() + .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; + let pub_key = VerifyingKey::from_bytes(&pub_key)?; + Ok(AuthorId(pub_key)) + } +} + +impl FromStr for NamespaceId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let pub_key: [u8; 32] = hex::decode(s)? + .try_into() + .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; + let pub_key = VerifyingKey::from_bytes(&pub_key)?; + Ok(NamespaceId(pub_key)) + } +} + impl From for Author { fn from(priv_key: SigningKey) -> Self { Self { priv_key } @@ -181,7 +206,7 @@ pub struct NamespaceId(VerifyingKey); impl Display for NamespaceId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "NamespaceId({})", hex::encode(self.0.as_bytes())) + write!(f, "{}", hex::encode(self.0.as_bytes())) } } @@ -214,6 +239,9 @@ pub type OnInsertCallback = Box> { inner: Arc>>, #[debug("on_insert: [Box; {}]", "self.on_insert.len()")] - on_insert: Arc>>, + on_insert: Arc>>, + on_insert_removal_id: Arc, } #[derive(derive_more::Debug)] @@ -248,12 +277,21 @@ impl> Replica { peer: Peer::from_store(store), })), on_insert: Default::default(), + on_insert_removal_id: Arc::new(AtomicU64::new(0)), } } - pub fn on_insert(&self, callback: OnInsertCallback) { + pub fn on_insert(&self, callback: OnInsertCallback) -> RemovalToken { let mut on_insert = self.on_insert.write(); - on_insert.push(callback); + let removal_id = self + .on_insert_removal_id + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + on_insert.insert(removal_id, callback); + RemovalToken(removal_id) + } + + pub fn remove_on_insert(&self, token: RemovalToken) -> bool { + self.on_insert.write().remove(&token.0).is_some() } /// Inserts a new record at the given key. @@ -275,7 +313,7 @@ impl> Replica { inner.peer.put(id, signed_entry.clone())?; drop(inner); let on_insert = self.on_insert.read(); - for cb in &*on_insert { + for cb in on_insert.values() { cb(InsertOrigin::Local, signed_entry.clone()); } Ok(()) @@ -313,7 +351,7 @@ impl> Replica { inner.peer.put(id, entry.clone()).map_err(Into::into)?; drop(inner); let on_insert = self.on_insert.read(); - for cb in &*on_insert { + for cb in on_insert.values() { cb(InsertOrigin::Sync(received_from), entry.clone()); } Ok(()) @@ -336,7 +374,7 @@ impl> Replica { .peer .process_message(message, |_key, entry| { let on_insert = self.on_insert.read(); - for cb in &*on_insert { + for cb in on_insert.values() { cb(InsertOrigin::Sync(from_peer), entry.clone()); } })?; @@ -347,6 +385,10 @@ impl> Replica { pub fn namespace(&self) -> NamespaceId { self.inner.read().namespace.id() } + + pub fn secret_key(&self) -> [u8; 32] { + self.inner.read().namespace.to_bytes() + } } /// A signed entry. @@ -382,6 +424,15 @@ impl SignedEntry { pub fn content_hash(&self) -> &Hash { self.entry().record().content_hash() } + pub fn content_len(&self) -> u64 { + self.entry().record().content_len() + } + pub fn author(&self) -> AuthorId { + self.entry().id().author() + } + pub fn key(&self) -> &[u8] { + self.entry().id().key() + } } /// Signature over an entry. diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index f750758471..0973844c75 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -19,6 +19,7 @@ bytes = "1" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } flume = "0.10.14" futures = "0.3.25" +genawaiter = { version = "0.99", default-features = false, features = ["futures03"] } hex = { version = "0.4.3" } iroh-bytes = { version = "0.5.0", path = "../iroh-bytes" } iroh-io = { version = "0.2.2" } @@ -28,6 +29,8 @@ num_cpus = { version = "1.15.0" } portable-atomic = "1" iroh-sync = { version = "0.5.1", path = "../iroh-sync" } iroh-gossip = { version = "0.5.1", path = "../iroh-gossip" } +once_cell = "1.18.0" +parking_lot = "0.12.1" postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quic-rpc = { version = "0.6", default-features = false, features = ["flume-transport"] } quinn = "0.10" @@ -54,11 +57,11 @@ data-encoding = "2.4.0" url = { version = "2.4", features = ["serde"] } # Examples -once_cell = { version = "1.18.0", optional = true } ed25519-dalek = { version = "=2.0.0-rc.3", features = ["serde", "rand_core"], optional = true } shell-words = { version = "1.1.0", optional = true } shellexpand = { version = "3.1.0", optional = true } rustyline = { version = "12.0.0", optional = true } +itertools = "0.11.0" [features] default = ["cli", "metrics", "sync"] @@ -69,7 +72,7 @@ mem-db = [] flat-db = [] iroh-collection = [] test = [] -example-sync = ["cli", "ed25519-dalek", "once_cell", "shell-words", "shellexpand", "sync", "rustyline"] +example-sync = ["cli", "ed25519-dalek", "shell-words", "shellexpand", "sync", "rustyline"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index d27076a584..ac5392ab16 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -13,17 +13,21 @@ use std::{ }; use anyhow::{anyhow, bail}; +use bytes::Bytes; use clap::{CommandFactory, FromArgMatches, Parser}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; -use iroh::sync::{ - BlobStore, Doc as SyncDoc, DocStore, DownloadMode, LiveSync, PeerSource, SYNC_ALPN, +use iroh::{ + database::flat::writable::WritableFileDatabase, + download::Downloader, + sync::{PeerSource, SyncEngine, SYNC_ALPN}, }; use iroh_bytes::util::runtime; use iroh_gossip::{ net::{Gossip, GOSSIP_ALPN}, proto::TopicId, }; +use iroh_io::AsyncSliceReaderExt; use iroh_metrics::{ core::{Counter, Metric}, struct_iterable::Iterable, @@ -34,7 +38,7 @@ use iroh_net::{ }; use iroh_sync::{ store::{self, Store as _}, - sync::{Author, Namespace, SignedEntry}, + sync::{Author, Namespace, Replica, SignedEntry}, }; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; @@ -51,7 +55,7 @@ use iroh_bytes_handlers::IrohBytesHandlers; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; -type Doc = SyncDoc; +type Doc = Replica<::Instance>; #[derive(Parser, Debug)] struct Args { @@ -219,24 +223,29 @@ async fn run(args: Args) -> anyhow::Result<()> { // create a runtime that can spawn tasks on a local-thread executors (to support !Send futures) let rt = iroh_bytes::util::runtime::Handle::from_currrent(num_cpus::get())?; - // create a blob store (with a iroh-bytes database inside) - let blobs = BlobStore::new(rt.clone(), storage_path.join("blobs"), endpoint.clone()).await?; - // create a doc store for the iroh-sync docs let author = Author::from(keypair.secret().clone()); - let docs_path = storage_path.join("docs"); - tokio::fs::create_dir_all(&docs_path).await?; - let docs = DocStore::new(blobs.clone(), author, docs_path)?; + let docs_path = storage_path.join("docs.db"); + let docs = iroh_sync::store::fs::Store::new(&docs_path)?; // create the live syncer - let live_sync = LiveSync::::spawn(endpoint.clone(), gossip.clone()); + let db = WritableFileDatabase::new(storage_path.join("blobs")).await?; + let downloader = Downloader::new(rt.clone(), endpoint.clone(), db.clone()); + let live_sync = SyncEngine::spawn( + rt.clone(), + endpoint.clone(), + gossip.clone(), + docs.clone(), + db.clone(), + downloader, + ); // construct the state that is passed to the endpoint loop and from there cloned // into to the connection handler task for incoming connections. let state = Arc::new(State { gossip: gossip.clone(), docs: docs.clone(), - bytes: IrohBytesHandlers::new(rt.clone(), blobs.db().clone()), + bytes: IrohBytesHandlers::new(rt.clone(), db.db().clone()), }); // spawn our endpoint loop that forwards incoming connections @@ -245,8 +254,11 @@ async fn run(args: Args) -> anyhow::Result<()> { // open our document and add to the live syncer let namespace = Namespace::from_bytes(topic.as_bytes()); println!("> opening doc {}", fmt_hash(namespace.id().as_bytes())); - let doc: Doc = docs.create_or_open(namespace, DownloadMode::Always).await?; - live_sync.add(doc.replica().clone(), peers.clone()).await?; + let doc = match docs.open_replica(&namespace.id()) { + Ok(Some(doc)) => doc, + Err(_) | Ok(None) => docs.new_replica(namespace)?, + }; + live_sync.start_sync(doc.namespace(), peers.clone()).await?; // spawn an repl thread that reads stdin and parses each line as a `Cmd` command let (cmd_tx, mut cmd_rx) = mpsc::channel(1); @@ -285,7 +297,7 @@ async fn run(args: Args) -> anyhow::Result<()> { _ = tokio::signal::ctrl_c() => { println!("> aborted"); } - res = handle_command(cmd, &rt, docs.store(), &doc, &our_ticket, &log_filter, ¤t_watch) => if let Err(err) = res { + res = handle_command(cmd, &rt, &docs, &author, &doc, &db, &our_ticket, &log_filter, ¤t_watch) => if let Err(err) = res { println!("> error: {err}"); }, }; @@ -294,11 +306,11 @@ async fn run(args: Args) -> anyhow::Result<()> { } // exit: cancel the sync and store blob database and document - if let Err(err) = live_sync.cancel().await { + if let Err(err) = live_sync.shutdown().await { println!("> syncer closed with error: {err:?}"); } println!("> persisting document and blob database at {storage_path:?}"); - blobs.save().await?; + db.save().await?; if let Some(metrics_fut) = metrics_fut { metrics_fut.abort(); @@ -312,14 +324,17 @@ async fn handle_command( cmd: Cmd, rt: &runtime::Handle, store: &store::fs::Store, + author: &Author, doc: &Doc, + db: &WritableFileDatabase, ticket: &Ticket, log_filter: &LogLevelReload, current_watch: &Arc>>, ) -> anyhow::Result<()> { match cmd { Cmd::Set { key, value } => { - doc.insert_bytes(&key, value.into_bytes().into()).await?; + let (hash, len) = db.put_bytes(value.into_bytes().into()).await?; + doc.insert(key, author, hash, len)?; } Cmd::Get { key, @@ -327,15 +342,15 @@ async fn handle_command( prefix, } => { let entries = if prefix { - store.get_all_by_prefix(doc.replica().namespace(), key.as_bytes())? + store.get_all_by_prefix(doc.namespace(), key.as_bytes())? } else { - store.get_all_by_key(doc.replica().namespace(), key.as_bytes())? + store.get_all_by_key(doc.namespace(), key.as_bytes())? }; for entry in entries { let (_id, entry) = entry?; println!("{}", fmt_entry(&entry)); if print_content { - println!("{}", fmt_content(doc, &entry).await); + println!("{}", fmt_content(db, &entry).await); } } } @@ -353,10 +368,8 @@ async fn handle_command( }, Cmd::Ls { prefix } => { let entries = match prefix { - None => store.get_all(doc.replica().namespace())?, - Some(prefix) => { - store.get_all_by_prefix(doc.replica().namespace(), prefix.as_bytes())? - } + None => store.get_all(doc.namespace())?, + Some(prefix) => store.get_all_by_prefix(doc.namespace(), prefix.as_bytes())?, }; let mut count = 0; for entry in entries { @@ -374,7 +387,7 @@ async fn handle_command( log_filter.modify(|layer| *layer = next_filter)?; } Cmd::Stats => get_stats(), - Cmd::Fs(cmd) => handle_fs_command(cmd, store, doc).await?, + Cmd::Fs(cmd) => handle_fs_command(cmd, store, db, doc, author).await?, Cmd::Hammer { prefix, threads, @@ -398,11 +411,14 @@ async fn handle_command( let prefix = prefix.clone(); let doc = doc.clone(); let bytes = bytes.clone(); + let db = db.clone(); + let author = author.clone(); let handle = rt.main().spawn(async move { for i in 0..count { let value = String::from_utf8(bytes.clone()).unwrap(); let key = format!("{}/{}/{}", prefix, t, i); - doc.insert_bytes(key, value.into_bytes().into()).await?; + let (hash, len) = db.put_bytes(value.into_bytes().into()).await?; + doc.insert(key, &author, hash, len)?; } Ok(count) }); @@ -418,8 +434,8 @@ async fn handle_command( let mut read = 0; for i in 0..count { let key = format!("{}/{}/{}", prefix, t, i); - let entries = store - .get_all_by_key(doc.replica().namespace(), key.as_bytes())?; + let entries = + store.get_all_by_key(doc.namespace(), key.as_bytes())?; for entry in entries { let (_id, entry) = entry?; let _content = fmt_content_simple(&doc, &entry); @@ -450,11 +466,19 @@ async fn handle_command( Ok(()) } -async fn handle_fs_command(cmd: FsCmd, store: &store::fs::Store, doc: &Doc) -> anyhow::Result<()> { +async fn handle_fs_command( + cmd: FsCmd, + store: &store::fs::Store, + db: &WritableFileDatabase, + doc: &Doc, + author: &Author, +) -> anyhow::Result<()> { match cmd { FsCmd::ImportFile { file_path, key } => { let file_path = canonicalize_path(&file_path)?.canonicalize()?; - let (hash, len) = doc.insert_from_file(&key, &file_path).await?; + let reader = tokio::fs::File::open(&file_path).await?; + let (hash, len) = db.put_reader(reader).await?; + doc.insert(key, author, hash, len)?; println!( "> imported {file_path:?}: {} ({})", fmt_hash(hash), @@ -480,7 +504,9 @@ async fn handle_fs_command(cmd: FsCmd, store: &store::fs::Store, doc: &Doc) -> a continue; } let key = format!("{key_prefix}/{relative}"); - let (hash, len) = doc.insert_from_file(key, file.path()).await?; + let reader = tokio::fs::File::open(file.path()).await?; + let (hash, len) = db.put_reader(reader).await?; + doc.insert(key, author, hash, len)?; println!( "> imported {relative}: {} ({})", fmt_hash(hash), @@ -498,15 +524,16 @@ async fn handle_fs_command(cmd: FsCmd, store: &store::fs::Store, doc: &Doc) -> a } let root = canonicalize_path(&dir_path)?; println!("> exporting {key_prefix} to {root:?}"); - let entries = - store.get_latest_by_prefix(doc.replica().namespace(), key_prefix.as_bytes())?; + let entries = store.get_latest_by_prefix(doc.namespace(), key_prefix.as_bytes())?; let mut checked_dirs = HashSet::new(); for entry in entries { let (id, entry) = entry?; let key = id.key(); let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; let len = entry.entry().record().content_len(); - if let Some(mut reader) = doc.get_content_reader(&entry).await { + let blob = db.db().get(entry.content_hash()); + if let Some(blob) = blob { + let mut reader = blob.data_reader().await?; let path = root.join(&relative); let parent = path.parent().unwrap(); if !checked_dirs.contains(parent) { @@ -526,19 +553,18 @@ async fn handle_fs_command(cmd: FsCmd, store: &store::fs::Store, doc: &Doc) -> a FsCmd::ExportFile { key, file_path } => { let path = canonicalize_path(&file_path)?; // TODO: Fix - let entry = store - .get_latest_by_key(doc.replica().namespace(), &key)? - .next(); + let entry = store.get_latest_by_key(doc.namespace(), &key)?.next(); if let Some(entry) = entry { let (_, entry) = entry?; println!("> exporting {key} to {path:?}"); let parent = path.parent().ok_or_else(|| anyhow!("Invalid path"))?; tokio::fs::create_dir_all(&parent).await?; let mut file = tokio::fs::File::create(&path).await?; - let mut reader = doc - .get_content_reader(&entry) - .await + let blob = db + .db() + .get(entry.content_hash()) .ok_or_else(|| anyhow!(format!("content for {key} is not available")))?; + let mut reader = blob.data_reader().await?; copy(&mut reader, &mut file).await?; } else { println!("> key not found, abort"); @@ -683,7 +709,7 @@ impl FromStr for Cmd { #[derive(Debug)] struct State { gossip: Gossip, - docs: DocStore, + docs: iroh_sync::store::fs::Store, bytes: IrohBytesHandlers, } @@ -704,7 +730,7 @@ async fn handle_connection(mut conn: quinn::Connecting, state: Arc) -> an println!("> incoming connection with alpn {alpn}"); match alpn.as_bytes() { GOSSIP_ALPN => state.gossip.handle_connection(conn.await?).await, - SYNC_ALPN => state.docs.handle_connection(conn).await, + SYNC_ALPN => iroh::sync::handle_connection(conn, state.docs.clone()).await, alpn if alpn == iroh_bytes::protocol::ALPN => state.bytes.handle_connection(conn).await, _ => bail!("ignoring connection: unsupported ALPN protocol"), } @@ -841,20 +867,32 @@ async fn fmt_content_simple(_doc: &Doc, entry: &SignedEntry) -> String { format!("<{}>", HumanBytes(len)) } -async fn fmt_content(doc: &Doc, entry: &SignedEntry) -> String { +async fn fmt_content(db: &WritableFileDatabase, entry: &SignedEntry) -> String { let len = entry.entry().record().content_len(); if len > MAX_DISPLAY_CONTENT_LEN { format!("<{}>", HumanBytes(len)) } else { - match doc.get_content_bytes(entry).await { - None => "".to_string(), - Some(content) => match String::from_utf8(content.into()) { + match read_content(db, entry).await { + Err(err) => format!(""), + Ok(content) => match String::from_utf8(content.into()) { Ok(str) => str, Err(_err) => format!("", HumanBytes(len)), }, } } } + +async fn read_content(db: &WritableFileDatabase, entry: &SignedEntry) -> anyhow::Result { + let data = db + .db() + .get(entry.content_hash()) + .ok_or_else(|| anyhow!("not found"))? + .data_reader() + .await? + .read_to_end() + .await?; + Ok(data) +} fn fmt_hash(hash: impl AsRef<[u8]>) -> String { let mut text = data_encoding::BASE32_NOPAD.encode(hash.as_ref()); text.make_ascii_lowercase(); diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs new file mode 100644 index 0000000000..d3a704b197 --- /dev/null +++ b/iroh/src/commands/sync.rs @@ -0,0 +1,227 @@ +use anyhow::anyhow; +use clap::Parser; +use futures::TryStreamExt; +use indicatif::HumanBytes; +use iroh::{ + rpc_protocol::{ProviderRequest, ProviderResponse, ShareMode}, + sync::PeerSource, +}; +use iroh_sync::{ + store::{GetFilter, KeyFilter}, + sync::{AuthorId, NamespaceId, SignedEntry}, +}; +use quic_rpc::transport::quinn::QuinnConnection; + +use super::RpcClient; + +// TODO: It is a bit unfortunate that we have to drag the generics all through. Maybe box the conn? +pub type Iroh = iroh::client::Iroh>; + +#[derive(Debug, Clone, Parser)] +pub enum Commands { + Author { + #[clap(subcommand)] + command: Author, + }, + Docs { + #[clap(subcommand)] + command: Docs, + }, + Doc { + id: NamespaceId, + #[clap(subcommand)] + command: Doc, + }, +} + +impl Commands { + pub async fn run(self, client: RpcClient) -> anyhow::Result<()> { + let iroh = Iroh::new(client); + match self { + Commands::Author { command } => command.run(iroh).await, + Commands::Docs { command } => command.run(iroh).await, + Commands::Doc { command, id } => command.run(iroh, id).await, + } + } +} + +#[derive(Debug, Clone, Parser)] +pub enum Author { + List, + Create, +} + +impl Author { + pub async fn run(self, iroh: Iroh) -> anyhow::Result<()> { + match self { + Author::List => { + let mut stream = iroh.list_authors().await?; + while let Some(author_id) = stream.try_next().await? { + println!("{}", author_id); + } + } + Author::Create => { + let author_id = iroh.create_author().await?; + println!("{}", author_id); + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Parser)] +pub enum Docs { + List, + Create, + Import { + key: String, + #[clap(short, long)] + peers: Vec, + }, +} + +impl Docs { + pub async fn run(self, iroh: Iroh) -> anyhow::Result<()> { + match self { + Docs::Create => { + let doc = iroh.create_doc().await?; + println!("created {}", doc.id()); + } + Docs::Import { key, peers } => { + let key = hex::decode(key)? + .try_into() + .map_err(|_| anyhow!("invalid length"))?; + let doc = iroh.import_doc(key, peers).await?; + println!("imported {}", doc.id()); + } + Docs::List => { + let mut stream = iroh.list_docs().await?; + while let Some(id) = stream.try_next().await? { + println!("{}", id) + } + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Parser)] +pub enum Doc { + StartSync { + peers: Vec, + }, + Share { + mode: ShareMode, + }, + /// Set an entry + Set { + /// Author of this entry. + author: AuthorId, + /// Key to the entry (parsed as UTF-8 string). + key: String, + /// Content to store for this entry (parsed as UTF-8 string) + value: String, + }, + /// Get entries by key + /// + /// Shows the author, content hash and content length for all entries for this key. + Get { + /// Key to the entry (parsed as UTF-8 string). + key: String, + /// If true, get all entries that start with KEY. + #[clap(short, long)] + prefix: bool, + /// Filter by author. + #[clap(short, long)] + author: Option, + /// If true, old entries will be included. By default only the latest value for each key is + /// shown. + #[clap(short, long)] + old: bool, + // TODO: get content? + }, + List { + /// If true, old entries will be included. By default only the latest value for each key is + /// shown. + #[clap(short, long)] + old: bool, + /// Optional key prefix (parsed as UTF-8 string) + prefix: Option, + }, +} + +impl Doc { + pub async fn run(self, iroh: Iroh, doc_id: NamespaceId) -> anyhow::Result<()> { + let doc = iroh.get_doc(doc_id)?; + match self { + Doc::StartSync { peers } => { + doc.start_sync(peers).await?; + println!("ok"); + } + Doc::Share { mode } => { + let res = doc.share(mode).await?; + println!("key: {}", hex::encode(res.key)); + println!("me: {}", res.peer); + } + Doc::Set { author, key, value } => { + let key = key.as_bytes().to_vec(); + let value = value.as_bytes().to_vec(); + let entry = doc.set_bytes(author, key, value).await?; + println!("{}", fmt_entry(&entry)); + } + Doc::Get { + key, + prefix, + author, + old, + } => { + let key = key.as_bytes().to_vec(); + let key = match prefix { + true => KeyFilter::Prefix(key), + false => KeyFilter::Key(key), + }; + let filter = GetFilter { + latest: !old, + author, + key, + }; + let mut stream = doc.get(filter).await?; + while let Some(entry) = stream.try_next().await? { + println!("{}", fmt_entry(&entry)); + } + } + Doc::List { old, prefix } => { + let key = match prefix { + Some(prefix) => KeyFilter::Prefix(prefix.as_bytes().to_vec()), + None => KeyFilter::All, + }; + let filter = GetFilter { + latest: !old, + author: None, + key, + }; + let mut stream = doc.get(filter).await?; + while let Some(entry) = stream.try_next().await? { + println!("{}", fmt_entry(&entry)); + } + } + } + Ok(()) + } +} + +fn fmt_entry(entry: &SignedEntry) -> String { + let id = entry.entry().id(); + let key = std::str::from_utf8(id.key()).unwrap_or(""); + let author = fmt_hash(id.author().as_bytes()); + let hash = entry.entry().record().content_hash(); + let hash = fmt_hash(hash.as_bytes()); + let len = HumanBytes(entry.entry().record().content_len()); + format!("@{author}: {key} = {hash} ({len})",) +} + +fn fmt_hash(hash: impl AsRef<[u8]>) -> String { + let mut text = data_encoding::BASE32_NOPAD.encode(&hash.as_ref()[..5]); + text.make_ascii_lowercase(); + format!("{}…", &text) +} diff --git a/iroh/src/database/flat/writable.rs b/iroh/src/database/flat/writable.rs index 6973d6e4fe..9427158b5f 100644 --- a/iroh/src/database/flat/writable.rs +++ b/iroh/src/database/flat/writable.rs @@ -6,7 +6,6 @@ use std::{ collections::HashMap, - io, path::{Path, PathBuf}, sync::Arc, }; @@ -57,8 +56,9 @@ impl WritableFileDatabase { &self.db } - pub async fn save(&self) -> io::Result<()> { - self.db.save(&self.storage.db_path).await + pub async fn save(&self) -> anyhow::Result<()> { + self.db.save(&self.storage.db_path).await?; + Ok(()) } pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64)> { @@ -73,12 +73,6 @@ impl WritableFileDatabase { Ok((hash, size)) } - pub async fn put_from_temp_file(&self, temp_path: &PathBuf) -> anyhow::Result<(Hash, u64)> { - let (hash, size, entry) = self.storage.move_to_blobs(temp_path).await?; - self.db.union_with(HashMap::from_iter([(hash, entry)])); - Ok((hash, size)) - } - pub async fn get_size(&self, hash: &Hash) -> Option { Some(self.db.get(hash)?.size().await) } @@ -111,17 +105,25 @@ impl WritableFileDatabase { .create(true) .open(path) }) - .await?; - - let (curr, _size) = header.next().await?; - let _curr = curr.write_all(&mut data_file).await?; + .await + .context("failed to create local tempfile")?; + + let (curr, _size) = header.next().await.context("failed to read blob content")?; + let _curr = curr + .write_all(&mut data_file) + .await + .context("failed to write blob content to tempfile")?; // Flush the data file first, it is the only thing that matters at this point - data_file.sync().await?; + data_file.sync().await.context("fsync failed")?; temp_path }; // 2. Insert into database - let (hash, size, entry) = self.storage.move_to_blobs(&temp_path).await?; + let (hash, size, entry) = self + .storage + .move_to_blobs(&temp_path) + .await + .context("failed to move to blobs dir")?; let entries = HashMap::from_iter([(hash, entry)]); self.db.union_with(entries); Ok(Some((hash, size))) diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 60be7b3f92..89a0db3180 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -181,7 +181,7 @@ impl DownloadActor { } fn on_peer_fail(&mut self, peer: &PeerId, err: anyhow::Error) { - warn!("download from {peer} failed: {err}"); + warn!("download from {peer} failed: {err:?}"); for hash in self.queue.on_peer_fail(peer) { self.reply(hash, None); } diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index b710b70da5..c3d13e8ceb 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -11,16 +11,16 @@ use iroh_sync::{ }; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::debug; +use tracing::{debug, trace}; /// The ALPN identifier for the iroh-sync protocol pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; -mod content; +mod engine; mod live; pub mod metrics; -pub use content::*; +pub use engine::*; pub use live::*; /// Sync Protocol @@ -81,7 +81,7 @@ pub async fn run_alice { @@ -106,9 +106,9 @@ pub async fn handle_connection( replica_store: S, ) -> Result<()> { let connection = connecting.await?; - debug!("> connection established!"); let peer_id = get_peer_id(&connection).await?; let (mut send_stream, mut recv_stream) = connection.accept_bi().await?; + debug!(peer = ?peer_id, "incoming sync: start"); run_bob( &mut send_stream, @@ -119,7 +119,7 @@ pub async fn handle_connection( .await?; send_stream.finish().await?; - debug!("done"); + debug!(peer = ?peer_id, "incoming sync: done"); Ok(()) } @@ -136,14 +136,14 @@ pub async fn run_bob { ensure!(replica.is_none(), "double init message"); - match replica_store.get_replica(&namespace)? { + match replica_store.open_replica(&namespace)? { Some(r) => { debug!("starting sync for {}", namespace); if let Some(msg) = diff --git a/iroh/src/sync/content.rs b/iroh/src/sync/content.rs deleted file mode 100644 index 7e04323807..0000000000 --- a/iroh/src/sync/content.rs +++ /dev/null @@ -1,278 +0,0 @@ -use std::{ - io, - path::{Path, PathBuf}, - sync::Arc, -}; - -use anyhow::Result; -use bytes::Bytes; -use iroh_bytes::util::Hash; -use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt}; -use iroh_metrics::{inc, inc_by}; -use iroh_net::{tls::PeerId, MagicEndpoint}; -use iroh_sync::{ - store::{self, Store as _}, - sync::{Author, InsertOrigin, Namespace, OnInsertCallback, PeerIdBytes, Replica, SignedEntry}, -}; -use tokio::io::AsyncRead; - -use super::metrics::Metrics; -use crate::{ - database::flat::{writable::WritableFileDatabase, Database}, - download::Downloader, -}; - -#[derive(Debug, Copy, Clone)] -pub enum DownloadMode { - Always, - Manual, -} - -#[derive(Debug, Clone)] -pub struct DocStore { - replicas: store::fs::Store, - blobs: BlobStore, - local_author: Arc, -} - -const REPLICA_DB_NAME: &str = "replica.db"; - -impl DocStore { - pub fn new(blobs: BlobStore, author: Author, storage_path: PathBuf) -> Result { - let replicas = store::fs::Store::new(storage_path.join(REPLICA_DB_NAME))?; - - Ok(Self { - replicas, - local_author: Arc::new(author), - blobs, - }) - } - - pub async fn create_or_open( - &self, - namespace: Namespace, - download_mode: DownloadMode, - ) -> Result> { - let replica = match self.replicas.get_replica(&namespace.id())? { - Some(replica) => replica, - None => self.replicas.new_replica(namespace)?, - }; - - let doc = Doc::new( - replica, - self.blobs.clone(), - self.local_author.clone(), - download_mode, - ); - Ok(doc) - } - - pub async fn handle_connection(&self, conn: quinn::Connecting) -> anyhow::Result<()> { - crate::sync::handle_connection(conn, self.replicas.clone()).await - } - - pub fn store(&self) -> &store::fs::Store { - &self.replicas - } -} - -/// A replica with a [`BlobStore`] for contents. -/// -/// This will also download missing content from peers. -/// -/// TODO: Currently content is only downloaded from the author of a entry. -/// We want to try other peers if the author is offline (or always). -/// We'll need some heuristics which peers to try. -#[derive(Clone, Debug)] -pub struct Doc { - replica: Replica, - blobs: BlobStore, - local_author: Arc, -} - -impl Doc { - pub fn new( - replica: Replica, - blobs: BlobStore, - local_author: Arc, - download_mode: DownloadMode, - ) -> Self { - let doc = Self { - replica, - blobs, - local_author, - }; - - // If download mode is set to always download: - // setup on_insert callback to trigger download on remote insert - if let DownloadMode::Always = download_mode { - let doc_clone = doc.clone(); - doc.replica - .on_insert(Box::new(move |origin, entry| match origin { - InsertOrigin::Sync(peer) => { - doc_clone.download_content_from_author_and_other_peer(&entry, peer); - } - InsertOrigin::Local => {} - })); - } - - // Collect metrics - doc.replica.on_insert(Box::new(move |origin, entry| { - let size = entry.entry().record().content_len(); - match origin { - InsertOrigin::Local => { - inc!(Metrics, new_entries_local); - inc_by!(Metrics, new_entries_local_size, size); - } - InsertOrigin::Sync(_) => { - inc!(Metrics, new_entries_remote); - inc_by!(Metrics, new_entries_remote_size, size); - } - } - })); - - doc - } - - pub fn on_insert(&self, callback: OnInsertCallback) { - self.replica.on_insert(callback); - } - - pub fn replica(&self) -> &Replica { - &self.replica - } - - pub fn local_author(&self) -> &Author { - &self.local_author - } - - pub async fn insert_bytes( - &self, - key: impl AsRef<[u8]>, - content: Bytes, - ) -> anyhow::Result<(Hash, u64)> { - let (hash, len) = self.blobs.put_bytes(content).await?; - self.replica - .insert(key, &self.local_author, hash, len) - .map_err(Into::into)?; - Ok((hash, len)) - } - - pub async fn insert_reader( - &self, - key: impl AsRef<[u8]>, - content: impl AsyncRead + Unpin, - ) -> anyhow::Result<(Hash, u64)> { - let (hash, len) = self.blobs.put_reader(content).await?; - self.replica - .insert(key, &self.local_author, hash, len) - .map_err(Into::into)?; - Ok((hash, len)) - } - - pub async fn insert_from_file( - &self, - key: impl AsRef<[u8]>, - file_path: impl AsRef, - ) -> anyhow::Result<(Hash, u64)> { - let reader = tokio::fs::File::open(&file_path).await?; - self.insert_reader(&key, reader).await - } - - pub fn download_content_from_author_and_other_peer( - &self, - entry: &SignedEntry, - other_peer: Option, - ) { - let author_peer_id = PeerId::from_bytes(entry.entry().id().author().as_bytes()) - .expect("failed to convert author to peer id"); - - let mut peers = vec![author_peer_id]; - - if let Some(other_peer) = other_peer { - let other_peer_id = - PeerId::from_bytes(&other_peer).expect("failed to convert author to peer id"); - if other_peer_id != peers[0] { - peers.push(other_peer_id); - } - } - - let hash = *entry.entry().record().content_hash(); - self.blobs.start_download(hash, peers); - } - - pub async fn get_content_bytes(&self, entry: &SignedEntry) -> Option { - let hash = entry.entry().record().content_hash(); - self.blobs.get_bytes(hash).await.ok().flatten() - } - - pub async fn get_content_reader(&self, entry: &SignedEntry) -> Option { - let hash = entry.entry().record().content_hash(); - self.blobs.get_reader(hash).await.ok().flatten() - } -} - -/// A blob database that can download missing blobs from peers. -/// -/// Blobs can be inserted either from bytes or by downloading from peers. -/// Downloads can be started and will be tracked in the blobstore. -/// New blobs will be saved as files with a filename based on their hash. -/// -/// TODO: This is similar to what is used in the iroh provider. -/// Unify once we know how the APIs should look like. -#[derive(Debug, Clone)] -pub struct BlobStore { - db: WritableFileDatabase, - downloader: Downloader, -} -impl BlobStore { - pub async fn new( - rt: iroh_bytes::util::runtime::Handle, - data_path: PathBuf, - endpoint: MagicEndpoint, - ) -> anyhow::Result { - let db = WritableFileDatabase::new(data_path).await?; - let downloader = Downloader::new(rt, endpoint, db.clone()); - Ok(Self { db, downloader }) - } - - pub async fn save(&self) -> io::Result<()> { - self.db.save().await - } - - pub fn db(&self) -> &Database { - self.db.db() - } - - pub fn start_download(&self, hash: Hash, peers: Vec) { - if !self.db.has(&hash) { - self.downloader.push(hash, peers); - } - } - - pub async fn get_bytes(&self, hash: &Hash) -> anyhow::Result> { - self.downloader.finished(hash).await; - let Some(entry) = self.db().get(hash) else { - return Ok(None) - }; - let bytes = entry.data_reader().await?.read_to_end().await?; - Ok(Some(bytes)) - } - - pub async fn get_reader(&self, hash: &Hash) -> anyhow::Result> { - self.downloader.finished(hash).await; - let Some(entry) = self.db().get(hash) else { - return Ok(None) - }; - let reader = entry.data_reader().await?; - Ok(Some(reader)) - } - - pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64)> { - self.db.put_bytes(data).await - } - - pub async fn put_reader(&self, data: impl AsyncRead + Unpin) -> anyhow::Result<(Hash, u64)> { - self.db.put_reader(data).await - } -} diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs new file mode 100644 index 0000000000..021a5d8d13 --- /dev/null +++ b/iroh/src/sync/engine.rs @@ -0,0 +1,113 @@ +use std::{collections::HashMap, sync::Arc}; + +use anyhow::anyhow; +use iroh_bytes::util::runtime::Handle; +use iroh_gossip::net::Gossip; +use iroh_net::{tls::PeerId, MagicEndpoint}; +use iroh_sync::{ + store::Store, + sync::{Author, AuthorId, InsertOrigin, NamespaceId, OnInsertCallback, RemovalToken, Replica}, +}; +use parking_lot::RwLock; + +use crate::{database::flat::writable::WritableFileDatabase, download::Downloader}; + +use super::{LiveSync, PeerSource}; + +/// The SyncEngine combines the [`LiveSync`] actor with the Iroh bytes database and [`Downloader`]. +/// +/// TODO: Replace the [`WritableFileDatabase`] with the real thing once +/// https://github.com/n0-computer/iroh/pull/1320 is merged +#[derive(Debug, Clone)] +pub struct SyncEngine { + pub(crate) rt: Handle, + pub(crate) store: S, + pub(crate) db: WritableFileDatabase, + pub(crate) endpoint: MagicEndpoint, + downloader: Downloader, + live: LiveSync, + active: Arc>>, +} + +impl SyncEngine { + pub fn spawn( + rt: Handle, + endpoint: MagicEndpoint, + gossip: Gossip, + store: S, + db: WritableFileDatabase, + downloader: Downloader, + ) -> Self { + let live = LiveSync::spawn(rt.clone(), endpoint.clone(), gossip); + Self { + live, + downloader, + store, + db, + rt, + endpoint, + active: Default::default(), + } + } + + pub async fn start_sync( + &self, + namespace: NamespaceId, + peers: Vec, + ) -> anyhow::Result<()> { + let replica = self.get_replica(&namespace)?; + if !self.active.read().contains_key(&namespace) { + // add download listener + let removal_token = replica.on_insert(on_insert_download(self.downloader.clone())); + self.active + .write() + .insert(replica.namespace(), removal_token); + // start to gossip updates + self.live.start_sync(replica.clone(), peers).await?; + } else if !peers.is_empty() { + self.live.join_peers(namespace, peers).await?; + } + Ok(()) + } + + pub async fn stop_sync(&self, namespace: NamespaceId) -> anyhow::Result<()> { + let replica = self.get_replica(&namespace)?; + if let Some(token) = self.active.write().remove(&replica.namespace()) { + replica.remove_on_insert(token); + self.live.stop_sync(namespace).await?; + } + Ok(()) + } + + pub async fn shutdown(&self) -> anyhow::Result<()> { + for (namespace, token) in self.active.write().drain() { + if let Ok(Some(replica)) = self.store.open_replica(&namespace) { + replica.remove_on_insert(token); + } + } + self.live.shutdown().await?; + Ok(()) + } + + pub fn get_replica(&self, id: &NamespaceId) -> anyhow::Result> { + self.store + .open_replica(id)? + .ok_or_else(|| anyhow!("doc not found")) + } + + pub fn get_author(&self, id: &AuthorId) -> anyhow::Result { + self.store + .get_author(id)? + .ok_or_else(|| anyhow!("author not found")) + } +} + +fn on_insert_download(downloader: Downloader) -> OnInsertCallback { + Box::new(move |origin, entry| { + if let InsertOrigin::Sync(Some(peer_id)) = origin { + let peer_id = PeerId::from_bytes(&peer_id).unwrap(); + let hash = *entry.entry().record().content_hash(); + downloader.push(hash, vec![peer_id]); + } + }) +} diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 320c236385..98283ab840 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use std::{collections::HashMap, fmt, net::SocketAddr, str::FromStr, sync::Arc}; use crate::sync::connect_and_sync; use anyhow::{anyhow, Result}; @@ -7,6 +7,7 @@ use futures::{ stream::{BoxStream, FuturesUnordered, StreamExt}, FutureExt, TryFutureExt, }; +use iroh_bytes::util::runtime::Handle; use iroh_gossip::{ net::{Event, Gossip}, proto::TopicId, @@ -15,11 +16,11 @@ use iroh_metrics::inc; use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::{ store, - sync::{InsertOrigin, Replica, SignedEntry}, + sync::{InsertOrigin, NamespaceId, RemovalToken, Replica, SignedEntry}, }; use serde::{Deserialize, Serialize}; use tokio::{sync::mpsc, task::JoinError}; -use tracing::{debug, error}; +use tracing::{debug, error, info, warn}; use super::metrics::Metrics; @@ -35,6 +36,53 @@ pub struct PeerSource { pub derp_region: Option, } +// /// A SyncId is a 32 byte array which is both a [`NamespaceId`] and a [`TopicId`]. +// pub struct SyncId([u8; 32]); + +impl PeerSource { + /// Deserializes from bytes. + fn from_bytes(bytes: &[u8]) -> anyhow::Result { + postcard::from_bytes(bytes).map_err(Into::into) + } + /// Serializes to bytes. + pub fn to_bytes(&self) -> Vec { + postcard::to_stdvec(self).expect("postcard::to_stdvec is infallible") + } + pub async fn from_endpoint(endpoint: &MagicEndpoint) -> anyhow::Result { + Ok(Self { + peer_id: endpoint.peer_id(), + derp_region: endpoint.my_derp().await, + addrs: endpoint + .local_endpoints() + .await? + .into_iter() + .map(|ep| ep.addr) + .collect(), + }) + } +} + +/// Serializes to base32. +impl fmt::Display for PeerSource { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let encoded = self.to_bytes(); + let mut text = data_encoding::BASE32_NOPAD.encode(&encoded); + text.make_ascii_lowercase(); + write!(f, "{text}") + } +} + +/// Deserializes from base32. +impl FromStr for PeerSource { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let bytes = data_encoding::BASE32_NOPAD.decode(s.to_ascii_uppercase().as_bytes())?; + let slf = Self::from_bytes(&bytes)?; + Ok(slf) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Op { Put(SignedEntry), @@ -49,9 +97,16 @@ enum SyncState { #[derive(Debug)] pub enum ToActor { - SyncDoc { - doc: Replica, - initial_peers: Vec, + StartSync { + replica: Replica, + peers: Vec, + }, + JoinPeers { + namespace: NamespaceId, + peers: Vec, + }, + StopSync { + namespace: NamespaceId, }, Shutdown, } @@ -64,10 +119,10 @@ pub struct LiveSync { } impl LiveSync { - pub fn spawn(endpoint: MagicEndpoint, gossip: Gossip) -> Self { + pub fn spawn(rt: Handle, endpoint: MagicEndpoint, gossip: Gossip) -> Self { let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); let mut actor = Actor::new(endpoint, gossip, to_actor_rx); - let task = tokio::spawn(async move { + let task = rt.main().spawn(async move { if let Err(err) = actor.run().await { error!("live sync failed: {err:?}"); } @@ -80,19 +135,33 @@ impl LiveSync { } /// Cancel the live sync. - pub async fn cancel(&self) -> Result<()> { + pub async fn shutdown(&self) -> Result<()> { self.to_actor_tx.send(ToActor::::Shutdown).await?; self.task.clone().await?; Ok(()) } - pub async fn add( + pub async fn start_sync( &self, - doc: Replica, - initial_peers: Vec, + replica: Replica, + peers: Vec, ) -> Result<()> { self.to_actor_tx - .send(ToActor::::SyncDoc { doc, initial_peers }) + .send(ToActor::::StartSync { replica, peers }) + .await?; + Ok(()) + } + + pub async fn join_peers(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { + self.to_actor_tx + .send(ToActor::::JoinPeers { namespace, peers }) + .await?; + Ok(()) + } + + pub async fn stop_sync(&self, namespace: NamespaceId) -> Result<()> { + self.to_actor_tx + .send(ToActor::::StopSync { namespace }) .await?; Ok(()) } @@ -104,7 +173,7 @@ struct Actor { endpoint: MagicEndpoint, gossip: Gossip, - docs: HashMap>, + replicas: HashMap, RemovalToken)>, subscription: BoxStream<'static, Result<(TopicId, Event)>>, sync_state: HashMap<(TopicId, PeerId), SyncState>, @@ -122,19 +191,19 @@ impl Actor { gossip: Gossip, to_actor_rx: mpsc::Receiver>, ) -> Self { - let (insert_tx, insert_rx) = flume::bounded(64); + let (insert_entry_tx, insert_entry_rx) = flume::bounded(64); let sub = gossip.clone().subscribe_all().boxed(); Self { gossip, endpoint, - insert_entry_rx: insert_rx, - insert_entry_tx: insert_tx, + insert_entry_rx, + insert_entry_tx, to_actor_rx, sync_state: Default::default(), pending_syncs: Default::default(), pending_joins: Default::default(), - docs: Default::default(), + replicas: Default::default(), subscription: sub, } } @@ -148,10 +217,12 @@ impl Actor { // received shutdown signal, or livesync handle was dropped: // break loop and exit Some(ToActor::Shutdown) | None => { - self.on_shutdown().await?; + self.shutdown().await?; break; } - Some(ToActor::SyncDoc { doc, initial_peers }) => self.insert_doc(doc, initial_peers).await?, + Some(ToActor::StartSync { replica, peers }) => self.start_sync(replica, peers).await?, + Some(ToActor::StopSync { namespace }) => self.stop_sync(&namespace).await?, + Some(ToActor::JoinPeers { namespace, peers }) => self.join_gossip_and_start_initial_sync(&namespace, peers).await?, } } // new gossip message @@ -173,6 +244,8 @@ impl Actor { Some((topic, res)) = self.pending_joins.next() => { if let Err(err) = res { error!("failed to join {topic:?}: {err:?}"); + } else { + info!("joined sync topic {topic:?}"); } // TODO: maintain some join state } @@ -182,7 +255,7 @@ impl Actor { } fn sync_with_peer(&mut self, topic: TopicId, peer: PeerId) { - let Some(doc) = self.docs.get(&topic) else { + let Some((replica, _token)) = self.replicas.get(&topic) else { return; }; // Check if we synced and only start sync if not yet synced @@ -192,16 +265,15 @@ impl Actor { if let Some(_state) = self.sync_state.get(&(topic, peer)) { return; }; - // TODO: fixme (doc_id, peer) self.sync_state.insert((topic, peer), SyncState::Running); let task = { let endpoint = self.endpoint.clone(); - let doc = doc.clone(); + let replica = replica.clone(); async move { - debug!("sync with {peer}"); + debug!("init sync with {peer}"); // TODO: Make sure that the peer is dialable. - let res = connect_and_sync::(&endpoint, &doc, peer, None, &[]).await; - debug!("> synced with {peer}: {res:?}"); + let res = connect_and_sync::(&endpoint, &replica, peer, None, &[]).await; + debug!("synced with {peer}: {res:?}"); // collect metrics match &res { Ok(_) => inc!(Metrics, initial_sync_success), @@ -214,30 +286,43 @@ impl Actor { self.pending_syncs.push(task); } - async fn on_shutdown(&mut self) -> anyhow::Result<()> { - for (topic, _doc) in self.docs.drain() { - // TODO: Remove the on_insert callbacks + async fn shutdown(&mut self) -> anyhow::Result<()> { + for (topic, (replica, removal_token)) in self.replicas.drain() { + replica.remove_on_insert(removal_token); self.gossip.quit(topic).await?; } Ok(()) } - async fn insert_doc( + async fn stop_sync(&mut self, namespace: &NamespaceId) -> anyhow::Result<()> { + let topic = TopicId::from_bytes(*namespace.as_bytes()); + if let Some((replica, removal_token)) = self.replicas.remove(&topic) { + replica.remove_on_insert(removal_token); + self.gossip.quit(topic).await?; + } + Ok(()) + } + + async fn join_gossip_and_start_initial_sync( &mut self, - doc: Replica, - initial_peers: Vec, - ) -> Result<()> { - let peer_ids: Vec = initial_peers.iter().map(|p| p.peer_id).collect(); + namespace: &NamespaceId, + peers: Vec, + ) -> anyhow::Result<()> { + let topic = TopicId::from_bytes(*namespace.as_bytes()); + let peer_ids: Vec = peers.iter().map(|p| p.peer_id).collect(); // add addresses of initial peers to our endpoint address book - for peer in &initial_peers { - self.endpoint + for peer in &peers { + if let Err(err) = self + .endpoint .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) - .await?; + .await + { + warn!(peer = ?peer.peer_id, "failed to add known addrs: {err:?}"); + } } // join gossip for the topic to receive and send message - let topic = TopicId::from_bytes(*doc.namespace().as_bytes()); self.pending_joins.push({ let peer_ids = peer_ids.clone(); let gossip = self.gossip.clone(); @@ -250,23 +335,6 @@ impl Actor { .boxed() }); - // setup replica insert notifications. - let insert_entry_tx = self.insert_entry_tx.clone(); - doc.on_insert(Box::new(move |origin, entry| { - // only care for local inserts, otherwise we'd do endless gossip loops - if let InsertOrigin::Local = origin { - // TODO: this is potentially blocking inside an async call. figure out a better solution - insert_entry_tx.send((topic, entry)).ok(); - } - })); - self.docs.insert(topic, doc); - // add addresses of initial peers to our endpoint address book - for peer in &initial_peers { - self.endpoint - .add_known_addrs(peer.peer_id, peer.derp_region, &peer.addrs) - .await?; - } - // trigger initial sync with initial peers for peer in peer_ids { self.sync_with_peer(topic, peer); @@ -274,6 +342,34 @@ impl Actor { Ok(()) } + async fn start_sync( + &mut self, + replica: Replica, + peers: Vec, + ) -> Result<()> { + let namespace = replica.namespace(); + let topic = TopicId::from_bytes(*namespace.as_bytes()); + if !self.replicas.contains_key(&topic) { + // setup replica insert notifications. + let insert_entry_tx = self.insert_entry_tx.clone(); + let removal_token = replica.on_insert(Box::new(move |origin, entry| { + // only care for local inserts, otherwise we'd do endless gossip loops + if let InsertOrigin::Local = origin { + // TODO: this is potentially blocking inside an async call. figure out a better solution + if let Err(err) = insert_entry_tx.send((topic, entry)) { + warn!("on_insert forward failed: {err} - LiveSync actor dropped"); + } + } + })); + self.replicas.insert(topic, (replica, removal_token)); + } + + self.join_gossip_and_start_initial_sync(&namespace, peers) + .await?; + + Ok(()) + } + fn on_sync_finished(&mut self, topic: TopicId, peer: PeerId, res: Result<()>) { let state = match res { Ok(_) => SyncState::Finished, @@ -283,7 +379,7 @@ impl Actor { } fn on_gossip_event(&mut self, topic: TopicId, event: Event) -> Result<()> { - let Some(doc) = self.docs.get(&topic) else { + let Some((replica, _token)) = self.replicas.get(&topic) else { return Err(anyhow!("Missing doc for {topic:?}")); }; match event { @@ -291,13 +387,19 @@ impl Actor { Event::Received(data, prev_peer) => { let op: Op = postcard::from_bytes(&data)?; match op { - Op::Put(entry) => doc.insert_remote_entry(entry, Some(prev_peer.to_bytes()))?, + Op::Put(entry) => { + debug!(peer = ?prev_peer, topic = ?topic, "received entry via gossip"); + replica.insert_remote_entry(entry, Some(prev_peer.to_bytes()))? + } } } // A new neighbor appeared in the gossip swarm. Try to sync with it directly. // [Self::sync_with_peer] will check to not resync with peers synced previously in the // same session. TODO: Maybe this is too broad and leads to too many sync requests. - Event::NeighborUp(peer) => self.sync_with_peer(topic, peer), + Event::NeighborUp(peer) => { + debug!(peer = ?peer, "new neighbor, init sync"); + self.sync_with_peer(topic, peer); + } _ => {} } Ok(()) @@ -307,6 +409,7 @@ impl Actor { async fn on_insert_entry(&mut self, topic: TopicId, entry: SignedEntry) -> Result<()> { let op = Op::Put(entry); let message = postcard::to_stdvec(&op)?.into(); + debug!(topic = ?topic, "broadcast new entry"); self.gossip.broadcast(topic, message).await?; Ok(()) } diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs new file mode 100644 index 0000000000..8aa4979f41 --- /dev/null +++ b/iroh/tests/sync.rs @@ -0,0 +1,215 @@ +#![cfg(all(feature = "sync"))] + +use std::{net::SocketAddr, time::Duration}; + +use anyhow::Result; +use futures::StreamExt; +use iroh::{ + client::Iroh, + collection::IrohCollectionParser, + database::flat::{writable::WritableFileDatabase, Database}, + node::{Builder, Node}, + rpc_protocol::{ProviderService, ShareMode}, +}; +use quic_rpc::{transport::misc::DummyServerEndpoint, ServiceConnection}; +use tempfile::TempDir; +use tracing_subscriber::{prelude::*, EnvFilter}; + +use iroh_bytes::{provider::BaoReadonlyDb, util::runtime}; +use iroh_sync::{ + store::{self, GetFilter, KeyFilter}, + sync::NamespaceId, +}; + +/// Pick up the tokio runtime from the thread local and add a +/// thread per core runtime. +fn test_runtime() -> runtime::Handle { + runtime::Handle::from_currrent(1).unwrap() +} + +struct Cancel(TempDir); + +fn test_node( + rt: runtime::Handle, + db: D, + writable_db_path: PathBuf, + addr: SocketAddr, +) -> Builder { + let store = iroh_sync::store::memory::Store::default(); + Node::builder(db, store, writable_db_path) + .collection_parser(IrohCollectionParser) + .runtime(&rt) + .bind_addr(addr) +} + +struct NodeDropGuard { + _dir: TempDir, + node: Node, +} +impl Drop for NodeDropGuard { + fn drop(&mut self) { + self.node.shutdown(); + } +} + +async fn spawn_node( + rt: runtime::Handle, +) -> anyhow::Result<(Node, NodeDropGuard)> { + let dir = tempfile::tempdir()?; + let db = WritableFileDatabase::new(dir.path().into()).await?; + let node = test_node( + rt, + db.db().clone(), + dir.path().into(), + "127.0.0.1:0".parse()?, + ); + let node = node.spawn().await?; + Ok((node.clone(), NodeDropGuard { node, _dir: dir })) +} + +async fn spawn_nodes( + rt: runtime::Handle, + n: usize, +) -> anyhow::Result<( + Vec>, + Vec, +)> { + let mut nodes = vec![]; + let mut guards = vec![]; + for _i in 0..n { + let (node, guard) = spawn_node(rt.clone()).await?; + nodes.push(node); + guards.push(guard); + } + Ok((nodes, guards)) +} + +#[tokio::test] +async fn sync_full_basic() -> Result<()> { + setup_logging(); + let rt = test_runtime(); + let (nodes, drop_guard) = spawn_nodes(rt, 3).await?; + let clients = nodes.iter().map(|node| node.client()).collect::>(); + + for (i, node) in nodes.iter().enumerate() { + println!( + "node {i}: {} {:?}", + node.peer_id(), + node.local_endpoints().await + ); + } + + // node1: create doc and ticket + let (id, ticket) = { + let iroh = &clients[0]; + let author = iroh.create_author().await?; + let doc = iroh.create_doc().await?; + let key = b"p1"; + let value = b"1"; + doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; + let res = doc.get_bytes_latest(author, key.to_vec()).await?; + assert_eq!(res.to_vec(), value.to_vec()); + let ticket = doc.share(ShareMode::Write).await?; + (doc.id(), ticket) + }; + + // node2: join in + { + let iroh = &clients[1]; + let author = iroh.create_author().await?; + println!("\n\n!!!! peer 1 joins !!!!"); + let doc = iroh + .import_doc(ticket.key, vec![ticket.peer.clone()]) + .await?; + tokio::time::sleep(Duration::from_secs(2)).await; + for (i, client) in clients.iter().enumerate() { + report(&client, id, format!("node{i}")).await; + } + + println!("\n\n!!!! peer 1 publishes !!!!"); + + let key = b"p2"; + let value = b"22"; + doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; + // todo: events + tokio::time::sleep(Duration::from_secs(2)).await; + for (i, client) in clients.iter().enumerate() { + report(&client, id, format!("node{i}")).await; + } + } + + println!("\n\n!!!! peer 2 joins !!!!"); + { + // node 3 joins & imports the doc from peer 1 + let iroh = &clients[2]; + let author = iroh.create_author().await?; + let doc = iroh + .import_doc(ticket.key, vec![ticket.peer.clone()]) + .await?; + + // now wait... + tokio::time::sleep(Duration::from_secs(5)).await; + println!("\n\n!!!! peer 2 publishes !!!!"); + let key = b"p3"; + let value = b"333"; + doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; + } + + tokio::time::sleep(Duration::from_secs(5)).await; + for (i, client) in clients.iter().enumerate() { + report(&client, id, format!("node{i}")).await; + } + + drop(drop_guard); + + Ok(()) +} + +async fn report>( + client: &Iroh, + id: NamespaceId, + label: impl ToString, +) { + let label = label.to_string(); + println!("report: {label} {id}"); + match try_report(client, id).await { + Ok(_) => {} + Err(err) => println!(" failed: {err}"), + } +} + +async fn try_report>( + client: &Iroh, + id: NamespaceId, +) -> anyhow::Result<()> { + let doc = client.get_doc(id)?; + let filter = GetFilter { + latest: false, + author: None, + key: KeyFilter::All, + }; + let mut stream = doc.get(filter).await?; + while let Some(entry) = stream.next().await { + let entry = entry?; + let text = match client.get_bytes(*entry.content_hash()).await { + Ok(bytes) => String::from_utf8(bytes.to_vec())?, + Err(err) => format!("<{err}>"), + }; + println!( + " @{} {} {:4} -- {}", + entry.author(), + entry.content_hash(), + entry.content_len(), + text + ); + } + Ok(()) +} + +fn setup_logging() { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) + .with(EnvFilter::from_default_env()) + .try_init() + .ok(); +} From c76a1daa66500faf7854ed05020d4231bfbaba14 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 4 Aug 2023 19:00:27 +0200 Subject: [PATCH 047/172] wip: integrate iroh sync and gossip wip: more methods wip: integration of iroh-sync in iroh changes after rebase continue integration of iroh-sync in iroh node * removed the `Doc`, `DocStore`, `BlobStore` abstractions, they were not the right way for now * Introduce `SyncEngine` that combines the gossip `LiveSync` with on_insert callbacks to download content, to replace the functionality from the above-mentioned abstractions * Add RPC handlers on the `SyncEngine` for the most important operations * Introduce `iroh::client::Iroh`, a wrapper around the RPC client, and `iroh::client::Doc`, a wrapper around RPC client plus document id * Use the new client structs in the CLI * Rename `iroh_sync::get_replica` to `iroh_sync::open_replica` and expect them to be singletons per namespace * Various other minor fixes and improvements to logging --- iroh-bytes/src/util.rs | 6 + iroh/examples/sync.rs | 2 - iroh/src/client.rs | 188 +++++++++++++++++++++ iroh/src/commands.rs | 24 ++- iroh/src/commands/get.rs | 5 +- iroh/src/commands/provide.rs | 15 +- iroh/src/config.rs | 5 + iroh/src/download.rs | 41 +++-- iroh/src/lib.rs | 4 +- iroh/src/node.rs | 280 ++++++++++++++++++++++++------- iroh/src/rpc_protocol.rs | 309 ++++++++++++++++++++++++++++++++++- iroh/src/sync.rs | 1 + iroh/src/sync/engine.rs | 5 +- iroh/src/sync/rpc.rs | 152 +++++++++++++++++ iroh/tests/provide.rs | 6 +- 15 files changed, 944 insertions(+), 99 deletions(-) create mode 100644 iroh/src/client.rs create mode 100644 iroh/src/sync/rpc.rs diff --git a/iroh-bytes/src/util.rs b/iroh-bytes/src/util.rs index 40244827fb..432ba03fb3 100644 --- a/iroh-bytes/src/util.rs +++ b/iroh-bytes/src/util.rs @@ -214,6 +214,12 @@ impl From for RpcError { } } +impl From for RpcError { + fn from(e: std::io::Error) -> Self { + RpcError(serde_error::Error::new(&e)) + } +} + /// A serializable result type for use in RPC responses. #[allow(dead_code)] pub type RpcResult = result::Result; diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index ac5392ab16..77622140ef 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -229,14 +229,12 @@ async fn run(args: Args) -> anyhow::Result<()> { let docs = iroh_sync::store::fs::Store::new(&docs_path)?; // create the live syncer - let db = WritableFileDatabase::new(storage_path.join("blobs")).await?; let downloader = Downloader::new(rt.clone(), endpoint.clone(), db.clone()); let live_sync = SyncEngine::spawn( rt.clone(), endpoint.clone(), gossip.clone(), docs.clone(), - db.clone(), downloader, ); diff --git a/iroh/src/client.rs b/iroh/src/client.rs new file mode 100644 index 0000000000..0cd686e200 --- /dev/null +++ b/iroh/src/client.rs @@ -0,0 +1,188 @@ +//! Client to an iroh node. Is generic over the connection (in-memory or RPC). +//! +//! TODO: Contains only iroh sync related methods. Add other methods. + +// TODO: fill out docs +#![allow(missing_docs)] + +use std::result::Result as StdResult; + +use anyhow::{anyhow, Result}; +use bytes::Bytes; +use futures::{Stream, StreamExt, TryStreamExt}; +use iroh_bytes::Hash; +use iroh_sync::store::{GetFilter, KeyFilter}; +use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; +use quic_rpc::{RpcClient, ServiceConnection}; + +use crate::rpc_protocol::{ + AuthorCreateRequest, AuthorListRequest, BytesGetRequest, DocGetRequest, DocImportRequest, + DocSetRequest, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocsCreateRequest, + DocsListRequest, ShareMode, +}; +use crate::rpc_protocol::{KeyBytes, ProviderService}; +use crate::sync::PeerSource; + +/// Iroh client +pub struct Iroh { + rpc: RpcClient, +} + +impl Iroh +where + C: ServiceConnection, +{ + pub fn new(rpc: RpcClient) -> Self { + Self { rpc } + } + pub async fn create_author(&self) -> Result { + let res = self.rpc.rpc(AuthorCreateRequest).await??; + Ok(res.author_id) + } + + pub async fn list_authors(&self) -> Result>> { + let stream = self.rpc.server_streaming(AuthorListRequest {}).await?; + Ok(flatten(stream).map_ok(|res| res.author_id)) + } + + pub async fn create_doc(&self) -> Result> { + let res = self.rpc.rpc(DocsCreateRequest {}).await??; + let doc = Doc { + id: res.id, + rpc: self.rpc.clone(), + }; + Ok(doc) + } + + pub async fn import_doc(&self, key: KeyBytes, peers: Vec) -> Result> { + let res = self.rpc.rpc(DocImportRequest { key, peers }).await??; + let doc = Doc { + id: res.doc_id, + rpc: self.rpc.clone(), + }; + Ok(doc) + } + + pub async fn list_docs(&self) -> Result>> { + let stream = self.rpc.server_streaming(DocsListRequest {}).await?; + Ok(flatten(stream).map_ok(|res| res.id)) + } + + pub fn get_doc(&self, id: NamespaceId) -> Result> { + // TODO: Check if doc exists? + let doc = Doc { + id, + rpc: self.rpc.clone(), + }; + Ok(doc) + } + + // TODO: add get_reader for streaming gets + pub async fn get_bytes(&self, hash: Hash) -> Result { + let res = self.rpc.rpc(BytesGetRequest { hash }).await??; + Ok(res.data) + } +} + +/// Document handle +pub struct Doc { + id: NamespaceId, + rpc: RpcClient, +} + +impl Doc +where + C: ServiceConnection, +{ + pub fn id(&self) -> NamespaceId { + self.id + } + + pub async fn set_bytes( + &self, + author_id: AuthorId, + key: Vec, + value: Vec, + ) -> Result { + let res = self + .rpc + .rpc(DocSetRequest { + doc_id: self.id, + author_id, + key, + value, + }) + .await??; + Ok(res.entry) + } + + // TODO: add get_content_reader + pub async fn get_content_bytes(&self, entry: &SignedEntry) -> Result { + let hash = *entry.content_hash(); + let bytes = self.rpc.rpc(BytesGetRequest { hash }).await??; + Ok(bytes.data) + } + + pub async fn get_latest(&self, author_id: AuthorId, key: Vec) -> Result { + let filter = GetFilter { + key: KeyFilter::Key(key), + author: Some(author_id), + latest: true, + }; + let mut stream = self.get(filter).await?; + let entry = stream + .next() + .await + .unwrap_or_else(|| Err(anyhow!("not found")))?; + Ok(entry) + } + + pub async fn get(&self, filter: GetFilter) -> Result>> { + let stream = self + .rpc + .server_streaming(DocGetRequest { + doc_id: self.id, + filter, + }) + .await?; + Ok(flatten(stream).map_ok(|res| res.entry)) + } + + pub async fn share(&self, mode: ShareMode) -> anyhow::Result { + let res = self + .rpc + .rpc(DocShareRequest { + doc_id: self.id, + mode, + }) + .await??; + Ok(res) + } + + pub async fn start_sync(&self, peers: Vec) -> Result<()> { + let _res = self + .rpc + .rpc(DocStartSyncRequest { + doc_id: self.id, + peers, + }) + .await??; + Ok(()) + } + + // TODO: add stop_sync +} + +fn flatten( + s: impl Stream, E2>>, +) -> impl Stream> +where + E1: std::error::Error + Send + Sync + 'static, + E2: std::error::Error + Send + Sync + 'static, +{ + s.map(|res| match res { + Ok(Ok(res)) => Ok(res), + Ok(Err(err)) => Err(err.into()), + Err(err) => Err(err.into()), + }) +} diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index b70b4cb7e6..54205ee2be 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -12,7 +12,6 @@ use iroh::rpc_protocol::*; use iroh_bytes::{protocol::RequestToken, util::runtime, Hash}; use iroh_net::tls::{Keypair, PeerId}; use quic_rpc::transport::quinn::QuinnConnection; -use quic_rpc::RpcClient; use crate::config::Config; @@ -28,8 +27,13 @@ pub mod doctor; pub mod get; pub mod list; pub mod provide; +pub mod sync; pub mod validate; +/// RPC client to an iroh node. +pub type RpcClient = + quic_rpc::RpcClient>; + /// Send data. /// /// The iroh command line tool has two modes: provide and get. @@ -216,6 +220,10 @@ impl Cli { Ok(()) } Commands::Doctor { command } => self::doctor::run(command, config).await, + Commands::Sync { command, rpc_port } => { + let client = make_rpc_client(rpc_port).await?; + command.run(client).await + } } } } @@ -390,18 +398,22 @@ pub enum Commands { #[clap(long, default_value_t = DEFAULT_RPC_PORT)] rpc_port: u16, }, + Sync { + /// RPC port + #[clap(long, default_value_t = DEFAULT_RPC_PORT)] + rpc_port: u16, + #[clap(subcommand)] + command: sync::Commands, + }, } -async fn make_rpc_client( - rpc_port: u16, -) -> anyhow::Result>> -{ +async fn make_rpc_client(rpc_port: u16) -> anyhow::Result { let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); let server_name = "localhost".to_string(); let connection = QuinnConnection::new(endpoint, addr, server_name); - let client = RpcClient::::new(connection); + let client = RpcClient::new(connection); // Do a version request to check if the server is running. let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) .await diff --git a/iroh/src/commands/get.rs b/iroh/src/commands/get.rs index 405a355df8..2d7076ad6f 100644 --- a/iroh/src/commands/get.rs +++ b/iroh/src/commands/get.rs @@ -69,8 +69,11 @@ impl GetInteractive { tokio::fs::create_dir_all(&temp_dir).await?; let db: iroh::baomap::flat::Store = iroh::baomap::flat::Store::load(temp_dir.clone(), temp_dir.clone(), &self.rt).await?; + // TODO: we don't need sync here, maybe disable completely? + let doc_store = iroh_sync::store::memory::Store::default(); // spin up temp node and ask it to download the data for us - let mut provider = iroh::node::Node::builder(db).collection_parser(IrohCollectionParser); + let mut provider = + iroh::node::Node::builder(db, doc_store).collection_parser(IrohCollectionParser); if let Some(ref dm) = self.opts.derp_map { provider = provider.derp_map(dm.clone()); } diff --git a/iroh/src/commands/provide.rs b/iroh/src/commands/provide.rs index 8056f4d32d..3223ec8536 100644 --- a/iroh/src/commands/provide.rs +++ b/iroh/src/commands/provide.rs @@ -13,8 +13,9 @@ use iroh::{ node::{Node, StaticTokenAuthHandler}, rpc_protocol::{ProvideRequest, ProviderRequest, ProviderResponse, ProviderService}, }; -use iroh_bytes::{baomap::Store, protocol::RequestToken, util::runtime}; +use iroh_bytes::{baomap::Store as BaoStore, protocol::RequestToken, util::runtime}; use iroh_net::{derp::DerpMap, tls::Keypair}; +use iroh_sync::store::Store as DocStore; use quic_rpc::{transport::quinn::QuinnServerEndpoint, ServiceEndpoint}; use tokio::io::AsyncWriteExt; use tracing::{info_span, Instrument}; @@ -57,8 +58,9 @@ pub async fn run( .await .with_context(|| format!("Failed to load iroh database from {}", blob_dir.display()))?; let key = Some(IrohPaths::Keypair.with_env()?); + let store = iroh_sync::store::fs::Store::new(IrohPaths::DocsDatabase.with_env()?)?; let token = opts.request_token.clone(); - let provider = provide(db.clone(), rt, key, opts).await?; + let provider = provide(db.clone(), store, rt, key, opts).await?; let controller = provider.controller(); if let Some(t) = token.as_ref() { println!("Request token: {}", t); @@ -125,15 +127,16 @@ pub async fn run( Ok(()) } -async fn provide( - db: D, +async fn provide( + bao_store: B, + doc_store: D, rt: &runtime::Handle, key: Option, opts: ProvideOptions, -) -> Result> { +) -> Result> { let keypair = get_keypair(key).await?; - let mut builder = Node::builder(db) + let mut builder = Node::builder(bao_store, doc_store) .collection_parser(IrohCollectionParser) .custom_auth_handler(Arc::new(StaticTokenAuthHandler::new(opts.request_token))) .keylog(opts.keylog); diff --git a/iroh/src/config.rs b/iroh/src/config.rs index c388d0b3e6..86ff92e0fb 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -32,6 +32,8 @@ pub enum IrohPaths { BaoFlatStoreComplete, /// Path to the node's [flat-file store](iroh::baomap::flat) for partial blobs. BaoFlatStorePartial, + /// Path to the [iroh-sync document database](iroh_sync::store::fs::Store) + DocsDatabase, } impl From<&IrohPaths> for &'static str { fn from(value: &IrohPaths) -> Self { @@ -39,6 +41,7 @@ impl From<&IrohPaths> for &'static str { IrohPaths::Keypair => "keypair", IrohPaths::BaoFlatStoreComplete => "blobs.v0", IrohPaths::BaoFlatStorePartial => "blobs-partial.v0", + IrohPaths::DocsDatabase => "docs.redb", } } } @@ -49,6 +52,7 @@ impl FromStr for IrohPaths { "keypair" => Self::Keypair, "blobs.v0" => Self::BaoFlatStoreComplete, "blobs-partial.v0" => Self::BaoFlatStorePartial, + "docs.redb" => Self::DocsDatabase, _ => bail!("unknown file or directory"), }) } @@ -262,6 +266,7 @@ mod tests { IrohPaths::BaoFlatStoreComplete, IrohPaths::BaoFlatStorePartial, IrohPaths::Keypair, + IrohPaths::DocsDatabase, ]; for iroh_path in &kinds { let root = PathBuf::from("/tmp"); diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 89a0db3180..5fd6222511 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -11,7 +11,10 @@ use futures::{ stream::FuturesUnordered, FutureExt, }; -use iroh_bytes::util::Hash; +use iroh_bytes::{ + baomap::{MapEntry, Store as BaoStore}, + util::Hash, +}; use iroh_gossip::net::util::Dialer; use iroh_metrics::{inc, inc_by}; use iroh_net::{tls::PeerId, MagicEndpoint}; @@ -21,9 +24,6 @@ use tracing::{debug, error, warn}; #[cfg(feature = "metrics")] use crate::metrics::Metrics; -// TODO: Will be replaced by proper persistent DB once -// https://github.com/n0-computer/iroh/pull/1320 is merged -use crate::database::flat::writable::WritableFileDatabase; /// Future for the completion of a download request pub type DownloadFuture = Shared>>; @@ -45,10 +45,10 @@ pub struct Downloader { impl Downloader { /// Create a new downloader - pub fn new( + pub fn new( rt: iroh_bytes::util::runtime::Handle, endpoint: MagicEndpoint, - db: WritableFileDatabase, + db: B, ) -> Self { let (tx, rx) = flume::bounded(64); // spawn the actor on a local pool @@ -118,21 +118,17 @@ struct DownloadRequest { } #[derive(Debug)] -struct DownloadActor { +struct DownloadActor { dialer: Dialer, - db: WritableFileDatabase, + db: B, conns: HashMap, replies: HashMap>, pending_download_futs: PendingDownloadsFutures, queue: DownloadQueue, rx: flume::Receiver, } -impl DownloadActor { - fn new( - endpoint: MagicEndpoint, - db: WritableFileDatabase, - rx: flume::Receiver, - ) -> Self { +impl DownloadActor { + fn new(endpoint: MagicEndpoint, db: B, rx: flume::Receiver) -> Self { Self { rx, db, @@ -205,11 +201,11 @@ impl DownloadActor { fn start_download_unchecked(&mut self, peer: PeerId, hash: Hash) { let conn = self.conns.get(&peer).unwrap().clone(); - let blobs = self.db.clone(); + let db = self.db.clone(); let fut = async move { #[cfg(feature = "metrics")] let start = Instant::now(); - let res = blobs.download_single(conn, hash).await; + let res = download_single(db, conn, hash).await; // record metrics #[cfg(feature = "metrics")] { @@ -231,8 +227,8 @@ impl DownloadActor { async fn on_download_request(&mut self, req: DownloadRequest) { let DownloadRequest { peers, hash, reply } = req; - if self.db.has(&hash) { - let size = self.db.get_size(&hash).await.unwrap(); + if let Some(entry) = self.db.get(&hash) { + let size = entry.size(); reply.send(Some((hash, size))).ok(); return; } @@ -247,6 +243,15 @@ impl DownloadActor { } } +// TODO: reimplement downloads +async fn download_single( + _store: B, + _conn: quinn::Connection, + _hash: Hash, +) -> anyhow::Result> { + todo!("Downloads not implemented") +} + #[derive(Debug, Default)] struct DownloadQueue { candidates_by_hash: HashMap>, diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index ef50b484cb..e1a0e6d123 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -8,14 +8,16 @@ pub mod baomap; #[cfg(feature = "iroh-collection")] pub mod collection; pub mod dial; -pub mod get; pub mod download; +pub mod get; pub mod node; pub mod rpc_protocol; #[allow(missing_docs)] pub mod sync; pub mod util; +pub mod client; + /// Expose metrics module #[cfg(feature = "metrics")] pub mod metrics; diff --git a/iroh/src/node.rs b/iroh/src/node.rs index c99d638ec6..394303f9d2 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -15,36 +15,36 @@ use std::sync::Arc; use std::task::Poll; use std::time::Duration; -use crate::dial::Ticket; -use crate::rpc_protocol::{ - AddrsRequest, AddrsResponse, IdRequest, IdResponse, ListBlobsRequest, ListBlobsResponse, - ListCollectionsRequest, ListCollectionsResponse, ListIncompleteBlobsRequest, - ListIncompleteBlobsResponse, ProvideRequest, ProviderRequest, ProviderResponse, - ProviderService, ShareRequest, ShutdownRequest, ValidateRequest, VersionRequest, - VersionResponse, WatchRequest, WatchResponse, -}; -use anyhow::{Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use bytes::Bytes; use futures::future::{BoxFuture, Shared}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; -use iroh_bytes::baomap::{ExportMode, Map, MapEntry, ReadableStore, Store, ValidateProgress}; +use iroh_bytes::baomap::{ + ExportMode, Map, MapEntry, ReadableStore, Store as BaoStore, ValidateProgress, +}; use iroh_bytes::collection::{CollectionParser, NoCollectionParser}; use iroh_bytes::get::Stats; use iroh_bytes::protocol::GetRequest; use iroh_bytes::provider::ShareProgress; use iroh_bytes::util::progress::{FlumeProgressSender, IdGenerator, ProgressSender}; +use iroh_bytes::util::{RpcError, RpcResult}; use iroh_bytes::{ protocol::{Closed, Request, RequestToken}, provider::{CustomGetHandler, ProvideProgress, RequestAuthorizationHandler}, util::runtime, util::Hash, }; +use iroh_gossip::net::{Gossip, GOSSIP_ALPN}; +use iroh_io::AsyncSliceReaderExt; +use iroh_net::magic_endpoint::get_alpn; use iroh_net::{ config::Endpoint, derp::DerpMap, tls::{self, Keypair, PeerId}, MagicEndpoint, }; +use iroh_sync::store::Store as DocStore; +use once_cell::sync::OnceCell; use quic_rpc::server::RpcChannel; use quic_rpc::transport::flume::FlumeConnection; use quic_rpc::transport::misc::DummyServerEndpoint; @@ -54,6 +54,17 @@ use tokio::task::JoinError; use tokio_util::sync::CancellationToken; use tracing::{debug, trace}; +use crate::dial::Ticket; +use crate::download::Downloader; +use crate::rpc_protocol::{ + AddrsRequest, AddrsResponse, BytesGetRequest, BytesGetResponse, IdRequest, IdResponse, + ListBlobsRequest, ListBlobsResponse, ListCollectionsRequest, ListCollectionsResponse, + ListIncompleteBlobsRequest, ListIncompleteBlobsResponse, ProvideRequest, ProviderRequest, + ProviderResponse, ProviderService, ShareRequest, ShutdownRequest, ValidateRequest, + VersionRequest, VersionResponse, WatchRequest, WatchResponse, +}; +use crate::sync::{SyncEngine, SYNC_ALPN}; + const MAX_CONNECTIONS: u32 = 1024; const MAX_STREAMS: u64 = 10; const HEALTH_POLL_WAIT: Duration = Duration::from_secs(1); @@ -75,9 +86,14 @@ const ENDPOINT_WAIT: Duration = Duration::from_secs(5); /// The returned [`Node`] is awaitable to know when it finishes. It can be terminated /// using [`Node::shutdown`]. #[derive(Debug)] -pub struct Builder -where +pub struct Builder< + D, + S = iroh_sync::store::memory::Store, + E = DummyServerEndpoint, + C = NoCollectionParser, +> where D: Map, + S: DocStore, E: ServiceEndpoint, C: CollectionParser, { @@ -91,9 +107,10 @@ where derp_map: Option, collection_parser: C, rt: Option, + docs: S, } -const PROTOCOLS: [&[u8]; 1] = [&iroh_bytes::protocol::ALPN]; +const PROTOCOLS: [&[u8]; 3] = [&iroh_bytes::protocol::ALPN, GOSSIP_ALPN, SYNC_ALPN]; /// A noop authorization handler that does not do any authorization. /// @@ -134,9 +151,9 @@ impl CustomGetHandler for NoopCustomGetHandler { } } -impl Builder { +impl Builder { /// Creates a new builder for [`Node`] using the given database. - fn with_db(db: D) -> Self { + fn with_db_and_store(db: D, docs: S) -> Self { Self { bind_addr: DEFAULT_BIND_ADDR.into(), keypair: Keypair::generate(), @@ -148,13 +165,15 @@ impl Builder { auth_handler: Arc::new(NoopRequestAuthorizationHandler), collection_parser: NoCollectionParser, rt: None, + docs, } } } -impl Builder +impl Builder where - D: Store, + D: BaoStore, + S: DocStore, E: ServiceEndpoint, C: CollectionParser, { @@ -162,7 +181,7 @@ where pub fn rpc_endpoint>( self, value: E2, - ) -> Builder { + ) -> Builder { // we can't use ..self here because the return type is different Builder { bind_addr: self.bind_addr, @@ -175,6 +194,7 @@ where derp_map: self.derp_map, collection_parser: self.collection_parser, rt: self.rt, + docs: self.docs, } } @@ -182,7 +202,7 @@ where pub fn collection_parser( self, collection_parser: C2, - ) -> Builder { + ) -> Builder { // we can't use ..self here because the return type is different Builder { collection_parser, @@ -195,6 +215,7 @@ where rpc_endpoint: self.rpc_endpoint, derp_map: self.derp_map, rt: self.rt, + docs: self.docs, } } @@ -257,7 +278,7 @@ where /// This will create the underlying network server and spawn a tokio task accepting /// connections. The returned [`Node`] can be used to control the task as well as /// get information about it. - pub async fn spawn(self) -> Result> { + pub async fn spawn(self) -> Result> { trace!("spawning node"); let rt = self.rt.context("runtime not set")?; @@ -267,6 +288,10 @@ where .max_concurrent_bidi_streams(MAX_STREAMS.try_into()?) .max_concurrent_uni_streams(0u32.into()); + // init a cell that will hold our gossip handle to be used in endpoint callbacks + let gossip_cell: OnceCell = OnceCell::new(); + let gossip_cell2 = gossip_cell.clone(); + let endpoint = MagicEndpoint::builder() .keypair(self.keypair.clone()) .alpns(PROTOCOLS.iter().map(|p| p.to_vec()).collect()) @@ -275,7 +300,14 @@ where .transport_config(transport_config) .concurrent_connections(MAX_CONNECTIONS) .on_endpoints(Box::new(move |eps| { - if !endpoints_update_s.is_disconnected() && !eps.is_empty() { + if eps.is_empty() { + return; + } + // send our updated endpoints to the gossip protocol to be sent as PeerData to peers + if let Some(gossip) = gossip_cell2.get() { + gossip.update_endpoints(eps).ok(); + } + if !endpoints_update_s.is_disconnected() { endpoints_update_s.send(()).ok(); } })) @@ -287,6 +319,23 @@ where let cancel_token = CancellationToken::new(); debug!("rpc listening on: {:?}", self.rpc_endpoint.local_addr()); + + // initialize the gossip protocol + let gossip = Gossip::from_endpoint(endpoint.clone(), Default::default()); + // insert into the gossip cell to be used in the endpoint callbacks above + gossip_cell.set(gossip.clone()).unwrap(); + + // spawn the sync engine + // TODO: Remove once persistence is merged + let downloader = Downloader::new(rt.clone(), endpoint.clone(), self.db.clone()); + let sync = SyncEngine::spawn( + rt.clone(), + endpoint.clone(), + gossip.clone(), + self.docs, + downloader, + ); + let (internal_rpc, controller) = quic_rpc::transport::flume::connection(1); let rt2 = rt.clone(); let rt3 = rt.clone(); @@ -300,6 +349,7 @@ where callbacks: callbacks.clone(), cb_sender, rt, + sync, }); let task = { let handler = RpcHandler { @@ -318,6 +368,7 @@ where self.auth_handler, self.collection_parser, rt3, + gossip, ) .await }) @@ -343,13 +394,14 @@ where server: MagicEndpoint, callbacks: Callbacks, mut cb_receiver: mpsc::Receiver, - handler: RpcHandler, + handler: RpcHandler, rpc: E, internal_rpc: impl ServiceEndpoint, custom_get_handler: Arc, auth_handler: Arc, collection_parser: C, rt: runtime::Handle, + gossip: Gossip, ) { let rpc = RpcServer::new(rpc); let internal_rpc = RpcServer::new(internal_rpc); @@ -362,6 +414,12 @@ where } let cancel_token = handler.inner.cancel_token.clone(); + // forward our initial endpoints to the gossip protocol + if let Ok(local_endpoints) = server.local_endpoints().await { + debug!(me = ?server.peer_id(), "gossip initial update: {local_endpoints:?}"); + gossip.update_endpoints(&local_endpoints).ok(); + } + loop { tokio::select! { biased; @@ -392,7 +450,6 @@ where }, // handle incoming p2p connections Some(mut connecting) = server.accept() => { - let alpn = match get_alpn(&mut connecting).await { Ok(alpn) => alpn, Err(err) => { @@ -400,19 +457,8 @@ where continue; } }; - if alpn.as_bytes() == iroh_bytes::protocol::ALPN.as_ref() { - let db = handler.inner.db.clone(); - let custom_get_handler = custom_get_handler.clone(); - let auth_handler = auth_handler.clone(); - let collection_parser = collection_parser.clone(); - let rt2 = rt.clone(); - let callbacks = callbacks.clone(); - rt.main().spawn(iroh_bytes::provider::handle_connection(connecting, db, callbacks, collection_parser, custom_get_handler, auth_handler, rt2)); - } else { - tracing::error!("unknown protocol: {}", alpn); - continue; - } - } + rt.main().spawn(handle_connection(connecting, alpn, handler.inner.clone(), gossip.clone(), collection_parser.clone(), custom_get_handler.clone(), auth_handler.clone())); + }, // Handle new callbacks Some(cb) = cb_receiver.recv() => { callbacks.push(cb).await; @@ -433,15 +479,33 @@ where } } -async fn get_alpn(connecting: &mut quinn::Connecting) -> Result { - let data = connecting.handshake_data().await?; - match data.downcast::() { - Ok(data) => match data.protocol { - Some(protocol) => std::string::String::from_utf8(protocol).map_err(Into::into), - None => anyhow::bail!("no ALPN protocol available"), - }, - Err(_) => anyhow::bail!("unknown handshake type"), +async fn handle_connection( + connecting: quinn::Connecting, + alpn: String, + node: Arc>, + gossip: Gossip, + collection_parser: C, + custom_get_handler: Arc, + auth_handler: Arc, +) -> Result<()> { + match alpn.as_bytes() { + GOSSIP_ALPN => gossip.handle_connection(connecting.await?).await?, + SYNC_ALPN => crate::sync::handle_connection(connecting, node.sync.store.clone()).await?, + alpn if alpn == iroh_bytes::protocol::ALPN => { + iroh_bytes::provider::handle_connection( + connecting, + node.db.clone(), + node.callbacks.clone(), + collection_parser, + custom_get_handler, + auth_handler, + node.rt.clone(), + ) + .await + } + _ => bail!("ignoring connection: unsupported ALPN protocol"), } + Ok(()) } type EventCallback = Box BoxFuture<'static, ()> + 'static + Sync + Send>; @@ -486,13 +550,13 @@ impl iroh_bytes::provider::EventSender for Callbacks { /// await the [`Node`] struct directly, it will complete when the task completes. If /// this is dropped the node task is not stopped but keeps running. #[derive(Debug, Clone)] -pub struct Node { - inner: Arc>, +pub struct Node { + inner: Arc>, task: Shared>>>, } #[derive(derive_more::Debug)] -struct NodeInner { +struct NodeInner { db: D, endpoint: MagicEndpoint, keypair: Keypair, @@ -503,6 +567,7 @@ struct NodeInner { #[allow(dead_code)] callbacks: Callbacks, rt: runtime::Handle, + pub(crate) sync: SyncEngine, } /// Events emitted by the [`Node`] informing about the current status. @@ -512,12 +577,14 @@ pub enum Event { ByteProvide(iroh_bytes::provider::Event), } -impl Node { +impl Node { /// Returns a new builder for the [`Node`]. /// /// Once the done with the builder call [`Builder::spawn`] to create the node. - pub fn builder(db: D) -> Builder { - Builder::with_db(db) + /// + /// TODO: remove blobs_path argument once peristence branch is merged + pub fn builder(bao_store: D, doc_store: S) -> Builder { + Builder::with_db_and_store(bao_store, doc_store) } /// The address on which the node socket is bound. @@ -557,12 +624,19 @@ impl Node { } /// Returns a handle that can be used to do RPC calls to the node internally. + /// + /// TODO: remove and replace with client? pub fn controller( &self, ) -> RpcClient> { RpcClient::new(self.inner.controller.clone()) } + /// + pub fn client(&self) -> super::client::Iroh> { + super::client::Iroh::new(self.controller()) + } + /// Return a single token containing everything needed to get a hash. /// /// See [`Ticket`] for more details of how it can be used. @@ -595,7 +669,7 @@ impl Node { } } -impl NodeInner { +impl NodeInner { async fn local_endpoints(&self) -> Result> { self.endpoint.local_endpoints().await } @@ -616,7 +690,7 @@ impl NodeInner { } /// The future completes when the spawned tokio task finishes. -impl Future for Node { +impl Future for Node { type Output = Result<(), Arc>; fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { @@ -625,12 +699,12 @@ impl Future for Node { } #[derive(Debug, Clone)] -struct RpcHandler { - inner: Arc>, +struct RpcHandler { + inner: Arc>, collection_parser: C, } -impl RpcHandler { +impl RpcHandler { fn rt(&self) -> runtime::Handle { self.inner.rt.clone() } @@ -775,7 +849,6 @@ impl RpcHandler { { use crate::collection::{Blob, Collection}; use crate::util::io::pathbuf_from_name; - use iroh_io::AsyncSliceReaderExt; tracing::trace!("exporting collection {} to {}", hash, path.display()); tokio::fs::create_dir_all(&path).await?; let collection = db.get(&hash).context("collection not there")?; @@ -1014,12 +1087,42 @@ impl RpcHandler { )) }) } + + // TODO: streaming + async fn bytes_get(self, req: BytesGetRequest) -> RpcResult { + let entry = self + .inner + .db + .get(&req.hash) + .ok_or_else(|| RpcError::from(anyhow!("not found")))?; + // TODO: size limit + // TODO: streaming + let data = self.inner.rt.local_pool().spawn_pinned(|| async move { + let data = entry + .data_reader() + .await + .map_err(anyhow::Error::from)? + .read_to_end() + .await + .map_err(anyhow::Error::from)?; + Result::<_, anyhow::Error>::Ok(data) + }); + let data = data + .await + .map_err(|_err| anyhow::anyhow!("task failed to complete"))??; + Ok(BytesGetResponse { data }) + } } -fn handle_rpc_request, C: CollectionParser>( +fn handle_rpc_request< + D: BaoStore, + S: DocStore, + E: ServiceEndpoint, + C: CollectionParser, +>( msg: ProviderRequest, chan: RpcChannel, - handler: &RpcHandler, + handler: &RpcHandler, rt: &runtime::Handle, ) { let handler = handler.clone(); @@ -1057,6 +1160,67 @@ fn handle_rpc_request, C: Collecti chan.server_streaming(msg, handler, RpcHandler::validate) .await } + PeerAdd(_msg) => todo!(), + PeerList(_msg) => todo!(), + AuthorList(msg) => { + chan.server_streaming(msg, handler, |handler, req| { + handler.inner.sync.author_list(req) + }) + .await + } + AuthorCreate(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.author_create(req) + }) + .await + } + AuthorImport(_msg) => { + todo!() + } + AuthorShare(_msg) => todo!(), + DocsList(msg) => { + chan.server_streaming(msg, handler, |handler, req| { + handler.inner.sync.docs_list(req) + }) + .await + } + DocsCreate(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.docs_create(req) + }) + .await + } + DocsImport(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_import(req).await + }) + .await + } + DocSet(msg) => { + let bao_store = handler.inner.db.clone(); + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_set(&bao_store, req).await + }) + .await + } + DocGet(msg) => { + chan.server_streaming(msg, handler, |handler, req| handler.inner.sync.doc_get(req)) + .await + } + DocStartSync(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_start_sync(req).await + }) + .await + } + DocShare(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_share(req).await + }) + .await + } + // TODO: make streaming + BytesGet(msg) => chan.rpc(msg, handler, RpcHandler::bytes_get).await, } }); } diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 6e8e443042..baca1d95a7 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -1,3 +1,5 @@ +#![allow(missing_docs)] + //! This defines the RPC protocol used for communication between a CLI and an iroh node. //! //! RPC using the [`quic-rpc`](https://docs.rs/quic-rpc) crate. @@ -9,17 +11,27 @@ //! Note that this is subject to change. The RPC protocol is not yet stable. use std::{net::SocketAddr, path::PathBuf}; +use bytes::Bytes; use derive_more::{From, TryInto}; use iroh_bytes::{protocol::RequestToken, provider::ShareProgress, Hash}; use iroh_net::tls::PeerId; +use iroh_sync::{ + store::GetFilter, + sync::{AuthorId, NamespaceId, SignedEntry}, +}; use quic_rpc::{ message::{Msg, RpcMsg, ServerStreaming, ServerStreamingMsg}, Service, }; use serde::{Deserialize, Serialize}; -pub use iroh_bytes::{baomap::ValidateProgress, provider::ProvideProgress}; +pub use iroh_bytes::{baomap::ValidateProgress, provider::ProvideProgress, util::RpcResult}; + +use crate::sync::PeerSource; + +/// A 32-byte key or token +pub type KeyBytes = [u8; 32]; /// A request to the node to provide the data at the given path /// @@ -256,6 +268,261 @@ pub struct VersionResponse { pub version: String, } +// peer + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct PeerAddRequest { + pub peer_id: PeerId, + pub addrs: Vec, + pub region: Option, +} + +impl RpcMsg for PeerAddRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct PeerAddResponse {} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct PeerListRequest {} + +impl Msg for PeerListRequest { + type Pattern = ServerStreaming; +} + +impl ServerStreamingMsg for PeerListRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct PeerListResponse { + pub peer_id: PeerId, +} + +// author + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorListRequest {} + +impl Msg for AuthorListRequest { + type Pattern = ServerStreaming; +} + +impl ServerStreamingMsg for AuthorListRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorListResponse { + pub author_id: AuthorId, + pub writable: bool, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorCreateRequest; + +impl RpcMsg for AuthorCreateRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorCreateResponse { + pub author_id: AuthorId, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorImportRequest { + // either a public or private key + pub key: KeyBytes, +} + +impl RpcMsg for AuthorImportRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorImportResponse { + pub author_id: AuthorId, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorShareRequest { + pub author: AuthorId, + pub mode: ShareMode, +} + +/// todo +#[derive(Serialize, Deserialize, Debug, Clone, clap::ValueEnum)] +pub enum ShareMode { + /// Read-only access + Read, + /// Write access + Write, +} + +impl RpcMsg for AuthorShareRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorShareResponse { + pub key: KeyBytes, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocsListRequest {} + +impl Msg for DocsListRequest { + type Pattern = ServerStreaming; +} + +impl ServerStreamingMsg for DocsListRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocsListResponse { + pub id: NamespaceId, + // pub writable: bool, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocsCreateRequest {} + +impl RpcMsg for DocsCreateRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocsCreateResponse { + pub id: NamespaceId, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocImportRequest { + // either a public or private key + pub key: KeyBytes, + pub peers: Vec, +} + +impl RpcMsg for DocImportRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocImportResponse { + pub doc_id: NamespaceId, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocShareRequest { + pub doc_id: NamespaceId, + pub mode: ShareMode, +} + +impl RpcMsg for DocShareRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocShareResponse { + pub key: KeyBytes, + pub peer: PeerSource, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocStartSyncRequest { + pub doc_id: NamespaceId, + pub peers: Vec, +} + +impl RpcMsg for DocStartSyncRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocStartSyncResponse {} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocSetRequest { + pub doc_id: NamespaceId, + pub author_id: AuthorId, + pub key: Vec, + // todo: different forms to supply value + pub value: Vec, +} + +impl RpcMsg for DocSetRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocSetResponse { + pub entry: SignedEntry, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocGetRequest { + pub doc_id: NamespaceId, + pub filter: GetFilter, +} + +impl Msg for DocGetRequest { + type Pattern = ServerStreaming; +} + +impl ServerStreamingMsg for DocGetRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocGetResponse { + pub entry: SignedEntry, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct BytesGetRequest { + pub hash: Hash, +} + +impl RpcMsg for BytesGetRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct BytesGetResponse { + pub data: Bytes, +} + /// The RPC service for the iroh provider process. #[derive(Debug, Clone)] pub struct ProviderService; @@ -275,6 +542,25 @@ pub enum ProviderRequest { Addrs(AddrsRequest), Shutdown(ShutdownRequest), Validate(ValidateRequest), + + PeerAdd(PeerAddRequest), + PeerList(PeerListRequest), + + AuthorList(AuthorListRequest), + AuthorCreate(AuthorCreateRequest), + AuthorImport(AuthorImportRequest), + AuthorShare(AuthorShareRequest), + + DocsList(DocsListRequest), + DocsCreate(DocsCreateRequest), + DocsImport(DocImportRequest), + + DocSet(DocSetRequest), + DocGet(DocGetRequest), + DocStartSync(DocStartSyncRequest), // DocGetContent(DocGetContentRequest), + DocShare(DocShareRequest), // DocGetContent(DocGetContentRequest), + + BytesGet(BytesGetRequest), } /// The response enum, listing all possible responses. @@ -292,6 +578,27 @@ pub enum ProviderResponse { Addrs(AddrsResponse), Validate(ValidateProgress), Shutdown(()), + + // TODO: I see I changed naming convention here but at least to me it becomes easier to parse + // with the subject in front if there's many commands + PeerAdd(RpcResult), + PeerList(RpcResult), + + AuthorList(RpcResult), + AuthorCreate(RpcResult), + AuthorImport(RpcResult), + AuthorShare(RpcResult), + + DocsList(RpcResult), + DocsCreate(RpcResult), + DocsImport(RpcResult), + + DocSet(RpcResult), + DocGet(RpcResult), + DocJoin(RpcResult), + DocShare(RpcResult), + + BytesGet(RpcResult), // DocGetContent(DocGetContentResponse), } impl Service for ProviderService { diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index c3d13e8ceb..d78ae7e388 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -19,6 +19,7 @@ pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; mod engine; mod live; pub mod metrics; +pub mod rpc; pub use engine::*; pub use live::*; diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index 021a5d8d13..a0719d26e6 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -10,7 +10,7 @@ use iroh_sync::{ }; use parking_lot::RwLock; -use crate::{database::flat::writable::WritableFileDatabase, download::Downloader}; +use crate::download::Downloader; use super::{LiveSync, PeerSource}; @@ -22,7 +22,6 @@ use super::{LiveSync, PeerSource}; pub struct SyncEngine { pub(crate) rt: Handle, pub(crate) store: S, - pub(crate) db: WritableFileDatabase, pub(crate) endpoint: MagicEndpoint, downloader: Downloader, live: LiveSync, @@ -35,7 +34,6 @@ impl SyncEngine { endpoint: MagicEndpoint, gossip: Gossip, store: S, - db: WritableFileDatabase, downloader: Downloader, ) -> Self { let live = LiveSync::spawn(rt.clone(), endpoint.clone(), gossip); @@ -43,7 +41,6 @@ impl SyncEngine { live, downloader, store, - db, rt, endpoint, active: Default::default(), diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs new file mode 100644 index 0000000000..9ad1b2b185 --- /dev/null +++ b/iroh/src/sync/rpc.rs @@ -0,0 +1,152 @@ +//! This module contains an impl block on [`SyncEngine`] with handlers for RPC requests + +use anyhow::anyhow; +use futures::Stream; +use iroh_bytes::{baomap::Store as BaoStore, util::RpcError}; +use iroh_sync::{store::Store, sync::Namespace}; +use itertools::Itertools; +use rand::rngs::OsRng; + +use crate::rpc_protocol::{ + AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, + DocGetRequest, DocGetResponse, DocImportRequest, DocImportResponse, DocSetRequest, + DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, + DocsCreateRequest, DocsCreateResponse, DocsListRequest, DocsListResponse, RpcResult, ShareMode, +}; + +use super::{engine::SyncEngine, PeerSource}; + +impl SyncEngine { + /// todo + pub fn author_create(&self, _req: AuthorCreateRequest) -> RpcResult { + // TODO: pass rng + let author = self.store.new_author(&mut rand::rngs::OsRng {})?; + Ok(AuthorCreateResponse { + author_id: author.id(), + }) + } + + /// todo + pub fn author_list( + &self, + _req: AuthorListRequest, + ) -> impl Stream> { + let ite = self.store.list_authors().map(|authors| authors.into_iter()); + let ite = inline_error(ite).map_ok(|author| AuthorListResponse { + author_id: author.id(), + writable: true, + }); + futures::stream::iter(ite) + } + + pub fn docs_create(&self, _req: DocsCreateRequest) -> RpcResult { + let doc = self.store.new_replica(Namespace::new(&mut OsRng {}))?; + Ok(DocsCreateResponse { + id: doc.namespace(), + }) + } + + pub fn docs_list( + &self, + _req: DocsListRequest, + ) -> impl Stream> { + let ite = self.store.list_replicas().map(|res| res.into_iter()); + let ite = inline_error(ite).map_ok(|id| DocsListResponse { id }); + futures::stream::iter(ite) + } + + pub async fn doc_share(&self, req: DocShareRequest) -> RpcResult { + let replica = self.get_replica(&req.doc_id)?; + let key = match req.mode { + ShareMode::Read => { + // TODO: support readonly docs + // *replica.namespace().as_bytes() + return Err(anyhow!("creating read-only shares is not yet supported").into()); + } + ShareMode::Write => replica.secret_key(), + }; + let me = PeerSource::from_endpoint(&self.endpoint).await?; + self.start_sync(replica.namespace(), vec![]).await?; + Ok(DocShareResponse { key, peer: me }) + } + + pub async fn doc_import(&self, req: DocImportRequest) -> RpcResult { + let DocImportRequest { key, peers } = req; + // TODO: support read-only docs + // if let Ok(namespace) = match NamespaceId::from_bytes(&key) {}; + let namespace = Namespace::from_bytes(&key); + let id = namespace.id(); + let replica = self.store.new_replica(namespace)?; + self.start_sync(replica.namespace(), peers).await?; + Ok(DocImportResponse { doc_id: id }) + } + + pub async fn doc_start_sync( + &self, + req: DocStartSyncRequest, + ) -> RpcResult { + let DocStartSyncRequest { doc_id, peers } = req; + let replica = self.get_replica(&doc_id)?; + self.start_sync(replica.namespace(), peers).await?; + Ok(DocStartSyncResponse {}) + } + + pub async fn doc_set( + &self, + bao_store: &B, + req: DocSetRequest, + ) -> RpcResult { + let DocSetRequest { + doc_id, + author_id, + key, + value, + } = req; + let replica = self.get_replica(&doc_id)?; + let author = self.get_author(&author_id)?; + let len = value.len(); + let hash = bao_store.import_bytes(value.into()).await?; + replica + .insert(&key, &author, hash, len as u64) + .map_err(Into::into)?; + let entry = self + .store + .get_latest_by_key_and_author(replica.namespace(), author.id(), &key)? + .ok_or_else(|| anyhow!("failed to get entry after insertion"))?; + Ok(DocSetResponse { entry }) + } + + pub fn doc_get(&self, req: DocGetRequest) -> impl Stream> { + let DocGetRequest { doc_id, filter } = req; + let (tx, rx) = flume::bounded(16); + let store = self.store.clone(); + self.rt.main().spawn_blocking(move || { + let ite = store.get(doc_id, filter); + let ite = inline_result(ite).map_ok(|entry| DocGetResponse { entry }); + for entry in ite { + if let Err(_err) = tx.send(entry) { + break; + } + } + }); + rx.into_stream() + } +} + +fn inline_result( + ite: Result>>, impl Into>, +) -> impl Iterator> { + match ite { + Ok(ite) => itertools::Either::Left(ite.map(|item| item.map_err(|err| err.into()))), + Err(err) => itertools::Either::Right(Some(Err(err.into())).into_iter()), + } +} + +fn inline_error( + ite: Result, impl Into>, +) -> impl Iterator> { + match ite { + Ok(ite) => itertools::Either::Left(ite.map(|item| Ok(item))), + Err(err) => itertools::Either::Right(Some(Err(err.into())).into_iter()), + } +} diff --git a/iroh/tests/provide.rs b/iroh/tests/provide.rs index 14a83028d4..2a6dcf26d8 100644 --- a/iroh/tests/provide.rs +++ b/iroh/tests/provide.rs @@ -37,6 +37,7 @@ use iroh_bytes::{ util::runtime, Hash, }; +use iroh_sync::store; /// Pick up the tokio runtime from the thread local and add a /// thread per core runtime. @@ -47,8 +48,9 @@ fn test_runtime() -> runtime::Handle { fn test_node( db: D, addr: SocketAddr, -) -> Builder { - Node::builder(db) +) -> Builder { + let store = iroh_sync::store::memory::Store::default(); + Node::builder(db, store) .collection_parser(IrohCollectionParser) .bind_addr(addr) } From 7784510ccd1f96531300dfd7a308db4e03a4455b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 9 Aug 2023 15:53:17 +0200 Subject: [PATCH 048/172] add ticket for doc share, and concrete type for node client/controller --- iroh/src/client.rs | 10 ++++---- iroh/src/commands/sync.rs | 44 +++++++++++++++++++++++------------ iroh/src/node.rs | 10 ++++---- iroh/src/rpc_protocol.rs | 48 ++++++++++++++++++++++++++++++++------- iroh/src/sync/rpc.rs | 10 +++++--- 5 files changed, 87 insertions(+), 35 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 0cd686e200..9a31c36c56 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -15,12 +15,12 @@ use iroh_sync::store::{GetFilter, KeyFilter}; use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; use quic_rpc::{RpcClient, ServiceConnection}; +use crate::rpc_protocol::ProviderService; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorListRequest, BytesGetRequest, DocGetRequest, DocImportRequest, - DocSetRequest, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocsCreateRequest, - DocsListRequest, ShareMode, + DocSetRequest, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocTicket, + DocsCreateRequest, DocsListRequest, ShareMode, }; -use crate::rpc_protocol::{KeyBytes, ProviderService}; use crate::sync::PeerSource; /// Iroh client @@ -54,8 +54,8 @@ where Ok(doc) } - pub async fn import_doc(&self, key: KeyBytes, peers: Vec) -> Result> { - let res = self.rpc.rpc(DocImportRequest { key, peers }).await??; + pub async fn import_doc(&self, ticket: DocTicket) -> Result> { + let res = self.rpc.rpc(DocImportRequest(ticket)).await??; let doc = Doc { id: res.doc_id, rpc: self.rpc.clone(), diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index d3a704b197..5c69da1ec5 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -1,9 +1,8 @@ -use anyhow::anyhow; use clap::Parser; use futures::TryStreamExt; use indicatif::HumanBytes; use iroh::{ - rpc_protocol::{ProviderRequest, ProviderResponse, ShareMode}, + rpc_protocol::{DocShareResponse, DocTicket, ProviderRequest, ProviderResponse, ShareMode}, sync::PeerSource, }; use iroh_sync::{ @@ -73,11 +72,12 @@ impl Author { pub enum Docs { List, Create, - Import { - key: String, - #[clap(short, long)] - peers: Vec, - }, + // Import { + // key: String, + // #[clap(short, long)] + // peers: Vec, + // }, + Import { ticket: DocTicket }, } impl Docs { @@ -87,11 +87,16 @@ impl Docs { let doc = iroh.create_doc().await?; println!("created {}", doc.id()); } - Docs::Import { key, peers } => { - let key = hex::decode(key)? - .try_into() - .map_err(|_| anyhow!("invalid length"))?; - let doc = iroh.import_doc(key, peers).await?; + // Docs::Import { key, peers } => { + // let key = hex::decode(key)? + // .try_into() + // .map_err(|_| anyhow!("invalid length"))?; + // let ticket = DocTicket::new(key, peers); + // let doc = iroh.import_doc(ticket).await?; + // println!("imported {}", doc.id()); + // } + Docs::Import { ticket } => { + let doc = iroh.import_doc(ticket).await?; println!("imported {}", doc.id()); } Docs::List => { @@ -159,9 +164,18 @@ impl Doc { println!("ok"); } Doc::Share { mode } => { - let res = doc.share(mode).await?; - println!("key: {}", hex::encode(res.key)); - println!("me: {}", res.peer); + let DocShareResponse(ticket) = doc.share(mode).await?; + // println!("key: {}", hex::encode(ticket.key)); + // println!( + // "peers: {}", + // ticket + // .peers + // .iter() + // .map(|p| p.to_string()) + // .collect::>() + // .join(", ") + // ); + println!("ticket: {}", ticket); } Doc::Set { author, key, value } => { let key = key.as_bytes().to_vec(); diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 394303f9d2..0572f0db75 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -48,7 +48,7 @@ use once_cell::sync::OnceCell; use quic_rpc::server::RpcChannel; use quic_rpc::transport::flume::FlumeConnection; use quic_rpc::transport::misc::DummyServerEndpoint; -use quic_rpc::{RpcClient, RpcServer, ServiceConnection, ServiceEndpoint}; +use quic_rpc::{RpcClient, RpcServer, ServiceEndpoint}; use tokio::sync::{mpsc, RwLock}; use tokio::task::JoinError; use tokio_util::sync::CancellationToken; @@ -628,12 +628,14 @@ impl Node { /// TODO: remove and replace with client? pub fn controller( &self, - ) -> RpcClient> { + ) -> RpcClient> { RpcClient::new(self.inner.controller.clone()) } - /// - pub fn client(&self) -> super::client::Iroh> { + /// Return a client to control this node over an in-memory channel. + pub fn client( + &self, + ) -> super::client::Iroh> { super::client::Iroh::new(self.controller()) } diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index baca1d95a7..4765f22718 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -9,11 +9,12 @@ //! response, while others like provide have a stream of responses. //! //! Note that this is subject to change. The RPC protocol is not yet stable. -use std::{net::SocketAddr, path::PathBuf}; +use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr}; use bytes::Bytes; use derive_more::{From, TryInto}; use iroh_bytes::{protocol::RequestToken, provider::ShareProgress, Hash}; +use iroh_gossip::proto::util::base32; use iroh_net::tls::PeerId; use iroh_sync::{ @@ -416,12 +417,46 @@ pub struct DocsCreateResponse { } /// todo -#[derive(Serialize, Deserialize, Debug)] -pub struct DocImportRequest { - // either a public or private key +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct DocTicket { + /// either a public or private key pub key: KeyBytes, + /// a list of peers pub peers: Vec, } +impl DocTicket { + /// Create a new doc ticket + pub fn new(key: KeyBytes, peers: Vec) -> Self { + Self { key, peers } + } + pub fn to_bytes(&self) -> anyhow::Result> { + let bytes = postcard::to_stdvec(&self)?; + Ok(bytes) + } + pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { + let slf = postcard::from_bytes(&bytes)?; + Ok(slf) + } +} +impl FromStr for DocTicket { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + Self::from_bytes(&base32::parse_vec(s)?) + } +} +impl fmt::Display for DocTicket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + base32::fmt(&self.to_bytes().expect("failed to serialize")) + ) + } +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocImportRequest(pub DocTicket); impl RpcMsg for DocImportRequest { type Response = RpcResult; @@ -446,10 +481,7 @@ impl RpcMsg for DocShareRequest { /// todo #[derive(Serialize, Deserialize, Debug)] -pub struct DocShareResponse { - pub key: KeyBytes, - pub peer: PeerSource, -} +pub struct DocShareResponse(pub DocTicket); /// todo #[derive(Serialize, Deserialize, Debug)] diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index 9ad1b2b185..f49cd0a47c 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -11,7 +11,8 @@ use crate::rpc_protocol::{ AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, DocGetRequest, DocGetResponse, DocImportRequest, DocImportResponse, DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, - DocsCreateRequest, DocsCreateResponse, DocsListRequest, DocsListResponse, RpcResult, ShareMode, + DocTicket, DocsCreateRequest, DocsCreateResponse, DocsListRequest, DocsListResponse, RpcResult, + ShareMode, }; use super::{engine::SyncEngine, PeerSource}; @@ -67,11 +68,14 @@ impl SyncEngine { }; let me = PeerSource::from_endpoint(&self.endpoint).await?; self.start_sync(replica.namespace(), vec![]).await?; - Ok(DocShareResponse { key, peer: me }) + Ok(DocShareResponse(DocTicket { + key, + peers: vec![me], + })) } pub async fn doc_import(&self, req: DocImportRequest) -> RpcResult { - let DocImportRequest { key, peers } = req; + let DocImportRequest(DocTicket { key, peers }) = req; // TODO: support read-only docs // if let Ok(namespace) = match NamespaceId::from_bytes(&key) {}; let namespace = Namespace::from_bytes(&key); From d1caf194839c4d536061b979312ea734cd958435 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 9 Aug 2023 16:48:56 +0200 Subject: [PATCH 049/172] add minimal client example --- iroh/examples/client.rs | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 iroh/examples/client.rs diff --git a/iroh/examples/client.rs b/iroh/examples/client.rs new file mode 100644 index 0000000000..7e44a739ee --- /dev/null +++ b/iroh/examples/client.rs @@ -0,0 +1,53 @@ +use indicatif::HumanBytes; +use iroh::{database::flat::writable::WritableFileDatabase, node::Node}; +use iroh_bytes::util::runtime; +use iroh_sync::{store::{GetFilter, KeyFilter}, sync::SignedEntry}; +use tokio_stream::StreamExt; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let rt = runtime::Handle::from_currrent(1)?; + let db = WritableFileDatabase::new(dir.path().into()).await?; + let store = iroh_sync::store::memory::Store::default(); + let node = Node::builder(db.db().clone(), store, dir.path().into()) + .runtime(&rt) + .spawn() + .await?; + let client = node.client(); + let doc = client.create_doc().await?; + let author = client.create_author().await?; + let key = b"hello".to_vec(); + let value = b"world".to_vec(); + doc.set_bytes(author, key.clone(), value).await?; + let mut stream = doc + .get(GetFilter { + latest: true, + key: KeyFilter::All, + author: None, + }) + .await?; + while let Some(entry) = stream.try_next().await? { + println!("entry {}", fmt_entry(&entry)); + let content = doc.get_content_bytes(&entry).await?; + println!(" content {}", String::from_utf8(content.to_vec())?) + } + + Ok(()) +} + +fn fmt_entry(entry: &SignedEntry) -> String { + let id = entry.entry().id(); + let key = std::str::from_utf8(id.key()).unwrap_or(""); + let author = fmt_hash(id.author().as_bytes()); + let hash = entry.entry().record().content_hash(); + let hash = fmt_hash(hash.as_bytes()); + let len = HumanBytes(entry.entry().record().content_len()); + format!("@{author}: {key} = {hash} ({len})",) +} + +fn fmt_hash(hash: impl AsRef<[u8]>) -> String { + let mut text = data_encoding::BASE32_NOPAD.encode(&hash.as_ref()[..5]); + text.make_ascii_lowercase(); + format!("{}…", &text) +} From f80617ac0a5dbc6b48f2d9d9d6cf1847879907be Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 9 Aug 2023 17:02:52 +0200 Subject: [PATCH 050/172] add util methods to obtain an iroh client --- iroh/examples/client.rs | 5 +++- iroh/src/client.rs | 65 +++++++++++++++++++++++++++++++++++++++-- iroh/src/commands.rs | 37 +++-------------------- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/iroh/examples/client.rs b/iroh/examples/client.rs index 7e44a739ee..ee9c123311 100644 --- a/iroh/examples/client.rs +++ b/iroh/examples/client.rs @@ -1,7 +1,10 @@ use indicatif::HumanBytes; use iroh::{database::flat::writable::WritableFileDatabase, node::Node}; use iroh_bytes::util::runtime; -use iroh_sync::{store::{GetFilter, KeyFilter}, sync::SignedEntry}; +use iroh_sync::{ + store::{GetFilter, KeyFilter}, + sync::SignedEntry, +}; use tokio_stream::StreamExt; #[tokio::main] diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 9a31c36c56..1e5713cfe9 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -5,24 +5,83 @@ // TODO: fill out docs #![allow(missing_docs)] +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::result::Result as StdResult; +use std::sync::Arc; +use std::time::Duration; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use bytes::Bytes; use futures::{Stream, StreamExt, TryStreamExt}; use iroh_bytes::Hash; use iroh_sync::store::{GetFilter, KeyFilter}; use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; +use quic_rpc::transport::flume::FlumeConnection; +use quic_rpc::transport::quinn::QuinnConnection; use quic_rpc::{RpcClient, ServiceConnection}; -use crate::rpc_protocol::ProviderService; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorListRequest, BytesGetRequest, DocGetRequest, DocImportRequest, DocSetRequest, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocTicket, - DocsCreateRequest, DocsListRequest, ShareMode, + DocsCreateRequest, DocsListRequest, ShareMode, VersionRequest, }; +use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; use crate::sync::PeerSource; +/// In-memory client to an iroh node running in the same process. +/// +/// This is obtained from [`iroh::node::Node::client`]. +pub type IrohMemClient = Iroh>; +/// RPC client to an iroh node running in a seperate process. +/// +/// This is obtained from [`connect`]. +pub type IrohRpcClient = Iroh>; + +/// TODO: Change to "/iroh-rpc/1" +pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; + +/// Connect to an iroh node running on the same computer, but in a different process. +pub async fn connect(rpc_port: u16) -> anyhow::Result { + let client = connect_raw(rpc_port).await?; + Ok(Iroh::new(client)) +} + +/// Create a raw RPC client to an iroh node running on the same computer, but in a different +/// process. +pub async fn connect_raw( + rpc_port: u16, +) -> anyhow::Result< + quic_rpc::RpcClient>, +> { + let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); + let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; + let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); + let server_name = "localhost".to_string(); + let connection = QuinnConnection::new(endpoint, addr, server_name); + let client = RpcClient::new(connection); + // Do a version request to check if the server is running. + let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) + .await + .context("iroh server is not running")??; + Ok(client) +} +fn create_quinn_client( + bind_addr: SocketAddr, + alpn_protocols: Vec>, + keylog: bool, +) -> Result { + let keypair = iroh_net::tls::Keypair::generate(); + let tls_client_config = + iroh_net::tls::make_client_config(&keypair, None, alpn_protocols, keylog)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); + let mut endpoint = quinn::Endpoint::client(bind_addr)?; + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(1))); + client_config.transport_config(Arc::new(transport_config)); + endpoint.set_default_client_config(client_config); + Ok(endpoint) +} + /// Iroh client pub struct Iroh { rpc: RpcClient, diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 54205ee2be..ead1ad320e 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -1,12 +1,10 @@ -use std::net::{Ipv4Addr, SocketAddrV4}; use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; use std::{net::SocketAddr, path::PathBuf}; -use anyhow::{Context, Result}; +use anyhow::Result; use clap::{Parser, Subcommand}; use futures::StreamExt; +use iroh::client::connect_raw; use iroh::dial::Ticket; use iroh::rpc_protocol::*; use iroh_bytes::{protocol::RequestToken, util::runtime, Hash}; @@ -407,35 +405,8 @@ pub enum Commands { }, } -async fn make_rpc_client(rpc_port: u16) -> anyhow::Result { - let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); - let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; - let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); - let server_name = "localhost".to_string(); - let connection = QuinnConnection::new(endpoint, addr, server_name); - let client = RpcClient::new(connection); - // Do a version request to check if the server is running. - let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) - .await - .context("iroh server is not running")??; - Ok(client) -} - -pub fn create_quinn_client( - bind_addr: SocketAddr, - alpn_protocols: Vec>, - keylog: bool, -) -> Result { - let keypair = iroh_net::tls::Keypair::generate(); - let tls_client_config = - iroh_net::tls::make_client_config(&keypair, None, alpn_protocols, keylog)?; - let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); - let mut endpoint = quinn::Endpoint::client(bind_addr)?; - let mut transport_config = quinn::TransportConfig::default(); - transport_config.keep_alive_interval(Some(Duration::from_secs(1))); - client_config.transport_config(Arc::new(transport_config)); - endpoint.set_default_client_config(client_config); - Ok(endpoint) +pub async fn make_rpc_client(rpc_port: u16) -> anyhow::Result { + connect_raw(rpc_port).await } #[cfg(feature = "metrics")] From 134477a9d858388b17555a6c0f04db2b2c295004 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 9 Aug 2023 17:23:23 +0200 Subject: [PATCH 051/172] fix: return DocTicket from share --- iroh/src/client.rs | 8 ++++---- iroh/src/commands/sync.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 1e5713cfe9..11843a6145 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -22,8 +22,8 @@ use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorListRequest, BytesGetRequest, DocGetRequest, DocImportRequest, - DocSetRequest, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocTicket, - DocsCreateRequest, DocsListRequest, ShareMode, VersionRequest, + DocSetRequest, DocShareRequest, DocStartSyncRequest, DocTicket, DocsCreateRequest, + DocsListRequest, ShareMode, VersionRequest, }; use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; use crate::sync::PeerSource; @@ -207,7 +207,7 @@ where Ok(flatten(stream).map_ok(|res| res.entry)) } - pub async fn share(&self, mode: ShareMode) -> anyhow::Result { + pub async fn share(&self, mode: ShareMode) -> anyhow::Result { let res = self .rpc .rpc(DocShareRequest { @@ -215,7 +215,7 @@ where mode, }) .await??; - Ok(res) + Ok(res.0) } pub async fn start_sync(&self, peers: Vec) -> Result<()> { diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 5c69da1ec5..ab3c7559d7 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -164,7 +164,7 @@ impl Doc { println!("ok"); } Doc::Share { mode } => { - let DocShareResponse(ticket) = doc.share(mode).await?; + let ticket = doc.share(mode).await?; // println!("key: {}", hex::encode(ticket.key)); // println!( // "peers: {}", From 5880f50e94f8c2374143dae54a8dd294e9f5e322 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 9 Aug 2023 18:24:47 +0200 Subject: [PATCH 052/172] feat: expose metrics over rpc --- Cargo.lock | 1 + iroh-sync/Cargo.toml | 4 +- iroh-sync/src/lib.rs | 38 ++++++++++++ {iroh/src/sync => iroh-sync/src}/metrics.rs | 0 iroh-sync/src/sync.rs | 18 ++++++ iroh/src/commands.rs | 12 +--- iroh/src/commands/sync.rs | 2 +- iroh/src/metrics.rs | 64 +++++++++++++++++++++ iroh/src/node.rs | 37 +++++++++++- iroh/src/rpc_protocol.rs | 28 ++++++++- iroh/src/sync.rs | 28 ++++++++- iroh/src/sync/live.rs | 8 --- 12 files changed, 213 insertions(+), 27 deletions(-) rename {iroh/src/sync => iroh-sync/src}/metrics.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 300e865321..ef630f708b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1964,6 +1964,7 @@ dependencies = [ "hex", "iroh-blake3", "iroh-bytes", + "iroh-metrics", "once_cell", "ouroboros", "parking_lot", diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 8ab9a1de20..70a1165781 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -15,6 +15,7 @@ crossbeam = "0.8.2" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } ed25519-dalek = { version = "2.0.0-rc.2", features = ["serde", "rand_core"] } iroh-bytes = { version = "0.5.0", path = "../iroh-bytes" } +iroh-metrics = { version = "0.5.0", path = "../iroh-metrics", optional = true } once_cell = "1.18.0" postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } rand = "0.8.5" @@ -34,5 +35,6 @@ tokio = { version = "1.28.2", features = ["sync", "macros"] } tempfile = "3.4" [features] -default = ["fs-store"] +default = ["fs-store", "metrics"] fs-store = ["redb", "ouroboros"] +metrics = ["iroh-metrics"] diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index 4c73579e2d..f0beb090b1 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -1,3 +1,41 @@ pub mod ranger; pub mod store; pub mod sync; +#[cfg(feature = "metrics")] +pub mod metrics; + +use iroh_metrics::{ + core::{Counter, Metric}, + struct_iterable::Iterable, +}; + +/// Metrics for iroh-sync +#[allow(missing_docs)] +#[derive(Debug, Clone, Iterable)] +pub struct Metrics { + pub new_entries_local: Counter, + pub new_entries_remote: Counter, + pub new_entries_local_size: Counter, + pub new_entries_remote_size: Counter, + pub initial_sync_success: Counter, + pub initial_sync_failed: Counter, +} + +impl Default for Metrics { + fn default() -> Self { + Self { + new_entries_local: Counter::new("Number of document entries added locally"), + new_entries_remote: Counter::new("Number of document entries added by peers"), + new_entries_local_size: Counter::new("Total size of entry contents added locally"), + new_entries_remote_size: Counter::new("Total size of entry contents added by peers"), + initial_sync_success: Counter::new("Number of successfull initial syncs "), + initial_sync_failed: Counter::new("Number of failed initial syncs"), + } + } +} + +impl Metric for Metrics { + fn name() -> &'static str { + "iroh-sync" + } +} diff --git a/iroh/src/sync/metrics.rs b/iroh-sync/src/metrics.rs similarity index 100% rename from iroh/src/sync/metrics.rs rename to iroh-sync/src/metrics.rs diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 84cbd513c6..1782dd90ff 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -13,6 +13,11 @@ use std::{ time::SystemTime, }; +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; +#[cfg(feature = "metrics")] +use iroh_metrics::{inc, inc_by}; + use parking_lot::RwLock; use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey}; @@ -316,6 +321,13 @@ impl> Replica { for cb in on_insert.values() { cb(InsertOrigin::Local, signed_entry.clone()); } + + #[cfg(feature = "metrics")] + { + inc!(Metrics, new_entries_local); + inc_by!(Metrics, new_entries_local_size, len); + } + Ok(()) } @@ -354,6 +366,12 @@ impl> Replica { for cb in on_insert.values() { cb(InsertOrigin::Sync(received_from), entry.clone()); } + #[cfg(feature = "metrics")] + { + inc!(Metrics, new_entries_remote); + inc_by!(Metrics, new_entries_remote_size, entry.content_len()); + } + Ok(()) } diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index ead1ad320e..fe6e146252 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -414,18 +414,10 @@ pub fn init_metrics_collection( metrics_addr: Option, rt: &iroh_bytes::util::runtime::Handle, ) -> Option> { - use iroh_metrics::core::Metric; - // doesn't start the server if the address is None if let Some(metrics_addr) = metrics_addr { - iroh_metrics::core::Core::init(|reg, metrics| { - metrics.insert(iroh::metrics::Metrics::new(reg)); - metrics.insert(iroh_net::metrics::MagicsockMetrics::new(reg)); - metrics.insert(iroh_net::metrics::NetcheckMetrics::new(reg)); - metrics.insert(iroh_net::metrics::PortmapMetrics::new(reg)); - metrics.insert(iroh_net::metrics::DerpMetrics::new(reg)); - }); - + // metrics are initilaized in iroh::node::Node::spawn + // here we only start the server return Some(rt.main().spawn(async move { if let Err(e) = iroh_metrics::metrics::start_metrics_server(metrics_addr).await { eprintln!("Failed to start metrics server: {e}"); diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index ab3c7559d7..ab3e230bec 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -2,7 +2,7 @@ use clap::Parser; use futures::TryStreamExt; use indicatif::HumanBytes; use iroh::{ - rpc_protocol::{DocShareResponse, DocTicket, ProviderRequest, ProviderResponse, ShareMode}, + rpc_protocol::{DocTicket, ProviderRequest, ProviderResponse, ShareMode}, sync::PeerSource, }; use iroh_sync::{ diff --git a/iroh/src/metrics.rs b/iroh/src/metrics.rs index 74355f2a09..69d7064dc1 100644 --- a/iroh/src/metrics.rs +++ b/iroh/src/metrics.rs @@ -1,8 +1,12 @@ +use std::collections::HashMap; + use iroh_metrics::{ core::{Counter, Metric}, struct_iterable::Iterable, }; +use crate::rpc_protocol::CounterStats; + /// Enum of metrics for the module #[allow(missing_docs)] #[derive(Debug, Clone, Iterable)] @@ -15,6 +19,8 @@ pub struct Metrics { pub downloads_success: Counter, pub downloads_error: Counter, pub downloads_notfound: Counter, + pub initial_sync_success: Counter, + pub initial_sync_failed: Counter, } impl Default for Metrics { @@ -28,6 +34,8 @@ impl Default for Metrics { downloads_success: Counter::new("Total number of successfull downloads"), downloads_error: Counter::new("Total number of downloads failed with error"), downloads_notfound: Counter::new("Total number of downloads failed with not found"), + initial_sync_success: Counter::new("Number of successfull initial syncs "), + initial_sync_failed: Counter::new("Number of failed initial syncs"), } } } @@ -37,3 +45,59 @@ impl Metric for Metrics { "Iroh" } } + +/// Initialize the metrics collection. +pub fn init_metrics_collection() { + iroh_metrics::core::Core::init(|reg, metrics| { + metrics.insert(crate::metrics::Metrics::new(reg)); + metrics.insert(iroh_sync::metrics::Metrics::new(reg)); + metrics.insert(iroh_net::metrics::MagicsockMetrics::new(reg)); + metrics.insert(iroh_net::metrics::NetcheckMetrics::new(reg)); + metrics.insert(iroh_net::metrics::PortmapMetrics::new(reg)); + metrics.insert(iroh_net::metrics::DerpMetrics::new(reg)); + }); +} + +/// Collect the current metrics into a hash map. +/// +/// TODO: Only counters are supported for now, other metrics will be skipped without error. +pub fn get_metrics() -> anyhow::Result> { + let mut map = HashMap::new(); + let core = + iroh_metrics::core::Core::get().ok_or_else(|| anyhow::anyhow!("metrics are disabled"))?; + collect( + core.get_collector::(), + &mut map, + ); + collect( + core.get_collector::(), + &mut map, + ); + collect( + core.get_collector::(), + &mut map, + ); + collect( + core.get_collector::(), + &mut map, + ); + collect( + core.get_collector::(), + &mut map, + ); + Ok(map) +} + +// TODO: support other things than counters +fn collect(metrics: Option<&impl Iterable>, map: &mut HashMap) { + let Some(metrics) = metrics else { + return; + }; + for (name, counter) in metrics.iter() { + if let Some(counter) = counter.downcast_ref::() { + let value = counter.get(); + let description = counter.description.to_string(); + map.insert(name.to_string(), CounterStats { value, description }); + } + } +} diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 0572f0db75..7994ce4fac 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -60,8 +60,9 @@ use crate::rpc_protocol::{ AddrsRequest, AddrsResponse, BytesGetRequest, BytesGetResponse, IdRequest, IdResponse, ListBlobsRequest, ListBlobsResponse, ListCollectionsRequest, ListCollectionsResponse, ListIncompleteBlobsRequest, ListIncompleteBlobsResponse, ProvideRequest, ProviderRequest, - ProviderResponse, ProviderService, ShareRequest, ShutdownRequest, ValidateRequest, - VersionRequest, VersionResponse, WatchRequest, WatchResponse, + ProviderResponse, ProviderService, ShareRequest, ShutdownRequest, StatsGetRequest, + StatsGetResponse, ValidateRequest, VersionRequest, VersionResponse, WatchRequest, + WatchResponse, }; use crate::sync::{SyncEngine, SYNC_ALPN}; @@ -76,6 +77,21 @@ pub const DEFAULT_BIND_ADDR: (Ipv4Addr, u16) = (Ipv4Addr::LOCALHOST, 11204); /// How long we wait at most for some endpoints to be discovered. const ENDPOINT_WAIT: Duration = Duration::from_secs(5); +#[cfg(feature = "metrics")] +/// Initialize the metrics collection. +pub fn init_metrics_collection() { + use iroh_metrics::core::Metric; + + iroh_metrics::core::Core::init(|reg, metrics| { + metrics.insert(crate::metrics::Metrics::new(reg)); + metrics.insert(iroh_sync::metrics::Metrics::new(reg)); + metrics.insert(iroh_net::metrics::MagicsockMetrics::new(reg)); + metrics.insert(iroh_net::metrics::NetcheckMetrics::new(reg)); + metrics.insert(iroh_net::metrics::PortmapMetrics::new(reg)); + metrics.insert(iroh_net::metrics::DerpMetrics::new(reg)); + }); +} + /// Builder for the [`Node`]. /// /// You must supply a blob store. Various store implementations are available @@ -282,6 +298,10 @@ where trace!("spawning node"); let rt = self.rt.context("runtime not set")?; + // TODO: this should actually run globally only once. + #[cfg(feature = "metrics")] + init_metrics_collection(); + let (endpoints_update_s, endpoints_update_r) = flume::bounded(1); let mut transport_config = quinn::TransportConfig::default(); transport_config @@ -1043,6 +1063,18 @@ impl RpcHandler { anyhow::bail!("collections not supported"); } + async fn stats(self, _req: StatsGetRequest) -> RpcResult { + #[cfg(feature = "metrics")] + let res = Ok(StatsGetResponse { + stats: crate::metrics::get_metrics()?, + }); + + #[cfg(not(feature = "metrics"))] + let res = Err(anyhow::anyhow!("metrics are disabled").into()); + + res + } + async fn version(self, _: VersionRequest) -> VersionResponse { VersionResponse { version: env!("CARGO_PKG_VERSION").to_string(), @@ -1158,6 +1190,7 @@ fn handle_rpc_request< Id(msg) => chan.rpc(msg, handler, RpcHandler::id).await, Addrs(msg) => chan.rpc(msg, handler, RpcHandler::addrs).await, Shutdown(msg) => chan.rpc(msg, handler, RpcHandler::shutdown).await, + Stats(msg) => chan.rpc(msg, handler, RpcHandler::stats).await, Validate(msg) => { chan.server_streaming(msg, handler, RpcHandler::validate) .await diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 4765f22718..f245d3ec13 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -9,7 +9,7 @@ //! response, while others like provide have a stream of responses. //! //! Note that this is subject to change. The RPC protocol is not yet stable. -use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr}; +use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr, collections::HashMap}; use bytes::Bytes; use derive_more::{From, TryInto}; @@ -555,6 +555,26 @@ pub struct BytesGetResponse { pub data: Bytes, } +#[derive(Serialize, Deserialize, Debug)] +pub struct StatsGetRequest {} + +impl RpcMsg for StatsGetRequest { + type Response = RpcResult; +} + +/// Counter stats +#[derive(Serialize, Deserialize, Debug)] +pub struct CounterStats { + pub value: u64, + pub description: String, +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct StatsGetResponse { + pub stats: HashMap, +} + /// The RPC service for the iroh provider process. #[derive(Debug, Clone)] pub struct ProviderService; @@ -593,6 +613,8 @@ pub enum ProviderRequest { DocShare(DocShareRequest), // DocGetContent(DocGetContentRequest), BytesGet(BytesGetRequest), + + Stats(StatsGetRequest), } /// The response enum, listing all possible responses. @@ -630,7 +652,9 @@ pub enum ProviderResponse { DocJoin(RpcResult), DocShare(RpcResult), - BytesGet(RpcResult), // DocGetContent(DocGetContentResponse), + BytesGet(RpcResult), + + Stats(RpcResult), } impl Service for ProviderService { diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index d78ae7e388..f7938af48a 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -13,12 +13,17 @@ use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{debug, trace}; +#[cfg(feature = "metrics")] +use crate::metrics::Metrics; +#[cfg(feature = "metrics")] +use iroh_metrics::inc; + + /// The ALPN identifier for the iroh-sync protocol pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; mod engine; mod live; -pub mod metrics; pub mod rpc; pub use engine::*; @@ -56,6 +61,14 @@ pub async fn connect_and_sync( .context("dial_and_sync")?; let (mut send_stream, mut recv_stream) = connection.open_bi().await?; let res = run_alice::(&mut send_stream, &mut recv_stream, doc, Some(peer_id)).await; + + #[cfg(feature = "metrics")] + if res.is_ok() { + inc!(Metrics, initial_sync_success); + } else { + inc!(Metrics, initial_sync_failed); + } + debug!("sync with peer {}: finish {:?}", peer_id, res); res } @@ -111,13 +124,22 @@ pub async fn handle_connection( let (mut send_stream, mut recv_stream) = connection.accept_bi().await?; debug!(peer = ?peer_id, "incoming sync: start"); - run_bob( + let res = run_bob( &mut send_stream, &mut recv_stream, replica_store, Some(peer_id), ) - .await?; + .await; + + #[cfg(feature = "metrics")] + if res.is_ok() { + inc!(Metrics, initial_sync_success); + } else { + inc!(Metrics, initial_sync_failed); + } + + res?; send_stream.finish().await?; debug!(peer = ?peer_id, "incoming sync: done"); diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 98283ab840..ed4d5f223f 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -12,7 +12,6 @@ use iroh_gossip::{ net::{Event, Gossip}, proto::TopicId, }; -use iroh_metrics::inc; use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::{ store, @@ -22,8 +21,6 @@ use serde::{Deserialize, Serialize}; use tokio::{sync::mpsc, task::JoinError}; use tracing::{debug, error, info, warn}; -use super::metrics::Metrics; - const CHANNEL_CAP: usize = 8; /// The address to connect to a peer @@ -274,11 +271,6 @@ impl Actor { // TODO: Make sure that the peer is dialable. let res = connect_and_sync::(&endpoint, &replica, peer, None, &[]).await; debug!("synced with {peer}: {res:?}"); - // collect metrics - match &res { - Ok(_) => inc!(Metrics, initial_sync_success), - Err(_) => inc!(Metrics, initial_sync_failed), - } (topic, peer, res) } .boxed() From e911696f8ad0dbe01b1a5bfeabdcfc75cd8264db Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 9 Aug 2023 18:28:05 +0200 Subject: [PATCH 053/172] expose stats in rpc client --- iroh-sync/src/lib.rs | 4 ++-- iroh/src/client.rs | 12 +++++++++--- iroh/src/rpc_protocol.rs | 2 +- iroh/src/sync.rs | 1 - 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index f0beb090b1..d236cc7330 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -1,8 +1,8 @@ +#[cfg(feature = "metrics")] +pub mod metrics; pub mod ranger; pub mod store; pub mod sync; -#[cfg(feature = "metrics")] -pub mod metrics; use iroh_metrics::{ core::{Counter, Metric}, diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 11843a6145..e3be3ac587 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -5,6 +5,7 @@ // TODO: fill out docs #![allow(missing_docs)] +use std::collections::HashMap; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::result::Result as StdResult; use std::sync::Arc; @@ -21,9 +22,9 @@ use quic_rpc::transport::quinn::QuinnConnection; use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ - AuthorCreateRequest, AuthorListRequest, BytesGetRequest, DocGetRequest, DocImportRequest, - DocSetRequest, DocShareRequest, DocStartSyncRequest, DocTicket, DocsCreateRequest, - DocsListRequest, ShareMode, VersionRequest, + AuthorCreateRequest, AuthorListRequest, BytesGetRequest, CounterStats, DocGetRequest, + DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocTicket, + DocsCreateRequest, DocsListRequest, ShareMode, StatsGetRequest, VersionRequest, }; use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; use crate::sync::PeerSource; @@ -141,6 +142,11 @@ where let res = self.rpc.rpc(BytesGetRequest { hash }).await??; Ok(res.data) } + + pub async fn stats(&self) -> Result> { + let res = self.rpc.rpc(StatsGetRequest {}).await??; + Ok(res.stats) + } } /// Document handle diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index f245d3ec13..adf5f1bb1f 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -9,7 +9,7 @@ //! response, while others like provide have a stream of responses. //! //! Note that this is subject to change. The RPC protocol is not yet stable. -use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr, collections::HashMap}; +use std::{collections::HashMap, fmt, net::SocketAddr, path::PathBuf, str::FromStr}; use bytes::Bytes; use derive_more::{From, TryInto}; diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index f7938af48a..887411e120 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -18,7 +18,6 @@ use crate::metrics::Metrics; #[cfg(feature = "metrics")] use iroh_metrics::inc; - /// The ALPN identifier for the iroh-sync protocol pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; From d021455adbba345bf468bd344653b58e2736be5c Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 12:02:14 +0200 Subject: [PATCH 054/172] fix sync example after rebase --- iroh/examples/sync.rs | 106 ++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 77622140ef..3efa01228f 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -18,20 +18,19 @@ use clap::{CommandFactory, FromArgMatches, Parser}; use ed25519_dalek::SigningKey; use indicatif::HumanBytes; use iroh::{ - database::flat::writable::WritableFileDatabase, download::Downloader, sync::{PeerSource, SyncEngine, SYNC_ALPN}, }; use iroh_bytes::util::runtime; +use iroh_bytes::{ + baomap::{ImportMode, Map, MapEntry, Store as BaoStore}, + util::progress::IgnoreProgressSender, +}; use iroh_gossip::{ net::{Gossip, GOSSIP_ALPN}, proto::TopicId, }; use iroh_io::AsyncSliceReaderExt; -use iroh_metrics::{ - core::{Counter, Metric}, - struct_iterable::Iterable, -}; use iroh_net::{ defaults::default_derp_map, derp::DerpMap, magic_endpoint::get_alpn, tls::Keypair, MagicEndpoint, @@ -99,11 +98,7 @@ async fn main() -> anyhow::Result<()> { pub fn init_metrics_collection( metrics_addr: Option, ) -> Option> { - iroh_metrics::core::Core::init(|reg, metrics| { - metrics.insert(iroh::sync::metrics::Metrics::new(reg)); - metrics.insert(iroh_gossip::metrics::Metrics::new(reg)); - }); - + iroh::metrics::init_metrics_collection(); // doesn't start the server if the address is None if let Some(metrics_addr) = metrics_addr { return Some(tokio::spawn(async move { @@ -214,7 +209,7 @@ async fn run(args: Args) -> anyhow::Result<()> { let name = format!("iroh-sync-{}", endpoint.peer_id()); let dir = std::env::temp_dir().join(name); if !dir.exists() { - std::fs::create_dir(&dir).expect("failed to create temp dir"); + std::fs::create_dir_all(&dir).expect("failed to create temp dir"); } dir }); @@ -228,6 +223,11 @@ async fn run(args: Args) -> anyhow::Result<()> { let docs_path = storage_path.join("docs.db"); let docs = iroh_sync::store::fs::Store::new(&docs_path)?; + // create a bao store for the iroh-bytes blobs + let blob_path = storage_path.join("blobs"); + std::fs::create_dir_all(&blob_path)?; + let db = iroh::baomap::flat::Store::load(&blob_path, &blob_path, &rt).await?; + // create the live syncer let downloader = Downloader::new(rt.clone(), endpoint.clone(), db.clone()); let live_sync = SyncEngine::spawn( @@ -243,7 +243,7 @@ async fn run(args: Args) -> anyhow::Result<()> { let state = Arc::new(State { gossip: gossip.clone(), docs: docs.clone(), - bytes: IrohBytesHandlers::new(rt.clone(), db.db().clone()), + bytes: IrohBytesHandlers::new(rt.clone(), db.clone()), }); // spawn our endpoint loop that forwards incoming connections @@ -307,9 +307,6 @@ async fn run(args: Args) -> anyhow::Result<()> { if let Err(err) = live_sync.shutdown().await { println!("> syncer closed with error: {err:?}"); } - println!("> persisting document and blob database at {storage_path:?}"); - db.save().await?; - if let Some(metrics_fut) = metrics_fut { metrics_fut.abort(); drop(metrics_fut); @@ -324,15 +321,17 @@ async fn handle_command( store: &store::fs::Store, author: &Author, doc: &Doc, - db: &WritableFileDatabase, + db: &iroh::baomap::flat::Store, ticket: &Ticket, log_filter: &LogLevelReload, current_watch: &Arc>>, ) -> anyhow::Result<()> { match cmd { Cmd::Set { key, value } => { - let (hash, len) = db.put_bytes(value.into_bytes().into()).await?; - doc.insert(key, author, hash, len)?; + let value = value.into_bytes(); + let len = value.len(); + let hash = db.import_bytes(value.into()).await?; + doc.insert(key, author, hash, len as u64)?; } Cmd::Get { key, @@ -413,10 +412,11 @@ async fn handle_command( let author = author.clone(); let handle = rt.main().spawn(async move { for i in 0..count { - let value = String::from_utf8(bytes.clone()).unwrap(); + let value = String::from_utf8(bytes.clone()).unwrap().into_bytes(); + let len = value.len(); let key = format!("{}/{}/{}", prefix, t, i); - let (hash, len) = db.put_bytes(value.into_bytes().into()).await?; - doc.insert(key, &author, hash, len)?; + let hash = db.import_bytes(value.into()).await?; + doc.insert(key, &author, hash, len as u64)?; } Ok(count) }); @@ -467,15 +467,20 @@ async fn handle_command( async fn handle_fs_command( cmd: FsCmd, store: &store::fs::Store, - db: &WritableFileDatabase, + db: &iroh::baomap::flat::Store, doc: &Doc, author: &Author, ) -> anyhow::Result<()> { match cmd { FsCmd::ImportFile { file_path, key } => { let file_path = canonicalize_path(&file_path)?.canonicalize()?; - let reader = tokio::fs::File::open(&file_path).await?; - let (hash, len) = db.put_reader(reader).await?; + let (hash, len) = db + .import( + file_path.clone(), + ImportMode::Copy, + IgnoreProgressSender::default(), + ) + .await?; doc.insert(key, author, hash, len)?; println!( "> imported {file_path:?}: {} ({})", @@ -502,8 +507,13 @@ async fn handle_fs_command( continue; } let key = format!("{key_prefix}/{relative}"); - let reader = tokio::fs::File::open(file.path()).await?; - let (hash, len) = db.put_reader(reader).await?; + let (hash, len) = db + .import( + file.path().into(), + ImportMode::Copy, + IgnoreProgressSender::default(), + ) + .await?; doc.insert(key, author, hash, len)?; println!( "> imported {relative}: {} ({})", @@ -529,7 +539,7 @@ async fn handle_fs_command( let key = id.key(); let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; let len = entry.entry().record().content_len(); - let blob = db.db().get(entry.content_hash()); + let blob = db.get(entry.content_hash()); if let Some(blob) = blob { let mut reader = blob.data_reader().await?; let path = root.join(&relative); @@ -559,7 +569,6 @@ async fn handle_fs_command( tokio::fs::create_dir_all(&parent).await?; let mut file = tokio::fs::File::create(&path).await?; let blob = db - .db() .get(entry.content_hash()) .ok_or_else(|| anyhow!(format!("content for {key} is not available")))?; let mut reader = blob.data_reader().await?; @@ -775,27 +784,15 @@ fn repl_loop(cmd_tx: mpsc::Sender<(Cmd, oneshot::Sender)>) -> anyhow::Re } fn get_stats() { - let core = iroh_metrics::core::Core::get().expect("Metrics core not initialized"); - println!("# sync"); - let metrics = core - .get_collector::() - .unwrap(); - fmt_metrics(metrics); - println!("# gossip"); - let metrics = core - .get_collector::() - .unwrap(); - fmt_metrics(metrics); -} - -fn fmt_metrics(metrics: &impl Iterable) { - for (name, counter) in metrics.iter() { - if let Some(counter) = counter.downcast_ref::() { - let value = counter.get(); - println!("{name:23} : {value:>6} ({})", counter.description); - } else { - println!("{name:23} : unsupported metric kind"); - } + let Ok(stats) = iroh::metrics::get_metrics() else { + println!("metrics collection is disabled"); + return; + }; + for (name, details) in stats.iter() { + println!( + "{:23} : {:>6} ({})", + name, details.value, details.description + ); } } @@ -865,7 +862,7 @@ async fn fmt_content_simple(_doc: &Doc, entry: &SignedEntry) -> String { format!("<{}>", HumanBytes(len)) } -async fn fmt_content(db: &WritableFileDatabase, entry: &SignedEntry) -> String { +async fn fmt_content(db: &B, entry: &SignedEntry) -> String { let len = entry.entry().record().content_len(); if len > MAX_DISPLAY_CONTENT_LEN { format!("<{}>", HumanBytes(len)) @@ -880,9 +877,8 @@ async fn fmt_content(db: &WritableFileDatabase, entry: &SignedEntry) -> String { } } -async fn read_content(db: &WritableFileDatabase, entry: &SignedEntry) -> anyhow::Result { +async fn read_content(db: &B, entry: &SignedEntry) -> anyhow::Result { let data = db - .db() .get(entry.content_hash()) .ok_or_else(|| anyhow!("not found"))? .data_reader() @@ -957,18 +953,18 @@ mod iroh_bytes_handlers { provider::{CustomGetHandler, EventSender, RequestAuthorizationHandler}, }; - use iroh::{collection::IrohCollectionParser, database::flat::Database}; + use iroh::collection::IrohCollectionParser; #[derive(Debug, Clone)] pub struct IrohBytesHandlers { - db: Database, + db: iroh::baomap::flat::Store, rt: iroh_bytes::util::runtime::Handle, event_sender: NoopEventSender, get_handler: Arc, auth_handler: Arc, } impl IrohBytesHandlers { - pub fn new(rt: iroh_bytes::util::runtime::Handle, db: Database) -> Self { + pub fn new(rt: iroh_bytes::util::runtime::Handle, db: iroh::baomap::flat::Store) -> Self { Self { db, rt, From c78fb5b88ee8b3ffc79188606de4cc33c1506426 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 13:18:13 +0200 Subject: [PATCH 055/172] feat: reimplement download --- iroh/src/download.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 5fd6222511..70ab2b2e2d 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -6,6 +6,7 @@ use std::{ time::Instant, }; +use anyhow::anyhow; use futures::{ future::{BoxFuture, LocalBoxFuture, Shared}, stream::FuturesUnordered, @@ -13,7 +14,7 @@ use futures::{ }; use iroh_bytes::{ baomap::{MapEntry, Store as BaoStore}, - util::Hash, + util::{progress::IgnoreProgressSender, Hash}, }; use iroh_gossip::net::util::Dialer; use iroh_metrics::{inc, inc_by}; @@ -32,11 +33,11 @@ pub type DownloadFuture = Shared>>; /// /// Spawns a background task that handles connecting to peers and performing get requests. /// -/// TODO: Move to iroh-bytes or replace with corresponding feature from iroh-bytes once available /// TODO: Support retries and backoff - become a proper queue... /// TODO: Download requests send via synchronous flume::Sender::send. Investigate if we want async /// here. We currently use [`Downloader::push`] from [`iroh_sync::Replica::on_insert`] callbacks, /// which are sync, thus we need a sync method on the Downloader to push new download requests. +/// TODO: Support collections, likely become generic over C: CollectionParser #[derive(Debug, Clone)] pub struct Downloader { pending_downloads: Arc>>, @@ -243,13 +244,18 @@ impl DownloadActor { } } -// TODO: reimplement downloads async fn download_single( - _store: B, - _conn: quinn::Connection, - _hash: Hash, + store: B, + conn: quinn::Connection, + hash: Hash, ) -> anyhow::Result> { - todo!("Downloads not implemented") + // TODO: Make use of progress + let progress_sender = IgnoreProgressSender::default(); + let _stats = crate::get::get_blob(&store, conn, &hash, progress_sender).await?; + let entry = store + .get(&hash) + .ok_or_else(|| anyhow!("downloaded blob is not in store"))?; + Ok(Some((hash, entry.size()))) } #[derive(Debug, Default)] From c039f72156938c00593a0b0b2b4d41c331b26749 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 13:18:34 +0200 Subject: [PATCH 056/172] feat(cli): support printing document content --- iroh/src/commands/sync.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index ab3e230bec..df83f0a5d5 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -16,6 +16,8 @@ use super::RpcClient; // TODO: It is a bit unfortunate that we have to drag the generics all through. Maybe box the conn? pub type Iroh = iroh::client::Iroh>; +const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; + #[derive(Debug, Clone, Parser)] pub enum Commands { Author { @@ -143,7 +145,10 @@ pub enum Doc { /// shown. #[clap(short, long)] old: bool, - // TODO: get content? + + /// Also print the content for each entry (but only if smaller than 1MB and valid UTf-8) + #[clap(short, long)] + content: bool, }, List { /// If true, old entries will be included. By default only the latest value for each key is @@ -188,6 +193,7 @@ impl Doc { prefix, author, old, + content, } => { let key = key.as_bytes().to_vec(); let key = match prefix { @@ -202,6 +208,23 @@ impl Doc { let mut stream = doc.get(filter).await?; while let Some(entry) = stream.try_next().await? { println!("{}", fmt_entry(&entry)); + if content { + if entry.content_len() < MAX_DISPLAY_CONTENT_LEN { + match doc.get_content_bytes(&entry).await { + Ok(content) => match String::from_utf8(content.into()) { + Ok(s) => println!("{s}"), + Err(_err) => println!(""), + }, + Err(err) => println!(""), + } + } else { + println!( + "", + HumanBytes(entry.content_len()) + ) + } + } + println!(""); } } Doc::List { old, prefix } => { From 910fd879eb9501e8064f6e6397f4461f02de237f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 13:38:35 +0200 Subject: [PATCH 057/172] remove unneeded todo comment --- iroh/src/sync.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index 887411e120..e40778da9f 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -178,7 +178,6 @@ pub async fn run_bob { - // TODO: this should be possible. bail!("unable to synchronize unknown namespace: {}", namespace); } } From f03b0619e32cdd89cd481230fcd3de43a279556d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 14:16:55 +0200 Subject: [PATCH 058/172] fix tests --- iroh-sync/src/store.rs | 31 +++++ iroh/examples/client.rs | 7 +- iroh/examples/collection.rs | 5 +- iroh/examples/hello-world.rs | 7 +- iroh/examples/rpc.rs | 3 +- iroh/src/commands/sync.rs | 2 +- iroh/src/node.rs | 11 +- iroh/tests/provide.rs | 3 +- iroh/tests/sync.rs | 249 +++++++++++++++++------------------ 9 files changed, 174 insertions(+), 144 deletions(-) diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index 465f341593..607124e955 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -104,6 +104,37 @@ pub struct GetFilter { pub key: KeyFilter, } +impl GetFilter { + /// Create a new get filter. Defaults to latest entries for all keys and authors. + pub fn new() -> Self { + Self { + latest: true, + author: None, + key: KeyFilter::All, + } + } + /// Filter by exact key match. + pub fn with_key(mut self, key: Vec) -> Self { + self.key = KeyFilter::Key(key); + self + } + /// Filter by prefix key match. + pub fn with_prefix(mut self, prefix: Vec) -> Self { + self.key = KeyFilter::Prefix(prefix); + self + } + /// Filter by author. + pub fn with_author(mut self, author: AuthorId) -> Self { + self.author = Some(author); + self + } + /// Include not only latest entries but also all historical entries. + pub fn with_history(mut self) -> Self { + self.latest = false; + self + } +} + /// Filter the keys in a namespace #[derive(Debug, Serialize, Deserialize)] pub enum KeyFilter { diff --git a/iroh/examples/client.rs b/iroh/examples/client.rs index ee9c123311..8c5896eb70 100644 --- a/iroh/examples/client.rs +++ b/iroh/examples/client.rs @@ -1,5 +1,5 @@ use indicatif::HumanBytes; -use iroh::{database::flat::writable::WritableFileDatabase, node::Node}; +use iroh::node::Node; use iroh_bytes::util::runtime; use iroh_sync::{ store::{GetFilter, KeyFilter}, @@ -9,11 +9,10 @@ use tokio_stream::StreamExt; #[tokio::main] async fn main() -> anyhow::Result<()> { - let dir = tempfile::tempdir()?; let rt = runtime::Handle::from_currrent(1)?; - let db = WritableFileDatabase::new(dir.path().into()).await?; + let db = iroh::baomap::mem::Store::new(rt.clone()); let store = iroh_sync::store::memory::Store::default(); - let node = Node::builder(db.db().clone(), store, dir.path().into()) + let node = Node::builder(db.clone(), store) .runtime(&rt) .spawn() .await?; diff --git a/iroh/examples/collection.rs b/iroh/examples/collection.rs index 51df19fcbf..353bab8904 100644 --- a/iroh/examples/collection.rs +++ b/iroh/examples/collection.rs @@ -41,9 +41,12 @@ async fn main() -> anyhow::Result<()> { // create a new iroh runtime with 1 worker thread, reusing the existing tokio runtime let rt = runtime::Handle::from_currrent(1)?; + // create an in-memory doc store for iroh sync (not used here) + let doc_store = iroh_sync::store::memory::Store::default(); + // create a new node // we must configure the iroh collection parser so the node understands iroh collections - let node = iroh::node::Node::builder(db) + let node = iroh::node::Node::builder(db, doc_store) .collection_parser(IrohCollectionParser) .runtime(&rt) .spawn() diff --git a/iroh/examples/hello-world.rs b/iroh/examples/hello-world.rs index 93ed838260..508569c643 100644 --- a/iroh/examples/hello-world.rs +++ b/iroh/examples/hello-world.rs @@ -22,12 +22,17 @@ async fn main() -> anyhow::Result<()> { setup_logging(); // create a new, empty in memory database let mut db = iroh::baomap::readonly_mem::Store::default(); + // create an in-memory doc store (not used in the example) + let doc_store = iroh_sync::store::memory::Store::default(); // create a new iroh runtime with 1 worker thread, reusing the existing tokio runtime let rt = runtime::Handle::from_currrent(1)?; // add some data and remember the hash let hash = db.insert(b"Hello, world!"); // create a new node - let node = iroh::node::Node::builder(db).runtime(&rt).spawn().await?; + let node = iroh::node::Node::builder(db, doc_store) + .runtime(&rt) + .spawn() + .await?; // create a ticket let ticket = node.ticket(hash).await?.with_recursive(false); // print some info about the node diff --git a/iroh/examples/rpc.rs b/iroh/examples/rpc.rs index 4ff3edd75a..dd09d5826a 100644 --- a/iroh/examples/rpc.rs +++ b/iroh/examples/rpc.rs @@ -45,7 +45,8 @@ async fn run(db: impl Store) -> anyhow::Result<()> { // create a new node // we must configure the iroh collection parser so the node understands iroh collections - let node = iroh::node::Node::builder(db) + let doc_store = iroh_sync::store::memory::Store::default(); + let node = iroh::node::Node::builder(db, doc_store) .keypair(keypair) .collection_parser(IrohCollectionParser) .runtime(&rt) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index df83f0a5d5..ff86e256d4 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -224,7 +224,7 @@ impl Doc { ) } } - println!(""); + println!(""); } } Doc::List { old, prefix } => { diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 7994ce4fac..bca8191306 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -82,14 +82,15 @@ const ENDPOINT_WAIT: Duration = Duration::from_secs(5); pub fn init_metrics_collection() { use iroh_metrics::core::Metric; - iroh_metrics::core::Core::init(|reg, metrics| { + iroh_metrics::core::Core::try_init(|reg, metrics| { metrics.insert(crate::metrics::Metrics::new(reg)); metrics.insert(iroh_sync::metrics::Metrics::new(reg)); metrics.insert(iroh_net::metrics::MagicsockMetrics::new(reg)); metrics.insert(iroh_net::metrics::NetcheckMetrics::new(reg)); metrics.insert(iroh_net::metrics::PortmapMetrics::new(reg)); metrics.insert(iroh_net::metrics::DerpMetrics::new(reg)); - }); + }) + .ok(); } /// Builder for the [`Node`]. @@ -1352,8 +1353,9 @@ mod tests { async fn test_ticket_multiple_addrs() { let rt = test_runtime(); let (db, hashes) = crate::baomap::readonly_mem::Store::new([("test", b"hello")]); + let doc_store = iroh_sync::store::memory::Store::default(); let hash = hashes["test"].into(); - let node = Node::builder(db) + let node = Node::builder(db, doc_store) .bind_addr((Ipv4Addr::UNSPECIFIED, 0).into()) .runtime(&rt) .spawn() @@ -1370,7 +1372,8 @@ mod tests { async fn test_node_add_collection_event() -> Result<()> { let rt = runtime::Handle::from_currrent(1)?; let db = crate::baomap::mem::Store::new(rt); - let node = Node::builder(db) + let doc_store = iroh_sync::store::memory::Store::default(); + let node = Node::builder(db, doc_store) .bind_addr((Ipv4Addr::UNSPECIFIED, 0).into()) .runtime(&test_runtime()) .spawn() diff --git a/iroh/tests/provide.rs b/iroh/tests/provide.rs index 2a6dcf26d8..5d7e5d992a 100644 --- a/iroh/tests/provide.rs +++ b/iroh/tests/provide.rs @@ -633,7 +633,8 @@ async fn test_custom_collection_parser() { let collection_bytes = postcard::to_allocvec(&collection).unwrap(); let collection_hash = db.insert(collection_bytes.clone()); let addr = "127.0.0.1:0".parse().unwrap(); - let node = Node::builder(db) + let doc_store = iroh_sync::store::memory::Store::default(); + let node = Node::builder(db, doc_store) .collection_parser(CollectionsAreJustLinks) .bind_addr(addr) .runtime(&rt) diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 8aa4979f41..e1bdbc8c1c 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -5,21 +5,15 @@ use std::{net::SocketAddr, time::Duration}; use anyhow::Result; use futures::StreamExt; use iroh::{ - client::Iroh, collection::IrohCollectionParser, - database::flat::{writable::WritableFileDatabase, Database}, node::{Builder, Node}, - rpc_protocol::{ProviderService, ShareMode}, + rpc_protocol::ShareMode, }; -use quic_rpc::{transport::misc::DummyServerEndpoint, ServiceConnection}; -use tempfile::TempDir; +use quic_rpc::transport::misc::DummyServerEndpoint; use tracing_subscriber::{prelude::*, EnvFilter}; -use iroh_bytes::{provider::BaoReadonlyDb, util::runtime}; -use iroh_sync::{ - store::{self, GetFilter, KeyFilter}, - sync::NamespaceId, -}; +use iroh_bytes::util::runtime; +use iroh_sync::store::{self, GetFilter}; /// Pick up the tokio runtime from the thread local and add a /// thread per core runtime. @@ -27,185 +21,178 @@ fn test_runtime() -> runtime::Handle { runtime::Handle::from_currrent(1).unwrap() } -struct Cancel(TempDir); - -fn test_node( +fn test_node( rt: runtime::Handle, - db: D, - writable_db_path: PathBuf, addr: SocketAddr, -) -> Builder { +) -> Builder< + iroh::baomap::mem::Store, + store::memory::Store, + DummyServerEndpoint, + IrohCollectionParser, +> { + let db = iroh::baomap::mem::Store::new(rt.clone()); let store = iroh_sync::store::memory::Store::default(); - Node::builder(db, store, writable_db_path) + Node::builder(db, store) .collection_parser(IrohCollectionParser) .runtime(&rt) .bind_addr(addr) } -struct NodeDropGuard { - _dir: TempDir, - node: Node, -} -impl Drop for NodeDropGuard { - fn drop(&mut self) { - self.node.shutdown(); - } -} - async fn spawn_node( rt: runtime::Handle, -) -> anyhow::Result<(Node, NodeDropGuard)> { - let dir = tempfile::tempdir()?; - let db = WritableFileDatabase::new(dir.path().into()).await?; - let node = test_node( - rt, - db.db().clone(), - dir.path().into(), - "127.0.0.1:0".parse()?, - ); +) -> anyhow::Result> { + let node = test_node(rt, "127.0.0.1:0".parse()?); let node = node.spawn().await?; - Ok((node.clone(), NodeDropGuard { node, _dir: dir })) + Ok(node) } async fn spawn_nodes( rt: runtime::Handle, n: usize, -) -> anyhow::Result<( - Vec>, - Vec, -)> { +) -> anyhow::Result>> { let mut nodes = vec![]; - let mut guards = vec![]; for _i in 0..n { - let (node, guard) = spawn_node(rt.clone()).await?; + let node = spawn_node(rt.clone()).await?; nodes.push(node); - guards.push(guard); } - Ok((nodes, guards)) + Ok(nodes) } #[tokio::test] async fn sync_full_basic() -> Result<()> { setup_logging(); let rt = test_runtime(); - let (nodes, drop_guard) = spawn_nodes(rt, 3).await?; + let nodes = spawn_nodes(rt, 3).await?; let clients = nodes.iter().map(|node| node.client()).collect::>(); - for (i, node) in nodes.iter().enumerate() { - println!( - "node {i}: {} {:?}", - node.peer_id(), - node.local_endpoints().await - ); - } + // for (i, node) in nodes.iter().enumerate() { + // println!( + // "node {i}: {} {:?}", + // node.peer_id(), + // node.local_endpoints().await + // ); + // } // node1: create doc and ticket - let (id, ticket) = { + let (ticket, doc2) = { let iroh = &clients[0]; let author = iroh.create_author().await?; let doc = iroh.create_doc().await?; - let key = b"p1"; - let value = b"1"; + let key = b"k1"; + let value = b"v1"; doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; - let res = doc.get_bytes_latest(author, key.to_vec()).await?; + let entry = doc.get_latest(author, key.to_vec()).await?; + let res = doc.get_content_bytes(&entry).await?; assert_eq!(res.to_vec(), value.to_vec()); let ticket = doc.share(ShareMode::Write).await?; - (doc.id(), ticket) + (ticket, doc) }; // node2: join in { let iroh = &clients[1]; let author = iroh.create_author().await?; - println!("\n\n!!!! peer 1 joins !!!!"); - let doc = iroh - .import_doc(ticket.key, vec![ticket.peer.clone()]) - .await?; - tokio::time::sleep(Duration::from_secs(2)).await; - for (i, client) in clients.iter().enumerate() { - report(&client, id, format!("node{i}")).await; - } - - println!("\n\n!!!! peer 1 publishes !!!!"); - - let key = b"p2"; - let value = b"22"; + let doc = iroh.import_doc(ticket.clone()).await?; + // todo: events over rpc to not use sleep... + tokio::time::sleep(Duration::from_secs(3)).await; + + let key = b"k1".to_vec(); + let filter = GetFilter::new().with_key(key); + let entry = doc.get(filter).await?.next().await.unwrap()?; + let res = doc.get_content_bytes(&entry).await?; + assert_eq!(res.to_vec(), b"v1".to_vec()); + + let key = b"k2"; + let value = b"v2"; doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; // todo: events - tokio::time::sleep(Duration::from_secs(2)).await; - for (i, client) in clients.iter().enumerate() { - report(&client, id, format!("node{i}")).await; + tokio::time::sleep(Duration::from_secs(3)).await; + + for doc in &[doc, doc2] { + let filter = GetFilter::new().with_key(key.to_vec()); + let entry = doc.get(filter).await?.next().await.unwrap()?; + let res = doc.get_content_bytes(&entry).await?; + assert_eq!(res.to_vec(), value.to_vec()); } } - println!("\n\n!!!! peer 2 joins !!!!"); { - // node 3 joins & imports the doc from peer 1 + // node 3 joins & imports the doc from peer 1 let iroh = &clients[2]; - let author = iroh.create_author().await?; - let doc = iroh - .import_doc(ticket.key, vec![ticket.peer.clone()]) - .await?; - - // now wait... - tokio::time::sleep(Duration::from_secs(5)).await; - println!("\n\n!!!! peer 2 publishes !!!!"); - let key = b"p3"; - let value = b"333"; - doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; - } - - tokio::time::sleep(Duration::from_secs(5)).await; - for (i, client) in clients.iter().enumerate() { - report(&client, id, format!("node{i}")).await; - } + let doc = iroh.import_doc(ticket).await?; - drop(drop_guard); + // todo: events + tokio::time::sleep(Duration::from_secs(3)).await; - Ok(()) -} + let key = b"k1"; + let value = b"v1"; + let filter = GetFilter::new().with_key(key.to_vec()); + let entry = doc.get(filter).await?.next().await.unwrap()?; + let res = doc.get_content_bytes(&entry).await?; + assert_eq!(res.to_vec(), value.to_vec()); -async fn report>( - client: &Iroh, - id: NamespaceId, - label: impl ToString, -) { - let label = label.to_string(); - println!("report: {label} {id}"); - match try_report(client, id).await { - Ok(_) => {} - Err(err) => println!(" failed: {err}"), + let key = b"k2"; + let value = b"v2"; + let filter = GetFilter::new().with_key(key.to_vec()); + let entry = doc.get(filter).await?.next().await.unwrap()?; + let res = doc.get_content_bytes(&entry).await?; + // TODO: This fails! seems reproviding is not working? + assert_eq!(res.to_vec(), value.to_vec()); } -} -async fn try_report>( - client: &Iroh, - id: NamespaceId, -) -> anyhow::Result<()> { - let doc = client.get_doc(id)?; - let filter = GetFilter { - latest: false, - author: None, - key: KeyFilter::All, - }; - let mut stream = doc.get(filter).await?; - while let Some(entry) = stream.next().await { - let entry = entry?; - let text = match client.get_bytes(*entry.content_hash()).await { - Ok(bytes) => String::from_utf8(bytes.to_vec())?, - Err(err) => format!("<{err}>"), - }; - println!( - " @{} {} {:4} -- {}", - entry.author(), - entry.content_hash(), - entry.content_len(), - text - ); + // TODO: + // - gossiping between multiple peers + // - better test utils + // - ... + + for node in nodes { + node.shutdown(); } + Ok(()) } +// async fn report>( +// client: &Iroh, +// id: NamespaceId, +// label: impl ToString, +// ) { +// let label = label.to_string(); +// println!("report: {label} {id}"); +// match try_report(client, id).await { +// Ok(_) => {} +// Err(err) => println!(" failed: {err}"), +// } +// } +// +// async fn try_report>( +// client: &Iroh, +// id: NamespaceId, +// ) -> anyhow::Result<()> { +// let doc = client.get_doc(id)?; +// let filter = GetFilter { +// latest: false, +// author: None, +// key: KeyFilter::All, +// }; +// let mut stream = doc.get(filter).await?; +// while let Some(entry) = stream.next().await { +// let entry = entry?; +// let text = match doc.get_content_bytes(&entry).await { +// Ok(bytes) => String::from_utf8(bytes.to_vec())?, +// Err(err) => format!("<{err}>"), +// }; +// println!( +// " @{} {} {:4} -- {}", +// entry.author(), +// entry.content_hash(), +// entry.content_len(), +// text +// ); +// } +// Ok(()) +// } + fn setup_logging() { tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) From 1f864b53380f5583d3f4308036da9fe8d0a72ae6 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 16:07:34 +0200 Subject: [PATCH 059/172] fix downloader and sync test --- iroh/src/client.rs | 10 ++++ iroh/src/download.rs | 70 +++++++++++++++++-------- iroh/tests/sync.rs | 120 ++++++++++++------------------------------- 3 files changed, 92 insertions(+), 108 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index e3be3ac587..0f76a70f31 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -38,6 +38,16 @@ pub type IrohMemClient = Iroh /// This is obtained from [`connect`]. pub type IrohRpcClient = Iroh>; +/// In-memory document client to an iroh node running in the same process. +/// +/// This is obtained from [`iroh::node::Node::client`]. +pub type DocMem = Doc>; + +/// RPC document client to an iroh node running in a seperate process. +/// +/// This is obtained from [`connect`]. +pub type DocRpc = Doc>; + /// TODO: Change to "/iroh-rpc/1" pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 70ab2b2e2d..6c30bddbeb 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -109,7 +109,7 @@ impl Downloader { type DownloadReply = oneshot::Sender>; type PendingDownloadsFutures = - FuturesUnordered>)>>; + FuturesUnordered>)>>; #[derive(Debug)] struct DownloadRequest { @@ -149,23 +149,37 @@ impl DownloadActor { }, (peer, conn) = self.dialer.next() => match conn { Ok(conn) => { - debug!("connection to {peer} established"); + debug!(peer = ?peer, "connection established"); self.conns.insert(peer, conn); self.on_peer_ready(peer); }, Err(err) => self.on_peer_fail(&peer, err), }, Some((peer, hash, res)) = self.pending_download_futs.next() => match res { - Ok(Some((hash, size))) => { + Ok(Some(size)) => { self.queue.on_success(hash, peer); self.reply(hash, Some((hash, size))); self.on_peer_ready(peer); } Ok(None) => { + // TODO: This case is currently never reached, because iroh::get::get_blob + // doesn't return an option but only a result, with no way (AFAICS) to discern + // between connection error and not found. + // self.on_not_found(&peer, hash); + // self.on_peer_ready(peer); + unreachable!() + } + Err(_err) => { self.on_not_found(&peer, hash); self.on_peer_ready(peer); + // TODO: In case of connection errors or similar we want to call + // on_peer_fail to not continue downloading from this peer. + // Currently however a "not found" is also an error, thus calling + // on_peer_fail would stop trying to get other hashes from this peer. + // This likely needs fixing in iroh::get::get to have a meaningful error to + // see if the connection failed or if it's just a "not found". + // self.on_peer_fail(&peer, err), } - Err(err) => self.on_peer_fail(&peer, err), } } } @@ -194,8 +208,10 @@ impl DownloadActor { fn on_peer_ready(&mut self, peer: PeerId) { if let Some(hash) = self.queue.try_next_for_peer(peer) { + debug!(peer = ?peer, hash = ?hash, "on_peer_ready: get next"); self.start_download_unchecked(peer, hash); } else { + debug!(peer = ?peer, "on_peer_ready: nothing left, disconnect"); self.conns.remove(&peer); } } @@ -203,16 +219,29 @@ impl DownloadActor { fn start_download_unchecked(&mut self, peer: PeerId, hash: Hash) { let conn = self.conns.get(&peer).unwrap().clone(); let db = self.db.clone(); + let progress_sender = IgnoreProgressSender::default(); + let fut = async move { + debug!(peer = ?peer, hash = ?hash, "start download"); + #[cfg(feature = "metrics")] let start = Instant::now(); - let res = download_single(db, conn, hash).await; + + // TODO: None for not found instead of error + let res = crate::get::get_blob(&db, conn, &hash, progress_sender).await; + let res = res.and_then(|_stats| { + db.get(&hash) + .ok_or_else(|| anyhow!("downloaded blob not found in store")) + .map(|entry| Some(entry.size())) + }); + debug!(peer = ?peer, hash = ?hash, "finish download: {res:?}"); + // record metrics #[cfg(feature = "metrics")] { let elapsed = start.elapsed().as_millis(); match &res { - Ok(Some((_hash, len))) => { + Ok(Some(len)) => { inc!(Metrics, downloads_success); inc_by!(Metrics, download_bytes_total, *len); inc_by!(Metrics, download_time_total, elapsed as u64); @@ -221,6 +250,7 @@ impl DownloadActor { Err(_) => inc!(Metrics, downloads_error), } } + (peer, hash, res) }; self.pending_download_futs.push(fut.boxed_local()); @@ -235,6 +265,7 @@ impl DownloadActor { } self.replies.entry(hash).or_default().push_back(reply); for peer in peers { + debug!(peer = ?peer, hash = ?hash, "queue download"); self.queue.push_candidate(hash, peer); // TODO: Don't dial all peers instantly. if self.conns.get(&peer).is_none() && !self.dialer.is_pending(&peer) { @@ -244,20 +275,6 @@ impl DownloadActor { } } -async fn download_single( - store: B, - conn: quinn::Connection, - hash: Hash, -) -> anyhow::Result> { - // TODO: Make use of progress - let progress_sender = IgnoreProgressSender::default(); - let _stats = crate::get::get_blob(&store, conn, &hash, progress_sender).await?; - let entry = store - .get(&hash) - .ok_or_else(|| anyhow!("downloaded blob is not in store"))?; - Ok(Some((hash, entry.size()))) -} - #[derive(Debug, Default)] struct DownloadQueue { candidates_by_hash: HashMap>, @@ -304,13 +321,22 @@ impl DownloadQueue { self.candidates_by_hash.get(hash).is_none() && self.running_by_hash.get(hash).is_none() } - pub fn on_success(&mut self, hash: Hash, peer: PeerId) -> Option<(PeerId, Hash)> { + /// Mark a download as successfull. + pub fn on_success(&mut self, hash: Hash, peer: PeerId) { let peer2 = self.running_by_hash.remove(&hash); debug_assert_eq!(peer2, Some(peer)); self.running_by_peer.remove(&peer); - self.try_next_for_peer(peer).map(|hash| (peer, hash)) + self.candidates_by_hash.remove(&hash); + for hashes in self.candidates_by_peer.values_mut() { + hashes.retain(|h| h != &hash); + } + self.ensure_no_empty(hash, peer); } + /// To be called when a peer failed (i.e. disconnected). + /// + /// Returns a list of hashes that have no other peers queue. Those hashes should thus be + /// considered failed. pub fn on_peer_fail(&mut self, peer: &PeerId) -> Vec { let mut failed = vec![]; for hash in self diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index e1bdbc8c1c..1ef2592d0d 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -2,9 +2,10 @@ use std::{net::SocketAddr, time::Duration}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use futures::StreamExt; use iroh::{ + client::DocMem, collection::IrohCollectionParser, node::{Builder, Node}, rpc_protocol::ShareMode, @@ -65,80 +66,51 @@ async fn sync_full_basic() -> Result<()> { let nodes = spawn_nodes(rt, 3).await?; let clients = nodes.iter().map(|node| node.client()).collect::>(); - // for (i, node) in nodes.iter().enumerate() { - // println!( - // "node {i}: {} {:?}", - // node.peer_id(), - // node.local_endpoints().await - // ); - // } - // node1: create doc and ticket - let (ticket, doc2) = { + let (ticket, doc1) = { let iroh = &clients[0]; let author = iroh.create_author().await?; let doc = iroh.create_doc().await?; let key = b"k1"; let value = b"v1"; doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; - let entry = doc.get_latest(author, key.to_vec()).await?; - let res = doc.get_content_bytes(&entry).await?; - assert_eq!(res.to_vec(), value.to_vec()); + assert_latest(&doc, key, value).await; let ticket = doc.share(ShareMode::Write).await?; (ticket, doc) }; // node2: join in - { + let _doc2 = { let iroh = &clients[1]; let author = iroh.create_author().await?; let doc = iroh.import_doc(ticket.clone()).await?; + // todo: events over rpc to not use sleep... tokio::time::sleep(Duration::from_secs(3)).await; - - let key = b"k1".to_vec(); - let filter = GetFilter::new().with_key(key); - let entry = doc.get(filter).await?.next().await.unwrap()?; - let res = doc.get_content_bytes(&entry).await?; - assert_eq!(res.to_vec(), b"v1".to_vec()); + assert_latest(&doc, b"k1", b"v1").await; let key = b"k2"; let value = b"v2"; doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; + assert_latest(&doc, key, value).await; // todo: events tokio::time::sleep(Duration::from_secs(3)).await; + assert_latest(&doc1, key, value).await; + doc + }; - for doc in &[doc, doc2] { - let filter = GetFilter::new().with_key(key.to_vec()); - let entry = doc.get(filter).await?.next().await.unwrap()?; - let res = doc.get_content_bytes(&entry).await?; - assert_eq!(res.to_vec(), value.to_vec()); - } - } - - { - // node 3 joins & imports the doc from peer 1 + // node 3 joins & imports the doc from peer 1 + let _doc3 = { let iroh = &clients[2]; + println!("!!!! DOC 3 JOIN !!!!!"); let doc = iroh.import_doc(ticket).await?; // todo: events tokio::time::sleep(Duration::from_secs(3)).await; - - let key = b"k1"; - let value = b"v1"; - let filter = GetFilter::new().with_key(key.to_vec()); - let entry = doc.get(filter).await?.next().await.unwrap()?; - let res = doc.get_content_bytes(&entry).await?; - assert_eq!(res.to_vec(), value.to_vec()); - - let key = b"k2"; - let value = b"v2"; - let filter = GetFilter::new().with_key(key.to_vec()); - let entry = doc.get(filter).await?.next().await.unwrap()?; - let res = doc.get_content_bytes(&entry).await?; - // TODO: This fails! seems reproviding is not working? - assert_eq!(res.to_vec(), value.to_vec()); - } + assert_latest(&doc, b"k1", b"v1").await; + assert_latest(&doc, b"k2", b"v2").await; + doc + }; // TODO: // - gossiping between multiple peers @@ -152,46 +124,22 @@ async fn sync_full_basic() -> Result<()> { Ok(()) } -// async fn report>( -// client: &Iroh, -// id: NamespaceId, -// label: impl ToString, -// ) { -// let label = label.to_string(); -// println!("report: {label} {id}"); -// match try_report(client, id).await { -// Ok(_) => {} -// Err(err) => println!(" failed: {err}"), -// } -// } -// -// async fn try_report>( -// client: &Iroh, -// id: NamespaceId, -// ) -> anyhow::Result<()> { -// let doc = client.get_doc(id)?; -// let filter = GetFilter { -// latest: false, -// author: None, -// key: KeyFilter::All, -// }; -// let mut stream = doc.get(filter).await?; -// while let Some(entry) = stream.next().await { -// let entry = entry?; -// let text = match doc.get_content_bytes(&entry).await { -// Ok(bytes) => String::from_utf8(bytes.to_vec())?, -// Err(err) => format!("<{err}>"), -// }; -// println!( -// " @{} {} {:4} -- {}", -// entry.author(), -// entry.content_hash(), -// entry.content_len(), -// text -// ); -// } -// Ok(()) -// } +async fn assert_latest(doc: &DocMem, key: &[u8], value: &[u8]) { + let content = get_latest(doc, key).await.unwrap(); + assert_eq!(content, value.to_vec()); +} + +async fn get_latest(doc: &DocMem, key: &[u8]) -> anyhow::Result> { + let filter = GetFilter::new().with_key(key.to_vec()); + let entry = doc + .get(filter) + .await? + .next() + .await + .ok_or_else(|| anyhow!("entry not found"))??; + let content = doc.get_content_bytes(&entry).await?; + Ok(content.to_vec()) +} fn setup_logging() { tracing_subscriber::registry() From 961bd3dbef4fcb4a326e21de9201855d68f9abcf Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 16:23:28 +0200 Subject: [PATCH 060/172] fix: feature flags, and more docs --- iroh/src/lib.rs | 4 +--- iroh/src/sync/engine.rs | 16 ++++++++++++++++ iroh/src/sync/live.rs | 20 +++++++++++++++++++- iroh/src/sync/rpc.rs | 3 +-- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index e1a0e6d123..852fbbca47 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -5,6 +5,7 @@ pub use iroh_bytes as bytes; pub use iroh_net as net; pub mod baomap; +pub mod client; #[cfg(feature = "iroh-collection")] pub mod collection; pub mod dial; @@ -12,12 +13,9 @@ pub mod download; pub mod get; pub mod node; pub mod rpc_protocol; -#[allow(missing_docs)] pub mod sync; pub mod util; -pub mod client; - /// Expose metrics module #[cfg(feature = "metrics")] pub mod metrics; diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index a0719d26e6..6dcce13642 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -29,6 +29,14 @@ pub struct SyncEngine { } impl SyncEngine { + /// Start the sync engine. + /// + /// This will spawn a background task for the [`LiveSync`]. When documents are added to the + /// engine with [`Self::start_sync`], then new entries inserted locally will be sent to peers + /// through iroh-gossip. + /// + /// The engine will also register [`Replica::on_insert`] callbacks to download content for new + /// entries from peers. pub fn spawn( rt: Handle, endpoint: MagicEndpoint, @@ -47,6 +55,10 @@ impl SyncEngine { } } + /// Start to sync a document. + /// + /// If `peers` is non-empty, it will both do an initial set-reconciliation sync with each peer, + /// and join an iroh-gossip swarm with these peers to receive and broadcast document updates. pub async fn start_sync( &self, namespace: NamespaceId, @@ -67,6 +79,7 @@ impl SyncEngine { Ok(()) } + /// Stop syncing a document. pub async fn stop_sync(&self, namespace: NamespaceId) -> anyhow::Result<()> { let replica = self.get_replica(&namespace)?; if let Some(token) = self.active.write().remove(&replica.namespace()) { @@ -76,6 +89,7 @@ impl SyncEngine { Ok(()) } + /// Shutdown the sync engine. pub async fn shutdown(&self) -> anyhow::Result<()> { for (namespace, token) in self.active.write().drain() { if let Ok(Some(replica)) = self.store.open_replica(&namespace) { @@ -86,12 +100,14 @@ impl SyncEngine { Ok(()) } + /// Get a [`Replica`] from the store, returning an error if the replica does not exist. pub fn get_replica(&self, id: &NamespaceId) -> anyhow::Result> { self.store .open_replica(id)? .ok_or_else(|| anyhow!("doc not found")) } + /// Get an [`Author`] from the store, returning an error if the replica does not exist. pub fn get_author(&self, id: &AuthorId) -> anyhow::Result { self.store .get_author(id)? diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index ed4d5f223f..4a85db016f 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -28,8 +28,11 @@ const CHANNEL_CAP: usize = 8; /// TODO: Make an enum and support DNS resolution #[derive(Clone, Debug, Serialize, Deserialize)] pub struct PeerSource { + /// The peer id (required) pub peer_id: PeerId, + /// Socket addresses for this peer (may be empty) pub addrs: Vec, + /// Derp region for this peer pub derp_region: Option, } @@ -45,6 +48,7 @@ impl PeerSource { pub fn to_bytes(&self) -> Vec { postcard::to_stdvec(self).expect("postcard::to_stdvec is infallible") } + /// Create with information gathered from a [`MagicEndpoint`] pub async fn from_endpoint(endpoint: &MagicEndpoint) -> anyhow::Result { Ok(Self { peer_id: endpoint.peer_id(), @@ -80,8 +84,12 @@ impl FromStr for PeerSource { } } +/// An iroh-sync operation +/// +/// This is the message that is broadcast over iroh-gossip. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Op { + /// A new entry was inserted into the document. Put(SignedEntry), } @@ -93,7 +101,7 @@ enum SyncState { } #[derive(Debug)] -pub enum ToActor { +enum ToActor { StartSync { replica: Replica, peers: Vec, @@ -116,6 +124,10 @@ pub struct LiveSync { } impl LiveSync { + /// Start the live sync. + /// + /// This spawn a background actor to handle gossip events and forward operations over broadcast + /// messages. pub fn spawn(rt: Handle, endpoint: MagicEndpoint, gossip: Gossip) -> Self { let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); let mut actor = Actor::new(endpoint, gossip, to_actor_rx); @@ -138,6 +150,8 @@ impl LiveSync { Ok(()) } + /// Start to sync a document with a set of peers, also joining the gossip swarm for that + /// document. pub async fn start_sync( &self, replica: Replica, @@ -149,6 +163,7 @@ impl LiveSync { Ok(()) } + /// Join and sync with a set of peers for a document that is already syncing. pub async fn join_peers(&self, namespace: NamespaceId, peers: Vec) -> Result<()> { self.to_actor_tx .send(ToActor::::JoinPeers { namespace, peers }) @@ -156,6 +171,9 @@ impl LiveSync { Ok(()) } + /// Stop the live sync for a document. + /// + /// This will leave the gossip swarm for this document. pub async fn stop_sync(&self, namespace: NamespaceId) -> Result<()> { self.to_actor_tx .send(ToActor::::StopSync { namespace }) diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index f49cd0a47c..c70ed60526 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -17,8 +17,8 @@ use crate::rpc_protocol::{ use super::{engine::SyncEngine, PeerSource}; +#[allow(missing_docs)] impl SyncEngine { - /// todo pub fn author_create(&self, _req: AuthorCreateRequest) -> RpcResult { // TODO: pass rng let author = self.store.new_author(&mut rand::rngs::OsRng {})?; @@ -27,7 +27,6 @@ impl SyncEngine { }) } - /// todo pub fn author_list( &self, _req: AuthorListRequest, From 67dd3427a5300bc00e2372060e44fb36be730376 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 17:36:10 +0200 Subject: [PATCH 061/172] remove sync feature --- iroh/Cargo.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 0973844c75..f4b88f7747 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -64,15 +64,14 @@ rustyline = { version = "12.0.0", optional = true } itertools = "0.11.0" [features] -default = ["cli", "metrics", "sync"] -sync = ["metrics", "iroh-sync/fs-store"] +default = ["cli", "metrics"] cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection"] metrics = ["iroh-metrics", "flat-db", "mem-db", "iroh-collection"] mem-db = [] flat-db = [] iroh-collection = [] test = [] -example-sync = ["cli", "ed25519-dalek", "shell-words", "shellexpand", "sync", "rustyline"] +example-sync = ["cli", "ed25519-dalek", "shell-words", "shellexpand", "rustyline"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } From cb24396f6625aecada73f6668e25a41a59260cb8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 17:44:05 +0200 Subject: [PATCH 062/172] chore: fmt & clippy fix --- iroh-sync/src/store.rs | 15 +++++++++++---- iroh/src/baomap/flat.rs | 8 ++------ iroh/src/baomap/mem.rs | 2 +- iroh/src/commands/sync.rs | 2 +- iroh/src/rpc_protocol.rs | 4 ++-- iroh/src/sync/live.rs | 4 ++-- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index 607124e955..5c864add31 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -104,15 +104,22 @@ pub struct GetFilter { pub key: KeyFilter, } -impl GetFilter { - /// Create a new get filter. Defaults to latest entries for all keys and authors. - pub fn new() -> Self { +impl Default for GetFilter { + fn default() -> Self { Self { latest: true, author: None, key: KeyFilter::All, } } +} + +impl GetFilter { + /// Create a new get filter. Defaults to latest entries for all keys and authors. + pub fn new() -> Self { + Self::default() + } + /// Filter by exact key match. pub fn with_key(mut self, key: Vec) -> Self { self.key = KeyFilter::Key(key); @@ -189,7 +196,7 @@ impl<'s, S: Store> GetIter<'s, S> { (Key(key), Some(author)) => Self::Single( store .get_latest_by_key_and_author(namespace, author, key)? - .map(|entry| Ok(entry)) + .map(Ok) .into_iter(), ), (All, Some(_)) | (Prefix(_), Some(_)) => { diff --git a/iroh/src/baomap/flat.rs b/iroh/src/baomap/flat.rs index 2b19fba89f..b1c6a6f2be 100644 --- a/iroh/src/baomap/flat.rs +++ b/iroh/src/baomap/flat.rs @@ -298,7 +298,7 @@ impl PartialMapEntry for PartialEntry { std::fs::OpenOptions::new() .write(true) .create(true) - .open(path.clone()) + .open(path) }) .boxed() } @@ -716,11 +716,7 @@ impl Store { Ok(progress2.try_send(ImportProgress::OutboardProgress { id, offset })?) })?; progress.blocking_send(ImportProgress::OutboardDone { id, hash })?; - ( - hash, - CompleteEntry::new_external(size, path.clone()), - outboard, - ) + (hash, CompleteEntry::new_external(size, path), outboard) } ImportMode::Copy => { let uuid = rand::thread_rng().gen::<[u8; 16]>(); diff --git a/iroh/src/baomap/mem.rs b/iroh/src/baomap/mem.rs index e5a6f75471..b0105554b1 100644 --- a/iroh/src/baomap/mem.rs +++ b/iroh/src/baomap/mem.rs @@ -378,7 +378,7 @@ impl PartialMap for Store { .write() .unwrap() .partial - .insert(hash, (data.clone(), ob2.clone())); + .insert(hash, (data.clone(), ob2)); Ok(PartialEntry { hash: hash.into(), outboard: PreOrderOutboard { diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index ff86e256d4..5e25c86f24 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -224,7 +224,7 @@ impl Doc { ) } } - println!(""); + println!(); } } Doc::List { old, prefix } => { diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index adf5f1bb1f..01473632dd 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -434,7 +434,7 @@ impl DocTicket { Ok(bytes) } pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { - let slf = postcard::from_bytes(&bytes)?; + let slf = postcard::from_bytes(bytes)?; Ok(slf) } } @@ -449,7 +449,7 @@ impl fmt::Display for DocTicket { write!( f, "{}", - base32::fmt(&self.to_bytes().expect("failed to serialize")) + base32::fmt(self.to_bytes().expect("failed to serialize")) ) } } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 4a85db016f..0bbbfd8220 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -359,7 +359,7 @@ impl Actor { ) -> Result<()> { let namespace = replica.namespace(); let topic = TopicId::from_bytes(*namespace.as_bytes()); - if !self.replicas.contains_key(&topic) { + if let std::collections::hash_map::Entry::Vacant(e) = self.replicas.entry(topic) { // setup replica insert notifications. let insert_entry_tx = self.insert_entry_tx.clone(); let removal_token = replica.on_insert(Box::new(move |origin, entry| { @@ -371,7 +371,7 @@ impl Actor { } } })); - self.replicas.insert(topic, (replica, removal_token)); + e.insert((replica, removal_token)); } self.join_gossip_and_start_initial_sync(&namespace, peers) From 75679dbf42ed6328a8adbbd6addc659b52ad3a90 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 17:49:11 +0200 Subject: [PATCH 063/172] fix: feature flags --- iroh/src/client.rs | 23 +++++++++++++++++------ iroh/src/download.rs | 4 +++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 0f76a70f31..66977511b3 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -11,14 +11,13 @@ use std::result::Result as StdResult; use std::sync::Arc; use std::time::Duration; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, Result}; use bytes::Bytes; use futures::{Stream, StreamExt, TryStreamExt}; use iroh_bytes::Hash; use iroh_sync::store::{GetFilter, KeyFilter}; use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; use quic_rpc::transport::flume::FlumeConnection; -use quic_rpc::transport::quinn::QuinnConnection; use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ @@ -33,10 +32,13 @@ use crate::sync::PeerSource; /// /// This is obtained from [`iroh::node::Node::client`]. pub type IrohMemClient = Iroh>; + /// RPC client to an iroh node running in a seperate process. /// /// This is obtained from [`connect`]. -pub type IrohRpcClient = Iroh>; +#[cfg(feature = "cli")] +pub type IrohRpcClient = + Iroh>; /// In-memory document client to an iroh node running in the same process. /// @@ -46,12 +48,15 @@ pub type DocMem = Doc>; /// RPC document client to an iroh node running in a seperate process. /// /// This is obtained from [`connect`]. -pub type DocRpc = Doc>; +#[cfg(feature = "cli")] +pub type DocRpc = + Doc>; /// TODO: Change to "/iroh-rpc/1" pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; /// Connect to an iroh node running on the same computer, but in a different process. +#[cfg(feature = "cli")] pub async fn connect(rpc_port: u16) -> anyhow::Result { let client = connect_raw(rpc_port).await?; Ok(Iroh::new(client)) @@ -59,16 +64,21 @@ pub async fn connect(rpc_port: u16) -> anyhow::Result { /// Create a raw RPC client to an iroh node running on the same computer, but in a different /// process. +#[cfg(feature = "cli")] pub async fn connect_raw( rpc_port: u16, ) -> anyhow::Result< - quic_rpc::RpcClient>, + quic_rpc::RpcClient< + ProviderService, + quic_rpc::transport::quinn::QuinnConnection, + >, > { + use anyhow::Context; let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); let server_name = "localhost".to_string(); - let connection = QuinnConnection::new(endpoint, addr, server_name); + let connection = quic_rpc::transport::quinn::QuinnConnection::new(endpoint, addr, server_name); let client = RpcClient::new(connection); // Do a version request to check if the server is running. let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) @@ -76,6 +86,7 @@ pub async fn connect_raw( .context("iroh server is not running")??; Ok(client) } +#[cfg(feature = "cli")] fn create_quinn_client( bind_addr: SocketAddr, alpn_protocols: Vec>, diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 6c30bddbeb..833a9865b5 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -1,9 +1,10 @@ //! Download queue +#[cfg(feature = "metrics")] +use std::time::Instant; use std::{ collections::{HashMap, VecDeque}, sync::{Arc, Mutex}, - time::Instant, }; use anyhow::anyhow; @@ -17,6 +18,7 @@ use iroh_bytes::{ util::{progress::IgnoreProgressSender, Hash}, }; use iroh_gossip::net::util::Dialer; +#[cfg(feature = "metrics")] use iroh_metrics::{inc, inc_by}; use iroh_net::{tls::PeerId, MagicEndpoint}; use tokio::sync::oneshot; From 69951d2f6bda015bfc65a73944b6ef5769a2e030 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 18:16:57 +0200 Subject: [PATCH 064/172] fix: feature flags --- iroh/src/client.rs | 121 ++++++++++++++++++++------------------- iroh/src/rpc_protocol.rs | 3 +- 2 files changed, 63 insertions(+), 61 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 66977511b3..059a8677f9 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -33,75 +33,76 @@ use crate::sync::PeerSource; /// This is obtained from [`iroh::node::Node::client`]. pub type IrohMemClient = Iroh>; -/// RPC client to an iroh node running in a seperate process. -/// -/// This is obtained from [`connect`]. -#[cfg(feature = "cli")] -pub type IrohRpcClient = - Iroh>; - /// In-memory document client to an iroh node running in the same process. /// /// This is obtained from [`iroh::node::Node::client`]. pub type DocMem = Doc>; -/// RPC document client to an iroh node running in a seperate process. -/// -/// This is obtained from [`connect`]. #[cfg(feature = "cli")] -pub type DocRpc = - Doc>; - -/// TODO: Change to "/iroh-rpc/1" -pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; +pub use quic::*; -/// Connect to an iroh node running on the same computer, but in a different process. #[cfg(feature = "cli")] -pub async fn connect(rpc_port: u16) -> anyhow::Result { - let client = connect_raw(rpc_port).await?; - Ok(Iroh::new(client)) -} +mod quic { + //! Utility methods to create an RPC client for a node running in a seperate process -/// Create a raw RPC client to an iroh node running on the same computer, but in a different -/// process. -#[cfg(feature = "cli")] -pub async fn connect_raw( - rpc_port: u16, -) -> anyhow::Result< - quic_rpc::RpcClient< - ProviderService, - quic_rpc::transport::quinn::QuinnConnection, - >, -> { - use anyhow::Context; - let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); - let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; - let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); - let server_name = "localhost".to_string(); - let connection = quic_rpc::transport::quinn::QuinnConnection::new(endpoint, addr, server_name); - let client = RpcClient::new(connection); - // Do a version request to check if the server is running. - let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) - .await - .context("iroh server is not running")??; - Ok(client) -} -#[cfg(feature = "cli")] -fn create_quinn_client( - bind_addr: SocketAddr, - alpn_protocols: Vec>, - keylog: bool, -) -> Result { - let keypair = iroh_net::tls::Keypair::generate(); - let tls_client_config = - iroh_net::tls::make_client_config(&keypair, None, alpn_protocols, keylog)?; - let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); - let mut endpoint = quinn::Endpoint::client(bind_addr)?; - let mut transport_config = quinn::TransportConfig::default(); - transport_config.keep_alive_interval(Some(Duration::from_secs(1))); - client_config.transport_config(Arc::new(transport_config)); - endpoint.set_default_client_config(client_config); - Ok(endpoint) + use quic_rpc::transport::quinn::QuinnConnection; + + use super::*; + /// RPC client to an iroh node running in a seperate process. + /// + /// This is obtained from [`connect`]. + pub type IrohRpcClient = Iroh>; + + /// RPC document client to an iroh node running in a seperate process. + /// + /// This is obtained from [`connect`]. + pub type DocRpc = Doc>; + + /// TODO: Change to "/iroh-rpc/1" + pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; + + /// Connect to an iroh node running on the same computer, but in a different process. + pub async fn connect(rpc_port: u16) -> anyhow::Result { + let client = connect_raw(rpc_port).await?; + Ok(Iroh::new(client)) + } + + /// Create a raw RPC client to an iroh node running on the same computer, but in a different + /// process. + pub async fn connect_raw( + rpc_port: u16, + ) -> anyhow::Result< + quic_rpc::RpcClient>, + > { + use anyhow::Context; + let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); + let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; + let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); + let server_name = "localhost".to_string(); + let connection = QuinnConnection::new(endpoint, addr, server_name); + let client = RpcClient::new(connection); + // Do a version request to check if the server is running. + let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) + .await + .context("iroh server is not running")??; + Ok(client) + } + fn create_quinn_client( + bind_addr: SocketAddr, + alpn_protocols: Vec>, + keylog: bool, + ) -> Result { + let keypair = iroh_net::tls::Keypair::generate(); + let tls_client_config = + iroh_net::tls::make_client_config(&keypair, None, alpn_protocols, keylog)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); + let mut endpoint = quinn::Endpoint::client(bind_addr)?; + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(1))); + client_config.transport_config(Arc::new(transport_config)); + endpoint.set_default_client_config(client_config); + Ok(endpoint) + } } /// Iroh client diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 01473632dd..652168e24f 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -365,7 +365,8 @@ pub struct AuthorShareRequest { } /// todo -#[derive(Serialize, Deserialize, Debug, Clone, clap::ValueEnum)] +#[derive(Serialize, Deserialize, Debug, Clone)] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum ShareMode { /// Read-only access Read, From 4d87cd305443cf27102bf4df5caa46a973e44f66 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 18:24:37 +0200 Subject: [PATCH 065/172] docs: fixes --- iroh/src/client.rs | 4 ++-- iroh/src/download.rs | 2 +- iroh/src/sync/engine.rs | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 059a8677f9..bc7eb69030 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -30,12 +30,12 @@ use crate::sync::PeerSource; /// In-memory client to an iroh node running in the same process. /// -/// This is obtained from [`iroh::node::Node::client`]. +/// This is obtained from [`crate::node::Node::client`]. pub type IrohMemClient = Iroh>; /// In-memory document client to an iroh node running in the same process. /// -/// This is obtained from [`iroh::node::Node::client`]. +/// This is obtained from [`crate::node::Node::client`]. pub type DocMem = Doc>; #[cfg(feature = "cli")] diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 833a9865b5..c57870e19b 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -37,7 +37,7 @@ pub type DownloadFuture = Shared>>; /// /// TODO: Support retries and backoff - become a proper queue... /// TODO: Download requests send via synchronous flume::Sender::send. Investigate if we want async -/// here. We currently use [`Downloader::push`] from [`iroh_sync::Replica::on_insert`] callbacks, +/// here. We currently use [`Downloader::push`] from [`iroh_sync::sync::Replica::on_insert`] callbacks, /// which are sync, thus we need a sync method on the Downloader to push new download requests. /// TODO: Support collections, likely become generic over C: CollectionParser #[derive(Debug, Clone)] diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index 6dcce13642..b2b452e4fd 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -16,8 +16,6 @@ use super::{LiveSync, PeerSource}; /// The SyncEngine combines the [`LiveSync`] actor with the Iroh bytes database and [`Downloader`]. /// -/// TODO: Replace the [`WritableFileDatabase`] with the real thing once -/// https://github.com/n0-computer/iroh/pull/1320 is merged #[derive(Debug, Clone)] pub struct SyncEngine { pub(crate) rt: Handle, From 14ce536c5e8ea595863beb75aa31fe0c99c7643f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 18:25:31 +0200 Subject: [PATCH 066/172] chore: clippy fix --- iroh/src/sync/engine.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index b2b452e4fd..19501197cc 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -80,7 +80,8 @@ impl SyncEngine { /// Stop syncing a document. pub async fn stop_sync(&self, namespace: NamespaceId) -> anyhow::Result<()> { let replica = self.get_replica(&namespace)?; - if let Some(token) = self.active.write().remove(&replica.namespace()) { + let token = self.active.write().remove(&replica.namespace()); + if let Some(token) = token { replica.remove_on_insert(token); self.live.stop_sync(namespace).await?; } From 610d09f9b8d5066c12d58a522b5dcff1645ac1a6 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 22:36:36 +0200 Subject: [PATCH 067/172] refactor: better modules for client --- iroh/src/client.rs | 91 ++++++++--------------------------------- iroh/src/client/quic.rs | 67 ++++++++++++++++++++++++++++++ iroh/src/commands.rs | 9 +--- iroh/tests/sync.rs | 6 +-- 4 files changed, 89 insertions(+), 84 deletions(-) create mode 100644 iroh/src/client/quic.rs diff --git a/iroh/src/client.rs b/iroh/src/client.rs index bc7eb69030..b303e2802c 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -6,10 +6,7 @@ #![allow(missing_docs)] use std::collections::HashMap; -use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::result::Result as StdResult; -use std::sync::Arc; -use std::time::Duration; use anyhow::{anyhow, Result}; use bytes::Bytes; @@ -17,94 +14,40 @@ use futures::{Stream, StreamExt, TryStreamExt}; use iroh_bytes::Hash; use iroh_sync::store::{GetFilter, KeyFilter}; use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; -use quic_rpc::transport::flume::FlumeConnection; use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorListRequest, BytesGetRequest, CounterStats, DocGetRequest, DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocTicket, - DocsCreateRequest, DocsListRequest, ShareMode, StatsGetRequest, VersionRequest, + DocsCreateRequest, DocsListRequest, ProviderService, ShareMode, StatsGetRequest, }; -use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; use crate::sync::PeerSource; -/// In-memory client to an iroh node running in the same process. -/// -/// This is obtained from [`crate::node::Node::client`]. -pub type IrohMemClient = Iroh>; +pub mod mem { + //! Type declarations for an in-memory client to an iroh node running in the same process. -/// In-memory document client to an iroh node running in the same process. -/// -/// This is obtained from [`crate::node::Node::client`]. -pub type DocMem = Doc>; + use quic_rpc::transport::flume::FlumeConnection; -#[cfg(feature = "cli")] -pub use quic::*; - -#[cfg(feature = "cli")] -mod quic { - //! Utility methods to create an RPC client for a node running in a seperate process - - use quic_rpc::transport::quinn::QuinnConnection; + use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; - use super::*; /// RPC client to an iroh node running in a seperate process. - /// - /// This is obtained from [`connect`]. - pub type IrohRpcClient = Iroh>; + pub type RpcClient = + quic_rpc::RpcClient>; - /// RPC document client to an iroh node running in a seperate process. + /// In-memory client to an iroh node running in the same process. /// - /// This is obtained from [`connect`]. - pub type DocRpc = Doc>; - - /// TODO: Change to "/iroh-rpc/1" - pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; - - /// Connect to an iroh node running on the same computer, but in a different process. - pub async fn connect(rpc_port: u16) -> anyhow::Result { - let client = connect_raw(rpc_port).await?; - Ok(Iroh::new(client)) - } + /// This is obtained from [`crate::node::Node::client`]. + pub type Iroh = super::Iroh>; - /// Create a raw RPC client to an iroh node running on the same computer, but in a different - /// process. - pub async fn connect_raw( - rpc_port: u16, - ) -> anyhow::Result< - quic_rpc::RpcClient>, - > { - use anyhow::Context; - let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); - let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; - let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); - let server_name = "localhost".to_string(); - let connection = QuinnConnection::new(endpoint, addr, server_name); - let client = RpcClient::new(connection); - // Do a version request to check if the server is running. - let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) - .await - .context("iroh server is not running")??; - Ok(client) - } - fn create_quinn_client( - bind_addr: SocketAddr, - alpn_protocols: Vec>, - keylog: bool, - ) -> Result { - let keypair = iroh_net::tls::Keypair::generate(); - let tls_client_config = - iroh_net::tls::make_client_config(&keypair, None, alpn_protocols, keylog)?; - let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); - let mut endpoint = quinn::Endpoint::client(bind_addr)?; - let mut transport_config = quinn::TransportConfig::default(); - transport_config.keep_alive_interval(Some(Duration::from_secs(1))); - client_config.transport_config(Arc::new(transport_config)); - endpoint.set_default_client_config(client_config); - Ok(endpoint) - } + /// In-memory document client to an iroh node running in the same process. + /// + /// This is obtained from [`crate::node::Node::client`]. + pub type Doc = super::Doc>; } +#[cfg(feature = "cli")] +pub mod quic; + /// Iroh client pub struct Iroh { rpc: RpcClient, diff --git a/iroh/src/client/quic.rs b/iroh/src/client/quic.rs new file mode 100644 index 0000000000..91db520954 --- /dev/null +++ b/iroh/src/client/quic.rs @@ -0,0 +1,67 @@ +//! Utility methods to create an RPC client to a node running in a seperate process over QUIC + +use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, + time::Duration, +}; + +use quic_rpc::transport::quinn::QuinnConnection; + +use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService, VersionRequest}; + +/// RPC client to an iroh node running in a seperate process. +pub type RpcClient = + quic_rpc::RpcClient>; + +/// Client to an iroh node running in a seperate process. +/// +/// This is obtained from [`connect`]. +pub type Iroh = super::Iroh>; + +/// RPC document client to an iroh node running in a seperate process. +/// +/// This is obtained from [`connect`]. +pub type Doc = super::Doc>; + +/// TODO: Change to "/iroh-rpc/1" +pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; + +/// Connect to an iroh node running on the same computer, but in a different process. +pub async fn connect(rpc_port: u16) -> anyhow::Result { + let client = connect_raw(rpc_port).await?; + Ok(Iroh::new(client)) +} + +/// Create a raw RPC client to an iroh node running on the same computer, but in a different +/// process. +pub async fn connect_raw(rpc_port: u16) -> anyhow::Result { + use anyhow::Context; + let bind_addr = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0).into(); + let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; + let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), rpc_port); + let server_name = "localhost".to_string(); + let connection = QuinnConnection::new(endpoint, addr, server_name); + let client = RpcClient::new(connection); + // Do a version request to check if the server is running. + let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(VersionRequest)) + .await + .context("iroh server is not running")??; + Ok(client) +} +fn create_quinn_client( + bind_addr: SocketAddr, + alpn_protocols: Vec>, + keylog: bool, +) -> anyhow::Result { + let keypair = iroh_net::tls::Keypair::generate(); + let tls_client_config = + iroh_net::tls::make_client_config(&keypair, None, alpn_protocols, keylog)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); + let mut endpoint = quinn::Endpoint::client(bind_addr)?; + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(1))); + client_config.transport_config(Arc::new(transport_config)); + endpoint.set_default_client_config(client_config); + Ok(endpoint) +} diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index fe6e146252..20ca19d797 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -4,12 +4,11 @@ use std::{net::SocketAddr, path::PathBuf}; use anyhow::Result; use clap::{Parser, Subcommand}; use futures::StreamExt; -use iroh::client::connect_raw; +use iroh::client::quic::RpcClient; use iroh::dial::Ticket; use iroh::rpc_protocol::*; use iroh_bytes::{protocol::RequestToken, util::runtime, Hash}; use iroh_net::tls::{Keypair, PeerId}; -use quic_rpc::transport::quinn::QuinnConnection; use crate::config::Config; @@ -28,10 +27,6 @@ pub mod provide; pub mod sync; pub mod validate; -/// RPC client to an iroh node. -pub type RpcClient = - quic_rpc::RpcClient>; - /// Send data. /// /// The iroh command line tool has two modes: provide and get. @@ -406,7 +401,7 @@ pub enum Commands { } pub async fn make_rpc_client(rpc_port: u16) -> anyhow::Result { - connect_raw(rpc_port).await + iroh::client::quic::connect_raw(rpc_port).await } #[cfg(feature = "metrics")] diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 1ef2592d0d..77de0fe7c4 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -5,7 +5,7 @@ use std::{net::SocketAddr, time::Duration}; use anyhow::{anyhow, Result}; use futures::StreamExt; use iroh::{ - client::DocMem, + client::mem::Doc, collection::IrohCollectionParser, node::{Builder, Node}, rpc_protocol::ShareMode, @@ -124,12 +124,12 @@ async fn sync_full_basic() -> Result<()> { Ok(()) } -async fn assert_latest(doc: &DocMem, key: &[u8], value: &[u8]) { +async fn assert_latest(doc: &Doc, key: &[u8], value: &[u8]) { let content = get_latest(doc, key).await.unwrap(); assert_eq!(content, value.to_vec()); } -async fn get_latest(doc: &DocMem, key: &[u8]) -> anyhow::Result> { +async fn get_latest(doc: &Doc, key: &[u8]) -> anyhow::Result> { let filter = GetFilter::new().with_key(key.to_vec()); let entry = doc .get(filter) From a16ec9a12c1905f1c084e59bc4d52e1973890f3b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 10 Aug 2023 22:49:03 +0200 Subject: [PATCH 068/172] fix: clippy --- iroh/src/commands/sync.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 5e25c86f24..1285a38ce0 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -18,6 +18,7 @@ pub type Iroh = iroh::client::Iroh Date: Thu, 10 Aug 2023 22:57:35 +0200 Subject: [PATCH 069/172] fix: clippy --- iroh/tests/sync.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 77de0fe7c4..837480364c 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -1,5 +1,3 @@ -#![cfg(all(feature = "sync"))] - use std::{net::SocketAddr, time::Duration}; use anyhow::{anyhow, Result}; From 3bc14831df1ab4f65d8681164a0248c0c20b9fd5 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 11 Aug 2023 10:08:06 +0200 Subject: [PATCH 070/172] fix: feature flags --- iroh/tests/sync.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 837480364c..667b7d9d55 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "mem-db")] + use std::{net::SocketAddr, time::Duration}; use anyhow::{anyhow, Result}; From 9d5d06aa5755ae42f714e6f23a74240d2ab4f688 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 11 Aug 2023 10:20:53 +0200 Subject: [PATCH 071/172] fix: cleanup imports --- iroh/src/client.rs | 23 +---------------------- iroh/src/client/mem.rs | 20 ++++++++++++++++++++ iroh/src/client/quic.rs | 10 ++++------ iroh/src/commands.rs | 1 - iroh/src/commands/provide.rs | 3 ++- iroh/src/node.rs | 12 +++--------- 6 files changed, 30 insertions(+), 39 deletions(-) create mode 100644 iroh/src/client/mem.rs diff --git a/iroh/src/client.rs b/iroh/src/client.rs index b303e2802c..1e34624e7f 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -23,28 +23,7 @@ use crate::rpc_protocol::{ }; use crate::sync::PeerSource; -pub mod mem { - //! Type declarations for an in-memory client to an iroh node running in the same process. - - use quic_rpc::transport::flume::FlumeConnection; - - use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; - - /// RPC client to an iroh node running in a seperate process. - pub type RpcClient = - quic_rpc::RpcClient>; - - /// In-memory client to an iroh node running in the same process. - /// - /// This is obtained from [`crate::node::Node::client`]. - pub type Iroh = super::Iroh>; - - /// In-memory document client to an iroh node running in the same process. - /// - /// This is obtained from [`crate::node::Node::client`]. - pub type Doc = super::Doc>; -} - +pub mod mem; #[cfg(feature = "cli")] pub mod quic; diff --git a/iroh/src/client/mem.rs b/iroh/src/client/mem.rs new file mode 100644 index 0000000000..1d72f7368a --- /dev/null +++ b/iroh/src/client/mem.rs @@ -0,0 +1,20 @@ +//! Type declarations for an in-memory client to an iroh node running in the same process. +//! +//! The in-memory client is obtained directly from a running node through +//! [`crate::node::Node::client`] + +use quic_rpc::transport::flume::FlumeConnection; + +use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService}; + +/// RPC client to an iroh node running in the same process. +pub type RpcClient = + quic_rpc::RpcClient>; + +/// In-memory client to an iroh node running in the same process. +/// +/// This is obtained from [`crate::node::Node::client`]. +pub type Iroh = super::Iroh>; + +/// In-memory document client to an iroh node running in the same process. +pub type Doc = super::Doc>; diff --git a/iroh/src/client/quic.rs b/iroh/src/client/quic.rs index 91db520954..09edd7e0e0 100644 --- a/iroh/src/client/quic.rs +++ b/iroh/src/client/quic.rs @@ -1,4 +1,4 @@ -//! Utility methods to create an RPC client to a node running in a seperate process over QUIC +//! Type declarations and utility functions for an RPC client to an iroh node running in a seperate process. use std::{ net::{Ipv4Addr, SocketAddr, SocketAddrV4}, @@ -10,6 +10,9 @@ use quic_rpc::transport::quinn::QuinnConnection; use crate::rpc_protocol::{ProviderRequest, ProviderResponse, ProviderService, VersionRequest}; +/// TODO: Change to "/iroh-rpc/1" +pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; + /// RPC client to an iroh node running in a seperate process. pub type RpcClient = quic_rpc::RpcClient>; @@ -20,13 +23,8 @@ pub type RpcClient = pub type Iroh = super::Iroh>; /// RPC document client to an iroh node running in a seperate process. -/// -/// This is obtained from [`connect`]. pub type Doc = super::Doc>; -/// TODO: Change to "/iroh-rpc/1" -pub const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; - /// Connect to an iroh node running on the same computer, but in a different process. pub async fn connect(rpc_port: u16) -> anyhow::Result { let client = connect_raw(rpc_port).await?; diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 20ca19d797..09544aa1ae 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -15,7 +15,6 @@ use crate::config::Config; use self::provide::{ProvideOptions, ProviderRpcPort}; const DEFAULT_RPC_PORT: u16 = 0x1337; -const RPC_ALPN: [u8; 17] = *b"n0/provider-rpc/1"; const MAX_RPC_CONNECTIONS: u32 = 16; const MAX_RPC_STREAMS: u64 = 1024; diff --git a/iroh/src/commands/provide.rs b/iroh/src/commands/provide.rs index 3223ec8536..26456bf7f6 100644 --- a/iroh/src/commands/provide.rs +++ b/iroh/src/commands/provide.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::{anyhow, ensure, Context, Result}; use iroh::{ baomap::flat, + client::quic::RPC_ALPN, collection::IrohCollectionParser, node::{Node, StaticTokenAuthHandler}, rpc_protocol::{ProvideRequest, ProviderRequest, ProviderResponse, ProviderService}, @@ -24,7 +25,7 @@ use crate::config::IrohPaths; use super::{ add::{aggregate_add_response, print_add_response}, - MAX_RPC_CONNECTIONS, MAX_RPC_STREAMS, RPC_ALPN, + MAX_RPC_CONNECTIONS, MAX_RPC_STREAMS, }; #[derive(Debug)] diff --git a/iroh/src/node.rs b/iroh/src/node.rs index bca8191306..aba1ed4614 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -645,19 +645,13 @@ impl Node { } /// Returns a handle that can be used to do RPC calls to the node internally. - /// - /// TODO: remove and replace with client? - pub fn controller( - &self, - ) -> RpcClient> { + pub fn controller(&self) -> crate::client::mem::RpcClient { RpcClient::new(self.inner.controller.clone()) } /// Return a client to control this node over an in-memory channel. - pub fn client( - &self, - ) -> super::client::Iroh> { - super::client::Iroh::new(self.controller()) + pub fn client(&self) -> crate::client::mem::Iroh { + crate::client::Iroh::new(self.controller()) } /// Return a single token containing everything needed to get a hash. From 23d138b51af0b39f6db23bfb3ee84d10704e9431 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 11 Aug 2023 10:37:33 +0200 Subject: [PATCH 072/172] fix: gossip endpoint init --- iroh/src/node.rs | 8 ++++++-- iroh/tests/sync.rs | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index aba1ed4614..d67cc7af9c 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -436,9 +436,13 @@ where let cancel_token = handler.inner.cancel_token.clone(); // forward our initial endpoints to the gossip protocol + // it may happen the the first endpoint update callback is missed because the gossip cell + // is only initialized once the endpoint is fully bound if let Ok(local_endpoints) = server.local_endpoints().await { - debug!(me = ?server.peer_id(), "gossip initial update: {local_endpoints:?}"); - gossip.update_endpoints(&local_endpoints).ok(); + if !local_endpoints.is_empty() { + debug!(me = ?server.peer_id(), "gossip initial update: {local_endpoints:?}"); + gossip.update_endpoints(&local_endpoints).ok(); + } } loop { diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 667b7d9d55..f717107fd6 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -86,7 +86,7 @@ async fn sync_full_basic() -> Result<()> { let doc = iroh.import_doc(ticket.clone()).await?; // todo: events over rpc to not use sleep... - tokio::time::sleep(Duration::from_secs(3)).await; + tokio::time::sleep(Duration::from_secs(2)).await; assert_latest(&doc, b"k1", b"v1").await; let key = b"k2"; @@ -94,7 +94,7 @@ async fn sync_full_basic() -> Result<()> { doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; assert_latest(&doc, key, value).await; // todo: events - tokio::time::sleep(Duration::from_secs(3)).await; + tokio::time::sleep(Duration::from_secs(2)).await; assert_latest(&doc1, key, value).await; doc }; @@ -106,7 +106,7 @@ async fn sync_full_basic() -> Result<()> { let doc = iroh.import_doc(ticket).await?; // todo: events - tokio::time::sleep(Duration::from_secs(3)).await; + tokio::time::sleep(Duration::from_secs(2)).await; assert_latest(&doc, b"k1", b"v1").await; assert_latest(&doc, b"k2", b"v2").await; doc From cc4c5a9b9d66ef09216cc50aea809d9e4aab780c Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 11 Aug 2023 10:50:33 +0200 Subject: [PATCH 073/172] fix: less arguments for clippy --- iroh/examples/sync.rs | 491 ++++++++++++++++++++++-------------------- 1 file changed, 256 insertions(+), 235 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 3efa01228f..4893dc66a4 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -278,6 +278,17 @@ async fn run(args: Args) -> anyhow::Result<()> { } })); + let repl_state = ReplState { + rt, + store: docs, + author, + doc, + db, + ticket: our_ticket, + log_filter, + current_watch, + }; + loop { // wait for a command from the input repl thread let Some((cmd, to_repl_tx)) = cmd_rx.recv().await else { @@ -295,7 +306,7 @@ async fn run(args: Args) -> anyhow::Result<()> { _ = tokio::signal::ctrl_c() => { println!("> aborted"); } - res = handle_command(cmd, &rt, &docs, &author, &doc, &db, &our_ticket, &log_filter, ¤t_watch) => if let Err(err) = res { + res = repl_state.handle_command(cmd) => if let Err(err) = res { println!("> error: {err}"); }, }; @@ -315,271 +326,281 @@ async fn run(args: Args) -> anyhow::Result<()> { Ok(()) } -async fn handle_command( - cmd: Cmd, - rt: &runtime::Handle, - store: &store::fs::Store, - author: &Author, - doc: &Doc, - db: &iroh::baomap::flat::Store, - ticket: &Ticket, - log_filter: &LogLevelReload, - current_watch: &Arc>>, -) -> anyhow::Result<()> { - match cmd { - Cmd::Set { key, value } => { - let value = value.into_bytes(); - let len = value.len(); - let hash = db.import_bytes(value.into()).await?; - doc.insert(key, author, hash, len as u64)?; - } - Cmd::Get { - key, - print_content, - prefix, - } => { - let entries = if prefix { - store.get_all_by_prefix(doc.namespace(), key.as_bytes())? - } else { - store.get_all_by_key(doc.namespace(), key.as_bytes())? - }; - for entry in entries { - let (_id, entry) = entry?; - println!("{}", fmt_entry(&entry)); - if print_content { - println!("{}", fmt_content(db, &entry).await); +struct ReplState { + rt: runtime::Handle, + store: store::fs::Store, + author: Author, + doc: Doc, + db: iroh::baomap::flat::Store, + ticket: Ticket, + log_filter: LogLevelReload, + current_watch: Arc>>, +} + +impl ReplState { + async fn handle_command(&self, cmd: Cmd) -> anyhow::Result<()> { + match cmd { + Cmd::Set { key, value } => { + let value = value.into_bytes(); + let len = value.len(); + let hash = self.db.import_bytes(value.into()).await?; + self.doc.insert(key, &self.author, hash, len as u64)?; + } + Cmd::Get { + key, + print_content, + prefix, + } => { + let entries = if prefix { + self.store + .get_all_by_prefix(self.doc.namespace(), key.as_bytes())? + } else { + self.store + .get_all_by_key(self.doc.namespace(), key.as_bytes())? + }; + for entry in entries { + let (_id, entry) = entry?; + println!("{}", fmt_entry(&entry)); + if print_content { + println!("{}", fmt_content(&self.db, &entry).await); + } } } - } - Cmd::Watch { key } => { - println!("watching key: '{key}'"); - current_watch.lock().unwrap().replace(key); - } - Cmd::WatchCancel => match current_watch.lock().unwrap().take() { - Some(key) => { - println!("canceled watching key: '{key}'"); + Cmd::Watch { key } => { + println!("watching key: '{key}'"); + self.current_watch.lock().unwrap().replace(key); } - None => { - println!("no watch active"); + Cmd::WatchCancel => match self.current_watch.lock().unwrap().take() { + Some(key) => { + println!("canceled watching key: '{key}'"); + } + None => { + println!("no watch active"); + } + }, + Cmd::Ls { prefix } => { + let entries = match prefix { + None => self.store.get_all(self.doc.namespace())?, + Some(prefix) => self + .store + .get_all_by_prefix(self.doc.namespace(), prefix.as_bytes())?, + }; + let mut count = 0; + for entry in entries { + let (_id, entry) = entry?; + count += 1; + println!("{}", fmt_entry(&entry),); + } + println!("> {} entries", count); } - }, - Cmd::Ls { prefix } => { - let entries = match prefix { - None => store.get_all(doc.namespace())?, - Some(prefix) => store.get_all_by_prefix(doc.namespace(), prefix.as_bytes())?, - }; - let mut count = 0; - for entry in entries { - let (_id, entry) = entry?; - count += 1; - println!("{}", fmt_entry(&entry),); + Cmd::Ticket => { + println!("Ticket: {}", self.ticket); } - println!("> {} entries", count); - } - Cmd::Ticket => { - println!("Ticket: {ticket}"); - } - Cmd::Log { directive } => { - let next_filter = EnvFilter::from_str(&directive)?; - log_filter.modify(|layer| *layer = next_filter)?; - } - Cmd::Stats => get_stats(), - Cmd::Fs(cmd) => handle_fs_command(cmd, store, db, doc, author).await?, - Cmd::Hammer { - prefix, - threads, - count, - size, - mode, - } => { - println!( + Cmd::Log { directive } => { + let next_filter = EnvFilter::from_str(&directive)?; + self.log_filter.modify(|layer| *layer = next_filter)?; + } + Cmd::Stats => get_stats(), + Cmd::Fs(cmd) => self.handle_fs_command(cmd).await?, + Cmd::Hammer { + prefix, + threads, + count, + size, + mode, + } => { + println!( "> Hammering with prefix \"{prefix}\" for {threads} x {count} messages of size {size} bytes in {mode} mode", mode = format!("{mode:?}").to_lowercase() ); - let start = Instant::now(); - let mut handles: Vec>> = Vec::new(); - match mode { - HammerMode::Set => { - let mut bytes = vec![0; size]; - // TODO: Add a flag to fill content differently per entry to be able to - // test downloading too - bytes.fill(97); - for t in 0..threads { - let prefix = prefix.clone(); - let doc = doc.clone(); - let bytes = bytes.clone(); - let db = db.clone(); - let author = author.clone(); - let handle = rt.main().spawn(async move { - for i in 0..count { - let value = String::from_utf8(bytes.clone()).unwrap().into_bytes(); - let len = value.len(); - let key = format!("{}/{}/{}", prefix, t, i); - let hash = db.import_bytes(value.into()).await?; - doc.insert(key, &author, hash, len as u64)?; - } - Ok(count) - }); - handles.push(handle); + let start = Instant::now(); + let mut handles: Vec>> = Vec::new(); + match mode { + HammerMode::Set => { + let mut bytes = vec![0; size]; + // TODO: Add a flag to fill content differently per entry to be able to + // test downloading too + bytes.fill(97); + for t in 0..threads { + let prefix = prefix.clone(); + let doc = self.doc.clone(); + let bytes = bytes.clone(); + let db = self.db.clone(); + let author = self.author.clone(); + let handle = self.rt.main().spawn(async move { + for i in 0..count { + let value = + String::from_utf8(bytes.clone()).unwrap().into_bytes(); + let len = value.len(); + let key = format!("{}/{}/{}", prefix, t, i); + let hash = db.import_bytes(value.into()).await?; + doc.insert(key, &author, hash, len as u64)?; + } + Ok(count) + }); + handles.push(handle); + } } - } - HammerMode::Get => { - for t in 0..threads { - let prefix = prefix.clone(); - let doc = doc.clone(); - let store = store.clone(); - let handle = rt.main().spawn(async move { - let mut read = 0; - for i in 0..count { - let key = format!("{}/{}/{}", prefix, t, i); - let entries = - store.get_all_by_key(doc.namespace(), key.as_bytes())?; - for entry in entries { - let (_id, entry) = entry?; - let _content = fmt_content_simple(&doc, &entry); - read += 1; + HammerMode::Get => { + for t in 0..threads { + let prefix = prefix.clone(); + let doc = self.doc.clone(); + let store = self.store.clone(); + let handle = self.rt.main().spawn(async move { + let mut read = 0; + for i in 0..count { + let key = format!("{}/{}/{}", prefix, t, i); + let entries = + store.get_all_by_key(doc.namespace(), key.as_bytes())?; + for entry in entries { + let (_id, entry) = entry?; + let _content = fmt_content_simple(&doc, &entry); + read += 1; + } } - } - Ok(read) - }); - handles.push(handle); + Ok(read) + }); + handles.push(handle); + } } } - } - let mut total_count = 0; - for result in futures::future::join_all(handles).await { - // Check that no errors ocurred and count rows inserted/read - total_count += result??; - } + let mut total_count = 0; + for result in futures::future::join_all(handles).await { + // Check that no errors ocurred and count rows inserted/read + total_count += result??; + } - let diff = start.elapsed().as_secs_f64(); - println!( + let diff = start.elapsed().as_secs_f64(); + println!( "> Hammering done in {diff:.2}s for {total_count} messages with total of {size}", size = HumanBytes(total_count as u64 * size as u64), ); + } + Cmd::Exit => {} } - Cmd::Exit => {} + Ok(()) } - Ok(()) -} -async fn handle_fs_command( - cmd: FsCmd, - store: &store::fs::Store, - db: &iroh::baomap::flat::Store, - doc: &Doc, - author: &Author, -) -> anyhow::Result<()> { - match cmd { - FsCmd::ImportFile { file_path, key } => { - let file_path = canonicalize_path(&file_path)?.canonicalize()?; - let (hash, len) = db - .import( - file_path.clone(), - ImportMode::Copy, - IgnoreProgressSender::default(), - ) - .await?; - doc.insert(key, author, hash, len)?; - println!( - "> imported {file_path:?}: {} ({})", - fmt_hash(hash), - HumanBytes(len) - ); - } - FsCmd::ImportDir { - dir_path, - mut key_prefix, - } => { - if key_prefix.ends_with('/') { - key_prefix.pop(); + async fn handle_fs_command(&self, cmd: FsCmd) -> anyhow::Result<()> { + match cmd { + FsCmd::ImportFile { file_path, key } => { + let file_path = canonicalize_path(&file_path)?.canonicalize()?; + let (hash, len) = self + .db + .import( + file_path.clone(), + ImportMode::Copy, + IgnoreProgressSender::default(), + ) + .await?; + self.doc.insert(key, &self.author, hash, len)?; + println!( + "> imported {file_path:?}: {} ({})", + fmt_hash(hash), + HumanBytes(len) + ); } - let root = canonicalize_path(&dir_path)?.canonicalize()?; - let files = walkdir::WalkDir::new(&root).into_iter(); - // TODO: parallelize - for file in files { - let file = file?; - if file.file_type().is_file() { - let relative = file.path().strip_prefix(&root)?.to_string_lossy(); - if relative.is_empty() { - warn!("invalid file path: {:?}", file.path()); - continue; + FsCmd::ImportDir { + dir_path, + mut key_prefix, + } => { + if key_prefix.ends_with('/') { + key_prefix.pop(); + } + let root = canonicalize_path(&dir_path)?.canonicalize()?; + let files = walkdir::WalkDir::new(&root).into_iter(); + // TODO: parallelize + for file in files { + let file = file?; + if file.file_type().is_file() { + let relative = file.path().strip_prefix(&root)?.to_string_lossy(); + if relative.is_empty() { + warn!("invalid file path: {:?}", file.path()); + continue; + } + let key = format!("{key_prefix}/{relative}"); + let (hash, len) = self + .db + .import( + file.path().into(), + ImportMode::Copy, + IgnoreProgressSender::default(), + ) + .await?; + self.doc.insert(key, &self.author, hash, len)?; + println!( + "> imported {relative}: {} ({})", + fmt_hash(hash), + HumanBytes(len) + ); } - let key = format!("{key_prefix}/{relative}"); - let (hash, len) = db - .import( - file.path().into(), - ImportMode::Copy, - IgnoreProgressSender::default(), - ) - .await?; - doc.insert(key, author, hash, len)?; - println!( - "> imported {relative}: {} ({})", - fmt_hash(hash), - HumanBytes(len) - ); } } - } - FsCmd::ExportDir { - mut key_prefix, - dir_path, - } => { - if !key_prefix.ends_with('/') { - key_prefix.push('/'); - } - let root = canonicalize_path(&dir_path)?; - println!("> exporting {key_prefix} to {root:?}"); - let entries = store.get_latest_by_prefix(doc.namespace(), key_prefix.as_bytes())?; - let mut checked_dirs = HashSet::new(); - for entry in entries { - let (id, entry) = entry?; - let key = id.key(); - let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; - let len = entry.entry().record().content_len(); - let blob = db.get(entry.content_hash()); - if let Some(blob) = blob { - let mut reader = blob.data_reader().await?; - let path = root.join(&relative); - let parent = path.parent().unwrap(); - if !checked_dirs.contains(parent) { - tokio::fs::create_dir_all(&parent).await?; - checked_dirs.insert(parent.to_owned()); + FsCmd::ExportDir { + mut key_prefix, + dir_path, + } => { + if !key_prefix.ends_with('/') { + key_prefix.push('/'); + } + let root = canonicalize_path(&dir_path)?; + println!("> exporting {key_prefix} to {root:?}"); + let entries = self + .store + .get_latest_by_prefix(self.doc.namespace(), key_prefix.as_bytes())?; + let mut checked_dirs = HashSet::new(); + for entry in entries { + let (id, entry) = entry?; + let key = id.key(); + let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; + let len = entry.entry().record().content_len(); + let blob = self.db.get(entry.content_hash()); + if let Some(blob) = blob { + let mut reader = blob.data_reader().await?; + let path = root.join(&relative); + let parent = path.parent().unwrap(); + if !checked_dirs.contains(parent) { + tokio::fs::create_dir_all(&parent).await?; + checked_dirs.insert(parent.to_owned()); + } + let mut file = tokio::fs::File::create(&path).await?; + copy(&mut reader, &mut file).await?; + println!( + "> exported {} to {path:?} ({})", + fmt_hash(entry.content_hash()), + HumanBytes(len) + ); } + } + } + FsCmd::ExportFile { key, file_path } => { + let path = canonicalize_path(&file_path)?; + // TODO: Fix + let entry = self + .store + .get_latest_by_key(self.doc.namespace(), &key)? + .next(); + if let Some(entry) = entry { + let (_, entry) = entry?; + println!("> exporting {key} to {path:?}"); + let parent = path.parent().ok_or_else(|| anyhow!("Invalid path"))?; + tokio::fs::create_dir_all(&parent).await?; let mut file = tokio::fs::File::create(&path).await?; + let blob = self + .db + .get(entry.content_hash()) + .ok_or_else(|| anyhow!(format!("content for {key} is not available")))?; + let mut reader = blob.data_reader().await?; copy(&mut reader, &mut file).await?; - println!( - "> exported {} to {path:?} ({})", - fmt_hash(entry.content_hash()), - HumanBytes(len) - ); + } else { + println!("> key not found, abort"); } } } - FsCmd::ExportFile { key, file_path } => { - let path = canonicalize_path(&file_path)?; - // TODO: Fix - let entry = store.get_latest_by_key(doc.namespace(), &key)?.next(); - if let Some(entry) = entry { - let (_, entry) = entry?; - println!("> exporting {key} to {path:?}"); - let parent = path.parent().ok_or_else(|| anyhow!("Invalid path"))?; - tokio::fs::create_dir_all(&parent).await?; - let mut file = tokio::fs::File::create(&path).await?; - let blob = db - .get(entry.content_hash()) - .ok_or_else(|| anyhow!(format!("content for {key} is not available")))?; - let mut reader = blob.data_reader().await?; - copy(&mut reader, &mut file).await?; - } else { - println!("> key not found, abort"); - } - } - } - Ok(()) + Ok(()) + } } #[derive(Parser, Debug)] From d05e32cb523f5f9d35580b56652be560881bff8c Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 11 Aug 2023 12:26:09 +0200 Subject: [PATCH 074/172] increase logging in cli tests --- iroh/tests/cli.rs | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/iroh/tests/cli.rs b/iroh/tests/cli.rs index 600b960ec6..4f190eee6b 100644 --- a/iroh/tests/cli.rs +++ b/iroh/tests/cli.rs @@ -90,16 +90,41 @@ fn cli_provide_tree() -> Result<()> { test_provide_get_loop(&dir, Input::Path, Output::Path) } -fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { +fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + let src = src.as_ref(); + let dst = dst.as_ref(); std::fs::create_dir_all(&dst)?; for entry in std::fs::read_dir(src)? { - let entry = entry?; - let ty = entry.file_type()?; + let entry = entry.with_context(|| { + format!( + "failed to read directory entry in `{}`", + src.to_string_lossy() + ) + })?; + let ty = entry.file_type().with_context(|| { + format!( + "failed to get file type for file `{}`", + entry.path().to_string_lossy() + ) + })?; + let src = entry.path(); + let dst = dst.join(entry.file_name()); if ty.is_dir() { - copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; + copy_dir_all(&src, &dst).with_context(|| { + format!( + "failed to copy directory `{}` to `{}`", + src.to_string_lossy(), + dst.to_string_lossy() + ) + })?; } else { - let to = dst.as_ref().join(entry.file_name()); - std::fs::copy(entry.path(), to)?; + std::fs::copy(&src, &dst).with_context(|| { + format!( + "failed to copy file `{}` to `{}`", + src.to_string_lossy(), + dst.to_string_lossy() + ) + })?; } } Ok(()) From ef0892d80ef3ba0ea6c0280d1873c998abc0341b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 11 Aug 2023 12:26:35 +0200 Subject: [PATCH 075/172] chore: remove obsolete file --- iroh/src/database/flat/writable.rs | 208 ----------------------------- 1 file changed, 208 deletions(-) delete mode 100644 iroh/src/database/flat/writable.rs diff --git a/iroh/src/database/flat/writable.rs b/iroh/src/database/flat/writable.rs deleted file mode 100644 index 9427158b5f..0000000000 --- a/iroh/src/database/flat/writable.rs +++ /dev/null @@ -1,208 +0,0 @@ -#![allow(missing_docs)] -//! Quick-and-dirty writable database -//! -//! I wrote this while diving into iroh-bytes, wildly copying code around. This will be solved much -//! nicer with the upcoming generic writable database branch by @rklaehn. - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; - -use anyhow::Context; -use bytes::Bytes; -use iroh_io::{AsyncSliceWriter, File}; -use range_collections::RangeSet2; -use tokio::io::AsyncRead; - -use iroh_bytes::{ - get::fsm, - protocol::{GetRequest, RangeSpecSeq, Request}, - Hash, -}; - -use crate::database::flat::{create_collection, DataSource, Database, DbEntry, FNAME_PATHS}; - -/// A blob database into which new blobs can be inserted. -/// -/// Blobs can be inserted either from bytes or by downloading from open connections to peers. -/// New blobs will be saved as files with a filename based on their hash. -/// -/// TODO: Replace with the generic writable database. -#[derive(Debug, Clone)] -pub struct WritableFileDatabase { - db: Database, - storage: Arc, -} - -impl WritableFileDatabase { - pub async fn new(data_path: PathBuf) -> anyhow::Result { - let storage = Arc::new(StoragePaths::new(data_path).await?); - let db = if storage.db_path.join(FNAME_PATHS).exists() { - Database::load(&storage.db_path).await.with_context(|| { - format!( - "Failed to load iroh database from {}", - storage.db_path.display() - ) - })? - } else { - Database::default() - }; - Ok(Self { db, storage }) - } - - pub fn db(&self) -> &Database { - &self.db - } - - pub async fn save(&self) -> anyhow::Result<()> { - self.db.save(&self.storage.db_path).await?; - Ok(()) - } - - pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64)> { - let (hash, size, entry) = self.storage.put_bytes(data).await?; - self.db.union_with(HashMap::from_iter([(hash, entry)])); - Ok((hash, size)) - } - - pub async fn put_reader(&self, data: impl AsyncRead + Unpin) -> anyhow::Result<(Hash, u64)> { - let (hash, size, entry) = self.storage.put_reader(data).await?; - self.db.union_with(HashMap::from_iter([(hash, entry)])); - Ok((hash, size)) - } - - pub async fn get_size(&self, hash: &Hash) -> Option { - Some(self.db.get(hash)?.size().await) - } - - pub fn has(&self, hash: &Hash) -> bool { - self.db.to_inner().contains_key(hash) - } - pub async fn download_single( - &self, - conn: quinn::Connection, - hash: Hash, - ) -> anyhow::Result> { - // 1. Download to temp file - let temp_path = { - let temp_path = self.storage.temp_path(); - let request = - Request::Get(GetRequest::new(hash, RangeSpecSeq::new([RangeSet2::all()]))); - let response = fsm::start(conn, request); - let connected = response.next().await?; - - let fsm::ConnectedNext::StartRoot(curr) = connected.next().await? else { - return Ok(None) - }; - let header = curr.next(); - - let path = temp_path.clone(); - let mut data_file = File::create(move || { - std::fs::OpenOptions::new() - .write(true) - .create(true) - .open(path) - }) - .await - .context("failed to create local tempfile")?; - - let (curr, _size) = header.next().await.context("failed to read blob content")?; - let _curr = curr - .write_all(&mut data_file) - .await - .context("failed to write blob content to tempfile")?; - // Flush the data file first, it is the only thing that matters at this point - data_file.sync().await.context("fsync failed")?; - temp_path - }; - - // 2. Insert into database - let (hash, size, entry) = self - .storage - .move_to_blobs(&temp_path) - .await - .context("failed to move to blobs dir")?; - let entries = HashMap::from_iter([(hash, entry)]); - self.db.union_with(entries); - Ok(Some((hash, size))) - } -} - -#[derive(Debug)] -pub struct StoragePaths { - blob_path: PathBuf, - temp_path: PathBuf, - db_path: PathBuf, -} - -impl StoragePaths { - pub async fn new(data_path: PathBuf) -> anyhow::Result { - let blob_path = data_path.join("blobs"); - let temp_path = data_path.join("temp"); - let db_path = data_path.join("db"); - tokio::fs::create_dir_all(&blob_path).await?; - tokio::fs::create_dir_all(&temp_path).await?; - tokio::fs::create_dir_all(&db_path).await?; - Ok(Self { - blob_path, - temp_path, - db_path, - }) - } - - pub async fn put_bytes(&self, data: Bytes) -> anyhow::Result<(Hash, u64, DbEntry)> { - let temp_path = self.temp_path(); - tokio::fs::write(&temp_path, &data).await?; - let (hash, size, entry) = self.move_to_blobs(&temp_path).await?; - Ok((hash, size, entry)) - } - - pub async fn put_reader( - &self, - mut reader: impl AsyncRead + Unpin, - ) -> anyhow::Result<(Hash, u64, DbEntry)> { - let temp_path = self.temp_path(); - let mut file = tokio::fs::OpenOptions::new() - .write(true) - .create(true) - .open(&temp_path) - .await?; - tokio::io::copy(&mut reader, &mut file).await?; - let (hash, size, entry) = self.move_to_blobs(&temp_path).await?; - Ok((hash, size, entry)) - } - - async fn move_to_blobs(&self, path: &PathBuf) -> anyhow::Result<(Hash, u64, DbEntry)> { - let datasource = DataSource::new(path.clone()); - // TODO: this needlessly creates a collection, but that's what's pub atm in iroh-bytes - let (db, _collection_hash) = create_collection(vec![datasource]).await?; - // the actual blob is the first entry in the external entries in the created collection - let (hash, _path, _len) = db.external().next().unwrap(); - let Some(DbEntry::External { outboard, size, .. }) = db.get(&hash) else { - unreachable!("just inserted"); - }; - - let final_path = prepare_hash_dir(&self.blob_path, &hash).await?; - tokio::fs::rename(&path, &final_path).await?; - let entry = DbEntry::External { - outboard, - path: final_path, - size, - }; - Ok((hash, size, entry)) - } - - fn temp_path(&self) -> PathBuf { - let name = hex::encode(rand::random::().to_be_bytes()); - self.temp_path.join(name) - } -} - -async fn prepare_hash_dir(path: &Path, hash: &Hash) -> anyhow::Result { - let hash = hex::encode(hash.as_ref()); - let path = path.join(&hash[0..2]).join(&hash[2..4]).join(&hash[4..]); - tokio::fs::create_dir_all(path.parent().unwrap()).await?; - Ok(path) -} From a285c6d051c2353d7be0b949ad456d23073113e5 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 11 Aug 2023 12:47:38 +0200 Subject: [PATCH 076/172] chore: clippy --- iroh/tests/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/tests/cli.rs b/iroh/tests/cli.rs index 4f190eee6b..976d0268c0 100644 --- a/iroh/tests/cli.rs +++ b/iroh/tests/cli.rs @@ -93,7 +93,7 @@ fn cli_provide_tree() -> Result<()> { fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { let src = src.as_ref(); let dst = dst.as_ref(); - std::fs::create_dir_all(&dst)?; + std::fs::create_dir_all(dst)?; for entry in std::fs::read_dir(src)? { let entry = entry.with_context(|| { format!( From 96c9a4335f90a27432b2d60b389415ef299865b3 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Mon, 14 Aug 2023 12:08:17 +0200 Subject: [PATCH 077/172] feat: implement minimal subscription for sync (#1346) --- iroh/src/client.rs | 14 +++++++++--- iroh/src/node.rs | 6 ++++++ iroh/src/rpc_protocol.rs | 24 ++++++++++++++++++++- iroh/src/sync/engine.rs | 2 +- iroh/src/sync/live.rs | 46 +++++++++++++++++++++++++++++++++++++++- iroh/src/sync/rpc.rs | 18 ++++++++++++++-- 6 files changed, 102 insertions(+), 8 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 1e34624e7f..9ebc77507a 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -18,10 +18,10 @@ use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorListRequest, BytesGetRequest, CounterStats, DocGetRequest, - DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocTicket, - DocsCreateRequest, DocsListRequest, ProviderService, ShareMode, StatsGetRequest, + DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocSubscribeRequest, + DocTicket, DocsCreateRequest, DocsListRequest, ProviderService, ShareMode, StatsGetRequest, }; -use crate::sync::PeerSource; +use crate::sync::{LiveEvent, PeerSource}; pub mod mem; #[cfg(feature = "cli")] @@ -180,6 +180,14 @@ where } // TODO: add stop_sync + + pub async fn subscribe(&self) -> anyhow::Result>> { + let stream = self + .rpc + .server_streaming(DocSubscribeRequest { doc_id: self.id }) + .await?; + Ok(stream.map_ok(|res| res.event).map_err(Into::into)) + } } fn flatten( diff --git a/iroh/src/node.rs b/iroh/src/node.rs index d67cc7af9c..3d36393a1c 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1253,6 +1253,12 @@ fn handle_rpc_request< }) .await } + DocSubscribe(msg) => { + chan.server_streaming(msg, handler, |handler, req| { + handler.inner.sync.doc_subscribe(req) + }) + .await + } // TODO: make streaming BytesGet(msg) => chan.rpc(msg, handler, RpcHandler::bytes_get).await, } diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 652168e24f..3098a14a7c 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -29,7 +29,7 @@ use serde::{Deserialize, Serialize}; pub use iroh_bytes::{baomap::ValidateProgress, provider::ProvideProgress, util::RpcResult}; -use crate::sync::PeerSource; +use crate::sync::{LiveEvent, PeerSource}; /// A 32-byte key or token pub type KeyBytes = [u8; 32]; @@ -384,6 +384,26 @@ pub struct AuthorShareResponse { pub key: KeyBytes, } +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocSubscribeRequest { + pub doc_id: NamespaceId, +} + +impl Msg for DocSubscribeRequest { + type Pattern = ServerStreaming; +} + +impl ServerStreamingMsg for DocSubscribeRequest { + type Response = DocSubscribeResponse; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocSubscribeResponse { + pub event: LiveEvent, +} + /// todo #[derive(Serialize, Deserialize, Debug)] pub struct DocsListRequest {} @@ -612,6 +632,7 @@ pub enum ProviderRequest { DocGet(DocGetRequest), DocStartSync(DocStartSyncRequest), // DocGetContent(DocGetContentRequest), DocShare(DocShareRequest), // DocGetContent(DocGetContentRequest), + DocSubscribe(DocSubscribeRequest), BytesGet(BytesGetRequest), @@ -652,6 +673,7 @@ pub enum ProviderResponse { DocGet(RpcResult), DocJoin(RpcResult), DocShare(RpcResult), + DocSubscribe(DocSubscribeResponse), BytesGet(RpcResult), diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index 19501197cc..3493f62534 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -22,7 +22,7 @@ pub struct SyncEngine { pub(crate) store: S, pub(crate) endpoint: MagicEndpoint, downloader: Downloader, - live: LiveSync, + pub(crate) live: LiveSync, active: Arc>>, } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 0bbbfd8220..694013cdb7 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -100,7 +100,7 @@ enum SyncState { Failed(anyhow::Error), } -#[derive(Debug)] +#[derive(derive_more::Debug)] enum ToActor { StartSync { replica: Replica, @@ -114,6 +114,20 @@ enum ToActor { namespace: NamespaceId, }, Shutdown, + Subscribe { + namespace: NamespaceId, + #[debug("cb")] + cb: Box, + }, +} + +/// Events informing about actions of the live sync progres. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum LiveEvent { + /// A local insertion. + InsertLocal, + /// Received a remote insert. + InsertRemote, } /// Handle to a running live sync actor @@ -180,6 +194,19 @@ impl LiveSync { .await?; Ok(()) } + + /// Subscribes `cb` to events on this `namespace`. + pub fn subscribe(&self, namespace: NamespaceId, cb: F) -> Result<()> + where + F: Fn(LiveEvent) + Send + Sync + 'static, + { + self.to_actor_tx.try_send(ToActor::::Subscribe { + namespace, + cb: Box::new(cb), + })?; + + Ok(()) + } } // TODO: Also add `handle_connection` to the replica and track incoming sync requests here too. @@ -238,6 +265,7 @@ impl Actor { Some(ToActor::StartSync { replica, peers }) => self.start_sync(replica, peers).await?, Some(ToActor::StopSync { namespace }) => self.stop_sync(&namespace).await?, Some(ToActor::JoinPeers { namespace, peers }) => self.join_gossip_and_start_initial_sync(&namespace, peers).await?, + Some(ToActor::Subscribe { namespace, cb }) => self.subscribe(&namespace, cb).await?, } } // new gossip message @@ -304,6 +332,22 @@ impl Actor { Ok(()) } + async fn subscribe( + &mut self, + namespace: &NamespaceId, + cb: Box, + ) -> anyhow::Result<()> { + let topic = TopicId::from_bytes(*namespace.as_bytes()); + if let Some((replica, tokens)) = self.replicas.get_mut(&topic) { + // TODO: handle unsubscribe + let token = replica.on_insert(Box::new(move |origin, entry| match origin { + InsertOrigin::Local => cb(LiveEvent::InsertLocal), + InsertOrigin::Sync(_) => cb(LiveEvent::InsertRemote), + })); + } + Ok(()) + } + async fn stop_sync(&mut self, namespace: &NamespaceId) -> anyhow::Result<()> { let topic = TopicId::from_bytes(*namespace.as_bytes()); if let Some((replica, removal_token)) = self.replicas.remove(&topic) { diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index c70ed60526..841a51dfd4 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -11,8 +11,8 @@ use crate::rpc_protocol::{ AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, DocGetRequest, DocGetResponse, DocImportRequest, DocImportResponse, DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, - DocTicket, DocsCreateRequest, DocsCreateResponse, DocsListRequest, DocsListResponse, RpcResult, - ShareMode, + DocSubscribeRequest, DocSubscribeResponse, DocTicket, DocsCreateRequest, DocsCreateResponse, + DocsListRequest, DocsListResponse, RpcResult, ShareMode, }; use super::{engine::SyncEngine, PeerSource}; @@ -73,6 +73,20 @@ impl SyncEngine { })) } + pub fn doc_subscribe( + &self, + req: DocSubscribeRequest, + ) -> impl Stream { + let (s, r) = flume::bounded(64); + self.live + .subscribe(req.doc_id, move |event| { + s.send(DocSubscribeResponse { event }).ok(); + }) + .unwrap(); // TODO: handle error + + r.into_stream() + } + pub async fn doc_import(&self, req: DocImportRequest) -> RpcResult { let DocImportRequest(DocTicket { key, peers }) = req; // TODO: support read-only docs From fbfdd272f6c0f9f844608617966a3ebeda31c49d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 14 Aug 2023 12:20:53 +0200 Subject: [PATCH 078/172] fix: unused code --- iroh/src/sync/live.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 694013cdb7..83fd5cc2c1 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -338,9 +338,9 @@ impl Actor { cb: Box, ) -> anyhow::Result<()> { let topic = TopicId::from_bytes(*namespace.as_bytes()); - if let Some((replica, tokens)) = self.replicas.get_mut(&topic) { + if let Some((replica, _token)) = self.replicas.get_mut(&topic) { // TODO: handle unsubscribe - let token = replica.on_insert(Box::new(move |origin, entry| match origin { + let _token = replica.on_insert(Box::new(move |origin, _entry| match origin { InsertOrigin::Local => cb(LiveEvent::InsertLocal), InsertOrigin::Sync(_) => cb(LiveEvent::InsertRemote), })); From 6daef61509455408bf1b697af07dfd8f94cca9cb Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 14 Aug 2023 12:58:53 +0200 Subject: [PATCH 079/172] test: use events in sync test --- iroh/tests/sync.rs | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index f717107fd6..0c3bd2324b 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -3,12 +3,13 @@ use std::{net::SocketAddr, time::Duration}; use anyhow::{anyhow, Result}; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use iroh::{ client::mem::Doc, collection::IrohCollectionParser, node::{Builder, Node}, rpc_protocol::ShareMode, + sync::LiveEvent, }; use quic_rpc::transport::misc::DummyServerEndpoint; use tracing_subscriber::{prelude::*, EnvFilter}; @@ -85,16 +86,28 @@ async fn sync_full_basic() -> Result<()> { let author = iroh.create_author().await?; let doc = iroh.import_doc(ticket.clone()).await?; - // todo: events over rpc to not use sleep... - tokio::time::sleep(Duration::from_secs(2)).await; + // wait for remote insert on doc2 + let mut events = doc.subscribe().await?; + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::InsertRemote)); + // TODO: emit event when download is complete instead of having a timeout + tokio::time::sleep(Duration::from_secs(1)).await; assert_latest(&doc, b"k1", b"v1").await; + + // setup event channel on on doc1 + let mut events = doc1.subscribe().await?; + let key = b"k2"; let value = b"v2"; doc.set_bytes(author, key.to_vec(), value.to_vec()).await?; assert_latest(&doc, key, value).await; - // todo: events - tokio::time::sleep(Duration::from_secs(2)).await; + + // wait for remote insert on doc1 + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::InsertRemote)); + // TODO: emit event when download is complete instead of having a timeout + tokio::time::sleep(Duration::from_secs(1)).await; assert_latest(&doc1, key, value).await; doc }; @@ -102,11 +115,17 @@ async fn sync_full_basic() -> Result<()> { // node 3 joins & imports the doc from peer 1 let _doc3 = { let iroh = &clients[2]; - println!("!!!! DOC 3 JOIN !!!!!"); let doc = iroh.import_doc(ticket).await?; - // todo: events - tokio::time::sleep(Duration::from_secs(2)).await; + // wait for 2 remote inserts + let mut events = doc.subscribe().await?; + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::InsertRemote)); + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::InsertRemote)); + + // TODO: emit event when download is complete instead of having a timeout + tokio::time::sleep(Duration::from_secs(1)).await; assert_latest(&doc, b"k1", b"v1").await; assert_latest(&doc, b"k2", b"v2").await; doc From e596d7bf569afba70260a4973e7fcd0fb2229d77 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 14 Aug 2023 13:26:39 +0200 Subject: [PATCH 080/172] chore: fmt --- iroh/tests/sync.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 0c3bd2324b..e0e4fac760 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -94,7 +94,6 @@ async fn sync_full_basic() -> Result<()> { tokio::time::sleep(Duration::from_secs(1)).await; assert_latest(&doc, b"k1", b"v1").await; - // setup event channel on on doc1 let mut events = doc1.subscribe().await?; From ec4e6195fd67b697b04b0eebdb8b1fc5ac5c16c6 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 14 Aug 2023 16:14:14 +0200 Subject: [PATCH 081/172] refactor: simplify iroh-sync store trait --- iroh-sync/src/store.rs | 146 ++++++---------------------------- iroh-sync/src/store/fs.rs | 73 ++++++++++++++--- iroh-sync/src/store/memory.rs | 58 +++++++++++++- iroh-sync/src/sync.rs | 69 ++++++++++++---- iroh/src/sync.rs | 13 +-- iroh/tests/sync.rs | 2 +- 6 files changed, 204 insertions(+), 157 deletions(-) diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index 5c864add31..86430dad44 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use rand_core::CryptoRngCore; use serde::{Deserialize, Serialize}; @@ -16,10 +16,7 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// The specialized instance scoped to a `Namespace`. type Instance: ranger::Store + Send + Sync + 'static + Clone; - type GetLatestIter<'a>: Iterator> - where - Self: 'a; - type GetAllIter<'a>: Iterator> + type GetIter<'a>: Iterator> where Self: 'a; @@ -40,60 +37,15 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { fn list_authors(&self) -> Result>; fn new_replica(&self, namespace: Namespace) -> Result>; - /// Gets all entries matching this key and author. + /// Returns an iterator over the entries in a namespace. + fn get(&self, namespace: NamespaceId, filter: GetFilter) -> Result>; + /// Gets the latest entry this key and author. fn get_latest_by_key_and_author( &self, namespace: NamespaceId, author: AuthorId, key: impl AsRef<[u8]>, ) -> Result>; - - /// Returns the latest version of the matching documents by key. - fn get_latest_by_key( - &self, - namespace: NamespaceId, - key: impl AsRef<[u8]>, - ) -> Result>; - - /// Returns the latest version of the matching documents by prefix. - fn get_latest_by_prefix( - &self, - namespace: NamespaceId, - prefix: impl AsRef<[u8]>, - ) -> Result>; - - /// Returns the latest versions of all documents. - fn get_latest(&self, namespace: NamespaceId) -> Result>; - - /// Returns all versions of the matching documents by author. - fn get_all_by_key_and_author<'a, 'b: 'a>( - &'a self, - namespace: NamespaceId, - author: AuthorId, - key: impl AsRef<[u8]> + 'b, - ) -> Result>; - - /// Returns all versions of the matching documents by key. - fn get_all_by_key( - &self, - namespace: NamespaceId, - key: impl AsRef<[u8]>, - ) -> Result>; - - /// Returns all versions of the matching documents by prefix. - fn get_all_by_prefix( - &self, - namespace: NamespaceId, - prefix: impl AsRef<[u8]>, - ) -> Result>; - - /// Returns all versions of all documents. - fn get_all(&self, namespace: NamespaceId) -> Result>; - - /// Returns an iterator over the entries in a namespace. - fn get(&self, namespace: NamespaceId, filter: GetFilter) -> Result> { - GetIter::new(self, namespace, filter) - } } /// Filter a get query onto a namespace @@ -106,28 +58,37 @@ pub struct GetFilter { impl Default for GetFilter { fn default() -> Self { + Self::latest() + } +} + +impl GetFilter { + /// No filter, iterate over all entries. + pub fn all() -> Self { Self { - latest: true, + latest: false, author: None, key: KeyFilter::All, } } -} -impl GetFilter { - /// Create a new get filter. Defaults to latest entries for all keys and authors. - pub fn new() -> Self { - Self::default() + /// Only include the latest entries. + pub fn latest() -> Self { + Self { + latest: true, + author: None, + key: KeyFilter::All, + } } /// Filter by exact key match. - pub fn with_key(mut self, key: Vec) -> Self { - self.key = KeyFilter::Key(key); + pub fn with_key(mut self, key: impl AsRef<[u8]>) -> Self { + self.key = KeyFilter::Key(key.as_ref().to_vec()); self } /// Filter by prefix key match. - pub fn with_prefix(mut self, prefix: Vec) -> Self { - self.key = KeyFilter::Prefix(prefix); + pub fn with_prefix(mut self, prefix: impl AsRef<[u8]>) -> Self { + self.key = KeyFilter::Prefix(prefix.as_ref().to_vec()); self } /// Filter by author. @@ -152,62 +113,3 @@ pub enum KeyFilter { /// Filter for exact key match Key(Vec), } - -/// Iterator over the entries in a namespace -pub enum GetIter<'s, S: Store> { - All(S::GetAllIter<'s>), - Latest(S::GetLatestIter<'s>), - Single(std::option::IntoIter>), -} - -impl<'s, S: Store> Iterator for GetIter<'s, S> { - type Item = anyhow::Result; - - fn next(&mut self) -> Option { - match self { - GetIter::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), - GetIter::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), - GetIter::Single(iter) => iter.next(), - } - } -} - -impl<'s, S: Store> GetIter<'s, S> { - fn new(store: &'s S, namespace: NamespaceId, filter: GetFilter) -> anyhow::Result { - use KeyFilter::*; - Ok(match filter.latest { - false => match (filter.key, filter.author) { - (All, None) => Self::All(store.get_all(namespace)?), - (Prefix(prefix), None) => Self::All(store.get_all_by_prefix(namespace, &prefix)?), - (Key(key), None) => Self::All(store.get_all_by_key(namespace, key)?), - (Key(key), Some(author)) => { - Self::All(store.get_all_by_key_and_author(namespace, author, key)?) - } - (All, Some(_)) | (Prefix(_), Some(_)) => { - bail!("This filter combination is not yet supported") - } - }, - true => match (filter.key, filter.author) { - (All, None) => Self::Latest(store.get_latest(namespace)?), - (Prefix(prefix), None) => { - Self::Latest(store.get_latest_by_prefix(namespace, &prefix)?) - } - (Key(key), None) => Self::Latest(store.get_latest_by_key(namespace, key)?), - (Key(key), Some(author)) => Self::Single( - store - .get_latest_by_key_and_author(namespace, author, key)? - .map(Ok) - .into_iter(), - ), - (All, Some(_)) | (Prefix(_), Some(_)) => { - bail!("This filter combination is not yet supported") - } - }, - }) - } - - /// Returns true if this iterator is known to return only a single result. - pub fn single(&self) -> bool { - matches!(self, Self::Single(_)) - } -} diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs index 42f85a0bd7..f0feea79a5 100644 --- a/iroh-sync/src/store/fs.rs +++ b/iroh-sync/src/store/fs.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, path::Path, sync::Arc}; -use anyhow::Result; +use anyhow::{bail, Result}; use ouroboros::self_referencing; use parking_lot::RwLock; use rand_core::CryptoRngCore; @@ -99,10 +99,28 @@ impl Store { } } +#[derive(Debug)] +pub enum GetIter<'s> { + All(RangeAllIterator<'s>), + Latest(RangeLatestIterator<'s>), + Single(std::option::IntoIter>), +} + +impl<'s> Iterator for GetIter<'s> { + type Item = anyhow::Result; + + fn next(&mut self) -> Option { + match self { + GetIter::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIter::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIter::Single(iter) => iter.next(), + } + } +} + impl super::Store for Store { type Instance = StoreInstance; - type GetAllIter<'a> = RangeAllIterator<'a>; - type GetLatestIter<'a> = RangeLatestIterator<'a>; + type GetIter<'a> = GetIter<'a>; fn open_replica(&self, namespace_id: &NamespaceId) -> Result>> { if let Some(replica) = self.replicas.read().get(namespace_id) { @@ -174,7 +192,38 @@ impl super::Store for Store { Ok(replica) } - /// Gets all entries matching this key and author. + fn get(&self, namespace: NamespaceId, filter: super::GetFilter) -> Result> { + use super::KeyFilter::*; + Ok(match filter.latest { + false => match (filter.key, filter.author) { + (All, None) => GetIter::All(self.get_all(namespace)?), + (Prefix(prefix), None) => GetIter::All(self.get_all_by_prefix(namespace, &prefix)?), + (Key(key), None) => GetIter::All(self.get_all_by_key(namespace, key)?), + (Key(key), Some(author)) => { + GetIter::All(self.get_all_by_key_and_author(namespace, author, key)?) + } + (All, Some(_)) | (Prefix(_), Some(_)) => { + bail!("This filter combination is not yet supported") + } + }, + true => match (filter.key, filter.author) { + (All, None) => GetIter::Latest(self.get_latest(namespace)?), + (Prefix(prefix), None) => { + GetIter::Latest(self.get_latest_by_prefix(namespace, &prefix)?) + } + (Key(key), None) => GetIter::Latest(self.get_latest_by_key(namespace, key)?), + (Key(key), Some(author)) => GetIter::Single( + self.get_latest_by_key_and_author(namespace, author, key)? + .map(Ok) + .into_iter(), + ), + (All, Some(_)) | (Prefix(_), Some(_)) => { + bail!("This filter combination is not yet supported") + } + }, + }) + } + fn get_latest_by_key_and_author( &self, namespace: NamespaceId, @@ -199,12 +248,14 @@ impl super::Store for Store { Ok(Some(signed_entry)) } +} +impl Store { fn get_latest_by_key( &self, namespace: NamespaceId, key: impl AsRef<[u8]>, - ) -> Result> { + ) -> Result> { let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); let iter = RangeLatestIterator::try_new( @@ -226,7 +277,7 @@ impl super::Store for Store { &self, namespace: NamespaceId, prefix: impl AsRef<[u8]>, - ) -> Result> { + ) -> Result> { let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); let iter = RangeLatestIterator::try_new( @@ -244,7 +295,7 @@ impl super::Store for Store { Ok(iter) } - fn get_latest(&self, namespace: NamespaceId) -> Result> { + fn get_latest(&self, namespace: NamespaceId) -> Result> { let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); let iter = RangeLatestIterator::try_new( @@ -267,7 +318,7 @@ impl super::Store for Store { namespace: NamespaceId, author: AuthorId, key: impl AsRef<[u8]> + 'b, - ) -> Result> { + ) -> Result> { let start = (namespace.as_bytes(), author.as_bytes(), key.as_ref()); let end = (namespace.as_bytes(), author.as_bytes(), key.as_ref()); let iter = RangeAllIterator::try_new( @@ -293,7 +344,7 @@ impl super::Store for Store { &self, namespace: NamespaceId, key: impl AsRef<[u8]>, - ) -> Result> { + ) -> Result> { let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); let iter = RangeAllIterator::try_new( @@ -319,7 +370,7 @@ impl super::Store for Store { &self, namespace: NamespaceId, prefix: impl AsRef<[u8]>, - ) -> Result> { + ) -> Result> { let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); let iter = RangeAllIterator::try_new( @@ -341,7 +392,7 @@ impl super::Store for Store { Ok(iter) } - fn get_all(&self, namespace: NamespaceId) -> Result> { + fn get_all(&self, namespace: NamespaceId) -> Result> { let start = (namespace.as_bytes(), &[0u8; 32], &[][..]); let end = (namespace.as_bytes(), &[255u8; 32], &[][..]); let iter = RangeAllIterator::try_new( diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs index 9dd39ec667..254a1a3876 100644 --- a/iroh-sync/src/store/memory.rs +++ b/iroh-sync/src/store/memory.rs @@ -6,7 +6,7 @@ use std::{ sync::Arc, }; -use anyhow::Result; +use anyhow::{bail, Result}; use parking_lot::{RwLock, RwLockReadGuard}; use rand_core::CryptoRngCore; @@ -29,8 +29,7 @@ type ReplicaRecordsOwned = impl super::Store for Store { type Instance = ReplicaStoreInstance; - type GetLatestIter<'a> = GetLatestIter<'a>; - type GetAllIter<'a> = GetAllIter<'a>; + type GetIter<'a> = GetIter<'a>; fn open_replica(&self, namespace: &NamespaceId) -> Result>> { let replicas = &*self.replicas.read(); @@ -65,6 +64,38 @@ impl super::Store for Store { Ok(replica) } + fn get(&self, namespace: NamespaceId, filter: super::GetFilter) -> Result> { + use super::KeyFilter::*; + Ok(match filter.latest { + false => match (filter.key, filter.author) { + (All, None) => GetIter::All(self.get_all(namespace)?), + (Prefix(prefix), None) => GetIter::All(self.get_all_by_prefix(namespace, &prefix)?), + (Key(key), None) => GetIter::All(self.get_all_by_key(namespace, key)?), + (Key(key), Some(author)) => { + GetIter::All(self.get_all_by_key_and_author(namespace, author, key)?) + } + (All, Some(_)) | (Prefix(_), Some(_)) => { + bail!("This filter combination is not yet supported") + } + }, + true => match (filter.key, filter.author) { + (All, None) => GetIter::Latest(self.get_latest(namespace)?), + (Prefix(prefix), None) => { + GetIter::Latest(self.get_latest_by_prefix(namespace, &prefix)?) + } + (Key(key), None) => GetIter::Latest(self.get_latest_by_key(namespace, key)?), + (Key(key), Some(author)) => GetIter::Single( + self.get_latest_by_key_and_author(namespace, author, key)? + .map(Ok) + .into_iter(), + ), + (All, Some(_)) | (Prefix(_), Some(_)) => { + bail!("This filter combination is not yet supported") + } + }, + }) + } + fn get_latest_by_key_and_author( &self, namespace: NamespaceId, @@ -80,7 +111,28 @@ impl super::Store for Store { Ok(value.map(|(_, v)| v.clone())) } +} + +#[derive(Debug)] +pub enum GetIter<'s> { + All(GetAllIter<'s>), + Latest(GetLatestIter<'s>), + Single(std::option::IntoIter>), +} + +impl<'s> Iterator for GetIter<'s> { + type Item = anyhow::Result; + + fn next(&mut self) -> Option { + match self { + GetIter::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIter::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIter::Single(iter) => iter.next(), + } + } +} +impl Store { fn get_latest_by_key( &self, namespace: NamespaceId, diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 1782dd90ff..6143e21c56 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -707,7 +707,10 @@ impl Record { mod tests { use anyhow::Result; - use crate::{ranger::Range, store}; + use crate::{ + ranger::Range, + store::{self, GetFilter}, + }; use super::*; @@ -773,37 +776,51 @@ mod tests { // Get All by author let entries: Vec<_> = store - .get_all_by_key_and_author(my_replica.namespace(), alice.id(), "/cool/path")? + .get( + my_replica.namespace(), + GetFilter::all() + .with_author(alice.id()) + .with_key("/cool/path"), + )? .collect::>()?; assert_eq!(entries.len(), 2); // Get All by key let entries: Vec<_> = store - .get_all_by_key(my_replica.namespace(), b"/cool/path")? + .get( + my_replica.namespace(), + GetFilter::all().with_key(b"/cool/path"), + )? .collect::>()?; assert_eq!(entries.len(), 2); // Get latest by key let entries: Vec<_> = store - .get_latest_by_key(my_replica.namespace(), b"/cool/path")? + .get( + my_replica.namespace(), + GetFilter::latest().with_key(b"/cool/path"), + )? .collect::>()?; assert_eq!(entries.len(), 1); // Get latest by prefix let entries: Vec<_> = store - .get_latest_by_prefix(my_replica.namespace(), b"/cool")? + .get( + my_replica.namespace(), + GetFilter::latest().with_prefix(b"/cool"), + )? .collect::>()?; assert_eq!(entries.len(), 1); // Get All let entries: Vec<_> = store - .get_all(my_replica.namespace())? + .get(my_replica.namespace(), GetFilter::all())? .collect::>()?; assert_eq!(entries.len(), 12); // Get All latest let entries: Vec<_> = store - .get_latest(my_replica.namespace())? + .get(my_replica.namespace(), GetFilter::latest())? .collect::>()?; assert_eq!(entries.len(), 11); @@ -814,48 +831,70 @@ mod tests { // Get All by author let entries: Vec<_> = store - .get_all_by_key_and_author(my_replica.namespace(), alice.id(), "/cool/path")? + .get( + my_replica.namespace(), + GetFilter::all() + .with_author(alice.id()) + .with_key("/cool/path"), + )? .collect::>()?; assert_eq!(entries.len(), 2); let entries: Vec<_> = store - .get_all_by_key_and_author(my_replica.namespace(), bob.id(), "/cool/path")? + .get( + my_replica.namespace(), + GetFilter::all() + .with_author(bob.id()) + .with_key("/cool/path"), + )? .collect::>()?; assert_eq!(entries.len(), 1); // Get All by key let entries: Vec<_> = store - .get_all_by_key(my_replica.namespace(), b"/cool/path")? + .get( + my_replica.namespace(), + GetFilter::all().with_key(b"/cool/path"), + )? .collect::>()?; assert_eq!(entries.len(), 3); // Get latest by key let entries: Vec<_> = store - .get_latest_by_key(my_replica.namespace(), b"/cool/path")? + .get( + my_replica.namespace(), + GetFilter::latest().with_key(b"/cool/path"), + )? .collect::>()?; assert_eq!(entries.len(), 2); // Get latest by prefix let entries: Vec<_> = store - .get_latest_by_prefix(my_replica.namespace(), b"/cool")? + .get( + my_replica.namespace(), + GetFilter::latest().with_prefix(b"/cool"), + )? .collect::>()?; assert_eq!(entries.len(), 2); // Get all by prefix let entries: Vec<_> = store - .get_all_by_prefix(my_replica.namespace(), b"/cool")? + .get( + my_replica.namespace(), + GetFilter::all().with_prefix(b"/cool"), + )? .collect::>()?; assert_eq!(entries.len(), 3); // Get All let entries: Vec<_> = store - .get_all(my_replica.namespace())? + .get(my_replica.namespace(), GetFilter::all())? .collect::>()?; assert_eq!(entries.len(), 13); // Get All latest let entries: Vec<_> = store - .get_latest(my_replica.namespace())? + .get(my_replica.namespace(), GetFilter::latest())? .collect::>()?; assert_eq!(entries.len(), 12); diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index e40778da9f..315c75479e 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -214,7 +214,10 @@ async fn send_sync_message( #[cfg(test)] mod tests { - use iroh_sync::{store::Store as _, sync::Namespace}; + use iroh_sync::{ + store::{GetFilter, Store as _}, + sync::Namespace, + }; use super::*; @@ -241,7 +244,7 @@ mod tests { assert_eq!( bob_replica_store - .get_all(bob_replica.namespace()) + .get(bob_replica.namespace(), GetFilter::all()) .unwrap() .collect::>>() .unwrap() @@ -250,7 +253,7 @@ mod tests { ); assert_eq!( alice_replica_store - .get_all(alice_replica.namespace()) + .get(alice_replica.namespace(), GetFilter::all()) .unwrap() .collect::>>() .unwrap() @@ -289,7 +292,7 @@ mod tests { assert_eq!( bob_replica_store - .get_all(bob_replica.namespace()) + .get(bob_replica.namespace(), GetFilter::all()) .unwrap() .collect::>>() .unwrap() @@ -298,7 +301,7 @@ mod tests { ); assert_eq!( alice_replica_store - .get_all(alice_replica.namespace()) + .get(alice_replica.namespace(), GetFilter::all()) .unwrap() .collect::>>() .unwrap() diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index e0e4fac760..abaa34b641 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -148,7 +148,7 @@ async fn assert_latest(doc: &Doc, key: &[u8], value: &[u8]) { } async fn get_latest(doc: &Doc, key: &[u8]) -> anyhow::Result> { - let filter = GetFilter::new().with_key(key.to_vec()); + let filter = GetFilter::latest().with_key(key); let entry = doc .get(filter) .await? From 35f32184fcec1d4a5dc5c1a5352e21f8c183e251 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 14 Aug 2023 16:39:01 +0200 Subject: [PATCH 082/172] fix: debug for client --- iroh/src/client.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 9ebc77507a..1c9d19a54e 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -28,6 +28,7 @@ pub mod mem; pub mod quic; /// Iroh client +#[derive(Debug, Clone)] pub struct Iroh { rpc: RpcClient, } @@ -94,6 +95,7 @@ where } /// Document handle +#[derive(Debug, Clone)] pub struct Doc { id: NamespaceId, rpc: RpcClient, From 4429435e5c01b76cc60a4123cbddf95b91d8c0eb Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 14 Aug 2023 16:51:14 +0200 Subject: [PATCH 083/172] fix: sync example --- iroh/examples/sync.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 4893dc66a4..d10eabdd94 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -36,7 +36,7 @@ use iroh_net::{ MagicEndpoint, }; use iroh_sync::{ - store::{self, Store as _}, + store::{self, GetFilter, Store as _}, sync::{Author, Namespace, Replica, SignedEntry}, }; use once_cell::sync::OnceCell; @@ -353,13 +353,13 @@ impl ReplState { } => { let entries = if prefix { self.store - .get_all_by_prefix(self.doc.namespace(), key.as_bytes())? + .get(self.doc.namespace(), GetFilter::all().with_prefix(key))? } else { self.store - .get_all_by_key(self.doc.namespace(), key.as_bytes())? + .get(self.doc.namespace(), GetFilter::all().with_key(key))? }; for entry in entries { - let (_id, entry) = entry?; + let entry = entry?; println!("{}", fmt_entry(&entry)); if print_content { println!("{}", fmt_content(&self.db, &entry).await); @@ -380,14 +380,14 @@ impl ReplState { }, Cmd::Ls { prefix } => { let entries = match prefix { - None => self.store.get_all(self.doc.namespace())?, + None => self.store.get(self.doc.namespace(), GetFilter::all())?, Some(prefix) => self .store - .get_all_by_prefix(self.doc.namespace(), prefix.as_bytes())?, + .get(self.doc.namespace(), GetFilter::all().with_prefix(prefix))?, }; let mut count = 0; for entry in entries { - let (_id, entry) = entry?; + let entry = entry?; count += 1; println!("{}", fmt_entry(&entry),); } @@ -450,10 +450,10 @@ impl ReplState { let mut read = 0; for i in 0..count { let key = format!("{}/{}/{}", prefix, t, i); - let entries = - store.get_all_by_key(doc.namespace(), key.as_bytes())?; + let entries = store + .get(doc.namespace(), GetFilter::all().with_key(key))?; for entry in entries { - let (_id, entry) = entry?; + let entry = entry?; let _content = fmt_content_simple(&doc, &entry); read += 1; } @@ -546,13 +546,14 @@ impl ReplState { } let root = canonicalize_path(&dir_path)?; println!("> exporting {key_prefix} to {root:?}"); - let entries = self - .store - .get_latest_by_prefix(self.doc.namespace(), key_prefix.as_bytes())?; + let entries = self.store.get( + self.doc.namespace(), + GetFilter::latest().with_prefix(&key_prefix), + )?; let mut checked_dirs = HashSet::new(); for entry in entries { - let (id, entry) = entry?; - let key = id.key(); + let entry = entry?; + let key = entry.entry().id().key(); let relative = String::from_utf8(key[key_prefix.len()..].to_vec())?; let len = entry.entry().record().content_len(); let blob = self.db.get(entry.content_hash()); @@ -579,10 +580,10 @@ impl ReplState { // TODO: Fix let entry = self .store - .get_latest_by_key(self.doc.namespace(), &key)? + .get(self.doc.namespace(), GetFilter::latest().with_key(&key))? .next(); if let Some(entry) = entry { - let (_, entry) = entry?; + let entry = entry?; println!("> exporting {key} to {path:?}"); let parent = path.parent().ok_or_else(|| anyhow!("Invalid path"))?; tokio::fs::create_dir_all(&parent).await?; From d15b61d02ac70305e01e317e15efbab22afd1c84 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 14 Aug 2023 14:39:22 +0200 Subject: [PATCH 084/172] refactor: improve subscription model for sync events --- Cargo.lock | 1 + iroh-sync/Cargo.toml | 1 + iroh-sync/src/sync.rs | 66 +++++------- iroh/examples/sync.rs | 43 ++++---- iroh/src/download.rs | 19 ++-- iroh/src/node.rs | 2 +- iroh/src/rpc_protocol.rs | 2 +- iroh/src/sync/engine.rs | 54 +++++----- iroh/src/sync/live.rs | 214 +++++++++++++++++++++++++++++---------- iroh/src/sync/rpc.rs | 11 +- iroh/tests/sync.rs | 8 +- 11 files changed, 262 insertions(+), 159 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef630f708b..c2da6b8490 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1961,6 +1961,7 @@ dependencies = [ "crossbeam", "derive_more", "ed25519-dalek", + "flume", "hex", "iroh-blake3", "iroh-bytes", diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 70a1165781..1b87a37aed 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -14,6 +14,7 @@ blake3 = { package = "iroh-blake3", version = "1.4.3"} crossbeam = "0.8.2" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } ed25519-dalek = { version = "2.0.0-rc.2", features = ["serde", "rand_core"] } +flume = "0.10" iroh-bytes = { version = "0.5.0", path = "../iroh-bytes" } iroh-metrics = { version = "0.5.0", path = "../iroh-metrics", optional = true } once_cell = "1.18.0" diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 6143e21c56..81ee8dbc57 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -6,10 +6,9 @@ use std::{ cmp::Ordering, - collections::HashMap, fmt::{Debug, Display}, str::FromStr, - sync::{atomic::AtomicU64, Arc}, + sync::Arc, time::SystemTime, }; @@ -18,7 +17,7 @@ use crate::metrics::Metrics; #[cfg(feature = "metrics")] use iroh_metrics::{inc, inc_by}; -use parking_lot::RwLock; +use parking_lot::{Mutex, RwLock}; use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey}; use iroh_bytes::Hash; @@ -235,18 +234,9 @@ impl NamespaceId { } } -/// TODO: Would potentially nice to pass a `&SignedEntry` reference, however that would make -/// everything `!Send`. -/// TODO: Not sure if the `Sync` requirement will be a problem for implementers. It comes from -/// [parking_lot::RwLock] requiring `Sync`. -pub type OnInsertCallback = Box; - /// TODO: PeerId is in iroh-net which iroh-sync doesn't depend on. Add iroh-common crate with `PeerId`. pub type PeerIdBytes = [u8; 32]; -#[derive(Debug, Clone)] -pub struct RemovalToken(u64); - #[derive(Debug, Clone)] pub enum InsertOrigin { Local, @@ -256,9 +246,9 @@ pub enum InsertOrigin { #[derive(derive_more::Debug, Clone)] pub struct Replica> { inner: Arc>>, - #[debug("on_insert: [Box; {}]", "self.on_insert.len()")] - on_insert: Arc>>, - on_insert_removal_id: Arc, + on_insert_sender: flume::Sender<(InsertOrigin, SignedEntry)>, + #[allow(clippy::type_complexity)] + on_insert_receiver: Arc>>>, } #[derive(derive_more::Debug)] @@ -276,27 +266,20 @@ struct ReplicaData { impl> Replica { // TODO: check that read only replicas are possible pub fn new(namespace: Namespace, store: S) -> Self { + let (s, r) = flume::bounded(16); // TODO: should this be configurable? Replica { inner: Arc::new(RwLock::new(InnerReplica { namespace, peer: Peer::from_store(store), })), - on_insert: Default::default(), - on_insert_removal_id: Arc::new(AtomicU64::new(0)), + on_insert_sender: s, + on_insert_receiver: Arc::new(Mutex::new(Some(r))), } } - pub fn on_insert(&self, callback: OnInsertCallback) -> RemovalToken { - let mut on_insert = self.on_insert.write(); - let removal_id = self - .on_insert_removal_id - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - on_insert.insert(removal_id, callback); - RemovalToken(removal_id) - } - - pub fn remove_on_insert(&self, token: RemovalToken) -> bool { - self.on_insert.write().remove(&token.0).is_some() + /// Subscribes to the events. Only one subscription can be active at a time. + pub fn subscribe(&self) -> Option> { + self.on_insert_receiver.lock().take() } /// Inserts a new record at the given key. @@ -317,10 +300,10 @@ impl> Replica { let signed_entry = entry.sign(&inner.namespace, author); inner.peer.put(id, signed_entry.clone())?; drop(inner); - let on_insert = self.on_insert.read(); - for cb in on_insert.values() { - cb(InsertOrigin::Local, signed_entry.clone()); - } + + self.on_insert_sender + .send((InsertOrigin::Local, signed_entry)) + .ok(); #[cfg(feature = "metrics")] { @@ -362,10 +345,10 @@ impl> Replica { let id = entry.entry.id.clone(); inner.peer.put(id, entry.clone()).map_err(Into::into)?; drop(inner); - let on_insert = self.on_insert.read(); - for cb in on_insert.values() { - cb(InsertOrigin::Sync(received_from), entry.clone()); - } + self.on_insert_sender + .send((InsertOrigin::Sync(received_from), entry.clone())) + .ok(); + #[cfg(feature = "metrics")] { inc!(Metrics, new_entries_remote); @@ -391,10 +374,9 @@ impl> Replica { .write() .peer .process_message(message, |_key, entry| { - let on_insert = self.on_insert.read(); - for cb in on_insert.values() { - cb(InsertOrigin::Sync(from_peer), entry.clone()); - } + self.on_insert_sender + .send((InsertOrigin::Sync(from_peer), entry)) + .ok(); })?; Ok(reply) @@ -522,6 +504,10 @@ impl Entry { &self.id } + pub fn namespace(&self) -> NamespaceId { + self.id.namespace() + } + pub fn record(&self) -> &Record { &self.record } diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index d10eabdd94..f9ce25fa9a 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -37,7 +37,7 @@ use iroh_net::{ }; use iroh_sync::{ store::{self, GetFilter, Store as _}, - sync::{Author, Namespace, Replica, SignedEntry}, + sync::{Author, Entry, Namespace, Replica, SignedEntry}, }; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; @@ -265,18 +265,23 @@ async fn run(args: Args) -> anyhow::Result<()> { println!("> ready to accept commands"); println!("> type `help` for a list of commands"); - let current_watch: Arc>> = - Arc::new(std::sync::Mutex::new(None)); + let current_watch: Arc>> = + Arc::new(tokio::sync::Mutex::new(None)); + let watch = current_watch.clone(); - doc.on_insert(Box::new(move |_origin, entry| { - let matcher = watch.lock().unwrap(); - if let Some(matcher) = &*matcher { - let key = entry.entry().id().key(); - if key.starts_with(matcher.as_bytes()) { - println!("change: {}", fmt_entry(&entry)); + let doc_events = doc.subscribe().expect("already subscribed"); + rt.main().spawn(async move { + while let Ok((_origin, entry)) = doc_events.recv_async().await { + let entry = entry.entry(); + let matcher = watch.lock().await; + if let Some(matcher) = &*matcher { + let key = entry.id().key(); + if key.starts_with(matcher.as_bytes()) { + println!("change: {}", fmt_entry(&entry)); + } } } - })); + }); let repl_state = ReplState { rt, @@ -334,7 +339,7 @@ struct ReplState { db: iroh::baomap::flat::Store, ticket: Ticket, log_filter: LogLevelReload, - current_watch: Arc>>, + current_watch: Arc>>, } impl ReplState { @@ -360,7 +365,7 @@ impl ReplState { }; for entry in entries { let entry = entry?; - println!("{}", fmt_entry(&entry)); + println!("{}", fmt_entry(entry.entry())); if print_content { println!("{}", fmt_content(&self.db, &entry).await); } @@ -368,9 +373,9 @@ impl ReplState { } Cmd::Watch { key } => { println!("watching key: '{key}'"); - self.current_watch.lock().unwrap().replace(key); + self.current_watch.lock().await.replace(key); } - Cmd::WatchCancel => match self.current_watch.lock().unwrap().take() { + Cmd::WatchCancel => match self.current_watch.lock().await.take() { Some(key) => { println!("canceled watching key: '{key}'"); } @@ -389,7 +394,7 @@ impl ReplState { for entry in entries { let entry = entry?; count += 1; - println!("{}", fmt_entry(&entry),); + println!("{}", fmt_entry(entry.entry()),); } println!("> {} entries", count); } @@ -869,13 +874,13 @@ fn init_logging() -> LogLevelReload { // helpers -fn fmt_entry(entry: &SignedEntry) -> String { - let id = entry.entry().id(); +fn fmt_entry(entry: &Entry) -> String { + let id = entry.id(); let key = std::str::from_utf8(id.key()).unwrap_or(""); let author = fmt_hash(id.author().as_bytes()); - let hash = entry.entry().record().content_hash(); + let hash = entry.record().content_hash(); let hash = fmt_hash(hash.as_bytes()); - let len = HumanBytes(entry.entry().record().content_len()); + let len = HumanBytes(entry.record().content_len()); format!("@{author}: {key} = {hash} ({len})",) } diff --git a/iroh/src/download.rs b/iroh/src/download.rs index c57870e19b..239eed2de9 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -4,7 +4,7 @@ use std::time::Instant; use std::{ collections::{HashMap, VecDeque}, - sync::{Arc, Mutex}, + sync::Arc, }; use anyhow::anyhow; @@ -21,7 +21,7 @@ use iroh_gossip::net::util::Dialer; #[cfg(feature = "metrics")] use iroh_metrics::{inc, inc_by}; use iroh_net::{tls::PeerId, MagicEndpoint}; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, Mutex}; use tokio_stream::StreamExt; use tracing::{debug, error, warn}; @@ -74,25 +74,24 @@ impl Downloader { /// Note: This method takes only [`PeerId`]s and will attempt to connect to those peers. For /// this to succeed, you need to add addresses for these peers to the magic endpoint's /// addressbook yourself. See [`MagicEndpoint::add_known_addrs`]. - pub fn push(&self, hash: Hash, peers: Vec) { + pub async fn push(&self, hash: Hash, peers: Vec) { let (reply, reply_rx) = oneshot::channel(); let req = DownloadRequest { hash, peers, reply }; - // TODO: this is potentially blocking inside an async call. figure out a better solution - if let Err(err) = self.to_actor_tx.send(req) { + if let Err(err) = self.to_actor_tx.send_async(req).await { warn!("download actor dropped: {err}"); } - if self.pending_downloads.lock().unwrap().get(&hash).is_none() { + if self.pending_downloads.lock().await.get(&hash).is_none() { let pending_downloads = self.pending_downloads.clone(); let fut = async move { let res = reply_rx.await; - pending_downloads.lock().unwrap().remove(&hash); + pending_downloads.lock().await.remove(&hash); res.ok().flatten() }; self.pending_downloads .lock() - .unwrap() + .await .insert(hash, fut.boxed().shared()); } } @@ -101,8 +100,8 @@ impl Downloader { /// requests for that blob have failed. /// /// NOTE: This does not start the download itself. Use [`Self::push`] for that. - pub fn finished(&self, hash: &Hash) -> DownloadFuture { - match self.pending_downloads.lock().unwrap().get(hash) { + pub async fn finished(&self, hash: &Hash) -> DownloadFuture { + match self.pending_downloads.lock().await.get(hash) { Some(fut) => fut.clone(), None => futures::future::ready(None).boxed().shared(), } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 3d36393a1c..a17000feaa 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1255,7 +1255,7 @@ fn handle_rpc_request< } DocSubscribe(msg) => { chan.server_streaming(msg, handler, |handler, req| { - handler.inner.sync.doc_subscribe(req) + async move { handler.inner.sync.doc_subscribe(req).await }.flatten_stream() }) .await } diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 3098a14a7c..3be460c1e5 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -640,7 +640,7 @@ pub enum ProviderRequest { } /// The response enum, listing all possible responses. -#[allow(missing_docs)] +#[allow(missing_docs, clippy::large_enum_variant)] #[derive(Debug, Serialize, Deserialize, From, TryInto)] pub enum ProviderResponse { Watch(WatchResponse), diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index 3493f62534..e9128b8cad 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -1,18 +1,19 @@ use std::{collections::HashMap, sync::Arc}; use anyhow::anyhow; +use futures::FutureExt; use iroh_bytes::util::runtime::Handle; use iroh_gossip::net::Gossip; -use iroh_net::{tls::PeerId, MagicEndpoint}; +use iroh_net::MagicEndpoint; use iroh_sync::{ store::Store, - sync::{Author, AuthorId, InsertOrigin, NamespaceId, OnInsertCallback, RemovalToken, Replica}, + sync::{Author, AuthorId, NamespaceId, Replica}, }; use parking_lot::RwLock; use crate::download::Downloader; -use super::{LiveSync, PeerSource}; +use super::{LiveEvent, LiveSync, OnLiveEventCallback, PeerSource, RemovalToken}; /// The SyncEngine combines the [`LiveSync`] actor with the Iroh bytes database and [`Downloader`]. /// @@ -64,13 +65,15 @@ impl SyncEngine { ) -> anyhow::Result<()> { let replica = self.get_replica(&namespace)?; if !self.active.read().contains_key(&namespace) { - // add download listener - let removal_token = replica.on_insert(on_insert_download(self.downloader.clone())); - self.active - .write() - .insert(replica.namespace(), removal_token); // start to gossip updates - self.live.start_sync(replica.clone(), peers).await?; + self.live.start_sync(replica, peers).await?; + // add download listener + let removal_token = self + .live + .subscribe(namespace, on_insert_download(self.downloader.clone())) + .await?; + + self.active.write().insert(namespace, removal_token); } else if !peers.is_empty() { self.live.join_peers(namespace, peers).await?; } @@ -80,21 +83,15 @@ impl SyncEngine { /// Stop syncing a document. pub async fn stop_sync(&self, namespace: NamespaceId) -> anyhow::Result<()> { let replica = self.get_replica(&namespace)?; - let token = self.active.write().remove(&replica.namespace()); - if let Some(token) = token { - replica.remove_on_insert(token); - self.live.stop_sync(namespace).await?; - } + self.active.write().remove(&replica.namespace()); + // `stop_sync` removes all callback listeners automatically + self.live.stop_sync(namespace).await?; + Ok(()) } /// Shutdown the sync engine. pub async fn shutdown(&self) -> anyhow::Result<()> { - for (namespace, token) in self.active.write().drain() { - if let Ok(Some(replica)) = self.store.open_replica(&namespace) { - replica.remove_on_insert(token); - } - } self.live.shutdown().await?; Ok(()) } @@ -114,12 +111,19 @@ impl SyncEngine { } } -fn on_insert_download(downloader: Downloader) -> OnInsertCallback { - Box::new(move |origin, entry| { - if let InsertOrigin::Sync(Some(peer_id)) = origin { - let peer_id = PeerId::from_bytes(&peer_id).unwrap(); - let hash = *entry.entry().record().content_hash(); - downloader.push(hash, vec![peer_id]); +fn on_insert_download(downloader: Downloader) -> OnLiveEventCallback { + Box::new(move |event: LiveEvent| { + let downloader = downloader.clone(); + async move { + if let LiveEvent::InsertRemote { + from: Some(peer_id), + entry, + } = event + { + let hash = *entry.record().content_hash(); + downloader.push(hash, vec![peer_id]).await; + } } + .boxed() }) } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 83fd5cc2c1..cfff1b7df8 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -1,7 +1,13 @@ -use std::{collections::HashMap, fmt, net::SocketAddr, str::FromStr, sync::Arc}; +use std::{ + collections::HashMap, + fmt, + net::SocketAddr, + str::FromStr, + sync::{atomic::AtomicU64, Arc}, +}; use crate::sync::connect_and_sync; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; use futures::{ future::{BoxFuture, Shared}, stream::{BoxStream, FuturesUnordered, StreamExt}, @@ -15,10 +21,13 @@ use iroh_gossip::{ use iroh_net::{tls::PeerId, MagicEndpoint}; use iroh_sync::{ store, - sync::{InsertOrigin, NamespaceId, RemovalToken, Replica, SignedEntry}, + sync::{Entry, InsertOrigin, NamespaceId, Replica, SignedEntry}, }; use serde::{Deserialize, Serialize}; -use tokio::{sync::mpsc, task::JoinError}; +use tokio::{ + sync::{self, mpsc}, + task::JoinError, +}; use tracing::{debug, error, info, warn}; const CHANNEL_CAP: usize = 8; @@ -117,17 +126,35 @@ enum ToActor { Subscribe { namespace: NamespaceId, #[debug("cb")] - cb: Box, + cb: OnLiveEventCallback, + s: sync::oneshot::Sender>, + }, + Unsubscribe { + namespace: NamespaceId, + token: RemovalToken, + s: sync::oneshot::Sender, }, } +/// Callback used for tracking [`LiveEvent`]s. +pub type OnLiveEventCallback = + Box BoxFuture<'static, ()> + Send + Sync + 'static>; + /// Events informing about actions of the live sync progres. #[derive(Serialize, Deserialize, Debug, Clone)] pub enum LiveEvent { /// A local insertion. - InsertLocal, + InsertLocal { + /// The inserted entry. + entry: Entry, + }, /// Received a remote insert. - InsertRemote, + InsertRemote { + /// The peer that sent us the entry. + from: Option, + /// The inserted entry. + entry: Entry, + }, } /// Handle to a running live sync actor @@ -196,16 +223,35 @@ impl LiveSync { } /// Subscribes `cb` to events on this `namespace`. - pub fn subscribe(&self, namespace: NamespaceId, cb: F) -> Result<()> + pub async fn subscribe(&self, namespace: NamespaceId, cb: F) -> Result where - F: Fn(LiveEvent) + Send + Sync + 'static, + F: Fn(LiveEvent) -> BoxFuture<'static, ()> + Send + Sync + 'static, { - self.to_actor_tx.try_send(ToActor::::Subscribe { - namespace, - cb: Box::new(cb), - })?; + let (s, r) = sync::oneshot::channel(); + self.to_actor_tx + .send(ToActor::::Subscribe { + namespace, + cb: Box::new(cb), + s, + }) + .await?; + let token = r.await??; + Ok(token) + } - Ok(()) + /// Unsubscribes `token` to events on this `namespace`. + /// Returns `true` if a callback was found + pub async fn unsubscribe(&self, namespace: NamespaceId, token: RemovalToken) -> Result { + let (s, r) = sync::oneshot::channel(); + self.to_actor_tx + .send(ToActor::::Unsubscribe { + namespace, + token, + s, + }) + .await?; + let token = r.await?; + Ok(token) } } @@ -215,38 +261,46 @@ struct Actor { endpoint: MagicEndpoint, gossip: Gossip, - replicas: HashMap, RemovalToken)>, + replicas: HashMap>, + replicas_subscription: futures::stream::SelectAll< + flume::r#async::RecvStream<'static, (InsertOrigin, SignedEntry)>, + >, subscription: BoxStream<'static, Result<(TopicId, Event)>>, sync_state: HashMap<(TopicId, PeerId), SyncState>, to_actor_rx: mpsc::Receiver>, - insert_entry_tx: flume::Sender<(TopicId, SignedEntry)>, - insert_entry_rx: flume::Receiver<(TopicId, SignedEntry)>, pending_syncs: FuturesUnordered)>>, pending_joins: FuturesUnordered)>>, + + event_subscriptions: HashMap>, + event_removal_id: AtomicU64, } +/// Token needed to remove inserted callbacks. +#[derive(Debug, Clone)] +pub struct RemovalToken(u64); + impl Actor { pub fn new( endpoint: MagicEndpoint, gossip: Gossip, to_actor_rx: mpsc::Receiver>, ) -> Self { - let (insert_entry_tx, insert_entry_rx) = flume::bounded(64); let sub = gossip.clone().subscribe_all().boxed(); Self { gossip, endpoint, - insert_entry_rx, - insert_entry_tx, to_actor_rx, sync_state: Default::default(), pending_syncs: Default::default(), pending_joins: Default::default(), replicas: Default::default(), + replicas_subscription: Default::default(), subscription: sub, + event_subscriptions: Default::default(), + event_removal_id: Default::default(), } } @@ -265,7 +319,14 @@ impl Actor { Some(ToActor::StartSync { replica, peers }) => self.start_sync(replica, peers).await?, Some(ToActor::StopSync { namespace }) => self.stop_sync(&namespace).await?, Some(ToActor::JoinPeers { namespace, peers }) => self.join_gossip_and_start_initial_sync(&namespace, peers).await?, - Some(ToActor::Subscribe { namespace, cb }) => self.subscribe(&namespace, cb).await?, + Some(ToActor::Subscribe { namespace, cb, s }) => { + let subscribe_result = self.subscribe(&namespace, cb).await; + s.send(subscribe_result).ok(); + }, + Some(ToActor::Unsubscribe { namespace, token, s }) => { + let result = self.unsubscribe(&namespace, token).await; + s.send(result).ok(); + }, } } // new gossip message @@ -275,9 +336,8 @@ impl Actor { error!("Failed to process gossip event: {err:?}"); } }, - entry = self.insert_entry_rx.recv_async() => { - let (topic, entry) = entry?; - self.on_insert_entry(topic, entry).await?; + Some((origin, entry)) = self.replicas_subscription.next() => { + self.on_replica_event(origin, entry).await?; } Some((topic, peer, res)) = self.pending_syncs.next() => { // let (topic, peer, res) = res.context("task sync_with_peer paniced")?; @@ -298,7 +358,7 @@ impl Actor { } fn sync_with_peer(&mut self, topic: TopicId, peer: PeerId) { - let Some((replica, _token)) = self.replicas.get(&topic) else { + let Some(replica) = self.replicas.get(&topic) else { return; }; // Check if we synced and only start sync if not yet synced @@ -325,33 +385,47 @@ impl Actor { } async fn shutdown(&mut self) -> anyhow::Result<()> { - for (topic, (replica, removal_token)) in self.replicas.drain() { - replica.remove_on_insert(removal_token); + for (topic, _replica) in self.replicas.drain() { + self.event_subscriptions.remove(&topic); self.gossip.quit(topic).await?; } + Ok(()) } async fn subscribe( &mut self, namespace: &NamespaceId, - cb: Box, - ) -> anyhow::Result<()> { + cb: OnLiveEventCallback, + ) -> anyhow::Result { let topic = TopicId::from_bytes(*namespace.as_bytes()); - if let Some((replica, _token)) = self.replicas.get_mut(&topic) { - // TODO: handle unsubscribe - let _token = replica.on_insert(Box::new(move |origin, _entry| match origin { - InsertOrigin::Local => cb(LiveEvent::InsertLocal), - InsertOrigin::Sync(_) => cb(LiveEvent::InsertRemote), - })); + if self.replicas.contains_key(&topic) { + let subs = self.event_subscriptions.entry(topic).or_default(); + let removal_id = self + .event_removal_id + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + subs.insert(removal_id, cb); + let token = RemovalToken(removal_id); + Ok(token) + } else { + bail!("cannot subscribe to unknown replica: {}", namespace); } - Ok(()) + } + + /// Returns `true` if a callback was found and removed + async fn unsubscribe(&mut self, namespace: &NamespaceId, token: RemovalToken) -> bool { + let topic = TopicId::from_bytes(*namespace.as_bytes()); + if let Some(subs) = self.event_subscriptions.get_mut(&topic) { + return subs.remove(&token.0).is_some(); + } + + false } async fn stop_sync(&mut self, namespace: &NamespaceId) -> anyhow::Result<()> { let topic = TopicId::from_bytes(*namespace.as_bytes()); - if let Some((replica, removal_token)) = self.replicas.remove(&topic) { - replica.remove_on_insert(removal_token); + if let Some(_replica) = self.replicas.remove(&topic) { + self.event_subscriptions.remove(&topic); self.gossip.quit(topic).await?; } Ok(()) @@ -405,17 +479,11 @@ impl Actor { let topic = TopicId::from_bytes(*namespace.as_bytes()); if let std::collections::hash_map::Entry::Vacant(e) = self.replicas.entry(topic) { // setup replica insert notifications. - let insert_entry_tx = self.insert_entry_tx.clone(); - let removal_token = replica.on_insert(Box::new(move |origin, entry| { - // only care for local inserts, otherwise we'd do endless gossip loops - if let InsertOrigin::Local = origin { - // TODO: this is potentially blocking inside an async call. figure out a better solution - if let Err(err) = insert_entry_tx.send((topic, entry)) { - warn!("on_insert forward failed: {err} - LiveSync actor dropped"); - } - } - })); - e.insert((replica, removal_token)); + let events = replica + .subscribe() + .ok_or_else(|| anyhow::anyhow!("trying to subscribe twice to the same replica"))?; + self.replicas_subscription.push(events.into_stream()); + e.insert(replica); } self.join_gossip_and_start_initial_sync(&namespace, peers) @@ -433,7 +501,7 @@ impl Actor { } fn on_gossip_event(&mut self, topic: TopicId, event: Event) -> Result<()> { - let Some((replica, _token)) = self.replicas.get(&topic) else { + let Some(replica) = self.replicas.get(&topic) else { return Err(anyhow!("Missing doc for {topic:?}")); }; match event { @@ -459,12 +527,46 @@ impl Actor { Ok(()) } - /// A new entry was inserted locally. Broadcast a gossip message. - async fn on_insert_entry(&mut self, topic: TopicId, entry: SignedEntry) -> Result<()> { - let op = Op::Put(entry); - let message = postcard::to_stdvec(&op)?.into(); - debug!(topic = ?topic, "broadcast new entry"); - self.gossip.broadcast(topic, message).await?; + async fn on_replica_event( + &mut self, + origin: InsertOrigin, + signed_entry: SignedEntry, + ) -> Result<()> { + let topic = TopicId::from_bytes(*signed_entry.entry().namespace().as_bytes()); + let subs = self.event_subscriptions.get(&topic); + match origin { + InsertOrigin::Local => { + let entry = signed_entry.entry().clone(); + + // A new entry was inserted locally. Broadcast a gossip message. + let op = Op::Put(signed_entry); + let message = postcard::to_stdvec(&op)?.into(); + debug!(topic = ?topic, "broadcast new entry"); + self.gossip.broadcast(topic, message).await?; + + if let Some(subs) = subs { + futures::future::join_all(subs.values().map(|sub| { + sub(LiveEvent::InsertLocal { + entry: entry.clone(), + }) + })) + .await; + } + } + InsertOrigin::Sync(peer_id) => { + if let Some(subs) = subs { + let from = peer_id.and_then(|id| PeerId::from_bytes(&id).ok()); + futures::future::join_all(subs.values().map(|sub| { + sub(LiveEvent::InsertRemote { + from, + entry: signed_entry.entry().clone(), + }) + })) + .await; + } + } + } + Ok(()) } } diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index 841a51dfd4..92278084eb 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -1,7 +1,7 @@ //! This module contains an impl block on [`SyncEngine`] with handlers for RPC requests use anyhow::anyhow; -use futures::Stream; +use futures::{FutureExt, Stream}; use iroh_bytes::{baomap::Store as BaoStore, util::RpcError}; use iroh_sync::{store::Store, sync::Namespace}; use itertools::Itertools; @@ -73,15 +73,20 @@ impl SyncEngine { })) } - pub fn doc_subscribe( + pub async fn doc_subscribe( &self, req: DocSubscribeRequest, ) -> impl Stream { let (s, r) = flume::bounded(64); self.live .subscribe(req.doc_id, move |event| { - s.send(DocSubscribeResponse { event }).ok(); + let s = s.clone(); + async move { + s.send_async(DocSubscribeResponse { event }).await.ok(); + } + .boxed() }) + .await .unwrap(); // TODO: handle error r.into_stream() diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index abaa34b641..6feb41d0d0 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -89,7 +89,7 @@ async fn sync_full_basic() -> Result<()> { // wait for remote insert on doc2 let mut events = doc.subscribe().await?; let event = events.try_next().await?.unwrap(); - assert!(matches!(event, LiveEvent::InsertRemote)); + assert!(matches!(event, LiveEvent::InsertRemote { .. })); // TODO: emit event when download is complete instead of having a timeout tokio::time::sleep(Duration::from_secs(1)).await; assert_latest(&doc, b"k1", b"v1").await; @@ -104,7 +104,7 @@ async fn sync_full_basic() -> Result<()> { // wait for remote insert on doc1 let event = events.try_next().await?.unwrap(); - assert!(matches!(event, LiveEvent::InsertRemote)); + assert!(matches!(event, LiveEvent::InsertRemote { .. })); // TODO: emit event when download is complete instead of having a timeout tokio::time::sleep(Duration::from_secs(1)).await; assert_latest(&doc1, key, value).await; @@ -119,9 +119,9 @@ async fn sync_full_basic() -> Result<()> { // wait for 2 remote inserts let mut events = doc.subscribe().await?; let event = events.try_next().await?.unwrap(); - assert!(matches!(event, LiveEvent::InsertRemote)); + assert!(matches!(event, LiveEvent::InsertRemote { .. })); let event = events.try_next().await?.unwrap(); - assert!(matches!(event, LiveEvent::InsertRemote)); + assert!(matches!(event, LiveEvent::InsertRemote { .. })); // TODO: emit event when download is complete instead of having a timeout tokio::time::sleep(Duration::from_secs(1)).await; From 4c649c10838f1b331c94df963ff54e6fc722a6c9 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 14 Aug 2023 21:50:12 +0200 Subject: [PATCH 085/172] docs and clippy --- iroh/examples/sync.rs | 2 +- iroh/src/download.rs | 3 --- iroh/src/sync/engine.rs | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index f9ce25fa9a..621277f791 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -277,7 +277,7 @@ async fn run(args: Args) -> anyhow::Result<()> { if let Some(matcher) = &*matcher { let key = entry.id().key(); if key.starts_with(matcher.as_bytes()) { - println!("change: {}", fmt_entry(&entry)); + println!("change: {}", fmt_entry(entry)); } } } diff --git a/iroh/src/download.rs b/iroh/src/download.rs index 239eed2de9..31f6c52afa 100644 --- a/iroh/src/download.rs +++ b/iroh/src/download.rs @@ -36,9 +36,6 @@ pub type DownloadFuture = Shared>>; /// Spawns a background task that handles connecting to peers and performing get requests. /// /// TODO: Support retries and backoff - become a proper queue... -/// TODO: Download requests send via synchronous flume::Sender::send. Investigate if we want async -/// here. We currently use [`Downloader::push`] from [`iroh_sync::sync::Replica::on_insert`] callbacks, -/// which are sync, thus we need a sync method on the Downloader to push new download requests. /// TODO: Support collections, likely become generic over C: CollectionParser #[derive(Debug, Clone)] pub struct Downloader { diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index e9128b8cad..5243bad2f9 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -34,7 +34,7 @@ impl SyncEngine { /// engine with [`Self::start_sync`], then new entries inserted locally will be sent to peers /// through iroh-gossip. /// - /// The engine will also register [`Replica::on_insert`] callbacks to download content for new + /// The engine will also register for [`Replica::subscribe`] events to download content for new /// entries from peers. pub fn spawn( rt: Handle, From ac2bd411b40ea105faa10f3209ca42fee0ae92f8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 15 Aug 2023 00:38:27 +0200 Subject: [PATCH 086/172] feat: content ready events --- iroh/src/node.rs | 1 + iroh/src/sync/engine.rs | 48 ++++--------------- iroh/src/sync/live.rs | 102 ++++++++++++++++++++++++++++++++-------- iroh/tests/sync.rs | 18 ++++--- 4 files changed, 105 insertions(+), 64 deletions(-) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index a17000feaa..1ccfa7b356 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -354,6 +354,7 @@ where endpoint.clone(), gossip.clone(), self.docs, + self.db.clone(), downloader, ); diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index 5243bad2f9..dfdc791163 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -1,8 +1,7 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashSet, sync::Arc}; use anyhow::anyhow; -use futures::FutureExt; -use iroh_bytes::util::runtime::Handle; +use iroh_bytes::{baomap::Store as BaoStore, util::runtime::Handle}; use iroh_gossip::net::Gossip; use iroh_net::MagicEndpoint; use iroh_sync::{ @@ -13,7 +12,7 @@ use parking_lot::RwLock; use crate::download::Downloader; -use super::{LiveEvent, LiveSync, OnLiveEventCallback, PeerSource, RemovalToken}; +use super::{LiveSync, PeerSource}; /// The SyncEngine combines the [`LiveSync`] actor with the Iroh bytes database and [`Downloader`]. /// @@ -22,9 +21,8 @@ pub struct SyncEngine { pub(crate) rt: Handle, pub(crate) store: S, pub(crate) endpoint: MagicEndpoint, - downloader: Downloader, pub(crate) live: LiveSync, - active: Arc>>, + active: Arc>>, } impl SyncEngine { @@ -36,17 +34,17 @@ impl SyncEngine { /// /// The engine will also register for [`Replica::subscribe`] events to download content for new /// entries from peers. - pub fn spawn( + pub fn spawn( rt: Handle, endpoint: MagicEndpoint, gossip: Gossip, store: S, + bao_store: B, downloader: Downloader, ) -> Self { - let live = LiveSync::spawn(rt.clone(), endpoint.clone(), gossip); + let live = LiveSync::spawn(rt.clone(), endpoint.clone(), gossip, bao_store, downloader); Self { live, - downloader, store, rt, endpoint, @@ -63,17 +61,10 @@ impl SyncEngine { namespace: NamespaceId, peers: Vec, ) -> anyhow::Result<()> { - let replica = self.get_replica(&namespace)?; - if !self.active.read().contains_key(&namespace) { - // start to gossip updates + if !self.active.read().contains(&namespace) { + let replica = self.get_replica(&namespace)?; self.live.start_sync(replica, peers).await?; - // add download listener - let removal_token = self - .live - .subscribe(namespace, on_insert_download(self.downloader.clone())) - .await?; - - self.active.write().insert(namespace, removal_token); + self.active.write().insert(namespace); } else if !peers.is_empty() { self.live.join_peers(namespace, peers).await?; } @@ -84,9 +75,7 @@ impl SyncEngine { pub async fn stop_sync(&self, namespace: NamespaceId) -> anyhow::Result<()> { let replica = self.get_replica(&namespace)?; self.active.write().remove(&replica.namespace()); - // `stop_sync` removes all callback listeners automatically self.live.stop_sync(namespace).await?; - Ok(()) } @@ -110,20 +99,3 @@ impl SyncEngine { .ok_or_else(|| anyhow!("author not found")) } } - -fn on_insert_download(downloader: Downloader) -> OnLiveEventCallback { - Box::new(move |event: LiveEvent| { - let downloader = downloader.clone(); - async move { - if let LiveEvent::InsertRemote { - from: Some(peer_id), - entry, - } = event - { - let hash = *entry.record().content_hash(); - downloader.push(hash, vec![peer_id]).await; - } - } - .boxed() - }) -} diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index cfff1b7df8..afb6677ca4 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -6,14 +6,14 @@ use std::{ sync::{atomic::AtomicU64, Arc}, }; -use crate::sync::connect_and_sync; +use crate::{download::Downloader, sync::connect_and_sync}; use anyhow::{anyhow, bail, Result}; use futures::{ future::{BoxFuture, Shared}, stream::{BoxStream, FuturesUnordered, StreamExt}, FutureExt, TryFutureExt, }; -use iroh_bytes::util::runtime::Handle; +use iroh_bytes::{baomap, util::runtime::Handle, Hash}; use iroh_gossip::{ net::{Event, Gossip}, proto::TopicId, @@ -154,7 +154,24 @@ pub enum LiveEvent { from: Option, /// The inserted entry. entry: Entry, + /// If the content is available at the local node + content_status: ContentStatus, }, + /// The content of an entry was downloaded and is now available at the local node + ContentReady { + /// The content hash of the newly available entry content + hash: Hash, + }, +} + +/// Availability status of an entry's content bytes +// TODO: Add NotRequested or similar +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum ContentStatus { + /// The content is available on the local node + Ready, + /// The content is not yet available on the local node + Pending, } /// Handle to a running live sync actor @@ -169,9 +186,15 @@ impl LiveSync { /// /// This spawn a background actor to handle gossip events and forward operations over broadcast /// messages. - pub fn spawn(rt: Handle, endpoint: MagicEndpoint, gossip: Gossip) -> Self { + pub fn spawn( + rt: Handle, + endpoint: MagicEndpoint, + gossip: Gossip, + bao_store: B, + downloader: Downloader, + ) -> Self { let (to_actor_tx, to_actor_rx) = mpsc::channel(CHANNEL_CAP); - let mut actor = Actor::new(endpoint, gossip, to_actor_rx); + let mut actor = Actor::new(endpoint, gossip, bao_store, downloader, to_actor_rx); let task = rt.main().spawn(async move { if let Err(err) = actor.run().await { error!("live sync failed: {err:?}"); @@ -257,9 +280,11 @@ impl LiveSync { // TODO: Also add `handle_connection` to the replica and track incoming sync requests here too. // Currently peers might double-sync in both directions. -struct Actor { +struct Actor { endpoint: MagicEndpoint, gossip: Gossip, + bao_store: B, + downloader: Downloader, replicas: HashMap>, replicas_subscription: futures::stream::SelectAll< @@ -275,16 +300,20 @@ struct Actor { event_subscriptions: HashMap>, event_removal_id: AtomicU64, + + pending_downloads: FuturesUnordered>>, } /// Token needed to remove inserted callbacks. #[derive(Debug, Clone)] pub struct RemovalToken(u64); -impl Actor { +impl Actor { pub fn new( endpoint: MagicEndpoint, gossip: Gossip, + bao_store: B, + downloader: Downloader, to_actor_rx: mpsc::Receiver>, ) -> Self { let sub = gossip.clone().subscribe_all().boxed(); @@ -292,6 +321,8 @@ impl Actor { Self { gossip, endpoint, + bao_store, + downloader, to_actor_rx, sync_state: Default::default(), pending_syncs: Default::default(), @@ -301,6 +332,7 @@ impl Actor { subscription: sub, event_subscriptions: Default::default(), event_removal_id: Default::default(), + pending_downloads: Default::default(), } } @@ -352,6 +384,15 @@ impl Actor { } // TODO: maintain some join state } + Some(res) = self.pending_downloads.next() => { + if let Some((topic, hash)) = res { + if let Some(subs) = self.event_subscriptions.get(&topic) { + let event = LiveEvent::ContentReady { hash }; + notify_all(&subs, event).await; + } + } + + } } } Ok(()) @@ -544,25 +585,44 @@ impl Actor { debug!(topic = ?topic, "broadcast new entry"); self.gossip.broadcast(topic, message).await?; + // Notify subscribers about the event if let Some(subs) = subs { - futures::future::join_all(subs.values().map(|sub| { - sub(LiveEvent::InsertLocal { - entry: entry.clone(), - }) - })) - .await; + let event = LiveEvent::InsertLocal { + entry: entry.clone(), + }; + notify_all(&subs, event).await; } } InsertOrigin::Sync(peer_id) => { + let from = peer_id.and_then(|id| PeerId::from_bytes(&id).ok()); + let entry = signed_entry.entry(); + let hash = *entry.record().content_hash(); + + // A new entry was inserted from initial sync or gossip. Queue downloading the + // content. + let content_status = if let Some(_bao_entry) = self.bao_store.get(&hash) { + ContentStatus::Ready + } else { + if let Some(from) = from { + self.downloader.push(hash, vec![from]).await; + let fut = self.downloader.finished(&hash).await; + let fut = fut + .map(move |res| res.map(move |(hash, _len)| (topic, hash))) + .boxed(); + self.pending_downloads.push(fut); + } + ContentStatus::Pending + }; + + // Notify subscribers about the event if let Some(subs) = subs { let from = peer_id.and_then(|id| PeerId::from_bytes(&id).ok()); - futures::future::join_all(subs.values().map(|sub| { - sub(LiveEvent::InsertRemote { - from, - entry: signed_entry.entry().clone(), - }) - })) - .await; + let event = LiveEvent::InsertRemote { + from, + entry: entry.clone(), + content_status, + }; + notify_all(&subs, event).await; } } } @@ -570,3 +630,7 @@ impl Actor { Ok(()) } } + +async fn notify_all(subs: &HashMap, event: LiveEvent) { + futures::future::join_all(subs.values().map(|sub| sub(event.clone()))).await; +} diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 6feb41d0d0..023a83d346 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -1,6 +1,6 @@ #![cfg(feature = "mem-db")] -use std::{net::SocketAddr, time::Duration}; +use std::net::SocketAddr; use anyhow::{anyhow, Result}; use futures::{StreamExt, TryStreamExt}; @@ -90,8 +90,9 @@ async fn sync_full_basic() -> Result<()> { let mut events = doc.subscribe().await?; let event = events.try_next().await?.unwrap(); assert!(matches!(event, LiveEvent::InsertRemote { .. })); - // TODO: emit event when download is complete instead of having a timeout - tokio::time::sleep(Duration::from_secs(1)).await; + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::ContentReady { .. })); + assert_latest(&doc, b"k1", b"v1").await; // setup event channel on on doc1 @@ -105,8 +106,9 @@ async fn sync_full_basic() -> Result<()> { // wait for remote insert on doc1 let event = events.try_next().await?.unwrap(); assert!(matches!(event, LiveEvent::InsertRemote { .. })); - // TODO: emit event when download is complete instead of having a timeout - tokio::time::sleep(Duration::from_secs(1)).await; + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::ContentReady { .. })); + assert_latest(&doc1, key, value).await; doc }; @@ -122,9 +124,11 @@ async fn sync_full_basic() -> Result<()> { assert!(matches!(event, LiveEvent::InsertRemote { .. })); let event = events.try_next().await?.unwrap(); assert!(matches!(event, LiveEvent::InsertRemote { .. })); + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::ContentReady { .. })); + let event = events.try_next().await?.unwrap(); + assert!(matches!(event, LiveEvent::ContentReady { .. })); - // TODO: emit event when download is complete instead of having a timeout - tokio::time::sleep(Duration::from_secs(1)).await; assert_latest(&doc, b"k1", b"v1").await; assert_latest(&doc, b"k2", b"v2").await; doc From 07653cab9107ba44052fc170f7eb4921775772fa Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 15 Aug 2023 00:50:27 +0200 Subject: [PATCH 087/172] chore: clippy --- iroh/src/sync/live.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index afb6677ca4..91311d811d 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -142,6 +142,7 @@ pub type OnLiveEventCallback = /// Events informing about actions of the live sync progres. #[derive(Serialize, Deserialize, Debug, Clone)] +#[allow(clippy::large_enum_variant)] pub enum LiveEvent { /// A local insertion. InsertLocal { @@ -388,7 +389,7 @@ impl Actor { if let Some((topic, hash)) = res { if let Some(subs) = self.event_subscriptions.get(&topic) { let event = LiveEvent::ContentReady { hash }; - notify_all(&subs, event).await; + notify_all(subs, event).await; } } @@ -590,7 +591,7 @@ impl Actor { let event = LiveEvent::InsertLocal { entry: entry.clone(), }; - notify_all(&subs, event).await; + notify_all(subs, event).await; } } InsertOrigin::Sync(peer_id) => { @@ -622,7 +623,7 @@ impl Actor { entry: entry.clone(), content_status, }; - notify_all(&subs, event).await; + notify_all(subs, event).await; } } } From 925ff835731e27f4db6ed8b9344f6979e800f53a Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 15 Aug 2023 01:21:57 +0200 Subject: [PATCH 088/172] fix: sync example --- iroh/examples/sync.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 621277f791..9d3cefb614 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -235,6 +235,7 @@ async fn run(args: Args) -> anyhow::Result<()> { endpoint.clone(), gossip.clone(), docs.clone(), + db.clone(), downloader, ); From f5c0dd91bfa05caca67a1fd5feeb3d47e4d49c02 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 15 Aug 2023 12:13:08 +0200 Subject: [PATCH 089/172] fix --- iroh/src/sync/live.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 91311d811d..43ece87689 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -601,7 +601,7 @@ impl Actor { // A new entry was inserted from initial sync or gossip. Queue downloading the // content. - let content_status = if let Some(_bao_entry) = self.bao_store.get(&hash) { + let content_status = if self.bao_store.get(&hash).is_some() { ContentStatus::Ready } else { if let Some(from) = from { From c08fef862efc9698678239ea25c30ebb13aa4c3f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 15 Aug 2023 12:29:19 +0200 Subject: [PATCH 090/172] refactor: make from peer id required --- iroh-net/src/tls.rs | 8 ++++++++ iroh-sync/src/sync.rs | 17 ++++++++++++----- iroh/src/sync.rs | 37 +++++++++++++++++++------------------ iroh/src/sync/live.rs | 25 ++++++++++++------------- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/iroh-net/src/tls.rs b/iroh-net/src/tls.rs index 83e1dde697..2308f1b2f3 100644 --- a/iroh-net/src/tls.rs +++ b/iroh-net/src/tls.rs @@ -79,6 +79,14 @@ impl Keypair { pub fn to_bytes(&self) -> [u8; 32] { self.secret.to_bytes() } + + /// Create a keypair by converting a byte array into the secret part. + /// The public part can always be recovered. + pub fn from_bytes(bytes: &[u8; 32]) -> Self { + let secret = SecretKey::from_bytes(bytes); + let public = secret.verifying_key(); + Self { secret, public } + } } impl From for Keypair { diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 81ee8dbc57..1ca881c41b 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -240,7 +240,7 @@ pub type PeerIdBytes = [u8; 32]; #[derive(Debug, Clone)] pub enum InsertOrigin { Local, - Sync(Option), + Sync(PeerIdBytes), } #[derive(derive_more::Debug, Clone)] @@ -338,7 +338,7 @@ impl> Replica { pub fn insert_remote_entry( &self, entry: SignedEntry, - received_from: Option, + received_from: PeerIdBytes, ) -> anyhow::Result<()> { entry.verify()?; let mut inner = self.inner.write(); @@ -367,7 +367,7 @@ impl> Replica { pub fn sync_process_message( &self, message: crate::ranger::Message, - from_peer: Option, + from_peer: PeerIdBytes, ) -> Result>, S::Error> { let reply = self .inner @@ -1010,6 +1010,8 @@ mod tests { alice_set: &[&str], bob_set: &[&str], ) -> Result<()> { + let alice_peer_id = [1u8; 32]; + let bob_peer_id = [2u8; 32]; // Sync alice - bob let mut next_to_bob = Some(alice.sync_initial_message().map_err(Into::into)?); let mut rounds = 0; @@ -1017,8 +1019,13 @@ mod tests { assert!(rounds < 100, "too many rounds"); rounds += 1; println!("round {}", rounds); - if let Some(msg) = bob.sync_process_message(msg, None).map_err(Into::into)? { - next_to_bob = alice.sync_process_message(msg, None).map_err(Into::into)?; + if let Some(msg) = bob + .sync_process_message(msg, alice_peer_id) + .map_err(Into::into)? + { + next_to_bob = alice + .sync_process_message(msg, bob_peer_id) + .map_err(Into::into)?; } } diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index 315c75479e..518f28e0be 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -59,7 +59,7 @@ pub async fn connect_and_sync( .await .context("dial_and_sync")?; let (mut send_stream, mut recv_stream) = connection.open_bi().await?; - let res = run_alice::(&mut send_stream, &mut recv_stream, doc, Some(peer_id)).await; + let res = run_alice::(&mut send_stream, &mut recv_stream, doc, peer_id).await; #[cfg(feature = "metrics")] if res.is_ok() { @@ -77,9 +77,9 @@ pub async fn run_alice, - peer: Option, + other_peer_id: PeerId, ) -> Result<()> { - let peer = peer.map(|peer| peer.to_bytes()); + let other_peer_id = other_peer_id.to_bytes(); let mut buffer = BytesMut::with_capacity(1024); // Init message @@ -101,7 +101,10 @@ pub async fn run_alice { - if let Some(msg) = alice.sync_process_message(msg, peer).map_err(Into::into)? { + if let Some(msg) = alice + .sync_process_message(msg, other_peer_id) + .map_err(Into::into)? + { send_sync_message(writer, msg).await?; } else { break; @@ -123,13 +126,7 @@ pub async fn handle_connection( let (mut send_stream, mut recv_stream) = connection.accept_bi().await?; debug!(peer = ?peer_id, "incoming sync: start"); - let res = run_bob( - &mut send_stream, - &mut recv_stream, - replica_store, - Some(peer_id), - ) - .await; + let res = run_bob(&mut send_stream, &mut recv_stream, replica_store, peer_id).await; #[cfg(feature = "metrics")] if res.is_ok() { @@ -151,9 +148,9 @@ pub async fn run_bob, + other_peer_id: PeerId, ) -> Result<()> { - let peer = peer.map(|peer| peer.to_bytes()); + let other_peer_id = other_peer_id.to_bytes(); let mut buffer = BytesMut::with_capacity(1024); let mut replica = None; @@ -168,8 +165,9 @@ pub async fn run_bob { debug!("starting sync for {}", namespace); - if let Some(msg) = - r.sync_process_message(message, peer).map_err(Into::into)? + if let Some(msg) = r + .sync_process_message(message, other_peer_id) + .map_err(Into::into)? { send_sync_message(writer, msg).await?; } else { @@ -185,7 +183,7 @@ pub async fn run_bob match replica { Some(ref replica) => { if let Some(msg) = replica - .sync_process_message(msg, peer) + .sync_process_message(msg, other_peer_id) .map_err(Into::into)? { send_sync_message(writer, msg).await?; @@ -214,6 +212,7 @@ async fn send_sync_message( #[cfg(test)] mod tests { + use iroh_net::tls::Keypair; use iroh_sync::{ store::{GetFilter, Store as _}, sync::Namespace, @@ -224,6 +223,8 @@ mod tests { #[tokio::test] async fn test_sync_simple() -> Result<()> { let mut rng = rand::thread_rng(); + let alice_peer_id = PeerId::from(Keypair::from_bytes(&[1u8; 32]).public()); + let bob_peer_id = PeerId::from(Keypair::from_bytes(&[2u8; 32]).public()); let alice_replica_store = store::memory::Store::default(); // For now uses same author on both sides. @@ -270,7 +271,7 @@ mod tests { &mut alice_writer, &mut alice_reader, &replica, - None, + bob_peer_id, ) .await }); @@ -282,7 +283,7 @@ mod tests { &mut bob_writer, &mut bob_reader, bob_replica_store_task, - None, + alice_peer_id, ) .await }); diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 43ece87689..9128f59ef1 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -152,7 +152,7 @@ pub enum LiveEvent { /// Received a remote insert. InsertRemote { /// The peer that sent us the entry. - from: Option, + from: PeerId, /// The inserted entry. entry: Entry, /// If the content is available at the local node @@ -370,7 +370,9 @@ impl Actor { } }, Some((origin, entry)) = self.replicas_subscription.next() => { - self.on_replica_event(origin, entry).await?; + if let Err(err) = self.on_replica_event(origin, entry).await { + error!("Failed to process replica event: {err:?}"); + } } Some((topic, peer, res)) = self.pending_syncs.next() => { // let (topic, peer, res) = res.context("task sync_with_peer paniced")?; @@ -553,7 +555,7 @@ impl Actor { match op { Op::Put(entry) => { debug!(peer = ?prev_peer, topic = ?topic, "received entry via gossip"); - replica.insert_remote_entry(entry, Some(prev_peer.to_bytes()))? + replica.insert_remote_entry(entry, prev_peer.to_bytes())? } } } @@ -595,7 +597,7 @@ impl Actor { } } InsertOrigin::Sync(peer_id) => { - let from = peer_id.and_then(|id| PeerId::from_bytes(&id).ok()); + let from = PeerId::from_bytes(&peer_id)?; let entry = signed_entry.entry(); let hash = *entry.record().content_hash(); @@ -604,20 +606,17 @@ impl Actor { let content_status = if self.bao_store.get(&hash).is_some() { ContentStatus::Ready } else { - if let Some(from) = from { - self.downloader.push(hash, vec![from]).await; - let fut = self.downloader.finished(&hash).await; - let fut = fut - .map(move |res| res.map(move |(hash, _len)| (topic, hash))) - .boxed(); - self.pending_downloads.push(fut); - } + self.downloader.push(hash, vec![from]).await; + let fut = self.downloader.finished(&hash).await; + let fut = fut + .map(move |res| res.map(move |(hash, _len)| (topic, hash))) + .boxed(); + self.pending_downloads.push(fut); ContentStatus::Pending }; // Notify subscribers about the event if let Some(subs) = subs { - let from = peer_id.and_then(|id| PeerId::from_bytes(&id).ok()); let event = LiveEvent::InsertRemote { from, entry: entry.clone(), From a8e809367245504787168d354f1714b11305920d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 12:22:50 +0200 Subject: [PATCH 091/172] fix: content status for partial entries --- iroh/src/sync/live.rs | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 9128f59ef1..23f4f3b656 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -13,7 +13,11 @@ use futures::{ stream::{BoxStream, FuturesUnordered, StreamExt}, FutureExt, TryFutureExt, }; -use iroh_bytes::{baomap, util::runtime::Handle, Hash}; +use iroh_bytes::{ + baomap::{self, EntryStatus}, + util::runtime::Handle, + Hash, +}; use iroh_gossip::{ net::{Event, Gossip}, proto::TopicId, @@ -166,13 +170,28 @@ pub enum LiveEvent { } /// Availability status of an entry's content bytes -// TODO: Add NotRequested or similar +// TODO: Add IsDownloading #[derive(Serialize, Deserialize, Debug, Clone)] pub enum ContentStatus { - /// The content is available on the local node - Ready, - /// The content is not yet available on the local node - Pending, + /// Fully available on the local node. + Complete, + /// Partially available on the local node. + Incomplete, + /// Not available on the local node. + /// + /// This currently means either that the content is about to be downloaded, failed to be + /// downloaded, or was never requested. + Missing, +} + +impl From for ContentStatus { + fn from(value: EntryStatus) -> Self { + match value { + EntryStatus::Complete => ContentStatus::Complete, + EntryStatus::Partial => ContentStatus::Incomplete, + EntryStatus::NotFound => ContentStatus::Missing, + } + } } /// Handle to a running live sync actor @@ -603,24 +622,22 @@ impl Actor { // A new entry was inserted from initial sync or gossip. Queue downloading the // content. - let content_status = if self.bao_store.get(&hash).is_some() { - ContentStatus::Ready - } else { + let entry_status = self.bao_store.contains(&hash).into(); + if matches!(entry_status, EntryStatus::NotFound) { self.downloader.push(hash, vec![from]).await; let fut = self.downloader.finished(&hash).await; let fut = fut .map(move |res| res.map(move |(hash, _len)| (topic, hash))) .boxed(); self.pending_downloads.push(fut); - ContentStatus::Pending - }; + } // Notify subscribers about the event if let Some(subs) = subs { let event = LiveEvent::InsertRemote { from, entry: entry.clone(), - content_status, + content_status: entry_status.into(), }; notify_all(subs, event).await; } From 839a8551a2c1b2bdc88629085b22e6d31d0298b8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 12:23:34 +0200 Subject: [PATCH 092/172] docs: fix --- iroh/src/sync/engine.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync/engine.rs index dfdc791163..8daa20d1f8 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync/engine.rs @@ -14,8 +14,10 @@ use crate::download::Downloader; use super::{LiveSync, PeerSource}; -/// The SyncEngine combines the [`LiveSync`] actor with the Iroh bytes database and [`Downloader`]. +/// The SyncEngine contains the [`LiveSync`] handle, and keeps a copy of the store and endpoint. /// +/// The RPC methods dealing with documents and sync operate on the `SyncEngine`, with method +/// implementations in [super::rpc]. #[derive(Debug, Clone)] pub struct SyncEngine { pub(crate) rt: Handle, From cde53c3ef1995fbddbcbda00a2665af9a04d3c51 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 12:30:13 +0200 Subject: [PATCH 093/172] chore: clippy --- iroh/src/sync/live.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 23f4f3b656..30c510ba60 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -622,7 +622,7 @@ impl Actor { // A new entry was inserted from initial sync or gossip. Queue downloading the // content. - let entry_status = self.bao_store.contains(&hash).into(); + let entry_status = self.bao_store.contains(&hash); if matches!(entry_status, EntryStatus::NotFound) { self.downloader.push(hash, vec![from]).await; let fut = self.downloader.finished(&hash).await; From fc91c5e7db3a57bb8ad5dc4b5f0f840cc31cec1e Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 16 Aug 2023 17:51:39 +0200 Subject: [PATCH 094/172] docs,refactor: add docs in iroh-sync and reduce public API surface --- iroh-sync/src/lib.rs | 41 +--------- iroh-sync/src/metrics.rs | 2 + iroh-sync/src/ranger.rs | 70 ++++++++-------- iroh-sync/src/store.rs | 61 +++++++++----- iroh-sync/src/store/fs.rs | 43 ++++++---- iroh-sync/src/store/memory.rs | 46 +++++++---- iroh-sync/src/sync.rs | 146 ++++++++++++++++++++++++++++++---- iroh/src/client.rs | 8 +- iroh/src/commands/sync.rs | 33 ++++---- 9 files changed, 288 insertions(+), 162 deletions(-) diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index d236cc7330..fa2a45831f 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -1,41 +1,8 @@ +//! Set reconciliation for multi-dimensional key-value stores +#![deny(missing_docs, rustdoc::broken_intra_doc_links)] + #[cfg(feature = "metrics")] pub mod metrics; -pub mod ranger; +mod ranger; pub mod store; pub mod sync; - -use iroh_metrics::{ - core::{Counter, Metric}, - struct_iterable::Iterable, -}; - -/// Metrics for iroh-sync -#[allow(missing_docs)] -#[derive(Debug, Clone, Iterable)] -pub struct Metrics { - pub new_entries_local: Counter, - pub new_entries_remote: Counter, - pub new_entries_local_size: Counter, - pub new_entries_remote_size: Counter, - pub initial_sync_success: Counter, - pub initial_sync_failed: Counter, -} - -impl Default for Metrics { - fn default() -> Self { - Self { - new_entries_local: Counter::new("Number of document entries added locally"), - new_entries_remote: Counter::new("Number of document entries added by peers"), - new_entries_local_size: Counter::new("Total size of entry contents added locally"), - new_entries_remote_size: Counter::new("Total size of entry contents added by peers"), - initial_sync_success: Counter::new("Number of successfull initial syncs "), - initial_sync_failed: Counter::new("Number of failed initial syncs"), - } - } -} - -impl Metric for Metrics { - fn name() -> &'static str { - "iroh-sync" - } -} diff --git a/iroh-sync/src/metrics.rs b/iroh-sync/src/metrics.rs index 37185e6cec..91af686c97 100644 --- a/iroh-sync/src/metrics.rs +++ b/iroh-sync/src/metrics.rs @@ -1,3 +1,5 @@ +//! Metrics for iroh-sync + use iroh_metrics::{ core::{Counter, Metric}, struct_iterable::Iterable, diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index aff7f66578..faaed06ce7 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -402,22 +402,23 @@ where } } -impl Peer -where - K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, - V: Clone + Debug, - S: Store + Default, -{ - pub fn with_limit(limit: Range) -> Self { - Peer { - store: S::default(), - max_set_size: 1, - split_factor: 2, - limit: Some(limit), - _phantom: Default::default(), - } - } -} +// TODO: with_limit is unused, remove? +// impl Peer +// where +// K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, +// V: Clone + Debug, +// S: Store + Default, +// { +// fn with_limit(limit: Range) -> Self { +// Peer { +// store: S::default(), +// max_set_size: 1, +// split_factor: 2, +// limit: Some(limit), +// _phantom: Default::default(), +// } +// } +// } impl Peer where K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, @@ -612,24 +613,25 @@ where self.store.put(k, v) } - pub fn get(&self, k: &K) -> Result, S::Error> { - self.store.get(k) - } - - /// Remove the given key. - pub fn remove(&mut self, k: &K) -> Result, S::Error> { - self.store.remove(k) - } - - /// List all existing key value pairs. - pub fn all(&self) -> Result> + '_, S::Error> { - self.store.all() - } - - /// Returns a refernce to the underlying store. - pub fn store(&self) -> &S { - &self.store - } + // TODO: these are unused, remove? + // pub fn get(&self, k: &K) -> Result, S::Error> { + // self.store.get(k) + // } + // + // /// Remove the given key. + // pub fn remove(&mut self, k: &K) -> Result, S::Error> { + // self.store.remove(k) + // } + // + // /// List all existing key value pairs. + // pub fn all(&self) -> Result> + '_, S::Error> { + // self.store.all() + // } + // + // /// Returns a refernce to the underlying store. + // pub fn store(&self) -> &S { + // &self.store + // } } /// Sadly is still unstable.. diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index 86430dad44..703126c205 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -1,3 +1,5 @@ +//! Storage trait and implementation for iroh-sync documents + use anyhow::Result; use rand_core::CryptoRngCore; use serde::{Deserialize, Serialize}; @@ -16,30 +18,43 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// The specialized instance scoped to a `Namespace`. type Instance: ranger::Store + Send + Sync + 'static + Clone; + /// The iterator returned from [`Self::get`]. type GetIter<'a>: Iterator> where Self: 'a; - /// Open a replica + /// Create a new replica for `namespace` and persist in this store. + fn new_replica(&self, namespace: Namespace) -> Result>; + + /// List all replicas in this store. + // TODO: return iterator + fn list_replicas(&self) -> Result>; + + /// Open a replica from this store. /// /// Store implementers must ensure that only a single instance of [`Replica`] is created per /// namespace. On subsequent calls, a clone of that singleton instance must be returned. /// - /// TODO: Add close_replica + // TODO: Add close_replica fn open_replica(&self, namespace: &NamespaceId) -> Result>>; - // TODO: return iterator - fn list_replicas(&self) -> Result>; - fn get_author(&self, author: &AuthorId) -> Result>; + /// Create a new author key key and persist it in the store. fn new_author(&self, rng: &mut R) -> Result; + /// List all author keys in this store. // TODO: return iterator fn list_authors(&self) -> Result>; - fn new_replica(&self, namespace: Namespace) -> Result>; - /// Returns an iterator over the entries in a namespace. + /// Get an author key from the store. + fn get_author(&self, author: &AuthorId) -> Result>; + + /// Iterate over entries of a replica. + /// + /// Returns an iterator. The [`GetFilter`] has several methods of filtering the returned + /// entries. fn get(&self, namespace: NamespaceId, filter: GetFilter) -> Result>; - /// Gets the latest entry this key and author. + + /// Gets the single latest entry for the specified key and author. fn get_latest_by_key_and_author( &self, namespace: NamespaceId, @@ -51,9 +66,9 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// Filter a get query onto a namespace #[derive(Debug, Serialize, Deserialize)] pub struct GetFilter { - pub latest: bool, - pub author: Option, - pub key: KeyFilter, + latest: bool, + author: Option, + key: KeyFilter, } impl Default for GetFilter { @@ -63,22 +78,28 @@ impl Default for GetFilter { } impl GetFilter { - /// No filter, iterate over all entries. - pub fn all() -> Self { - Self { - latest: false, + /// Create a new get filter, either for only latest or all entries. + pub fn new(latest: bool) -> Self { + GetFilter { + latest, author: None, key: KeyFilter::All, } } + /// No filter, iterate over all entries. + pub fn all() -> Self { + Self::new(false) + } /// Only include the latest entries. pub fn latest() -> Self { - Self { - latest: true, - author: None, - key: KeyFilter::All, - } + Self::new(true) + } + + /// Set the key filter. + pub fn with_key_filter(mut self, key_filter: KeyFilter) -> Self { + self.key = key_filter; + self } /// Filter by exact key match. diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs index f0feea79a5..a3a9acfe5e 100644 --- a/iroh-sync/src/store/fs.rs +++ b/iroh-sync/src/store/fs.rs @@ -58,6 +58,9 @@ const RECORDS_TABLE: MultimapTableDefinition = MultimapTableDefinition::new("records-1"); impl Store { + /// Create or open a store from a `path` to a database file. + /// + /// The file will be created if it does not exist, otherwise it will be opened. pub fn new(path: impl AsRef) -> Result { let db = Database::create(path)?; @@ -99,8 +102,15 @@ impl Store { } } +/// Iterator for entries in [`Store`] +// We use a struct with an inner enum to exclude the enum variants from the public API. #[derive(Debug)] -pub enum GetIter<'s> { +pub struct GetIter<'s> { + inner: GetIterInner<'s>, +} + +#[derive(Debug)] +enum GetIterInner<'s> { All(RangeAllIterator<'s>), Latest(RangeLatestIterator<'s>), Single(std::option::IntoIter>), @@ -110,10 +120,10 @@ impl<'s> Iterator for GetIter<'s> { type Item = anyhow::Result; fn next(&mut self) -> Option { - match self { - GetIter::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), - GetIter::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), - GetIter::Single(iter) => iter.next(), + match &mut self.inner { + GetIterInner::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIterInner::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIterInner::Single(iter) => iter.next(), } } } @@ -194,25 +204,27 @@ impl super::Store for Store { fn get(&self, namespace: NamespaceId, filter: super::GetFilter) -> Result> { use super::KeyFilter::*; - Ok(match filter.latest { + let inner = match filter.latest { false => match (filter.key, filter.author) { - (All, None) => GetIter::All(self.get_all(namespace)?), - (Prefix(prefix), None) => GetIter::All(self.get_all_by_prefix(namespace, &prefix)?), - (Key(key), None) => GetIter::All(self.get_all_by_key(namespace, key)?), + (All, None) => GetIterInner::All(self.get_all(namespace)?), + (Prefix(prefix), None) => { + GetIterInner::All(self.get_all_by_prefix(namespace, &prefix)?) + } + (Key(key), None) => GetIterInner::All(self.get_all_by_key(namespace, key)?), (Key(key), Some(author)) => { - GetIter::All(self.get_all_by_key_and_author(namespace, author, key)?) + GetIterInner::All(self.get_all_by_key_and_author(namespace, author, key)?) } (All, Some(_)) | (Prefix(_), Some(_)) => { bail!("This filter combination is not yet supported") } }, true => match (filter.key, filter.author) { - (All, None) => GetIter::Latest(self.get_latest(namespace)?), + (All, None) => GetIterInner::Latest(self.get_latest(namespace)?), (Prefix(prefix), None) => { - GetIter::Latest(self.get_latest_by_prefix(namespace, &prefix)?) + GetIterInner::Latest(self.get_latest_by_prefix(namespace, &prefix)?) } - (Key(key), None) => GetIter::Latest(self.get_latest_by_key(namespace, key)?), - (Key(key), Some(author)) => GetIter::Single( + (Key(key), None) => GetIterInner::Latest(self.get_latest_by_key(namespace, key)?), + (Key(key), Some(author)) => GetIterInner::Single( self.get_latest_by_key_and_author(namespace, author, key)? .map(Ok) .into_iter(), @@ -221,7 +233,8 @@ impl super::Store for Store { bail!("This filter combination is not yet supported") } }, - }) + }; + Ok(GetIter { inner }) } fn get_latest_by_key_and_author( diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs index 254a1a3876..ad93f055f8 100644 --- a/iroh-sync/src/store/memory.rs +++ b/iroh-sync/src/store/memory.rs @@ -66,25 +66,27 @@ impl super::Store for Store { fn get(&self, namespace: NamespaceId, filter: super::GetFilter) -> Result> { use super::KeyFilter::*; - Ok(match filter.latest { + let inner = match filter.latest { false => match (filter.key, filter.author) { - (All, None) => GetIter::All(self.get_all(namespace)?), - (Prefix(prefix), None) => GetIter::All(self.get_all_by_prefix(namespace, &prefix)?), - (Key(key), None) => GetIter::All(self.get_all_by_key(namespace, key)?), + (All, None) => GetIterInner::All(self.get_all(namespace)?), + (Prefix(prefix), None) => { + GetIterInner::All(self.get_all_by_prefix(namespace, &prefix)?) + } + (Key(key), None) => GetIterInner::All(self.get_all_by_key(namespace, key)?), (Key(key), Some(author)) => { - GetIter::All(self.get_all_by_key_and_author(namespace, author, key)?) + GetIterInner::All(self.get_all_by_key_and_author(namespace, author, key)?) } (All, Some(_)) | (Prefix(_), Some(_)) => { bail!("This filter combination is not yet supported") } }, true => match (filter.key, filter.author) { - (All, None) => GetIter::Latest(self.get_latest(namespace)?), + (All, None) => GetIterInner::Latest(self.get_latest(namespace)?), (Prefix(prefix), None) => { - GetIter::Latest(self.get_latest_by_prefix(namespace, &prefix)?) + GetIterInner::Latest(self.get_latest_by_prefix(namespace, &prefix)?) } - (Key(key), None) => GetIter::Latest(self.get_latest_by_key(namespace, key)?), - (Key(key), Some(author)) => GetIter::Single( + (Key(key), None) => GetIterInner::Latest(self.get_latest_by_key(namespace, key)?), + (Key(key), Some(author)) => GetIterInner::Single( self.get_latest_by_key_and_author(namespace, author, key)? .map(Ok) .into_iter(), @@ -93,7 +95,8 @@ impl super::Store for Store { bail!("This filter combination is not yet supported") } }, - }) + }; + Ok(GetIter { inner }) } fn get_latest_by_key_and_author( @@ -113,8 +116,15 @@ impl super::Store for Store { } } +/// Iterator for entries in [`Store`] +// We use a struct with an inner enum to exclude the enum variants from the public API. #[derive(Debug)] -pub enum GetIter<'s> { +pub struct GetIter<'s> { + inner: GetIterInner<'s>, +} + +#[derive(Debug)] +enum GetIterInner<'s> { All(GetAllIter<'s>), Latest(GetLatestIter<'s>), Single(std::option::IntoIter>), @@ -124,10 +134,10 @@ impl<'s> Iterator for GetIter<'s> { type Item = anyhow::Result; fn next(&mut self) -> Option { - match self { - GetIter::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), - GetIter::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), - GetIter::Single(iter) => iter.next(), + match &mut self.inner { + GetIterInner::All(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIterInner::Latest(iter) => iter.next().map(|x| x.map(|(_id, entry)| entry)), + GetIterInner::Single(iter) => iter.next(), } } } @@ -266,7 +276,7 @@ impl GetFilter { } #[derive(Debug)] -pub struct GetLatestIter<'a> { +struct GetLatestIter<'a> { records: ReplicaRecords<'a>, filter: GetFilter, /// Current iteration index. @@ -321,7 +331,7 @@ impl<'a> Iterator for GetLatestIter<'a> { } #[derive(Debug)] -pub struct GetAllIter<'a> { +struct GetAllIter<'a> { records: ReplicaRecords<'a>, filter: GetFilter, /// Current iteration index. @@ -369,6 +379,7 @@ impl<'a> Iterator for GetAllIter<'a> { } } +/// Instance of a [`Store`] #[derive(Debug, Clone)] pub struct ReplicaStoreInstance { namespace: NamespaceId, @@ -529,6 +540,7 @@ impl crate::ranger::Store for ReplicaStoreInstanc } } +/// Range iterator for a [`ReplicaStoreInstance`] #[derive(Debug)] pub struct RangeIterator<'a> { iter: RecordsIter<'a>, diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 1ca881c41b..241acaf65a 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -1,3 +1,5 @@ +//! API for iroh-sync replicas + // Names and concepts are roughly based on Willows design at the moment: // // https://hackmd.io/DTtck8QOQm6tZaQBBtTf7w @@ -26,8 +28,14 @@ use serde::{Deserialize, Serialize}; use crate::ranger::{self, AsFingerprint, Fingerprint, Peer, RangeKey}; +/// Protocol message for the set reconciliation protocol +/// +/// Can be serialized to bytes with [serde] to transfer between peers. pub type ProtocolMessage = crate::ranger::Message; +/// Author key to insert entries in a [`Replica`] +/// +/// Internally, an author is a [`SigningKey`] which is used to sign entries. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Author { priv_key: SigningKey, @@ -40,12 +48,14 @@ impl Display for Author { } impl Author { + /// Create a new author with a random key. pub fn new(rng: &mut R) -> Self { let priv_key = SigningKey::generate(rng); Author { priv_key } } + /// Create an author from a byte array. pub fn from_bytes(bytes: &[u8; 32]) -> Self { SigningKey::from_bytes(bytes).into() } @@ -60,19 +70,26 @@ impl Author { self.priv_key.verifying_key().to_bytes() } + /// Get the [`AuthorId`] for this author. pub fn id(&self) -> AuthorId { AuthorId(self.priv_key.verifying_key()) } + /// Sign a message with this author key. pub fn sign(&self, msg: &[u8]) -> Signature { self.priv_key.sign(msg) } + /// Strictly verify a signature on a message with this author's public key. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.priv_key.verify_strict(msg, signature) } } +/// Identifier for an [`Author`] +/// +/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can +/// be used to verify [`Signature`]s. #[derive(Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct AuthorId(VerifyingKey); @@ -89,19 +106,33 @@ impl Display for AuthorId { } impl AuthorId { + /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] + /// that corresponds to this [`AuthorId`]. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.0.verify_strict(msg, signature) } + /// Get the byte representation of this [`AuthorId`]. pub fn as_bytes(&self) -> &[u8; 32] { self.0.as_bytes() } + /// Construct an `AuthorId` from a slice of bytes. + /// + /// # Warning + /// + /// The caller is responsible for ensuring that the bytes passed into this method actually + /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from + /// [`Self::as_bytes`]. pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { Ok(AuthorId(VerifyingKey::from_bytes(bytes)?)) } } +/// Namespace key of a [`Replica`]. +/// +/// Holders of this key can insert new entries into a [`Replica`]. +/// Internally, a namespace is a [`SigningKey`] which is used to sign entries. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Namespace { priv_key: SigningKey, @@ -172,39 +203,48 @@ impl From for Namespace { } impl Namespace { + /// Create a new namespace with a random key. pub fn new(rng: &mut R) -> Self { let priv_key = SigningKey::generate(rng); Namespace { priv_key } } + /// Create a namespace from a byte array. pub fn from_bytes(bytes: &[u8; 32]) -> Self { SigningKey::from_bytes(bytes).into() } - /// Returns the Namespace byte representation. + /// Returns the namespace byte representation. pub fn to_bytes(&self) -> [u8; 32] { self.priv_key.to_bytes() } - /// Returns the NamespaceId byte representation. + /// Returns the [`NamespaceId`] byte representation. pub fn id_bytes(&self) -> [u8; 32] { self.priv_key.verifying_key().to_bytes() } + /// Get the [`NamespaceId`] for this namespace. pub fn id(&self) -> NamespaceId { NamespaceId(self.priv_key.verifying_key()) } + /// Sign a message with this namespace key. pub fn sign(&self, msg: &[u8]) -> Signature { self.priv_key.sign(msg) } + /// Strictly verify a signature on a message with this namespaces's public key. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.priv_key.verify_strict(msg, signature) } } +/// Identifier for a [`Namespace`] +/// +/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can +/// be used to verify [`Signature`]s. #[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct NamespaceId(VerifyingKey); @@ -221,28 +261,43 @@ impl Debug for NamespaceId { } impl NamespaceId { + /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] + /// that corresponds to this [`NamespaceId`]. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.0.verify_strict(msg, signature) } + /// Get the byte representation of this [`NamespaceId`]. pub fn as_bytes(&self) -> &[u8; 32] { self.0.as_bytes() } + /// Construct a `NamespaceId` from a slice of bytes. + /// + /// # Warning + /// + /// The caller is responsible for ensuring that the bytes passed into this method actually + /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from + /// [`Self::as_bytes`]. pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { Ok(NamespaceId(VerifyingKey::from_bytes(bytes)?)) } } -/// TODO: PeerId is in iroh-net which iroh-sync doesn't depend on. Add iroh-common crate with `PeerId`. +/// Byte represenation of a `PeerId` from `iroh-net` +// TODO: PeerId is in iroh-net which iroh-sync doesn't depend on. Add iroh-common crate with `PeerId`. pub type PeerIdBytes = [u8; 32]; +/// Whether an entry was inserted locally or by a remote peer. #[derive(Debug, Clone)] pub enum InsertOrigin { + /// The entry was inserted locally. Local, + /// The entry was received from the remote peer identified by [`PeerIdBytes`]. Sync(PeerIdBytes), } +/// Local representation of a mutable, synchronizable key-value store. #[derive(derive_more::Debug, Clone)] pub struct Replica> { inner: Arc>>, @@ -264,7 +319,8 @@ struct ReplicaData { } impl> Replica { - // TODO: check that read only replicas are possible + /// Create a new replica. + // TODO: make read only replicas possible pub fn new(namespace: Namespace, store: S) -> Self { let (s, r) = flume::bounded(16); // TODO: should this be configurable? Replica { @@ -277,12 +333,19 @@ impl> Replica { } } - /// Subscribes to the events. Only one subscription can be active at a time. + /// Subscribe to insert events. + /// + /// Only one subscription can be active at a time. If a previous subscription was created, this + /// will return `None`. + // TODO: Allow to clear a previous subscription? pub fn subscribe(&self) -> Option> { self.on_insert_receiver.lock().take() } - /// Inserts a new record at the given key. + /// Insert a new record at the given key. + /// + /// The entry will by signed by the provided `author`. + /// The `len` must be the byte length of the data identified by `hash`. pub fn insert( &self, key: impl AsRef<[u8]>, @@ -315,8 +378,8 @@ impl> Replica { } /// Hashes the given data and inserts it. - /// This does not store the content, just the record of it. /// + /// This does not store the content, just the record of it. /// Returns the calculated hash. pub fn hash_and_insert( &self, @@ -330,11 +393,16 @@ impl> Replica { Ok(hash) } + /// Get the identifier for an entry in this replica. pub fn id(&self, key: impl AsRef<[u8]>, author: &Author) -> RecordIdentifier { let inner = self.inner.read(); RecordIdentifier::new(key, inner.namespace.id(), author.id()) } + /// Insert an entry into this replica which was received from a remote peer. + /// + /// This will verify both the namespace and author signatures of the entry, emit an `on_insert` + /// event, and insert the entry into the replica store. pub fn insert_remote_entry( &self, entry: SignedEntry, @@ -358,12 +426,16 @@ impl> Replica { Ok(()) } + /// Create the initial message for the set reconciliation flow with a remote peer. pub fn sync_initial_message( &self, ) -> Result, S::Error> { self.inner.read().peer.initial_message() } + /// Process a set reconciliation message from a remote peer. + /// + /// Returns the next message to be sent to the peer, if any. pub fn sync_process_message( &self, message: crate::ranger::Message, @@ -382,10 +454,13 @@ impl> Replica { Ok(reply) } + /// Get the namespace identifier for this [`Replica`]. pub fn namespace(&self) -> NamespaceId { self.inner.read().namespace.id() } + /// Get the byte represenation of the [`Namespace`] key for this replica. + // TODO: Why return [u8; 32] and not `Namespace` here? pub fn secret_key(&self) -> [u8; 32] { self.inner.read().namespace.to_bytes() } @@ -399,37 +474,48 @@ pub struct SignedEntry { } impl SignedEntry { - pub fn new(signature: EntrySignature, entry: Entry) -> Self { + pub(crate) fn new(signature: EntrySignature, entry: Entry) -> Self { SignedEntry { signature, entry } } + /// Create a new signed entry by signing an entry with a namespace and author. pub fn from_entry(entry: Entry, namespace: &Namespace, author: &Author) -> Self { let signature = EntrySignature::from_entry(&entry, namespace, author); SignedEntry { signature, entry } } + /// Verify the signatures on this entry. pub fn verify(&self) -> Result<(), SignatureError> { self.signature .verify(&self.entry, &self.entry.id.namespace, &self.entry.id.author) } + /// Get the signature. pub fn signature(&self) -> &EntrySignature { &self.signature } + /// Get the entry. pub fn entry(&self) -> &Entry { &self.entry } + /// Get the content hash of the entry. pub fn content_hash(&self) -> &Hash { self.entry().record().content_hash() } + + /// Get the content length of the entry. pub fn content_len(&self) -> u64 { self.entry().record().content_len() } + + /// Get the author of the entry. pub fn author(&self) -> AuthorId { self.entry().id().author() } + + /// Get the key of the entry. pub fn key(&self) -> &[u8] { self.entry().id().key() } @@ -443,6 +529,7 @@ pub struct EntrySignature { } impl EntrySignature { + /// Create a new signature by signing an entry with a namespace and author. pub fn from_entry(entry: &Entry, namespace: &Namespace, author: &Author) -> Self { // TODO: this should probably include a namespace prefix // namespace in the cryptographic sense. @@ -456,6 +543,8 @@ impl EntrySignature { } } + /// Verify that this signature was created by signing the `entry` with the + /// secret keys of the specified `author` and `namespace`. pub fn verify( &self, entry: &Entry, @@ -469,7 +558,7 @@ impl EntrySignature { Ok(()) } - pub fn from_parts(namespace_sig: &[u8; 64], author_sig: &[u8; 64]) -> Self { + pub(crate) fn from_parts(namespace_sig: &[u8; 64], author_sig: &[u8; 64]) -> Self { let namespace_signature = Signature::from_bytes(namespace_sig); let author_signature = Signature::from_bytes(author_sig); @@ -479,11 +568,11 @@ impl EntrySignature { } } - pub fn author_signature(&self) -> &Signature { + pub(crate) fn author_signature(&self) -> &Signature { &self.author_signature } - pub fn namespace_signature(&self) -> &Signature { + pub(crate) fn namespace_signature(&self) -> &Signature { &self.namespace_signature } } @@ -496,18 +585,22 @@ pub struct Entry { } impl Entry { + /// Create a new entry pub fn new(id: RecordIdentifier, record: Record) -> Self { Entry { id, record } } + /// Get the [`RecordIdentifier`] for this entry. pub fn id(&self) -> &RecordIdentifier { &self.id } + /// Get the [`NamespaceId`] of this entry. pub fn namespace(&self) -> NamespaceId { self.id.namespace() } + /// Get the [`Record`] contained in this entry. pub fn record(&self) -> &Record { &self.record } @@ -518,12 +611,14 @@ impl Entry { self.record.as_bytes(out); } + /// Serialize this entry into a new vector with its canonical byte representation. pub fn to_vec(&self) -> Vec { let mut out = Vec::new(); self.into_vec(&mut out); out } + /// Sign this entry with a [`Namespace`] and [`Author`]. pub fn sign(self, namespace: &Namespace, author: &Author) -> SignedEntry { SignedEntry::from_entry(self, namespace, author) } @@ -589,6 +684,7 @@ impl RangeKey for RecordIdentifier { } impl RecordIdentifier { + /// Create a new record identifier. pub fn new(key: impl AsRef<[u8]>, namespace: NamespaceId, author: AuthorId) -> Self { RecordIdentifier { key: key.as_ref().to_vec(), @@ -597,7 +693,11 @@ impl RecordIdentifier { } } - pub fn from_parts(key: &[u8], namespace: &[u8; 32], author: &[u8; 32]) -> anyhow::Result { + pub(crate) fn from_parts( + key: &[u8], + namespace: &[u8; 32], + author: &[u8; 32], + ) -> anyhow::Result { Ok(RecordIdentifier { key: key.to_vec(), namespace: NamespaceId::from_bytes(namespace)?, @@ -605,33 +705,38 @@ impl RecordIdentifier { }) } - pub fn as_bytes(&self, out: &mut Vec) { + /// Serialize this record identifier into a mutable byte array. + pub(crate) fn as_bytes(&self, out: &mut Vec) { out.extend_from_slice(self.namespace.as_bytes()); out.extend_from_slice(self.author.as_bytes()); out.extend_from_slice(&self.key); } + /// Get the key of this record. pub fn key(&self) -> &[u8] { &self.key } + /// Get the namespace of this record. pub fn namespace(&self) -> NamespaceId { self.namespace } - pub fn namespace_bytes(&self) -> &[u8; 32] { + pub(crate) fn namespace_bytes(&self) -> &[u8; 32] { self.namespace.as_bytes() } + /// Get the author of this record. pub fn author(&self) -> AuthorId { self.author } - pub fn author_bytes(&self) -> &[u8; 32] { + pub(crate) fn author_bytes(&self) -> &[u8; 32] { self.author.as_bytes() } } +/// The data part of an entry in a [`Replica`]. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Record { /// Record creation timestamp. Counted as micros since the Unix epoch. @@ -642,6 +747,7 @@ pub struct Record { } impl Record { + /// Create a new record. pub fn new(timestamp: u64, len: u64, hash: Hash) -> Self { Record { timestamp, @@ -650,18 +756,22 @@ impl Record { } } + /// Get the timestamp of this record. pub fn timestamp(&self) -> u64 { self.timestamp } + /// Get the length of the data to which this record's content hash refers to. pub fn content_len(&self) -> u64 { self.len } + /// Get the content hash of this record. pub fn content_hash(&self) -> &Hash { &self.hash } + /// Create a new record with a timestamp of the current system date. pub fn from_hash(hash: Hash, len: u64) -> Self { let timestamp = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) @@ -671,7 +781,8 @@ impl Record { } // TODO: remove - pub fn from_data(data: impl AsRef<[u8]>, namespace: NamespaceId) -> Self { + #[cfg(test)] + pub(crate) fn from_data(data: impl AsRef<[u8]>, namespace: NamespaceId) -> Self { // Salted hash // TODO: do we actually want this? // TODO: this should probably use a namespace prefix if used @@ -682,7 +793,8 @@ impl Record { Self::from_hash(hash.into(), data.as_ref().len() as u64) } - pub fn as_bytes(&self, out: &mut Vec) { + /// Serialize this record into a mutable byte array. + pub(crate) fn as_bytes(&self, out: &mut Vec) { out.extend_from_slice(&self.timestamp.to_be_bytes()); out.extend_from_slice(&self.len.to_be_bytes()); out.extend_from_slice(self.hash.as_ref()); diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 1c9d19a54e..bfb0ffed4e 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -12,7 +12,7 @@ use anyhow::{anyhow, Result}; use bytes::Bytes; use futures::{Stream, StreamExt, TryStreamExt}; use iroh_bytes::Hash; -use iroh_sync::store::{GetFilter, KeyFilter}; +use iroh_sync::store::{GetFilter}; use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; use quic_rpc::{RpcClient, ServiceConnection}; @@ -135,11 +135,7 @@ where } pub async fn get_latest(&self, author_id: AuthorId, key: Vec) -> Result { - let filter = GetFilter { - key: KeyFilter::Key(key), - author: Some(author_id), - latest: true, - }; + let filter = GetFilter::latest().with_key(key).with_author(author_id); let mut stream = self.get(filter).await?; let entry = stream .next() diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 1285a38ce0..93abdbea0d 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -6,7 +6,7 @@ use iroh::{ sync::PeerSource, }; use iroh_sync::{ - store::{GetFilter, KeyFilter}, + store::{GetFilter}, sync::{AuthorId, NamespaceId, SignedEntry}, }; use quic_rpc::transport::quinn::QuinnConnection; @@ -196,16 +196,18 @@ impl Doc { old, content, } => { - let key = key.as_bytes().to_vec(); - let key = match prefix { - true => KeyFilter::Prefix(key), - false => KeyFilter::Key(key), + let mut filter = match old { + true => GetFilter::all(), + false => GetFilter::latest(), + }; + if let Some(author) = author { + filter = filter.with_author(author); }; - let filter = GetFilter { - latest: !old, - author, - key, + let filter = match prefix { + true => filter.with_prefix(key), + false => filter.with_key(key), }; + let mut stream = doc.get(filter).await?; while let Some(entry) = stream.try_next().await? { println!("{}", fmt_entry(&entry)); @@ -229,14 +231,13 @@ impl Doc { } } Doc::List { old, prefix } => { - let key = match prefix { - Some(prefix) => KeyFilter::Prefix(prefix.as_bytes().to_vec()), - None => KeyFilter::All, + let filter = match old { + true => GetFilter::all(), + false => GetFilter::latest(), }; - let filter = GetFilter { - latest: !old, - author: None, - key, + let filter = match prefix { + Some(prefix) => filter.with_prefix(prefix), + None => filter, }; let mut stream = doc.get(filter).await?; while let Some(entry) = stream.try_next().await? { From ff90fe60f0a0e2547a85582837a7986a982e8c02 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 16 Aug 2023 18:07:43 +0200 Subject: [PATCH 095/172] refactor: move keys into module --- iroh-sync/src/keys.rs | 280 ++++++++++++++++++++++++++++++++++++++++ iroh-sync/src/lib.rs | 1 + iroh-sync/src/sync.rs | 288 +----------------------------------------- 3 files changed, 285 insertions(+), 284 deletions(-) create mode 100644 iroh-sync/src/keys.rs diff --git a/iroh-sync/src/keys.rs b/iroh-sync/src/keys.rs new file mode 100644 index 0000000000..210819e6fc --- /dev/null +++ b/iroh-sync/src/keys.rs @@ -0,0 +1,280 @@ +//! Keys used in iroh-sync + +use std::{cmp::Ordering, fmt, str::FromStr}; + +use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey}; +use rand_core::CryptoRngCore; +use serde::{Deserialize, Serialize}; + +/// Author key to insert entries in a [`Replica`] +/// +/// Internally, an author is a [`SigningKey`] which is used to sign entries. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Author { + priv_key: SigningKey, +} +impl Author { + /// Create a new author with a random key. + pub fn new(rng: &mut R) -> Self { + let priv_key = SigningKey::generate(rng); + Author { priv_key } + } + + /// Create an author from a byte array. + pub fn from_bytes(bytes: &[u8; 32]) -> Self { + SigningKey::from_bytes(bytes).into() + } + + /// Returns the Author byte representation. + pub fn to_bytes(&self) -> [u8; 32] { + self.priv_key.to_bytes() + } + + /// Returns the AuthorId byte representation. + pub fn id_bytes(&self) -> [u8; 32] { + self.priv_key.verifying_key().to_bytes() + } + + /// Get the [`AuthorId`] for this author. + pub fn id(&self) -> AuthorId { + AuthorId(self.priv_key.verifying_key()) + } + + /// Sign a message with this author key. + pub fn sign(&self, msg: &[u8]) -> Signature { + self.priv_key.sign(msg) + } + + /// Strictly verify a signature on a message with this author's public key. + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.priv_key.verify_strict(msg, signature) + } +} + +/// Identifier for an [`Author`] +/// +/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can +/// be used to verify [`Signature`]s. +#[derive(Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct AuthorId(VerifyingKey); + +impl AuthorId { + /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] + /// that corresponds to this [`AuthorId`]. + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.0.verify_strict(msg, signature) + } + + /// Get the byte representation of this [`AuthorId`]. + pub fn as_bytes(&self) -> &[u8; 32] { + self.0.as_bytes() + } + + /// Construct an `AuthorId` from a slice of bytes. + /// + /// # Warning + /// + /// The caller is responsible for ensuring that the bytes passed into this method actually + /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from + /// [`Self::as_bytes`]. + pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { + Ok(AuthorId(VerifyingKey::from_bytes(bytes)?)) + } +} + +/// Namespace key of a [`Replica`]. +/// +/// Holders of this key can insert new entries into a [`Replica`]. +/// Internally, a namespace is a [`SigningKey`] which is used to sign entries. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Namespace { + priv_key: SigningKey, +} + +impl Namespace { + /// Create a new namespace with a random key. + pub fn new(rng: &mut R) -> Self { + let priv_key = SigningKey::generate(rng); + + Namespace { priv_key } + } + + /// Create a namespace from a byte array. + pub fn from_bytes(bytes: &[u8; 32]) -> Self { + SigningKey::from_bytes(bytes).into() + } + + /// Returns the namespace byte representation. + pub fn to_bytes(&self) -> [u8; 32] { + self.priv_key.to_bytes() + } + + /// Returns the [`NamespaceId`] byte representation. + pub fn id_bytes(&self) -> [u8; 32] { + self.priv_key.verifying_key().to_bytes() + } + + /// Get the [`NamespaceId`] for this namespace. + pub fn id(&self) -> NamespaceId { + NamespaceId(self.priv_key.verifying_key()) + } + + /// Sign a message with this namespace key. + pub fn sign(&self, msg: &[u8]) -> Signature { + self.priv_key.sign(msg) + } + + /// Strictly verify a signature on a message with this namespaces's public key. + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.priv_key.verify_strict(msg, signature) + } +} + +/// Identifier for a [`Namespace`] +/// +/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can +/// be used to verify [`Signature`]s. +#[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct NamespaceId(VerifyingKey); + +impl NamespaceId { + /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] + /// that corresponds to this [`NamespaceId`]. + pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { + self.0.verify_strict(msg, signature) + } + + /// Get the byte representation of this [`NamespaceId`]. + pub fn as_bytes(&self) -> &[u8; 32] { + self.0.as_bytes() + } + + /// Construct a `NamespaceId` from a slice of bytes. + /// + /// # Warning + /// + /// The caller is responsible for ensuring that the bytes passed into this method actually + /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from + /// [`Self::as_bytes`]. + pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { + Ok(NamespaceId(VerifyingKey::from_bytes(bytes)?)) + } +} + +impl fmt::Display for Author { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Author({})", hex::encode(self.priv_key.to_bytes())) + } +} + +impl fmt::Display for Namespace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Namespace({})", hex::encode(self.priv_key.to_bytes())) + } +} + +impl fmt::Display for AuthorId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", hex::encode(self.0.as_bytes())) + } +} + +impl fmt::Display for NamespaceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", hex::encode(self.0.as_bytes())) + } +} + +impl fmt::Debug for NamespaceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "NamespaceId({})", hex::encode(self.0.as_bytes())) + } +} + +impl fmt::Debug for AuthorId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "AuthorId({})", hex::encode(self.0.as_bytes())) + } +} + +impl FromStr for Author { + type Err = (); + + fn from_str(s: &str) -> Result { + let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; + let priv_key = SigningKey::from_bytes(&priv_key); + + Ok(Author { priv_key }) + } +} + +impl FromStr for Namespace { + type Err = (); + + fn from_str(s: &str) -> Result { + let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; + let priv_key = SigningKey::from_bytes(&priv_key); + + Ok(Namespace { priv_key }) + } +} + +impl FromStr for AuthorId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let pub_key: [u8; 32] = hex::decode(s)? + .try_into() + .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; + let pub_key = VerifyingKey::from_bytes(&pub_key)?; + Ok(AuthorId(pub_key)) + } +} + +impl FromStr for NamespaceId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let pub_key: [u8; 32] = hex::decode(s)? + .try_into() + .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; + let pub_key = VerifyingKey::from_bytes(&pub_key)?; + Ok(NamespaceId(pub_key)) + } +} + +impl From for Author { + fn from(priv_key: SigningKey) -> Self { + Self { priv_key } + } +} + +impl From for Namespace { + fn from(priv_key: SigningKey) -> Self { + Self { priv_key } + } +} + +impl PartialOrd for NamespaceId { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for NamespaceId { + fn cmp(&self, other: &Self) -> Ordering { + self.0.as_bytes().cmp(other.0.as_bytes()) + } +} + +impl PartialOrd for AuthorId { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for AuthorId { + fn cmp(&self, other: &Self) -> Ordering { + self.0.as_bytes().cmp(other.0.as_bytes()) + } +} diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index fa2a45831f..d57b499908 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -6,3 +6,4 @@ pub mod metrics; mod ranger; pub mod store; pub mod sync; +mod keys; diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 241acaf65a..7d0d53ce8d 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -6,13 +6,7 @@ // // This is going to change! -use std::{ - cmp::Ordering, - fmt::{Debug, Display}, - str::FromStr, - sync::Arc, - time::SystemTime, -}; +use std::{fmt::Debug, sync::Arc, time::SystemTime}; #[cfg(feature = "metrics")] use crate::metrics::Metrics; @@ -21,269 +15,19 @@ use iroh_metrics::{inc, inc_by}; use parking_lot::{Mutex, RwLock}; -use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey}; +use ed25519_dalek::{Signature, SignatureError}; use iroh_bytes::Hash; -use rand_core::CryptoRngCore; use serde::{Deserialize, Serialize}; use crate::ranger::{self, AsFingerprint, Fingerprint, Peer, RangeKey}; +pub use crate::keys::*; + /// Protocol message for the set reconciliation protocol /// /// Can be serialized to bytes with [serde] to transfer between peers. pub type ProtocolMessage = crate::ranger::Message; -/// Author key to insert entries in a [`Replica`] -/// -/// Internally, an author is a [`SigningKey`] which is used to sign entries. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Author { - priv_key: SigningKey, -} - -impl Display for Author { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Author({})", hex::encode(self.priv_key.to_bytes())) - } -} - -impl Author { - /// Create a new author with a random key. - pub fn new(rng: &mut R) -> Self { - let priv_key = SigningKey::generate(rng); - - Author { priv_key } - } - - /// Create an author from a byte array. - pub fn from_bytes(bytes: &[u8; 32]) -> Self { - SigningKey::from_bytes(bytes).into() - } - - /// Returns the Author byte representation. - pub fn to_bytes(&self) -> [u8; 32] { - self.priv_key.to_bytes() - } - - /// Returns the AuthorId byte representation. - pub fn id_bytes(&self) -> [u8; 32] { - self.priv_key.verifying_key().to_bytes() - } - - /// Get the [`AuthorId`] for this author. - pub fn id(&self) -> AuthorId { - AuthorId(self.priv_key.verifying_key()) - } - - /// Sign a message with this author key. - pub fn sign(&self, msg: &[u8]) -> Signature { - self.priv_key.sign(msg) - } - - /// Strictly verify a signature on a message with this author's public key. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.priv_key.verify_strict(msg, signature) - } -} - -/// Identifier for an [`Author`] -/// -/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can -/// be used to verify [`Signature`]s. -#[derive(Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] -pub struct AuthorId(VerifyingKey); - -impl Debug for AuthorId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "AuthorId({})", hex::encode(self.0.as_bytes())) - } -} - -impl Display for AuthorId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", hex::encode(self.0.as_bytes())) - } -} - -impl AuthorId { - /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] - /// that corresponds to this [`AuthorId`]. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.0.verify_strict(msg, signature) - } - - /// Get the byte representation of this [`AuthorId`]. - pub fn as_bytes(&self) -> &[u8; 32] { - self.0.as_bytes() - } - - /// Construct an `AuthorId` from a slice of bytes. - /// - /// # Warning - /// - /// The caller is responsible for ensuring that the bytes passed into this method actually - /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from - /// [`Self::as_bytes`]. - pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { - Ok(AuthorId(VerifyingKey::from_bytes(bytes)?)) - } -} - -/// Namespace key of a [`Replica`]. -/// -/// Holders of this key can insert new entries into a [`Replica`]. -/// Internally, a namespace is a [`SigningKey`] which is used to sign entries. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Namespace { - priv_key: SigningKey, -} - -impl Display for Namespace { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Namespace({})", hex::encode(self.priv_key.to_bytes())) - } -} - -impl FromStr for Namespace { - type Err = (); - - fn from_str(s: &str) -> Result { - let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; - let priv_key = SigningKey::from_bytes(&priv_key); - - Ok(Namespace { priv_key }) - } -} - -impl FromStr for Author { - type Err = (); - - fn from_str(s: &str) -> Result { - let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; - let priv_key = SigningKey::from_bytes(&priv_key); - - Ok(Author { priv_key }) - } -} - -impl FromStr for AuthorId { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - let pub_key: [u8; 32] = hex::decode(s)? - .try_into() - .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; - let pub_key = VerifyingKey::from_bytes(&pub_key)?; - Ok(AuthorId(pub_key)) - } -} - -impl FromStr for NamespaceId { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - let pub_key: [u8; 32] = hex::decode(s)? - .try_into() - .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; - let pub_key = VerifyingKey::from_bytes(&pub_key)?; - Ok(NamespaceId(pub_key)) - } -} - -impl From for Author { - fn from(priv_key: SigningKey) -> Self { - Self { priv_key } - } -} - -impl From for Namespace { - fn from(priv_key: SigningKey) -> Self { - Self { priv_key } - } -} - -impl Namespace { - /// Create a new namespace with a random key. - pub fn new(rng: &mut R) -> Self { - let priv_key = SigningKey::generate(rng); - - Namespace { priv_key } - } - - /// Create a namespace from a byte array. - pub fn from_bytes(bytes: &[u8; 32]) -> Self { - SigningKey::from_bytes(bytes).into() - } - - /// Returns the namespace byte representation. - pub fn to_bytes(&self) -> [u8; 32] { - self.priv_key.to_bytes() - } - - /// Returns the [`NamespaceId`] byte representation. - pub fn id_bytes(&self) -> [u8; 32] { - self.priv_key.verifying_key().to_bytes() - } - - /// Get the [`NamespaceId`] for this namespace. - pub fn id(&self) -> NamespaceId { - NamespaceId(self.priv_key.verifying_key()) - } - - /// Sign a message with this namespace key. - pub fn sign(&self, msg: &[u8]) -> Signature { - self.priv_key.sign(msg) - } - - /// Strictly verify a signature on a message with this namespaces's public key. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.priv_key.verify_strict(msg, signature) - } -} - -/// Identifier for a [`Namespace`] -/// -/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can -/// be used to verify [`Signature`]s. -#[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] -pub struct NamespaceId(VerifyingKey); - -impl Display for NamespaceId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", hex::encode(self.0.as_bytes())) - } -} - -impl Debug for NamespaceId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "NamespaceId({})", hex::encode(self.0.as_bytes())) - } -} - -impl NamespaceId { - /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] - /// that corresponds to this [`NamespaceId`]. - pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.0.verify_strict(msg, signature) - } - - /// Get the byte representation of this [`NamespaceId`]. - pub fn as_bytes(&self) -> &[u8; 32] { - self.0.as_bytes() - } - - /// Construct a `NamespaceId` from a slice of bytes. - /// - /// # Warning - /// - /// The caller is responsible for ensuring that the bytes passed into this method actually - /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from - /// [`Self::as_bytes`]. - pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { - Ok(NamespaceId(VerifyingKey::from_bytes(bytes)?)) - } -} - /// Byte represenation of a `PeerId` from `iroh-net` // TODO: PeerId is in iroh-net which iroh-sync doesn't depend on. Add iroh-common crate with `PeerId`. pub type PeerIdBytes = [u8; 32]; @@ -645,30 +389,6 @@ impl AsFingerprint for RecordIdentifier { } } -impl PartialOrd for NamespaceId { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for NamespaceId { - fn cmp(&self, other: &Self) -> Ordering { - self.0.as_bytes().cmp(other.0.as_bytes()) - } -} - -impl PartialOrd for AuthorId { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for AuthorId { - fn cmp(&self, other: &Self) -> Ordering { - self.0.as_bytes().cmp(other.0.as_bytes()) - } -} - impl RangeKey for RecordIdentifier { fn contains(&self, range: &crate::ranger::Range) -> bool { use crate::ranger::contains; From 1fb6f8f45cf65d262e2e1d49855d0feebd4aa70d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 16 Aug 2023 18:11:50 +0200 Subject: [PATCH 096/172] chore: fmt --- iroh-sync/src/lib.rs | 2 +- iroh/src/client.rs | 2 +- iroh/src/commands/sync.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index d57b499908..43eafeffa8 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -1,9 +1,9 @@ //! Set reconciliation for multi-dimensional key-value stores #![deny(missing_docs, rustdoc::broken_intra_doc_links)] +mod keys; #[cfg(feature = "metrics")] pub mod metrics; mod ranger; pub mod store; pub mod sync; -mod keys; diff --git a/iroh/src/client.rs b/iroh/src/client.rs index bfb0ffed4e..c662369c25 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -12,7 +12,7 @@ use anyhow::{anyhow, Result}; use bytes::Bytes; use futures::{Stream, StreamExt, TryStreamExt}; use iroh_bytes::Hash; -use iroh_sync::store::{GetFilter}; +use iroh_sync::store::GetFilter; use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; use quic_rpc::{RpcClient, ServiceConnection}; diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 93abdbea0d..57a33b1ca5 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -6,7 +6,7 @@ use iroh::{ sync::PeerSource, }; use iroh_sync::{ - store::{GetFilter}, + store::GetFilter, sync::{AuthorId, NamespaceId, SignedEntry}, }; use quic_rpc::transport::quinn::QuinnConnection; From b666f87bca199b19907449e314802453fa9913fa Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 16 Aug 2023 20:59:38 +0200 Subject: [PATCH 097/172] refactor: align variable names with rust crypto --- iroh-sync/src/keys.rs | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/iroh-sync/src/keys.rs b/iroh-sync/src/keys.rs index 210819e6fc..a08b4da592 100644 --- a/iroh-sync/src/keys.rs +++ b/iroh-sync/src/keys.rs @@ -11,13 +11,13 @@ use serde::{Deserialize, Serialize}; /// Internally, an author is a [`SigningKey`] which is used to sign entries. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Author { - priv_key: SigningKey, + signing_key: SigningKey, } impl Author { /// Create a new author with a random key. pub fn new(rng: &mut R) -> Self { - let priv_key = SigningKey::generate(rng); - Author { priv_key } + let signing_key = SigningKey::generate(rng); + Author { signing_key } } /// Create an author from a byte array. @@ -27,27 +27,27 @@ impl Author { /// Returns the Author byte representation. pub fn to_bytes(&self) -> [u8; 32] { - self.priv_key.to_bytes() + self.signing_key.to_bytes() } /// Returns the AuthorId byte representation. pub fn id_bytes(&self) -> [u8; 32] { - self.priv_key.verifying_key().to_bytes() + self.signing_key.verifying_key().to_bytes() } /// Get the [`AuthorId`] for this author. pub fn id(&self) -> AuthorId { - AuthorId(self.priv_key.verifying_key()) + AuthorId(self.signing_key.verifying_key()) } /// Sign a message with this author key. pub fn sign(&self, msg: &[u8]) -> Signature { - self.priv_key.sign(msg) + self.signing_key.sign(msg) } /// Strictly verify a signature on a message with this author's public key. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.priv_key.verify_strict(msg, signature) + self.signing_key.verify_strict(msg, signature) } } @@ -88,15 +88,15 @@ impl AuthorId { /// Internally, a namespace is a [`SigningKey`] which is used to sign entries. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Namespace { - priv_key: SigningKey, + signing_key: SigningKey, } impl Namespace { /// Create a new namespace with a random key. pub fn new(rng: &mut R) -> Self { - let priv_key = SigningKey::generate(rng); + let signing_key = SigningKey::generate(rng); - Namespace { priv_key } + Namespace { signing_key } } /// Create a namespace from a byte array. @@ -106,27 +106,27 @@ impl Namespace { /// Returns the namespace byte representation. pub fn to_bytes(&self) -> [u8; 32] { - self.priv_key.to_bytes() + self.signing_key.to_bytes() } /// Returns the [`NamespaceId`] byte representation. pub fn id_bytes(&self) -> [u8; 32] { - self.priv_key.verifying_key().to_bytes() + self.signing_key.verifying_key().to_bytes() } /// Get the [`NamespaceId`] for this namespace. pub fn id(&self) -> NamespaceId { - NamespaceId(self.priv_key.verifying_key()) + NamespaceId(self.signing_key.verifying_key()) } /// Sign a message with this namespace key. pub fn sign(&self, msg: &[u8]) -> Signature { - self.priv_key.sign(msg) + self.signing_key.sign(msg) } /// Strictly verify a signature on a message with this namespaces's public key. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { - self.priv_key.verify_strict(msg, signature) + self.signing_key.verify_strict(msg, signature) } } @@ -163,13 +163,13 @@ impl NamespaceId { impl fmt::Display for Author { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Author({})", hex::encode(self.priv_key.to_bytes())) + write!(f, "Author({})", hex::encode(self.signing_key.to_bytes())) } } impl fmt::Display for Namespace { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Namespace({})", hex::encode(self.priv_key.to_bytes())) + write!(f, "Namespace({})", hex::encode(self.signing_key.to_bytes())) } } @@ -201,10 +201,10 @@ impl FromStr for Author { type Err = (); fn from_str(s: &str) -> Result { - let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; - let priv_key = SigningKey::from_bytes(&priv_key); + let signing_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; + let signing_key = SigningKey::from_bytes(&signing_key); - Ok(Author { priv_key }) + Ok(Author { signing_key }) } } @@ -212,10 +212,10 @@ impl FromStr for Namespace { type Err = (); fn from_str(s: &str) -> Result { - let priv_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; - let priv_key = SigningKey::from_bytes(&priv_key); + let signing_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; + let signing_key = SigningKey::from_bytes(&signing_key); - Ok(Namespace { priv_key }) + Ok(Namespace { signing_key }) } } @@ -223,11 +223,11 @@ impl FromStr for AuthorId { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let pub_key: [u8; 32] = hex::decode(s)? + let verifying_key: [u8; 32] = hex::decode(s)? .try_into() .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; - let pub_key = VerifyingKey::from_bytes(&pub_key)?; - Ok(AuthorId(pub_key)) + let verifying_key = VerifyingKey::from_bytes(&verifying_key)?; + Ok(AuthorId(verifying_key)) } } @@ -235,23 +235,23 @@ impl FromStr for NamespaceId { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let pub_key: [u8; 32] = hex::decode(s)? + let verifying_key: [u8; 32] = hex::decode(s)? .try_into() .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; - let pub_key = VerifyingKey::from_bytes(&pub_key)?; - Ok(NamespaceId(pub_key)) + let verifying_key = VerifyingKey::from_bytes(&verifying_key)?; + Ok(NamespaceId(verifying_key)) } } impl From for Author { - fn from(priv_key: SigningKey) -> Self { - Self { priv_key } + fn from(signing_key: SigningKey) -> Self { + Self { signing_key } } } impl From for Namespace { - fn from(priv_key: SigningKey) -> Self { - Self { priv_key } + fn from(signing_key: SigningKey) -> Self { + Self { signing_key } } } From 4b50b64c2715af8f2625d6d193dcc545da48a029 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 16 Aug 2023 21:16:43 +0200 Subject: [PATCH 098/172] docs: improve docs further --- iroh-sync/src/keys.rs | 26 +++++++++++--------------- iroh-sync/src/lib.rs | 33 ++++++++++++++++++++++++++++++++- iroh-sync/src/sync.rs | 9 +++++++-- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/iroh-sync/src/keys.rs b/iroh-sync/src/keys.rs index a08b4da592..922f60736f 100644 --- a/iroh-sync/src/keys.rs +++ b/iroh-sync/src/keys.rs @@ -6,7 +6,7 @@ use ed25519_dalek::{Signature, SignatureError, Signer, SigningKey, VerifyingKey} use rand_core::CryptoRngCore; use serde::{Deserialize, Serialize}; -/// Author key to insert entries in a [`Replica`] +/// Author key to insert entries in a [`crate::Replica`] /// /// Internally, an author is a [`SigningKey`] which is used to sign entries. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -70,21 +70,19 @@ impl AuthorId { self.0.as_bytes() } - /// Construct an `AuthorId` from a slice of bytes. + /// Create from a slice of bytes. /// - /// # Warning - /// - /// The caller is responsible for ensuring that the bytes passed into this method actually - /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from - /// [`Self::as_bytes`]. + /// Will return an error if the input bytes do not represent a valid [`ed25519_dalek`] + /// curve point. Will never fail for a byte array returned from [`Self::as_bytes`]. + /// See [`VerifyingKey::from_bytes`] for details. pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { Ok(AuthorId(VerifyingKey::from_bytes(bytes)?)) } } -/// Namespace key of a [`Replica`]. +/// Namespace key of a [`crate::Replica`]. /// -/// Holders of this key can insert new entries into a [`Replica`]. +/// Holders of this key can insert new entries into a [`crate::Replica`]. /// Internally, a namespace is a [`SigningKey`] which is used to sign entries. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Namespace { @@ -149,13 +147,11 @@ impl NamespaceId { self.0.as_bytes() } - /// Construct a `NamespaceId` from a slice of bytes. - /// - /// # Warning + /// Create from a slice of bytes. /// - /// The caller is responsible for ensuring that the bytes passed into this method actually - /// represent a valid [`ed25591`] curve point. This will never fail for bytes returned from - /// [`Self::as_bytes`]. + /// Will return an error if the input bytes do not represent a valid [`ed25519_dalek`] + /// curve point. Will never fail for a byte array returned from [`Self::as_bytes`]. + /// See [`VerifyingKey::from_bytes`] for details. pub fn from_bytes(bytes: &[u8; 32]) -> anyhow::Result { Ok(NamespaceId(VerifyingKey::from_bytes(bytes)?)) } diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index 43eafeffa8..3e3803f667 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -1,4 +1,32 @@ -//! Set reconciliation for multi-dimensional key-value stores +//! Multi-dimensional key-value documents with an efficient synchronization protocol +//! +//! The crate operates on [Replicas](Replica). A replica contains an unlimited number of +//! [Entrys][Entry]. Each entry is identified by a key, its author, and the replica's +//! namespace. Its value is the [32-byte BLAKE3 hash](iroh_bytes::Hash) +//! of the entry's content data, the size of this content data, and a timestamp. +//! The content data itself is not stored or transfered through a replica. +//! +//! All entries in a replica are signed with two keypairs: +//! +//! * The [Namespace] key, as a token of write capability. The public key is the [NamespaceId], which +//! also serves as the unique identifier for a replica. +//! * The [Author] key, as a proof of authorship. Any number of authors may be created, and an +//! their semantic meaning is application-specific. The public key of an author is the [AuthorId]. +//! +//! Replicas can be synchronized between peers by exchanging messages. The synchronization algorithm +//! is based on a technique called *range-based set reconciliation*, based on [this paper][paper] by +//! Aljoscha Meyer: +//! +//! > Range-based set reconciliation is a simple approach to efficiently computing the union of two +//! sets over a network, based on recursively partitioning the sets and comparing fingerprints of +//! the partitions to probabilistically detect whether a partition requires further work. +//! +//! The crate exposes a [generic storage interface](store::Store) with +//! [in-memory](store::memory::Store) and [persistent, file-based](store::fs::Store) +//! implementations. The latter makes use of [`redb`], an embedded key-value store, and persists +//! the whole store with all replicas to a single file. +//! +//! [paper]: https://arxiv.org/abs/2212.13567 #![deny(missing_docs, rustdoc::broken_intra_doc_links)] mod keys; @@ -7,3 +35,6 @@ pub mod metrics; mod ranger; pub mod store; pub mod sync; + +pub use keys::*; +pub use sync::*; diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 7d0d53ce8d..0077d15410 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -322,6 +322,10 @@ impl EntrySignature { } /// A single entry in a replica. +/// +/// An entry is identified by a key, its author, and the replica's +/// namespace. Its value is the [32-byte BLAKE3 hash](iroh_bytes::Hash) +/// of the entry's content data, the size of this content data, and a timestamp. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Entry { id: RecordIdentifier, @@ -463,6 +467,7 @@ pub struct Record { timestamp: u64, /// Length of the data referenced by `hash`. len: u64, + /// Hash of the content data. hash: Hash, } @@ -481,12 +486,12 @@ impl Record { self.timestamp } - /// Get the length of the data to which this record's content hash refers to. + /// Get the length of the data addressed by this record's content hash. pub fn content_len(&self) -> u64 { self.len } - /// Get the content hash of this record. + /// Get the hash of the content data of this record. pub fn content_hash(&self) -> &Hash { &self.hash } From 337fc1d58841234f18c82b364a16fe68ed827e4e Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 16 Aug 2023 23:45:54 +0200 Subject: [PATCH 099/172] fix: item visibility in iroh-sync --- iroh-sync/src/ranger.rs | 51 ++++++++++++++++++++++------------------- iroh/examples/client.rs | 13 ++--------- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/iroh-sync/src/ranger.rs b/iroh-sync/src/ranger.rs index faaed06ce7..cbec0c359b 100644 --- a/iroh-sync/src/ranger.rs +++ b/iroh-sync/src/ranger.rs @@ -402,23 +402,25 @@ where } } -// TODO: with_limit is unused, remove? -// impl Peer -// where -// K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, -// V: Clone + Debug, -// S: Store + Default, -// { -// fn with_limit(limit: Range) -> Self { -// Peer { -// store: S::default(), -// max_set_size: 1, -// split_factor: 2, -// limit: Some(limit), -// _phantom: Default::default(), -// } -// } -// } +// currently unused outside of tests +#[cfg(test)] +impl Peer +where + K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, + V: Clone + Debug, + S: Store + Default, +{ + fn with_limit(limit: Range) -> Self { + Peer { + store: S::default(), + max_set_size: 1, + split_factor: 2, + limit: Some(limit), + _phantom: Default::default(), + } + } +} + impl Peer where K: PartialEq + RangeKey + Clone + Default + Debug + AsFingerprint, @@ -613,21 +615,22 @@ where self.store.put(k, v) } - // TODO: these are unused, remove? + /// List all existing key value pairs. + // currently unused outside of tests + #[cfg(test)] + pub fn all(&self) -> Result> + '_, S::Error> { + self.store.all() + } + + // /// Get the entry for the given key. // pub fn get(&self, k: &K) -> Result, S::Error> { // self.store.get(k) // } - // // /// Remove the given key. // pub fn remove(&mut self, k: &K) -> Result, S::Error> { // self.store.remove(k) // } // - // /// List all existing key value pairs. - // pub fn all(&self) -> Result> + '_, S::Error> { - // self.store.all() - // } - // // /// Returns a refernce to the underlying store. // pub fn store(&self) -> &S { // &self.store diff --git a/iroh/examples/client.rs b/iroh/examples/client.rs index 8c5896eb70..e72a67bfcf 100644 --- a/iroh/examples/client.rs +++ b/iroh/examples/client.rs @@ -1,10 +1,7 @@ use indicatif::HumanBytes; use iroh::node::Node; use iroh_bytes::util::runtime; -use iroh_sync::{ - store::{GetFilter, KeyFilter}, - sync::SignedEntry, -}; +use iroh_sync::{store::GetFilter, sync::SignedEntry}; use tokio_stream::StreamExt; #[tokio::main] @@ -22,13 +19,7 @@ async fn main() -> anyhow::Result<()> { let key = b"hello".to_vec(); let value = b"world".to_vec(); doc.set_bytes(author, key.clone(), value).await?; - let mut stream = doc - .get(GetFilter { - latest: true, - key: KeyFilter::All, - author: None, - }) - .await?; + let mut stream = doc.get(GetFilter::latest()).await?; while let Some(entry) = stream.try_next().await? { println!("entry {}", fmt_entry(&entry)); let content = doc.get_content_bytes(&entry).await?; From 4df7796ff63bc8a4dc1db2362700a94d4c2597d8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 01:34:25 +0200 Subject: [PATCH 100/172] remove obsolete comments --- iroh/src/commands/sync.rs | 7 ++----- iroh/src/node.rs | 3 --- iroh/src/sync/live.rs | 7 +------ 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 57a33b1ca5..cac2b8943b 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -2,20 +2,17 @@ use clap::Parser; use futures::TryStreamExt; use indicatif::HumanBytes; use iroh::{ - rpc_protocol::{DocTicket, ProviderRequest, ProviderResponse, ShareMode}, + rpc_protocol::{DocTicket, ShareMode}, + client::quic::Iroh, sync::PeerSource, }; use iroh_sync::{ store::GetFilter, sync::{AuthorId, NamespaceId, SignedEntry}, }; -use quic_rpc::transport::quinn::QuinnConnection; use super::RpcClient; -// TODO: It is a bit unfortunate that we have to drag the generics all through. Maybe box the conn? -pub type Iroh = iroh::client::Iroh>; - const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; #[allow(clippy::large_enum_variant)] diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 1ccfa7b356..8f6effc6c1 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -347,7 +347,6 @@ where gossip_cell.set(gossip.clone()).unwrap(); // spawn the sync engine - // TODO: Remove once persistence is merged let downloader = Downloader::new(rt.clone(), endpoint.clone(), self.db.clone()); let sync = SyncEngine::spawn( rt.clone(), @@ -607,8 +606,6 @@ impl Node { /// Returns a new builder for the [`Node`]. /// /// Once the done with the builder call [`Builder::spawn`] to create the node. - /// - /// TODO: remove blobs_path argument once peristence branch is merged pub fn builder(bao_store: D, doc_store: S) -> Builder { Builder::with_db_and_store(bao_store, doc_store) } diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 30c510ba60..6270e693a6 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -37,8 +37,7 @@ use tracing::{debug, error, info, warn}; const CHANNEL_CAP: usize = 8; /// The address to connect to a peer -/// TODO: Move into iroh-net -/// TODO: Make an enum and support DNS resolution +// TODO: Move into iroh-net #[derive(Clone, Debug, Serialize, Deserialize)] pub struct PeerSource { /// The peer id (required) @@ -49,9 +48,6 @@ pub struct PeerSource { pub derp_region: Option, } -// /// A SyncId is a 32 byte array which is both a [`NamespaceId`] and a [`TopicId`]. -// pub struct SyncId([u8; 32]); - impl PeerSource { /// Deserializes from bytes. fn from_bytes(bytes: &[u8]) -> anyhow::Result { @@ -298,7 +294,6 @@ impl LiveSync { } } -// TODO: Also add `handle_connection` to the replica and track incoming sync requests here too. // Currently peers might double-sync in both directions. struct Actor { endpoint: MagicEndpoint, From b103077fd91784b23bc311ae69b521f7423a6187 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 02:21:36 +0200 Subject: [PATCH 101/172] refactor: make list_authors and list_namespaces return an interator --- iroh-sync/src/store.rs | 17 ++++++++++--- iroh-sync/src/store/fs.rs | 40 ++++++++++++++++------------- iroh-sync/src/store/memory.rs | 26 ++++++++++++++++--- iroh/src/commands/sync.rs | 2 +- iroh/src/sync/rpc.rs | 48 ++++++++++++++++++++++------------- 5 files changed, 88 insertions(+), 45 deletions(-) diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index 703126c205..ec5c6fdf49 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -18,17 +18,26 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// The specialized instance scoped to a `Namespace`. type Instance: ranger::Store + Send + Sync + 'static + Clone; - /// The iterator returned from [`Self::get`]. + /// Iterator over entries in the store, returned from [`Self::get`] type GetIter<'a>: Iterator> where Self: 'a; + /// Iterator over replicas in the store, returned from [`Self::list_replicas`] + type NamespaceIter<'a>: Iterator> + where + Self: 'a; + + /// Iterator over authors in the store, returned from [`Self::list_replicas`] + type AuthorsIter<'a>: Iterator> + where + Self: 'a; + /// Create a new replica for `namespace` and persist in this store. fn new_replica(&self, namespace: Namespace) -> Result>; /// List all replicas in this store. - // TODO: return iterator - fn list_replicas(&self) -> Result>; + fn list_namespaces(&self) -> Result>; /// Open a replica from this store. /// @@ -43,7 +52,7 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// List all author keys in this store. // TODO: return iterator - fn list_authors(&self) -> Result>; + fn list_authors(&self) -> Result>; /// Get an author key from the store. fn get_author(&self, author: &AuthorId) -> Result>; diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs index a3a9acfe5e..e279d421f5 100644 --- a/iroh-sync/src/store/fs.rs +++ b/iroh-sync/src/store/fs.rs @@ -131,6 +131,8 @@ impl<'s> Iterator for GetIter<'s> { impl super::Store for Store { type Instance = StoreInstance; type GetIter<'a> = GetIter<'a>; + type AuthorsIter<'a> = std::vec::IntoIter>; + type NamespaceIter<'a> = std::vec::IntoIter>; fn open_replica(&self, namespace_id: &NamespaceId) -> Result>> { if let Some(replica) = self.replicas.read().get(namespace_id) { @@ -148,15 +150,18 @@ impl super::Store for Store { Ok(Some(replica)) } - // TODO: return iterator - fn list_replicas(&self) -> Result> { + fn list_namespaces(&self) -> Result> { + // TODO: avoid collect let read_tx = self.db.begin_read()?; let namespace_table = read_tx.open_table(NAMESPACES_TABLE)?; - let namespaces = namespace_table + let namespaces: Vec<_> = namespace_table .iter()? - .filter_map(|entry| entry.ok()) - .map(|(_key, value)| Namespace::from_bytes(value.value()).id()); - Ok(namespaces.collect()) + .map(|res| match res { + Ok((_key, value)) => Ok(Namespace::from_bytes(value.value()).id()), + Err(err) => Err(err.into()), + }) + .collect(); + Ok(namespaces.into_iter()) } fn get_author(&self, author_id: &AuthorId) -> Result> { @@ -170,26 +175,25 @@ impl super::Store for Store { Ok(Some(author)) } - /// Generates a new author, using the passed in randomness. fn new_author(&self, rng: &mut R) -> Result { let author = Author::new(rng); self.insert_author(author.clone())?; Ok(author) } - /// Generates a new author, using the passed in randomness. - fn list_authors(&self) -> Result> { + fn list_authors(&self) -> Result> { + // TODO: avoid collect let read_tx = self.db.begin_read()?; - let author_table = read_tx.open_table(AUTHORS_TABLE)?; + let authors_table = read_tx.open_table(AUTHORS_TABLE)?; + let authors: Vec<_> = authors_table + .iter()? + .map(|res| match res { + Ok((_key, value)) => Ok(Author::from_bytes(value.value())), + Err(err) => Err(err.into()), + }) + .collect(); - let mut authors = vec![]; - let iter = author_table.iter()?; - for entry in iter { - let (_key, value) = entry?; - let author = Author::from_bytes(value.value()); - authors.push(author); - } - Ok(authors) + Ok(authors.into_iter()) } fn new_replica(&self, namespace: Namespace) -> Result> { diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs index ad93f055f8..618192ac5d 100644 --- a/iroh-sync/src/store/memory.rs +++ b/iroh-sync/src/store/memory.rs @@ -30,14 +30,24 @@ type ReplicaRecordsOwned = impl super::Store for Store { type Instance = ReplicaStoreInstance; type GetIter<'a> = GetIter<'a>; + type AuthorsIter<'a> = std::vec::IntoIter>; + type NamespaceIter<'a> = std::vec::IntoIter>; fn open_replica(&self, namespace: &NamespaceId) -> Result>> { let replicas = &*self.replicas.read(); Ok(replicas.get(namespace).cloned()) } - fn list_replicas(&self) -> Result> { - Ok(self.replicas.read().keys().cloned().collect()) + fn list_namespaces(&self) -> Result> { + // TODO: avoid collect? + Ok(self + .replicas + .read() + .keys() + .cloned() + .map(|n| Ok(n)) + .collect::>() + .into_iter()) } fn get_author(&self, author: &AuthorId) -> Result> { @@ -51,8 +61,16 @@ impl super::Store for Store { Ok(author) } - fn list_authors(&self) -> Result> { - Ok(self.authors.read().values().cloned().collect()) + fn list_authors(&self) -> Result> { + // TODO: avoid collect? + Ok(self + .authors + .read() + .values() + .cloned() + .map(|n| Ok(n)) + .collect::>() + .into_iter()) } fn new_replica(&self, namespace: Namespace) -> Result> { diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index cac2b8943b..60f7b6b6b2 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -2,8 +2,8 @@ use clap::Parser; use futures::TryStreamExt; use indicatif::HumanBytes; use iroh::{ - rpc_protocol::{DocTicket, ShareMode}, client::quic::Iroh, + rpc_protocol::{DocTicket, ShareMode}, sync::PeerSource, }; use iroh_sync::{ diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index 92278084eb..1a0c755707 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -17,6 +17,9 @@ use crate::rpc_protocol::{ use super::{engine::SyncEngine, PeerSource}; +/// Capacity for the flume channels to forward sync store iterators to async RPC streams. +const ITER_CHANNEL_CAP: usize = 64; + #[allow(missing_docs)] impl SyncEngine { pub fn author_create(&self, _req: AuthorCreateRequest) -> RpcResult { @@ -31,12 +34,21 @@ impl SyncEngine { &self, _req: AuthorListRequest, ) -> impl Stream> { - let ite = self.store.list_authors().map(|authors| authors.into_iter()); - let ite = inline_error(ite).map_ok(|author| AuthorListResponse { - author_id: author.id(), - writable: true, + let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); + let store = self.store.clone(); + self.rt.main().spawn_blocking(move || { + let ite = store.list_authors(); + let ite = inline_result(ite).map_ok(|author| AuthorListResponse { + author_id: author.id(), + writable: true, + }); + for entry in ite { + if let Err(_err) = tx.send(entry) { + break; + } + } }); - futures::stream::iter(ite) + rx.into_stream() } pub fn docs_create(&self, _req: DocsCreateRequest) -> RpcResult { @@ -50,9 +62,18 @@ impl SyncEngine { &self, _req: DocsListRequest, ) -> impl Stream> { - let ite = self.store.list_replicas().map(|res| res.into_iter()); - let ite = inline_error(ite).map_ok(|id| DocsListResponse { id }); - futures::stream::iter(ite) + let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); + let store = self.store.clone(); + self.rt.main().spawn_blocking(move || { + let ite = store.list_namespaces(); + let ite = inline_result(ite).map_ok(|id| DocsListResponse { id }); + for entry in ite { + if let Err(_err) = tx.send(entry) { + break; + } + } + }); + rx.into_stream() } pub async fn doc_share(&self, req: DocShareRequest) -> RpcResult { @@ -140,7 +161,7 @@ impl SyncEngine { pub fn doc_get(&self, req: DocGetRequest) -> impl Stream> { let DocGetRequest { doc_id, filter } = req; - let (tx, rx) = flume::bounded(16); + let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); let store = self.store.clone(); self.rt.main().spawn_blocking(move || { let ite = store.get(doc_id, filter); @@ -163,12 +184,3 @@ fn inline_result( Err(err) => itertools::Either::Right(Some(Err(err.into())).into_iter()), } } - -fn inline_error( - ite: Result, impl Into>, -) -> impl Iterator> { - match ite { - Ok(ite) => itertools::Either::Left(ite.map(|item| Ok(item))), - Err(err) => itertools::Either::Right(Some(Err(err.into())).into_iter()), - } -} From 036c16813083714263a94a8ff7f50fb52fafd052 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 02:27:00 +0200 Subject: [PATCH 102/172] fix: doc links --- iroh-sync/src/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index ec5c6fdf49..28d41ba3c8 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -23,12 +23,12 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { where Self: 'a; - /// Iterator over replicas in the store, returned from [`Self::list_replicas`] + /// Iterator over replica namespaces in the store, returned from [`Self::list_namespaces`] type NamespaceIter<'a>: Iterator> where Self: 'a; - /// Iterator over authors in the store, returned from [`Self::list_replicas`] + /// Iterator over authors in the store, returned from [`Self::list_authors`] type AuthorsIter<'a>: Iterator> where Self: 'a; From b990a721e4afb8d48527e3b30faf8a88d40d2533 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 02:37:30 +0200 Subject: [PATCH 103/172] chore: clippy --- iroh-sync/src/store/memory.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs index 618192ac5d..f1140b3df2 100644 --- a/iroh-sync/src/store/memory.rs +++ b/iroh-sync/src/store/memory.rs @@ -45,7 +45,7 @@ impl super::Store for Store { .read() .keys() .cloned() - .map(|n| Ok(n)) + .map(Ok) .collect::>() .into_iter()) } @@ -68,7 +68,7 @@ impl super::Store for Store { .read() .values() .cloned() - .map(|n| Ok(n)) + .map(Ok) .collect::>() .into_iter()) } From a3bd92e5eea5d2f34c396d18af40fb1d305ce2c2 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 17 Aug 2023 12:08:57 +0200 Subject: [PATCH 104/172] doc: fixes --- iroh-sync/src/store.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index 28d41ba3c8..dc2d2acd0c 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -36,7 +36,7 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// Create a new replica for `namespace` and persist in this store. fn new_replica(&self, namespace: Namespace) -> Result>; - /// List all replicas in this store. + /// List all replica namespaces in this store. fn list_namespaces(&self) -> Result>; /// Open a replica from this store. @@ -51,7 +51,6 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { fn new_author(&self, rng: &mut R) -> Result; /// List all author keys in this store. - // TODO: return iterator fn list_authors(&self) -> Result>; /// Get an author key from the store. @@ -59,8 +58,7 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// Iterate over entries of a replica. /// - /// Returns an iterator. The [`GetFilter`] has several methods of filtering the returned - /// entries. + /// The [`GetFilter`] has several methods of filtering the returne entries. fn get(&self, namespace: NamespaceId, filter: GetFilter) -> Result>; /// Gets the single latest entry for the specified key and author. From 5bf0e9e5fd5f36fb213019a259c33fba50a7f070 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 22 Aug 2023 12:40:53 +0200 Subject: [PATCH 105/172] fixes after merging main --- iroh-net/src/key.rs | 9 +++++++-- iroh/src/sync.rs | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/iroh-net/src/key.rs b/iroh-net/src/key.rs index bd6d1a8e04..d747d03e13 100644 --- a/iroh-net/src/key.rs +++ b/iroh-net/src/key.rs @@ -254,6 +254,12 @@ impl SecretKey { self.secret.to_bytes() } + /// Create a secret key from its byte representation. + pub fn from_bytes(bytes: &[u8; 32]) -> Self { + let secret = SigningKey::from_bytes(bytes); + secret.into() + } + fn secret_crypto_box(&self) -> &crypto_box::SecretKey { self.secret_crypto_box .get_or_init(|| secret_ed_box(&self.secret)) @@ -271,8 +277,7 @@ impl From for SecretKey { impl From<[u8; 32]> for SecretKey { fn from(value: [u8; 32]) -> Self { - let secret = SigningKey::from(value); - secret.into() + Self::from_bytes(&value) } } diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index 0896564d88..c70eb7aeff 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -212,7 +212,7 @@ async fn send_sync_message( #[cfg(test)] mod tests { - use iroh_net::tls::Keypair; + use iroh_net::key::SecretKey; use iroh_sync::{ store::{GetFilter, Store as _}, sync::Namespace, @@ -223,8 +223,8 @@ mod tests { #[tokio::test] async fn test_sync_simple() -> Result<()> { let mut rng = rand::thread_rng(); - let alice_peer_id = PublicKey::from(Keypair::from_bytes(&[1u8; 32]).public()); - let bob_peer_id = PublicKey::from(Keypair::from_bytes(&[2u8; 32]).public()); + let alice_peer_id = PublicKey::from(SecretKey::from_bytes(&[1u8; 32]).public()); + let bob_peer_id = PublicKey::from(SecretKey::from_bytes(&[2u8; 32]).public()); let alice_replica_store = store::memory::Store::default(); // For now uses same author on both sides. From 05035e3610c31854a7e4d5c9b2985725fdb681e0 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 22 Aug 2023 14:59:41 +0200 Subject: [PATCH 106/172] chore: clippy --- iroh/src/sync.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index c70eb7aeff..a36a579c9f 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -223,8 +223,8 @@ mod tests { #[tokio::test] async fn test_sync_simple() -> Result<()> { let mut rng = rand::thread_rng(); - let alice_peer_id = PublicKey::from(SecretKey::from_bytes(&[1u8; 32]).public()); - let bob_peer_id = PublicKey::from(SecretKey::from_bytes(&[2u8; 32]).public()); + let alice_peer_id = SecretKey::from_bytes(&[1u8; 32]).public(); + let bob_peer_id = SecretKey::from_bytes(&[2u8; 32]).public(); let alice_replica_store = store::memory::Store::default(); // For now uses same author on both sides. From e65582806467246200757db04d9faeecc9ffe6dd Mon Sep 17 00:00:00 2001 From: Kasey Date: Thu, 17 Aug 2023 23:37:05 -0400 Subject: [PATCH 107/172] more docs clean up --- iroh-sync/src/keys.rs | 30 +++++++++++++++--------------- iroh-sync/src/lib.rs | 4 ++-- iroh-sync/src/store.rs | 13 ++++++++++--- iroh-sync/src/store/fs.rs | 1 + iroh-sync/src/store/memory.rs | 2 ++ iroh-sync/src/sync.rs | 34 +++++++++++++++++----------------- 6 files changed, 47 insertions(+), 37 deletions(-) diff --git a/iroh-sync/src/keys.rs b/iroh-sync/src/keys.rs index 922f60736f..1acf56f4fe 100644 --- a/iroh-sync/src/keys.rs +++ b/iroh-sync/src/keys.rs @@ -14,23 +14,23 @@ pub struct Author { signing_key: SigningKey, } impl Author { - /// Create a new author with a random key. + /// Create a new [`Author`] with a random key. pub fn new(rng: &mut R) -> Self { let signing_key = SigningKey::generate(rng); Author { signing_key } } - /// Create an author from a byte array. + /// Create an [`Author`] from a byte array. pub fn from_bytes(bytes: &[u8; 32]) -> Self { SigningKey::from_bytes(bytes).into() } - /// Returns the Author byte representation. + /// Returns the [`Author`] byte representation. pub fn to_bytes(&self) -> [u8; 32] { self.signing_key.to_bytes() } - /// Returns the AuthorId byte representation. + /// Returns the [`AuthorId`] byte representation. pub fn id_bytes(&self) -> [u8; 32] { self.signing_key.verifying_key().to_bytes() } @@ -40,12 +40,12 @@ impl Author { AuthorId(self.signing_key.verifying_key()) } - /// Sign a message with this author key. + /// Sign a message with this [`Author`] key. pub fn sign(&self, msg: &[u8]) -> Signature { self.signing_key.sign(msg) } - /// Strictly verify a signature on a message with this author's public key. + /// Strictly verify a signature on a message with this [`Author`]'s public key. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.signing_key.verify_strict(msg, signature) } @@ -59,7 +59,7 @@ impl Author { pub struct AuthorId(VerifyingKey); impl AuthorId { - /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] + /// Verify that a signature matches the `msg` bytes and was created with the [`Author`] /// that corresponds to this [`AuthorId`]. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.0.verify_strict(msg, signature) @@ -83,26 +83,26 @@ impl AuthorId { /// Namespace key of a [`crate::Replica`]. /// /// Holders of this key can insert new entries into a [`crate::Replica`]. -/// Internally, a namespace is a [`SigningKey`] which is used to sign entries. +/// Internally, a [`Namespace`] is a [`SigningKey`] which is used to sign entries. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Namespace { signing_key: SigningKey, } impl Namespace { - /// Create a new namespace with a random key. + /// Create a new [`Namespace`] with a random key. pub fn new(rng: &mut R) -> Self { let signing_key = SigningKey::generate(rng); Namespace { signing_key } } - /// Create a namespace from a byte array. + /// Create a [`Namespace`] from a byte array. pub fn from_bytes(bytes: &[u8; 32]) -> Self { SigningKey::from_bytes(bytes).into() } - /// Returns the namespace byte representation. + /// Returns the [`Namespace`] byte representation. pub fn to_bytes(&self) -> [u8; 32] { self.signing_key.to_bytes() } @@ -117,12 +117,12 @@ impl Namespace { NamespaceId(self.signing_key.verifying_key()) } - /// Sign a message with this namespace key. + /// Sign a message with this [`Namespace`] key. pub fn sign(&self, msg: &[u8]) -> Signature { self.signing_key.sign(msg) } - /// Strictly verify a signature on a message with this namespaces's public key. + /// Strictly verify a signature on a message with this [`Namespace`]'s public key. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.signing_key.verify_strict(msg, signature) } @@ -130,13 +130,13 @@ impl Namespace { /// Identifier for a [`Namespace`] /// -/// This is the corresponding [`VerifyingKey`] for an author. It is used as an identifier, and can +/// This is the corresponding [`VerifyingKey`] for a [`Namespace`]. It is used as an identifier, and can /// be used to verify [`Signature`]s. #[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct NamespaceId(VerifyingKey); impl NamespaceId { - /// Verify that a signature matches the `msg` bytes and was created with this the [`Author`] + /// Verify that a signature matches the `msg` bytes and was created with the [`Namespace`] /// that corresponds to this [`NamespaceId`]. pub fn verify(&self, msg: &[u8], signature: &Signature) -> Result<(), SignatureError> { self.0.verify_strict(msg, signature) diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index 3e3803f667..adab62046b 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -10,14 +10,14 @@ //! //! * The [Namespace] key, as a token of write capability. The public key is the [NamespaceId], which //! also serves as the unique identifier for a replica. -//! * The [Author] key, as a proof of authorship. Any number of authors may be created, and an +//! * The [Author] key, as a proof of authorship. Any number of authors may be created, and //! their semantic meaning is application-specific. The public key of an author is the [AuthorId]. //! //! Replicas can be synchronized between peers by exchanging messages. The synchronization algorithm //! is based on a technique called *range-based set reconciliation*, based on [this paper][paper] by //! Aljoscha Meyer: //! -//! > Range-based set reconciliation is a simple approach to efficiently computing the union of two +//! > Range-based set reconciliation is a simple approach to efficiently compute the union of two //! sets over a network, based on recursively partitioning the sets and comparing fingerprints of //! the partitions to probabilistically detect whether a partition requires further work. //! diff --git a/iroh-sync/src/store.rs b/iroh-sync/src/store.rs index dc2d2acd0c..888a8574b0 100644 --- a/iroh-sync/src/store.rs +++ b/iroh-sync/src/store.rs @@ -47,7 +47,7 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { // TODO: Add close_replica fn open_replica(&self, namespace: &NamespaceId) -> Result>>; - /// Create a new author key key and persist it in the store. + /// Create a new author key and persist it in the store. fn new_author(&self, rng: &mut R) -> Result; /// List all author keys in this store. @@ -58,7 +58,7 @@ pub trait Store: std::fmt::Debug + Clone + Send + Sync + 'static { /// Iterate over entries of a replica. /// - /// The [`GetFilter`] has several methods of filtering the returne entries. + /// The [`GetFilter`] has several methods of filtering the returned entries. fn get(&self, namespace: NamespaceId, filter: GetFilter) -> Result>; /// Gets the single latest entry for the specified key and author. @@ -85,7 +85,10 @@ impl Default for GetFilter { } impl GetFilter { - /// Create a new get filter, either for only latest or all entries. + /// Create a new get filter. + /// + /// When `latest` is `true`, it will iterate over the latest entries, otherwise it will + /// iterate over all entires. pub fn new(latest: bool) -> Self { GetFilter { latest, @@ -93,6 +96,7 @@ impl GetFilter { key: KeyFilter::All, } } + /// No filter, iterate over all entries. pub fn all() -> Self { Self::new(false) @@ -114,16 +118,19 @@ impl GetFilter { self.key = KeyFilter::Key(key.as_ref().to_vec()); self } + /// Filter by prefix key match. pub fn with_prefix(mut self, prefix: impl AsRef<[u8]>) -> Self { self.key = KeyFilter::Prefix(prefix.as_ref().to_vec()); self } + /// Filter by author. pub fn with_author(mut self, author: AuthorId) -> Self { self.author = Some(author); self } + /// Include not only latest entries but also all historical entries. pub fn with_history(mut self) -> Self { self.latest = false; diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs index e279d421f5..2833c7dbb8 100644 --- a/iroh-sync/src/store/fs.rs +++ b/iroh-sync/src/store/fs.rs @@ -78,6 +78,7 @@ impl Store { replicas: Default::default(), }) } + /// Stores a new namespace fn insert_namespace(&self, namespace: Namespace) -> Result<()> { let write_tx = self.db.begin_write()?; diff --git a/iroh-sync/src/store/memory.rs b/iroh-sync/src/store/memory.rs index f1140b3df2..b909555151 100644 --- a/iroh-sync/src/store/memory.rs +++ b/iroh-sync/src/store/memory.rs @@ -264,6 +264,7 @@ impl Store { }) } } + #[derive(Debug)] enum GetFilter { /// All entries. @@ -526,6 +527,7 @@ impl crate::ranger::Store for ReplicaStoreInstanc } type RangeIterator<'a> = RangeIterator<'a>; + fn get_range( &self, range: Range, diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 0077d15410..29c21822b0 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -23,12 +23,12 @@ use crate::ranger::{self, AsFingerprint, Fingerprint, Peer, RangeKey}; pub use crate::keys::*; -/// Protocol message for the set reconciliation protocol +/// Protocol message for the set reconciliation protocol. /// /// Can be serialized to bytes with [serde] to transfer between peers. pub type ProtocolMessage = crate::ranger::Message; -/// Byte represenation of a `PeerId` from `iroh-net` +/// Byte represenation of a `PeerId` from `iroh-net`. // TODO: PeerId is in iroh-net which iroh-sync doesn't depend on. Add iroh-common crate with `PeerId`. pub type PeerIdBytes = [u8; 32]; @@ -222,7 +222,7 @@ impl SignedEntry { SignedEntry { signature, entry } } - /// Create a new signed entry by signing an entry with a namespace and author. + /// Create a new signed entry by signing an entry with the `namespace` and `author`. pub fn from_entry(entry: Entry, namespace: &Namespace, author: &Author) -> Self { let signature = EntrySignature::from_entry(&entry, namespace, author); SignedEntry { signature, entry } @@ -239,12 +239,12 @@ impl SignedEntry { &self.signature } - /// Get the entry. + /// Get the [`Entry`]. pub fn entry(&self) -> &Entry { &self.entry } - /// Get the content hash of the entry. + /// Get the content [`Hash`] of the entry. pub fn content_hash(&self) -> &Hash { self.entry().record().content_hash() } @@ -254,7 +254,7 @@ impl SignedEntry { self.entry().record().content_len() } - /// Get the author of the entry. + /// Get the [`AuthorId`] of the entry. pub fn author(&self) -> AuthorId { self.entry().id().author() } @@ -273,7 +273,7 @@ pub struct EntrySignature { } impl EntrySignature { - /// Create a new signature by signing an entry with a namespace and author. + /// Create a new signature by signing an entry with the `namespace` and `author`. pub fn from_entry(entry: &Entry, namespace: &Namespace, author: &Author) -> Self { // TODO: this should probably include a namespace prefix // namespace in the cryptographic sense. @@ -321,10 +321,10 @@ impl EntrySignature { } } -/// A single entry in a replica. +/// A single entry in a [`Replica`] /// -/// An entry is identified by a key, its author, and the replica's -/// namespace. Its value is the [32-byte BLAKE3 hash](iroh_bytes::Hash) +/// An entry is identified by a key, its [`Author`], and the [`Replica`]'s +/// [`Namespace`]. Its value is the [32-byte BLAKE3 hash](iroh_bytes::Hash) /// of the entry's content data, the size of this content data, and a timestamp. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Entry { @@ -377,9 +377,9 @@ impl Entry { pub struct RecordIdentifier { /// The key of the record. key: Vec, - /// The namespace this record belongs to. + /// The [`NamespaceId`] of the namespace this record belongs to. namespace: NamespaceId, - /// The author that wrote this record. + /// The [`AuthorId`] of the author that wrote this record. author: AuthorId, } @@ -408,7 +408,7 @@ impl RangeKey for RecordIdentifier { } impl RecordIdentifier { - /// Create a new record identifier. + /// Create a new [`RecordIdentifier`]. pub fn new(key: impl AsRef<[u8]>, namespace: NamespaceId, author: AuthorId) -> Self { RecordIdentifier { key: key.as_ref().to_vec(), @@ -429,7 +429,7 @@ impl RecordIdentifier { }) } - /// Serialize this record identifier into a mutable byte array. + /// Serialize this [`RecordIdentifier`] into a mutable byte array. pub(crate) fn as_bytes(&self, out: &mut Vec) { out.extend_from_slice(self.namespace.as_bytes()); out.extend_from_slice(self.author.as_bytes()); @@ -441,7 +441,7 @@ impl RecordIdentifier { &self.key } - /// Get the namespace of this record. + /// Get the [`NamespaceId`] of this record. pub fn namespace(&self) -> NamespaceId { self.namespace } @@ -450,7 +450,7 @@ impl RecordIdentifier { self.namespace.as_bytes() } - /// Get the author of this record. + /// Get the [`AuthorId`] of this record. pub fn author(&self) -> AuthorId { self.author } @@ -491,7 +491,7 @@ impl Record { self.len } - /// Get the hash of the content data of this record. + /// Get the [`Hash`] of the content data of this record. pub fn content_hash(&self) -> &Hash { &self.hash } From 4c16c41d1074a4d1422c02dae465b88ca8a93377 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 14 Aug 2023 12:16:00 +0200 Subject: [PATCH 108/172] feat: iroh REPL console --- Cargo.lock | 10 + iroh/Cargo.toml | 13 +- iroh/src/commands.rs | 434 +++++++++++++++++----------------- iroh/src/commands/add.rs | 7 +- iroh/src/commands/list.rs | 36 +-- iroh/src/commands/repl.rs | 255 ++++++++++++++++++++ iroh/src/commands/sync.rs | 85 ++++--- iroh/src/commands/validate.rs | 7 +- iroh/src/config.rs | 5 + 9 files changed, 566 insertions(+), 286 deletions(-) create mode 100644 iroh/src/commands/repl.rs diff --git a/Cargo.lock b/Cargo.lock index d073176c51..551118b09b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anstream" version = "0.3.2" @@ -1858,6 +1867,7 @@ checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" name = "iroh" version = "0.5.1" dependencies = [ + "ansi_term", "anyhow", "bao-tree", "bytes", diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index ed35d5cccf..16c228bcfe 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -25,6 +25,7 @@ iroh-bytes = { version = "0.5.0", path = "../iroh-bytes" } iroh-io = { version = "0.2.2" } iroh-metrics = { version = "0.5.0", path = "../iroh-metrics", optional = true } iroh-net = { version = "0.5.1", path = "../iroh-net" } +itertools = "0.11.0" num_cpus = { version = "1.15.0" } portable-atomic = "1" iroh-sync = { version = "0.5.1", path = "../iroh-sync" } @@ -45,12 +46,16 @@ tracing = "0.1" walkdir = "2" # CLI +ansi_term = { version = "0.12.1", optional = true } clap = { version = "4", features = ["derive"], optional = true } config = { version = "0.13.1", default-features = false, features = ["toml", "preserve_order"], optional = true } console = { version = "0.15.5", optional = true } dirs-next = { version = "2.0.0", optional = true } indicatif = { version = "0.17", features = ["tokio"], optional = true } multibase = { version = "0.9.1", optional = true } +rustyline = { version = "12.0.0", optional = true } +shell-words = { version = "1.1.0", optional = true } +shellexpand = { version = "3.1.0", optional = true } tempfile = { version = "3.4", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } data-encoding = "2.4.0" @@ -58,20 +63,16 @@ url = { version = "2.4", features = ["serde"] } # Examples ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"], optional = true } -shell-words = { version = "1.1.0", optional = true } -shellexpand = { version = "3.1.0", optional = true } -rustyline = { version = "12.0.0", optional = true } -itertools = "0.11.0" [features] default = ["cli", "metrics"] -cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection"] +cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection", "shell-words", "shellexpand", "rustyline", "ansi_term"] metrics = ["iroh-metrics", "flat-db", "mem-db", "iroh-collection"] mem-db = [] flat-db = [] iroh-collection = [] test = [] -example-sync = ["cli", "ed25519-dalek", "shell-words", "shellexpand", "rustyline"] +example-sync = ["cli", "ed25519-dalek"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 57473b8e64..9d5f07a0bf 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -23,6 +23,7 @@ pub mod doctor; pub mod get; pub mod list; pub mod provide; +pub mod repl; pub mod sync; pub mod validate; @@ -42,7 +43,7 @@ pub mod validate; #[clap(version)] pub struct Cli { #[clap(subcommand)] - pub command: Commands, + pub command: Option, /// Log SSL pre-master key to file in SSLKEYLOGFILE environment variable. #[clap(long)] pub keylog: bool, @@ -52,69 +53,140 @@ pub struct Cli { pub metrics_addr: Option, #[clap(long)] pub cfg: Option, + /// RPC port of the Iroh node + #[clap(long, default_value_t = DEFAULT_RPC_PORT)] + rpc_port: u16, } impl Cli { pub async fn run(self, rt: &runtime::Handle, config: &Config) -> Result<()> { match self.command { - Commands::Share { - hash, - recursive, - rpc_port, - peer, + None => { + let client = iroh::client::quic::connect_raw(self.rpc_port).await?; + repl::run(client).await + } + Some(Commands::Rpc(command)) => { + let client = iroh::client::quic::connect_raw(self.rpc_port).await?; + command.run(client).await + } + Some(Commands::Full(command)) => command.run(rt, config, self.keylog).await, + } + } +} + +#[derive(Parser, Debug, Clone)] +pub enum Commands { + #[clap(flatten)] + Full(#[clap(subcommand)] FullCommands), + #[clap(flatten)] + Rpc(#[clap(subcommands)] RpcCommand), +} + +#[derive(Subcommand, Debug, Clone)] +pub enum FullCommands { + /// Serve data from the given path. + /// + /// If PATH is a folder all files in that folder will be served. If no PATH is + /// specified reads from STDIN. + Provide { + /// Path to initial file or directory to provide + path: Option, + /// Serve data in place + /// + /// Set this to true only if you are sure that the data in its current location + /// will not change. + #[clap(long, default_value_t = false)] + in_place: bool, + #[clap(long, short)] + /// Listening address to bind to + #[clap(long, short, default_value_t = SocketAddr::from(iroh::node::DEFAULT_BIND_ADDR))] + addr: SocketAddr, + /// RPC port, set to "disabled" to disable RPC + #[clap(long, default_value_t = ProviderRpcPort::Enabled(DEFAULT_RPC_PORT))] + rpc_port: ProviderRpcPort, + /// Use a token to authenticate requests for data + /// + /// Pass "random" to generate a random token, or base32-encoded bytes to use as a token + #[clap(long)] + request_token: Option, + }, + /// Fetch the data identified by HASH from a provider + Get { + /// The hash to retrieve, as a Blake3 CID + #[clap(conflicts_with = "ticket", required_unless_present = "ticket")] + hash: Option, + /// PublicKey of the provider + #[clap( + long, + short, + conflicts_with = "ticket", + required_unless_present = "ticket" + )] + peer: Option, + /// Addresses of the provider + #[clap(long, short)] + addrs: Vec, + /// base32-encoded Request token to use for authentication, if any + #[clap(long)] + token: Option, + /// DERP region of the provider + #[clap(long)] + region: Option, + /// Directory in which to save the file(s), defaults to writing to STDOUT + /// + /// If the directory exists and contains a partial download, the download will + /// be resumed. + /// + /// Otherwise, all files in the collection will be overwritten. Other files + /// in the directory will be left untouched. + #[clap(long, short)] + out: Option, + #[clap(conflicts_with_all = &["hash", "peer", "addrs", "token"])] + /// Ticket containing everything to retrieve the data from a provider. + #[clap(long)] + ticket: Option, + /// True to download a single blob, false (default) to download a collection and its children. + #[clap(long, default_value_t = false)] + single: bool, + }, + /// Diagnostic commands for the derp relay protocol. + Doctor { + /// Commands for doctor - defined in the mod + #[clap(subcommand)] + command: self::doctor::Commands, + }, +} + +impl FullCommands { + pub async fn run(self, rt: &runtime::Handle, config: &Config, keylog: bool) -> Result<()> { + match self { + FullCommands::Provide { + path, + in_place, addr, - token, - ticket, - derp_region, - mut out, - stable: in_place, + rpc_port, + request_token, } => { - if let Some(out) = out.as_mut() { - tracing::info!("canonicalizing output path"); - let absolute = std::env::current_dir()?.join(&out); - tracing::info!("output path is {} -> {}", out.display(), absolute.display()); - *out = absolute; - } - let client = make_rpc_client(rpc_port).await?; - let (peer, addr, token, derp_region, hash, recursive) = - if let Some(ticket) = ticket.as_ref() { - ( - ticket.peer(), - ticket.addrs().to_vec(), - ticket.token(), - ticket.derp_region(), - ticket.hash(), - ticket.recursive(), - ) - } else { - ( - peer.unwrap(), - addr, - token.as_ref(), - derp_region, - hash.unwrap(), - recursive.unwrap_or_default(), - ) - }; - let mut stream = client - .server_streaming(ShareRequest { - hash, - recursive, - peer, - addrs: addr, - derp_region, - token: token.cloned(), - out: out.map(|x| x.display().to_string()), - in_place, - }) - .await?; - while let Some(item) = stream.next().await { - let item = item?; - println!("{:?}", item); - } - Ok(()) + let request_token = match request_token { + Some(RequestTokenOptions::Random) => Some(RequestToken::generate()), + Some(RequestTokenOptions::Token(token)) => Some(token), + None => None, + }; + self::provide::run( + rt, + path, + in_place, + ProvideOptions { + addr, + rpc_port, + keylog, + request_token, + derp_map: config.derp_map(), + }, + ) + .await } - Commands::Get { + FullCommands::Get { hash, peer, addrs, @@ -139,7 +211,7 @@ impl Cli { opts: iroh::dial::Options { addrs, peer_id: peer, - keylog: self.keylog, + keylog, derp_region: region, derp_map: config.derp_map(), secret_key: SecretKey::generate(), @@ -159,110 +231,19 @@ impl Cli { } } } - Commands::Provide { - path, - addr, - rpc_port, - request_token, - in_place, - } => { - let request_token = match request_token { - Some(RequestTokenOptions::Random) => Some(RequestToken::generate()), - Some(RequestTokenOptions::Token(token)) => Some(token), - None => None, - }; - self::provide::run( - rt, - path, - in_place, - ProvideOptions { - addr, - rpc_port, - keylog: self.keylog, - request_token, - derp_map: config.derp_map(), - }, - ) - .await - } - Commands::List(cmd) => cmd.run().await, - Commands::Validate { rpc_port, repair } => self::validate::run(rpc_port, repair).await, - Commands::Shutdown { force, rpc_port } => { - let client = make_rpc_client(rpc_port).await?; - client.rpc(ShutdownRequest { force }).await?; - Ok(()) - } - Commands::Id { rpc_port } => { - let client = make_rpc_client(rpc_port).await?; - let response = client.rpc(IdRequest).await?; - - println!("Listening address: {:#?}", response.listen_addrs); - println!("PeerID: {}", response.peer_id); - Ok(()) - } - Commands::Add { - path, - rpc_port, - in_place, - } => self::add::run(path, in_place, rpc_port).await, - Commands::Addresses { rpc_port } => { - let client = make_rpc_client(rpc_port).await?; - let response = client.rpc(AddrsRequest).await?; - println!("Listening addresses: {:?}", response.addrs); - Ok(()) - } - Commands::Doctor { command } => self::doctor::run(command, config).await, - Commands::Sync { command, rpc_port } => { - let client = make_rpc_client(rpc_port).await?; - command.run(client).await - } + FullCommands::Doctor { command } => self::doctor::run(command, config).await, } } } + #[derive(Subcommand, Debug, Clone)] #[allow(clippy::large_enum_variant)] -pub enum Commands { - /// Diagnostic commands for the derp relay protocol. - Doctor { - /// Commands for doctor - defined in the mod - #[clap(subcommand)] - command: self::doctor::Commands, - }, - - /// Serve data from the given path. - /// - /// If PATH is a folder all files in that folder will be served. If no PATH is - /// specified reads from STDIN. - Provide { - /// Path to initial file or directory to provide - path: Option, - /// Serve data in place - /// - /// Set this to true only if you are sure that the data in its current location - /// will not change. - #[clap(long, default_value_t = false)] - in_place: bool, - #[clap(long, short)] - /// Listening address to bind to - #[clap(long, short, default_value_t = SocketAddr::from(iroh::node::DEFAULT_BIND_ADDR))] - addr: SocketAddr, - /// RPC port, set to "disabled" to disable RPC - #[clap(long, default_value_t = ProviderRpcPort::Enabled(DEFAULT_RPC_PORT))] - rpc_port: ProviderRpcPort, - /// Use a token to authenticate requests for data - /// - /// Pass "random" to generate a random token, or base32-encoded bytes to use as a token - #[clap(long)] - request_token: Option, - }, +pub enum RpcCommand { /// List availble content on the provider. #[clap(subcommand)] List(self::list::Commands), /// Validate hashes on the running provider. Validate { - /// RPC port of the provider - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, /// Repair the store by removing invalid data #[clap(long, default_value_t = false)] repair: bool, @@ -275,16 +256,9 @@ pub enum Commands { /// for all connections to close. #[clap(long, default_value_t = false)] force: bool, - /// RPC port of the provider - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, }, /// Identify the running provider. - Id { - /// RPC port of the provider - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, - }, + Id, /// Add data from PATH to the running provider's database. Add { /// The path to the file or folder to add @@ -295,48 +269,6 @@ pub enum Commands { /// will not change. #[clap(long, default_value_t = false)] in_place: bool, - /// RPC port - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, - }, - /// Fetch the data identified by HASH from a provider - Get { - /// The hash to retrieve, as a Blake3 CID - #[clap(conflicts_with = "ticket", required_unless_present = "ticket")] - hash: Option, - /// PublicKey of the provider - #[clap( - long, - short, - conflicts_with = "ticket", - required_unless_present = "ticket" - )] - peer: Option, - /// Addresses of the provider - #[clap(long, short)] - addrs: Vec, - /// base32-encoded Request token to use for authentication, if any - #[clap(long)] - token: Option, - /// DERP region of the provider - #[clap(long)] - region: Option, - /// Directory in which to save the file(s), defaults to writing to STDOUT - /// - /// If the directory exists and contains a partial download, the download will - /// be resumed. - /// - /// Otherwise, all files in the collection will be overwritten. Other files - /// in the directory will be left untouched. - #[clap(long, short)] - out: Option, - #[clap(conflicts_with_all = &["hash", "peer", "addrs", "token"])] - /// Ticket containing everything to retrieve the data from a provider. - #[clap(long)] - ticket: Option, - /// True to download a single blob, false (default) to download a collection and its children. - #[clap(long, default_value_t = false)] - single: bool, }, /// Download data to the running provider's database and provide it. /// @@ -380,27 +312,93 @@ pub enum Commands { /// and iroh will assume that it will not change. #[clap(long, default_value_t = false)] stable: bool, - /// RPC port - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, }, /// List listening addresses of the provider. - Addresses { - /// RPC port - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, - }, - Sync { - /// RPC port - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, - #[clap(subcommand)] - command: sync::Commands, - }, + Addresses, + #[clap(flatten)] + Sync(#[clap(subcommand)] sync::Commands), } -pub async fn make_rpc_client(rpc_port: u16) -> anyhow::Result { - iroh::client::quic::connect_raw(rpc_port).await +impl RpcCommand { + pub async fn run(self, client: RpcClient) -> Result<()> { + match self { + RpcCommand::Share { + hash, + recursive, + peer, + addr, + token, + ticket, + derp_region, + mut out, + stable: in_place, + } => { + if let Some(out) = out.as_mut() { + tracing::info!("canonicalizing output path"); + let absolute = std::env::current_dir()?.join(&out); + tracing::info!("output path is {} -> {}", out.display(), absolute.display()); + *out = absolute; + } + let (peer, addr, token, derp_region, hash, recursive) = + if let Some(ticket) = ticket.as_ref() { + ( + ticket.peer(), + ticket.addrs().to_vec(), + ticket.token(), + ticket.derp_region(), + ticket.hash(), + ticket.recursive(), + ) + } else { + ( + peer.unwrap(), + addr, + token.as_ref(), + derp_region, + hash.unwrap(), + recursive.unwrap_or_default(), + ) + }; + let mut stream = client + .server_streaming(ShareRequest { + hash, + recursive, + peer, + addrs: addr, + derp_region, + token: token.cloned(), + out: out.map(|x| x.display().to_string()), + in_place, + }) + .await?; + while let Some(item) = stream.next().await { + let item = item?; + println!("{:?}", item); + } + Ok(()) + } + RpcCommand::List(cmd) => cmd.run(client).await, + RpcCommand::Validate { repair } => self::validate::run(client, repair).await, + RpcCommand::Shutdown { force } => { + client.rpc(ShutdownRequest { force }).await?; + Ok(()) + } + RpcCommand::Id {} => { + let response = client.rpc(IdRequest).await?; + + println!("Listening address: {:#?}", response.listen_addrs); + println!("PeerID: {}", response.peer_id); + Ok(()) + } + RpcCommand::Add { path, in_place } => self::add::run(client, path, in_place).await, + RpcCommand::Addresses {} => { + let response = client.rpc(AddrsRequest).await?; + println!("Listening addresses: {:?}", response.addrs); + Ok(()) + } + RpcCommand::Sync(command) => command.run(client).await, + } + } } #[cfg(feature = "metrics")] diff --git a/iroh/src/commands/add.rs b/iroh/src/commands/add.rs index 1e2b94086c..1269400e20 100644 --- a/iroh/src/commands/add.rs +++ b/iroh/src/commands/add.rs @@ -7,13 +7,10 @@ use std::{ use anyhow::{Context, Result}; use futures::{Stream, StreamExt}; use indicatif::{HumanBytes, MultiProgress, ProgressBar, ProgressStyle}; -use iroh::rpc_protocol::ProvideRequest; +use iroh::{client::quic::RpcClient, rpc_protocol::ProvideRequest}; use iroh_bytes::{provider::ProvideProgress, Hash}; -use crate::commands::make_rpc_client; - -pub async fn run(path: PathBuf, in_place: bool, rpc_port: u16) -> Result<()> { - let client = make_rpc_client(rpc_port).await?; +pub async fn run(client: RpcClient, path: PathBuf, in_place: bool) -> Result<()> { let absolute = path.canonicalize()?; println!("Adding {} as {}...", path.display(), absolute.display()); let stream = client diff --git a/iroh/src/commands/list.rs b/iroh/src/commands/list.rs index 68de5dd7a8..ef69955492 100644 --- a/iroh/src/commands/list.rs +++ b/iroh/src/commands/list.rs @@ -2,53 +2,39 @@ use anyhow::Result; use clap::Subcommand; use futures::StreamExt; use indicatif::HumanBytes; -use iroh::rpc_protocol::{ListBlobsRequest, ListCollectionsRequest, ListIncompleteBlobsRequest}; - -use super::{make_rpc_client, DEFAULT_RPC_PORT}; +use iroh::{ + client::quic::RpcClient, + rpc_protocol::{ListBlobsRequest, ListCollectionsRequest, ListIncompleteBlobsRequest}, +}; #[derive(Subcommand, Debug, Clone)] pub enum Commands { /// List the available blobs on the running provider. - Blobs { - /// RPC port of the provider - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, - }, + Blobs, /// List the available blobs on the running provider. - IncompleteBlobs { - /// RPC port of the provider - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, - }, + IncompleteBlobs, /// List the available collections on the running provider. - Collections { - /// RPC port of the provider - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, - }, + Collections, } impl Commands { - pub async fn run(self) -> Result<()> { + pub async fn run(self, client: RpcClient) -> Result<()> { match self { - Commands::Blobs { rpc_port } => { - let client = make_rpc_client(rpc_port).await?; + Commands::Blobs => { let mut response = client.server_streaming(ListBlobsRequest).await?; while let Some(item) = response.next().await { let item = item?; println!("{} {} ({})", item.path, item.hash, HumanBytes(item.size),); } } - Commands::IncompleteBlobs { rpc_port } => { - let client = make_rpc_client(rpc_port).await?; + Commands::IncompleteBlobs => { let mut response = client.server_streaming(ListIncompleteBlobsRequest).await?; while let Some(item) = response.next().await { let item = item?; println!("{} {}", item.hash, item.size); } } - Commands::Collections { rpc_port } => { - let client = make_rpc_client(rpc_port).await?; + Commands::Collections => { let mut response = client.server_streaming(ListCollectionsRequest).await?; while let Some(collection) = response.next().await { let collection = collection?; diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs new file mode 100644 index 0000000000..223939ea04 --- /dev/null +++ b/iroh/src/commands/repl.rs @@ -0,0 +1,255 @@ +use ansi_term::Colour; +use clap::{Parser, Subcommand}; +use iroh::client::quic::{Iroh, RpcClient}; +use iroh_gossip::proto::util::base32; +use iroh_sync::sync::{AuthorId, NamespaceId}; +use rustyline::{error::ReadlineError, Config, DefaultEditor}; +use tokio::sync::{mpsc, oneshot}; + +use super::sync::DocEnv; + +pub async fn run(client: RpcClient) -> anyhow::Result<()> { + println!("Welcome to the Iroh console!"); + println!("Type `help` for a list of commands."); + let mut repl_rx = Repl::spawn(); + while let Some((event, reply)) = repl_rx.recv().await { + let (next, res) = match event { + FromRepl::DocCmd { id, cmd, doc_env } => { + let iroh = Iroh::new(client.clone()); + let res = cmd.run(&iroh, id, doc_env).await; + (ToRepl::Continue, res) + } + FromRepl::RpcCmd { cmd } => { + let res = cmd.run(client.clone()).await; + (ToRepl::Continue, res) + } + FromRepl::Exit => (ToRepl::Exit, Ok(())), + }; + + if let Err(err) = res { + println!( + "{} {:?}", + ansi_term::Colour::Red.bold().paint("Error:"), + err + ) + } + + reply.send(next).ok(); + } + Ok(()) +} + +#[derive(Debug)] +pub enum FromRepl { + RpcCmd { + cmd: super::RpcCommand, + }, + DocCmd { + id: NamespaceId, + doc_env: DocEnv, + cmd: super::sync::Doc, + }, + Exit, +} + +#[derive(Debug)] +pub enum ToRepl { + Continue, + Exit, +} + +pub struct Repl { + state: ReplState, + cmd_tx: mpsc::Sender<(FromRepl, oneshot::Sender)>, +} +impl Repl { + pub fn spawn() -> mpsc::Receiver<(FromRepl, oneshot::Sender)> { + let (cmd_tx, cmd_rx) = mpsc::channel(1); + let repl = Repl { + state: ReplState::from_env(), + cmd_tx, + }; + std::thread::spawn(move || { + if let Err(err) = repl.run() { + println!("> repl crashed: {err}"); + } + }); + cmd_rx + } + pub fn run(mut self) -> anyhow::Result<()> { + let mut rl = + DefaultEditor::with_config(Config::builder().check_cursor_position(true).build())?; + loop { + // prepare a channel to receive a signal from the main thread when a command completed + let (to_repl_tx, to_repl_rx) = oneshot::channel(); + let readline = rl.readline(&self.state.prompt()); + match readline { + Ok(line) if line.is_empty() => continue, + Ok(line) => { + rl.add_history_entry(line.as_str())?; + let cmd = self.state.parse_command(&line); + if let Some(cmd) = cmd { + self.cmd_tx.blocking_send((cmd, to_repl_tx))?; + } else { + continue; + } + } + Err(ReadlineError::Interrupted | ReadlineError::Eof) => { + self.cmd_tx.blocking_send((FromRepl::Exit, to_repl_tx))?; + } + Err(ReadlineError::WindowResized) => continue, + Err(err) => return Err(err.into()), + } + // wait for reply from main thread + match to_repl_rx.blocking_recv()? { + ToRepl::Continue => continue, + ToRepl::Exit => break, + } + } + Ok(()) + } +} + +pub struct ReplState { + pwd: Pwd, + doc_env: DocEnv, +} + +impl ReplState { + fn from_env() -> Self { + Self { + pwd: Pwd::Home, + doc_env: DocEnv::from_env().unwrap_or_default(), + } + } +} + +#[derive(Debug)] +enum Pwd { + Home, + Doc { id: NamespaceId }, +} + +impl ReplState { + pub fn prompt(&self) -> String { + let bang = Colour::Blue.paint("> "); + let pwd = match self.pwd { + Pwd::Home => None, + Pwd::Doc { id } => { + let author = self + .doc_env + .author + .map(|author| format!(" author:{}", fmt_short(author.as_bytes()))) + .map(|author| Colour::Red.paint(author).to_string()) + .unwrap_or_default(); + let pwd = format!("doc:{}{author}", fmt_short(id.as_bytes())); + let pwd = Colour::Blue.paint(pwd); + Some(pwd.to_string()) + } + }; + let pwd = pwd.map(|pwd| format!("{}\n", pwd)).unwrap_or_default(); + format!("\n{pwd}{bang}") + } + + pub fn parse_command(&mut self, line: &str) -> Option { + match self.pwd { + Pwd::Home => match parse_cmd::(line)? { + HomeCommands::Repl(cmd) => self.process_repl_command(cmd), + HomeCommands::Rpc(cmd) => Some(FromRepl::RpcCmd { cmd }), + }, + Pwd::Doc { id } => match parse_cmd::(line)? { + DocCommands::Repl(cmd) => self.process_repl_command(cmd), + DocCommands::Sync(cmd) => Some(FromRepl::RpcCmd { + cmd: super::RpcCommand::Sync(cmd), + }), + DocCommands::Doc(cmd) => Some(FromRepl::DocCmd { + id, + cmd, + doc_env: self.doc_env.clone(), + }), + }, + } + } + + fn process_repl_command(&mut self, command: ReplCommand) -> Option { + match command { + ReplCommand::SetDoc { id } => { + self.pwd = Pwd::Doc { id }; + None + } + ReplCommand::SetAuthor { id } => { + self.doc_env.author = Some(id); + None + } + ReplCommand::Close => { + self.pwd = Pwd::Home; + None + } + ReplCommand::Exit => Some(FromRepl::Exit), + } + } +} + +#[derive(Debug, Parser)] +pub enum ReplCommand { + /// Open a document + #[clap(next_help_heading = "foo")] + SetDoc { id: NamespaceId }, + /// Set the active author for doc insertion + SetAuthor { id: AuthorId }, + /// Close the open document + Close, + + /// Quit the Iroh console + #[clap(alias = "quit")] + Exit, +} + +#[derive(Debug, Parser)] +pub enum DocCommands { + #[clap(flatten)] + Doc(#[clap(subcommand)] super::sync::Doc), + #[clap(flatten)] + Repl(#[clap(subcommand)] ReplCommand), + // TODO: We can't embed RpcCommand here atm because there'd be a conflict between + // `list` top level and `list` doc command + // Thus for now only embedding sync commands + #[clap(flatten)] + Sync(#[clap(subcommand)] super::sync::Commands), +} + +#[derive(Debug, Parser)] +pub enum HomeCommands { + #[clap(flatten)] + Repl(#[clap(subcommand)] ReplCommand), + #[clap(flatten)] + Rpc(#[clap(subcommand)] super::RpcCommand), +} + +fn try_parse_cmd(s: &str) -> anyhow::Result { + let args = shell_words::split(s)?; + let cmd = clap::Command::new("repl"); + let cmd = C::augment_subcommands(cmd); + let matches = cmd + .multicall(true) + .subcommand_required(true) + .try_get_matches_from(args)?; + let cmd = C::from_arg_matches(&matches)?; + Ok(cmd) +} + +fn parse_cmd(s: &str) -> Option { + match try_parse_cmd::(s) { + Ok(cmd) => Some(cmd), + Err(err) => { + println!("{err}"); + None + } + } +} + +fn fmt_short(bytes: impl AsRef<[u8]>) -> String { + let bytes = bytes.as_ref(); + let len = bytes.len().min(5); + base32::fmt(&bytes[..len]) +} diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 60f7b6b6b2..974536163a 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -1,3 +1,6 @@ +use std::str::FromStr; + +use anyhow::{anyhow, Result}; use clap::Parser; use futures::TryStreamExt; use indicatif::HumanBytes; @@ -11,6 +14,8 @@ use iroh_sync::{ sync::{AuthorId, NamespaceId, SignedEntry}, }; +use crate::config::env_var; + use super::RpcClient; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; @@ -18,14 +23,17 @@ const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Parser)] pub enum Commands { + /// Manage document authors Author { #[clap(subcommand)] command: Author, }, + /// Manage documents Docs { #[clap(subcommand)] command: Docs, }, + /// Manage a single document Doc { id: NamespaceId, #[clap(subcommand)] @@ -34,24 +42,30 @@ pub enum Commands { } impl Commands { - pub async fn run(self, client: RpcClient) -> anyhow::Result<()> { + pub async fn run(self, client: RpcClient) -> Result<()> { let iroh = Iroh::new(client); match self { - Commands::Author { command } => command.run(iroh).await, - Commands::Docs { command } => command.run(iroh).await, - Commands::Doc { command, id } => command.run(iroh, id).await, + Commands::Author { command } => command.run(&iroh).await, + Commands::Docs { command } => command.run(&iroh).await, + Commands::Doc { command, id } => { + let doc_env = DocEnv::from_env()?; + command.run(&iroh, id, doc_env).await + } } } } #[derive(Debug, Clone, Parser)] pub enum Author { + /// List authors + #[clap(alias = "ls")] List, + /// Create a new author Create, } impl Author { - pub async fn run(self, iroh: Iroh) -> anyhow::Result<()> { + pub async fn run(self, iroh: &Iroh) -> Result<()> { match self { Author::List => { let mut stream = iroh.list_authors().await?; @@ -70,31 +84,22 @@ impl Author { #[derive(Debug, Clone, Parser)] pub enum Docs { + /// List documents + #[clap(alias = "ls")] List, + /// Create a new document Create, - // Import { - // key: String, - // #[clap(short, long)] - // peers: Vec, - // }, + /// Import a document from peers Import { ticket: DocTicket }, } impl Docs { - pub async fn run(self, iroh: Iroh) -> anyhow::Result<()> { + pub async fn run(self, iroh: &Iroh) -> Result<()> { match self { Docs::Create => { let doc = iroh.create_doc().await?; println!("created {}", doc.id()); } - // Docs::Import { key, peers } => { - // let key = hex::decode(key)? - // .try_into() - // .map_err(|_| anyhow!("invalid length"))?; - // let ticket = DocTicket::new(key, peers); - // let doc = iroh.import_doc(ticket).await?; - // println!("imported {}", doc.id()); - // } Docs::Import { ticket } => { let doc = iroh.import_doc(ticket).await?; println!("imported {}", doc.id()); @@ -110,18 +115,41 @@ impl Docs { } } +#[derive(Debug, Clone, Default)] +pub struct DocEnv { + pub author: Option, +} + +impl DocEnv { + pub fn from_env() -> anyhow::Result { + let author = if let Some(author) = env_var("AUTHOR").ok() { + Some(AuthorId::from_str(&author)?) + } else { + None + }; + Ok(Self { author }) + } + + pub fn author(&self, arg: Option) -> Result { + arg.or(self.author.clone()) + .ok_or_else(|| anyhow!("Author is required but not set")) + } +} + #[derive(Debug, Clone, Parser)] pub enum Doc { - StartSync { - peers: Vec, - }, - Share { - mode: ShareMode, - }, + /// Start to synchronize a document with peers + StartSync { peers: Vec }, + /// Share a document and print a ticket to share with peers + Share { mode: ShareMode }, /// Set an entry Set { /// Author of this entry. - author: AuthorId, + /// + /// Required unless the author is set through the REPL environment or the IROH_AUTHOR + /// environment variable. + #[clap(short, long)] + author: Option, /// Key to the entry (parsed as UTF-8 string). key: String, /// Content to store for this entry (parsed as UTF-8 string) @@ -148,6 +176,8 @@ pub enum Doc { #[clap(short, long)] content: bool, }, + /// List all entries in the document + #[clap(alias = "ls")] List { /// If true, old entries will be included. By default only the latest value for each key is /// shown. @@ -159,7 +189,7 @@ pub enum Doc { } impl Doc { - pub async fn run(self, iroh: Iroh, doc_id: NamespaceId) -> anyhow::Result<()> { + pub async fn run(self, iroh: &Iroh, doc_id: NamespaceId, env: DocEnv) -> Result<()> { let doc = iroh.get_doc(doc_id)?; match self { Doc::StartSync { peers } => { @@ -183,6 +213,7 @@ impl Doc { Doc::Set { author, key, value } => { let key = key.as_bytes().to_vec(); let value = value.as_bytes().to_vec(); + let author = env.author(author)?; let entry = doc.set_bytes(author, key, value).await?; println!("{}", fmt_entry(&entry)); } diff --git a/iroh/src/commands/validate.rs b/iroh/src/commands/validate.rs index 4d91163a90..9e90013d2b 100644 --- a/iroh/src/commands/validate.rs +++ b/iroh/src/commands/validate.rs @@ -4,13 +4,10 @@ use anyhow::Result; use console::{style, Emoji}; use futures::StreamExt; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use iroh::rpc_protocol::ValidateRequest; +use iroh::{client::quic::RpcClient, rpc_protocol::ValidateRequest}; use iroh_bytes::{baomap::ValidateProgress, Hash}; -use super::make_rpc_client; - -pub async fn run(rpc_port: u16, repair: bool) -> Result<()> { - let client = make_rpc_client(rpc_port).await?; +pub async fn run(client: RpcClient, repair: bool) -> Result<()> { let mut state = ValidateProgressState::new(); let mut response = client.server_streaming(ValidateRequest { repair }).await?; diff --git a/iroh/src/config.rs b/iroh/src/config.rs index dc41ec999c..a40a1b7158 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -23,6 +23,11 @@ pub const CONFIG_FILE_NAME: &str = "iroh.config.toml"; /// For example, `IROH_PATH=/path/to/config` would set the value of the `Config.path` field pub const ENV_PREFIX: &str = "IROH"; +/// Fetches the environment variable `IROH_` from the current process. +pub fn env_var(key: &str) -> std::result::Result { + env::var(&format!("{ENV_PREFIX}_{key}")) +} + /// Paths to files or directory within the [`iroh_data_root`] used by Iroh. #[derive(Debug, Clone, Eq, PartialEq)] pub enum IrohPaths { From e48174cad0adb10a6535e7d8966f533f89790f78 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 15 Aug 2023 16:28:05 +0200 Subject: [PATCH 109/172] fix: improve argument handling, add headings to help --- iroh/src/commands.rs | 93 +++++++++++++++++++++++++++++---------- iroh/src/commands/repl.rs | 6 +-- iroh/src/config.rs | 19 +++++++- iroh/src/main.rs | 34 ++------------ 4 files changed, 95 insertions(+), 57 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 9d5f07a0bf..af1e1a7740 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use std::{net::SocketAddr, path::PathBuf}; use anyhow::Result; -use clap::{Parser, Subcommand}; +use clap::{Args, Parser, Subcommand}; use futures::StreamExt; use iroh::client::quic::RpcClient; use iroh::dial::Ticket; @@ -44,6 +44,27 @@ pub mod validate; pub struct Cli { #[clap(subcommand)] pub command: Option, + + #[clap(flatten)] + #[clap(next_help_heading = "Options for all commands (except `provide`, `get`, and `doctor`)")] + pub rpc_args: RpcArgs, + + #[clap(flatten)] + #[clap(next_help_heading = "Options for `provide`, `get` and `doctor`")] + pub full_args: FullArgs, +} + +/// Options for commands that talk to a running Iroh node over RPC +#[derive(Args, Debug, Clone)] +pub struct RpcArgs { + /// RPC port of the Iroh node + #[clap(long, default_value_t = DEFAULT_RPC_PORT)] + pub rpc_port: u16, +} + +/// Options for commands that may start an Iroh node +#[derive(Args, Debug, Clone)] +pub struct FullArgs { /// Log SSL pre-master key to file in SSLKEYLOGFILE environment variable. #[clap(long)] pub keylog: bool, @@ -53,33 +74,53 @@ pub struct Cli { pub metrics_addr: Option, #[clap(long)] pub cfg: Option, - /// RPC port of the Iroh node - #[clap(long, default_value_t = DEFAULT_RPC_PORT)] - rpc_port: u16, } impl Cli { - pub async fn run(self, rt: &runtime::Handle, config: &Config) -> Result<()> { - match self.command { - None => { - let client = iroh::client::quic::connect_raw(self.rpc_port).await?; + pub async fn run(self, rt: &runtime::Handle) -> Result<()> { + let command = self.command.unwrap_or(Commands::Console); + match command { + Commands::Console => { + let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; repl::run(client).await } - Some(Commands::Rpc(command)) => { - let client = iroh::client::quic::connect_raw(self.rpc_port).await?; + Commands::Rpc(command) => { + let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; command.run(client).await } - Some(Commands::Full(command)) => command.run(rt, config, self.keylog).await, + Commands::Full(command) => { + let FullArgs { + cfg, + metrics_addr, + keylog, + } = self.full_args; + + let config = Config::from_env(cfg.as_deref())?; + + #[cfg(feature = "metrics")] + let metrics_fut = start_metrics_server(metrics_addr, &rt); + + let res = command.run(rt, &config, keylog).await; + + #[cfg(feature = "metrics")] + if let Some(metrics_fut) = metrics_fut { + metrics_fut.abort(); + } + + res + } } } } #[derive(Parser, Debug, Clone)] pub enum Commands { + /// Start the Iroh console + Console, #[clap(flatten)] Full(#[clap(subcommand)] FullCommands), #[clap(flatten)] - Rpc(#[clap(subcommands)] RpcCommand), + Rpc(#[clap(subcommands)] RpcCommands), } #[derive(Subcommand, Debug, Clone)] @@ -236,9 +277,15 @@ impl FullCommands { } } +#[derive(Subcommand, Debug, Clone)] +pub enum ConsoleCommands { + /// Start the Iroh console. This is the default command. + Console, +} + #[derive(Subcommand, Debug, Clone)] #[allow(clippy::large_enum_variant)] -pub enum RpcCommand { +pub enum RpcCommands { /// List availble content on the provider. #[clap(subcommand)] List(self::list::Commands), @@ -319,10 +366,10 @@ pub enum RpcCommand { Sync(#[clap(subcommand)] sync::Commands), } -impl RpcCommand { +impl RpcCommands { pub async fn run(self, client: RpcClient) -> Result<()> { match self { - RpcCommand::Share { + RpcCommands::Share { hash, recursive, peer, @@ -377,32 +424,32 @@ impl RpcCommand { } Ok(()) } - RpcCommand::List(cmd) => cmd.run(client).await, - RpcCommand::Validate { repair } => self::validate::run(client, repair).await, - RpcCommand::Shutdown { force } => { + RpcCommands::List(cmd) => cmd.run(client).await, + RpcCommands::Validate { repair } => self::validate::run(client, repair).await, + RpcCommands::Shutdown { force } => { client.rpc(ShutdownRequest { force }).await?; Ok(()) } - RpcCommand::Id {} => { + RpcCommands::Id {} => { let response = client.rpc(IdRequest).await?; println!("Listening address: {:#?}", response.listen_addrs); println!("PeerID: {}", response.peer_id); Ok(()) } - RpcCommand::Add { path, in_place } => self::add::run(client, path, in_place).await, - RpcCommand::Addresses {} => { + RpcCommands::Add { path, in_place } => self::add::run(client, path, in_place).await, + RpcCommands::Addresses {} => { let response = client.rpc(AddrsRequest).await?; println!("Listening addresses: {:?}", response.addrs); Ok(()) } - RpcCommand::Sync(command) => command.run(client).await, + RpcCommands::Sync(command) => command.run(client).await, } } } #[cfg(feature = "metrics")] -pub fn init_metrics_collection( +pub fn start_metrics_server( metrics_addr: Option, rt: &iroh_bytes::util::runtime::Handle, ) -> Option> { diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 223939ea04..c1d125c54e 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -42,7 +42,7 @@ pub async fn run(client: RpcClient) -> anyhow::Result<()> { #[derive(Debug)] pub enum FromRepl { RpcCmd { - cmd: super::RpcCommand, + cmd: super::RpcCommands, }, DocCmd { id: NamespaceId, @@ -160,7 +160,7 @@ impl ReplState { Pwd::Doc { id } => match parse_cmd::(line)? { DocCommands::Repl(cmd) => self.process_repl_command(cmd), DocCommands::Sync(cmd) => Some(FromRepl::RpcCmd { - cmd: super::RpcCommand::Sync(cmd), + cmd: super::RpcCommands::Sync(cmd), }), DocCommands::Doc(cmd) => Some(FromRepl::DocCmd { id, @@ -223,7 +223,7 @@ pub enum HomeCommands { #[clap(flatten)] Repl(#[clap(subcommand)] ReplCommand), #[clap(flatten)] - Rpc(#[clap(subcommand)] super::RpcCommand), + Rpc(#[clap(subcommand)] super::RpcCommands), } fn try_parse_cmd(s: &str) -> anyhow::Result { diff --git a/iroh/src/config.rs b/iroh/src/config.rs index a40a1b7158..9bdc236549 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -7,7 +7,7 @@ use std::{ str::FromStr, }; -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, bail, Context, Result}; use config::{Environment, File, Value}; use iroh_net::{ defaults::{default_eu_derp_region, default_na_derp_region}, @@ -109,6 +109,23 @@ impl Default for Config { } impl Config { + /// Make a config from the default environment variables. + /// + /// Optionally provide an additional configuration source. + pub fn from_env(additional_config_source: Option<&Path>) -> anyhow::Result { + let config_path = iroh_config_path(CONFIG_FILE_NAME).context("invalid config path")?; + let sources = [Some(config_path.as_path()), additional_config_source]; + let config = Config::load( + // potential config files + &sources, + // env var prefix for this config + ENV_PREFIX, + // map of present command line arguments + // args.make_overrides_map(), + HashMap::::new(), + )?; + Ok(config) + } /// Make a config using a default, files, environment variables, and commandline flags. /// /// Later items in the *file_paths* slice will have a higher priority than earlier ones. diff --git a/iroh/src/main.rs b/iroh/src/main.rs index ce986cde08..23d0cb9c64 100644 --- a/iroh/src/main.rs +++ b/iroh/src/main.rs @@ -1,16 +1,13 @@ -use std::{collections::HashMap, time::Duration}; +use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::Result; use clap::Parser; use tracing_subscriber::{prelude::*, EnvFilter}; mod commands; mod config; -use crate::{ - commands::{init_metrics_collection, Cli}, - config::{iroh_config_path, Config, CONFIG_FILE_NAME, ENV_PREFIX}, -}; +use crate::commands::Cli; fn main() -> Result<()> { let rt = tokio::runtime::Builder::new_multi_thread() @@ -36,28 +33,5 @@ async fn main_impl() -> Result<()> { .init(); let cli = Cli::parse(); - - let config_path = iroh_config_path(CONFIG_FILE_NAME).context("invalid config path")?; - let sources = [Some(config_path.as_path()), cli.cfg.as_deref()]; - let config = Config::load( - // potential config files - &sources, - // env var prefix for this config - ENV_PREFIX, - // map of present command line arguments - // args.make_overrides_map(), - HashMap::::new(), - )?; - - #[cfg(feature = "metrics")] - let metrics_fut = init_metrics_collection(cli.metrics_addr, &rt); - - let r = cli.run(&rt, &config).await; - - #[cfg(feature = "metrics")] - if let Some(metrics_fut) = metrics_fut { - metrics_fut.abort(); - drop(metrics_fut); - } - r + cli.run(&rt).await } From bd06fcbcc69dd6fa60460af028521e188adccfc5 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 15 Aug 2023 16:30:28 +0200 Subject: [PATCH 110/172] test: fix cli test --- iroh/tests/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/tests/cli.rs b/iroh/tests/cli.rs index 19f6bf8f5b..9676b0ba88 100644 --- a/iroh/tests/cli.rs +++ b/iroh/tests/cli.rs @@ -396,7 +396,7 @@ fn cli_provide_addresses() -> Result<()> { let _ticket = match_provide_output(&mut provider, 1)?; // test output - let get_output = cmd(iroh_bin(), ["addresses", "--rpc-port", RPC_PORT]) + let get_output = cmd(iroh_bin(), ["--rpc-port", RPC_PORT, "addresses"]) // .stderr_file(std::io::stderr().as_raw_fd()) for debug output .stdout_capture() .run()?; From 2612cf878f5b495677bc86968af8f9719ccbcc84 Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Wed, 16 Aug 2023 20:29:28 -0400 Subject: [PATCH 111/172] remove outdated help text (#1369) --- iroh/src/commands.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index af1e1a7740..64c9b46f10 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -27,20 +27,10 @@ pub mod repl; pub mod sync; pub mod validate; -/// Send data. -/// -/// The iroh command line tool has two modes: provide and get. -/// -/// The provide mode is a long-running process binding to a socket which the get mode -/// contacts to request data. By default the provide process also binds to an RPC port -/// which allows adding additional data to be provided as well as a few other maintenance -/// commands. -/// -/// The get mode retrieves data from the provider, for this it needs the hash, provider -/// address and PeerID as well as an authentication code. The get --ticket option is a -/// shortcut to provide all this information conveniently in a single ticket. +/// Iroh is a tool for syncing bytes. +/// https://iroh.computer/docs #[derive(Parser, Debug, Clone)] -#[clap(version)] +#[clap(version, verbatim_doc_comment)] pub struct Cli { #[clap(subcommand)] pub command: Option, From 6615092caa14408fb542d382c9225bb8c5594172 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 22 Aug 2023 15:24:36 +0200 Subject: [PATCH 112/172] fix: sync example --- Cargo.lock | 1 - iroh-sync/src/sync.rs | 4 ++-- iroh/Cargo.toml | 3 +-- iroh/examples/sync.rs | 36 +++++++++++------------------------- 4 files changed, 14 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d073176c51..dd08866778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1868,7 +1868,6 @@ dependencies = [ "derive_more", "dirs-next", "duct", - "ed25519-dalek", "flume", "futures", "genawaiter", diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 29c21822b0..5298718a66 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -244,7 +244,7 @@ impl SignedEntry { &self.entry } - /// Get the content [`Hash`] of the entry. + /// Get the content [`struct@Hash`] of the entry. pub fn content_hash(&self) -> &Hash { self.entry().record().content_hash() } @@ -491,7 +491,7 @@ impl Record { self.len } - /// Get the [`Hash`] of the content data of this record. + /// Get the [`struct@Hash`] of the content data of this record. pub fn content_hash(&self) -> &Hash { &self.hash } diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index ed35d5cccf..08c3a63327 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -57,7 +57,6 @@ data-encoding = "2.4.0" url = { version = "2.4", features = ["serde"] } # Examples -ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"], optional = true } shell-words = { version = "1.1.0", optional = true } shellexpand = { version = "3.1.0", optional = true } rustyline = { version = "12.0.0", optional = true } @@ -71,7 +70,7 @@ mem-db = [] flat-db = [] iroh-collection = [] test = [] -example-sync = ["cli", "ed25519-dalek", "shell-words", "shellexpand", "rustyline"] +example-sync = ["cli", "shell-words", "shellexpand", "rustyline"] [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 9d3cefb614..d78ce6dea0 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -1,7 +1,7 @@ //! Live edit a p2p document //! //! By default a new peer id is created when starting the example. To reuse your identity, -//! set the `--private-key` CLI flag with the private key printed on a previous invocation. +//! set the `--secret-key` CLI flag with the secret key printed on a previous invocation. //! //! You can use this with a local DERP server. To do so, run //! `cargo run --bin derper -- --dev` @@ -15,7 +15,6 @@ use std::{ use anyhow::{anyhow, bail}; use bytes::Bytes; use clap::{CommandFactory, FromArgMatches, Parser}; -use ed25519_dalek::SigningKey; use indicatif::HumanBytes; use iroh::{ download::Downloader, @@ -32,7 +31,7 @@ use iroh_gossip::{ }; use iroh_io::AsyncSliceReaderExt; use iroh_net::{ - defaults::default_derp_map, derp::DerpMap, magic_endpoint::get_alpn, tls::Keypair, + defaults::default_derp_map, derp::DerpMap, key::SecretKey, magic_endpoint::get_alpn, MagicEndpoint, }; use iroh_sync::{ @@ -58,9 +57,9 @@ type Doc = Replica<::Instance>; #[derive(Parser, Debug)] struct Args { - /// Private key to derive our peer id from + /// Secret key for this node #[clap(long)] - private_key: Option, + secret_key: Option, /// Path to a data directory where blobs will be persisted #[clap(short, long)] storage_path: Option, @@ -117,12 +116,12 @@ async fn run(args: Args) -> anyhow::Result<()> { let metrics_fut = init_metrics_collection(args.metrics_addr); - // parse or generate our keypair - let keypair = match args.private_key { - None => Keypair::generate(), - Some(key) => parse_keypair(&key)?, + // parse or generate our secret_key + let secret_key = match args.secret_key { + None => SecretKey::generate(), + Some(key) => SecretKey::from_str(&key)?, }; - println!("> our private key: {}", fmt_secret(&keypair)); + println!("> our secret key: {}", secret_key); // configure our derp map let derp_map = match (args.no_derp, args.derp) { @@ -141,7 +140,7 @@ async fn run(args: Args) -> anyhow::Result<()> { let (initial_endpoints_tx, mut initial_endpoints_rx) = mpsc::channel(1); // build the magic endpoint let endpoint = MagicEndpoint::builder() - .keypair(keypair.clone()) + .secret_key(secret_key.clone()) .alpns(vec![ GOSSIP_ALPN.to_vec(), SYNC_ALPN.to_vec(), @@ -219,7 +218,7 @@ async fn run(args: Args) -> anyhow::Result<()> { let rt = iroh_bytes::util::runtime::Handle::from_currrent(num_cpus::get())?; // create a doc store for the iroh-sync docs - let author = Author::from(keypair.secret().clone()); + let author = Author::from_bytes(&secret_key.to_bytes()); let docs_path = storage_path.join("docs.db"); let docs = iroh_sync::store::fs::Store::new(&docs_path)?; @@ -920,19 +919,6 @@ fn fmt_hash(hash: impl AsRef<[u8]>) -> String { text.make_ascii_lowercase(); format!("{}…{}", &text[..5], &text[(text.len() - 2)..]) } -fn fmt_secret(keypair: &Keypair) -> String { - let mut text = data_encoding::BASE32_NOPAD.encode(&keypair.secret().to_bytes()); - text.make_ascii_lowercase(); - text -} -fn parse_keypair(secret: &str) -> anyhow::Result { - let bytes: [u8; 32] = data_encoding::BASE32_NOPAD - .decode(secret.to_ascii_uppercase().as_bytes())? - .try_into() - .map_err(|_| anyhow::anyhow!("Invalid secret"))?; - let key = SigningKey::from_bytes(&bytes); - Ok(key.into()) -} fn fmt_derp_map(derp_map: &Option) -> String { match derp_map { None => "None".to_string(), From 6fa839dff285c2f9c9b9296a6256769b56f218f5 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 22 Aug 2023 16:12:21 +0200 Subject: [PATCH 113/172] deps: be less specific in required versions --- iroh-sync/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 333918a0d6..9c854eafcb 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -9,7 +9,7 @@ authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" [dependencies] -anyhow = "1.0.71" +anyhow = "1" blake3 = { package = "iroh-blake3", version = "1.4.3"} crossbeam = "0.8.2" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } @@ -22,8 +22,8 @@ postcard = { version = "1", default-features = false, features = ["alloc", "use- rand = "0.8.5" rand_core = "0.6.4" serde = { version = "1.0.164", features = ["derive"] } -url = "2.4.0" -bytes = "1.4.0" +url = "2.4" +bytes = "1" parking_lot = "0.12.1" hex = "0.4" @@ -32,7 +32,7 @@ redb = { version = "1.0.5", optional = true } ouroboros = { version = "0.17", optional = true } [dev-dependencies] -tokio = { version = "1.28.2", features = ["sync", "macros"] } +tokio = { version = "1", features = ["sync", "macros"] } tempfile = "3.4" [features] From ab2d2db0192860dea07a53b0f469690e90d05da0 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 22 Aug 2023 16:14:16 +0200 Subject: [PATCH 114/172] fix: do not require features for metrics --- iroh/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 08c3a63327..0220641ac0 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -65,7 +65,7 @@ itertools = "0.11.0" [features] default = ["cli", "metrics"] cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection"] -metrics = ["iroh-metrics", "flat-db", "mem-db", "iroh-collection"] +metrics = ["iroh-metrics"] mem-db = [] flat-db = [] iroh-collection = [] From 629289bd173ce658ac3e0d87a9b8cc14255eb1e1 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 22 Aug 2023 16:22:20 +0200 Subject: [PATCH 115/172] docs(iroh-sync): copy prose from lib.rs to README.md --- iroh-sync/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/iroh-sync/README.md b/iroh-sync/README.md index 7c79e368f2..27484946f0 100644 --- a/iroh-sync/README.md +++ b/iroh-sync/README.md @@ -1,5 +1,34 @@ # iroh-sync +Multi-dimensional key-value documents with an efficient synchronization protocol. + +The crate operates on *Replicas*. A replica contains an unlimited number of +*Entrys*. Each entry is identified by a key, its author, and the replica's +namespace. Its value is the 32-byte BLAKE3 hash of the entry's content data, +the size of this content data, and a timestamp. +The content data itself is not stored or transfered through a replica. + +All entries in a replica are signed with two keypairs: + +* The *Namespace* key, as a token of write capability. The public key is the *NamespaceId*, which + also serves as the unique identifier for a replica. +* The *Author* key, as a proof of authorship. Any number of authors may be created, and + their semantic meaning is application-specific. The public key of an author is the [AuthorId]. + +Replicas can be synchronized between peers by exchanging messages. The synchronization algorithm +is based on a technique called *range-based set reconciliation*, based on [this paper][paper] by +Aljoscha Meyer: + +> Range-based set reconciliation is a simple approach to efficiently compute the union of two +sets over a network, based on recursively partitioning the sets and comparing fingerprints of +the partitions to probabilistically detect whether a partition requires further work. + +The crate exposes a generic storage interface with in-memory and persistent, file-based +implementations. The latter makes use of [`redb`], an embedded key-value store, and persists +the whole store with all replicas to a single file. + +[paper]: https://arxiv.org/abs/2212.13567 + # License From 3779d2753136a594702d49bd4269f1938cdaa72c Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 22 Aug 2023 16:32:31 +0200 Subject: [PATCH 116/172] fix: idbytes macro --- iroh-gossip/src/proto/util.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/iroh-gossip/src/proto/util.rs b/iroh-gossip/src/proto/util.rs index 03a759ad01..a93f15d12a 100644 --- a/iroh-gossip/src/proto/util.rs +++ b/iroh-gossip/src/proto/util.rs @@ -56,20 +56,20 @@ macro_rules! idbytes_impls { } } - impl> std::convert::From for $ty { + impl> ::std::convert::From for $ty { fn from(value: T) -> Self { Self::from_bytes(value.into()) } } - impl std::fmt::Display for $ty { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + impl ::std::fmt::Display for $ty { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { write!(f, "{}", $crate::proto::util::base32::fmt(&self.0)) } } - impl std::fmt::Debug for $ty { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + impl ::std::fmt::Debug for $ty { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { write!( f, "{}({})", @@ -79,22 +79,22 @@ macro_rules! idbytes_impls { } } - impl std::str::FromStr for $ty { + impl ::std::str::FromStr for $ty { type Err = ::anyhow::Error; - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> ::std::result::Result { Ok(Self::from_bytes($crate::proto::util::base32::parse_array( s, )?)) } } - impl AsRef<[u8]> for $ty { + impl ::std::convert::AsRef<[u8]> for $ty { fn as_ref(&self) -> &[u8] { &self.0 } } - impl AsRef<[u8; 32]> for $ty { + impl ::std::convert::AsRef<[u8; 32]> for $ty { fn as_ref(&self) -> &[u8; 32] { &self.0 } From 62e3c8d6b1be885de74f536594e4489d1b84fa01 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 10:54:06 +0200 Subject: [PATCH 117/172] feat: persist default author in repl --- iroh/src/commands.rs | 13 +++-- iroh/src/commands/repl.rs | 108 ++++++++++++++++++++------------------ iroh/src/commands/sync.rs | 84 ++++++++++++++++++----------- iroh/src/config.rs | 54 +++++++++++++++++++ 4 files changed, 175 insertions(+), 84 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 64c9b46f10..7e39da5ee2 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -4,7 +4,7 @@ use std::{net::SocketAddr, path::PathBuf}; use anyhow::Result; use clap::{Args, Parser, Subcommand}; use futures::StreamExt; -use iroh::client::quic::RpcClient; +use iroh::client::quic::{Iroh, RpcClient}; use iroh::dial::Ticket; use iroh::rpc_protocol::*; use iroh_bytes::{protocol::RequestToken, util::runtime, Hash}; @@ -13,6 +13,7 @@ use iroh_net::key::{PublicKey, SecretKey}; use crate::config::Config; use self::provide::{ProvideOptions, ProviderRpcPort}; +use self::sync::SyncEnv; const DEFAULT_RPC_PORT: u16 = 0x1337; const MAX_RPC_CONNECTIONS: u32 = 16; @@ -72,7 +73,9 @@ impl Cli { match command { Commands::Console => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; - repl::run(client).await + let iroh = Iroh::new(client.clone()); + let env = SyncEnv::load_from_env(&iroh).await?; + repl::run(client, env).await } Commands::Rpc(command) => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; @@ -433,7 +436,11 @@ impl RpcCommands { println!("Listening addresses: {:?}", response.addrs); Ok(()) } - RpcCommands::Sync(command) => command.run(client).await, + RpcCommands::Sync(command) => { + let iroh = Iroh::new(client.clone()); + let env = SyncEnv::load_from_env(&iroh).await?; + command.run(client, env).await + } } } } diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index c1d125c54e..c542b67ce8 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -1,4 +1,5 @@ use ansi_term::Colour; +use anyhow::Result; use clap::{Parser, Subcommand}; use iroh::client::quic::{Iroh, RpcClient}; use iroh_gossip::proto::util::base32; @@ -6,24 +7,39 @@ use iroh_sync::sync::{AuthorId, NamespaceId}; use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; -use super::sync::DocEnv; +use super::sync::SyncEnv; -pub async fn run(client: RpcClient) -> anyhow::Result<()> { +pub async fn run(client: RpcClient, env: SyncEnv) -> Result<()> { println!("Welcome to the Iroh console!"); println!("Type `help` for a list of commands."); - let mut repl_rx = Repl::spawn(); + let mut state = ReplState::with_env(env); + let mut repl_rx = Repl::spawn(state.clone()); while let Some((event, reply)) = repl_rx.recv().await { let (next, res) = match event { - FromRepl::DocCmd { id, cmd, doc_env } => { + FromRepl::DocCmd { id, cmd, env } => { let iroh = Iroh::new(client.clone()); - let res = cmd.run(&iroh, id, doc_env).await; + let res = cmd.run(&iroh, id, env).await; (ToRepl::Continue, res) } FromRepl::RpcCmd { cmd } => { let res = cmd.run(client.clone()).await; (ToRepl::Continue, res) } - FromRepl::Exit => (ToRepl::Exit, Ok(())), + FromRepl::ReplCmd { cmd } => match cmd { + ReplCmd::SetDoc { id } => { + state.pwd = Pwd::Doc { id }; + (ToRepl::UpdateState(state.clone()), Ok(())) + } + ReplCmd::SetAuthor { id } => { + let res = state.env.set_author(id).await; + (ToRepl::UpdateState(state.clone()), res) + } + ReplCmd::Close => { + state.pwd = Pwd::Home; + (ToRepl::UpdateState(state.clone()), Ok(())) + } + ReplCmd::Exit => (ToRepl::Exit, Ok(())), + }, }; if let Err(err) = res { @@ -46,15 +62,22 @@ pub enum FromRepl { }, DocCmd { id: NamespaceId, - doc_env: DocEnv, + env: SyncEnv, cmd: super::sync::Doc, }, - Exit, + ReplCmd { + cmd: ReplCmd, + }, } +/// Reply to the repl after a command completed #[derive(Debug)] pub enum ToRepl { + /// Continue execution by reading the next command Continue, + /// Continue execution by reading the next command, and update the repl state + UpdateState(ReplState), + /// Exit the repl Exit, } @@ -63,12 +86,9 @@ pub struct Repl { cmd_tx: mpsc::Sender<(FromRepl, oneshot::Sender)>, } impl Repl { - pub fn spawn() -> mpsc::Receiver<(FromRepl, oneshot::Sender)> { + pub fn spawn(state: ReplState) -> mpsc::Receiver<(FromRepl, oneshot::Sender)> { let (cmd_tx, cmd_rx) = mpsc::channel(1); - let repl = Repl { - state: ReplState::from_env(), - cmd_tx, - }; + let repl = Repl { state, cmd_tx }; std::thread::spawn(move || { if let Err(err) = repl.run() { println!("> repl crashed: {err}"); @@ -87,7 +107,7 @@ impl Repl { Ok(line) if line.is_empty() => continue, Ok(line) => { rl.add_history_entry(line.as_str())?; - let cmd = self.state.parse_command(&line); + let cmd = self.state.handle_command(&line); if let Some(cmd) = cmd { self.cmd_tx.blocking_send((cmd, to_repl_tx))?; } else { @@ -95,13 +115,18 @@ impl Repl { } } Err(ReadlineError::Interrupted | ReadlineError::Eof) => { - self.cmd_tx.blocking_send((FromRepl::Exit, to_repl_tx))?; + self.cmd_tx + .blocking_send((FromRepl::ReplCmd { cmd: ReplCmd::Exit }, to_repl_tx))?; } Err(ReadlineError::WindowResized) => continue, Err(err) => return Err(err.into()), } // wait for reply from main thread match to_repl_rx.blocking_recv()? { + ToRepl::UpdateState(state) => { + self.state = state; + continue; + } ToRepl::Continue => continue, ToRepl::Exit => break, } @@ -110,21 +135,22 @@ impl Repl { } } +#[derive(Debug, Clone)] pub struct ReplState { pwd: Pwd, - doc_env: DocEnv, + env: SyncEnv, } impl ReplState { - fn from_env() -> Self { + fn with_env(env: SyncEnv) -> Self { Self { pwd: Pwd::Home, - doc_env: DocEnv::from_env().unwrap_or_default(), + env, } } } -#[derive(Debug)] +#[derive(Debug, Clone)] enum Pwd { Home, Doc { id: NamespaceId }, @@ -136,13 +162,11 @@ impl ReplState { let pwd = match self.pwd { Pwd::Home => None, Pwd::Doc { id } => { - let author = self - .doc_env - .author - .map(|author| format!(" author:{}", fmt_short(author.as_bytes()))) - .map(|author| Colour::Red.paint(author).to_string()) - .unwrap_or_default(); - let pwd = format!("doc:{}{author}", fmt_short(id.as_bytes())); + let author = match self.env.author().as_ref() { + Some(author) => fmt_short(author.as_bytes()), + None => "".to_string(), + }; + let pwd = format!("doc:{} author:{}", fmt_short(id.as_bytes()), author); let pwd = Colour::Blue.paint(pwd); Some(pwd.to_string()) } @@ -151,47 +175,29 @@ impl ReplState { format!("\n{pwd}{bang}") } - pub fn parse_command(&mut self, line: &str) -> Option { + pub fn handle_command(&mut self, line: &str) -> Option { match self.pwd { Pwd::Home => match parse_cmd::(line)? { - HomeCommands::Repl(cmd) => self.process_repl_command(cmd), + HomeCommands::Repl(cmd) => Some(FromRepl::ReplCmd { cmd }), HomeCommands::Rpc(cmd) => Some(FromRepl::RpcCmd { cmd }), }, Pwd::Doc { id } => match parse_cmd::(line)? { - DocCommands::Repl(cmd) => self.process_repl_command(cmd), + DocCommands::Repl(cmd) => Some(FromRepl::ReplCmd { cmd }), DocCommands::Sync(cmd) => Some(FromRepl::RpcCmd { cmd: super::RpcCommands::Sync(cmd), }), DocCommands::Doc(cmd) => Some(FromRepl::DocCmd { id, cmd, - doc_env: self.doc_env.clone(), + env: self.env.clone(), }), }, } } - - fn process_repl_command(&mut self, command: ReplCommand) -> Option { - match command { - ReplCommand::SetDoc { id } => { - self.pwd = Pwd::Doc { id }; - None - } - ReplCommand::SetAuthor { id } => { - self.doc_env.author = Some(id); - None - } - ReplCommand::Close => { - self.pwd = Pwd::Home; - None - } - ReplCommand::Exit => Some(FromRepl::Exit), - } - } } #[derive(Debug, Parser)] -pub enum ReplCommand { +pub enum ReplCmd { /// Open a document #[clap(next_help_heading = "foo")] SetDoc { id: NamespaceId }, @@ -210,7 +216,7 @@ pub enum DocCommands { #[clap(flatten)] Doc(#[clap(subcommand)] super::sync::Doc), #[clap(flatten)] - Repl(#[clap(subcommand)] ReplCommand), + Repl(#[clap(subcommand)] ReplCmd), // TODO: We can't embed RpcCommand here atm because there'd be a conflict between // `list` top level and `list` doc command // Thus for now only embedding sync commands @@ -221,7 +227,7 @@ pub enum DocCommands { #[derive(Debug, Parser)] pub enum HomeCommands { #[clap(flatten)] - Repl(#[clap(subcommand)] ReplCommand), + Repl(#[clap(subcommand)] ReplCmd), #[clap(flatten)] Rpc(#[clap(subcommand)] super::RpcCommands), } diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 974536163a..c983628e7d 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -1,4 +1,4 @@ -use std::str::FromStr; +use std::{path::PathBuf, str::FromStr}; use anyhow::{anyhow, Result}; use clap::Parser; @@ -13,13 +13,59 @@ use iroh_sync::{ store::GetFilter, sync::{AuthorId, NamespaceId, SignedEntry}, }; +use tokio::fs; -use crate::config::env_var; +use crate::config::{ConsolePaths, IrohPaths}; use super::RpcClient; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; +#[derive(Debug, Clone, Default)] +pub struct SyncEnv { + pub dir: PathBuf, + pub author: Option, + pub repl_history_path: PathBuf, +} + +impl SyncEnv { + pub async fn load_from_env(_client: &Iroh) -> Result { + let dir: PathBuf = IrohPaths::Console.with_env()?; + if std::fs::metadata(&dir).is_err() { + std::fs::create_dir_all(&dir)?; + }; + let repl_history_path = ConsolePaths::History.with_root(&dir); + let author_path = ConsolePaths::DefaultAuthor.with_root(&dir); + let author = match fs::read(author_path).await { + Ok(s) => Some(AuthorId::from_str(&String::from_utf8(s)?)?), + Err(_err) => None, + }; + let slf = Self { + dir, + author, + repl_history_path, + }; + Ok(slf) + } + + async fn persist(&self) -> Result<()> { + if let Some(author) = &self.author { + let author_path = ConsolePaths::DefaultAuthor.with_root(&self.dir); + fs::write(author_path, format!("{}", author)).await?; + } + Ok(()) + } + + pub async fn set_author(&mut self, author: AuthorId) -> Result<()> { + self.author = Some(author); + self.persist().await + } + + pub fn author(&self) -> &Option { + &self.author + } +} + #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Parser)] pub enum Commands { @@ -42,15 +88,12 @@ pub enum Commands { } impl Commands { - pub async fn run(self, client: RpcClient) -> Result<()> { + pub async fn run(self, client: RpcClient, env: SyncEnv) -> Result<()> { let iroh = Iroh::new(client); match self { Commands::Author { command } => command.run(&iroh).await, Commands::Docs { command } => command.run(&iroh).await, - Commands::Doc { command, id } => { - let doc_env = DocEnv::from_env()?; - command.run(&iroh, id, doc_env).await - } + Commands::Doc { command, id } => command.run(&iroh, id, env).await, } } } @@ -115,27 +158,6 @@ impl Docs { } } -#[derive(Debug, Clone, Default)] -pub struct DocEnv { - pub author: Option, -} - -impl DocEnv { - pub fn from_env() -> anyhow::Result { - let author = if let Some(author) = env_var("AUTHOR").ok() { - Some(AuthorId::from_str(&author)?) - } else { - None - }; - Ok(Self { author }) - } - - pub fn author(&self, arg: Option) -> Result { - arg.or(self.author.clone()) - .ok_or_else(|| anyhow!("Author is required but not set")) - } -} - #[derive(Debug, Clone, Parser)] pub enum Doc { /// Start to synchronize a document with peers @@ -189,7 +211,7 @@ pub enum Doc { } impl Doc { - pub async fn run(self, iroh: &Iroh, doc_id: NamespaceId, env: DocEnv) -> Result<()> { + pub async fn run(self, iroh: &Iroh, doc_id: NamespaceId, env: SyncEnv) -> Result<()> { let doc = iroh.get_doc(doc_id)?; match self { Doc::StartSync { peers } => { @@ -213,7 +235,9 @@ impl Doc { Doc::Set { author, key, value } => { let key = key.as_bytes().to_vec(); let value = value.as_bytes().to_vec(); - let author = env.author(author)?; + let author = author + .or_else(|| *env.author()) + .ok_or_else(|| anyhow!("No author provided"))?; let entry = doc.set_bytes(author, key, value).await?; println!("{}", fmt_entry(&entry)); } diff --git a/iroh/src/config.rs b/iroh/src/config.rs index 9bdc236549..495a3e8e06 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -24,6 +24,7 @@ pub const CONFIG_FILE_NAME: &str = "iroh.config.toml"; pub const ENV_PREFIX: &str = "IROH"; /// Fetches the environment variable `IROH_` from the current process. +#[allow(dead_code)] pub fn env_var(key: &str) -> std::result::Result { env::var(&format!("{ENV_PREFIX}_{key}")) } @@ -39,7 +40,10 @@ pub enum IrohPaths { BaoFlatStorePartial, /// Path to the [iroh-sync document database](iroh_sync::store::fs::Store) DocsDatabase, + /// Path to a directory with the [`ConsolePaths`] + Console, } + impl From<&IrohPaths> for &'static str { fn from(value: &IrohPaths) -> Self { match value { @@ -47,6 +51,7 @@ impl From<&IrohPaths> for &'static str { IrohPaths::BaoFlatStoreComplete => "blobs.v0", IrohPaths::BaoFlatStorePartial => "blobs-partial.v0", IrohPaths::DocsDatabase => "docs.redb", + IrohPaths::Console => "console", } } } @@ -58,6 +63,7 @@ impl FromStr for IrohPaths { "blobs.v0" => Self::BaoFlatStoreComplete, "blobs-partial.v0" => Self::BaoFlatStorePartial, "docs.redb" => Self::DocsDatabase, + "console" => Self::Console, _ => bail!("unknown file or directory"), }) } @@ -91,6 +97,53 @@ impl IrohPaths { } } +#[derive(Debug, Clone, Copy)] +pub enum ConsolePaths { + DefaultAuthor, + History, +} + +impl From<&ConsolePaths> for &'static str { + fn from(value: &ConsolePaths) -> Self { + match value { + ConsolePaths::DefaultAuthor => "default_author.pubkey", + ConsolePaths::History => "history", + } + } +} +impl FromStr for ConsolePaths { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + Ok(match s { + "default_author.pubkey" => Self::DefaultAuthor, + "history" => Self::History, + _ => bail!("unknown file or directory"), + }) + } +} + +impl fmt::Display for ConsolePaths { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &str = self.into(); + write!(f, "{s}") + } +} +impl AsRef for ConsolePaths { + fn as_ref(&self) -> &Path { + let s: &str = self.into(); + Path::new(s) + } +} + +impl ConsolePaths { + pub fn with_root(self, root: impl AsRef) -> PathBuf { + PathBuf::from(root.as_ref()).join(self) + } + pub fn with_env(self) -> Result { + Ok(self.with_root(IrohPaths::Console.with_env()?)) + } +} + /// The configuration for the iroh cli. #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] #[serde(default)] @@ -289,6 +342,7 @@ mod tests { IrohPaths::BaoFlatStoreComplete, IrohPaths::BaoFlatStorePartial, IrohPaths::DocsDatabase, + IrohPaths::Console, ]; for iroh_path in &kinds { let root = PathBuf::from("/tmp"); From 7dc64c4b4bad9bf4a12313f3258d6caff4162727 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 10:56:48 +0200 Subject: [PATCH 118/172] feat: persist repl history --- iroh/src/commands/repl.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index c542b67ce8..c97e31bd18 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -99,6 +99,7 @@ impl Repl { pub fn run(mut self) -> anyhow::Result<()> { let mut rl = DefaultEditor::with_config(Config::builder().check_cursor_position(true).build())?; + rl.load_history(&self.state.env.repl_history_path).ok(); loop { // prepare a channel to receive a signal from the main thread when a command completed let (to_repl_tx, to_repl_rx) = oneshot::channel(); @@ -131,6 +132,7 @@ impl Repl { ToRepl::Exit => break, } } + rl.save_history(&self.state.env.repl_history_path).ok(); Ok(()) } } From 09683847bec9cb1c37527fd4217ef2bafb560fd1 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 11:16:38 +0200 Subject: [PATCH 119/172] refactor: use base32 for Author and Namespace encoding --- Cargo.lock | 1 + iroh-sync/Cargo.toml | 1 + iroh-sync/src/keys.rs | 83 +++++++++++++++++++++------------------ iroh-sync/src/store/fs.rs | 4 +- 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31bd98f4a0..a8dc6c995d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2097,6 +2097,7 @@ dependencies = [ "anyhow", "bytes", "crossbeam", + "data-encoding", "derive_more", "ed25519-dalek", "flume", diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 9c854eafcb..8d229c788a 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -12,6 +12,7 @@ repository = "https://github.com/n0-computer/iroh" anyhow = "1" blake3 = { package = "iroh-blake3", version = "1.4.3"} crossbeam = "0.8.2" +data-encoding = "2.4.0" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"] } flume = "0.10" diff --git a/iroh-sync/src/keys.rs b/iroh-sync/src/keys.rs index 1acf56f4fe..8232217fbe 100644 --- a/iroh-sync/src/keys.rs +++ b/iroh-sync/src/keys.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; /// Author key to insert entries in a [`crate::Replica`] /// /// Internally, an author is a [`SigningKey`] which is used to sign entries. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct Author { signing_key: SigningKey, } @@ -30,11 +30,6 @@ impl Author { self.signing_key.to_bytes() } - /// Returns the [`AuthorId`] byte representation. - pub fn id_bytes(&self) -> [u8; 32] { - self.signing_key.verifying_key().to_bytes() - } - /// Get the [`AuthorId`] for this author. pub fn id(&self) -> AuthorId { AuthorId(self.signing_key.verifying_key()) @@ -84,7 +79,7 @@ impl AuthorId { /// /// Holders of this key can insert new entries into a [`crate::Replica`]. /// Internally, a [`Namespace`] is a [`SigningKey`] which is used to sign entries. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct Namespace { signing_key: SigningKey, } @@ -107,11 +102,6 @@ impl Namespace { self.signing_key.to_bytes() } - /// Returns the [`NamespaceId`] byte representation. - pub fn id_bytes(&self) -> [u8; 32] { - self.signing_key.verifying_key().to_bytes() - } - /// Get the [`NamespaceId`] for this namespace. pub fn id(&self) -> NamespaceId { NamespaceId(self.signing_key.verifying_key()) @@ -159,59 +149,65 @@ impl NamespaceId { impl fmt::Display for Author { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Author({})", hex::encode(self.signing_key.to_bytes())) + write!(f, "{}", base32::fmt(&self.to_bytes())) } } impl fmt::Display for Namespace { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Namespace({})", hex::encode(self.signing_key.to_bytes())) + write!(f, "{}", base32::fmt(&self.to_bytes())) } } impl fmt::Display for AuthorId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", hex::encode(self.0.as_bytes())) + write!(f, "{}", base32::fmt(self.as_bytes())) } } impl fmt::Display for NamespaceId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", hex::encode(self.0.as_bytes())) + write!(f, "{}", base32::fmt(self.as_bytes())) + } +} + +impl fmt::Debug for Namespace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Namespace({})", self) + } +} + +impl fmt::Debug for Author { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Author({})", self) } } impl fmt::Debug for NamespaceId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NamespaceId({})", hex::encode(self.0.as_bytes())) + write!(f, "NamespaceId({})", self) } } impl fmt::Debug for AuthorId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "AuthorId({})", hex::encode(self.0.as_bytes())) + write!(f, "AuthorId({})", self) } } impl FromStr for Author { - type Err = (); + type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let signing_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; - let signing_key = SigningKey::from_bytes(&signing_key); - - Ok(Author { signing_key }) + Ok(Self::from_bytes(&base32::parse_array(s)?)) } } impl FromStr for Namespace { - type Err = (); + type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let signing_key: [u8; 32] = hex::decode(s).map_err(|_| ())?.try_into().map_err(|_| ())?; - let signing_key = SigningKey::from_bytes(&signing_key); - - Ok(Namespace { signing_key }) + Ok(Self::from_bytes(&base32::parse_array(s)?)) } } @@ -219,11 +215,7 @@ impl FromStr for AuthorId { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let verifying_key: [u8; 32] = hex::decode(s)? - .try_into() - .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; - let verifying_key = VerifyingKey::from_bytes(&verifying_key)?; - Ok(AuthorId(verifying_key)) + Self::from_bytes(&base32::parse_array(s)?) } } @@ -231,11 +223,7 @@ impl FromStr for NamespaceId { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - let verifying_key: [u8; 32] = hex::decode(s)? - .try_into() - .map_err(|_| anyhow::anyhow!("failed to parse: invalid key length"))?; - let verifying_key = VerifyingKey::from_bytes(&verifying_key)?; - Ok(NamespaceId(verifying_key)) + Self::from_bytes(&base32::parse_array(s)?) } } @@ -274,3 +262,22 @@ impl Ord for AuthorId { self.0.as_bytes().cmp(other.0.as_bytes()) } } + +/// Utilities for working with byte array identifiers +// TODO: copy-pasted from iroh-gossip/src/proto/util.rs +// Unify into iroh-common crate or similar +mod base32 { + /// Convert to a base32 string + pub fn fmt(bytes: impl AsRef<[u8]>) -> String { + let mut text = data_encoding::BASE32_NOPAD.encode(bytes.as_ref()); + text.make_ascii_lowercase(); + text + } + /// Parse from a base32 string into a byte array + pub fn parse_array(input: &str) -> anyhow::Result<[u8; N]> { + data_encoding::BASE32_NOPAD + .decode(input.to_ascii_uppercase().as_bytes())? + .try_into() + .map_err(|_| ::anyhow::anyhow!("Failed to parse: invalid byte length")) + } +} diff --git a/iroh-sync/src/store/fs.rs b/iroh-sync/src/store/fs.rs index 2833c7dbb8..0e9e68c407 100644 --- a/iroh-sync/src/store/fs.rs +++ b/iroh-sync/src/store/fs.rs @@ -84,7 +84,7 @@ impl Store { let write_tx = self.db.begin_write()?; { let mut namespace_table = write_tx.open_table(NAMESPACES_TABLE)?; - namespace_table.insert(&namespace.id_bytes(), &namespace.to_bytes())?; + namespace_table.insert(namespace.id().as_bytes(), &namespace.to_bytes())?; } write_tx.commit()?; @@ -95,7 +95,7 @@ impl Store { let write_tx = self.db.begin_write()?; { let mut author_table = write_tx.open_table(AUTHORS_TABLE)?; - author_table.insert(&author.id_bytes(), &author.to_bytes())?; + author_table.insert(author.id().as_bytes(), &author.to_bytes())?; } write_tx.commit()?; From fbec016c5fd35912fcd6b10f78d7219718f57953 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 15:26:14 +0200 Subject: [PATCH 120/172] feat: rework CLI and repl structure --- Cargo.lock | 1 + iroh/Cargo.toml | 3 +- iroh/src/commands.rs | 147 ++++++++++++--------- iroh/src/commands/doctor.rs | 27 ++-- iroh/src/commands/repl.rs | 185 ++++++++------------------ iroh/src/commands/sync.rs | 257 +++++++++++++++--------------------- iroh/src/config.rs | 222 +++++++++++++++++++++++-------- iroh/src/node.rs | 27 ++-- iroh/src/rpc_protocol.rs | 29 +--- 9 files changed, 439 insertions(+), 459 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffd9abec25..48be76ccc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1914,6 +1914,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", + "toml 0.7.6", "tracing", "tracing-subscriber", "url", diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 19e300d15e..e5c41bce5e 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -57,6 +57,7 @@ rustyline = { version = "12.0.0", optional = true } shell-words = { version = "1.1.0", optional = true } shellexpand = { version = "3.1.0", optional = true } tempfile = { version = "3.4", optional = true } +toml = { version = "0.7.3", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } data-encoding = "2.4.0" url = { version = "2.4", features = ["serde"] } @@ -66,7 +67,7 @@ ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"], optional [features] default = ["cli", "metrics"] -cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection", "shell-words", "shellexpand", "rustyline", "ansi_term"] +cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection", "shell-words", "shellexpand", "rustyline", "ansi_term", "toml"] metrics = ["iroh-metrics"] mem-db = [] flat-db = [] diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 7e39da5ee2..2b29f46093 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -4,16 +4,16 @@ use std::{net::SocketAddr, path::PathBuf}; use anyhow::Result; use clap::{Args, Parser, Subcommand}; use futures::StreamExt; -use iroh::client::quic::{Iroh, RpcClient}; +use iroh::client::quic::RpcClient; use iroh::dial::Ticket; use iroh::rpc_protocol::*; use iroh_bytes::{protocol::RequestToken, util::runtime, Hash}; use iroh_net::key::{PublicKey, SecretKey}; -use crate::config::Config; +use crate::config::{ConsoleEnv, NodeConfig}; use self::provide::{ProvideOptions, ProviderRpcPort}; -use self::sync::SyncEnv; +// use self::sync::SyncEnv; const DEFAULT_RPC_PORT: u16 = 0x1337; const MAX_RPC_CONNECTIONS: u32 = 16; @@ -34,14 +34,14 @@ pub mod validate; #[clap(version, verbatim_doc_comment)] pub struct Cli { #[clap(subcommand)] - pub command: Option, + pub command: Commands, #[clap(flatten)] - #[clap(next_help_heading = "Options for all commands (except `provide`, `get`, and `doctor`)")] + #[clap(next_help_heading = "Options for console, doc, author, blob, node")] pub rpc_args: RpcArgs, #[clap(flatten)] - #[clap(next_help_heading = "Options for `provide`, `get` and `doctor`")] + #[clap(next_help_heading = "Options for start, get, doctor")] pub full_args: FullArgs, } @@ -69,17 +69,16 @@ pub struct FullArgs { impl Cli { pub async fn run(self, rt: &runtime::Handle) -> Result<()> { - let command = self.command.unwrap_or(Commands::Console); - match command { + match self.command { Commands::Console => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; - let iroh = Iroh::new(client.clone()); - let env = SyncEnv::load_from_env(&iroh).await?; + let env = ConsoleEnv::from_env_and_peristent_state()?; repl::run(client, env).await } Commands::Rpc(command) => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; - command.run(client).await + let env = ConsoleEnv::from_env()?; + command.run(client, env).await } Commands::Full(command) => { let FullArgs { @@ -88,7 +87,7 @@ impl Cli { keylog, } = self.full_args; - let config = Config::from_env(cfg.as_deref())?; + let config = NodeConfig::from_env(cfg.as_deref())?; #[cfg(feature = "metrics")] let metrics_fut = start_metrics_server(metrics_addr, &rt); @@ -118,11 +117,11 @@ pub enum Commands { #[derive(Subcommand, Debug, Clone)] pub enum FullCommands { - /// Serve data from the given path. + /// Start a Iroh node /// /// If PATH is a folder all files in that folder will be served. If no PATH is /// specified reads from STDIN. - Provide { + Start { /// Path to initial file or directory to provide path: Option, /// Serve data in place @@ -144,7 +143,9 @@ pub enum FullCommands { #[clap(long)] request_token: Option, }, - /// Fetch the data identified by HASH from a provider + /// Fetch data from a provider + /// + /// Starts a temporary Iroh node and fetches the content identified by HASH. Get { /// The hash to retrieve, as a Blake3 CID #[clap(conflicts_with = "ticket", required_unless_present = "ticket")] @@ -192,9 +193,9 @@ pub enum FullCommands { } impl FullCommands { - pub async fn run(self, rt: &runtime::Handle, config: &Config, keylog: bool) -> Result<()> { + pub async fn run(self, rt: &runtime::Handle, config: &NodeConfig, keylog: bool) -> Result<()> { match self { - FullCommands::Provide { + FullCommands::Start { path, in_place, addr, @@ -270,25 +271,29 @@ impl FullCommands { } } -#[derive(Subcommand, Debug, Clone)] -pub enum ConsoleCommands { - /// Start the Iroh console. This is the default command. - Console, -} - #[derive(Subcommand, Debug, Clone)] #[allow(clippy::large_enum_variant)] pub enum RpcCommands { - /// List availble content on the provider. - #[clap(subcommand)] - List(self::list::Commands), - /// Validate hashes on the running provider. - Validate { - /// Repair the store by removing invalid data - #[clap(long, default_value_t = false)] - repair: bool, + /// Doc and author commands + #[clap(flatten)] + Sync(#[clap(subcommand)] sync::Commands), + /// Manage blobs + Blob { + #[clap(subcommand)] + command: BlobCommands, + }, + /// Manage a running Iroh node + Node { + #[clap(subcommand)] + command: NodeCommands, }, - /// Shutdown provider. +} + +#[derive(Subcommand, Debug, Clone)] +pub enum NodeCommands { + /// Get status of the running node. + Status, + /// Shutdown the running node. Shutdown { /// Shutdown mode. /// @@ -297,8 +302,38 @@ pub enum RpcCommands { #[clap(long, default_value_t = false)] force: bool, }, - /// Identify the running provider. - Id, +} + +impl NodeCommands { + pub async fn run(self, client: RpcClient) -> Result<()> { + match self { + Self::Shutdown { force } => { + client.rpc(ShutdownRequest { force }).await?; + Ok(()) + } + Self::Status {} => { + let response = client.rpc(StatusRequest).await?; + + println!("Listening address: {:#?}", response.listen_addrs); + println!("PeerID: {}", response.peer_id); + Ok(()) + } + } + } +} + +impl RpcCommands { + pub async fn run(self, client: RpcClient, env: ConsoleEnv) -> Result<()> { + match self { + Self::Node { command } => command.run(client).await, + Self::Blob { command } => command.run(client).await, + Self::Sync(command) => command.run(client, env).await, + } + } +} + +#[derive(Subcommand, Debug, Clone)] +pub enum BlobCommands { /// Add data from PATH to the running provider's database. Add { /// The path to the file or folder to add @@ -353,16 +388,21 @@ pub enum RpcCommands { #[clap(long, default_value_t = false)] stable: bool, }, - /// List listening addresses of the provider. - Addresses, - #[clap(flatten)] - Sync(#[clap(subcommand)] sync::Commands), + /// List availble content on the node. + #[clap(subcommand)] + List(self::list::Commands), + /// Validate hashes on the running node. + Validate { + /// Repair the store by removing invalid data + #[clap(long, default_value_t = false)] + repair: bool, + }, } -impl RpcCommands { +impl BlobCommands { pub async fn run(self, client: RpcClient) -> Result<()> { match self { - RpcCommands::Share { + Self::Share { hash, recursive, peer, @@ -417,30 +457,9 @@ impl RpcCommands { } Ok(()) } - RpcCommands::List(cmd) => cmd.run(client).await, - RpcCommands::Validate { repair } => self::validate::run(client, repair).await, - RpcCommands::Shutdown { force } => { - client.rpc(ShutdownRequest { force }).await?; - Ok(()) - } - RpcCommands::Id {} => { - let response = client.rpc(IdRequest).await?; - - println!("Listening address: {:#?}", response.listen_addrs); - println!("PeerID: {}", response.peer_id); - Ok(()) - } - RpcCommands::Add { path, in_place } => self::add::run(client, path, in_place).await, - RpcCommands::Addresses {} => { - let response = client.rpc(AddrsRequest).await?; - println!("Listening addresses: {:?}", response.addrs); - Ok(()) - } - RpcCommands::Sync(command) => { - let iroh = Iroh::new(client.clone()); - let env = SyncEnv::load_from_env(&iroh).await?; - command.run(client, env).await - } + Self::List(cmd) => cmd.run(client).await, + Self::Validate { repair } => self::validate::run(client, repair).await, + Self::Add { path, in_place } => self::add::run(client, path, in_place).await, } } } diff --git a/iroh/src/commands/doctor.rs b/iroh/src/commands/doctor.rs index 9dcca51b95..604cd55407 100644 --- a/iroh/src/commands/doctor.rs +++ b/iroh/src/commands/doctor.rs @@ -1,13 +1,12 @@ //! Tool to get information about the current network environment of a node, //! and to test connectivity to specific other nodes. use std::{ - collections::HashMap, net::SocketAddr, num::NonZeroU16, time::{Duration, Instant}, }; -use crate::config::{iroh_config_path, Config, IrohPaths, CONFIG_FILE_NAME, ENV_PREFIX}; +use crate::config::{IrohPaths, NodeConfig}; use anyhow::Context; use clap::Subcommand; @@ -233,7 +232,11 @@ async fn send_blocks( Ok(()) } -async fn report(stun_host: Option, stun_port: u16, config: &Config) -> anyhow::Result<()> { +async fn report( + stun_host: Option, + stun_port: u16, + config: &NodeConfig, +) -> anyhow::Result<()> { let port_mapper = portmapper::Client::default().await; let mut client = netcheck::Client::new(Some(port_mapper)).await?; @@ -653,7 +656,7 @@ async fn port_map_probe(config: portmapper::Config) -> anyhow::Result<()> { Ok(()) } -async fn derp_regions(config: Config) -> anyhow::Result<()> { +async fn derp_regions(config: NodeConfig) -> anyhow::Result<()> { let key = SecretKey::generate(); let mut set = tokio::task::JoinSet::new(); if config.derp_regions.is_empty() { @@ -789,7 +792,7 @@ fn create_secret_key(secret_key: SecretKeyOption) -> anyhow::Result { }) } -pub async fn run(command: Commands, config: &Config) -> anyhow::Result<()> { +pub async fn run(command: Commands, config: &NodeConfig) -> anyhow::Result<()> { match command { Commands::Report { stun_host, @@ -844,19 +847,7 @@ pub async fn run(command: Commands, config: &Config) -> anyhow::Result<()> { port_map_probe(config).await } Commands::DerpRegions => { - let default_config_path = - iroh_config_path(CONFIG_FILE_NAME).context("invalid config path")?; - - let sources = [Some(default_config_path.as_path())]; - let config = Config::load( - // potential config files - &sources, - // env var prefix for this config - ENV_PREFIX, - // map of present command line arguments - // args.make_overrides_map(), - HashMap::::new(), - )?; + let config = NodeConfig::from_env(None)?; derp_regions(config).await } } diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index c97e31bd18..692bde2d59 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -1,45 +1,39 @@ -use ansi_term::Colour; +use ansi_term::{Colour, Style}; use anyhow::Result; use clap::{Parser, Subcommand}; -use iroh::client::quic::{Iroh, RpcClient}; +use iroh::client::quic::RpcClient; use iroh_gossip::proto::util::base32; use iroh_sync::sync::{AuthorId, NamespaceId}; use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; -use super::sync::SyncEnv; - -pub async fn run(client: RpcClient, env: SyncEnv) -> Result<()> { - println!("Welcome to the Iroh console!"); - println!("Type `help` for a list of commands."); - let mut state = ReplState::with_env(env); - let mut repl_rx = Repl::spawn(state.clone()); +use crate::config::{ConsoleEnv, ConsolePaths}; + +pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { + println!( + "{}", + Colour::Purple.bold().paint("Welcome to the Iroh console!") + ); + println!( + "Type `{}` for a list of commands.", + Style::new().bold().paint("help") + ); + let mut repl_rx = Repl::spawn(ReplState::with_env(env.clone())); while let Some((event, reply)) = repl_rx.recv().await { let (next, res) = match event { - FromRepl::DocCmd { id, cmd, env } => { - let iroh = Iroh::new(client.clone()); - let res = cmd.run(&iroh, id, env).await; + ReplCmd::Rpc(cmd) => { + let res = cmd.run(client.clone(), env.clone()).await; (ToRepl::Continue, res) } - FromRepl::RpcCmd { cmd } => { - let res = cmd.run(client.clone()).await; - (ToRepl::Continue, res) + ReplCmd::SetDoc { id } => { + env.set_doc(id); + (ToRepl::UpdateEnv(env.clone()), Ok(())) } - FromRepl::ReplCmd { cmd } => match cmd { - ReplCmd::SetDoc { id } => { - state.pwd = Pwd::Doc { id }; - (ToRepl::UpdateState(state.clone()), Ok(())) - } - ReplCmd::SetAuthor { id } => { - let res = state.env.set_author(id).await; - (ToRepl::UpdateState(state.clone()), res) - } - ReplCmd::Close => { - state.pwd = Pwd::Home; - (ToRepl::UpdateState(state.clone()), Ok(())) - } - ReplCmd::Exit => (ToRepl::Exit, Ok(())), - }, + ReplCmd::SetAuthor { id } => { + let res = env.save_author(id); + (ToRepl::UpdateEnv(env.clone()), res) + } + ReplCmd::Exit => (ToRepl::Exit, Ok(())), }; if let Err(err) = res { @@ -55,38 +49,23 @@ pub async fn run(client: RpcClient, env: SyncEnv) -> Result<()> { Ok(()) } -#[derive(Debug)] -pub enum FromRepl { - RpcCmd { - cmd: super::RpcCommands, - }, - DocCmd { - id: NamespaceId, - env: SyncEnv, - cmd: super::sync::Doc, - }, - ReplCmd { - cmd: ReplCmd, - }, -} - /// Reply to the repl after a command completed #[derive(Debug)] pub enum ToRepl { /// Continue execution by reading the next command Continue, /// Continue execution by reading the next command, and update the repl state - UpdateState(ReplState), + UpdateEnv(ConsoleEnv), /// Exit the repl Exit, } pub struct Repl { state: ReplState, - cmd_tx: mpsc::Sender<(FromRepl, oneshot::Sender)>, + cmd_tx: mpsc::Sender<(ReplCmd, oneshot::Sender)>, } impl Repl { - pub fn spawn(state: ReplState) -> mpsc::Receiver<(FromRepl, oneshot::Sender)> { + pub fn spawn(state: ReplState) -> mpsc::Receiver<(ReplCmd, oneshot::Sender)> { let (cmd_tx, cmd_rx) = mpsc::channel(1); let repl = Repl { state, cmd_tx }; std::thread::spawn(move || { @@ -99,7 +78,8 @@ impl Repl { pub fn run(mut self) -> anyhow::Result<()> { let mut rl = DefaultEditor::with_config(Config::builder().check_cursor_position(true).build())?; - rl.load_history(&self.state.env.repl_history_path).ok(); + let history_path = ConsolePaths::History.with_env()?; + rl.load_history(&history_path).ok(); loop { // prepare a channel to receive a signal from the main thread when a command completed let (to_repl_tx, to_repl_rx) = oneshot::channel(); @@ -108,7 +88,7 @@ impl Repl { Ok(line) if line.is_empty() => continue, Ok(line) => { rl.add_history_entry(line.as_str())?; - let cmd = self.state.handle_command(&line); + let cmd = parse_cmd::(&line); if let Some(cmd) = cmd { self.cmd_tx.blocking_send((cmd, to_repl_tx))?; } else { @@ -116,124 +96,74 @@ impl Repl { } } Err(ReadlineError::Interrupted | ReadlineError::Eof) => { - self.cmd_tx - .blocking_send((FromRepl::ReplCmd { cmd: ReplCmd::Exit }, to_repl_tx))?; + self.cmd_tx.blocking_send((ReplCmd::Exit, to_repl_tx))?; } Err(ReadlineError::WindowResized) => continue, Err(err) => return Err(err.into()), } // wait for reply from main thread match to_repl_rx.blocking_recv()? { - ToRepl::UpdateState(state) => { - self.state = state; + ToRepl::UpdateEnv(env) => { + self.state.env = env; continue; } ToRepl::Continue => continue, ToRepl::Exit => break, } } - rl.save_history(&self.state.env.repl_history_path).ok(); + rl.save_history(&history_path).ok(); Ok(()) } } #[derive(Debug, Clone)] pub struct ReplState { - pwd: Pwd, - env: SyncEnv, + env: ConsoleEnv, } impl ReplState { - fn with_env(env: SyncEnv) -> Self { - Self { - pwd: Pwd::Home, - env, - } + fn with_env(env: ConsoleEnv) -> Self { + Self { env } } } -#[derive(Debug, Clone)] -enum Pwd { - Home, - Doc { id: NamespaceId }, -} - impl ReplState { pub fn prompt(&self) -> String { let bang = Colour::Blue.paint("> "); - let pwd = match self.pwd { - Pwd::Home => None, - Pwd::Doc { id } => { - let author = match self.env.author().as_ref() { - Some(author) => fmt_short(author.as_bytes()), - None => "".to_string(), - }; - let pwd = format!("doc:{} author:{}", fmt_short(id.as_bytes()), author); - let pwd = Colour::Blue.paint(pwd); - Some(pwd.to_string()) - } - }; - let pwd = pwd.map(|pwd| format!("{}\n", pwd)).unwrap_or_default(); - format!("\n{pwd}{bang}") - } - - pub fn handle_command(&mut self, line: &str) -> Option { - match self.pwd { - Pwd::Home => match parse_cmd::(line)? { - HomeCommands::Repl(cmd) => Some(FromRepl::ReplCmd { cmd }), - HomeCommands::Rpc(cmd) => Some(FromRepl::RpcCmd { cmd }), - }, - Pwd::Doc { id } => match parse_cmd::(line)? { - DocCommands::Repl(cmd) => Some(FromRepl::ReplCmd { cmd }), - DocCommands::Sync(cmd) => Some(FromRepl::RpcCmd { - cmd: super::RpcCommands::Sync(cmd), - }), - DocCommands::Doc(cmd) => Some(FromRepl::DocCmd { - id, - cmd, - env: self.env.clone(), - }), - }, + let mut pwd = String::new(); + if let Some(doc) = &self.env.doc { + pwd.push_str(&format!( + "doc:{} ", + Style::new().bold().paint(fmt_short(doc.as_bytes())) + )); } + if let Some(author) = &self.env.author { + pwd.push_str(&format!( + "author:{} ", + Style::new().bold().paint(fmt_short(author.as_bytes())) + )); + } + if !pwd.is_empty() { + pwd = format!("{}\n", Colour::Blue.paint(pwd)); + } + format!("\n{pwd}{bang}") } } #[derive(Debug, Parser)] pub enum ReplCmd { - /// Open a document + /// Set the active document #[clap(next_help_heading = "foo")] SetDoc { id: NamespaceId }, - /// Set the active author for doc insertion + /// Set the active author SetAuthor { id: AuthorId }, - /// Close the open document - Close, - + #[clap(flatten)] + Rpc(#[clap(subcommand)] super::RpcCommands), /// Quit the Iroh console #[clap(alias = "quit")] Exit, } -#[derive(Debug, Parser)] -pub enum DocCommands { - #[clap(flatten)] - Doc(#[clap(subcommand)] super::sync::Doc), - #[clap(flatten)] - Repl(#[clap(subcommand)] ReplCmd), - // TODO: We can't embed RpcCommand here atm because there'd be a conflict between - // `list` top level and `list` doc command - // Thus for now only embedding sync commands - #[clap(flatten)] - Sync(#[clap(subcommand)] super::sync::Commands), -} - -#[derive(Debug, Parser)] -pub enum HomeCommands { - #[clap(flatten)] - Repl(#[clap(subcommand)] ReplCmd), - #[clap(flatten)] - Rpc(#[clap(subcommand)] super::RpcCommands), -} - fn try_parse_cmd(s: &str) -> anyhow::Result { let args = shell_words::split(s)?; let cmd = clap::Command::new("repl"); @@ -258,6 +188,7 @@ fn parse_cmd(s: &str) -> Option { fn fmt_short(bytes: impl AsRef<[u8]>) -> String { let bytes = bytes.as_ref(); + // we use 5 bytes because this always results in 8 character string in base32 let len = bytes.len().min(5); base32::fmt(&bytes[..len]) } diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index c983628e7d..01b8b83ffa 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -1,7 +1,5 @@ -use std::{path::PathBuf, str::FromStr}; - -use anyhow::{anyhow, Result}; -use clap::Parser; +use anyhow::Result; +use clap::{Parser, Subcommand}; use futures::TryStreamExt; use indicatif::HumanBytes; use iroh::{ @@ -13,159 +11,67 @@ use iroh_sync::{ store::GetFilter, sync::{AuthorId, NamespaceId, SignedEntry}, }; -use tokio::fs; -use crate::config::{ConsolePaths, IrohPaths}; +use crate::config::ConsoleEnv; use super::RpcClient; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; -#[derive(Debug, Clone, Default)] -pub struct SyncEnv { - pub dir: PathBuf, - pub author: Option, - pub repl_history_path: PathBuf, -} - -impl SyncEnv { - pub async fn load_from_env(_client: &Iroh) -> Result { - let dir: PathBuf = IrohPaths::Console.with_env()?; - if std::fs::metadata(&dir).is_err() { - std::fs::create_dir_all(&dir)?; - }; - let repl_history_path = ConsolePaths::History.with_root(&dir); - let author_path = ConsolePaths::DefaultAuthor.with_root(&dir); - let author = match fs::read(author_path).await { - Ok(s) => Some(AuthorId::from_str(&String::from_utf8(s)?)?), - Err(_err) => None, - }; - let slf = Self { - dir, - author, - repl_history_path, - }; - Ok(slf) - } - - async fn persist(&self) -> Result<()> { - if let Some(author) = &self.author { - let author_path = ConsolePaths::DefaultAuthor.with_root(&self.dir); - fs::write(author_path, format!("{}", author)).await?; - } - Ok(()) - } - - pub async fn set_author(&mut self, author: AuthorId) -> Result<()> { - self.author = Some(author); - self.persist().await - } - - pub fn author(&self) -> &Option { - &self.author - } -} - -#[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, Parser)] +#[derive(Subcommand, Debug, Clone)] pub enum Commands { - /// Manage document authors - Author { - #[clap(subcommand)] - command: Author, - }, /// Manage documents - Docs { + Doc { #[clap(subcommand)] - command: Docs, + command: DocCommands, }, - /// Manage a single document - Doc { - id: NamespaceId, + + /// Manage document authors + Author { #[clap(subcommand)] - command: Doc, + command: AuthorCommands, }, } impl Commands { - pub async fn run(self, client: RpcClient, env: SyncEnv) -> Result<()> { + pub async fn run(self, client: RpcClient, env: ConsoleEnv) -> Result<()> { let iroh = Iroh::new(client); match self { - Commands::Author { command } => command.run(&iroh).await, - Commands::Docs { command } => command.run(&iroh).await, - Commands::Doc { command, id } => command.run(&iroh, id, env).await, - } - } -} - -#[derive(Debug, Clone, Parser)] -pub enum Author { - /// List authors - #[clap(alias = "ls")] - List, - /// Create a new author - Create, -} - -impl Author { - pub async fn run(self, iroh: &Iroh) -> Result<()> { - match self { - Author::List => { - let mut stream = iroh.list_authors().await?; - while let Some(author_id) = stream.try_next().await? { - println!("{}", author_id); - } - } - Author::Create => { - let author_id = iroh.create_author().await?; - println!("{}", author_id); - } + Self::Doc { command } => command.run(&iroh, env).await, + Self::Author { command } => command.run(&iroh).await, } - Ok(()) } } +#[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Parser)] -pub enum Docs { - /// List documents - #[clap(alias = "ls")] - List, +pub enum DocCommands { /// Create a new document - Create, + Init, /// Import a document from peers Import { ticket: DocTicket }, -} - -impl Docs { - pub async fn run(self, iroh: &Iroh) -> Result<()> { - match self { - Docs::Create => { - let doc = iroh.create_doc().await?; - println!("created {}", doc.id()); - } - Docs::Import { ticket } => { - let doc = iroh.import_doc(ticket).await?; - println!("imported {}", doc.id()); - } - Docs::List => { - let mut stream = iroh.list_docs().await?; - while let Some(id) = stream.try_next().await? { - println!("{}", id) - } - } - } - Ok(()) - } -} - -#[derive(Debug, Clone, Parser)] -pub enum Doc { + /// List documents + List, /// Start to synchronize a document with peers - StartSync { peers: Vec }, + Sync { + /// Set the document + #[clap(short, long)] + doc_id: Option, + + peers: Vec, + }, /// Share a document and print a ticket to share with peers - Share { mode: ShareMode }, + Share { + /// Set the document + #[clap(short, long)] + doc_id: Option, + mode: ShareMode, + }, /// Set an entry Set { + /// Set the document + #[clap(short, long)] + doc_id: Option, /// Author of this entry. /// /// Required unless the author is set through the REPL environment or the IROH_AUTHOR @@ -181,6 +87,9 @@ pub enum Doc { /// /// Shows the author, content hash and content length for all entries for this key. Get { + /// Set the document + #[clap(short, long)] + doc_id: Option, /// Key to the entry (parsed as UTF-8 string). key: String, /// If true, get all entries that start with KEY. @@ -200,7 +109,10 @@ pub enum Doc { }, /// List all entries in the document #[clap(alias = "ls")] - List { + Keys { + /// Set the document + #[clap(short, long)] + doc_id: Option, /// If true, old entries will be included. By default only the latest value for each key is /// shown. #[clap(short, long)] @@ -210,44 +122,55 @@ pub enum Doc { }, } -impl Doc { - pub async fn run(self, iroh: &Iroh, doc_id: NamespaceId, env: SyncEnv) -> Result<()> { - let doc = iroh.get_doc(doc_id)?; +impl DocCommands { + pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { - Doc::StartSync { peers } => { + Self::Init => { + let doc = iroh.create_doc().await?; + println!("{}", doc.id()); + } + Self::Import { ticket } => { + let doc = iroh.import_doc(ticket).await?; + println!("{}", doc.id()); + } + Self::List => { + let mut stream = iroh.list_docs().await?; + while let Some(id) = stream.try_next().await? { + println!("{}", id) + } + } + Self::Sync { doc_id, peers } => { + let doc = iroh.get_doc(env.doc(doc_id)?)?; doc.start_sync(peers).await?; println!("ok"); } - Doc::Share { mode } => { + Self::Share { doc_id, mode } => { + let doc = iroh.get_doc(env.doc(doc_id)?)?; let ticket = doc.share(mode).await?; - // println!("key: {}", hex::encode(ticket.key)); - // println!( - // "peers: {}", - // ticket - // .peers - // .iter() - // .map(|p| p.to_string()) - // .collect::>() - // .join(", ") - // ); - println!("ticket: {}", ticket); + println!("{}", ticket); } - Doc::Set { author, key, value } => { + Self::Set { + doc_id, + author, + key, + value, + } => { + let doc = iroh.get_doc(env.doc(doc_id)?)?; + let author = env.author(author)?; let key = key.as_bytes().to_vec(); let value = value.as_bytes().to_vec(); - let author = author - .or_else(|| *env.author()) - .ok_or_else(|| anyhow!("No author provided"))?; let entry = doc.set_bytes(author, key, value).await?; println!("{}", fmt_entry(&entry)); } - Doc::Get { + Self::Get { + doc_id, key, prefix, author, old, content, } => { + let doc = iroh.get_doc(env.doc(doc_id)?)?; let mut filter = match old { true => GetFilter::all(), false => GetFilter::latest(), @@ -282,7 +205,12 @@ impl Doc { println!(); } } - Doc::List { old, prefix } => { + Self::Keys { + doc_id, + old, + prefix, + } => { + let doc = iroh.get_doc(env.doc(doc_id)?)?; let filter = match old { true => GetFilter::all(), false => GetFilter::latest(), @@ -301,6 +229,33 @@ impl Doc { } } +#[derive(Debug, Clone, Parser)] +pub enum AuthorCommands { + /// Create a new author + Create, + /// List authors + #[clap(alias = "ls")] + List, +} + +impl AuthorCommands { + pub async fn run(self, iroh: &Iroh) -> Result<()> { + match self { + AuthorCommands::List => { + let mut stream = iroh.list_authors().await?; + while let Some(author_id) = stream.try_next().await? { + println!("{}", author_id); + } + } + AuthorCommands::Create => { + let author_id = iroh.create_author().await?; + println!("{}", author_id); + } + } + Ok(()) + } +} + fn fmt_entry(entry: &SignedEntry) -> String { let id = entry.entry().id(); let key = std::str::from_utf8(id.key()).unwrap_or(""); diff --git a/iroh/src/config.rs b/iroh/src/config.rs index 495a3e8e06..0f390c640c 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -13,18 +13,22 @@ use iroh_net::{ defaults::{default_eu_derp_region, default_na_derp_region}, derp::{DerpMap, DerpRegion}, }; -use serde::{Deserialize, Serialize}; +use iroh_sync::{AuthorId, NamespaceId}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tracing::debug; /// CONFIG_FILE_NAME is the name of the optional config file located in the iroh home directory pub const CONFIG_FILE_NAME: &str = "iroh.config.toml"; + /// ENV_PREFIX should be used along side the config field name to set a config field using /// environment variables /// For example, `IROH_PATH=/path/to/config` would set the value of the `Config.path` field pub const ENV_PREFIX: &str = "IROH"; +const ENV_AUTHOR: &str = "AUTHOR"; +const ENV_DOC: &str = "DOC"; + /// Fetches the environment variable `IROH_` from the current process. -#[allow(dead_code)] pub fn env_var(key: &str) -> std::result::Result { env::var(&format!("{ENV_PREFIX}_{key}")) } @@ -40,7 +44,7 @@ pub enum IrohPaths { BaoFlatStorePartial, /// Path to the [iroh-sync document database](iroh_sync::store::fs::Store) DocsDatabase, - /// Path to a directory with the [`ConsolePaths`] + /// Path to the console state Console, } @@ -140,19 +144,36 @@ impl ConsolePaths { PathBuf::from(root.as_ref()).join(self) } pub fn with_env(self) -> Result { + Self::ensure_env_dir()?; Ok(self.with_root(IrohPaths::Console.with_env()?)) } + pub fn ensure_env_dir() -> Result<()> { + let p = IrohPaths::Console.with_env()?; + match std::fs::metadata(&p) { + Ok(meta) => match meta.is_dir() { + true => Ok(()), + false => Err(anyhow!(format!( + "Expected directory but found file at `{}`", + p.to_string_lossy() + ))), + }, + Err(_) => { + std::fs::create_dir_all(&p)?; + Ok(()) + } + } + } } -/// The configuration for the iroh cli. +/// The configuration for an iroh node. #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] #[serde(default)] -pub struct Config { +pub struct NodeConfig { /// The regions for DERP to use. pub derp_regions: Vec, } -impl Default for Config { +impl Default for NodeConfig { fn default() -> Self { Self { // TODO(ramfox): this should probably just be a derp map @@ -161,14 +182,14 @@ impl Default for Config { } } -impl Config { +impl NodeConfig { /// Make a config from the default environment variables. /// /// Optionally provide an additional configuration source. pub fn from_env(additional_config_source: Option<&Path>) -> anyhow::Result { let config_path = iroh_config_path(CONFIG_FILE_NAME).context("invalid config path")?; let sources = [Some(config_path.as_path()), additional_config_source]; - let config = Config::load( + let config = load_config( // potential config files &sources, // env var prefix for this config @@ -179,63 +200,151 @@ impl Config { )?; Ok(config) } - /// Make a config using a default, files, environment variables, and commandline flags. - /// - /// Later items in the *file_paths* slice will have a higher priority than earlier ones. - /// - /// Environment variables are expected to start with the *env_prefix*. Nested fields can be - /// accessed using `.`, if your environment allows env vars with `.` - /// - /// Note: For the metrics configuration env vars, it is recommended to use the metrics - /// specific prefix `IROH_METRICS` to set a field in the metrics config. You can use the - /// above dot notation to set a metrics field, eg, `IROH_CONFIG_METRICS.SERVICE_NAME`, but - /// only if your environment allows it - pub fn load( - file_paths: &[Option<&Path>], - env_prefix: &str, - flag_overrides: HashMap, - ) -> Result - where - S: AsRef, - V: Into, - { - let mut builder = config::Config::builder(); - - // layer on config options from files - for path in file_paths.iter().flatten() { - if path.exists() { - let p = path.to_str().ok_or_else(|| anyhow::anyhow!("empty path"))?; - builder = builder.add_source(File::with_name(p)); - } + /// Constructs a `DerpMap` based on the current configuration. + pub fn derp_map(&self) -> Option { + if self.derp_regions.is_empty() { + return None; } - // next, add any environment variables - builder = builder.add_source( - Environment::with_prefix(env_prefix) - .separator("__") - .try_parsing(true), - ); + let dm: DerpMap = self.derp_regions.iter().cloned().into(); + Some(dm) + } +} + +#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] +pub struct ConsoleEnv { + pub author: Option, + pub doc: Option, +} +impl ConsoleEnv { + /// Read from environment variables and the console config file. + pub fn from_env_and_peristent_state() -> Result { + let author = match Self::read_author_from_persistent_state()? { + Some(author) => Some(author), + None => env_author()?, + }; + Ok(Self { + author, + doc: env_doc()?, + }) + } - // finally, override any values - for (flag, val) in flag_overrides.into_iter() { - builder = builder.set_override(flag, val)?; + fn read_author_from_persistent_state() -> anyhow::Result> { + let author_path = ConsolePaths::DefaultAuthor.with_env()?; + if let Ok(s) = std::fs::read(&author_path) { + let author = String::from_utf8(s) + .map_err(Into::into) + .and_then(|s| AuthorId::from_str(&s)) + .with_context(|| { + format!( + "Failed to parse author file at {}", + author_path.to_string_lossy() + ) + })?; + Ok(Some(author)) + } else { + Ok(None) } + } - let cfg = builder.build()?; - debug!("make_config:\n{:#?}\n", cfg); - let cfg = cfg.try_deserialize()?; - Ok(cfg) + /// Read only from environment variables. + pub fn from_env() -> Result { + Ok(Self { + author: env_author()?, + doc: env_doc()?, + }) } - /// Constructs a `DerpMap` based on the current configuration. - pub fn derp_map(&self) -> Option { - if self.derp_regions.is_empty() { - return None; + pub fn save_author(&mut self, author: AuthorId) -> anyhow::Result<()> { + self.author = Some(author); + std::fs::write( + ConsolePaths::DefaultAuthor.with_env()?, + author.to_string().as_bytes(), + )?; + Ok(()) + } + + pub fn set_doc(&mut self, doc: NamespaceId) { + self.doc = Some(doc); + } + + pub fn doc(&self, arg: Option) -> anyhow::Result { + let doc_id = arg.or(self.doc).ok_or_else(|| { + anyhow!("Missing document id. Set the current document with the `IROH_DOC` environment variable or by passing the `-d` flag. In the console, you can set the active document with `set-doc`.") + })?; + Ok(doc_id) + } + + pub fn author(&self, arg: Option) -> anyhow::Result { + let author_id = arg.or(self.author).ok_or_else(|| { + anyhow!("Missing author id. Set the current author with the `IROH_AUTHOR` environment variable or by passing the `-a` flag. In the console, you can set the active author with `set-author`.") + +})?; + Ok(author_id) + } +} + +pub fn env_author() -> Result> { + match env_var(ENV_AUTHOR) { + Ok(s) => Ok(Some(AuthorId::from_str(&s)?)), + Err(_) => Ok(None), + } +} + +pub fn env_doc() -> Result> { + match env_var(ENV_DOC) { + Ok(s) => Ok(Some(NamespaceId::from_str(&s)?)), + Err(_) => Ok(None), + } +} + +/// Make a config using a default, files, environment variables, and commandline flags. +/// +/// Later items in the *file_paths* slice will have a higher priority than earlier ones. +/// +/// Environment variables are expected to start with the *env_prefix*. Nested fields can be +/// accessed using `.`, if your environment allows env vars with `.` +/// +/// Note: For the metrics configuration env vars, it is recommended to use the metrics +/// specific prefix `IROH_METRICS` to set a field in the metrics config. You can use the +/// above dot notation to set a metrics field, eg, `IROH_CONFIG_METRICS.SERVICE_NAME`, but +/// only if your environment allows it +pub fn load_config( + file_paths: &[Option<&Path>], + env_prefix: &str, + flag_overrides: HashMap, +) -> Result +where + C: DeserializeOwned, + S: AsRef, + V: Into, +{ + let mut builder = config::Config::builder(); + + // layer on config options from files + for path in file_paths.iter().flatten() { + if path.exists() { + let p = path.to_str().ok_or_else(|| anyhow::anyhow!("empty path"))?; + builder = builder.add_source(File::with_name(p)); } + } - let dm: DerpMap = self.derp_regions.iter().cloned().into(); - Some(dm) + // next, add any environment variables + builder = builder.add_source( + Environment::with_prefix(env_prefix) + .separator("__") + .try_parsing(true), + ); + + // finally, override any values + for (flag, val) in flag_overrides.into_iter() { + builder = builder.set_override(flag, val)?; } + + let cfg = builder.build()?; + debug!("make_config:\n{:#?}\n", cfg); + let cfg = cfg.try_deserialize()?; + Ok(cfg) } /// Name of directory that wraps all iroh files in a given application directory @@ -330,7 +439,8 @@ mod tests { #[test] fn test_default_settings() { - let config = Config::load::(&[][..], "__FOO", Default::default()).unwrap(); + let config: NodeConfig = + load_config(&[][..], "__FOO", HashMap::::new()).unwrap(); assert_eq!(config.derp_regions.len(), 2); } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 2e1c03fc5c..086c75ade3 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -57,12 +57,11 @@ use tracing::{debug, trace}; use crate::dial::Ticket; use crate::download::Downloader; use crate::rpc_protocol::{ - AddrsRequest, AddrsResponse, BytesGetRequest, BytesGetResponse, IdRequest, IdResponse, - ListBlobsRequest, ListBlobsResponse, ListCollectionsRequest, ListCollectionsResponse, - ListIncompleteBlobsRequest, ListIncompleteBlobsResponse, ProvideRequest, ProviderRequest, - ProviderResponse, ProviderService, ShareRequest, ShutdownRequest, StatsGetRequest, - StatsGetResponse, ValidateRequest, VersionRequest, VersionResponse, WatchRequest, - WatchResponse, + BytesGetRequest, BytesGetResponse, ListBlobsRequest, ListBlobsResponse, ListCollectionsRequest, + ListCollectionsResponse, ListIncompleteBlobsRequest, ListIncompleteBlobsResponse, + ProvideRequest, ProviderRequest, ProviderResponse, ProviderService, ShareRequest, + ShutdownRequest, StatsGetRequest, StatsGetResponse, StatusRequest, StatusResponse, + ValidateRequest, VersionRequest, VersionResponse, WatchRequest, WatchResponse, }; use crate::sync::{SyncEngine, SYNC_ALPN}; @@ -1077,8 +1076,8 @@ impl RpcHandler { version: env!("CARGO_PKG_VERSION").to_string(), } } - async fn id(self, _: IdRequest) -> IdResponse { - IdResponse { + async fn status(self, _: StatusRequest) -> StatusResponse { + StatusResponse { peer_id: Box::new(self.inner.secret_key.public()), listen_addrs: self .inner @@ -1088,15 +1087,6 @@ impl RpcHandler { version: env!("CARGO_PKG_VERSION").to_string(), } } - async fn addrs(self, _: AddrsRequest) -> AddrsResponse { - AddrsResponse { - addrs: self - .inner - .local_endpoint_addresses() - .await - .unwrap_or_default(), - } - } async fn shutdown(self, request: ShutdownRequest) { if request.force { tracing::info!("hard shutdown requested"); @@ -1184,8 +1174,7 @@ fn handle_rpc_request< Share(msg) => chan.server_streaming(msg, handler, RpcHandler::share).await, Watch(msg) => chan.server_streaming(msg, handler, RpcHandler::watch).await, Version(msg) => chan.rpc(msg, handler, RpcHandler::version).await, - Id(msg) => chan.rpc(msg, handler, RpcHandler::id).await, - Addrs(msg) => chan.rpc(msg, handler, RpcHandler::addrs).await, + Status(msg) => chan.rpc(msg, handler, RpcHandler::status).await, Shutdown(msg) => chan.rpc(msg, handler, RpcHandler::shutdown).await, Stats(msg) => chan.rpc(msg, handler, RpcHandler::stats).await, Validate(msg) => { diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index e45501f699..083530fcb3 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -215,18 +215,10 @@ impl RpcMsg for ShutdownRequest { /// /// See [`IdResponse`] for the response. #[derive(Serialize, Deserialize, Debug)] -pub struct IdRequest; +pub struct StatusRequest; -impl RpcMsg for IdRequest { - type Response = IdResponse; -} - -/// A request to get the addresses of the node -#[derive(Serialize, Deserialize, Debug)] -pub struct AddrsRequest; - -impl RpcMsg for AddrsRequest { - type Response = AddrsResponse; +impl RpcMsg for StatusRequest { + type Response = StatusResponse; } /// The response to a watch request @@ -238,7 +230,7 @@ pub struct WatchResponse { /// The response to a version request #[derive(Serialize, Deserialize, Debug)] -pub struct IdResponse { +pub struct StatusResponse { /// The peer id of the node pub peer_id: Box, /// The addresses of the node @@ -247,13 +239,6 @@ pub struct IdResponse { pub version: String, } -/// The response to an addrs request -#[derive(Serialize, Deserialize, Debug)] -pub struct AddrsResponse { - /// The addresses of the node - pub addrs: Vec, -} - impl Msg for WatchRequest { type Pattern = ServerStreaming; } @@ -611,8 +596,7 @@ pub enum ProviderRequest { ListCollections(ListCollectionsRequest), Provide(ProvideRequest), Share(ShareRequest), - Id(IdRequest), - Addrs(AddrsRequest), + Status(StatusRequest), Shutdown(ShutdownRequest), Validate(ValidateRequest), @@ -650,8 +634,7 @@ pub enum ProviderResponse { ListCollections(ListCollectionsResponse), Provide(ProvideProgress), Share(ShareProgress), - Id(IdResponse), - Addrs(AddrsResponse), + Status(StatusResponse), Validate(ValidateProgress), Shutdown(()), From 5434229e9559f2bbc1d83fcb0def532de137dda6 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 15:33:16 +0200 Subject: [PATCH 121/172] fix: prompt styling --- iroh/src/commands/repl.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 692bde2d59..952a520bc0 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -131,16 +131,18 @@ impl ReplState { pub fn prompt(&self) -> String { let bang = Colour::Blue.paint("> "); let mut pwd = String::new(); - if let Some(doc) = &self.env.doc { + if let Some(author) = &self.env.author { pwd.push_str(&format!( - "doc:{} ", - Style::new().bold().paint(fmt_short(doc.as_bytes())) + "{}{} ", + Colour::Blue.paint("author:"), + Colour::Blue.bold().paint(fmt_short(author.as_bytes())), )); } - if let Some(author) = &self.env.author { + if let Some(doc) = &self.env.doc { pwd.push_str(&format!( - "author:{} ", - Style::new().bold().paint(fmt_short(author.as_bytes())) + "{}{} ", + Colour::Blue.paint("doc:"), + Colour::Blue.bold().paint(fmt_short(doc.as_bytes())), )); } if !pwd.is_empty() { From ba15a054b8c7f779bb1dd5cc079f94cbbe28f9a2 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 15:46:37 +0200 Subject: [PATCH 122/172] doc: cli help text --- iroh/src/commands/sync.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 01b8b83ffa..3aa8819af3 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -54,7 +54,10 @@ pub enum DocCommands { List, /// Start to synchronize a document with peers Sync { - /// Set the document + /// Document to operate on. + /// + /// Required unless the document is set through the IROH_DOC environment variable. + /// Within the Iroh console, the active document can also set with `set-doc`. #[clap(short, long)] doc_id: Option, @@ -69,13 +72,16 @@ pub enum DocCommands { }, /// Set an entry Set { - /// Set the document + /// Document to operate on. + /// + /// Required unless the document is set through the IROH_DOC environment variable. + /// Within the Iroh console, the active document can also set with `set-doc`. #[clap(short, long)] doc_id: Option, /// Author of this entry. /// - /// Required unless the author is set through the REPL environment or the IROH_AUTHOR - /// environment variable. + /// Required unless the author is set through the IROH_AUTHOR environment variable. + /// Within the Iroh console, the active author can also set with `set-author`. #[clap(short, long)] author: Option, /// Key to the entry (parsed as UTF-8 string). @@ -87,7 +93,10 @@ pub enum DocCommands { /// /// Shows the author, content hash and content length for all entries for this key. Get { - /// Set the document + /// Document to operate on. + /// + /// Required unless the document is set through the IROH_DOC environment variable. + /// Within the Iroh console, the active document can also set with `set-doc`. #[clap(short, long)] doc_id: Option, /// Key to the entry (parsed as UTF-8 string). @@ -110,7 +119,10 @@ pub enum DocCommands { /// List all entries in the document #[clap(alias = "ls")] Keys { - /// Set the document + /// Document to operate on. + /// + /// Required unless the document is set through the IROH_DOC environment variable. + /// Within the Iroh console, the active document can also set with `set-doc`. #[clap(short, long)] doc_id: Option, /// If true, old entries will be included. By default only the latest value for each key is From bbdf7bfa4c03b137c3acb13797f01fe852dccb0f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 15:55:36 +0200 Subject: [PATCH 123/172] chore: clippy --- iroh-sync/src/keys.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh-sync/src/keys.rs b/iroh-sync/src/keys.rs index 8232217fbe..830e17be3e 100644 --- a/iroh-sync/src/keys.rs +++ b/iroh-sync/src/keys.rs @@ -149,13 +149,13 @@ impl NamespaceId { impl fmt::Display for Author { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(&self.to_bytes())) + write!(f, "{}", base32::fmt(self.to_bytes())) } } impl fmt::Display for Namespace { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", base32::fmt(&self.to_bytes())) + write!(f, "{}", base32::fmt(self.to_bytes())) } } From 7bb22fa80884f63ca9649eaaf3e6cdfd81fbdb60 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 16:00:09 +0200 Subject: [PATCH 124/172] fix: replace unmaintained ansi_term with colored --- Cargo.lock | 22 ++++++++++++---------- iroh/Cargo.toml | 6 +++--- iroh/src/commands/repl.rs | 31 ++++++++++--------------------- 3 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48be76ccc5..8847b48154 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,15 +75,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "anstream" version = "0.3.2" @@ -557,6 +548,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" +dependencies = [ + "is-terminal", + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "config" version = "0.13.3" @@ -1867,11 +1869,11 @@ checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" name = "iroh" version = "0.5.1" dependencies = [ - "ansi_term", "anyhow", "bao-tree", "bytes", "clap", + "colored", "config", "console", "data-encoding", diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index e5c41bce5e..a148b65e63 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -16,6 +16,7 @@ rust-version = "1.67" anyhow = { version = "1", features = ["backtrace"] } bao-tree = { version = "0.6.3", features = ["tokio_fsm"], default-features = false } bytes = "1" +data-encoding = "2.4.0" derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } flume = "0.10.14" futures = "0.3.25" @@ -46,7 +47,6 @@ tracing = "0.1" walkdir = "2" # CLI -ansi_term = { version = "0.12.1", optional = true } clap = { version = "4", features = ["derive"], optional = true } config = { version = "0.13.1", default-features = false, features = ["toml", "preserve_order"], optional = true } console = { version = "0.15.5", optional = true } @@ -59,15 +59,15 @@ shellexpand = { version = "3.1.0", optional = true } tempfile = { version = "3.4", optional = true } toml = { version = "0.7.3", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } -data-encoding = "2.4.0" url = { version = "2.4", features = ["serde"] } +colored = { version = "2.0.4", optional = true } # Examples ed25519-dalek = { version = "2.0.0", features = ["serde", "rand_core"], optional = true } [features] default = ["cli", "metrics"] -cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection", "shell-words", "shellexpand", "rustyline", "ansi_term", "toml"] +cli = ["clap", "config", "console", "dirs-next", "indicatif", "multibase", "quic-rpc/quinn-transport", "tempfile", "tokio/rt-multi-thread", "tracing-subscriber", "flat-db", "mem-db", "iroh-collection", "shell-words", "shellexpand", "rustyline", "colored", "toml"] metrics = ["iroh-metrics"] mem-db = [] flat-db = [] diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 952a520bc0..a6ccafcba4 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -1,6 +1,6 @@ -use ansi_term::{Colour, Style}; use anyhow::Result; use clap::{Parser, Subcommand}; +use colored::Colorize; use iroh::client::quic::RpcClient; use iroh_gossip::proto::util::base32; use iroh_sync::sync::{AuthorId, NamespaceId}; @@ -10,14 +10,8 @@ use tokio::sync::{mpsc, oneshot}; use crate::config::{ConsoleEnv, ConsolePaths}; pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { - println!( - "{}", - Colour::Purple.bold().paint("Welcome to the Iroh console!") - ); - println!( - "Type `{}` for a list of commands.", - Style::new().bold().paint("help") - ); + println!("{}", "Welcome to the Iroh console!".purple().bold()); + println!("Type `{}` for a list of commands.", "help".bold()); let mut repl_rx = Repl::spawn(ReplState::with_env(env.clone())); while let Some((event, reply)) = repl_rx.recv().await { let (next, res) = match event { @@ -37,11 +31,7 @@ pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { }; if let Err(err) = res { - println!( - "{} {:?}", - ansi_term::Colour::Red.bold().paint("Error:"), - err - ) + println!("{} {:?}", "Error:".red().bold(), err) } reply.send(next).ok(); @@ -129,26 +119,25 @@ impl ReplState { impl ReplState { pub fn prompt(&self) -> String { - let bang = Colour::Blue.paint("> "); let mut pwd = String::new(); if let Some(author) = &self.env.author { pwd.push_str(&format!( "{}{} ", - Colour::Blue.paint("author:"), - Colour::Blue.bold().paint(fmt_short(author.as_bytes())), + "author:".blue(), + fmt_short(author.as_bytes()).blue().bold(), )); } if let Some(doc) = &self.env.doc { pwd.push_str(&format!( "{}{} ", - Colour::Blue.paint("doc:"), - Colour::Blue.bold().paint(fmt_short(doc.as_bytes())), + "doc:".blue(), + fmt_short(doc.as_bytes()).blue().bold(), )); } if !pwd.is_empty() { - pwd = format!("{}\n", Colour::Blue.paint(pwd)); + pwd.push_str("\n"); } - format!("\n{pwd}{bang}") + format!("\n{pwd}{}", "> ".blue()) } } From dfbdb0f99f7d5bf67fc0b1f4af0b7bf875d2140d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 16:13:07 +0200 Subject: [PATCH 125/172] refactor: simplify repl --- iroh/src/commands/repl.rs | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index a6ccafcba4..1fe69a7290 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -12,7 +12,7 @@ use crate::config::{ConsoleEnv, ConsolePaths}; pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { println!("{}", "Welcome to the Iroh console!".purple().bold()); println!("Type `{}` for a list of commands.", "help".bold()); - let mut repl_rx = Repl::spawn(ReplState::with_env(env.clone())); + let mut repl_rx = Repl::spawn(env.clone()); while let Some((event, reply)) = repl_rx.recv().await { let (next, res) = match event { ReplCmd::Rpc(cmd) => { @@ -51,13 +51,13 @@ pub enum ToRepl { } pub struct Repl { - state: ReplState, + env: ConsoleEnv, cmd_tx: mpsc::Sender<(ReplCmd, oneshot::Sender)>, } impl Repl { - pub fn spawn(state: ReplState) -> mpsc::Receiver<(ReplCmd, oneshot::Sender)> { + pub fn spawn(env: ConsoleEnv) -> mpsc::Receiver<(ReplCmd, oneshot::Sender)> { let (cmd_tx, cmd_rx) = mpsc::channel(1); - let repl = Repl { state, cmd_tx }; + let repl = Repl { env, cmd_tx }; std::thread::spawn(move || { if let Err(err) = repl.run() { println!("> repl crashed: {err}"); @@ -72,52 +72,38 @@ impl Repl { rl.load_history(&history_path).ok(); loop { // prepare a channel to receive a signal from the main thread when a command completed - let (to_repl_tx, to_repl_rx) = oneshot::channel(); - let readline = rl.readline(&self.state.prompt()); + let (reply_tx, reply_rx) = oneshot::channel(); + let readline = rl.readline(&self.prompt()); match readline { Ok(line) if line.is_empty() => continue, Ok(line) => { rl.add_history_entry(line.as_str())?; let cmd = parse_cmd::(&line); if let Some(cmd) = cmd { - self.cmd_tx.blocking_send((cmd, to_repl_tx))?; + self.cmd_tx.blocking_send((cmd, reply_tx))?; } else { continue; } } Err(ReadlineError::Interrupted | ReadlineError::Eof) => { - self.cmd_tx.blocking_send((ReplCmd::Exit, to_repl_tx))?; + self.cmd_tx.blocking_send((ReplCmd::Exit, reply_tx))?; } Err(ReadlineError::WindowResized) => continue, Err(err) => return Err(err.into()), } // wait for reply from main thread - match to_repl_rx.blocking_recv()? { + match reply_rx.blocking_recv()? { ToRepl::UpdateEnv(env) => { - self.state.env = env; - continue; + self.env = env; } - ToRepl::Continue => continue, + ToRepl::Continue => {} ToRepl::Exit => break, } } rl.save_history(&history_path).ok(); Ok(()) } -} - -#[derive(Debug, Clone)] -pub struct ReplState { - env: ConsoleEnv, -} - -impl ReplState { - fn with_env(env: ConsoleEnv) -> Self { - Self { env } - } -} -impl ReplState { pub fn prompt(&self) -> String { let mut pwd = String::new(); if let Some(author) = &self.env.author { From 4f6953becab0979fc24a57159e12c285fd2c51fe Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 16:18:17 +0200 Subject: [PATCH 126/172] refactor: cli command names and help --- iroh/src/commands/sync.rs | 40 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 3aa8819af3..5fb84df978 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -46,31 +46,20 @@ impl Commands { #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Parser)] pub enum DocCommands { - /// Create a new document + /// Create a new document. Init, - /// Import a document from peers - Import { ticket: DocTicket }, - /// List documents + /// Join a document from a ticket. + Join { ticket: DocTicket }, + /// List documents. List, - /// Start to synchronize a document with peers - Sync { - /// Document to operate on. - /// - /// Required unless the document is set through the IROH_DOC environment variable. - /// Within the Iroh console, the active document can also set with `set-doc`. - #[clap(short, long)] - doc_id: Option, - - peers: Vec, - }, - /// Share a document and print a ticket to share with peers + /// Share a document with peers over a ticket. Share { /// Set the document #[clap(short, long)] doc_id: Option, mode: ShareMode, }, - /// Set an entry + /// Set an entry in a document. Set { /// Document to operate on. /// @@ -78,7 +67,7 @@ pub enum DocCommands { /// Within the Iroh console, the active document can also set with `set-doc`. #[clap(short, long)] doc_id: Option, - /// Author of this entry. + /// Author of the entry. /// /// Required unless the author is set through the IROH_AUTHOR environment variable. /// Within the Iroh console, the active author can also set with `set-author`. @@ -89,7 +78,7 @@ pub enum DocCommands { /// Content to store for this entry (parsed as UTF-8 string) value: String, }, - /// Get entries by key + /// Get entries in a document. /// /// Shows the author, content hash and content length for all entries for this key. Get { @@ -116,7 +105,7 @@ pub enum DocCommands { #[clap(short, long)] content: bool, }, - /// List all entries in the document + /// List all keys in a document. #[clap(alias = "ls")] Keys { /// Document to operate on. @@ -141,7 +130,7 @@ impl DocCommands { let doc = iroh.create_doc().await?; println!("{}", doc.id()); } - Self::Import { ticket } => { + Self::Join { ticket } => { let doc = iroh.import_doc(ticket).await?; println!("{}", doc.id()); } @@ -151,11 +140,6 @@ impl DocCommands { println!("{}", id) } } - Self::Sync { doc_id, peers } => { - let doc = iroh.get_doc(env.doc(doc_id)?)?; - doc.start_sync(peers).await?; - println!("ok"); - } Self::Share { doc_id, mode } => { let doc = iroh.get_doc(env.doc(doc_id)?)?; let ticket = doc.share(mode).await?; @@ -243,9 +227,9 @@ impl DocCommands { #[derive(Debug, Clone, Parser)] pub enum AuthorCommands { - /// Create a new author + /// Create a new author. Create, - /// List authors + /// List authors. #[clap(alias = "ls")] List, } From f10efa4c28c7da9b188c320582c3f675c8338adf Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 16:19:25 +0200 Subject: [PATCH 127/172] chore: fmt --- iroh/src/commands/sync.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 5fb84df978..b42f717bb1 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -5,7 +5,6 @@ use indicatif::HumanBytes; use iroh::{ client::quic::Iroh, rpc_protocol::{DocTicket, ShareMode}, - sync::PeerSource, }; use iroh_sync::{ store::GetFilter, From 0fe633eb2a273df6ddbf658ed84d03e27596e1c4 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 16:23:58 +0200 Subject: [PATCH 128/172] refactor: better method naming --- iroh/src/commands.rs | 4 ++-- iroh/src/config.rs | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 2b29f46093..ca8f29952a 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -72,12 +72,12 @@ impl Cli { match self.command { Commands::Console => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; - let env = ConsoleEnv::from_env_and_peristent_state()?; + let env = ConsoleEnv::console_env()?; repl::run(client, env).await } Commands::Rpc(command) => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; - let env = ConsoleEnv::from_env()?; + let env = ConsoleEnv::cli_env()?; command.run(client, env).await } Commands::Full(command) => { diff --git a/iroh/src/config.rs b/iroh/src/config.rs index 0f390c640c..28c384eabb 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -218,8 +218,8 @@ pub struct ConsoleEnv { } impl ConsoleEnv { /// Read from environment variables and the console config file. - pub fn from_env_and_peristent_state() -> Result { - let author = match Self::read_author_from_persistent_state()? { + pub fn console_env() -> Result { + let author = match Self::get_console_default_author()? { Some(author) => Some(author), None => env_author()?, }; @@ -229,7 +229,15 @@ impl ConsoleEnv { }) } - fn read_author_from_persistent_state() -> anyhow::Result> { + /// Read only from environment variables. + pub fn cli_env() -> Result { + Ok(Self { + author: env_author()?, + doc: env_doc()?, + }) + } + + fn get_console_default_author() -> anyhow::Result> { let author_path = ConsolePaths::DefaultAuthor.with_env()?; if let Ok(s) = std::fs::read(&author_path) { let author = String::from_utf8(s) @@ -247,14 +255,6 @@ impl ConsoleEnv { } } - /// Read only from environment variables. - pub fn from_env() -> Result { - Ok(Self { - author: env_author()?, - doc: env_doc()?, - }) - } - pub fn save_author(&mut self, author: AuthorId) -> anyhow::Result<()> { self.author = Some(author); std::fs::write( @@ -284,14 +284,14 @@ impl ConsoleEnv { } } -pub fn env_author() -> Result> { +fn env_author() -> Result> { match env_var(ENV_AUTHOR) { Ok(s) => Ok(Some(AuthorId::from_str(&s)?)), Err(_) => Ok(None), } } -pub fn env_doc() -> Result> { +fn env_doc() -> Result> { match env_var(ENV_DOC) { Ok(s) => Ok(Some(NamespaceId::from_str(&s)?)), Err(_) => Ok(None), From 3dba9ce1212cb8ffb27ce3d89c4366fb66fe894b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 16:35:14 +0200 Subject: [PATCH 129/172] tests: fix cli tests --- iroh/src/commands.rs | 2 +- iroh/tests/cli.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index ca8f29952a..bde0b622d7 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -314,7 +314,7 @@ impl NodeCommands { Self::Status {} => { let response = client.rpc(StatusRequest).await?; - println!("Listening address: {:#?}", response.listen_addrs); + println!("Listening addresses: {:#?}", response.listen_addrs); println!("PeerID: {}", response.peer_id); Ok(()) } diff --git a/iroh/tests/cli.rs b/iroh/tests/cli.rs index 9676b0ba88..eb4e94b232 100644 --- a/iroh/tests/cli.rs +++ b/iroh/tests/cli.rs @@ -332,7 +332,7 @@ fn cli_provide_persistence() -> anyhow::Result<()> { cmd( iroh_bin(), [ - "provide", + "start", "--addr", ADDR, "--rpc-port", @@ -396,7 +396,7 @@ fn cli_provide_addresses() -> Result<()> { let _ticket = match_provide_output(&mut provider, 1)?; // test output - let get_output = cmd(iroh_bin(), ["--rpc-port", RPC_PORT, "addresses"]) + let get_output = cmd(iroh_bin(), ["--rpc-port", RPC_PORT, "node", "status"]) // .stderr_file(std::io::stderr().as_raw_fd()) for debug output .stdout_capture() .run()?; @@ -460,7 +460,7 @@ fn make_provider_in( let res = cmd( iroh_bin(), [ - "provide", + "start", path.to_str().unwrap(), "--addr", addr.unwrap_or(ADDR), From f571631e24364640caf6291398634cb5cc989bbc Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 16:56:33 +0200 Subject: [PATCH 130/172] fix: docs for client, add stop_sync rpc call --- iroh-sync/Cargo.toml | 3 +++ iroh/src/client.rs | 37 +++++++++++++++++++++++++++++++------ iroh/src/node.rs | 6 ++++++ iroh/src/rpc_protocol.rs | 22 +++++++++++++++++++--- iroh/src/sync/rpc.rs | 11 +++++++++-- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index 8d229c788a..b02d4b1550 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -8,6 +8,9 @@ license = "MIT/Apache-2.0" authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" +# Sadly this also needs to be updated in .github/workflows/ci.yml +rust-version = "1.67" + [dependencies] anyhow = "1" blake3 = { package = "iroh-blake3", version = "1.4.3"} diff --git a/iroh/src/client.rs b/iroh/src/client.rs index c662369c25..70f62a8482 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -2,9 +2,6 @@ //! //! TODO: Contains only iroh sync related methods. Add other methods. -// TODO: fill out docs -#![allow(missing_docs)] - use std::collections::HashMap; use std::result::Result as StdResult; @@ -18,8 +15,9 @@ use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorListRequest, BytesGetRequest, CounterStats, DocGetRequest, - DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocSubscribeRequest, - DocTicket, DocsCreateRequest, DocsListRequest, ProviderService, ShareMode, StatsGetRequest, + DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocStopSyncRequest, + DocSubscribeRequest, DocTicket, DocsCreateRequest, DocsListRequest, ProviderService, ShareMode, + StatsGetRequest, }; use crate::sync::{LiveEvent, PeerSource}; @@ -37,19 +35,24 @@ impl Iroh where C: ServiceConnection, { + /// Create a new high-level client to a Iroh node from the low-level RPC client. pub fn new(rpc: RpcClient) -> Self { Self { rpc } } + + /// Create a new document author. pub async fn create_author(&self) -> Result { let res = self.rpc.rpc(AuthorCreateRequest).await??; Ok(res.author_id) } + /// List document authors for which we have a secret key. pub async fn list_authors(&self) -> Result>> { let stream = self.rpc.server_streaming(AuthorListRequest {}).await?; Ok(flatten(stream).map_ok(|res| res.author_id)) } + /// Create a new document. pub async fn create_doc(&self) -> Result> { let res = self.rpc.rpc(DocsCreateRequest {}).await??; let doc = Doc { @@ -59,6 +62,7 @@ where Ok(doc) } + /// Import a document from a ticket and join all peers in the ticket. pub async fn import_doc(&self, ticket: DocTicket) -> Result> { let res = self.rpc.rpc(DocImportRequest(ticket)).await??; let doc = Doc { @@ -68,11 +72,13 @@ where Ok(doc) } + /// List all documents. pub async fn list_docs(&self) -> Result>> { let stream = self.rpc.server_streaming(DocsListRequest {}).await?; Ok(flatten(stream).map_ok(|res| res.id)) } + /// Get a [`Doc`] client for a single document. pub fn get_doc(&self, id: NamespaceId) -> Result> { // TODO: Check if doc exists? let doc = Doc { @@ -82,12 +88,16 @@ where Ok(doc) } + /// Get the bytes for a hash. + /// + /// NOTE: This reads the full blob into memory. // TODO: add get_reader for streaming gets pub async fn get_bytes(&self, hash: Hash) -> Result { let res = self.rpc.rpc(BytesGetRequest { hash }).await??; Ok(res.data) } + /// Get statistics of the running node. pub async fn stats(&self) -> Result> { let res = self.rpc.rpc(StatsGetRequest {}).await??; Ok(res.stats) @@ -105,10 +115,12 @@ impl Doc where C: ServiceConnection, { + /// Get the document id of this doc. pub fn id(&self) -> NamespaceId { self.id } + /// Set the content of a key to a byte array. pub async fn set_bytes( &self, author_id: AuthorId, @@ -127,6 +139,7 @@ where Ok(res.entry) } + /// Get the contents of an entry as a byte array. // TODO: add get_content_reader pub async fn get_content_bytes(&self, entry: &SignedEntry) -> Result { let hash = *entry.content_hash(); @@ -134,6 +147,7 @@ where Ok(bytes.data) } + /// Get the latest entry for a key and author. pub async fn get_latest(&self, author_id: AuthorId, key: Vec) -> Result { let filter = GetFilter::latest().with_key(key).with_author(author_id); let mut stream = self.get(filter).await?; @@ -144,6 +158,7 @@ where Ok(entry) } + /// Get entries. pub async fn get(&self, filter: GetFilter) -> Result>> { let stream = self .rpc @@ -155,6 +170,7 @@ where Ok(flatten(stream).map_ok(|res| res.entry)) } + /// Share this document with peers over a ticket. pub async fn share(&self, mode: ShareMode) -> anyhow::Result { let res = self .rpc @@ -166,6 +182,7 @@ where Ok(res.0) } + /// Start to sync this document with a list of peers. pub async fn start_sync(&self, peers: Vec) -> Result<()> { let _res = self .rpc @@ -177,8 +194,16 @@ where Ok(()) } - // TODO: add stop_sync + /// Stop the live sync for this document. + pub async fn stop_sync(&self) -> Result<()> { + let _res = self + .rpc + .rpc(DocStopSyncRequest { doc_id: self.id }) + .await??; + Ok(()) + } + /// Subscribe to events for this document. pub async fn subscribe(&self) -> anyhow::Result>> { let stream = self .rpc diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 2e1c03fc5c..81acd8a4f0 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1245,6 +1245,12 @@ fn handle_rpc_request< }) .await } + DocStopSync(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_stop_sync(req).await + }) + .await + } DocShare(msg) => { chan.rpc(msg, handler, |handler, req| async move { handler.inner.sync.doc_share(req).await diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index e45501f699..862c667cbb 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -519,6 +519,20 @@ impl RpcMsg for DocStartSyncRequest { #[derive(Serialize, Deserialize, Debug)] pub struct DocStartSyncResponse {} +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocStopSyncRequest { + pub doc_id: NamespaceId, +} + +impl RpcMsg for DocStopSyncRequest { + type Response = RpcResult; +} + +/// todo +#[derive(Serialize, Deserialize, Debug)] +pub struct DocStopSyncResponse {} + /// todo #[derive(Serialize, Deserialize, Debug)] pub struct DocSetRequest { @@ -630,8 +644,9 @@ pub enum ProviderRequest { DocSet(DocSetRequest), DocGet(DocGetRequest), - DocStartSync(DocStartSyncRequest), // DocGetContent(DocGetContentRequest), - DocShare(DocShareRequest), // DocGetContent(DocGetContentRequest), + DocStartSync(DocStartSyncRequest), + DocStopSync(DocStopSyncRequest), + DocShare(DocShareRequest), DocSubscribe(DocSubscribeRequest), BytesGet(BytesGetRequest), @@ -671,8 +686,9 @@ pub enum ProviderResponse { DocSet(RpcResult), DocGet(RpcResult), - DocJoin(RpcResult), DocShare(RpcResult), + DocStartSync(RpcResult), + DocStopSync(RpcResult), DocSubscribe(DocSubscribeResponse), BytesGet(RpcResult), diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index 1a0c755707..b8437ea044 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -11,8 +11,8 @@ use crate::rpc_protocol::{ AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, DocGetRequest, DocGetResponse, DocImportRequest, DocImportResponse, DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, - DocSubscribeRequest, DocSubscribeResponse, DocTicket, DocsCreateRequest, DocsCreateResponse, - DocsListRequest, DocsListResponse, RpcResult, ShareMode, + DocStopSyncRequest, DocStopSyncResponse, DocSubscribeRequest, DocSubscribeResponse, DocTicket, + DocsCreateRequest, DocsCreateResponse, DocsListRequest, DocsListResponse, RpcResult, ShareMode, }; use super::{engine::SyncEngine, PeerSource}; @@ -134,6 +134,13 @@ impl SyncEngine { Ok(DocStartSyncResponse {}) } + pub async fn doc_stop_sync(&self, req: DocStopSyncRequest) -> RpcResult { + let DocStopSyncRequest { doc_id } = req; + let replica = self.get_replica(&doc_id)?; + self.stop_sync(replica.namespace()).await?; + Ok(DocStopSyncResponse {}) + } + pub async fn doc_set( &self, bao_store: &B, From f065781146fd1f0f16b941d45e79ce7868d4e96b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 17:04:38 +0200 Subject: [PATCH 131/172] remove unimplemented PeerAdd and PeerList commands for now --- iroh/src/node.rs | 2 -- iroh/src/rpc_protocol.rs | 41 ---------------------------------------- 2 files changed, 43 deletions(-) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 81acd8a4f0..55c5c47488 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1192,8 +1192,6 @@ fn handle_rpc_request< chan.server_streaming(msg, handler, RpcHandler::validate) .await } - PeerAdd(_msg) => todo!(), - PeerList(_msg) => todo!(), AuthorList(msg) => { chan.server_streaming(msg, handler, |handler, req| { handler.inner.sync.author_list(req) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 862c667cbb..1286d89497 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -1,5 +1,3 @@ -#![allow(missing_docs)] - //! This defines the RPC protocol used for communication between a CLI and an iroh node. //! //! RPC using the [`quic-rpc`](https://docs.rs/quic-rpc) crate. @@ -269,42 +267,6 @@ pub struct VersionResponse { pub version: String, } -// peer - -/// todo -#[derive(Serialize, Deserialize, Debug)] -pub struct PeerAddRequest { - pub peer_id: PublicKey, - pub addrs: Vec, - pub region: Option, -} - -impl RpcMsg for PeerAddRequest { - type Response = RpcResult; -} - -/// todo -#[derive(Serialize, Deserialize, Debug)] -pub struct PeerAddResponse {} - -/// todo -#[derive(Serialize, Deserialize, Debug)] -pub struct PeerListRequest {} - -impl Msg for PeerListRequest { - type Pattern = ServerStreaming; -} - -impl ServerStreamingMsg for PeerListRequest { - type Response = RpcResult; -} - -/// todo -#[derive(Serialize, Deserialize, Debug)] -pub struct PeerListResponse { - pub peer_id: PublicKey, -} - // author /// todo @@ -630,9 +592,6 @@ pub enum ProviderRequest { Shutdown(ShutdownRequest), Validate(ValidateRequest), - PeerAdd(PeerAddRequest), - PeerList(PeerListRequest), - AuthorList(AuthorListRequest), AuthorCreate(AuthorCreateRequest), AuthorImport(AuthorImportRequest), From f00db5804db4ee1055f45e8a59b52cb5493cbc27 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 17:18:55 +0200 Subject: [PATCH 132/172] docs and cleanup of RPC interface --- iroh/src/node.rs | 1 - iroh/src/rpc_protocol.rs | 114 +++++++++++++++++++++------------------ iroh/src/sync/rpc.rs | 1 - 3 files changed, 63 insertions(+), 53 deletions(-) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 55c5c47488..867dd0a26d 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1207,7 +1207,6 @@ fn handle_rpc_request< AuthorImport(_msg) => { todo!() } - AuthorShare(_msg) => todo!(), DocsList(msg) => { chan.server_streaming(msg, handler, |handler, req| { handler.inner.sync.docs_list(req) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 1286d89497..66b30c2665 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -269,7 +269,7 @@ pub struct VersionResponse { // author -/// todo +/// List document authors for which we have a secret key. #[derive(Serialize, Deserialize, Debug)] pub struct AuthorListRequest {} @@ -281,14 +281,14 @@ impl ServerStreamingMsg for AuthorListRequest { type Response = RpcResult; } -/// todo +/// Response for [`AuthorListRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct AuthorListResponse { + /// The author id pub author_id: AuthorId, - pub writable: bool, } -/// todo +/// Create a new document author. #[derive(Serialize, Deserialize, Debug)] pub struct AuthorCreateRequest; @@ -296,16 +296,17 @@ impl RpcMsg for AuthorCreateRequest { type Response = RpcResult; } -/// todo +/// Response for [`AuthorCreateRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct AuthorCreateResponse { + /// The id of the created author pub author_id: AuthorId, } -/// todo +/// Import author from secret key #[derive(Serialize, Deserialize, Debug)] pub struct AuthorImportRequest { - // either a public or private key + /// The secret key for the author pub key: KeyBytes, } @@ -313,20 +314,14 @@ impl RpcMsg for AuthorImportRequest { type Response = RpcResult; } -/// todo +/// Response to [`AuthorImportRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct AuthorImportResponse { + /// The author id of the imported author pub author_id: AuthorId, } -/// todo -#[derive(Serialize, Deserialize, Debug)] -pub struct AuthorShareRequest { - pub author: AuthorId, - pub mode: ShareMode, -} - -/// todo +/// Intended capability for document share tickets #[derive(Serialize, Deserialize, Debug, Clone)] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum ShareMode { @@ -335,20 +330,17 @@ pub enum ShareMode { /// Write access Write, } - -impl RpcMsg for AuthorShareRequest { - type Response = RpcResult; -} - -/// todo +/// Response to [`AuthorShareRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct AuthorShareResponse { + /// The secret key of the author pub key: KeyBytes, } -/// todo +/// Subscribe to events for a document. #[derive(Serialize, Deserialize, Debug)] pub struct DocSubscribeRequest { + /// The document id pub doc_id: NamespaceId, } @@ -360,13 +352,14 @@ impl ServerStreamingMsg for DocSubscribeRequest { type Response = DocSubscribeResponse; } -/// todo +/// Response to [`DocSubscribeRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocSubscribeResponse { + /// The event that occured on the document pub event: LiveEvent, } -/// todo +/// List all documents #[derive(Serialize, Deserialize, Debug)] pub struct DocsListRequest {} @@ -378,14 +371,14 @@ impl ServerStreamingMsg for DocsListRequest { type Response = RpcResult; } -/// todo +/// Response to [`DocsListRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocsListResponse { + /// The document id pub id: NamespaceId, - // pub writable: bool, } -/// todo +/// Create a new document #[derive(Serialize, Deserialize, Debug)] pub struct DocsCreateRequest {} @@ -393,13 +386,14 @@ impl RpcMsg for DocsCreateRequest { type Response = RpcResult; } -/// todo +/// Response to [`DocsCreateRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocsCreateResponse { + /// The document id pub id: NamespaceId, } -/// todo +/// Contains both a key (either secret or public) to a document, and a list of peers to join. #[derive(Serialize, Deserialize, Clone, Debug)] pub struct DocTicket { /// either a public or private key @@ -412,10 +406,12 @@ impl DocTicket { pub fn new(key: KeyBytes, peers: Vec) -> Self { Self { key, peers } } + /// Serialize the ticket to a byte array. pub fn to_bytes(&self) -> anyhow::Result> { let bytes = postcard::to_stdvec(&self)?; Ok(bytes) } + /// Parse ticket from a byte array. pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { let slf = postcard::from_bytes(bytes)?; Ok(slf) @@ -437,7 +433,7 @@ impl fmt::Display for DocTicket { } } -/// todo +/// Import a document from a ticket. #[derive(Serialize, Deserialize, Debug)] pub struct DocImportRequest(pub DocTicket); @@ -445,16 +441,19 @@ impl RpcMsg for DocImportRequest { type Response = RpcResult; } -/// todo +/// Response to [`DocImportRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocImportResponse { + /// the document id pub doc_id: NamespaceId, } -/// todo +/// Share a document with peers over a ticket. #[derive(Serialize, Deserialize, Debug)] pub struct DocShareRequest { + /// The document id pub doc_id: NamespaceId, + /// Whether to share read or write access to the document pub mode: ShareMode, } @@ -462,14 +461,16 @@ impl RpcMsg for DocShareRequest { type Response = RpcResult; } -/// todo +/// The response to [`DocShareRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocShareResponse(pub DocTicket); -/// todo +/// Start to sync a doc with peers. #[derive(Serialize, Deserialize, Debug)] pub struct DocStartSyncRequest { + /// The document id pub doc_id: NamespaceId, + /// List of peers to join pub peers: Vec, } @@ -477,13 +478,14 @@ impl RpcMsg for DocStartSyncRequest { type Response = RpcResult; } -/// todo +/// Response to [`DocStartSyncRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocStartSyncResponse {} -/// todo +/// Stop the live sync for a doc. #[derive(Serialize, Deserialize, Debug)] pub struct DocStopSyncRequest { + /// The document id pub doc_id: NamespaceId, } @@ -491,17 +493,22 @@ impl RpcMsg for DocStopSyncRequest { type Response = RpcResult; } -/// todo +/// Response to [`DocStopSyncRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocStopSyncResponse {} -/// todo +/// Set an entry in a document #[derive(Serialize, Deserialize, Debug)] pub struct DocSetRequest { + /// The document id pub doc_id: NamespaceId, + /// Author of this entry. pub author_id: AuthorId, + /// Key of this entry. pub key: Vec, - // todo: different forms to supply value + /// Value of this entry. + // TODO: Allow to provide the hash directly + // TODO: Add a way to provide content as stream pub value: Vec, } @@ -509,16 +516,19 @@ impl RpcMsg for DocSetRequest { type Response = RpcResult; } -/// todo +/// Response to [`DocSetRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocSetResponse { + /// The newly-created entry. pub entry: SignedEntry, } -/// todo +/// Get entries from a document #[derive(Serialize, Deserialize, Debug)] pub struct DocGetRequest { + /// The document id pub doc_id: NamespaceId, + /// Filter entries by this [`GetFilter`] pub filter: GetFilter, } @@ -530,15 +540,17 @@ impl ServerStreamingMsg for DocGetRequest { type Response = RpcResult; } -/// todo +/// Response to [`DocGetRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocGetResponse { + /// The document entry pub entry: SignedEntry, } -/// todo +/// Get the bytes for a hash #[derive(Serialize, Deserialize, Debug)] pub struct BytesGetRequest { + /// Hash to get bytes for pub hash: Hash, } @@ -546,12 +558,14 @@ impl RpcMsg for BytesGetRequest { type Response = RpcResult; } -/// todo +/// Response to [`BytesGetRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct BytesGetResponse { + /// The blob data pub data: Bytes, } +/// Get stats for the running Iroh node #[derive(Serialize, Deserialize, Debug)] pub struct StatsGetRequest {} @@ -562,13 +576,16 @@ impl RpcMsg for StatsGetRequest { /// Counter stats #[derive(Serialize, Deserialize, Debug)] pub struct CounterStats { + /// The counter value pub value: u64, + /// The counter description pub description: String, } -/// todo +/// Response to [`StatsGetRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct StatsGetResponse { + /// Map of statistics pub stats: HashMap, } @@ -595,7 +612,6 @@ pub enum ProviderRequest { AuthorList(AuthorListRequest), AuthorCreate(AuthorCreateRequest), AuthorImport(AuthorImportRequest), - AuthorShare(AuthorShareRequest), DocsList(DocsListRequest), DocsCreate(DocsCreateRequest), @@ -631,13 +647,9 @@ pub enum ProviderResponse { // TODO: I see I changed naming convention here but at least to me it becomes easier to parse // with the subject in front if there's many commands - PeerAdd(RpcResult), - PeerList(RpcResult), - AuthorList(RpcResult), AuthorCreate(RpcResult), AuthorImport(RpcResult), - AuthorShare(RpcResult), DocsList(RpcResult), DocsCreate(RpcResult), diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index b8437ea044..058e604305 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -40,7 +40,6 @@ impl SyncEngine { let ite = store.list_authors(); let ite = inline_result(ite).map_ok(|author| AuthorListResponse { author_id: author.id(), - writable: true, }); for entry in ite { if let Err(_err) = tx.send(entry) { From 5e126591dd00798f10f9a519987b67fb397d4707 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 17:24:10 +0200 Subject: [PATCH 133/172] cleanup --- iroh/src/commands/sync.rs | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 60f7b6b6b2..45e1ef3aab 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -72,11 +72,6 @@ impl Author { pub enum Docs { List, Create, - // Import { - // key: String, - // #[clap(short, long)] - // peers: Vec, - // }, Import { ticket: DocTicket }, } @@ -87,14 +82,6 @@ impl Docs { let doc = iroh.create_doc().await?; println!("created {}", doc.id()); } - // Docs::Import { key, peers } => { - // let key = hex::decode(key)? - // .try_into() - // .map_err(|_| anyhow!("invalid length"))?; - // let ticket = DocTicket::new(key, peers); - // let doc = iroh.import_doc(ticket).await?; - // println!("imported {}", doc.id()); - // } Docs::Import { ticket } => { let doc = iroh.import_doc(ticket).await?; println!("imported {}", doc.id()); @@ -121,6 +108,7 @@ pub enum Doc { /// Set an entry Set { /// Author of this entry. + #[clap(short, long)] author: AuthorId, /// Key to the entry (parsed as UTF-8 string). key: String, @@ -168,16 +156,6 @@ impl Doc { } Doc::Share { mode } => { let ticket = doc.share(mode).await?; - // println!("key: {}", hex::encode(ticket.key)); - // println!( - // "peers: {}", - // ticket - // .peers - // .iter() - // .map(|p| p.to_string()) - // .collect::>() - // .join(", ") - // ); println!("ticket: {}", ticket); } Doc::Set { author, key, value } => { From 2c540be25fd1e351a2dfb5b8c74dae4891eab73b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 17:28:33 +0200 Subject: [PATCH 134/172] fix: log errors for incoming connections --- iroh/src/node.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 867dd0a26d..33f0ec2f46 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -52,7 +52,7 @@ use quic_rpc::{RpcClient, RpcServer, ServiceEndpoint}; use tokio::sync::{mpsc, RwLock}; use tokio::task::JoinError; use tokio_util::sync::CancellationToken; -use tracing::{debug, trace}; +use tracing::{debug, trace, warn}; use crate::dial::Ticket; use crate::download::Downloader; @@ -482,7 +482,16 @@ where continue; } }; - rt.main().spawn(handle_connection(connecting, alpn, handler.inner.clone(), gossip.clone(), collection_parser.clone(), custom_get_handler.clone(), auth_handler.clone())); + let gossip = gossip.clone(); + let inner = handler.inner.clone(); + let collection_parser = collection_parser.clone(); + let custom_get_handler = custom_get_handler.clone(); + let auth_handler = auth_handler.clone(); + rt.main().spawn(async move { + if let Err(err) = handle_connection(connecting, alpn, inner, gossip, collection_parser, custom_get_handler, auth_handler).await { + warn!("Handling incoming connection ended with error: {err}"); + } + }); }, // Handle new callbacks Some(cb) = cb_receiver.recv() => { From 82317f31fa0bde8ab4d323ee8d9468fe91bcb8f3 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 17:52:15 +0200 Subject: [PATCH 135/172] add author switch and doc switch commands --- iroh/src/commands/repl.rs | 27 ++++++++++++++------------- iroh/src/commands/sync.rs | 16 +++++++++++++--- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 1fe69a7290..3792e6cef8 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -3,11 +3,13 @@ use clap::{Parser, Subcommand}; use colored::Colorize; use iroh::client::quic::RpcClient; use iroh_gossip::proto::util::base32; -use iroh_sync::sync::{AuthorId, NamespaceId}; use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; -use crate::config::{ConsoleEnv, ConsolePaths}; +use crate::{ + commands::sync, + config::{ConsoleEnv, ConsolePaths}, +}; pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { println!("{}", "Welcome to the Iroh console!".purple().bold()); @@ -15,18 +17,22 @@ pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { let mut repl_rx = Repl::spawn(env.clone()); while let Some((event, reply)) = repl_rx.recv().await { let (next, res) = match event { - ReplCmd::Rpc(cmd) => { - let res = cmd.run(client.clone(), env.clone()).await; - (ToRepl::Continue, res) - } - ReplCmd::SetDoc { id } => { + ReplCmd::Rpc(super::RpcCommands::Sync(sync::Commands::Doc { + command: sync::DocCommands::Switch { id }, + })) => { env.set_doc(id); (ToRepl::UpdateEnv(env.clone()), Ok(())) } - ReplCmd::SetAuthor { id } => { + ReplCmd::Rpc(super::RpcCommands::Sync(sync::Commands::Author { + command: sync::AuthorCommands::Switch { id }, + })) => { let res = env.save_author(id); (ToRepl::UpdateEnv(env.clone()), res) } + ReplCmd::Rpc(cmd) => { + let res = cmd.run(client.clone(), env.clone()).await; + (ToRepl::Continue, res) + } ReplCmd::Exit => (ToRepl::Exit, Ok(())), }; @@ -129,11 +135,6 @@ impl Repl { #[derive(Debug, Parser)] pub enum ReplCmd { - /// Set the active document - #[clap(next_help_heading = "foo")] - SetDoc { id: NamespaceId }, - /// Set the active author - SetAuthor { id: AuthorId }, #[clap(flatten)] Rpc(#[clap(subcommand)] super::RpcCommands), /// Quit the Iroh console diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 2efb4f5b08..3837e7cf72 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{bail, Result}; use clap::{Parser, Subcommand}; use futures::TryStreamExt; use indicatif::HumanBytes; @@ -44,6 +44,8 @@ impl Commands { #[derive(Debug, Clone, Parser)] pub enum DocCommands { + /// Set the active document (only works within the Iroh console). + Switch { id: NamespaceId }, /// Create a new document. Init, /// Join a document from a ticket. @@ -124,6 +126,9 @@ pub enum DocCommands { impl DocCommands { pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { + Self::Switch { .. } => { + bail!("This command is only supported in the Iroh console") + } Self::Init => { let doc = iroh.create_doc().await?; println!("{}", doc.id()); @@ -225,6 +230,8 @@ impl DocCommands { #[derive(Debug, Clone, Parser)] pub enum AuthorCommands { + /// Set the active author (only works within the Iroh console). + Switch { id: AuthorId }, /// Create a new author. Create, /// List authors. @@ -235,13 +242,16 @@ pub enum AuthorCommands { impl AuthorCommands { pub async fn run(self, iroh: &Iroh) -> Result<()> { match self { - AuthorCommands::List => { + Self::Switch { .. } => { + bail!("This command is only supported in the Iroh console") + } + Self::List => { let mut stream = iroh.list_authors().await?; while let Some(author_id) = stream.try_next().await? { println!("{}", author_id); } } - AuthorCommands::Create => { + Self::Create => { let author_id = iroh.create_author().await?; println!("{}", author_id); } From 23911fc2d1497c2c7b049503fd5e98325d1ccadc Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 17:59:26 +0200 Subject: [PATCH 136/172] chore: remove leftover code --- iroh/src/rpc_protocol.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 66b30c2665..bac71e2fa9 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -330,12 +330,6 @@ pub enum ShareMode { /// Write access Write, } -/// Response to [`AuthorShareRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub struct AuthorShareResponse { - /// The secret key of the author - pub key: KeyBytes, -} /// Subscribe to events for a document. #[derive(Serialize, Deserialize, Debug)] From eb40aa4377266c4731ccc655ea2ed52aed9f6001 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:03:29 +0200 Subject: [PATCH 137/172] chore: clippy --- iroh/src/commands.rs | 8 +++++--- iroh/src/commands/repl.rs | 3 ++- iroh/src/commands/sync.rs | 1 + iroh/src/config.rs | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index bde0b622d7..377fb9bf4c 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -72,12 +72,12 @@ impl Cli { match self.command { Commands::Console => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; - let env = ConsoleEnv::console_env()?; + let env = ConsoleEnv::for_console()?; repl::run(client, env).await } Commands::Rpc(command) => { let client = iroh::client::quic::connect_raw(self.rpc_args.rpc_port).await?; - let env = ConsoleEnv::cli_env()?; + let env = ConsoleEnv::for_cli()?; command.run(client, env).await } Commands::Full(command) => { @@ -90,7 +90,7 @@ impl Cli { let config = NodeConfig::from_env(cfg.as_deref())?; #[cfg(feature = "metrics")] - let metrics_fut = start_metrics_server(metrics_addr, &rt); + let metrics_fut = start_metrics_server(metrics_addr, rt); let res = command.run(rt, &config, keylog).await; @@ -115,6 +115,7 @@ pub enum Commands { Rpc(#[clap(subcommands)] RpcCommands), } +#[allow(clippy::large_enum_variant)] #[derive(Subcommand, Debug, Clone)] pub enum FullCommands { /// Start a Iroh node @@ -332,6 +333,7 @@ impl RpcCommands { } } +#[allow(clippy::large_enum_variant)] #[derive(Subcommand, Debug, Clone)] pub enum BlobCommands { /// Add data from PATH to the running provider's database. diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 3792e6cef8..96d1f879cd 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -46,6 +46,7 @@ pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { } /// Reply to the repl after a command completed +#[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum ToRepl { /// Continue execution by reading the next command @@ -127,7 +128,7 @@ impl Repl { )); } if !pwd.is_empty() { - pwd.push_str("\n"); + pwd.push('\n'); } format!("\n{pwd}{}", "> ".blue()) } diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 3837e7cf72..c880b5dddf 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -17,6 +17,7 @@ use super::RpcClient; const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; +#[allow(clippy::large_enum_variant)] #[derive(Subcommand, Debug, Clone)] pub enum Commands { /// Manage documents diff --git a/iroh/src/config.rs b/iroh/src/config.rs index 28c384eabb..ac0927b95c 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -30,7 +30,7 @@ const ENV_DOC: &str = "DOC"; /// Fetches the environment variable `IROH_` from the current process. pub fn env_var(key: &str) -> std::result::Result { - env::var(&format!("{ENV_PREFIX}_{key}")) + env::var(format!("{ENV_PREFIX}_{key}")) } /// Paths to files or directory within the [`iroh_data_root`] used by Iroh. @@ -218,7 +218,7 @@ pub struct ConsoleEnv { } impl ConsoleEnv { /// Read from environment variables and the console config file. - pub fn console_env() -> Result { + pub fn for_console() -> Result { let author = match Self::get_console_default_author()? { Some(author) => Some(author), None => env_author()?, @@ -230,7 +230,7 @@ impl ConsoleEnv { } /// Read only from environment variables. - pub fn cli_env() -> Result { + pub fn for_cli() -> Result { Ok(Self { author: env_author()?, doc: env_doc()?, From e44fb8394d69d358f59592ab4e597819dbb65d43 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:12:54 +0200 Subject: [PATCH 138/172] chore: disable cross tests for armv7 and aarch64 until https://github.com/cross-rs/cross/issues/1311 is fixed --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb4c3e8155..92da20cbfb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,6 +245,9 @@ jobs: run: cross check --all --target ${{ matrix.target }} - name: test + # cross tests are currently broken vor armv7 and aarch64 + # see https://github.com/cross-rs/cross/issues/1311 + if: matrix.target == 'i686-unkown-linux-gnu' run: cross test --all --target ${{ matrix.target }} -- --test-threads=12 check_fmt_and_docs: From 0d432dab917fd2f3ea5e348bc7228765d1753fa8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:18:13 +0200 Subject: [PATCH 139/172] feat: filter by author on --- iroh/src/commands/sync.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 45e1ef3aab..b07aa76f20 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -137,6 +137,9 @@ pub enum Doc { content: bool, }, List { + /// Filter by author. + #[clap(short, long)] + author: Option, /// If true, old entries will be included. By default only the latest value for each key is /// shown. #[clap(short, long)] @@ -205,15 +208,22 @@ impl Doc { println!(); } } - Doc::List { old, prefix } => { + Doc::List { + old, + prefix, + author, + } => { let filter = match old { true => GetFilter::all(), false => GetFilter::latest(), }; - let filter = match prefix { + let mut filter = match prefix { Some(prefix) => filter.with_prefix(prefix), None => filter, }; + if let Some(author) = author { + filter = filter.with_author(author); + }; let mut stream = doc.get(filter).await?; while let Some(entry) = stream.try_next().await? { println!("{}", fmt_entry(&entry)); From 502da40ff124e4f7e0885827c251e26b4586c9a5 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:22:50 +0200 Subject: [PATCH 140/172] cleanup naming --- iroh/src/client.rs | 6 +++--- iroh/src/node.rs | 6 +++--- iroh/src/rpc_protocol.rs | 36 +++++++++++++++++------------------- iroh/src/sync/rpc.rs | 12 ++++++------ 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 70f62a8482..2987196e6b 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -16,7 +16,7 @@ use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorListRequest, BytesGetRequest, CounterStats, DocGetRequest, DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocStopSyncRequest, - DocSubscribeRequest, DocTicket, DocsCreateRequest, DocsListRequest, ProviderService, ShareMode, + DocSubscribeRequest, DocTicket, DocCreateRequest, DocListRequest, ProviderService, ShareMode, StatsGetRequest, }; use crate::sync::{LiveEvent, PeerSource}; @@ -54,7 +54,7 @@ where /// Create a new document. pub async fn create_doc(&self) -> Result> { - let res = self.rpc.rpc(DocsCreateRequest {}).await??; + let res = self.rpc.rpc(DocCreateRequest {}).await??; let doc = Doc { id: res.id, rpc: self.rpc.clone(), @@ -74,7 +74,7 @@ where /// List all documents. pub async fn list_docs(&self) -> Result>> { - let stream = self.rpc.server_streaming(DocsListRequest {}).await?; + let stream = self.rpc.server_streaming(DocListRequest {}).await?; Ok(flatten(stream).map_ok(|res| res.id)) } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 33f0ec2f46..6b21163da4 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1216,19 +1216,19 @@ fn handle_rpc_request< AuthorImport(_msg) => { todo!() } - DocsList(msg) => { + DocList(msg) => { chan.server_streaming(msg, handler, |handler, req| { handler.inner.sync.docs_list(req) }) .await } - DocsCreate(msg) => { + DocCreate(msg) => { chan.rpc(msg, handler, |handler, req| async move { handler.inner.sync.docs_create(req) }) .await } - DocsImport(msg) => { + DocImport(msg) => { chan.rpc(msg, handler, |handler, req| async move { handler.inner.sync.doc_import(req).await }) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index bac71e2fa9..ee5c441da9 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -355,34 +355,34 @@ pub struct DocSubscribeResponse { /// List all documents #[derive(Serialize, Deserialize, Debug)] -pub struct DocsListRequest {} +pub struct DocListRequest {} -impl Msg for DocsListRequest { +impl Msg for DocListRequest { type Pattern = ServerStreaming; } -impl ServerStreamingMsg for DocsListRequest { - type Response = RpcResult; +impl ServerStreamingMsg for DocListRequest { + type Response = RpcResult; } /// Response to [`DocsListRequest`] #[derive(Serialize, Deserialize, Debug)] -pub struct DocsListResponse { +pub struct DocListResponse { /// The document id pub id: NamespaceId, } /// Create a new document #[derive(Serialize, Deserialize, Debug)] -pub struct DocsCreateRequest {} +pub struct DocCreateRequest {} -impl RpcMsg for DocsCreateRequest { - type Response = RpcResult; +impl RpcMsg for DocCreateRequest { + type Response = RpcResult; } /// Response to [`DocsCreateRequest`] #[derive(Serialize, Deserialize, Debug)] -pub struct DocsCreateResponse { +pub struct DocCreateResponse { /// The document id pub id: NamespaceId, } @@ -603,14 +603,15 @@ pub enum ProviderRequest { Shutdown(ShutdownRequest), Validate(ValidateRequest), + // TODO: I see I changed naming convention here but at least to me it becomes easier to parse + // with the subject in front if there's many commands AuthorList(AuthorListRequest), AuthorCreate(AuthorCreateRequest), AuthorImport(AuthorImportRequest), - DocsList(DocsListRequest), - DocsCreate(DocsCreateRequest), - DocsImport(DocImportRequest), - + DocList(DocListRequest), + DocCreate(DocCreateRequest), + DocImport(DocImportRequest), DocSet(DocSetRequest), DocGet(DocGetRequest), DocStartSync(DocStartSyncRequest), @@ -639,16 +640,13 @@ pub enum ProviderResponse { Validate(ValidateProgress), Shutdown(()), - // TODO: I see I changed naming convention here but at least to me it becomes easier to parse - // with the subject in front if there's many commands AuthorList(RpcResult), AuthorCreate(RpcResult), AuthorImport(RpcResult), - DocsList(RpcResult), - DocsCreate(RpcResult), - DocsImport(RpcResult), - + DocList(RpcResult), + DocCreate(RpcResult), + DocImport(RpcResult), DocSet(RpcResult), DocGet(RpcResult), DocShare(RpcResult), diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index 058e604305..b0be6d02fe 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -12,7 +12,7 @@ use crate::rpc_protocol::{ DocGetRequest, DocGetResponse, DocImportRequest, DocImportResponse, DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, DocStopSyncRequest, DocStopSyncResponse, DocSubscribeRequest, DocSubscribeResponse, DocTicket, - DocsCreateRequest, DocsCreateResponse, DocsListRequest, DocsListResponse, RpcResult, ShareMode, + DocCreateRequest, DocCreateResponse, DocListRequest, DocListResponse, RpcResult, ShareMode, }; use super::{engine::SyncEngine, PeerSource}; @@ -50,22 +50,22 @@ impl SyncEngine { rx.into_stream() } - pub fn docs_create(&self, _req: DocsCreateRequest) -> RpcResult { + pub fn docs_create(&self, _req: DocCreateRequest) -> RpcResult { let doc = self.store.new_replica(Namespace::new(&mut OsRng {}))?; - Ok(DocsCreateResponse { + Ok(DocCreateResponse { id: doc.namespace(), }) } pub fn docs_list( &self, - _req: DocsListRequest, - ) -> impl Stream> { + _req: DocListRequest, + ) -> impl Stream> { let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); let store = self.store.clone(); self.rt.main().spawn_blocking(move || { let ite = store.list_namespaces(); - let ite = inline_result(ite).map_ok(|id| DocsListResponse { id }); + let ite = inline_result(ite).map_ok(|id| DocListResponse { id }); for entry in ite { if let Err(_err) = tx.send(entry) { break; From a63fbf857c8f8049a30be6158c66359924fdaacc Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:25:26 +0200 Subject: [PATCH 141/172] chore: fix CI skip statement --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92da20cbfb..eb5a9ff801 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -247,7 +247,7 @@ jobs: - name: test # cross tests are currently broken vor armv7 and aarch64 # see https://github.com/cross-rs/cross/issues/1311 - if: matrix.target == 'i686-unkown-linux-gnu' + if: matrix.target == 'i686-unknown-linux-gnu' run: cross test --all --target ${{ matrix.target }} -- --test-threads=12 check_fmt_and_docs: From 3339add08d415fca675d3c014e10752de5e08468 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:38:39 +0200 Subject: [PATCH 142/172] fix: check if doc exists in client --- iroh/src/client.rs | 28 +++++++++++++++++++--------- iroh/src/commands/sync.rs | 2 +- iroh/src/node.rs | 6 ++++++ iroh/src/rpc_protocol.rs | 18 ++++++++++++++++++ iroh/src/sync/rpc.rs | 15 +++++++++++---- 5 files changed, 55 insertions(+), 14 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 2987196e6b..224e97695c 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -14,10 +14,10 @@ use iroh_sync::sync::{AuthorId, NamespaceId, SignedEntry}; use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ - AuthorCreateRequest, AuthorListRequest, BytesGetRequest, CounterStats, DocGetRequest, - DocImportRequest, DocSetRequest, DocShareRequest, DocStartSyncRequest, DocStopSyncRequest, - DocSubscribeRequest, DocTicket, DocCreateRequest, DocListRequest, ProviderService, ShareMode, - StatsGetRequest, + AuthorCreateRequest, AuthorListRequest, BytesGetRequest, CounterStats, DocCreateRequest, + DocGetRequest, DocImportRequest, DocInfoRequest, DocListRequest, DocSetRequest, + DocShareRequest, DocStartSyncRequest, DocStopSyncRequest, DocSubscribeRequest, DocTicket, + ProviderService, ShareMode, StatsGetRequest, }; use crate::sync::{LiveEvent, PeerSource}; @@ -78,19 +78,29 @@ where Ok(flatten(stream).map_ok(|res| res.id)) } - /// Get a [`Doc`] client for a single document. - pub fn get_doc(&self, id: NamespaceId) -> Result> { - // TODO: Check if doc exists? + /// Get a [`Doc`] client for a single document. Return an error if the document cannot be found. + pub async fn get_doc(&self, id: NamespaceId) -> Result> { + match self.try_get_doc(id).await? { + Some(doc) => Ok(doc), + None => Err(anyhow!("Document not found")), + } + } + + /// Get a [`Doc`] client for a single document. Return None if the document cannot be found. + pub async fn try_get_doc(&self, id: NamespaceId) -> Result>> { + if let Err(_err) = self.rpc.rpc(DocInfoRequest { doc_id: id }).await? { + return Ok(None); + } let doc = Doc { id, rpc: self.rpc.clone(), }; - Ok(doc) + Ok(Some(doc)) } /// Get the bytes for a hash. /// - /// NOTE: This reads the full blob into memory. + /// Note: This reads the full blob into memory. // TODO: add get_reader for streaming gets pub async fn get_bytes(&self, hash: Hash) -> Result { let res = self.rpc.rpc(BytesGetRequest { hash }).await??; diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index b07aa76f20..29efeb7c65 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -151,7 +151,7 @@ pub enum Doc { impl Doc { pub async fn run(self, iroh: Iroh, doc_id: NamespaceId) -> anyhow::Result<()> { - let doc = iroh.get_doc(doc_id)?; + let doc = iroh.get_doc(doc_id).await?; match self { Doc::StartSync { peers } => { doc.start_sync(peers).await?; diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 6b21163da4..e2e5d64a19 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1216,6 +1216,12 @@ fn handle_rpc_request< AuthorImport(_msg) => { todo!() } + DocInfo(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.doc_info(req).await + }) + .await + } DocList(msg) => { chan.server_streaming(msg, handler, |handler, req| { handler.inner.sync.docs_list(req) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index ee5c441da9..4ace8ec224 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -459,6 +459,22 @@ impl RpcMsg for DocShareRequest { #[derive(Serialize, Deserialize, Debug)] pub struct DocShareResponse(pub DocTicket); +/// Get info on a document +#[derive(Serialize, Deserialize, Debug)] +pub struct DocInfoRequest { + /// The document id + pub doc_id: NamespaceId, +} + +impl RpcMsg for DocInfoRequest { + type Response = RpcResult; +} + +/// Response to [`DocInfoRequest`] +// TODO: actually provide info +#[derive(Serialize, Deserialize, Debug)] +pub struct DocInfoResponse {} + /// Start to sync a doc with peers. #[derive(Serialize, Deserialize, Debug)] pub struct DocStartSyncRequest { @@ -609,6 +625,7 @@ pub enum ProviderRequest { AuthorCreate(AuthorCreateRequest), AuthorImport(AuthorImportRequest), + DocInfo(DocInfoRequest), DocList(DocListRequest), DocCreate(DocCreateRequest), DocImport(DocImportRequest), @@ -644,6 +661,7 @@ pub enum ProviderResponse { AuthorCreate(RpcResult), AuthorImport(RpcResult), + DocInfo(RpcResult), DocList(RpcResult), DocCreate(RpcResult), DocImport(RpcResult), diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index b0be6d02fe..3378a18be4 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -9,10 +9,11 @@ use rand::rngs::OsRng; use crate::rpc_protocol::{ AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, - DocGetRequest, DocGetResponse, DocImportRequest, DocImportResponse, DocSetRequest, - DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, DocStartSyncResponse, - DocStopSyncRequest, DocStopSyncResponse, DocSubscribeRequest, DocSubscribeResponse, DocTicket, - DocCreateRequest, DocCreateResponse, DocListRequest, DocListResponse, RpcResult, ShareMode, + DocCreateRequest, DocCreateResponse, DocGetRequest, DocGetResponse, DocImportRequest, + DocImportResponse, DocInfoRequest, DocInfoResponse, DocListRequest, DocListResponse, + DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, + DocStartSyncResponse, DocStopSyncRequest, DocStopSyncResponse, DocSubscribeRequest, + DocSubscribeResponse, DocTicket, RpcResult, ShareMode, }; use super::{engine::SyncEngine, PeerSource}; @@ -75,6 +76,12 @@ impl SyncEngine { rx.into_stream() } + pub async fn doc_info(&self, req: DocInfoRequest) -> RpcResult { + let replica = self.get_replica(&req.doc_id)?; + self.start_sync(replica.namespace(), vec![]).await?; + Ok(DocInfoResponse {}) + } + pub async fn doc_share(&self, req: DocShareRequest) -> RpcResult { let replica = self.get_replica(&req.doc_id)?; let key = match req.mode { From b4d21641ce873734d4dfbad34a07f0c0cb5c9964 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:42:50 +0200 Subject: [PATCH 143/172] cleanup --- iroh/src/rpc_protocol.rs | 4 ++-- iroh/src/sync/rpc.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 4ace8ec224..0ddef01f4f 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -365,7 +365,7 @@ impl ServerStreamingMsg for DocListRequest { type Response = RpcResult; } -/// Response to [`DocsListRequest`] +/// Response to [`DocListRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocListResponse { /// The document id @@ -380,7 +380,7 @@ impl RpcMsg for DocCreateRequest { type Response = RpcResult; } -/// Response to [`DocsCreateRequest`] +/// Response to [`DocCreateRequest`] #[derive(Serialize, Deserialize, Debug)] pub struct DocCreateResponse { /// The document id diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index 3378a18be4..7c522476b7 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -51,14 +51,14 @@ impl SyncEngine { rx.into_stream() } - pub fn docs_create(&self, _req: DocCreateRequest) -> RpcResult { + pub fn doc_create(&self, _req: DocCreateRequest) -> RpcResult { let doc = self.store.new_replica(Namespace::new(&mut OsRng {}))?; Ok(DocCreateResponse { id: doc.namespace(), }) } - pub fn docs_list( + pub fn doc_list( &self, _req: DocListRequest, ) -> impl Stream> { From fe781be2bc4890e343e3717a8359ff86686a6c30 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 18:43:49 +0200 Subject: [PATCH 144/172] chore: fmt --- iroh/src/sync/rpc.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index 7c522476b7..ed7ca06087 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -58,10 +58,7 @@ impl SyncEngine { }) } - pub fn doc_list( - &self, - _req: DocListRequest, - ) -> impl Stream> { + pub fn doc_list(&self, _req: DocListRequest) -> impl Stream> { let (tx, rx) = flume::bounded(ITER_CHANNEL_CAP); let store = self.store.clone(); self.rt.main().spawn_blocking(move || { From 264dadf423888aa4a6f2d0c5065ed011c5a84b2f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 19:00:05 +0200 Subject: [PATCH 145/172] fix: last commit was broken --- iroh/src/node.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index e2e5d64a19..235f39e6ca 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -1224,13 +1224,13 @@ fn handle_rpc_request< } DocList(msg) => { chan.server_streaming(msg, handler, |handler, req| { - handler.inner.sync.docs_list(req) + handler.inner.sync.doc_list(req) }) .await } DocCreate(msg) => { chan.rpc(msg, handler, |handler, req| async move { - handler.inner.sync.docs_create(req) + handler.inner.sync.doc_create(req) }) .await } From 503ff23b6439a4787bd4d02e7d605c65cbfd8022 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 19:14:11 +0200 Subject: [PATCH 146/172] fix: clarify metrics initialization --- iroh/examples/sync.rs | 2 +- iroh/src/commands.rs | 2 +- iroh/src/main.rs | 4 ++-- iroh/src/metrics.rs | 10 ++++++---- iroh/src/node.rs | 25 +++++++------------------ 5 files changed, 17 insertions(+), 26 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index d78ce6dea0..3d33dabb31 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -97,7 +97,7 @@ async fn main() -> anyhow::Result<()> { pub fn init_metrics_collection( metrics_addr: Option, ) -> Option> { - iroh::metrics::init_metrics_collection(); + iroh::metrics::try_init_metrics_collection().ok(); // doesn't start the server if the address is None if let Some(metrics_addr) = metrics_addr { return Some(tokio::spawn(async move { diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 57473b8e64..6a08d66a34 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -404,7 +404,7 @@ pub async fn make_rpc_client(rpc_port: u16) -> anyhow::Result { } #[cfg(feature = "metrics")] -pub fn init_metrics_collection( +pub fn start_metrics_server( metrics_addr: Option, rt: &iroh_bytes::util::runtime::Handle, ) -> Option> { diff --git a/iroh/src/main.rs b/iroh/src/main.rs index ce986cde08..441e3df973 100644 --- a/iroh/src/main.rs +++ b/iroh/src/main.rs @@ -8,7 +8,7 @@ mod commands; mod config; use crate::{ - commands::{init_metrics_collection, Cli}, + commands::{start_metrics_server, Cli}, config::{iroh_config_path, Config, CONFIG_FILE_NAME, ENV_PREFIX}, }; @@ -50,7 +50,7 @@ async fn main_impl() -> Result<()> { )?; #[cfg(feature = "metrics")] - let metrics_fut = init_metrics_collection(cli.metrics_addr, &rt); + let metrics_fut = start_metrics_server(cli.metrics_addr, &rt); let r = cli.run(&rt, &config).await; diff --git a/iroh/src/metrics.rs b/iroh/src/metrics.rs index 69d7064dc1..6085483918 100644 --- a/iroh/src/metrics.rs +++ b/iroh/src/metrics.rs @@ -46,16 +46,18 @@ impl Metric for Metrics { } } -/// Initialize the metrics collection. -pub fn init_metrics_collection() { - iroh_metrics::core::Core::init(|reg, metrics| { +/// Initialize the global metrics collection. +/// +/// Will return an error if the global metrics collection was already initialized. +pub fn try_init_metrics_collection() -> std::io::Result<()> { + iroh_metrics::core::Core::try_init(|reg, metrics| { metrics.insert(crate::metrics::Metrics::new(reg)); metrics.insert(iroh_sync::metrics::Metrics::new(reg)); metrics.insert(iroh_net::metrics::MagicsockMetrics::new(reg)); metrics.insert(iroh_net::metrics::NetcheckMetrics::new(reg)); metrics.insert(iroh_net::metrics::PortmapMetrics::new(reg)); metrics.insert(iroh_net::metrics::DerpMetrics::new(reg)); - }); + }) } /// Collect the current metrics into a hash map. diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 235f39e6ca..9e91aad8cb 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -77,22 +77,6 @@ pub const DEFAULT_BIND_ADDR: (Ipv4Addr, u16) = (Ipv4Addr::LOCALHOST, 11204); /// How long we wait at most for some endpoints to be discovered. const ENDPOINT_WAIT: Duration = Duration::from_secs(5); -#[cfg(feature = "metrics")] -/// Initialize the metrics collection. -pub fn init_metrics_collection() { - use iroh_metrics::core::Metric; - - iroh_metrics::core::Core::try_init(|reg, metrics| { - metrics.insert(crate::metrics::Metrics::new(reg)); - metrics.insert(iroh_sync::metrics::Metrics::new(reg)); - metrics.insert(iroh_net::metrics::MagicsockMetrics::new(reg)); - metrics.insert(iroh_net::metrics::NetcheckMetrics::new(reg)); - metrics.insert(iroh_net::metrics::PortmapMetrics::new(reg)); - metrics.insert(iroh_net::metrics::DerpMetrics::new(reg)); - }) - .ok(); -} - /// Builder for the [`Node`]. /// /// You must supply a blob store. Various store implementations are available @@ -299,9 +283,14 @@ where trace!("spawning node"); let rt = self.rt.context("runtime not set")?; - // TODO: this should actually run globally only once. + // Initialize the metrics collection. + // + // The metrics are global per process. Subsequent calls do not change the metrics + // collection and will return an error. We ignore this error. This means that if you'd + // spawn multiple Iroh nodes in the same process, the metrics would be shared between the + // nodes. #[cfg(feature = "metrics")] - init_metrics_collection(); + crate::metrics::try_init_metrics_collection().ok(); let (endpoints_update_s, endpoints_update_r) = flume::bounded(1); let mut transport_config = quinn::TransportConfig::default(); From 5106b5ae5b94004206f80900de13431b51d2c900 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 20:59:23 +0200 Subject: [PATCH 147/172] feat: add node stats cli command --- iroh/src/commands.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index 377fb9bf4c..abe05dcdb9 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -294,6 +294,8 @@ pub enum RpcCommands { pub enum NodeCommands { /// Get status of the running node. Status, + /// Get statistics and metrics from the running node. + Stats, /// Shutdown the running node. Shutdown { /// Shutdown mode. @@ -310,16 +312,24 @@ impl NodeCommands { match self { Self::Shutdown { force } => { client.rpc(ShutdownRequest { force }).await?; - Ok(()) } - Self::Status {} => { + Self::Stats => { + let response = client.rpc(StatsGetRequest {}).await??; + for (name, details) in response.stats.iter() { + println!( + "{:23} : {:>6} ({})", + name, details.value, details.description + ); + } + } + Self::Status => { let response = client.rpc(StatusRequest).await?; println!("Listening addresses: {:#?}", response.listen_addrs); println!("PeerID: {}", response.peer_id); - Ok(()) } } + Ok(()) } } From a428a3b5d2d6c54289bb80fa139d0124aba8a59b Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 23 Aug 2023 21:02:50 +0200 Subject: [PATCH 148/172] feat: add doc watch command --- iroh/src/commands/sync.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 684016fc52..394a060bb9 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -1,6 +1,6 @@ use anyhow::{bail, Result}; use clap::{Parser, Subcommand}; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use indicatif::HumanBytes; use iroh::{ client::quic::Iroh, @@ -125,6 +125,15 @@ pub enum DocCommands { /// Optional key prefix (parsed as UTF-8 string) prefix: Option, }, + /// Watch for changes and events on a document + Watch { + /// Document to operate on. + /// + /// Required unless the document is set through the IROH_DOC environment variable. + /// Within the Iroh console, the active document can also set with `set-doc`. + #[clap(short, long)] + doc_id: Option, + }, } impl DocCommands { @@ -231,6 +240,14 @@ impl DocCommands { println!("{}", fmt_entry(&entry)); } } + Self::Watch { doc_id } => { + let doc = iroh.get_doc(env.doc(doc_id)?)?; + let mut stream = doc.subscribe().await?; + while let Some(event) = stream.next().await { + let event = event?; + println!("{event:?}"); + } + } } Ok(()) } From 0e3f169601f65c77073716f30a5b09e878fe9bef Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Thu, 24 Aug 2023 10:21:37 +0200 Subject: [PATCH 149/172] fix(sync): properly drop rpc subscriptions, add doc status (#1396) ## Description So far in #1333, if a RPC or in-memory client called `doc.subscribe()` the event callback would never be dropped, even if the client did drop the event stream. This PR fixes this, by having the event callbacks return whether the callback should stay active or not. We can't use the removal token here, because calling `LiveSync::unsubscribe` from within the event callback would deadlock the actor. Also adds a a `LiveStatus` to the doc info RPC call. For now only contains the number of subscribers. More info, e.g. on peers, can come later. ## Notes & open questions * As of #1333 and unchanged by this PR: `doc.subscribe` will fail for documents that are not in the `LiveSync` (they are added via `doc.import` or `doc.start_sync`). This is unfortunate, because you'd often want to setup a subscription before starting sync, to catch all events. ## Change checklist - [x] Self-review. - [x] Documentation updates if relevant. - [x] Tests if relevant. --- iroh/src/client.rs | 10 ++++- iroh/src/rpc_protocol.rs | 11 ++++-- iroh/src/sync/live.rs | 84 ++++++++++++++++++++++++++++++++++------ iroh/src/sync/rpc.rs | 59 ++++++++++++++++++---------- iroh/tests/sync.rs | 29 ++++++++++++++ 5 files changed, 156 insertions(+), 37 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 224e97695c..83ed81a8df 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -19,7 +19,7 @@ use crate::rpc_protocol::{ DocShareRequest, DocStartSyncRequest, DocStopSyncRequest, DocSubscribeRequest, DocTicket, ProviderService, ShareMode, StatsGetRequest, }; -use crate::sync::{LiveEvent, PeerSource}; +use crate::sync::{LiveEvent, LiveStatus, PeerSource}; pub mod mem; #[cfg(feature = "cli")] @@ -219,7 +219,13 @@ where .rpc .server_streaming(DocSubscribeRequest { doc_id: self.id }) .await?; - Ok(stream.map_ok(|res| res.event).map_err(Into::into)) + Ok(flatten(stream).map_ok(|res| res.event).map_err(Into::into)) + } + + /// Get status info for this document + pub async fn status(&self) -> anyhow::Result { + let res = self.rpc.rpc(DocInfoRequest { doc_id: self.id }).await??; + Ok(res.status) } } diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 0ddef01f4f..ef4880b992 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -27,7 +27,7 @@ use serde::{Deserialize, Serialize}; pub use iroh_bytes::{baomap::ValidateProgress, provider::ProvideProgress, util::RpcResult}; -use crate::sync::{LiveEvent, PeerSource}; +use crate::sync::{LiveEvent, LiveStatus, PeerSource}; /// A 32-byte key or token pub type KeyBytes = [u8; 32]; @@ -343,7 +343,7 @@ impl Msg for DocSubscribeRequest { } impl ServerStreamingMsg for DocSubscribeRequest { - type Response = DocSubscribeResponse; + type Response = RpcResult; } /// Response to [`DocSubscribeRequest`] @@ -473,7 +473,10 @@ impl RpcMsg for DocInfoRequest { /// Response to [`DocInfoRequest`] // TODO: actually provide info #[derive(Serialize, Deserialize, Debug)] -pub struct DocInfoResponse {} +pub struct DocInfoResponse { + /// Live sync status + pub status: LiveStatus, +} /// Start to sync a doc with peers. #[derive(Serialize, Deserialize, Debug)] @@ -670,7 +673,7 @@ pub enum ProviderResponse { DocShare(RpcResult), DocStartSync(RpcResult), DocStopSync(RpcResult), - DocSubscribe(DocSubscribeResponse), + DocSubscribe(RpcResult), BytesGet(RpcResult), diff --git a/iroh/src/sync/live.rs b/iroh/src/sync/live.rs index 36e23971a7..583cfeaaad 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync/live.rs @@ -109,8 +109,21 @@ enum SyncState { Failed(anyhow::Error), } +/// Sync status for a document +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct LiveStatus { + /// Whether this document is in the live sync + pub active: bool, + /// Number of event listeners registered + pub subscriptions: usize, +} + #[derive(derive_more::Debug)] enum ToActor { + Status { + namespace: NamespaceId, + s: sync::oneshot::Sender>, + }, StartSync { replica: Replica, peers: Vec, @@ -136,9 +149,18 @@ enum ToActor { }, } +/// Whether to keep a live event callback active. +#[derive(Debug)] +pub enum KeepCallback { + /// Keep active + Keep, + /// Drop this callback + Drop, +} + /// Callback used for tracking [`LiveEvent`]s. pub type OnLiveEventCallback = - Box BoxFuture<'static, ()> + Send + Sync + 'static>; + Box BoxFuture<'static, KeepCallback> + Send + Sync + 'static>; /// Events informing about actions of the live sync progres. #[derive(Serialize, Deserialize, Debug, Clone)] @@ -264,7 +286,7 @@ impl LiveSync { /// Subscribes `cb` to events on this `namespace`. pub async fn subscribe(&self, namespace: NamespaceId, cb: F) -> Result where - F: Fn(LiveEvent) -> BoxFuture<'static, ()> + Send + Sync + 'static, + F: Fn(LiveEvent) -> BoxFuture<'static, KeepCallback> + Send + Sync + 'static, { let (s, r) = sync::oneshot::channel(); self.to_actor_tx @@ -292,6 +314,16 @@ impl LiveSync { let token = r.await?; Ok(token) } + + /// Get status for a document + pub async fn status(&self, namespace: NamespaceId) -> Result> { + let (s, r) = sync::oneshot::channel(); + self.to_actor_tx + .send(ToActor::::Status { namespace, s }) + .await?; + let status = r.await?; + Ok(status) + } } // Currently peers might double-sync in both directions. @@ -320,7 +352,7 @@ struct Actor { } /// Token needed to remove inserted callbacks. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct RemovalToken(u64); impl Actor { @@ -367,13 +399,17 @@ impl Actor { Some(ToActor::StopSync { namespace }) => self.stop_sync(&namespace).await?, Some(ToActor::JoinPeers { namespace, peers }) => self.join_gossip_and_start_initial_sync(&namespace, peers).await?, Some(ToActor::Subscribe { namespace, cb, s }) => { - let subscribe_result = self.subscribe(&namespace, cb).await; - s.send(subscribe_result).ok(); + let result = self.subscribe(&namespace, cb).await; + s.send(result).ok(); }, Some(ToActor::Unsubscribe { namespace, token, s }) => { let result = self.unsubscribe(&namespace, token).await; s.send(result).ok(); }, + Some(ToActor::Status { namespace , s }) => { + let result = self.status(&namespace).await; + s.send(result).ok(); + }, } } // new gossip message @@ -389,7 +425,6 @@ impl Actor { } } Some((topic, peer, res)) = self.pending_syncs.next() => { - // let (topic, peer, res) = res.context("task sync_with_peer paniced")?; self.on_sync_finished(topic, peer, res); } @@ -403,7 +438,7 @@ impl Actor { } Some(res) = self.pending_downloads.next() => { if let Some((topic, hash)) = res { - if let Some(subs) = self.event_subscriptions.get(&topic) { + if let Some(subs) = self.event_subscriptions.get_mut(&topic) { let event = LiveEvent::ContentReady { hash }; notify_all(subs, event).await; } @@ -451,6 +486,23 @@ impl Actor { Ok(()) } + async fn status(&mut self, namespace: &NamespaceId) -> Option { + let topic = TopicId::from_bytes(*namespace.as_bytes()); + if self.replicas.contains_key(&topic) { + let subscriptions = self + .event_subscriptions + .get(&topic) + .map(|map| map.len()) + .unwrap_or_default(); + Some(LiveStatus { + active: true, + subscriptions, + }) + } else { + None + } + } + async fn subscribe( &mut self, namespace: &NamespaceId, @@ -474,7 +526,8 @@ impl Actor { async fn unsubscribe(&mut self, namespace: &NamespaceId, token: RemovalToken) -> bool { let topic = TopicId::from_bytes(*namespace.as_bytes()); if let Some(subs) = self.event_subscriptions.get_mut(&topic) { - return subs.remove(&token.0).is_some(); + let res = subs.remove(&token.0).is_some(); + return res; } false @@ -591,7 +644,7 @@ impl Actor { signed_entry: SignedEntry, ) -> Result<()> { let topic = TopicId::from_bytes(*signed_entry.entry().namespace().as_bytes()); - let subs = self.event_subscriptions.get(&topic); + let subs = self.event_subscriptions.get_mut(&topic); match origin { InsertOrigin::Local => { let entry = signed_entry.entry().clone(); @@ -643,6 +696,15 @@ impl Actor { } } -async fn notify_all(subs: &HashMap, event: LiveEvent) { - futures::future::join_all(subs.values().map(|sub| sub(event.clone()))).await; +async fn notify_all(subs: &mut HashMap, event: LiveEvent) { + let res = futures::future::join_all( + subs.iter() + .map(|(idx, sub)| sub(event.clone()).map(|res| (*idx, res))), + ) + .await; + for (idx, res) in res { + if matches!(res, KeepCallback::Drop) { + subs.remove(&idx); + } + } } diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync/rpc.rs index ed7ca06087..2ad7b264e1 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync/rpc.rs @@ -7,16 +7,19 @@ use iroh_sync::{store::Store, sync::Namespace}; use itertools::Itertools; use rand::rngs::OsRng; -use crate::rpc_protocol::{ - AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, - DocCreateRequest, DocCreateResponse, DocGetRequest, DocGetResponse, DocImportRequest, - DocImportResponse, DocInfoRequest, DocInfoResponse, DocListRequest, DocListResponse, - DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, - DocStartSyncResponse, DocStopSyncRequest, DocStopSyncResponse, DocSubscribeRequest, - DocSubscribeResponse, DocTicket, RpcResult, ShareMode, +use crate::{ + rpc_protocol::{ + AuthorCreateRequest, AuthorCreateResponse, AuthorListRequest, AuthorListResponse, + DocCreateRequest, DocCreateResponse, DocGetRequest, DocGetResponse, DocImportRequest, + DocImportResponse, DocInfoRequest, DocInfoResponse, DocListRequest, DocListResponse, + DocSetRequest, DocSetResponse, DocShareRequest, DocShareResponse, DocStartSyncRequest, + DocStartSyncResponse, DocStopSyncRequest, DocStopSyncResponse, DocSubscribeRequest, + DocSubscribeResponse, DocTicket, RpcResult, ShareMode, + }, + sync::KeepCallback, }; -use super::{engine::SyncEngine, PeerSource}; +use super::{engine::SyncEngine, LiveStatus, PeerSource}; /// Capacity for the flume channels to forward sync store iterators to async RPC streams. const ITER_CHANNEL_CAP: usize = 64; @@ -74,9 +77,13 @@ impl SyncEngine { } pub async fn doc_info(&self, req: DocInfoRequest) -> RpcResult { - let replica = self.get_replica(&req.doc_id)?; - self.start_sync(replica.namespace(), vec![]).await?; - Ok(DocInfoResponse {}) + let _replica = self.get_replica(&req.doc_id)?; + let status = self.live.status(req.doc_id).await?; + let status = status.unwrap_or(LiveStatus { + active: false, + subscriptions: 0, + }); + Ok(DocInfoResponse { status }) } pub async fn doc_share(&self, req: DocShareRequest) -> RpcResult { @@ -100,19 +107,31 @@ impl SyncEngine { pub async fn doc_subscribe( &self, req: DocSubscribeRequest, - ) -> impl Stream { + ) -> impl Stream> { let (s, r) = flume::bounded(64); - self.live - .subscribe(req.doc_id, move |event| { + let res = self + .live + .subscribe(req.doc_id, { let s = s.clone(); - async move { - s.send_async(DocSubscribeResponse { event }).await.ok(); + move |event| { + let s = s.clone(); + async move { + // Send event over the channel, unsubscribe if the channel is closed. + match s.send_async(Ok(DocSubscribeResponse { event })).await { + Err(_err) => KeepCallback::Drop, + Ok(()) => KeepCallback::Keep, + } + } + .boxed() } - .boxed() }) - .await - .unwrap(); // TODO: handle error - + .await; + match res { + Err(err) => { + s.send_async(Err(err.into())).await.ok(); + } + Ok(_token) => {} + }; r.into_stream() } diff --git a/iroh/tests/sync.rs b/iroh/tests/sync.rs index 023a83d346..6cf21a9d9b 100644 --- a/iroh/tests/sync.rs +++ b/iroh/tests/sync.rs @@ -146,6 +146,35 @@ async fn sync_full_basic() -> Result<()> { Ok(()) } +#[tokio::test] +async fn sync_subscribe_stop() -> Result<()> { + setup_logging(); + let rt = test_runtime(); + let node = spawn_node(rt).await?; + let client = node.client(); + + let doc = client.create_doc().await?; + let author = client.create_author().await?; + doc.start_sync(vec![]).await?; + + let status = doc.status().await?; + assert!(status.active); + assert_eq!(status.subscriptions, 0); + + let sub = doc.subscribe().await?; + let status = doc.status().await?; + assert_eq!(status.subscriptions, 1); + drop(sub); + + doc.set_bytes(author, b"x".to_vec(), b"x".to_vec()).await?; + let status = doc.status().await?; + assert_eq!(status.subscriptions, 0); + + node.shutdown(); + + Ok(()) +} + async fn assert_latest(doc: &Doc, key: &[u8], value: &[u8]) { let content = get_latest(doc, key).await.unwrap(); assert_eq!(content, value.to_vec()); From a799e653dc1526510ee1c9930e35c37959d6ae73 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 11:32:12 +0200 Subject: [PATCH 150/172] fix: properly format events on doc watch command --- iroh/src/commands/sync.rs | 46 ++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index acb78e0a8c..951a156be2 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -6,10 +6,7 @@ use iroh::{ client::quic::Iroh, rpc_protocol::{DocTicket, ShareMode}, }; -use iroh_sync::{ - store::GetFilter, - sync::{AuthorId, NamespaceId, SignedEntry}, -}; +use iroh_sync::{store::GetFilter, AuthorId, Entry, NamespaceId}; use crate::config::ConsoleEnv; @@ -172,7 +169,7 @@ impl DocCommands { let key = key.as_bytes().to_vec(); let value = value.as_bytes().to_vec(); let entry = doc.set_bytes(author, key, value).await?; - println!("{}", fmt_entry(&entry)); + println!("{}", fmt_entry(entry.entry())); } Self::Get { doc_id, @@ -197,7 +194,7 @@ impl DocCommands { let mut stream = doc.get(filter).await?; while let Some(entry) = stream.try_next().await? { - println!("{}", fmt_entry(&entry)); + println!("{}", fmt_entry(entry.entry())); if content { if entry.content_len() < MAX_DISPLAY_CONTENT_LEN { match doc.get_content_bytes(&entry).await { @@ -237,7 +234,7 @@ impl DocCommands { }; let mut stream = doc.get(filter).await?; while let Some(entry) = stream.try_next().await? { - println!("{}", fmt_entry(&entry)); + println!("{}", fmt_entry(entry.entry())); } } Self::Watch { doc_id } => { @@ -245,7 +242,26 @@ impl DocCommands { let mut stream = doc.subscribe().await?; while let Some(event) = stream.next().await { let event = event?; - println!("{event:?}"); + match event { + iroh::sync::LiveEvent::InsertLocal { entry } => { + println!("local change: {}", fmt_entry(&entry)) + } + iroh::sync::LiveEvent::InsertRemote { + entry, + from, + content_status, + } => { + println!( + "remote change: {} (via @{}, content {:?})", + fmt_entry(&entry), + fmt_short(from.as_bytes()), + content_status + ) + } + iroh::sync::LiveEvent::ContentReady { hash } => { + println!("content ready: {}", fmt_short(hash.as_bytes())) + } + } } } } @@ -285,17 +301,17 @@ impl AuthorCommands { } } -fn fmt_entry(entry: &SignedEntry) -> String { - let id = entry.entry().id(); +fn fmt_entry(entry: &Entry) -> String { + let id = entry.id(); let key = std::str::from_utf8(id.key()).unwrap_or(""); - let author = fmt_hash(id.author().as_bytes()); - let hash = entry.entry().record().content_hash(); - let hash = fmt_hash(hash.as_bytes()); - let len = HumanBytes(entry.entry().record().content_len()); + let author = fmt_short(id.author().as_bytes()); + let hash = entry.record().content_hash(); + let hash = fmt_short(hash.as_bytes()); + let len = HumanBytes(entry.record().content_len()); format!("@{author}: {key} = {hash} ({len})",) } -fn fmt_hash(hash: impl AsRef<[u8]>) -> String { +fn fmt_short(hash: impl AsRef<[u8]>) -> String { let mut text = data_encoding::BASE32_NOPAD.encode(&hash.as_ref()[..5]); text.make_ascii_lowercase(); format!("{}…", &text) From aa511b0c96759d205796ca3355c7316b03652b4d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 11:40:10 +0200 Subject: [PATCH 151/172] feat(repl): abort running command with ctrl-c --- iroh/src/commands/repl.rs | 53 ++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 96d1f879cd..2511e6bb0c 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use colored::Colorize; use iroh::client::quic::RpcClient; @@ -7,7 +7,7 @@ use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; use crate::{ - commands::sync, + commands::{sync::{self, DocCommands, AuthorCommands}, RpcCommands}, config::{ConsoleEnv, ConsolePaths}, }; @@ -16,24 +16,37 @@ pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { println!("Type `{}` for a list of commands.", "help".bold()); let mut repl_rx = Repl::spawn(env.clone()); while let Some((event, reply)) = repl_rx.recv().await { - let (next, res) = match event { - ReplCmd::Rpc(super::RpcCommands::Sync(sync::Commands::Doc { - command: sync::DocCommands::Switch { id }, - })) => { - env.set_doc(id); - (ToRepl::UpdateEnv(env.clone()), Ok(())) - } - ReplCmd::Rpc(super::RpcCommands::Sync(sync::Commands::Author { - command: sync::AuthorCommands::Switch { id }, - })) => { - let res = env.save_author(id); - (ToRepl::UpdateEnv(env.clone()), res) - } - ReplCmd::Rpc(cmd) => { - let res = cmd.run(client.clone(), env.clone()).await; - (ToRepl::Continue, res) + let fut = async { + match event { + // handle doc switch command + ReplCmd::Rpc(RpcCommands::Sync(sync::Commands::Doc { + command: DocCommands::Switch { id }, + })) => { + env.set_doc(id); + (ToRepl::UpdateEnv(env.clone()), Ok(())) + } + // handle author switch command + ReplCmd::Rpc(RpcCommands::Sync(sync::Commands::Author { + command: AuthorCommands::Switch { id }, + })) => { + let res = env.save_author(id); + (ToRepl::UpdateEnv(env.clone()), res) + } + // handle any other comand + ReplCmd::Rpc(cmd) => { + let res = cmd.run(client.clone(), env.clone()).await; + (ToRepl::Continue, res) + } + // handle exit + ReplCmd::Exit => (ToRepl::Exit, Ok(())), } - ReplCmd::Exit => (ToRepl::Exit, Ok(())), + }; + + // allow to abort a running command with Ctrl-C + let (next, res) = tokio::select! { + biased; + _ = tokio::signal::ctrl_c() => (ToRepl::Continue, Err(anyhow!("aborted"))), + (next, res) = fut => (next, res) }; if let Err(err) = res { @@ -137,7 +150,7 @@ impl Repl { #[derive(Debug, Parser)] pub enum ReplCmd { #[clap(flatten)] - Rpc(#[clap(subcommand)] super::RpcCommands), + Rpc(#[clap(subcommand)] RpcCommands), /// Quit the Iroh console #[clap(alias = "quit")] Exit, From 5d41edbd4d9a4e93561a33b87d7e29a60d6ac61f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 11:42:56 +0200 Subject: [PATCH 152/172] docs: config --- iroh/src/config.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iroh/src/config.rs b/iroh/src/config.rs index ac0927b95c..4a0b225e09 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -211,9 +211,13 @@ impl NodeConfig { } } +/// Environment for CLI and REPL #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] pub struct ConsoleEnv { + /// Active author. Read from IROH_AUTHOR env variable. + /// For console also read from/persisted to a file (see [`ConsolePaths::DefaultAuthor`]) pub author: Option, + /// Active doc. Read from IROH_DOC env variable. Not persisted. pub doc: Option, } impl ConsoleEnv { From ce63a3465e6a957fefc20a2686d579e78a43eabb Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 11:50:00 +0200 Subject: [PATCH 153/172] fix: docs --- iroh/src/commands/repl.rs | 12 +++++++++--- iroh/src/commands/sync.rs | 15 +++++++++------ iroh/src/config.rs | 6 +++--- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 2511e6bb0c..bdb612afce 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Result}; +use anyhow::Result; use clap::{Parser, Subcommand}; use colored::Colorize; use iroh::client::quic::RpcClient; @@ -7,7 +7,10 @@ use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; use crate::{ - commands::{sync::{self, DocCommands, AuthorCommands}, RpcCommands}, + commands::{ + sync::{self, AuthorCommands, DocCommands}, + RpcCommands, + }, config::{ConsoleEnv, ConsolePaths}, }; @@ -45,7 +48,10 @@ pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { // allow to abort a running command with Ctrl-C let (next, res) = tokio::select! { biased; - _ = tokio::signal::ctrl_c() => (ToRepl::Continue, Err(anyhow!("aborted"))), + _ = tokio::signal::ctrl_c() => { + println!("aborted"); + (ToRepl::Continue, Ok(())) + } (next, res) = fut => (next, res) }; diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 951a156be2..40a1be7afd 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -52,7 +52,10 @@ pub enum DocCommands { List, /// Share a document with peers. Share { - /// Set the document + /// Document to operate on. + /// + /// Required unless the document is set through the IROH_DOC environment variable. + /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] doc_id: Option, mode: ShareMode, @@ -62,13 +65,13 @@ pub enum DocCommands { /// Document to operate on. /// /// Required unless the document is set through the IROH_DOC environment variable. - /// Within the Iroh console, the active document can also set with `set-doc`. + /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] doc_id: Option, /// Author of the entry. /// /// Required unless the author is set through the IROH_AUTHOR environment variable. - /// Within the Iroh console, the active author can also set with `set-author`. + /// Within the Iroh console, the active author can also set with `author switch`. #[clap(short, long)] author: Option, /// Key to the entry (parsed as UTF-8 string). @@ -83,7 +86,7 @@ pub enum DocCommands { /// Document to operate on. /// /// Required unless the document is set through the IROH_DOC environment variable. - /// Within the Iroh console, the active document can also set with `set-doc`. + /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] doc_id: Option, /// Key to the entry (parsed as UTF-8 string). @@ -109,7 +112,7 @@ pub enum DocCommands { /// Document to operate on. /// /// Required unless the document is set through the IROH_DOC environment variable. - /// Within the Iroh console, the active document can also set with `set-doc`. + /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] doc_id: Option, /// Filter by author. @@ -127,7 +130,7 @@ pub enum DocCommands { /// Document to operate on. /// /// Required unless the document is set through the IROH_DOC environment variable. - /// Within the Iroh console, the active document can also set with `set-doc`. + /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] doc_id: Option, }, diff --git a/iroh/src/config.rs b/iroh/src/config.rs index 4a0b225e09..67778c1e2e 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -214,7 +214,7 @@ impl NodeConfig { /// Environment for CLI and REPL #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] pub struct ConsoleEnv { - /// Active author. Read from IROH_AUTHOR env variable. + /// Active author. Read from IROH_AUTHOR env variable. /// For console also read from/persisted to a file (see [`ConsolePaths::DefaultAuthor`]) pub author: Option, /// Active doc. Read from IROH_DOC env variable. Not persisted. @@ -274,14 +274,14 @@ impl ConsoleEnv { pub fn doc(&self, arg: Option) -> anyhow::Result { let doc_id = arg.or(self.doc).ok_or_else(|| { - anyhow!("Missing document id. Set the current document with the `IROH_DOC` environment variable or by passing the `-d` flag. In the console, you can set the active document with `set-doc`.") + anyhow!("Missing document id. Set the current document with the `IROH_DOC` environment variable or by passing the `-d` flag.\nIn the console, you can set the active document with `doc switch`.") })?; Ok(doc_id) } pub fn author(&self, arg: Option) -> anyhow::Result { let author_id = arg.or(self.author).ok_or_else(|| { - anyhow!("Missing author id. Set the current author with the `IROH_AUTHOR` environment variable or by passing the `-a` flag. In the console, you can set the active author with `set-author`.") + anyhow!("Missing author id. Set the current author with the `IROH_AUTHOR` environment variable or by passing the `-a` flag.\nIn the console, you can set the active author with `author switch`.") })?; Ok(author_id) From 6a443fcab044b50b9ecfaa8fe5d716870d8955c8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 12:01:49 +0200 Subject: [PATCH 154/172] fix: better errors --- iroh/src/config.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/iroh/src/config.rs b/iroh/src/config.rs index 67778c1e2e..da2af2328b 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -219,17 +219,20 @@ pub struct ConsoleEnv { pub author: Option, /// Active doc. Read from IROH_DOC env variable. Not persisted. pub doc: Option, + /// True if running in a Iroh console session, false for a CLI command + pub is_console: bool, } impl ConsoleEnv { /// Read from environment variables and the console config file. pub fn for_console() -> Result { - let author = match Self::get_console_default_author()? { + let author = match env_author()? { Some(author) => Some(author), - None => env_author()?, + None => Self::get_console_default_author()?, }; Ok(Self { author, doc: env_doc()?, + is_console: true, }) } @@ -238,6 +241,7 @@ impl ConsoleEnv { Ok(Self { author: env_author()?, doc: env_doc()?, + is_console: false, }) } @@ -274,30 +278,39 @@ impl ConsoleEnv { pub fn doc(&self, arg: Option) -> anyhow::Result { let doc_id = arg.or(self.doc).ok_or_else(|| { - anyhow!("Missing document id. Set the current document with the `IROH_DOC` environment variable or by passing the `-d` flag.\nIn the console, you can set the active document with `doc switch`.") + anyhow!( + "Missing document id. Set the active document with the `IROH_DOC` environment variable or the `-d` option.\n\ + In the console, you can also set the active document with `doc switch`." + ) })?; Ok(doc_id) } pub fn author(&self, arg: Option) -> anyhow::Result { let author_id = arg.or(self.author).ok_or_else(|| { - anyhow!("Missing author id. Set the current author with the `IROH_AUTHOR` environment variable or by passing the `-a` flag.\nIn the console, you can set the active author with `author switch`.") - -})?; + anyhow!( + "Missing author id. Set the active author with the `IROH_AUTHOR` environment variable or the `-a` option.\n\ + In the console, you can also set the active author with `author switch`." + ) + })?; Ok(author_id) } } fn env_author() -> Result> { match env_var(ENV_AUTHOR) { - Ok(s) => Ok(Some(AuthorId::from_str(&s)?)), + Ok(s) => Ok(Some( + AuthorId::from_str(&s).context("Failed to parse IROH_AUTHOR environment variable")?, + )), Err(_) => Ok(None), } } fn env_doc() -> Result> { match env_var(ENV_DOC) { - Ok(s) => Ok(Some(NamespaceId::from_str(&s)?)), + Ok(s) => Ok(Some( + NamespaceId::from_str(&s).context("Failed to parse IROH_DOC environment variable")?, + )), Err(_) => Ok(None), } } From 9ecaa07aa4619e5fe30cccae6c56fcc00788db25 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 12:34:46 +0200 Subject: [PATCH 155/172] feat: better env handling for Iroh console --- iroh/src/commands/repl.rs | 36 ++++----------------- iroh/src/commands/sync.rs | 61 +++++++++++++++++++++++++++++------- iroh/src/config.rs | 66 ++++++++++++++++++++++++++++++--------- 3 files changed, 106 insertions(+), 57 deletions(-) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index bdb612afce..1c33647c05 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -7,40 +7,21 @@ use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; use crate::{ - commands::{ - sync::{self, AuthorCommands, DocCommands}, - RpcCommands, - }, + commands::RpcCommands, config::{ConsoleEnv, ConsolePaths}, }; -pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { +pub async fn run(client: RpcClient, env: ConsoleEnv) -> Result<()> { println!("{}", "Welcome to the Iroh console!".purple().bold()); println!("Type `{}` for a list of commands.", "help".bold()); let mut repl_rx = Repl::spawn(env.clone()); while let Some((event, reply)) = repl_rx.recv().await { let fut = async { match event { - // handle doc switch command - ReplCmd::Rpc(RpcCommands::Sync(sync::Commands::Doc { - command: DocCommands::Switch { id }, - })) => { - env.set_doc(id); - (ToRepl::UpdateEnv(env.clone()), Ok(())) - } - // handle author switch command - ReplCmd::Rpc(RpcCommands::Sync(sync::Commands::Author { - command: AuthorCommands::Switch { id }, - })) => { - let res = env.save_author(id); - (ToRepl::UpdateEnv(env.clone()), res) - } - // handle any other comand ReplCmd::Rpc(cmd) => { let res = cmd.run(client.clone(), env.clone()).await; (ToRepl::Continue, res) } - // handle exit ReplCmd::Exit => (ToRepl::Exit, Ok(())), } }; @@ -70,8 +51,6 @@ pub async fn run(client: RpcClient, mut env: ConsoleEnv) -> Result<()> { pub enum ToRepl { /// Continue execution by reading the next command Continue, - /// Continue execution by reading the next command, and update the repl state - UpdateEnv(ConsoleEnv), /// Exit the repl Exit, } @@ -91,7 +70,7 @@ impl Repl { }); cmd_rx } - pub fn run(mut self) -> anyhow::Result<()> { + pub fn run(self) -> anyhow::Result<()> { let mut rl = DefaultEditor::with_config(Config::builder().check_cursor_position(true).build())?; let history_path = ConsolePaths::History.with_env()?; @@ -119,10 +98,7 @@ impl Repl { } // wait for reply from main thread match reply_rx.blocking_recv()? { - ToRepl::UpdateEnv(env) => { - self.env = env; - } - ToRepl::Continue => {} + ToRepl::Continue => continue, ToRepl::Exit => break, } } @@ -132,14 +108,14 @@ impl Repl { pub fn prompt(&self) -> String { let mut pwd = String::new(); - if let Some(author) = &self.env.author { + if let Some(author) = &self.env.author(None).ok() { pwd.push_str(&format!( "{}{} ", "author:".blue(), fmt_short(author.as_bytes()).blue().bold(), )); } - if let Some(doc) = &self.env.doc { + if let Some(doc) = &self.env.doc(None).ok() { pwd.push_str(&format!( "{}{} ", "doc:".blue(), diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 40a1be7afd..b71a56f615 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -35,7 +35,7 @@ impl Commands { let iroh = Iroh::new(client); match self { Self::Doc { command } => command.run(&iroh, env).await, - Self::Author { command } => command.run(&iroh).await, + Self::Author { command } => command.run(&iroh, env).await, } } } @@ -45,9 +45,18 @@ pub enum DocCommands { /// Set the active document (only works within the Iroh console). Switch { id: NamespaceId }, /// Create a new document. - Init, + Init { + /// Switch to the created document (only in the Iroh console). + #[clap(long)] + switch: bool, + }, /// Join a document from a ticket. - Join { ticket: DocTicket }, + Join { + ticket: DocTicket, + /// Switch to the joined document (only in the Iroh console). + #[clap(long)] + switch: bool, + }, /// List documents. List, /// Share a document with peers. @@ -139,16 +148,32 @@ pub enum DocCommands { impl DocCommands { pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { - Self::Switch { .. } => { - bail!("This command is only supported in the Iroh console") + Self::Switch { id } => { + env.set_doc(id)?; } - Self::Init => { + Self::Init { switch } => { + if switch && !env.is_console() { + bail!("The --switch flag is only supported within the Iroh console."); + } + let doc = iroh.create_doc().await?; println!("{}", doc.id()); + + if switch { + env.set_doc(doc.id())?; + } } - Self::Join { ticket } => { + Self::Join { ticket, switch } => { + if switch && !env.is_console() { + bail!("The --switch flag is only supported within the Iroh console."); + } + let doc = iroh.import_doc(ticket).await?; println!("{}", doc.id()); + + if switch { + env.set_doc(doc.id())?; + } } Self::List => { let mut stream = iroh.list_docs().await?; @@ -277,17 +302,21 @@ pub enum AuthorCommands { /// Set the active author (only works within the Iroh console). Switch { id: AuthorId }, /// Create a new author. - Create, + Create { + /// Switch to the created author (only in the Iroh console). + #[clap(long)] + switch: bool, + }, /// List authors. #[clap(alias = "ls")] List, } impl AuthorCommands { - pub async fn run(self, iroh: &Iroh) -> Result<()> { + pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { - Self::Switch { .. } => { - bail!("This command is only supported in the Iroh console") + Self::Switch { id } => { + env.set_author(id)?; } Self::List => { let mut stream = iroh.list_authors().await?; @@ -295,9 +324,17 @@ impl AuthorCommands { println!("{}", author_id); } } - Self::Create => { + Self::Create { switch } => { + if switch && !env.is_console() { + bail!("The --switch flag is only supported within the Iroh console."); + } + let author_id = iroh.create_author().await?; println!("{}", author_id); + + if switch { + env.set_author(author_id)?; + } } } Ok(()) diff --git a/iroh/src/config.rs b/iroh/src/config.rs index da2af2328b..5c4c06c334 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -5,6 +5,7 @@ use std::{ env, fmt, path::{Path, PathBuf}, str::FromStr, + sync::Arc, }; use anyhow::{anyhow, bail, Context, Result}; @@ -14,6 +15,7 @@ use iroh_net::{ derp::{DerpMap, DerpRegion}, }; use iroh_sync::{AuthorId, NamespaceId}; +use parking_lot::Mutex; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tracing::debug; @@ -212,15 +214,21 @@ impl NodeConfig { } /// Environment for CLI and REPL +/// +/// This is cheaply cloneable and has interior mutability. If not running in the console +/// environment, [Self::set_doc] and [Self::set_author] will lead to an error, as changing the +/// environment is only supported within the console. +#[derive(Clone, Debug)] +pub struct ConsoleEnv(Arc>); + #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] -pub struct ConsoleEnv { +struct ConsoleEnvInner { /// Active author. Read from IROH_AUTHOR env variable. /// For console also read from/persisted to a file (see [`ConsolePaths::DefaultAuthor`]) - pub author: Option, + author: Option, /// Active doc. Read from IROH_DOC env variable. Not persisted. - pub doc: Option, - /// True if running in a Iroh console session, false for a CLI command - pub is_console: bool, + doc: Option, + is_console: bool, } impl ConsoleEnv { /// Read from environment variables and the console config file. @@ -229,20 +237,22 @@ impl ConsoleEnv { Some(author) => Some(author), None => Self::get_console_default_author()?, }; - Ok(Self { + let env = ConsoleEnvInner { author, doc: env_doc()?, is_console: true, - }) + }; + Ok(Self(Arc::new(Mutex::new(env)))) } /// Read only from environment variables. pub fn for_cli() -> Result { - Ok(Self { + let env = ConsoleEnvInner { author: env_author()?, doc: env_doc()?, is_console: false, - }) + }; + Ok(Self(Arc::new(Mutex::new(env)))) } fn get_console_default_author() -> anyhow::Result> { @@ -263,8 +273,21 @@ impl ConsoleEnv { } } - pub fn save_author(&mut self, author: AuthorId) -> anyhow::Result<()> { - self.author = Some(author); + /// True if running in a Iroh console session, false for a CLI command + pub fn is_console(&self) -> bool { + self.0.lock().is_console + } + + /// Set the active author. + /// + /// Will error if not running in the Iroh console. + /// Will persist to a file in the Iroh data dir otherwise. + pub fn set_author(&self, author: AuthorId) -> anyhow::Result<()> { + let mut inner = self.0.lock(); + if !inner.is_console { + bail!("Switching the author is only supported within the Iroh console, not on the command line"); + } + inner.author = Some(author); std::fs::write( ConsolePaths::DefaultAuthor.with_env()?, author.to_string().as_bytes(), @@ -272,12 +295,23 @@ impl ConsoleEnv { Ok(()) } - pub fn set_doc(&mut self, doc: NamespaceId) { - self.doc = Some(doc); + /// Set the active document. + /// + /// Will error if not running in the Iroh console. + /// Will not persist, only valid for the current console session. + pub fn set_doc(&self, doc: NamespaceId) -> anyhow::Result<()> { + let mut inner = self.0.lock(); + if !inner.is_console { + bail!("Switching the document is only supported within the Iroh console, not on the command line"); + } + inner.doc = Some(doc); + Ok(()) } + /// Get the active document. pub fn doc(&self, arg: Option) -> anyhow::Result { - let doc_id = arg.or(self.doc).ok_or_else(|| { + let inner = self.0.lock(); + let doc_id = arg.or(inner.doc).ok_or_else(|| { anyhow!( "Missing document id. Set the active document with the `IROH_DOC` environment variable or the `-d` option.\n\ In the console, you can also set the active document with `doc switch`." @@ -286,8 +320,10 @@ impl ConsoleEnv { Ok(doc_id) } + /// Get the active author. pub fn author(&self, arg: Option) -> anyhow::Result { - let author_id = arg.or(self.author).ok_or_else(|| { + let inner = self.0.lock(); + let author_id = arg.or(inner.author).ok_or_else(|| { anyhow!( "Missing author id. Set the active author with the `IROH_AUTHOR` environment variable or the `-a` option.\n\ In the console, you can also set the active author with `author switch`." From f44987796ec59ab3523c0352210a8b4d0f9832ae Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 12:50:49 +0200 Subject: [PATCH 156/172] chore: doc links --- iroh/src/rpc_protocol.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index d037c2fd76..24b1d38ffc 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -186,10 +186,6 @@ impl ServerStreamingMsg for ListCollectionsRequest { type Response = ListCollectionsResponse; } -/// A request to watch for the node status -#[derive(Serialize, Deserialize, Debug)] -pub struct WatchRequest; - /// A request to get the version of the node #[derive(Serialize, Deserialize, Debug)] pub struct VersionRequest; @@ -211,7 +207,7 @@ impl RpcMsg for ShutdownRequest { /// A request to get information about the identity of the node /// -/// See [`IdResponse`] for the response. +/// See [`SttausResponse`] for the response. #[derive(Serialize, Deserialize, Debug)] pub struct StatusRequest; @@ -219,13 +215,6 @@ impl RpcMsg for StatusRequest { type Response = StatusResponse; } -/// The response to a watch request -#[derive(Serialize, Deserialize, Debug)] -pub struct WatchResponse { - /// The version of the node - pub version: String, -} - /// The response to a version request #[derive(Serialize, Deserialize, Debug)] pub struct StatusResponse { @@ -237,6 +226,10 @@ pub struct StatusResponse { pub version: String, } +/// A request to watch for the node status +#[derive(Serialize, Deserialize, Debug)] +pub struct WatchRequest; + impl Msg for WatchRequest { type Pattern = ServerStreaming; } @@ -245,6 +238,14 @@ impl ServerStreamingMsg for WatchRequest { type Response = WatchResponse; } +/// The response to a watch request +#[derive(Serialize, Deserialize, Debug)] +pub struct WatchResponse { + /// The version of the node + pub version: String, +} + + /// The response to a version request #[derive(Serialize, Deserialize, Debug)] pub struct VersionResponse { From a6b7068ebc24200ed5cece7b2601f77cc10d5a13 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 12:52:14 +0200 Subject: [PATCH 157/172] chore: fmt --- iroh/src/rpc_protocol.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 24b1d38ffc..8fa67d2204 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -245,7 +245,6 @@ pub struct WatchResponse { pub version: String, } - /// The response to a version request #[derive(Serialize, Deserialize, Debug)] pub struct VersionResponse { From 8400c90efbe0332baf9ae82358aee90795cfd2b8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 12:58:27 +0200 Subject: [PATCH 158/172] chore: undo unneeded change --- iroh/src/config.rs | 102 ++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/iroh/src/config.rs b/iroh/src/config.rs index 5c4c06c334..f142f8768c 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -16,7 +16,7 @@ use iroh_net::{ }; use iroh_sync::{AuthorId, NamespaceId}; use parking_lot::Mutex; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use tracing::debug; /// CONFIG_FILE_NAME is the name of the optional config file located in the iroh home directory @@ -191,7 +191,7 @@ impl NodeConfig { pub fn from_env(additional_config_source: Option<&Path>) -> anyhow::Result { let config_path = iroh_config_path(CONFIG_FILE_NAME).context("invalid config path")?; let sources = [Some(config_path.as_path()), additional_config_source]; - let config = load_config( + let config = Self::load( // potential config files &sources, // env var prefix for this config @@ -202,6 +202,55 @@ impl NodeConfig { )?; Ok(config) } + + /// Make a config using a default, files, environment variables, and commandline flags. + /// + /// Later items in the *file_paths* slice will have a higher priority than earlier ones. + /// + /// Environment variables are expected to start with the *env_prefix*. Nested fields can be + /// accessed using `.`, if your environment allows env vars with `.` + /// + /// Note: For the metrics configuration env vars, it is recommended to use the metrics + /// specific prefix `IROH_METRICS` to set a field in the metrics config. You can use the + /// above dot notation to set a metrics field, eg, `IROH_CONFIG_METRICS.SERVICE_NAME`, but + /// only if your environment allows it + pub fn load( + file_paths: &[Option<&Path>], + env_prefix: &str, + flag_overrides: HashMap, + ) -> Result + where + S: AsRef, + V: Into, + { + let mut builder = config::Config::builder(); + + // layer on config options from files + for path in file_paths.iter().flatten() { + if path.exists() { + let p = path.to_str().ok_or_else(|| anyhow::anyhow!("empty path"))?; + builder = builder.add_source(File::with_name(p)); + } + } + + // next, add any environment variables + builder = builder.add_source( + Environment::with_prefix(env_prefix) + .separator("__") + .try_parsing(true), + ); + + // finally, override any values + for (flag, val) in flag_overrides.into_iter() { + builder = builder.set_override(flag, val)?; + } + + let cfg = builder.build()?; + debug!("make_config:\n{:#?}\n", cfg); + let cfg = cfg.try_deserialize()?; + Ok(cfg) + } + /// Constructs a `DerpMap` based on the current configuration. pub fn derp_map(&self) -> Option { if self.derp_regions.is_empty() { @@ -351,55 +400,6 @@ fn env_doc() -> Result> { } } -/// Make a config using a default, files, environment variables, and commandline flags. -/// -/// Later items in the *file_paths* slice will have a higher priority than earlier ones. -/// -/// Environment variables are expected to start with the *env_prefix*. Nested fields can be -/// accessed using `.`, if your environment allows env vars with `.` -/// -/// Note: For the metrics configuration env vars, it is recommended to use the metrics -/// specific prefix `IROH_METRICS` to set a field in the metrics config. You can use the -/// above dot notation to set a metrics field, eg, `IROH_CONFIG_METRICS.SERVICE_NAME`, but -/// only if your environment allows it -pub fn load_config( - file_paths: &[Option<&Path>], - env_prefix: &str, - flag_overrides: HashMap, -) -> Result -where - C: DeserializeOwned, - S: AsRef, - V: Into, -{ - let mut builder = config::Config::builder(); - - // layer on config options from files - for path in file_paths.iter().flatten() { - if path.exists() { - let p = path.to_str().ok_or_else(|| anyhow::anyhow!("empty path"))?; - builder = builder.add_source(File::with_name(p)); - } - } - - // next, add any environment variables - builder = builder.add_source( - Environment::with_prefix(env_prefix) - .separator("__") - .try_parsing(true), - ); - - // finally, override any values - for (flag, val) in flag_overrides.into_iter() { - builder = builder.set_override(flag, val)?; - } - - let cfg = builder.build()?; - debug!("make_config:\n{:#?}\n", cfg); - let cfg = cfg.try_deserialize()?; - Ok(cfg) -} - /// Name of directory that wraps all iroh files in a given application directory const IROH_DIR: &str = "iroh"; From 5bb254a888a0550dfca651a3da17712ace7e11fb Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 13:17:20 +0200 Subject: [PATCH 159/172] refactor: simplify repl even further --- iroh/src/commands/repl.rs | 63 +++++++++++---------------------------- 1 file changed, 17 insertions(+), 46 deletions(-) diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index 1c33647c05..fe75405228 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -14,53 +14,29 @@ use crate::{ pub async fn run(client: RpcClient, env: ConsoleEnv) -> Result<()> { println!("{}", "Welcome to the Iroh console!".purple().bold()); println!("Type `{}` for a list of commands.", "help".bold()); - let mut repl_rx = Repl::spawn(env.clone()); - while let Some((event, reply)) = repl_rx.recv().await { - let fut = async { - match event { - ReplCmd::Rpc(cmd) => { - let res = cmd.run(client.clone(), env.clone()).await; - (ToRepl::Continue, res) - } - ReplCmd::Exit => (ToRepl::Exit, Ok(())), - } - }; - + let mut from_repl = Repl::spawn(env.clone()); + while let Some((cmd, reply)) = from_repl.recv().await { // allow to abort a running command with Ctrl-C - let (next, res) = tokio::select! { + tokio::select! { biased; - _ = tokio::signal::ctrl_c() => { - println!("aborted"); - (ToRepl::Continue, Ok(())) + _ = tokio::signal::ctrl_c() => {}, + res = cmd.run(client.clone(), env.clone()) => { + if let Err(err) = res { + println!("{} {:?}", "Error:".red().bold(), err) + } } - (next, res) = fut => (next, res) - }; - - if let Err(err) = res { - println!("{} {:?}", "Error:".red().bold(), err) } - - reply.send(next).ok(); + reply.send(()).ok(); } Ok(()) } -/// Reply to the repl after a command completed -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] -pub enum ToRepl { - /// Continue execution by reading the next command - Continue, - /// Exit the repl - Exit, -} - pub struct Repl { env: ConsoleEnv, - cmd_tx: mpsc::Sender<(ReplCmd, oneshot::Sender)>, + cmd_tx: mpsc::Sender<(RpcCommands, oneshot::Sender<()>)>, } impl Repl { - pub fn spawn(env: ConsoleEnv) -> mpsc::Receiver<(ReplCmd, oneshot::Sender)> { + pub fn spawn(env: ConsoleEnv) -> mpsc::Receiver<(RpcCommands, oneshot::Sender<()>)> { let (cmd_tx, cmd_rx) = mpsc::channel(1); let repl = Repl { env, cmd_tx }; std::thread::spawn(move || { @@ -84,23 +60,18 @@ impl Repl { Ok(line) => { rl.add_history_entry(line.as_str())?; let cmd = parse_cmd::(&line); - if let Some(cmd) = cmd { - self.cmd_tx.blocking_send((cmd, reply_tx))?; - } else { - continue; + match cmd { + None => continue, + Some(ReplCmd::Exit) => break, + Some(ReplCmd::Rpc(cmd)) => self.cmd_tx.blocking_send((cmd, reply_tx))?, } } - Err(ReadlineError::Interrupted | ReadlineError::Eof) => { - self.cmd_tx.blocking_send((ReplCmd::Exit, reply_tx))?; - } + Err(ReadlineError::Interrupted | ReadlineError::Eof) => break, Err(ReadlineError::WindowResized) => continue, Err(err) => return Err(err.into()), } // wait for reply from main thread - match reply_rx.blocking_recv()? { - ToRepl::Continue => continue, - ToRepl::Exit => break, - } + reply_rx.blocking_recv()?; } rl.save_history(&history_path).ok(); Ok(()) From 547ebdede859d1c28a0f2f67cc48c46f5ab37223 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 13:21:03 +0200 Subject: [PATCH 160/172] fix: leftover --- iroh/src/config.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iroh/src/config.rs b/iroh/src/config.rs index f142f8768c..c71342d6be 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -492,8 +492,7 @@ mod tests { #[test] fn test_default_settings() { - let config: NodeConfig = - load_config(&[][..], "__FOO", HashMap::::new()).unwrap(); + let config = NodeConfig::load(&[][..], "__FOO", HashMap::::new()).unwrap(); assert_eq!(config.derp_regions.len(), 2); } From 30abfea385fe2b702eff4a1cac983e656255ebc7 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 13:21:45 +0200 Subject: [PATCH 161/172] fix: doc link --- iroh/src/rpc_protocol.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 8fa67d2204..b687e534b5 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -207,7 +207,7 @@ impl RpcMsg for ShutdownRequest { /// A request to get information about the identity of the node /// -/// See [`SttausResponse`] for the response. +/// See [`StatusResponse`] for the response. #[derive(Serialize, Deserialize, Debug)] pub struct StatusRequest; From c02ecbbe183cfe3f6e9521b8672d372a651e2c69 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 16:59:11 +0200 Subject: [PATCH 162/172] refactor: code structure --- iroh/src/commands.rs | 22 ++++++++++---- iroh/src/commands/repl.rs | 10 +------ iroh/src/commands/sync.rs | 63 +++++++++++---------------------------- 3 files changed, 35 insertions(+), 60 deletions(-) diff --git a/iroh/src/commands.rs b/iroh/src/commands.rs index abe05dcdb9..eeb1fb831f 100644 --- a/iroh/src/commands.rs +++ b/iroh/src/commands.rs @@ -4,7 +4,7 @@ use std::{net::SocketAddr, path::PathBuf}; use anyhow::Result; use clap::{Args, Parser, Subcommand}; use futures::StreamExt; -use iroh::client::quic::RpcClient; +use iroh::client::quic::{Iroh, RpcClient}; use iroh::dial::Ticket; use iroh::rpc_protocol::*; use iroh_bytes::{protocol::RequestToken, util::runtime, Hash}; @@ -13,7 +13,7 @@ use iroh_net::key::{PublicKey, SecretKey}; use crate::config::{ConsoleEnv, NodeConfig}; use self::provide::{ProvideOptions, ProviderRpcPort}; -// use self::sync::SyncEnv; +use self::sync::{AuthorCommands, DocCommands}; const DEFAULT_RPC_PORT: u16 = 0x1337; const MAX_RPC_CONNECTIONS: u32 = 16; @@ -275,9 +275,17 @@ impl FullCommands { #[derive(Subcommand, Debug, Clone)] #[allow(clippy::large_enum_variant)] pub enum RpcCommands { - /// Doc and author commands - #[clap(flatten)] - Sync(#[clap(subcommand)] sync::Commands), + /// Manage documents + Doc { + #[clap(subcommand)] + command: DocCommands, + }, + + /// Manage document authors + Author { + #[clap(subcommand)] + command: AuthorCommands, + }, /// Manage blobs Blob { #[clap(subcommand)] @@ -335,10 +343,12 @@ impl NodeCommands { impl RpcCommands { pub async fn run(self, client: RpcClient, env: ConsoleEnv) -> Result<()> { + let iroh = Iroh::new(client.clone()); match self { Self::Node { command } => command.run(client).await, Self::Blob { command } => command.run(client).await, - Self::Sync(command) => command.run(client, env).await, + Self::Doc { command } => command.run(&iroh, env).await, + Self::Author { command } => command.run(&iroh, env).await, } } } diff --git a/iroh/src/commands/repl.rs b/iroh/src/commands/repl.rs index fe75405228..3f5f9f8f52 100644 --- a/iroh/src/commands/repl.rs +++ b/iroh/src/commands/repl.rs @@ -2,12 +2,11 @@ use anyhow::Result; use clap::{Parser, Subcommand}; use colored::Colorize; use iroh::client::quic::RpcClient; -use iroh_gossip::proto::util::base32; use rustyline::{error::ReadlineError, Config, DefaultEditor}; use tokio::sync::{mpsc, oneshot}; use crate::{ - commands::RpcCommands, + commands::{sync::fmt_short, RpcCommands}, config::{ConsoleEnv, ConsolePaths}, }; @@ -130,10 +129,3 @@ fn parse_cmd(s: &str) -> Option { } } } - -fn fmt_short(bytes: impl AsRef<[u8]>) -> String { - let bytes = bytes.as_ref(); - // we use 5 bytes because this always results in 8 character string in base32 - let len = bytes.len().min(5); - base32::fmt(&bytes[..len]) -} diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index b71a56f615..fe9b11f35c 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -1,5 +1,5 @@ use anyhow::{bail, Result}; -use clap::{Parser, Subcommand}; +use clap::Parser; use futures::{StreamExt, TryStreamExt}; use indicatif::HumanBytes; use iroh::{ @@ -10,36 +10,8 @@ use iroh_sync::{store::GetFilter, AuthorId, Entry, NamespaceId}; use crate::config::ConsoleEnv; -use super::RpcClient; - const MAX_DISPLAY_CONTENT_LEN: u64 = 1024 * 1024; -#[allow(clippy::large_enum_variant)] -#[derive(Subcommand, Debug, Clone)] -pub enum Commands { - /// Manage documents - Doc { - #[clap(subcommand)] - command: DocCommands, - }, - - /// Manage document authors - Author { - #[clap(subcommand)] - command: AuthorCommands, - }, -} - -impl Commands { - pub async fn run(self, client: RpcClient, env: ConsoleEnv) -> Result<()> { - let iroh = Iroh::new(client); - match self { - Self::Doc { command } => command.run(&iroh, env).await, - Self::Author { command } => command.run(&iroh, env).await, - } - } -} - #[derive(Debug, Clone, Parser)] pub enum DocCommands { /// Set the active document (only works within the Iroh console). @@ -145,6 +117,21 @@ pub enum DocCommands { }, } +#[derive(Debug, Clone, Parser)] +pub enum AuthorCommands { + /// Set the active author (only works within the Iroh console). + Switch { id: AuthorId }, + /// Create a new author. + Create { + /// Switch to the created author (only in the Iroh console). + #[clap(long)] + switch: bool, + }, + /// List authors. + #[clap(alias = "ls")] + List, +} + impl DocCommands { pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { @@ -297,21 +284,6 @@ impl DocCommands { } } -#[derive(Debug, Clone, Parser)] -pub enum AuthorCommands { - /// Set the active author (only works within the Iroh console). - Switch { id: AuthorId }, - /// Create a new author. - Create { - /// Switch to the created author (only in the Iroh console). - #[clap(long)] - switch: bool, - }, - /// List authors. - #[clap(alias = "ls")] - List, -} - impl AuthorCommands { pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { @@ -351,7 +323,8 @@ fn fmt_entry(entry: &Entry) -> String { format!("@{author}: {key} = {hash} ({len})",) } -fn fmt_short(hash: impl AsRef<[u8]>) -> String { +/// Format the first 5 bytes of a byte string in bas32 +pub fn fmt_short(hash: impl AsRef<[u8]>) -> String { let mut text = data_encoding::BASE32_NOPAD.encode(&hash.as_ref()[..5]); text.make_ascii_lowercase(); format!("{}…", &text) From fc20a0b11da21becd507011d1954a290a7ec8a35 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 16:59:32 +0200 Subject: [PATCH 163/172] fix: doc not doc_id for arg names --- iroh/src/commands/sync.rs | 45 ++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index fe9b11f35c..3cbce21840 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -38,7 +38,7 @@ pub enum DocCommands { /// Required unless the document is set through the IROH_DOC environment variable. /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] - doc_id: Option, + doc: Option, mode: ShareMode, }, /// Set an entry in a document. @@ -48,7 +48,7 @@ pub enum DocCommands { /// Required unless the document is set through the IROH_DOC environment variable. /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] - doc_id: Option, + doc: Option, /// Author of the entry. /// /// Required unless the author is set through the IROH_AUTHOR environment variable. @@ -69,7 +69,7 @@ pub enum DocCommands { /// Required unless the document is set through the IROH_DOC environment variable. /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] - doc_id: Option, + doc: Option, /// Key to the entry (parsed as UTF-8 string). key: String, /// If true, get all entries that start with KEY. @@ -95,7 +95,7 @@ pub enum DocCommands { /// Required unless the document is set through the IROH_DOC environment variable. /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] - doc_id: Option, + doc: Option, /// Filter by author. #[clap(short, long)] author: Option, @@ -113,14 +113,14 @@ pub enum DocCommands { /// Required unless the document is set through the IROH_DOC environment variable. /// Within the Iroh console, the active document can also set with `doc switch`. #[clap(short, long)] - doc_id: Option, + doc: Option, }, } #[derive(Debug, Clone, Parser)] pub enum AuthorCommands { /// Set the active author (only works within the Iroh console). - Switch { id: AuthorId }, + Switch { author: AuthorId }, /// Create a new author. Create { /// Switch to the created author (only in the Iroh console). @@ -135,8 +135,9 @@ pub enum AuthorCommands { impl DocCommands { pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { - Self::Switch { id } => { - env.set_doc(id)?; + Self::Switch { id: doc } => { + env.set_doc(doc)?; + println!("Active doc is now {}", fmt_short(doc.as_bytes())); } Self::Init { switch } => { if switch && !env.is_console() { @@ -148,6 +149,7 @@ impl DocCommands { if switch { env.set_doc(doc.id())?; + println!("Active doc is now {}", fmt_short(doc.id().as_bytes())); } } Self::Join { ticket, switch } => { @@ -160,6 +162,7 @@ impl DocCommands { if switch { env.set_doc(doc.id())?; + println!("Active doc is now {}", fmt_short(doc.id().as_bytes())); } } Self::List => { @@ -168,18 +171,18 @@ impl DocCommands { println!("{}", id) } } - Self::Share { doc_id, mode } => { - let doc = iroh.get_doc(env.doc(doc_id)?).await?; + Self::Share { doc, mode } => { + let doc = iroh.get_doc(env.doc(doc)?).await?; let ticket = doc.share(mode).await?; println!("{}", ticket); } Self::Set { - doc_id, + doc, author, key, value, } => { - let doc = iroh.get_doc(env.doc(doc_id)?).await?; + let doc = iroh.get_doc(env.doc(doc)?).await?; let author = env.author(author)?; let key = key.as_bytes().to_vec(); let value = value.as_bytes().to_vec(); @@ -187,14 +190,14 @@ impl DocCommands { println!("{}", fmt_entry(entry.entry())); } Self::Get { - doc_id, + doc, key, prefix, author, old, content, } => { - let doc = iroh.get_doc(env.doc(doc_id)?).await?; + let doc = iroh.get_doc(env.doc(doc)?).await?; let mut filter = match old { true => GetFilter::all(), false => GetFilter::latest(), @@ -230,12 +233,12 @@ impl DocCommands { } } Self::Keys { - doc_id, + doc, old, prefix, author, } => { - let doc = iroh.get_doc(env.doc(doc_id)?).await?; + let doc = iroh.get_doc(env.doc(doc)?).await?; let filter = match old { true => GetFilter::all(), false => GetFilter::latest(), @@ -252,8 +255,8 @@ impl DocCommands { println!("{}", fmt_entry(entry.entry())); } } - Self::Watch { doc_id } => { - let doc = iroh.get_doc(env.doc(doc_id)?).await?; + Self::Watch { doc } => { + let doc = iroh.get_doc(env.doc(doc)?).await?; let mut stream = doc.subscribe().await?; while let Some(event) = stream.next().await { let event = event?; @@ -287,8 +290,9 @@ impl DocCommands { impl AuthorCommands { pub async fn run(self, iroh: &Iroh, env: ConsoleEnv) -> Result<()> { match self { - Self::Switch { id } => { - env.set_author(id)?; + Self::Switch { author } => { + env.set_author(author)?; + println!("Active author is now {}", fmt_short(author.as_bytes())); } Self::List => { let mut stream = iroh.list_authors().await?; @@ -306,6 +310,7 @@ impl AuthorCommands { if switch { env.set_author(author_id)?; + println!("Active author is now {}", fmt_short(author_id.as_bytes())); } } } From 9efb09e3b91c1429c5f1de237c1b42733acfafd8 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 17:03:32 +0200 Subject: [PATCH 164/172] refactor: use RwLock not Mutex --- iroh/src/config.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/iroh/src/config.rs b/iroh/src/config.rs index c71342d6be..99498a7cfe 100644 --- a/iroh/src/config.rs +++ b/iroh/src/config.rs @@ -15,7 +15,7 @@ use iroh_net::{ derp::{DerpMap, DerpRegion}, }; use iroh_sync::{AuthorId, NamespaceId}; -use parking_lot::Mutex; +use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use tracing::debug; @@ -268,7 +268,7 @@ impl NodeConfig { /// environment, [Self::set_doc] and [Self::set_author] will lead to an error, as changing the /// environment is only supported within the console. #[derive(Clone, Debug)] -pub struct ConsoleEnv(Arc>); +pub struct ConsoleEnv(Arc>); #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, Clone)] struct ConsoleEnvInner { @@ -291,7 +291,7 @@ impl ConsoleEnv { doc: env_doc()?, is_console: true, }; - Ok(Self(Arc::new(Mutex::new(env)))) + Ok(Self(Arc::new(RwLock::new(env)))) } /// Read only from environment variables. @@ -301,7 +301,7 @@ impl ConsoleEnv { doc: env_doc()?, is_console: false, }; - Ok(Self(Arc::new(Mutex::new(env)))) + Ok(Self(Arc::new(RwLock::new(env)))) } fn get_console_default_author() -> anyhow::Result> { @@ -324,7 +324,7 @@ impl ConsoleEnv { /// True if running in a Iroh console session, false for a CLI command pub fn is_console(&self) -> bool { - self.0.lock().is_console + self.0.read().is_console } /// Set the active author. @@ -332,7 +332,7 @@ impl ConsoleEnv { /// Will error if not running in the Iroh console. /// Will persist to a file in the Iroh data dir otherwise. pub fn set_author(&self, author: AuthorId) -> anyhow::Result<()> { - let mut inner = self.0.lock(); + let mut inner = self.0.write(); if !inner.is_console { bail!("Switching the author is only supported within the Iroh console, not on the command line"); } @@ -349,7 +349,7 @@ impl ConsoleEnv { /// Will error if not running in the Iroh console. /// Will not persist, only valid for the current console session. pub fn set_doc(&self, doc: NamespaceId) -> anyhow::Result<()> { - let mut inner = self.0.lock(); + let mut inner = self.0.write(); if !inner.is_console { bail!("Switching the document is only supported within the Iroh console, not on the command line"); } @@ -359,7 +359,7 @@ impl ConsoleEnv { /// Get the active document. pub fn doc(&self, arg: Option) -> anyhow::Result { - let inner = self.0.lock(); + let inner = self.0.read(); let doc_id = arg.or(inner.doc).ok_or_else(|| { anyhow!( "Missing document id. Set the active document with the `IROH_DOC` environment variable or the `-d` option.\n\ @@ -371,7 +371,7 @@ impl ConsoleEnv { /// Get the active author. pub fn author(&self, arg: Option) -> anyhow::Result { - let inner = self.0.lock(); + let inner = self.0.read(); let author_id = arg.or(inner.author).ok_or_else(|| { anyhow!( "Missing author id. Set the active author with the `IROH_AUTHOR` environment variable or the `-a` option.\n\ From d20625290dff04450cc744be878f289f395de0cb Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Thu, 24 Aug 2023 17:17:26 +0200 Subject: [PATCH 165/172] refactor(iroh): simplify and cleanup the sync algorithm (#1401) switches to also use tokio::codec under the hood now. Also introduces a max message length of 1GiB for now. --- iroh/Cargo.toml | 2 +- iroh/src/sync.rs | 246 +---------------------------------- iroh/src/sync/codec.rs | 288 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 294 insertions(+), 242 deletions(-) create mode 100644 iroh/src/sync/codec.rs diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 0220641ac0..651a714e61 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -40,7 +40,7 @@ serde = { version = "1", features = ["derive"] } thiserror = "1" tokio = { version = "1", features = ["io-util", "rt"] } tokio-stream = "0.1" -tokio-util = { version = "0.7", features = ["io-util", "io"] } +tokio-util = { version = "0.7", features = ["codec", "io-util", "io"] } tracing = "0.1" walkdir = "2" diff --git a/iroh/src/sync.rs b/iroh/src/sync.rs index a36a579c9f..696bf6d937 100644 --- a/iroh/src/sync.rs +++ b/iroh/src/sync.rs @@ -2,25 +2,21 @@ use std::net::SocketAddr; -use anyhow::{bail, ensure, Context, Result}; -use bytes::BytesMut; +use anyhow::{Context, Result}; use iroh_net::{key::PublicKey, magic_endpoint::get_peer_id, MagicEndpoint}; -use iroh_sync::{ - store, - sync::{NamespaceId, Replica}, -}; -use serde::{Deserialize, Serialize}; -use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::{debug, trace}; +use iroh_sync::{store, sync::Replica}; +use tracing::debug; #[cfg(feature = "metrics")] use crate::metrics::Metrics; +use crate::sync::codec::{run_alice, run_bob}; #[cfg(feature = "metrics")] use iroh_metrics::inc; /// The ALPN identifier for the iroh-sync protocol pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; +mod codec; mod engine; mod live; pub mod rpc; @@ -28,23 +24,6 @@ pub mod rpc; pub use engine::*; pub use live::*; -/// Sync Protocol -/// -/// - Init message: signals which namespace is being synced -/// - N Sync messages -/// -/// On any error and on success the substream is closed. -#[derive(Debug, Clone, Serialize, Deserialize)] -enum Message { - Init { - /// Namespace to sync - namespace: NamespaceId, - /// Initial message - message: iroh_sync::sync::ProtocolMessage, - }, - Sync(iroh_sync::sync::ProtocolMessage), -} - /// Connect to a peer and sync a replica pub async fn connect_and_sync( endpoint: &MagicEndpoint, @@ -72,50 +51,6 @@ pub async fn connect_and_sync( res } -/// Runs the initiator side of the sync protocol. -pub async fn run_alice( - writer: &mut W, - reader: &mut R, - alice: &Replica, - other_peer_id: PublicKey, -) -> Result<()> { - let other_peer_id = *other_peer_id.as_bytes(); - let mut buffer = BytesMut::with_capacity(1024); - - // Init message - - let init_message = Message::Init { - namespace: alice.namespace(), - message: alice.sync_initial_message().map_err(Into::into)?, - }; - let msg_bytes = postcard::to_stdvec(&init_message)?; - iroh_bytes::protocol::write_lp(writer, &msg_bytes).await?; - - // Sync message loop - - while let Some(read) = iroh_bytes::protocol::read_lp(&mut *reader, &mut buffer).await? { - trace!("read {}", read.len()); - let msg = postcard::from_bytes(&read)?; - match msg { - Message::Init { .. } => { - bail!("unexpected message: init"); - } - Message::Sync(msg) => { - if let Some(msg) = alice - .sync_process_message(msg, other_peer_id) - .map_err(Into::into)? - { - send_sync_message(writer, msg).await?; - } else { - break; - } - } - } - } - - Ok(()) -} - /// Handle an iroh-sync connection and sync all shared documents in the replica store. pub async fn handle_connection( connecting: quinn::Connecting, @@ -142,174 +77,3 @@ pub async fn handle_connection( Ok(()) } - -/// Runs the receiver side of the sync protocol. -pub async fn run_bob( - writer: &mut W, - reader: &mut R, - replica_store: S, - other_peer_id: PublicKey, -) -> Result<()> { - let other_peer_id = *other_peer_id.as_bytes(); - let mut buffer = BytesMut::with_capacity(1024); - - let mut replica = None; - while let Some(read) = iroh_bytes::protocol::read_lp(&mut *reader, &mut buffer).await? { - trace!("read {}", read.len()); - let msg = postcard::from_bytes(&read)?; - - match msg { - Message::Init { namespace, message } => { - ensure!(replica.is_none(), "double init message"); - - match replica_store.open_replica(&namespace)? { - Some(r) => { - debug!("starting sync for {}", namespace); - if let Some(msg) = r - .sync_process_message(message, other_peer_id) - .map_err(Into::into)? - { - send_sync_message(writer, msg).await?; - } else { - break; - } - replica = Some(r); - } - None => { - bail!("unable to synchronize unknown namespace: {}", namespace); - } - } - } - Message::Sync(msg) => match replica { - Some(ref replica) => { - if let Some(msg) = replica - .sync_process_message(msg, other_peer_id) - .map_err(Into::into)? - { - send_sync_message(writer, msg).await?; - } else { - break; - } - } - None => { - bail!("unexpected sync message without init"); - } - }, - } - } - - Ok(()) -} - -async fn send_sync_message( - stream: &mut W, - msg: iroh_sync::sync::ProtocolMessage, -) -> Result<()> { - let msg_bytes = postcard::to_stdvec(&Message::Sync(msg))?; - iroh_bytes::protocol::write_lp(stream, &msg_bytes).await?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use iroh_net::key::SecretKey; - use iroh_sync::{ - store::{GetFilter, Store as _}, - sync::Namespace, - }; - - use super::*; - - #[tokio::test] - async fn test_sync_simple() -> Result<()> { - let mut rng = rand::thread_rng(); - let alice_peer_id = SecretKey::from_bytes(&[1u8; 32]).public(); - let bob_peer_id = SecretKey::from_bytes(&[2u8; 32]).public(); - - let alice_replica_store = store::memory::Store::default(); - // For now uses same author on both sides. - let author = alice_replica_store.new_author(&mut rng).unwrap(); - - let namespace = Namespace::new(&mut rng); - - let alice_replica = alice_replica_store.new_replica(namespace.clone()).unwrap(); - alice_replica - .hash_and_insert("hello bob", &author, "from alice") - .unwrap(); - - let bob_replica_store = store::memory::Store::default(); - let bob_replica = bob_replica_store.new_replica(namespace.clone()).unwrap(); - bob_replica - .hash_and_insert("hello alice", &author, "from bob") - .unwrap(); - - assert_eq!( - bob_replica_store - .get(bob_replica.namespace(), GetFilter::all()) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 1 - ); - assert_eq!( - alice_replica_store - .get(alice_replica.namespace(), GetFilter::all()) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 1 - ); - - let (alice, bob) = tokio::io::duplex(64); - - let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); - let replica = alice_replica.clone(); - let alice_task = tokio::task::spawn(async move { - run_alice::( - &mut alice_writer, - &mut alice_reader, - &replica, - bob_peer_id, - ) - .await - }); - - let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); - let bob_replica_store_task = bob_replica_store.clone(); - let bob_task = tokio::task::spawn(async move { - run_bob::( - &mut bob_writer, - &mut bob_reader, - bob_replica_store_task, - alice_peer_id, - ) - .await - }); - - alice_task.await??; - bob_task.await??; - - assert_eq!( - bob_replica_store - .get(bob_replica.namespace(), GetFilter::all()) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 2 - ); - assert_eq!( - alice_replica_store - .get(alice_replica.namespace(), GetFilter::all()) - .unwrap() - .collect::>>() - .unwrap() - .len(), - 2 - ); - - Ok(()) - } -} diff --git a/iroh/src/sync/codec.rs b/iroh/src/sync/codec.rs new file mode 100644 index 0000000000..367c733647 --- /dev/null +++ b/iroh/src/sync/codec.rs @@ -0,0 +1,288 @@ +use anyhow::{bail, ensure, Result}; +use bytes::{Buf, BufMut, BytesMut}; +use futures::SinkExt; +use iroh_net::key::PublicKey; +use iroh_sync::{store, NamespaceId, Replica}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_stream::StreamExt; +use tokio_util::codec::{Decoder, Encoder, FramedRead, FramedWrite}; +use tracing::debug; + +#[derive(Debug, Default)] +struct SyncCodec; + +const MAX_MESSAGE_SIZE: usize = 1024 * 1024 * 1024; // This is likely too large, but lets have some restrictions + +impl Decoder for SyncCodec { + type Item = Message; + type Error = anyhow::Error; + fn decode( + &mut self, + src: &mut BytesMut, + ) -> std::result::Result, Self::Error> { + if src.len() < 4 { + return Ok(None); + } + let bytes: [u8; 4] = src[..4].try_into().unwrap(); + let frame_len = u32::from_be_bytes(bytes) as usize; + ensure!( + frame_len <= MAX_MESSAGE_SIZE, + "received message that is too large: {}", + frame_len + ); + if src.len() < 4 + frame_len { + return Ok(None); + } + + let message: Message = postcard::from_bytes(&src[4..4 + frame_len])?; + src.advance(4 + frame_len); + Ok(Some(message)) + } +} + +impl Encoder for SyncCodec { + type Error = anyhow::Error; + + fn encode( + &mut self, + item: Message, + dst: &mut BytesMut, + ) -> std::result::Result<(), Self::Error> { + let len = + postcard::serialize_with_flavor(&item, postcard::ser_flavors::Size::default()).unwrap(); + ensure!( + len <= MAX_MESSAGE_SIZE, + "attempting to send message that is too large {}", + len + ); + + dst.put_u32(u32::try_from(len).expect("already checked")); + if dst.len() < 4 + len { + dst.resize(4 + len, 0u8); + } + postcard::to_slice(&item, &mut dst[4..])?; + + Ok(()) + } +} + +/// Sync Protocol +/// +/// - Init message: signals which namespace is being synced +/// - N Sync messages +/// +/// On any error and on success the substream is closed. +#[derive(Debug, Clone, Serialize, Deserialize)] +enum Message { + Init { + /// Namespace to sync + namespace: NamespaceId, + /// Initial message + message: iroh_sync::sync::ProtocolMessage, + }, + Sync(iroh_sync::sync::ProtocolMessage), +} + +/// Runs the initiator side of the sync protocol. +pub(super) async fn run_alice( + writer: &mut W, + reader: &mut R, + alice: &Replica, + other_peer_id: PublicKey, +) -> Result<()> { + let other_peer_id = *other_peer_id.as_bytes(); + let mut reader = FramedRead::new(reader, SyncCodec); + let mut writer = FramedWrite::new(writer, SyncCodec); + + // Init message + + let init_message = Message::Init { + namespace: alice.namespace(), + message: alice.sync_initial_message().map_err(Into::into)?, + }; + writer.send(init_message).await?; + + // Sync message loop + + while let Some(msg) = reader.next().await { + match msg? { + Message::Init { .. } => { + bail!("unexpected message: init"); + } + Message::Sync(msg) => { + if let Some(msg) = alice + .sync_process_message(msg, other_peer_id) + .map_err(Into::into)? + { + writer.send(Message::Sync(msg)).await?; + } else { + break; + } + } + } + } + + Ok(()) +} + +/// Runs the receiver side of the sync protocol. +pub(super) async fn run_bob( + writer: &mut W, + reader: &mut R, + replica_store: S, + other_peer_id: PublicKey, +) -> Result<()> { + let other_peer_id = *other_peer_id.as_bytes(); + let mut reader = FramedRead::new(reader, SyncCodec); + let mut writer = FramedWrite::new(writer, SyncCodec); + + let mut replica = None; + + while let Some(msg) = reader.next().await { + match msg? { + Message::Init { namespace, message } => { + ensure!(replica.is_none(), "double init message"); + + match replica_store.open_replica(&namespace)? { + Some(r) => { + debug!("starting sync for {}", namespace); + if let Some(msg) = r + .sync_process_message(message, other_peer_id) + .map_err(Into::into)? + { + writer.send(Message::Sync(msg)).await?; + } else { + break; + } + replica = Some(r); + } + None => { + bail!("unable to synchronize unknown namespace: {}", namespace); + } + } + } + Message::Sync(msg) => match replica { + Some(ref replica) => { + if let Some(msg) = replica + .sync_process_message(msg, other_peer_id) + .map_err(Into::into)? + { + writer.send(Message::Sync(msg)).await?; + } else { + break; + } + } + None => { + bail!("unexpected sync message without init"); + } + }, + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use iroh_net::key::SecretKey; + use iroh_sync::{ + store::{GetFilter, Store as _}, + sync::Namespace, + }; + + use super::*; + + #[tokio::test] + async fn test_sync_simple() -> Result<()> { + let mut rng = rand::thread_rng(); + let alice_peer_id = SecretKey::from_bytes(&[1u8; 32]).public(); + let bob_peer_id = SecretKey::from_bytes(&[2u8; 32]).public(); + + let alice_replica_store = store::memory::Store::default(); + // For now uses same author on both sides. + let author = alice_replica_store.new_author(&mut rng).unwrap(); + + let namespace = Namespace::new(&mut rng); + + let alice_replica = alice_replica_store.new_replica(namespace.clone()).unwrap(); + alice_replica + .hash_and_insert("hello bob", &author, "from alice") + .unwrap(); + + let bob_replica_store = store::memory::Store::default(); + let bob_replica = bob_replica_store.new_replica(namespace.clone()).unwrap(); + bob_replica + .hash_and_insert("hello alice", &author, "from bob") + .unwrap(); + + assert_eq!( + bob_replica_store + .get(bob_replica.namespace(), GetFilter::all()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 1 + ); + assert_eq!( + alice_replica_store + .get(alice_replica.namespace(), GetFilter::all()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 1 + ); + + let (alice, bob) = tokio::io::duplex(64); + + let (mut alice_reader, mut alice_writer) = tokio::io::split(alice); + let replica = alice_replica.clone(); + let alice_task = tokio::task::spawn(async move { + run_alice::( + &mut alice_writer, + &mut alice_reader, + &replica, + bob_peer_id, + ) + .await + }); + + let (mut bob_reader, mut bob_writer) = tokio::io::split(bob); + let bob_replica_store_task = bob_replica_store.clone(); + let bob_task = tokio::task::spawn(async move { + run_bob::( + &mut bob_writer, + &mut bob_reader, + bob_replica_store_task, + alice_peer_id, + ) + .await + }); + + alice_task.await??; + bob_task.await??; + + assert_eq!( + bob_replica_store + .get(bob_replica.namespace(), GetFilter::all()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 2 + ); + assert_eq!( + alice_replica_store + .get(alice_replica.namespace(), GetFilter::all()) + .unwrap() + .collect::>>() + .unwrap() + .len(), + 2 + ); + + Ok(()) + } +} From 8d803b06520bb77f3504cc9eea0403bad8e90ad7 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 17:56:10 +0200 Subject: [PATCH 166/172] refactor: move sync net code into iroh_sync --- Cargo.lock | 6 ++++++ iroh-sync/Cargo.toml | 12 +++++++++++- iroh-sync/src/lib.rs | 2 ++ iroh/src/sync.rs => iroh-sync/src/net.rs | 16 +++++++--------- {iroh/src/sync => iroh-sync/src/net}/codec.rs | 7 ++++--- iroh/src/client.rs | 2 +- iroh/src/commands/sync.rs | 2 +- iroh/src/lib.rs | 3 ++- iroh/src/metrics.rs | 4 ---- iroh/src/node.rs | 4 ++-- iroh/src/rpc_protocol.rs | 2 +- iroh/src/{sync/engine.rs => sync_engine.rs} | 10 +++++++++- iroh/src/{sync => sync_engine}/live.rs | 3 ++- iroh/src/{sync => sync_engine}/rpc.rs | 4 +--- 14 files changed, 49 insertions(+), 28 deletions(-) rename iroh/src/sync.rs => iroh-sync/src/net.rs (90%) rename {iroh/src/sync => iroh-sync/src/net}/codec.rs (98%) rename iroh/src/{sync/engine.rs => sync_engine.rs} (94%) rename iroh/src/{sync => sync_engine}/live.rs (99%) rename iroh/src/{sync => sync_engine}/rpc.rs (98%) diff --git a/Cargo.lock b/Cargo.lock index a8dc6c995d..c8e83d3f52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2101,20 +2101,26 @@ dependencies = [ "derive_more", "ed25519-dalek", "flume", + "futures", "hex", "iroh-blake3", "iroh-bytes", "iroh-metrics", + "iroh-net", "once_cell", "ouroboros", "parking_lot", "postcard", + "quinn", "rand", "rand_core", "redb", "serde", "tempfile", "tokio", + "tokio-stream", + "tokio-util", + "tracing", "url", ] diff --git a/iroh-sync/Cargo.toml b/iroh-sync/Cargo.toml index b02d4b1550..1335064ec6 100644 --- a/iroh-sync/Cargo.toml +++ b/iroh-sync/Cargo.toml @@ -30,16 +30,26 @@ url = "2.4" bytes = "1" parking_lot = "0.12.1" hex = "0.4" +tracing = "0.1" # fs-store redb = { version = "1.0.5", optional = true } ouroboros = { version = "0.17", optional = true } +# net +iroh-net = { version = "0.5.1", optional = true, path = "../iroh-net" } +tokio = { version = "1", optional = true, features = ["io-util", "sync"] } +tokio-util = { version = "0.7", optional = true, features = ["codec", "io-util", "io"] } +tokio-stream = { version = "0.1", optional = true, features = ["sync"]} +quinn = { version = "0.10", optional = true } +futures = { version = "0.3", optional = true } + [dev-dependencies] tokio = { version = "1", features = ["sync", "macros"] } tempfile = "3.4" [features] -default = ["fs-store", "metrics"] +default = ["net", "fs-store", "metrics"] +net = ["iroh-net", "tokio", "tokio-stream", "tokio-util", "quinn", "futures"] fs-store = ["redb", "ouroboros"] metrics = ["iroh-metrics"] diff --git a/iroh-sync/src/lib.rs b/iroh-sync/src/lib.rs index adab62046b..391b6b09b7 100644 --- a/iroh-sync/src/lib.rs +++ b/iroh-sync/src/lib.rs @@ -32,6 +32,8 @@ mod keys; #[cfg(feature = "metrics")] pub mod metrics; +#[cfg(feature = "net")] +pub mod net; mod ranger; pub mod store; pub mod sync; diff --git a/iroh/src/sync.rs b/iroh-sync/src/net.rs similarity index 90% rename from iroh/src/sync.rs rename to iroh-sync/src/net.rs index 696bf6d937..25534ae8ed 100644 --- a/iroh/src/sync.rs +++ b/iroh-sync/src/net.rs @@ -1,15 +1,19 @@ -//! Implementation of the iroh-sync protocol +//! Network implementation of the iroh-sync protocol use std::net::SocketAddr; use anyhow::{Context, Result}; use iroh_net::{key::PublicKey, magic_endpoint::get_peer_id, MagicEndpoint}; -use iroh_sync::{store, sync::Replica}; use tracing::debug; +use crate::{ + net::codec::{run_alice, run_bob}, + store, + sync::Replica, +}; + #[cfg(feature = "metrics")] use crate::metrics::Metrics; -use crate::sync::codec::{run_alice, run_bob}; #[cfg(feature = "metrics")] use iroh_metrics::inc; @@ -17,12 +21,6 @@ use iroh_metrics::inc; pub const SYNC_ALPN: &[u8] = b"/iroh-sync/1"; mod codec; -mod engine; -mod live; -pub mod rpc; - -pub use engine::*; -pub use live::*; /// Connect to a peer and sync a replica pub async fn connect_and_sync( diff --git a/iroh/src/sync/codec.rs b/iroh-sync/src/net/codec.rs similarity index 98% rename from iroh/src/sync/codec.rs rename to iroh-sync/src/net/codec.rs index 367c733647..9579d65a41 100644 --- a/iroh/src/sync/codec.rs +++ b/iroh-sync/src/net/codec.rs @@ -2,13 +2,14 @@ use anyhow::{bail, ensure, Result}; use bytes::{Buf, BufMut, BytesMut}; use futures::SinkExt; use iroh_net::key::PublicKey; -use iroh_sync::{store, NamespaceId, Replica}; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_stream::StreamExt; use tokio_util::codec::{Decoder, Encoder, FramedRead, FramedWrite}; use tracing::debug; +use crate::{store, NamespaceId, Replica}; + #[derive(Debug, Default)] struct SyncCodec; @@ -79,9 +80,9 @@ enum Message { /// Namespace to sync namespace: NamespaceId, /// Initial message - message: iroh_sync::sync::ProtocolMessage, + message: crate::sync::ProtocolMessage, }, - Sync(iroh_sync::sync::ProtocolMessage), + Sync(crate::sync::ProtocolMessage), } /// Runs the initiator side of the sync protocol. diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 83ed81a8df..8cf1b10ff4 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -19,7 +19,7 @@ use crate::rpc_protocol::{ DocShareRequest, DocStartSyncRequest, DocStopSyncRequest, DocSubscribeRequest, DocTicket, ProviderService, ShareMode, StatsGetRequest, }; -use crate::sync::{LiveEvent, LiveStatus, PeerSource}; +use crate::sync_engine::{LiveEvent, LiveStatus, PeerSource}; pub mod mem; #[cfg(feature = "cli")] diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index 29efeb7c65..46cb5af9dd 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -4,7 +4,7 @@ use indicatif::HumanBytes; use iroh::{ client::quic::Iroh, rpc_protocol::{DocTicket, ShareMode}, - sync::PeerSource, + sync_engine::PeerSource, }; use iroh_sync::{ store::GetFilter, diff --git a/iroh/src/lib.rs b/iroh/src/lib.rs index 852fbbca47..9f12a9fecf 100644 --- a/iroh/src/lib.rs +++ b/iroh/src/lib.rs @@ -3,6 +3,7 @@ pub use iroh_bytes as bytes; pub use iroh_net as net; +pub use iroh_sync as sync; pub mod baomap; pub mod client; @@ -13,7 +14,7 @@ pub mod download; pub mod get; pub mod node; pub mod rpc_protocol; -pub mod sync; +pub mod sync_engine; pub mod util; /// Expose metrics module diff --git a/iroh/src/metrics.rs b/iroh/src/metrics.rs index 6085483918..4dd559ba37 100644 --- a/iroh/src/metrics.rs +++ b/iroh/src/metrics.rs @@ -19,8 +19,6 @@ pub struct Metrics { pub downloads_success: Counter, pub downloads_error: Counter, pub downloads_notfound: Counter, - pub initial_sync_success: Counter, - pub initial_sync_failed: Counter, } impl Default for Metrics { @@ -34,8 +32,6 @@ impl Default for Metrics { downloads_success: Counter::new("Total number of successfull downloads"), downloads_error: Counter::new("Total number of downloads failed with error"), downloads_notfound: Counter::new("Total number of downloads failed with not found"), - initial_sync_success: Counter::new("Number of successfull initial syncs "), - initial_sync_failed: Counter::new("Number of failed initial syncs"), } } } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 9e91aad8cb..73c22f08ab 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -64,7 +64,7 @@ use crate::rpc_protocol::{ StatsGetResponse, ValidateRequest, VersionRequest, VersionResponse, WatchRequest, WatchResponse, }; -use crate::sync::{SyncEngine, SYNC_ALPN}; +use crate::sync_engine::{SyncEngine, SYNC_ALPN}; const MAX_CONNECTIONS: u32 = 1024; const MAX_STREAMS: u64 = 10; @@ -513,7 +513,7 @@ async fn handle_connection( ) -> Result<()> { match alpn.as_bytes() { GOSSIP_ALPN => gossip.handle_connection(connecting.await?).await?, - SYNC_ALPN => crate::sync::handle_connection(connecting, node.sync.store.clone()).await?, + SYNC_ALPN => iroh_sync::net::handle_connection(connecting, node.sync.store.clone()).await?, alpn if alpn == iroh_bytes::protocol::ALPN => { iroh_bytes::provider::handle_connection( connecting, diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index ef4880b992..d5ca45fb41 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -27,7 +27,7 @@ use serde::{Deserialize, Serialize}; pub use iroh_bytes::{baomap::ValidateProgress, provider::ProvideProgress, util::RpcResult}; -use crate::sync::{LiveEvent, LiveStatus, PeerSource}; +use crate::sync_engine::{LiveEvent, LiveStatus, PeerSource}; /// A 32-byte key or token pub type KeyBytes = [u8; 32]; diff --git a/iroh/src/sync/engine.rs b/iroh/src/sync_engine.rs similarity index 94% rename from iroh/src/sync/engine.rs rename to iroh/src/sync_engine.rs index 8daa20d1f8..6321d1d358 100644 --- a/iroh/src/sync/engine.rs +++ b/iroh/src/sync_engine.rs @@ -1,3 +1,7 @@ +//! Handlers and actors to for live syncing [`iroh_sync`] replicas. +//! +//! [`iroh_sync::Replica`] is also called documents here. + use std::{collections::HashSet, sync::Arc}; use anyhow::anyhow; @@ -12,7 +16,11 @@ use parking_lot::RwLock; use crate::download::Downloader; -use super::{LiveSync, PeerSource}; +mod live; +mod rpc; + +pub use iroh_sync::net::SYNC_ALPN; +pub use live::*; /// The SyncEngine contains the [`LiveSync`] handle, and keeps a copy of the store and endpoint. /// diff --git a/iroh/src/sync/live.rs b/iroh/src/sync_engine/live.rs similarity index 99% rename from iroh/src/sync/live.rs rename to iroh/src/sync_engine/live.rs index 583cfeaaad..e7dda80110 100644 --- a/iroh/src/sync/live.rs +++ b/iroh/src/sync_engine/live.rs @@ -6,7 +6,7 @@ use std::{ sync::{atomic::AtomicU64, Arc}, }; -use crate::{download::Downloader, sync::connect_and_sync}; +use crate::download::Downloader; use anyhow::{anyhow, bail, Result}; use futures::{ future::{BoxFuture, Shared}, @@ -24,6 +24,7 @@ use iroh_gossip::{ }; use iroh_net::{key::PublicKey, MagicEndpoint}; use iroh_sync::{ + net::connect_and_sync, store, sync::{Entry, InsertOrigin, NamespaceId, Replica, SignedEntry}, }; diff --git a/iroh/src/sync/rpc.rs b/iroh/src/sync_engine/rpc.rs similarity index 98% rename from iroh/src/sync/rpc.rs rename to iroh/src/sync_engine/rpc.rs index 2ad7b264e1..04fc86e0cd 100644 --- a/iroh/src/sync/rpc.rs +++ b/iroh/src/sync_engine/rpc.rs @@ -16,11 +16,9 @@ use crate::{ DocStartSyncResponse, DocStopSyncRequest, DocStopSyncResponse, DocSubscribeRequest, DocSubscribeResponse, DocTicket, RpcResult, ShareMode, }, - sync::KeepCallback, + sync_engine::{KeepCallback, LiveStatus, PeerSource, SyncEngine}, }; -use super::{engine::SyncEngine, LiveStatus, PeerSource}; - /// Capacity for the flume channels to forward sync store iterators to async RPC streams. const ITER_CHANNEL_CAP: usize = 64; From 2026eca0862a0931ee0fe6ea0256ad2dcfa05939 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 18:04:11 +0200 Subject: [PATCH 167/172] refactor: doc/author new, not create --- iroh/src/commands/sync.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iroh/src/commands/sync.rs b/iroh/src/commands/sync.rs index e63578ce21..1b70817d32 100644 --- a/iroh/src/commands/sync.rs +++ b/iroh/src/commands/sync.rs @@ -18,7 +18,7 @@ pub enum DocCommands { /// Set the active document (only works within the Iroh console). Switch { id: NamespaceId }, /// Create a new document. - Init { + New { /// Switch to the created document (only in the Iroh console). #[clap(long)] switch: bool, @@ -123,7 +123,7 @@ pub enum AuthorCommands { /// Set the active author (only works within the Iroh console). Switch { author: AuthorId }, /// Create a new author. - Create { + New { /// Switch to the created author (only in the Iroh console). #[clap(long)] switch: bool, @@ -140,7 +140,7 @@ impl DocCommands { env.set_doc(doc)?; println!("Active doc is now {}", fmt_short(doc.as_bytes())); } - Self::Init { switch } => { + Self::New { switch } => { if switch && !env.is_console() { bail!("The --switch flag is only supported within the Iroh console."); } @@ -301,7 +301,7 @@ impl AuthorCommands { println!("{}", author_id); } } - Self::Create { switch } => { + Self::New { switch } => { if switch && !env.is_console() { bail!("The --switch flag is only supported within the Iroh console."); } From 3921eac6809854d0e507e76ff31ae093b5cf8221 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 18:12:36 +0200 Subject: [PATCH 168/172] fix: doc links --- iroh/src/sync_engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/sync_engine.rs b/iroh/src/sync_engine.rs index 6321d1d358..0f1108effa 100644 --- a/iroh/src/sync_engine.rs +++ b/iroh/src/sync_engine.rs @@ -25,7 +25,7 @@ pub use live::*; /// The SyncEngine contains the [`LiveSync`] handle, and keeps a copy of the store and endpoint. /// /// The RPC methods dealing with documents and sync operate on the `SyncEngine`, with method -/// implementations in [super::rpc]. +/// implementations in [rpc]. #[derive(Debug, Clone)] pub struct SyncEngine { pub(crate) rt: Handle, From 39fda80d44b410c4eae5c7b42e7d564cf682f890 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 18:15:35 +0200 Subject: [PATCH 169/172] fix: imports in tests --- iroh-sync/src/net/codec.rs | 4 ++-- iroh/tests/sync.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iroh-sync/src/net/codec.rs b/iroh-sync/src/net/codec.rs index 9579d65a41..fef102a378 100644 --- a/iroh-sync/src/net/codec.rs +++ b/iroh-sync/src/net/codec.rs @@ -186,11 +186,11 @@ pub(super) async fn run_bob Date: Thu, 24 Aug 2023 18:42:35 +0200 Subject: [PATCH 170/172] fix: visibility --- iroh/src/sync_engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/sync_engine.rs b/iroh/src/sync_engine.rs index 0f1108effa..8646ce6f82 100644 --- a/iroh/src/sync_engine.rs +++ b/iroh/src/sync_engine.rs @@ -17,7 +17,7 @@ use parking_lot::RwLock; use crate::download::Downloader; mod live; -mod rpc; +pub mod rpc; pub use iroh_sync::net::SYNC_ALPN; pub use live::*; From 0bc1efb53e2ac0fb288252d7a1a47745c3b3337d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 18:44:22 +0200 Subject: [PATCH 171/172] fix: sync example --- iroh/examples/sync.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/examples/sync.rs b/iroh/examples/sync.rs index 3d33dabb31..7e073b5271 100644 --- a/iroh/examples/sync.rs +++ b/iroh/examples/sync.rs @@ -18,7 +18,7 @@ use clap::{CommandFactory, FromArgMatches, Parser}; use indicatif::HumanBytes; use iroh::{ download::Downloader, - sync::{PeerSource, SyncEngine, SYNC_ALPN}, + sync_engine::{PeerSource, SyncEngine, SYNC_ALPN}, }; use iroh_bytes::util::runtime; use iroh_bytes::{ @@ -764,7 +764,7 @@ async fn handle_connection(mut conn: quinn::Connecting, state: Arc) -> an println!("> incoming connection with alpn {alpn}"); match alpn.as_bytes() { GOSSIP_ALPN => state.gossip.handle_connection(conn.await?).await, - SYNC_ALPN => iroh::sync::handle_connection(conn, state.docs.clone()).await, + SYNC_ALPN => iroh_sync::net::handle_connection(conn, state.docs.clone()).await, alpn if alpn == iroh_bytes::protocol::ALPN => state.bytes.handle_connection(conn).await, _ => bail!("ignoring connection: unsupported ALPN protocol"), } From b7c76235893354bbfb7a8509a3d8a6516fb9a5fe Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 24 Aug 2023 18:54:41 +0200 Subject: [PATCH 172/172] chore: make clippy happy --- iroh-sync/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh-sync/src/sync.rs b/iroh-sync/src/sync.rs index 5298718a66..2b1e28a6e5 100644 --- a/iroh-sync/src/sync.rs +++ b/iroh-sync/src/sync.rs @@ -728,7 +728,7 @@ mod tests { fn test_multikey() { let mut rng = rand::thread_rng(); - let k = vec!["a", "c", "z"]; + let k = ["a", "c", "z"]; let mut n: Vec<_> = (0..3).map(|_| Namespace::new(&mut rng)).collect(); n.sort_by_key(|n| n.id());