Merge pull request #3139 from autonomys/gateway-rpc

Add an initial gateway RPC crate

crates/sc-consensus-subspace-rpc/src/lib.rs

@@ -172,7 +172,7 @@ pub trait SubspaceRpcApi {
     #[method(name = "subspace_lastSegmentHeaders")]
     async fn last_segment_headers(&self, limit: u32) -> Result<Vec<Option<SegmentHeader>>, Error>;
 
-    /// Block/transaction object mappings subscription
+    /// DSN object mappings subscription
     #[subscription(
         name = "subspace_subscribeObjectMappings" => "subspace_object_mappings",
         unsubscribe = "subspace_unsubscribeObjectMappings",
@@ -181,7 +181,7 @@ pub trait SubspaceRpcApi {
     )]
     fn subscribe_object_mappings(&self);
 
-    /// Filtered block/transaction object mappings subscription
+    /// Filtered DSN object mappings subscription
     #[subscription(
         name = "subspace_subscribeFilteredObjectMappings" => "subspace_filtered_object_mappings",
         unsubscribe = "subspace_unsubscribeFilteredObjectMappings",

crates/subspace-gateway-rpc/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "subspace-gateway-rpc"
+version = "0.1.0"
+authors = ["Teor <[email protected]>"]
+description = "A Subspace Network data gateway."
+edition = "2021"
+license = "MIT OR Apache-2.0"
+homepage = "https://subspace.network"
+repository = "https://github.com/autonomys/subspace"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+hex = "0.4.3"
+jsonrpsee = { version = "0.24.5", features = ["server", "macros"] }
+serde = { version = "1.0.110", default-features = false, features = ["alloc", "derive"] }
+subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" }
+subspace-data-retrieval = { version = "0.1.0", path = "../../shared/subspace-data-retrieval" }
+thiserror = "1.0.64"
+tracing = "0.1.40"

crates/subspace-gateway-rpc/README.md

@@ -0,0 +1,43 @@
+## Subspace Gateway RPCs
+
+RPC API for Subspace Gateway.
+
+### Using the gateway RPCs
+
+The gateway RPCs can fetch data using object mappings supplied by a node.
+
+Launch a node using the instructions in its README, and wait for mappings from the node RPCs:
+```bash
+$ websocat --jsonrpc ws://127.0.0.1:9944
+subspace_subscribeObjectMappings
+```
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "subspace_archived_object_mappings",
+  "params": {
+    "subscription": "o7M85uu9ir39R5PJ",
+    "result": {
+      "v0": {
+        "objects": [
+          ["0000000000000000000000000000000000000000000000000000000000000000", 0, 0]
+        ]
+      }
+    }
+  }
+}
+```
+
+Then use those mappings to get object data from the gateway RPCs:
+```bash
+$ websocat --jsonrpc ws://127.0.0.1:9955
+subspace_fetchObject ["v0": { "objects": [["0000000000000000000000000000000000000000000000000000000000000000", 0, 0]]}]
+```
+
+```json
+{
+  "jsonrpc": "2.0",
+  "result": ["00000000"]
+}
+```

crates/subspace-gateway-rpc/src/lib.rs

@@ -0,0 +1,157 @@
+//! RPC API for the Subspace Gateway.
+
+use jsonrpsee::core::async_trait;
+use jsonrpsee::proc_macros::rpc;
+use jsonrpsee::types::{ErrorObject, ErrorObjectOwned};
+use std::fmt;
+use std::ops::{Deref, DerefMut};
+use subspace_core_primitives::hashes::{blake3_hash, Blake3Hash};
+use subspace_core_primitives::objects::GlobalObjectMapping;
+use subspace_data_retrieval::object_fetcher::{self, ObjectFetcher};
+use tracing::debug;
+
+const SUBSPACE_ERROR: i32 = 9000;
+
+/// The maximum number of objects that can be requested in a single RPC call.
+///
+/// If the returned objects are large, they could overflow the RPC server (or client) buffers,
+/// despite this limit.
+// TODO: turn this into a CLI option
+const MAX_OBJECTS_PER_REQUEST: usize = 100;
+
+/// Top-level error type for the RPC handler.
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    /// Too many mappings were supplied.
+    #[error("Mapping count {count} exceeded request limit {MAX_OBJECTS_PER_REQUEST}")]
+    TooManyMappings {
+        /// The number of supplied mappings.
+        count: usize,
+    },
+
+    /// The object fetcher failed.
+    #[error(transparent)]
+    ObjectFetcherError(#[from] object_fetcher::Error),
+
+    /// The returned object data did not match the hash in the mapping.
+    #[error(
+        "Invalid object hash, mapping had {mapping_hash:?}, but fetched data had {data_hash:?}"
+    )]
+    InvalidObjectHash {
+        /// The expected hash from the mapping.
+        mapping_hash: Blake3Hash,
+        /// The actual hash of the returned object data.
+        data_hash: Blake3Hash,
+    },
+}
+
+impl From<Error> for ErrorObjectOwned {
+    fn from(error: Error) -> Self {
+        ErrorObject::owned(SUBSPACE_ERROR + 1, format!("{error:?}"), None::<()>)
+    }
+}
+
+/// Binary data, encoded as hex.
+#[derive(Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
+#[serde(transparent)]
+pub struct HexData {
+    #[serde(with = "hex")]
+    pub data: Vec<u8>,
+}
+
+impl fmt::Debug for HexData {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "HexData({})", hex::encode(&self.data))
+    }
+}
+
+impl fmt::Display for HexData {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", hex::encode(&self.data))
+    }
+}
+
+impl From<Vec<u8>> for HexData {
+    fn from(data: Vec<u8>) -> Self {
+        Self { data }
+    }
+}
+
+impl Deref for HexData {
+    type Target = Vec<u8>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.data
+    }
+}
+
+impl DerefMut for HexData {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.data
+    }
+}
+
+/// Provides rpc methods for interacting with a Subspace DSN Gateway.
+#[rpc(client, server)]
+pub trait SubspaceGatewayRpcApi {
+    /// Get object data from DSN object mappings.
+    /// Returns an error if any object fetch was unsuccessful.
+    #[method(name = "subspace_fetchObject")]
+    async fn fetch_object(&self, mappings: GlobalObjectMapping) -> Result<Vec<HexData>, Error>;
+}
+/// Subspace Gateway RPC configuration
+pub struct SubspaceGatewayRpcConfig {
+    /// DSN object fetcher instance.
+    pub object_fetcher: ObjectFetcher,
+}
+
+/// Implements the [`SubspaceGatewayRpcApiServer`] trait for interacting with the Subspace Gateway.
+pub struct SubspaceGatewayRpc {
+    /// DSN object fetcher instance.
+    object_fetcher: ObjectFetcher,
+}
+
+/// [`SubspaceGatewayRpc`] is used to fetch objects from the DSN.
+impl SubspaceGatewayRpc {
+    /// Creates a new instance of the `SubspaceGatewayRpc` handler.
+    pub fn new(config: SubspaceGatewayRpcConfig) -> Self {
+        Self {
+            object_fetcher: config.object_fetcher,
+        }
+    }
+}
+
+#[async_trait]
+impl SubspaceGatewayRpcApiServer for SubspaceGatewayRpc {
+    async fn fetch_object(&self, mappings: GlobalObjectMapping) -> Result<Vec<HexData>, Error> {
+        // TODO: deny unsafe RPC calls
+
+        let count = mappings.objects().len();
+        if count > MAX_OBJECTS_PER_REQUEST {
+            debug!(%count, %MAX_OBJECTS_PER_REQUEST, "Too many mappings in request");
+            return Err(Error::TooManyMappings { count });
+        }
+
+        let mut objects = Vec::with_capacity(count);
+        // TODO: fetch concurrently
+        for mapping in mappings.objects() {
+            let data = self
+                .object_fetcher
+                .fetch_object(mapping.piece_index, mapping.offset)
+                .await?;
+
+            let data_hash = blake3_hash(&data);
+            if data_hash != mapping.hash {
+                debug!(?data_hash, ?mapping.hash, "Retrieved data did not match mapping hash");
+                return Err(Error::InvalidObjectHash {
+                    mapping_hash: mapping.hash,
+                    data_hash,
+                });
+            }
+
+            objects.push(data.into());
+        }
+
+        Ok(objects)
+    }
+}