From 26cf0da7017b35250a601c9c8d8f2928be206a23 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 8 May 2024 12:29:11 -0400 Subject: [PATCH 01/21] feat: given a MARF key constructed client side, retrieve clarity value --- stackslib/src/net/api/getclaritymarfvalue.rs | 234 +++++++++++++++++++ stackslib/src/net/api/mod.rs | 4 + stackslib/src/net/httpcore.rs | 11 + 3 files changed, 249 insertions(+) create mode 100644 stackslib/src/net/api/getclaritymarfvalue.rs diff --git a/stackslib/src/net/api/getclaritymarfvalue.rs b/stackslib/src/net/api/getclaritymarfvalue.rs new file mode 100644 index 0000000000..d2369f2b31 --- /dev/null +++ b/stackslib/src/net/api/getclaritymarfvalue.rs @@ -0,0 +1,234 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::io::{Read, Write}; + +use clarity::vm::ast::parser::v1::CLARITY_NAME_REGEX; +use clarity::vm::clarity::ClarityConnection; +use clarity::vm::costs::LimitedCostTracker; +use clarity::vm::database::{ClarityDatabase, STXBalance, StoreType}; +use clarity::vm::representations::{ + CONTRACT_NAME_REGEX_STRING, PRINCIPAL_DATA_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING, +}; +use clarity::vm::types::{PrincipalData, QualifiedContractIdentifier, StandardPrincipalData}; +use clarity::vm::{ClarityName, ClarityVersion, ContractName}; +use regex::{Captures, Regex}; +use stacks_common::types::chainstate::{StacksAddress, StacksBlockId}; +use stacks_common::types::net::PeerHost; +use stacks_common::types::Address; +use stacks_common::util::hash::{to_hex, Sha256Sum}; + +use crate::burnchains::Burnchain; +use crate::chainstate::burn::db::sortdb::SortitionDB; +use crate::chainstate::stacks::db::StacksChainState; +use crate::chainstate::stacks::Error as ChainError; +use crate::core::mempool::MemPoolDB; +use crate::net::http::{ + parse_json, Error, HttpNotFound, HttpRequest, HttpRequestContents, HttpRequestPreamble, + HttpResponse, HttpResponseContents, HttpResponsePayload, HttpResponsePreamble, HttpServerError, +}; +use crate::net::httpcore::{ + request, HttpPreambleExtensions, HttpRequestContentsExtensions, RPCRequestHandler, StacksHttp, + StacksHttpRequest, StacksHttpResponse, +}; +use crate::net::p2p::PeerNetwork; +use crate::net::{Error as NetError, StacksNodeState, TipRequest}; +use crate::util_lib::boot::boot_code_id; +use crate::util_lib::db::Error as DBError; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ClarityMarfValueResponse { + pub data: String, + #[serde(rename = "proof")] + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub marf_proof: Option, +} + +#[derive(Clone)] +pub struct RPCGetClarityMarfValueRequestHandler { + pub clarity_marf_key: Option, +} +impl RPCGetClarityMarfValueRequestHandler { + pub fn new() -> Self { + Self { + clarity_marf_key: None, + } + } +} + +/// Decode the HTTP request +impl HttpRequest for RPCGetClarityMarfValueRequestHandler { + fn verb(&self) -> &'static str { + "GET" + } + + fn path_regex(&self) -> Regex { + // TODO: write regex validating the following patterns + // format!("vm::{}::{}::{}", contract_identifier, data as u8, var_name) + // format!("vm::{}::{}::{}::{}", contract_identifier, data as u8, var_name, key_value) + // format!("vm-metadata::{}::{}", data as u8, var_name) + // format!("vm-epoch::epoch-version") + + Regex::new(&format!("^/v2/clarity_marf_value/(?.*)$",)).unwrap() + } + + fn metrics_identifier(&self) -> &str { + "/v2/clarity_marf_value/:clarity_marf_key" + } + + /// Try to decode this request. + /// There's nothing to load here, so just make sure the request is well-formed. + fn try_parse_request( + &mut self, + preamble: &HttpRequestPreamble, + captures: &Captures, + query: Option<&str>, + _body: &[u8], + ) -> Result { + if preamble.get_content_length() != 0 { + return Err(Error::DecodeError( + "Invalid Http request: expected 0-length body".to_string(), + )); + } + + let marf_key = request::get_marf_key(captures, "clarity_marf_key")?; + + self.clarity_marf_key = Some(marf_key); + + let contents = HttpRequestContents::new().query_string(query); + Ok(contents) + } +} + +/// Handle the HTTP request +impl RPCRequestHandler for RPCGetClarityMarfValueRequestHandler { + /// Reset internal state + fn restart(&mut self) { + self.clarity_marf_key = None; + } + + /// Make the response + fn try_handle_request( + &mut self, + preamble: HttpRequestPreamble, + contents: HttpRequestContents, + node: &mut StacksNodeState, + ) -> Result<(HttpResponsePreamble, HttpResponseContents), NetError> { + let clarity_marf_key = self.clarity_marf_key.take().ok_or(NetError::SendError( + "`clarity_marf_key` not set".to_string(), + ))?; + + let tip = match node.load_stacks_chain_tip(&preamble, &contents) { + Ok(tip) => tip, + Err(error_resp) => { + return error_resp.try_into_contents().map_err(NetError::from); + } + }; + + let with_proof = contents.get_with_proof(); + + let data_opt = node.with_node_state(|_network, sortdb, chainstate, _mempool, _rpc_args| { + chainstate.maybe_read_only_clarity_tx(&sortdb.index_conn(), &tip, |clarity_tx| { + clarity_tx.with_clarity_db_readonly(|clarity_db| { + let (value_hex, marf_proof): (String, _) = if with_proof { + clarity_db + .get_data_with_proof(&clarity_marf_key) + .ok() + .flatten() + .map(|(a, b)| (a, Some(format!("0x{}", to_hex(&b)))))? + } else { + clarity_db + .get_data(&clarity_marf_key) + .ok() + .flatten() + .map(|a| (a, None))? + }; + + let data = format!("0x{}", value_hex); + Some(ClarityMarfValueResponse { data, marf_proof }) + }) + }) + }); + + let data_resp = match data_opt { + Ok(Some(Some(data))) => data, + Ok(Some(None)) => { + return StacksHttpResponse::new_error( + &preamble, + &HttpNotFound::new("Data var not found".to_string()), + ) + .try_into_contents() + .map_err(NetError::from); + } + Ok(None) | Err(_) => { + return StacksHttpResponse::new_error( + &preamble, + &HttpNotFound::new("Chain tip not found".to_string()), + ) + .try_into_contents() + .map_err(NetError::from); + } + }; + + let mut preamble = HttpResponsePreamble::ok_json(&preamble); + preamble.set_canonical_stacks_tip_height(Some(node.canonical_stacks_tip_height())); + let body = HttpResponseContents::try_from_json(&data_resp)?; + Ok((preamble, body)) + } +} + +/// Decode the HTTP response +impl HttpResponse for RPCGetClarityMarfValueRequestHandler { + fn try_parse_response( + &self, + preamble: &HttpResponsePreamble, + body: &[u8], + ) -> Result { + let marf_value: ClarityMarfValueResponse = parse_json(preamble, body)?; + Ok(HttpResponsePayload::try_from_json(marf_value)?) + } +} + +impl StacksHttpRequest { + /// Make a new request for a data var + pub fn new_getclaritymarfvalue( + host: PeerHost, + clarity_marf_key: String, + tip_req: TipRequest, + with_proof: bool, + ) -> StacksHttpRequest { + StacksHttpRequest::new_for_peer( + host, + "GET".into(), + format!("/v2/clarity_marf_value/{}", &clarity_marf_key), + HttpRequestContents::new() + .for_tip(tip_req) + .query_arg("proof".into(), if with_proof { "1" } else { "0" }.into()), + ) + .expect("FATAL: failed to construct request from infallible data") + } +} + +impl StacksHttpResponse { + pub fn decode_clarity_marf_value_response(self) -> Result { + let contents = self.get_http_payload_ok()?; + let contents_json: serde_json::Value = contents.try_into()?; + let resp: ClarityMarfValueResponse = serde_json::from_value(contents_json) + .map_err(|_e| NetError::DeserializeError("Failed to load from JSON".to_string()))?; + Ok(resp) + } +} diff --git a/stackslib/src/net/api/mod.rs b/stackslib/src/net/api/mod.rs index d256c15b97..fc7e1f64a3 100644 --- a/stackslib/src/net/api/mod.rs +++ b/stackslib/src/net/api/mod.rs @@ -42,6 +42,7 @@ pub mod getattachment; pub mod getattachmentsinv; pub mod getblock; pub mod getblock_v3; +pub mod getclaritymarfvalue; pub mod getconstantval; pub mod getcontractabi; pub mod getcontractsrc; @@ -90,6 +91,9 @@ impl StacksHttp { self.register_rpc_endpoint(getattachmentsinv::RPCGetAttachmentsInvRequestHandler::new()); self.register_rpc_endpoint(getblock::RPCBlocksRequestHandler::new()); self.register_rpc_endpoint(getblock_v3::RPCNakamotoBlockRequestHandler::new()); + self.register_rpc_endpoint( + getclaritymarfvalue::RPCGetClarityMarfValueRequestHandler::new(), + ); self.register_rpc_endpoint(getconstantval::RPCGetConstantValRequestHandler::new()); self.register_rpc_endpoint(getcontractabi::RPCGetContractAbiRequestHandler::new()); self.register_rpc_endpoint(getcontractsrc::RPCGetContractSrcRequestHandler::new()); diff --git a/stackslib/src/net/httpcore.rs b/stackslib/src/net/httpcore.rs index dec51df42a..fb92b03b18 100644 --- a/stackslib/src/net/httpcore.rs +++ b/stackslib/src/net/httpcore.rs @@ -247,6 +247,17 @@ pub mod request { Ok(txid) } + /// Get and parse a MARF key from a path's captures, given the name of the regex field. + pub fn get_marf_key(captures: &Captures, key: &str) -> Result { + let marf_key = if let Some(marf_key_str) = captures.name(key) { + marf_key_str.as_str().to_string() + } else { + return Err(HttpError::Http(404, format!("Missing `{}`", key))); + }; + + Ok(marf_key) + } + /// Get and parse a Clarity name from a path's captures, given the name of the regex field. pub fn get_clarity_name(captures: &Captures, key: &str) -> Result { let clarity_name = if let Some(name_str) = captures.name(key) { From 931f315d8a49df3a9c9dbc165d1f84f025bee6d6 Mon Sep 17 00:00:00 2001 From: Hugo Caillard <911307+hugocaillard@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:52:45 +0200 Subject: [PATCH 02/21] feat: rpc endpoint to retrieve clarity metadata --- clarity/src/vm/representations.rs | 12 + stackslib/src/net/api/getclaritymarfvalue.rs | 90 +++----- stackslib/src/net/api/getclaritymetadata.rs | 217 ++++++++++++++++++ stackslib/src/net/api/mod.rs | 2 + .../src/net/api/tests/getclaritymarfvalue.rs | 180 +++++++++++++++ .../src/net/api/tests/getclaritymetadata.rs | 166 ++++++++++++++ stackslib/src/net/api/tests/mod.rs | 4 +- stackslib/src/net/httpcore.rs | 2 +- 8 files changed, 618 insertions(+), 55 deletions(-) create mode 100644 stackslib/src/net/api/getclaritymetadata.rs create mode 100644 stackslib/src/net/api/tests/getclaritymarfvalue.rs create mode 100644 stackslib/src/net/api/tests/getclaritymetadata.rs diff --git a/clarity/src/vm/representations.rs b/clarity/src/vm/representations.rs index c80e3c7467..8f140a6a4d 100644 --- a/clarity/src/vm/representations.rs +++ b/clarity/src/vm/representations.rs @@ -64,6 +64,18 @@ lazy_static! { Regex::new(format!("^{}$|^__transient$", CONTRACT_NAME_REGEX_STRING.as_str()).as_str()) .unwrap() }; + pub static ref MARF_KEY_FOR_TRIP_REGEX_STRING: String = format!( + r"vm::{}::\d+::.*", + *CONTRACT_PRINCIPAL_REGEX_STRING, + ); + pub static ref MARF_KEY_FOR_QUAD_REGEX_STRING: String = format!( + r"{}::.*", + *MARF_KEY_FOR_TRIP_REGEX_STRING, + ); + pub static ref METADATA_KEY_REGEX_STRING: String = format!( + r"vm-metadata::\d+::.*", + + ); } guarded_string!( diff --git a/stackslib/src/net/api/getclaritymarfvalue.rs b/stackslib/src/net/api/getclaritymarfvalue.rs index d2369f2b31..b950f7ffcc 100644 --- a/stackslib/src/net/api/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/getclaritymarfvalue.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// Copyright (C) 2020-2024 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -14,40 +14,23 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::io::{Read, Write}; - -use clarity::vm::ast::parser::v1::CLARITY_NAME_REGEX; use clarity::vm::clarity::ClarityConnection; -use clarity::vm::costs::LimitedCostTracker; -use clarity::vm::database::{ClarityDatabase, STXBalance, StoreType}; use clarity::vm::representations::{ - CONTRACT_NAME_REGEX_STRING, PRINCIPAL_DATA_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING, + MARF_KEY_FOR_QUAD_REGEX_STRING, MARF_KEY_FOR_TRIP_REGEX_STRING, }; -use clarity::vm::types::{PrincipalData, QualifiedContractIdentifier, StandardPrincipalData}; -use clarity::vm::{ClarityName, ClarityVersion, ContractName}; use regex::{Captures, Regex}; -use stacks_common::types::chainstate::{StacksAddress, StacksBlockId}; use stacks_common::types::net::PeerHost; -use stacks_common::types::Address; -use stacks_common::util::hash::{to_hex, Sha256Sum}; - -use crate::burnchains::Burnchain; -use crate::chainstate::burn::db::sortdb::SortitionDB; -use crate::chainstate::stacks::db::StacksChainState; -use crate::chainstate::stacks::Error as ChainError; -use crate::core::mempool::MemPoolDB; +use stacks_common::util::hash::to_hex; + use crate::net::http::{ parse_json, Error, HttpNotFound, HttpRequest, HttpRequestContents, HttpRequestPreamble, - HttpResponse, HttpResponseContents, HttpResponsePayload, HttpResponsePreamble, HttpServerError, + HttpResponse, HttpResponseContents, HttpResponsePayload, HttpResponsePreamble, }; use crate::net::httpcore::{ - request, HttpPreambleExtensions, HttpRequestContentsExtensions, RPCRequestHandler, StacksHttp, + request, HttpPreambleExtensions, HttpRequestContentsExtensions, RPCRequestHandler, StacksHttpRequest, StacksHttpResponse, }; -use crate::net::p2p::PeerNetwork; use crate::net::{Error as NetError, StacksNodeState, TipRequest}; -use crate::util_lib::boot::boot_code_id; -use crate::util_lib::db::Error as DBError; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ClarityMarfValueResponse { @@ -77,13 +60,11 @@ impl HttpRequest for RPCGetClarityMarfValueRequestHandler { } fn path_regex(&self) -> Regex { - // TODO: write regex validating the following patterns - // format!("vm::{}::{}::{}", contract_identifier, data as u8, var_name) - // format!("vm::{}::{}::{}::{}", contract_identifier, data as u8, var_name, key_value) - // format!("vm-metadata::{}::{}", data as u8, var_name) - // format!("vm-epoch::epoch-version") - - Regex::new(&format!("^/v2/clarity_marf_value/(?.*)$",)).unwrap() + Regex::new(&format!( + r"^/v2/clarity_marf_value/(?P(vm-epoch::epoch-version)|({})|({}))$", + *MARF_KEY_FOR_TRIP_REGEX_STRING, *MARF_KEY_FOR_QUAD_REGEX_STRING + )) + .unwrap() } fn metrics_identifier(&self) -> &str { @@ -105,7 +86,7 @@ impl HttpRequest for RPCGetClarityMarfValueRequestHandler { )); } - let marf_key = request::get_marf_key(captures, "clarity_marf_key")?; + let marf_key = request::get_key(captures, "clarity_marf_key")?; self.clarity_marf_key = Some(marf_key); @@ -142,26 +123,30 @@ impl RPCRequestHandler for RPCGetClarityMarfValueRequestHandler { let with_proof = contents.get_with_proof(); let data_opt = node.with_node_state(|_network, sortdb, chainstate, _mempool, _rpc_args| { - chainstate.maybe_read_only_clarity_tx(&sortdb.index_conn(), &tip, |clarity_tx| { - clarity_tx.with_clarity_db_readonly(|clarity_db| { - let (value_hex, marf_proof): (String, _) = if with_proof { - clarity_db - .get_data_with_proof(&clarity_marf_key) - .ok() - .flatten() - .map(|(a, b)| (a, Some(format!("0x{}", to_hex(&b)))))? - } else { - clarity_db - .get_data(&clarity_marf_key) - .ok() - .flatten() - .map(|a| (a, None))? - }; - - let data = format!("0x{}", value_hex); - Some(ClarityMarfValueResponse { data, marf_proof }) - }) - }) + chainstate.maybe_read_only_clarity_tx( + &sortdb.index_handle_at_block(chainstate, &tip)?, + &tip, + |clarity_tx| { + clarity_tx.with_clarity_db_readonly(|clarity_db| { + let (value_hex, marf_proof): (String, _) = if with_proof { + clarity_db + .get_data_with_proof(&clarity_marf_key) + .ok() + .flatten() + .map(|(a, b)| (a, Some(format!("0x{}", to_hex(&b)))))? + } else { + clarity_db + .get_data(&clarity_marf_key) + .ok() + .flatten() + .map(|a| (a, None))? + }; + + let data = format!("0x{}", value_hex); + Some(ClarityMarfValueResponse { data, marf_proof }) + }) + }, + ) }); let data_resp = match data_opt { @@ -169,7 +154,7 @@ impl RPCRequestHandler for RPCGetClarityMarfValueRequestHandler { Ok(Some(None)) => { return StacksHttpResponse::new_error( &preamble, - &HttpNotFound::new("Data var not found".to_string()), + &HttpNotFound::new("Marf key not found".to_string()), ) .try_into_contents() .map_err(NetError::from); @@ -204,7 +189,6 @@ impl HttpResponse for RPCGetClarityMarfValueRequestHandler { } impl StacksHttpRequest { - /// Make a new request for a data var pub fn new_getclaritymarfvalue( host: PeerHost, clarity_marf_key: String, diff --git a/stackslib/src/net/api/getclaritymetadata.rs b/stackslib/src/net/api/getclaritymetadata.rs new file mode 100644 index 0000000000..2feb81613a --- /dev/null +++ b/stackslib/src/net/api/getclaritymetadata.rs @@ -0,0 +1,217 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2024 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use clarity::vm::clarity::ClarityConnection; +use clarity::vm::representations::{ + CONTRACT_NAME_REGEX_STRING, METADATA_KEY_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING, +}; +use clarity::vm::types::QualifiedContractIdentifier; +use clarity::vm::ContractName; +use regex::{Captures, Regex}; +use stacks_common::types::chainstate::StacksAddress; +use stacks_common::types::net::PeerHost; + +use crate::net::http::{ + parse_json, Error, HttpNotFound, HttpRequest, HttpRequestContents, HttpRequestPreamble, + HttpResponse, HttpResponseContents, HttpResponsePayload, HttpResponsePreamble, +}; +use crate::net::httpcore::{ + request, HttpPreambleExtensions, HttpRequestContentsExtensions, RPCRequestHandler, + StacksHttpRequest, StacksHttpResponse, +}; +use crate::net::{Error as NetError, StacksNodeState, TipRequest}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ClarityMetadataResponse { + pub data: String, +} + +#[derive(Clone)] +pub struct RPCGetClarityMetadataRequestHandler { + pub clarity_metadata_key: Option, + pub contract_identifier: Option, +} +impl RPCGetClarityMetadataRequestHandler { + pub fn new() -> Self { + Self { + clarity_metadata_key: None, + contract_identifier: None, + } + } +} + +/// Decode the HTTP request +impl HttpRequest for RPCGetClarityMetadataRequestHandler { + fn verb(&self) -> &'static str { + "GET" + } + + fn path_regex(&self) -> Regex { + Regex::new(&format!( + r"^/v2/clarity_metadata/(?P
{})/(?P{})/(?P(analysis)|({}))$", + *STANDARD_PRINCIPAL_REGEX_STRING, + *CONTRACT_NAME_REGEX_STRING, + *METADATA_KEY_REGEX_STRING + )) + .unwrap() + } + + fn metrics_identifier(&self) -> &str { + "/v2/clarity_metadata/:principal/:contract_name/:clarity_metadata_key" + } + + /// Try to decode this request. + /// There's nothing to load here, so just make sure the request is well-formed. + fn try_parse_request( + &mut self, + preamble: &HttpRequestPreamble, + captures: &Captures, + query: Option<&str>, + _body: &[u8], + ) -> Result { + if preamble.get_content_length() != 0 { + return Err(Error::DecodeError( + "Invalid Http request: expected 0-length body".to_string(), + )); + } + + let contract_identifier = request::get_contract_address(captures, "address", "contract")?; + let metadata_key = request::get_key(captures, "clarity_metadata_key")?; + + self.contract_identifier = Some(contract_identifier); + self.clarity_metadata_key = Some(metadata_key); + + let contents = HttpRequestContents::new().query_string(query); + Ok(contents) + } +} + +/// Handle the HTTP request +impl RPCRequestHandler for RPCGetClarityMetadataRequestHandler { + /// Reset internal state + fn restart(&mut self) { + self.contract_identifier = None; + self.clarity_metadata_key = None; + } + + /// Make the response + fn try_handle_request( + &mut self, + preamble: HttpRequestPreamble, + contents: HttpRequestContents, + node: &mut StacksNodeState, + ) -> Result<(HttpResponsePreamble, HttpResponseContents), NetError> { + let contract_identifier = self.contract_identifier.take().ok_or(NetError::SendError( + "`contract_identifier` not set".to_string(), + ))?; + let clarity_metadata_key = self.clarity_metadata_key.take().ok_or(NetError::SendError( + "`clarity_metadata_key` not set".to_string(), + ))?; + + let tip = match node.load_stacks_chain_tip(&preamble, &contents) { + Ok(tip) => tip, + Err(error_resp) => { + return error_resp.try_into_contents().map_err(NetError::from); + } + }; + + let data_opt = node.with_node_state(|_network, sortdb, chainstate, _mempool, _rpc_args| { + chainstate.maybe_read_only_clarity_tx( + &sortdb.index_handle_at_block(chainstate, &tip)?, + &tip, + |clarity_tx| { + clarity_tx.with_clarity_db_readonly(|clarity_db| { + let data = clarity_db + .store + .get_metadata(&contract_identifier, &clarity_metadata_key) + .ok() + .flatten()?; + + Some(ClarityMetadataResponse { data }) + }) + }, + ) + }); + + let data_resp = match data_opt { + Ok(Some(Some(data))) => data, + Ok(Some(None)) => { + return StacksHttpResponse::new_error( + &preamble, + &HttpNotFound::new("Metadata not found".to_string()), + ) + .try_into_contents() + .map_err(NetError::from); + } + Ok(None) | Err(_) => { + return StacksHttpResponse::new_error( + &preamble, + &HttpNotFound::new("Chain tip not found".to_string()), + ) + .try_into_contents() + .map_err(NetError::from); + } + }; + + let mut preamble = HttpResponsePreamble::ok_json(&preamble); + preamble.set_canonical_stacks_tip_height(Some(node.canonical_stacks_tip_height())); + let body = HttpResponseContents::try_from_json(&data_resp)?; + Ok((preamble, body)) + } +} + +/// Decode the HTTP response +impl HttpResponse for RPCGetClarityMetadataRequestHandler { + fn try_parse_response( + &self, + preamble: &HttpResponsePreamble, + body: &[u8], + ) -> Result { + let metadata: ClarityMetadataResponse = parse_json(preamble, body)?; + Ok(HttpResponsePayload::try_from_json(metadata)?) + } +} + +impl StacksHttpRequest { + pub fn new_getclaritymetadata( + host: PeerHost, + contract_addr: StacksAddress, + contract_name: ContractName, + clarity_metadata_key: String, + tip_req: TipRequest, + ) -> StacksHttpRequest { + StacksHttpRequest::new_for_peer( + host, + "GET".into(), + format!( + "/v2/clarity_metadata/{}/{}/{}", + &contract_addr, &contract_name, &clarity_metadata_key + ), + HttpRequestContents::new().for_tip(tip_req), + ) + .expect("FATAL: failed to construct request from infallible data") + } +} + +impl StacksHttpResponse { + pub fn decode_clarity_metadata_response(self) -> Result { + let contents = self.get_http_payload_ok()?; + let contents_json: serde_json::Value = contents.try_into()?; + let resp: ClarityMetadataResponse = serde_json::from_value(contents_json) + .map_err(|_e| NetError::DeserializeError("Failed to load from JSON".to_string()))?; + Ok(resp) + } +} diff --git a/stackslib/src/net/api/mod.rs b/stackslib/src/net/api/mod.rs index fc7e1f64a3..9e6be1bf11 100644 --- a/stackslib/src/net/api/mod.rs +++ b/stackslib/src/net/api/mod.rs @@ -43,6 +43,7 @@ pub mod getattachmentsinv; pub mod getblock; pub mod getblock_v3; pub mod getclaritymarfvalue; +pub mod getclaritymetadata; pub mod getconstantval; pub mod getcontractabi; pub mod getcontractsrc; @@ -94,6 +95,7 @@ impl StacksHttp { self.register_rpc_endpoint( getclaritymarfvalue::RPCGetClarityMarfValueRequestHandler::new(), ); + self.register_rpc_endpoint(getclaritymetadata::RPCGetClarityMetadataRequestHandler::new()); self.register_rpc_endpoint(getconstantval::RPCGetConstantValRequestHandler::new()); self.register_rpc_endpoint(getcontractabi::RPCGetContractAbiRequestHandler::new()); self.register_rpc_endpoint(getcontractsrc::RPCGetContractSrcRequestHandler::new()); diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs new file mode 100644 index 0000000000..b4129c7d93 --- /dev/null +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -0,0 +1,180 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2024 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions}; +use clarity::vm::{ClarityName, ContractName}; +use stacks_common::codec::StacksMessageCodec; +use stacks_common::types::chainstate::StacksAddress; +use stacks_common::types::net::PeerHost; +use stacks_common::types::Address; + +use super::test_rpc; +use crate::net::api::*; +use crate::net::connection::ConnectionOptions; +use crate::net::httpcore::{ + HttpPreambleExtensions, HttpRequestContentsExtensions, RPCRequestHandler, StacksHttp, + StacksHttpRequest, +}; +use crate::net::{ProtocolFamily, TipRequest}; + +#[test] +fn test_try_parse_request() { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); + let mut http = StacksHttp::new(addr.clone(), &ConnectionOptions::default()); + + let vm_key_epoch = "vm-epoch::epoch-version"; + let vm_key_trip = "vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::1::count"; + let vm_key_quad = "vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::0::data::1234"; + let valid_keys = [vm_key_epoch, vm_key_trip, vm_key_quad]; + + for key in valid_keys { + let request = StacksHttpRequest::new_getclaritymarfvalue( + addr.into(), + key.to_string(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])), + true, + ); + assert_eq!( + request.contents().tip_request(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])) + ); + assert_eq!(request.contents().get_with_proof(), true); + + let bytes = request.try_serialize().unwrap(); + + let (parsed_preamble, offset) = http.read_preamble(&bytes).unwrap(); + let mut handler = getclaritymarfvalue::RPCGetClarityMarfValueRequestHandler::new(); + let mut parsed_request = http + .handle_try_parse_request( + &mut handler, + &parsed_preamble.expect_request(), + &bytes[offset..], + ) + .unwrap(); + + // parsed request consumes headers that would not be in a constructed request + parsed_request.clear_headers(); + let (preamble, contents) = parsed_request.destruct(); + + // consumed path args + assert_eq!(handler.clarity_marf_key, Some(key.to_string())); + + assert_eq!(&preamble, request.preamble()); + + handler.restart(); + assert!(handler.clarity_marf_key.is_none()); + } +} + +#[test] +fn test_try_make_response() { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); + + let mut requests = vec![]; + + // query existing + let request = StacksHttpRequest::new_getclaritymarfvalue( + addr.into(), + "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::bar".to_string(), + TipRequest::UseLatestAnchoredTip, + true, + ); + requests.push(request); + + // query existing unconfirmed + let request = StacksHttpRequest::new_getclaritymarfvalue( + addr.into(), + "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world-unconfirmed::1::bar-unconfirmed" + .to_string(), + TipRequest::UseLatestUnconfirmedTip, + true, + ); + requests.push(request); + + // query non-existant var + let request = StacksHttpRequest::new_getclaritymarfvalue( + addr.into(), + "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::does-not-exist".to_string(), + TipRequest::UseLatestAnchoredTip, + true, + ); + requests.push(request); + + // query non-existant contract + let request = StacksHttpRequest::new_getclaritymarfvalue( + addr.into(), + "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.does-not-exist::1::bar".to_string(), + TipRequest::UseLatestAnchoredTip, + true, + ); + requests.push(request); + + let mut responses = test_rpc(function_name!(), requests); + + // existing data + let response = responses.remove(0); + println!( + "Response:\n{}\n", + std::str::from_utf8(&response.try_serialize().unwrap()).unwrap() + ); + + assert_eq!( + response.preamble().get_canonical_stacks_tip_height(), + Some(1) + ); + + let resp = response.decode_data_var_response().unwrap(); + assert_eq!(resp.data, "0x0000000000000000000000000000000000"); + assert!(resp.marf_proof.is_some()); + + // unconfirmed data + let response = responses.remove(0); + debug!( + "Response:\n{}\n", + std::str::from_utf8(&response.try_serialize().unwrap()).unwrap() + ); + + assert_eq!( + response.preamble().get_canonical_stacks_tip_height(), + Some(1) + ); + + let resp = response.decode_data_var_response().unwrap(); + assert_eq!(resp.data, "0x0100000000000000000000000000000001"); + assert!(resp.marf_proof.is_some()); + + // no such var + let response = responses.remove(0); + debug!( + "Response:\n{}\n", + std::str::from_utf8(&response.try_serialize().unwrap()).unwrap() + ); + + let (preamble, body) = response.destruct(); + assert_eq!(preamble.status_code, 404); + + // no such contract + let response = responses.remove(0); + debug!( + "Response:\n{}\n", + std::str::from_utf8(&response.try_serialize().unwrap()).unwrap() + ); + + let (preamble, body) = response.destruct(); + assert_eq!(preamble.status_code, 404); +} diff --git a/stackslib/src/net/api/tests/getclaritymetadata.rs b/stackslib/src/net/api/tests/getclaritymetadata.rs new file mode 100644 index 0000000000..3de5949a87 --- /dev/null +++ b/stackslib/src/net/api/tests/getclaritymetadata.rs @@ -0,0 +1,166 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2024 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions}; +use clarity::vm::{ClarityName, ContractName}; +use stacks_common::codec::StacksMessageCodec; +use stacks_common::types::chainstate::StacksAddress; +use stacks_common::types::net::PeerHost; +use stacks_common::types::Address; + +use super::test_rpc; +use crate::net::api::*; +use crate::net::connection::ConnectionOptions; +use crate::net::httpcore::{ + HttpPreambleExtensions, HttpRequestContentsExtensions, RPCRequestHandler, StacksHttp, + StacksHttpRequest, +}; +use crate::net::{ProtocolFamily, TipRequest}; + +#[test] +fn test_try_parse_request() { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); + let mut http = StacksHttp::new(addr.clone(), &ConnectionOptions::default()); + + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::9::contract-size".to_string(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])), + ); + assert_eq!( + request.contents().tip_request(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])) + ); + let bytes = request.try_serialize().unwrap(); + + let (parsed_preamble, offset) = http.read_preamble(&bytes).unwrap(); + let mut handler = getclaritymetadata::RPCGetClarityMetadataRequestHandler::new(); + let mut parsed_request = http + .handle_try_parse_request( + &mut handler, + &parsed_preamble.expect_request(), + &bytes[offset..], + ) + .unwrap(); + + // parsed request consumes headers that would not be in a constructed request + parsed_request.clear_headers(); + let (preamble, contents) = parsed_request.destruct(); + + // consumed path args + assert_eq!( + handler.clarity_metadata_key, + Some("vm-metadata::9::contract-size".to_string()) + ); + assert_eq!( + handler.contract_identifier, + Some( + QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world" + ) + .unwrap() + ) + ); + + assert_eq!(&preamble, request.preamble()); + + handler.restart(); + assert!(handler.clarity_metadata_key.is_none()); +} + +#[test] +fn test_try_parse_request_for_analysis() { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); + let mut http = StacksHttp::new(addr.clone(), &ConnectionOptions::default()); + + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "analysis".to_string(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])), + ); + assert_eq!( + request.contents().tip_request(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])) + ); + let bytes = request.try_serialize().unwrap(); + + let (parsed_preamble, offset) = http.read_preamble(&bytes).unwrap(); + let mut handler = getclaritymetadata::RPCGetClarityMetadataRequestHandler::new(); + let mut parsed_request = http + .handle_try_parse_request( + &mut handler, + &parsed_preamble.expect_request(), + &bytes[offset..], + ) + .unwrap(); + + // parsed request consumes headers that would not be in a constructed request + parsed_request.clear_headers(); + let (preamble, contents) = parsed_request.destruct(); + + // consumed path args + assert_eq!(handler.clarity_metadata_key, Some("analysis".to_string())); + assert_eq!( + handler.contract_identifier, + Some( + QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world" + ) + .unwrap() + ) + ); + + assert_eq!(&preamble, request.preamble()); + + handler.restart(); + assert!(handler.clarity_metadata_key.is_none()); +} + +#[test] +fn test_try_make_response() { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); + + let mut requests = vec![]; + + // query existing + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::9::contract-size".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + + let mut responses = test_rpc(function_name!(), requests); + + // latest data + let response = responses.remove(0); + + assert_eq!( + response.preamble().get_canonical_stacks_tip_height(), + Some(1) + ); + + let resp = response.decode_clarity_metadata_response().unwrap(); + assert_eq!(resp.data, "1432"); +} diff --git a/stackslib/src/net/api/tests/mod.rs b/stackslib/src/net/api/tests/mod.rs index f0a537d045..72800164f3 100644 --- a/stackslib/src/net/api/tests/mod.rs +++ b/stackslib/src/net/api/tests/mod.rs @@ -60,6 +60,8 @@ mod getattachment; mod getattachmentsinv; mod getblock; mod getblock_v3; +mod getclaritymarfvalue; +mod getclaritymetadata; mod getconstantval; mod getcontractabi; mod getcontractsrc; @@ -117,7 +119,7 @@ const TEST_CONTRACT: &'static str = " (ok 1))) (begin (map-set unit-map { account: 'ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R } { units: 123 })) - + (define-read-only (ro-confirmed) u1) (define-public (do-test) (ok u0)) diff --git a/stackslib/src/net/httpcore.rs b/stackslib/src/net/httpcore.rs index fb92b03b18..71bcfff284 100644 --- a/stackslib/src/net/httpcore.rs +++ b/stackslib/src/net/httpcore.rs @@ -248,7 +248,7 @@ pub mod request { } /// Get and parse a MARF key from a path's captures, given the name of the regex field. - pub fn get_marf_key(captures: &Captures, key: &str) -> Result { + pub fn get_key(captures: &Captures, key: &str) -> Result { let marf_key = if let Some(marf_key_str) = captures.name(key) { marf_key_str.as_str().to_string() } else { From ccf60f18cadf4757a915ef5b91e37a6ed55a178c Mon Sep 17 00:00:00 2001 From: Hugo Caillard <911307+hugocaillard@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:20:39 +0200 Subject: [PATCH 03/21] docs: add get_clarity_mark_value and get_clarity_metadata documentation --- docs/rpc-endpoints.md | 29 ++++++ .../get-clarity-marf-value.example.json | 4 + .../get-clarity-marf-value.schema.json | 17 ++++ .../get-clarity-metadata.example.json | 3 + .../get-clarity-metadata.schema.json | 13 +++ docs/rpc/openapi.yaml | 93 ++++++++++++++++++- 6 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 docs/rpc/api/core-node/get-clarity-marf-value.example.json create mode 100644 docs/rpc/api/core-node/get-clarity-marf-value.schema.json create mode 100644 docs/rpc/api/core-node/get-clarity-metadata.example.json create mode 100644 docs/rpc/api/core-node/get-clarity-metadata.schema.json diff --git a/docs/rpc-endpoints.md b/docs/rpc-endpoints.md index 6163f27b75..117d74d0dc 100644 --- a/docs/rpc-endpoints.md +++ b/docs/rpc-endpoints.md @@ -172,6 +172,35 @@ Where data is the hex serialization of the variable value. This endpoint also accepts a querystring parameter `?proof=` which when supplied `0`, will return the JSON object _without_ the `proof` field. +### GET /v2/clarity_marf_value/[Clarity MARF Key] +Attempt to fetch the value of a MARF key. The key is identified with [Clarity MARF Key]. + +Returns JSON data in the form: + +```json +{ + "data": "0x01ce...", + "proof": "0x01ab...", +} +``` + +Where data is the hex serialization of the value. + +### GET /v2/clarity_metadata/[Stacks Address]/[Contract Name]/[Clarity Metadata Key] +Attempt to fetch the metadata of a contract. + The contract is identified with [Stacks Address] and [Contract Name] in the URL path. + The metadata key is identified with [Clarity Metadata Key]. + +Returns JSON data in the form: + +```json +{ + "data": "'{\"contract_identifier\":{...}'", +} +``` + +Where data is the metadata formatted as a JSON string. + ### GET /v2/constant_val/[Stacks Address]/[Contract Name]/[Constant Name] Attempt to fetch a constant from a contract. The contract is identified with [Stacks Address] and [Contract Name] in the URL path. The constant is identified with [Constant Name]. diff --git a/docs/rpc/api/core-node/get-clarity-marf-value.example.json b/docs/rpc/api/core-node/get-clarity-marf-value.example.json new file mode 100644 index 0000000000..d0e233416f --- /dev/null +++ b/docs/rpc/api/core-node/get-clarity-marf-value.example.json @@ -0,0 +1,4 @@ +{ + "data": "0x0a0c000000010a6d6f6e737465722d69640100000000000000000000000000000001", + "proof": "0x123..." +} diff --git a/docs/rpc/api/core-node/get-clarity-marf-value.schema.json b/docs/rpc/api/core-node/get-clarity-marf-value.schema.json new file mode 100644 index 0000000000..ea7e7894fb --- /dev/null +++ b/docs/rpc/api/core-node/get-clarity-marf-value.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Response of get Clarity MARF value request", + "title": "ClarityMARFValueResponse", + "type": "object", + "required": ["data"], + "properties": { + "data": { + "type": "string", + "description": "Hex-encoded string" + }, + "proof": { + "type": "string", + "description": "Hex-encoded string of the MARF proof for the data" + } + } +} diff --git a/docs/rpc/api/core-node/get-clarity-metadata.example.json b/docs/rpc/api/core-node/get-clarity-metadata.example.json new file mode 100644 index 0000000000..5bb4bd5c47 --- /dev/null +++ b/docs/rpc/api/core-node/get-clarity-metadata.example.json @@ -0,0 +1,3 @@ +{ + "data": "'{\"contract_identifier\":{...}, \"private_function_types\":{...}'" +} diff --git a/docs/rpc/api/core-node/get-clarity-metadata.schema.json b/docs/rpc/api/core-node/get-clarity-metadata.schema.json new file mode 100644 index 0000000000..3c0104fa41 --- /dev/null +++ b/docs/rpc/api/core-node/get-clarity-metadata.schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Response of get clarity metadata request", + "title": "ClarityMetadataResponse", + "type": "object", + "required": ["data"], + "properties": { + "data": { + "type": "string", + "description": "Metadata value formatted as a JSON string" + } + } +} diff --git a/docs/rpc/openapi.yaml b/docs/rpc/openapi.yaml index f33e0dca73..e77fc31ade 100644 --- a/docs/rpc/openapi.yaml +++ b/docs/rpc/openapi.yaml @@ -519,7 +519,94 @@ paths: description: | The Stacks chain tip to query from. If tip == "latest", the query will be run from the latest known tip (includes unconfirmed state). - If the tip is left unspecified, the stacks chain tip will be selected (only includes confirmed state). + If the tip is left unspecified, the stacks chain tip will be selected (only includes confirmed state). + + /v2/clarity_marf_value/{clarity_marf_key}: + post: + summary: Get the MARF value for a given key + tags: + - Smart Contracts + operationId: get_clarity_marf_value + description: | + Attempt to fetch the value of a MARF key. The key is identified with [Clarity MARF Key]. + + In the response, `data` is the hex serialization of the value. + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: ./api/core-node/get-clarity-marf-value.schema.json + example: + $ref: ./api/core-node/get-clarity-marf-value.example.json + 400: + description: Failed to retrieve MARF key + parameters: + - name: clarity_marf_key + in: path + required: true + description: MARF key + schema: + type: string + - name: proof + in: query + description: Returns object without the proof field when set to 0 + schema: + type: integer + - name: tip + in: query + schema: + type: string + description: The Stacks chain tip to query from. If tip == latest, the query will be run from the latest + known tip (includes unconfirmed state). + + /v2/clarity_metadata/{contract_address}/{contract_name}/{clarity_metadata_key}: + post: + summary: Get the contract metadata for the metadata key + tags: + - Smart Contracts + operationId: get_clarity_metadata_key + description: | + Attempt to fetch the metadata of a contract. The contract is identified with [Stacks Address] and [Contract Name] in the URL path. The metadata key is identified with [Clarity Metadata Key]. + + In the response, `data` is formatted as JSON. + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: ./api/core-node/get-clarity-metadata.schema.json + example: + $ref: ./api/core-node/get-clarity-metadata.example.json + 400: + description: Failed to retrieve constant value from contract + parameters: + - name: contract_address + in: path + required: true + description: Stacks address + schema: + type: string + - name: contract_name + in: path + required: true + description: Contract name + schema: + type: string + - name: clarity_metadata_key + in: path + required: true + description: Metadata key + schema: + type: string + - name: tip + in: query + schema: + type: string + description: The Stacks chain tip to query from. If tip == latest, the query will be run from the latest + known tip (includes unconfirmed state). /v2/constant_val/{contract_address}/{contract_name}/{constant_name}: post: @@ -633,7 +720,7 @@ paths: /v3/blocks/{block_id}: get: - summary: Fetch a Nakamoto block + summary: Fetch a Nakamoto block tags: - Blocks operationId: get_block_v3 @@ -674,7 +761,7 @@ paths: application/json: example: $ref: ./api/core-node/get_tenure_info.json - + /v3/tenures/{block_id}: get: summary: Fetch a sequence of Nakamoto blocks in a tenure From 26c448727def84bbef34ba4d14ec4fd5244a2390 Mon Sep 17 00:00:00 2001 From: Hugo Caillard <911307+hugocaillard@users.noreply.github.com> Date: Tue, 23 Jul 2024 18:12:47 +0200 Subject: [PATCH 04/21] refactor: improve clarity_mark_key and clarity_metadata request parsing --- clarity/src/vm/representations.rs | 11 +++++++---- stackslib/src/net/api/getclaritymarfvalue.rs | 2 +- stackslib/src/net/api/getclaritymetadata.rs | 2 +- stackslib/src/net/api/tests/getclaritymarfvalue.rs | 2 +- stackslib/src/net/httpcore.rs | 12 ++++++------ 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/clarity/src/vm/representations.rs b/clarity/src/vm/representations.rs index 8f140a6a4d..ee3387d629 100644 --- a/clarity/src/vm/representations.rs +++ b/clarity/src/vm/representations.rs @@ -51,6 +51,8 @@ lazy_static! { "({})|({})", *STANDARD_PRINCIPAL_REGEX_STRING, *CONTRACT_PRINCIPAL_REGEX_STRING ); + static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = + "[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?".into(); pub static ref CLARITY_NAME_REGEX_STRING: String = "^[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*$|^[-+=/*]$|^[<>]=?$".into(); pub static ref CLARITY_NAME_REGEX: Regex = @@ -65,16 +67,17 @@ lazy_static! { .unwrap() }; pub static ref MARF_KEY_FOR_TRIP_REGEX_STRING: String = format!( - r"vm::{}::\d+::.*", + r"vm::{}::\d+::({})", *CONTRACT_PRINCIPAL_REGEX_STRING, + *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, ); pub static ref MARF_KEY_FOR_QUAD_REGEX_STRING: String = format!( - r"{}::.*", + r"{}::[0-9a-fA-F]+", *MARF_KEY_FOR_TRIP_REGEX_STRING, ); pub static ref METADATA_KEY_REGEX_STRING: String = format!( - r"vm-metadata::\d+::.*", - + r"vm-metadata::\d+::(contract|contract-size|contract-src|contract-data-size|({}))", + *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, ); } diff --git a/stackslib/src/net/api/getclaritymarfvalue.rs b/stackslib/src/net/api/getclaritymarfvalue.rs index b950f7ffcc..470c5872ba 100644 --- a/stackslib/src/net/api/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/getclaritymarfvalue.rs @@ -86,7 +86,7 @@ impl HttpRequest for RPCGetClarityMarfValueRequestHandler { )); } - let marf_key = request::get_key(captures, "clarity_marf_key")?; + let marf_key = request::get_clarity_key(captures, "clarity_marf_key")?; self.clarity_marf_key = Some(marf_key); diff --git a/stackslib/src/net/api/getclaritymetadata.rs b/stackslib/src/net/api/getclaritymetadata.rs index 2feb81613a..8a08a48503 100644 --- a/stackslib/src/net/api/getclaritymetadata.rs +++ b/stackslib/src/net/api/getclaritymetadata.rs @@ -89,7 +89,7 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { } let contract_identifier = request::get_contract_address(captures, "address", "contract")?; - let metadata_key = request::get_key(captures, "clarity_metadata_key")?; + let metadata_key = request::get_clarity_key(captures, "clarity_metadata_key")?; self.contract_identifier = Some(contract_identifier); self.clarity_metadata_key = Some(metadata_key); diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index b4129c7d93..40e73c41e4 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -128,7 +128,7 @@ fn test_try_make_response() { // existing data let response = responses.remove(0); - println!( + debug!( "Response:\n{}\n", std::str::from_utf8(&response.try_serialize().unwrap()).unwrap() ); diff --git a/stackslib/src/net/httpcore.rs b/stackslib/src/net/httpcore.rs index 71bcfff284..731b5fb771 100644 --- a/stackslib/src/net/httpcore.rs +++ b/stackslib/src/net/httpcore.rs @@ -247,15 +247,15 @@ pub mod request { Ok(txid) } - /// Get and parse a MARF key from a path's captures, given the name of the regex field. - pub fn get_key(captures: &Captures, key: &str) -> Result { - let marf_key = if let Some(marf_key_str) = captures.name(key) { - marf_key_str.as_str().to_string() + /// Get a clarity key (MARF or Metadata) from a path's captures, given the name of the regex field. + pub fn get_clarity_key(captures: &Captures, clarity_key: &str) -> Result { + let key = if let Some(key_str) = captures.name(clarity_key) { + key_str.as_str().to_string() } else { - return Err(HttpError::Http(404, format!("Missing `{}`", key))); + return Err(HttpError::Http(404, format!("Missing `{}`", clarity_key))); }; - Ok(marf_key) + Ok(key) } /// Get and parse a Clarity name from a path's captures, given the name of the regex field. From 71cad36a21b3e1d6c47b57645a8857f61e938af9 Mon Sep 17 00:00:00 2001 From: Hugo Caillard <911307+hugocaillard@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:29:10 +0200 Subject: [PATCH 05/21] refactor: rename get clarity marf value and metadata endpoints --- clarity/src/vm/representations.rs | 15 ------ stackslib/src/net/api/getclaritymarfvalue.rs | 50 ++++++++++++------- stackslib/src/net/api/getclaritymetadata.rs | 30 ++++++++--- stackslib/src/net/api/mod.rs | 4 +- .../src/net/api/tests/getclaritymarfvalue.rs | 12 ++--- stackslib/src/net/httpcore.rs | 11 ---- 6 files changed, 62 insertions(+), 60 deletions(-) diff --git a/clarity/src/vm/representations.rs b/clarity/src/vm/representations.rs index ee3387d629..c80e3c7467 100644 --- a/clarity/src/vm/representations.rs +++ b/clarity/src/vm/representations.rs @@ -51,8 +51,6 @@ lazy_static! { "({})|({})", *STANDARD_PRINCIPAL_REGEX_STRING, *CONTRACT_PRINCIPAL_REGEX_STRING ); - static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = - "[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?".into(); pub static ref CLARITY_NAME_REGEX_STRING: String = "^[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*$|^[-+=/*]$|^[<>]=?$".into(); pub static ref CLARITY_NAME_REGEX: Regex = @@ -66,19 +64,6 @@ lazy_static! { Regex::new(format!("^{}$|^__transient$", CONTRACT_NAME_REGEX_STRING.as_str()).as_str()) .unwrap() }; - pub static ref MARF_KEY_FOR_TRIP_REGEX_STRING: String = format!( - r"vm::{}::\d+::({})", - *CONTRACT_PRINCIPAL_REGEX_STRING, - *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, - ); - pub static ref MARF_KEY_FOR_QUAD_REGEX_STRING: String = format!( - r"{}::[0-9a-fA-F]+", - *MARF_KEY_FOR_TRIP_REGEX_STRING, - ); - pub static ref METADATA_KEY_REGEX_STRING: String = format!( - r"vm-metadata::\d+::(contract|contract-size|contract-src|contract-data-size|({}))", - *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, - ); } guarded_string!( diff --git a/stackslib/src/net/api/getclaritymarfvalue.rs b/stackslib/src/net/api/getclaritymarfvalue.rs index 470c5872ba..ff584a0ccf 100644 --- a/stackslib/src/net/api/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/getclaritymarfvalue.rs @@ -15,9 +15,8 @@ // along with this program. If not, see . use clarity::vm::clarity::ClarityConnection; -use clarity::vm::representations::{ - MARF_KEY_FOR_QUAD_REGEX_STRING, MARF_KEY_FOR_TRIP_REGEX_STRING, -}; +use clarity::vm::representations::CONTRACT_PRINCIPAL_REGEX_STRING; +use lazy_static::lazy_static; use regex::{Captures, Regex}; use stacks_common::types::net::PeerHost; use stacks_common::util::hash::to_hex; @@ -32,8 +31,19 @@ use crate::net::httpcore::{ }; use crate::net::{Error as NetError, StacksNodeState, TipRequest}; +lazy_static! { + static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = + "[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?".into(); + static ref MARF_KEY_FOR_TRIP_REGEX_STRING: String = format!( + r"vm::{}::\d+::({})", + *CONTRACT_PRINCIPAL_REGEX_STRING, *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, + ); + static ref MARF_KEY_FOR_QUAD_REGEX_STRING: String = + format!(r"{}::[0-9a-fA-F]+", *MARF_KEY_FOR_TRIP_REGEX_STRING,); +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ClarityMarfValueResponse { +pub struct ClarityMarfResponse { pub data: String, #[serde(rename = "proof")] #[serde(default)] @@ -42,10 +52,10 @@ pub struct ClarityMarfValueResponse { } #[derive(Clone)] -pub struct RPCGetClarityMarfValueRequestHandler { +pub struct RPCGetClarityMarfRequestHandler { pub clarity_marf_key: Option, } -impl RPCGetClarityMarfValueRequestHandler { +impl RPCGetClarityMarfRequestHandler { pub fn new() -> Self { Self { clarity_marf_key: None, @@ -54,21 +64,21 @@ impl RPCGetClarityMarfValueRequestHandler { } /// Decode the HTTP request -impl HttpRequest for RPCGetClarityMarfValueRequestHandler { +impl HttpRequest for RPCGetClarityMarfRequestHandler { fn verb(&self) -> &'static str { "GET" } fn path_regex(&self) -> Regex { Regex::new(&format!( - r"^/v2/clarity_marf_value/(?P(vm-epoch::epoch-version)|({})|({}))$", + r"^/v2/clarity/marf/(?P(vm-epoch::epoch-version)|({})|({}))$", *MARF_KEY_FOR_TRIP_REGEX_STRING, *MARF_KEY_FOR_QUAD_REGEX_STRING )) .unwrap() } fn metrics_identifier(&self) -> &str { - "/v2/clarity_marf_value/:clarity_marf_key" + "/v2/clarity/marf/:clarity_marf_key" } /// Try to decode this request. @@ -86,7 +96,11 @@ impl HttpRequest for RPCGetClarityMarfValueRequestHandler { )); } - let marf_key = request::get_clarity_key(captures, "clarity_marf_key")?; + let marf_key = if let Some(key_str) = captures.name("clarity_marf_key") { + key_str.as_str().to_string() + } else { + return Err(Error::Http(404, "Missing `clarity_marf_key`".to_string())); + }; self.clarity_marf_key = Some(marf_key); @@ -96,7 +110,7 @@ impl HttpRequest for RPCGetClarityMarfValueRequestHandler { } /// Handle the HTTP request -impl RPCRequestHandler for RPCGetClarityMarfValueRequestHandler { +impl RPCRequestHandler for RPCGetClarityMarfRequestHandler { /// Reset internal state fn restart(&mut self) { self.clarity_marf_key = None; @@ -143,7 +157,7 @@ impl RPCRequestHandler for RPCGetClarityMarfValueRequestHandler { }; let data = format!("0x{}", value_hex); - Some(ClarityMarfValueResponse { data, marf_proof }) + Some(ClarityMarfResponse { data, marf_proof }) }) }, ) @@ -177,19 +191,19 @@ impl RPCRequestHandler for RPCGetClarityMarfValueRequestHandler { } /// Decode the HTTP response -impl HttpResponse for RPCGetClarityMarfValueRequestHandler { +impl HttpResponse for RPCGetClarityMarfRequestHandler { fn try_parse_response( &self, preamble: &HttpResponsePreamble, body: &[u8], ) -> Result { - let marf_value: ClarityMarfValueResponse = parse_json(preamble, body)?; + let marf_value: ClarityMarfResponse = parse_json(preamble, body)?; Ok(HttpResponsePayload::try_from_json(marf_value)?) } } impl StacksHttpRequest { - pub fn new_getclaritymarfvalue( + pub fn new_getclaritymarf( host: PeerHost, clarity_marf_key: String, tip_req: TipRequest, @@ -198,7 +212,7 @@ impl StacksHttpRequest { StacksHttpRequest::new_for_peer( host, "GET".into(), - format!("/v2/clarity_marf_value/{}", &clarity_marf_key), + format!("/v2/clarity/marf/{}", &clarity_marf_key), HttpRequestContents::new() .for_tip(tip_req) .query_arg("proof".into(), if with_proof { "1" } else { "0" }.into()), @@ -208,10 +222,10 @@ impl StacksHttpRequest { } impl StacksHttpResponse { - pub fn decode_clarity_marf_value_response(self) -> Result { + pub fn decode_clarity_marf_response(self) -> Result { let contents = self.get_http_payload_ok()?; let contents_json: serde_json::Value = contents.try_into()?; - let resp: ClarityMarfValueResponse = serde_json::from_value(contents_json) + let resp: ClarityMarfResponse = serde_json::from_value(contents_json) .map_err(|_e| NetError::DeserializeError("Failed to load from JSON".to_string()))?; Ok(resp) } diff --git a/stackslib/src/net/api/getclaritymetadata.rs b/stackslib/src/net/api/getclaritymetadata.rs index 8a08a48503..5ef3feee6e 100644 --- a/stackslib/src/net/api/getclaritymetadata.rs +++ b/stackslib/src/net/api/getclaritymetadata.rs @@ -15,11 +15,10 @@ // along with this program. If not, see . use clarity::vm::clarity::ClarityConnection; -use clarity::vm::representations::{ - CONTRACT_NAME_REGEX_STRING, METADATA_KEY_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING, -}; +use clarity::vm::representations::{CONTRACT_NAME_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING}; use clarity::vm::types::QualifiedContractIdentifier; use clarity::vm::ContractName; +use lazy_static::lazy_static; use regex::{Captures, Regex}; use stacks_common::types::chainstate::StacksAddress; use stacks_common::types::net::PeerHost; @@ -34,6 +33,15 @@ use crate::net::httpcore::{ }; use crate::net::{Error as NetError, StacksNodeState, TipRequest}; +lazy_static! { + static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = + "[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?".into(); + static ref METADATA_KEY_REGEX_STRING: String = format!( + r"vm-metadata::\d+::(contract|contract-size|contract-src|contract-data-size|({}))", + *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, + ); +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ClarityMetadataResponse { pub data: String, @@ -61,7 +69,7 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { fn path_regex(&self) -> Regex { Regex::new(&format!( - r"^/v2/clarity_metadata/(?P
{})/(?P{})/(?P(analysis)|({}))$", + r"^/v2/clarity/metadata/(?P
{})/(?P{})/(?P(analysis)|({}))$", *STANDARD_PRINCIPAL_REGEX_STRING, *CONTRACT_NAME_REGEX_STRING, *METADATA_KEY_REGEX_STRING @@ -70,7 +78,7 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { } fn metrics_identifier(&self) -> &str { - "/v2/clarity_metadata/:principal/:contract_name/:clarity_metadata_key" + "/v2/clarity/metadata/:principal/:contract_name/:clarity_metadata_key" } /// Try to decode this request. @@ -89,7 +97,15 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { } let contract_identifier = request::get_contract_address(captures, "address", "contract")?; - let metadata_key = request::get_clarity_key(captures, "clarity_metadata_key")?; + + let metadata_key = if let Some(key_str) = captures.name("clarity_metadata_key") { + key_str.as_str().to_string() + } else { + return Err(Error::Http( + 404, + "Missing `clarity_metadata_key`".to_string(), + )); + }; self.contract_identifier = Some(contract_identifier); self.clarity_metadata_key = Some(metadata_key); @@ -197,7 +213,7 @@ impl StacksHttpRequest { host, "GET".into(), format!( - "/v2/clarity_metadata/{}/{}/{}", + "/v2/clarity/metadata/{}/{}/{}", &contract_addr, &contract_name, &clarity_metadata_key ), HttpRequestContents::new().for_tip(tip_req), diff --git a/stackslib/src/net/api/mod.rs b/stackslib/src/net/api/mod.rs index 9e6be1bf11..0057f9047d 100644 --- a/stackslib/src/net/api/mod.rs +++ b/stackslib/src/net/api/mod.rs @@ -92,9 +92,7 @@ impl StacksHttp { self.register_rpc_endpoint(getattachmentsinv::RPCGetAttachmentsInvRequestHandler::new()); self.register_rpc_endpoint(getblock::RPCBlocksRequestHandler::new()); self.register_rpc_endpoint(getblock_v3::RPCNakamotoBlockRequestHandler::new()); - self.register_rpc_endpoint( - getclaritymarfvalue::RPCGetClarityMarfValueRequestHandler::new(), - ); + self.register_rpc_endpoint(getclaritymarfvalue::RPCGetClarityMarfRequestHandler::new()); self.register_rpc_endpoint(getclaritymetadata::RPCGetClarityMetadataRequestHandler::new()); self.register_rpc_endpoint(getconstantval::RPCGetConstantValRequestHandler::new()); self.register_rpc_endpoint(getcontractabi::RPCGetContractAbiRequestHandler::new()); diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index 40e73c41e4..ce342b7442 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -43,7 +43,7 @@ fn test_try_parse_request() { let valid_keys = [vm_key_epoch, vm_key_trip, vm_key_quad]; for key in valid_keys { - let request = StacksHttpRequest::new_getclaritymarfvalue( + let request = StacksHttpRequest::new_getclaritymarf( addr.into(), key.to_string(), TipRequest::SpecificTip(StacksBlockId([0x22; 32])), @@ -58,7 +58,7 @@ fn test_try_parse_request() { let bytes = request.try_serialize().unwrap(); let (parsed_preamble, offset) = http.read_preamble(&bytes).unwrap(); - let mut handler = getclaritymarfvalue::RPCGetClarityMarfValueRequestHandler::new(); + let mut handler = getclaritymarfvalue::RPCGetClarityMarfRequestHandler::new(); let mut parsed_request = http .handle_try_parse_request( &mut handler, @@ -88,7 +88,7 @@ fn test_try_make_response() { let mut requests = vec![]; // query existing - let request = StacksHttpRequest::new_getclaritymarfvalue( + let request = StacksHttpRequest::new_getclaritymarf( addr.into(), "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::bar".to_string(), TipRequest::UseLatestAnchoredTip, @@ -97,7 +97,7 @@ fn test_try_make_response() { requests.push(request); // query existing unconfirmed - let request = StacksHttpRequest::new_getclaritymarfvalue( + let request = StacksHttpRequest::new_getclaritymarf( addr.into(), "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world-unconfirmed::1::bar-unconfirmed" .to_string(), @@ -107,7 +107,7 @@ fn test_try_make_response() { requests.push(request); // query non-existant var - let request = StacksHttpRequest::new_getclaritymarfvalue( + let request = StacksHttpRequest::new_getclaritymarf( addr.into(), "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::does-not-exist".to_string(), TipRequest::UseLatestAnchoredTip, @@ -116,7 +116,7 @@ fn test_try_make_response() { requests.push(request); // query non-existant contract - let request = StacksHttpRequest::new_getclaritymarfvalue( + let request = StacksHttpRequest::new_getclaritymarf( addr.into(), "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.does-not-exist::1::bar".to_string(), TipRequest::UseLatestAnchoredTip, diff --git a/stackslib/src/net/httpcore.rs b/stackslib/src/net/httpcore.rs index 731b5fb771..dec51df42a 100644 --- a/stackslib/src/net/httpcore.rs +++ b/stackslib/src/net/httpcore.rs @@ -247,17 +247,6 @@ pub mod request { Ok(txid) } - /// Get a clarity key (MARF or Metadata) from a path's captures, given the name of the regex field. - pub fn get_clarity_key(captures: &Captures, clarity_key: &str) -> Result { - let key = if let Some(key_str) = captures.name(clarity_key) { - key_str.as_str().to_string() - } else { - return Err(HttpError::Http(404, format!("Missing `{}`", clarity_key))); - }; - - Ok(key) - } - /// Get and parse a Clarity name from a path's captures, given the name of the regex field. pub fn get_clarity_name(captures: &Captures, key: &str) -> Result { let clarity_name = if let Some(name_str) = captures.name(key) { From 74ac33fed796250db7642a3c1a3492a84fa856f8 Mon Sep 17 00:00:00 2001 From: Hugo Caillard <911307+hugocaillard@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:49:23 +0100 Subject: [PATCH 06/21] feat: get data from hash --- clarity/src/vm/database/clarity_store.rs | 13 ++++- clarity/src/vm/database/sqlite.rs | 9 ++- stackslib/src/chainstate/stacks/index/marf.rs | 58 +++++++++++++++++++ stackslib/src/clarity_vm/database/marf.rs | 51 ++++++++++++++++ stackslib/src/clarity_vm/database/mod.rs | 9 +++ 5 files changed, 138 insertions(+), 2 deletions(-) diff --git a/clarity/src/vm/database/clarity_store.rs b/clarity/src/vm/database/clarity_store.rs index b6a45ee764..6e8f878f6e 100644 --- a/clarity/src/vm/database/clarity_store.rs +++ b/clarity/src/vm/database/clarity_store.rs @@ -18,7 +18,7 @@ use std::path::PathBuf; #[cfg(feature = "canonical")] use rusqlite::Connection; -use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, VRFSeed}; +use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash, VRFSeed}; use stacks_common::util::hash::{hex_bytes, to_hex, Hash160, Sha512Trunc256Sum}; use crate::vm::analysis::AnalysisDatabase; @@ -67,6 +67,10 @@ pub trait ClarityBackingStore { /// fetch K-V out of the committed datastore, along with the byte representation /// of the Merkle proof for that key-value pair fn get_data_with_proof(&mut self, key: &str) -> Result)>>; + fn get_data_with_proof_from_path( + &mut self, + hash: &TrieHash, + ) -> Result)>>; fn has_entry(&mut self, key: &str) -> Result { Ok(self.get_data(key)?.is_some()) } @@ -213,6 +217,13 @@ impl ClarityBackingStore for NullBackingStore { panic!("NullBackingStore can't retrieve data") } + fn get_data_with_proof_from_path( + &mut self, + _hash: &TrieHash, + ) -> Result)>> { + panic!("NullBackingStore can't retrieve data") + } + #[cfg(feature = "canonical")] fn get_side_store(&mut self) -> &Connection { panic!("NullBackingStore has no side store") diff --git a/clarity/src/vm/database/sqlite.rs b/clarity/src/vm/database/sqlite.rs index dc3ad4f5bd..92a932676d 100644 --- a/clarity/src/vm/database/sqlite.rs +++ b/clarity/src/vm/database/sqlite.rs @@ -19,7 +19,7 @@ use rusqlite::{ params, Connection, Error as SqliteError, ErrorCode as SqliteErrorCode, OptionalExtension, Row, Savepoint, }; -use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId}; +use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash}; use stacks_common::types::sqlite::NO_PARAMS; use stacks_common::util::db_common::tx_busy_handler; use stacks_common::util::hash::Sha512Trunc256Sum; @@ -328,6 +328,13 @@ impl ClarityBackingStore for MemoryBackingStore { Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![]))) } + fn get_data_with_proof_from_path( + &mut self, + hash: &TrieHash, + ) -> Result)>> { + self.get_data_with_proof(&hash.to_string()) + } + fn get_side_store(&mut self) -> &Connection { &self.side_store } diff --git a/stackslib/src/chainstate/stacks/index/marf.rs b/stackslib/src/chainstate/stacks/index/marf.rs index d5dd77c51f..81506a08c3 100644 --- a/stackslib/src/chainstate/stacks/index/marf.rs +++ b/stackslib/src/chainstate/stacks/index/marf.rs @@ -142,6 +142,22 @@ pub trait MarfConnection { }) } + fn get_with_proof_from_hash( + &mut self, + block_hash: &T, + hash: &TrieHash, + ) -> Result)>, Error> { + self.with_conn(|conn| { + let path = TriePath::from_bytes(hash.as_bytes()).ok_or(Error::BadSeekValue)?; + let marf_value = match MARF::get_by_path(conn, block_hash, &path)? { + None => return Ok(None), + Some(x) => x, + }; + let proof = TrieMerkleProof::from_path(conn, &path, &marf_value, block_hash)?; + Ok(Some((marf_value, proof))) + }) + } + fn get_block_at_height(&mut self, height: u32, tip: &T) -> Result, Error> { self.with_conn(|c| MARF::get_block_at_height(c, height, tip)) } @@ -1123,6 +1139,33 @@ impl MARF { Ok(MARF::from_storage(file_storage)) } + pub fn get_by_path( + storage: &mut TrieStorageConnection, + block_hash: &T, + path: &TriePath, + ) -> Result, Error> { + let (cur_block_hash, cur_block_id) = storage.get_cur_block_and_id(); + + let result = MARF::get_path(storage, block_hash, &path).or_else(|e| match e { + Error::NotFoundError => Ok(None), + _ => Err(e), + }); + + // restore + storage + .open_block_maybe_id(&cur_block_hash, cur_block_id) + .map_err(|e| { + warn!( + "Failed to re-open {} {:?}: {:?}", + &cur_block_hash, cur_block_id, &e + ); + warn!("Result of failed path lookup '{}': {:?}", path, &result); + e + })?; + + result.map(|option_result| option_result.map(|leaf| leaf.data)) + } + pub fn get_by_key( storage: &mut TrieStorageConnection, block_hash: &T, @@ -1320,6 +1363,21 @@ impl MARF { Ok(Some((marf_value, proof))) } + pub fn get_with_proof_from_hash( + &mut self, + block_hash: &T, + hash: &TrieHash, + ) -> Result)>, Error> { + let mut conn = self.storage.connection(); + let path = TriePath::from_bytes(hash.as_bytes()).ok_or(Error::BadSeekValue)?; + let marf_value = match MARF::get_by_path(&mut conn, block_hash, &path)? { + None => return Ok(None), + Some(x) => x, + }; + let proof = TrieMerkleProof::from_path(&mut conn, &path, &marf_value, block_hash)?; + Ok(Some((marf_value, proof))) + } + pub fn get_bhh_at_height(&mut self, block_hash: &T, height: u32) -> Result, Error> { MARF::get_block_at_height(&mut self.storage.connection(), height, block_hash) } diff --git a/stackslib/src/clarity_vm/database/marf.rs b/stackslib/src/clarity_vm/database/marf.rs index fed0e70e95..be0f60ff56 100644 --- a/stackslib/src/clarity_vm/database/marf.rs +++ b/stackslib/src/clarity_vm/database/marf.rs @@ -20,6 +20,7 @@ use stacks_common::codec::StacksMessageCodec; use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash}; use crate::chainstate::stacks::index::marf::{MARFOpenOpts, MarfConnection, MarfTransaction, MARF}; +use crate::chainstate::stacks::index::node::TriePath; use crate::chainstate::stacks::index::{ ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieMerkleProof, }; @@ -422,6 +423,31 @@ impl<'a> ClarityBackingStore for ReadOnlyMarfStore<'a> { .transpose() } + fn get_data_with_proof_from_path( + &mut self, + hash: &TrieHash, + ) -> InterpreterResult)>> { + self.marf + .get_with_proof_from_hash(&self.chain_tip, hash) + .or_else(|e| match e { + Error::NotFoundError => Ok(None), + _ => Err(e), + }) + .map_err(|_| InterpreterError::Expect("ERROR: Unexpected MARF Failure on GET".into()))? + .map(|(marf_value, proof)| { + let side_key = marf_value.to_hex(); + let data = + SqliteConnection::get(self.get_side_store(), &side_key)?.ok_or_else(|| { + InterpreterError::Expect(format!( + "ERROR: MARF contained value_hash not found in side storage: {}", + side_key + )) + })?; + Ok((data, proof.serialize_to_vec())) + }) + .transpose() + } + fn get_data(&mut self, key: &str) -> InterpreterResult> { trace!("MarfedKV get: {:?} tip={}", key, &self.chain_tip); self.marf @@ -653,6 +679,31 @@ impl<'a> ClarityBackingStore for WritableMarfStore<'a> { .transpose() } + fn get_data_with_proof_from_path( + &mut self, + hash: &TrieHash, + ) -> InterpreterResult)>> { + self.marf + .get_with_proof_from_hash(&self.chain_tip, hash) + .or_else(|e| match e { + Error::NotFoundError => Ok(None), + _ => Err(e), + }) + .map_err(|_| InterpreterError::Expect("ERROR: Unexpected MARF Failure on GET".into()))? + .map(|(marf_value, proof)| { + let side_key = marf_value.to_hex(); + let data = + SqliteConnection::get(self.marf.sqlite_tx(), &side_key)?.ok_or_else(|| { + InterpreterError::Expect(format!( + "ERROR: MARF contained value_hash not found in side storage: {}", + side_key + )) + })?; + Ok((data, proof.serialize_to_vec())) + }) + .transpose() + } + fn get_side_store(&mut self) -> &Connection { self.marf.sqlite_tx() } diff --git a/stackslib/src/clarity_vm/database/mod.rs b/stackslib/src/clarity_vm/database/mod.rs index 81f0bac43c..49a29773bd 100644 --- a/stackslib/src/clarity_vm/database/mod.rs +++ b/stackslib/src/clarity_vm/database/mod.rs @@ -1,5 +1,6 @@ use std::ops::{Deref, DerefMut}; +use clarity::types::chainstate::TrieHash; use clarity::util::hash::Sha512Trunc256Sum; use clarity::vm::analysis::AnalysisDatabase; use clarity::vm::database::sqlite::{ @@ -1138,6 +1139,14 @@ impl ClarityBackingStore for MemoryBackingStore { Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![]))) } + fn get_data_with_proof_from_path( + &mut self, + key: &TrieHash, + ) -> InterpreterResult)>> { + // Ok(SqliteConnection::get(self.get_side_store(), )?.map(|x| (x, vec![]))) + Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![]))) + } + fn get_side_store(&mut self) -> &Connection { &self.side_store } From 14535c41085c6681ede71404d9efb691b2166050 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 8 Nov 2024 15:07:47 -0500 Subject: [PATCH 07/21] chore: remove needless TriePath type and use TrieHash everywhere. Also, add `MARF::get_by_path()` to look up `MARFValue`s by `TrieHash` instead of by `&str` keys, and add the relevant `ClarityBackingStore` implementation to the stackslib's read-only and writeable MARF stores --- stacks-common/src/types/chainstate.rs | 62 +++++++++++++++ stackslib/src/burnchains/tests/burnchain.rs | 1 - .../src/chainstate/burn/db/processing.rs | 1 - stackslib/src/chainstate/burn/db/sortdb.rs | 1 - stackslib/src/chainstate/burn/distribution.rs | 1 - stackslib/src/chainstate/burn/mod.rs | 1 - .../burn/operations/leader_block_commit.rs | 1 - .../burn/operations/leader_key_register.rs | 1 - stackslib/src/chainstate/burn/sortition.rs | 2 +- stackslib/src/chainstate/stacks/index/bits.rs | 10 +-- .../src/chainstate/stacks/index/cache.rs | 6 +- stackslib/src/chainstate/stacks/index/file.rs | 2 +- stackslib/src/chainstate/stacks/index/marf.rs | 74 +++++++++++++----- stackslib/src/chainstate/stacks/index/mod.rs | 65 ---------------- stackslib/src/chainstate/stacks/index/node.rs | 23 +----- .../src/chainstate/stacks/index/proofs.rs | 20 ++--- .../src/chainstate/stacks/index/storage.rs | 4 +- .../src/chainstate/stacks/index/test/cache.rs | 4 +- .../src/chainstate/stacks/index/test/file.rs | 4 +- .../src/chainstate/stacks/index/test/marf.rs | 66 ++++++++-------- .../src/chainstate/stacks/index/test/mod.rs | 8 +- .../src/chainstate/stacks/index/test/node.rs | 20 ++--- .../chainstate/stacks/index/test/proofs.rs | 10 +-- .../chainstate/stacks/index/test/storage.rs | 6 +- .../src/chainstate/stacks/index/test/trie.rs | 76 +++++++++---------- stackslib/src/chainstate/stacks/index/trie.rs | 2 +- .../src/chainstate/stacks/index/trie_sql.rs | 2 +- stackslib/src/core/tests/mod.rs | 2 +- 28 files changed, 241 insertions(+), 234 deletions(-) diff --git a/stacks-common/src/types/chainstate.rs b/stacks-common/src/types/chainstate.rs index 47d6c3c499..59347ed36a 100644 --- a/stacks-common/src/types/chainstate.rs +++ b/stacks-common/src/types/chainstate.rs @@ -30,6 +30,68 @@ impl_byte_array_serde!(TrieHash); pub const TRIEHASH_ENCODED_SIZE: usize = 32; +impl TrieHash { + pub fn from_key(k: &str) -> Self { + Self::from_data(k.as_bytes()) + } + + /// TrieHash of zero bytes + pub fn from_empty_data() -> TrieHash { + // sha2-512/256 hash of empty string. + // this is used so frequently it helps performance if we just have a constant for it. + TrieHash([ + 0xc6, 0x72, 0xb8, 0xd1, 0xef, 0x56, 0xed, 0x28, 0xab, 0x87, 0xc3, 0x62, 0x2c, 0x51, + 0x14, 0x06, 0x9b, 0xdd, 0x3a, 0xd7, 0xb8, 0xf9, 0x73, 0x74, 0x98, 0xd0, 0xc0, 0x1e, + 0xce, 0xf0, 0x96, 0x7a, + ]) + } + + /// TrieHash from bytes + pub fn from_data(data: &[u8]) -> TrieHash { + if data.len() == 0 { + return TrieHash::from_empty_data(); + } + + let mut tmp = [0u8; 32]; + + let mut hasher = Sha512_256::new(); + hasher.update(data); + tmp.copy_from_slice(hasher.finalize().as_slice()); + + TrieHash(tmp) + } + + pub fn from_data_array>(data: &[B]) -> TrieHash { + if data.len() == 0 { + return TrieHash::from_empty_data(); + } + + let mut tmp = [0u8; 32]; + + let mut hasher = Sha512_256::new(); + + for item in data.iter() { + hasher.update(item); + } + tmp.copy_from_slice(hasher.finalize().as_slice()); + TrieHash(tmp) + } + + /// Convert to a String that can be used in e.g. sqlite + pub fn to_string(&self) -> String { + let s = format!("{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", + self.0[0], self.0[1], self.0[2], self.0[3], + self.0[4], self.0[5], self.0[6], self.0[7], + self.0[8], self.0[9], self.0[10], self.0[11], + self.0[12], self.0[13], self.0[14], self.0[15], + self.0[16], self.0[17], self.0[18], self.0[19], + self.0[20], self.0[21], self.0[22], self.0[23], + self.0[24], self.0[25], self.0[26], self.0[27], + self.0[28], self.0[29], self.0[30], self.0[31]); + s + } +} + #[derive(Serialize, Deserialize)] pub struct BurnchainHeaderHash(pub [u8; 32]); impl_array_newtype!(BurnchainHeaderHash, u8, 32); diff --git a/stackslib/src/burnchains/tests/burnchain.rs b/stackslib/src/burnchains/tests/burnchain.rs index b08d7a097e..1dd8d283ef 100644 --- a/stackslib/src/burnchains/tests/burnchain.rs +++ b/stackslib/src/burnchains/tests/burnchain.rs @@ -44,7 +44,6 @@ use crate::chainstate::burn::{ BlockSnapshot, ConsensusHash, ConsensusHashExtensions, OpsHash, SortitionHash, }; use crate::chainstate::stacks::address::StacksAddressExtensions; -use crate::chainstate::stacks::index::TrieHashExtension; use crate::chainstate::stacks::StacksPublicKey; use crate::util_lib::db::Error as db_error; diff --git a/stackslib/src/chainstate/burn/db/processing.rs b/stackslib/src/chainstate/burn/db/processing.rs index 82318bfe37..4aa0d64929 100644 --- a/stackslib/src/chainstate/burn/db/processing.rs +++ b/stackslib/src/chainstate/burn/db/processing.rs @@ -353,7 +353,6 @@ mod tests { use crate::chainstate::burn::operations::{LeaderBlockCommitOp, LeaderKeyRegisterOp}; use crate::chainstate::burn::*; use crate::chainstate::stacks::address::StacksAddressExtensions; - use crate::chainstate::stacks::index::TrieHashExtension; use crate::chainstate::stacks::StacksPublicKey; use crate::core::MICROSTACKS_PER_STACKS; diff --git a/stackslib/src/chainstate/burn/db/sortdb.rs b/stackslib/src/chainstate/burn/db/sortdb.rs index 53dc2d0547..7e697a7dff 100644 --- a/stackslib/src/chainstate/burn/db/sortdb.rs +++ b/stackslib/src/chainstate/burn/db/sortdb.rs @@ -6594,7 +6594,6 @@ pub mod tests { BlockstackOperationType, LeaderBlockCommitOp, LeaderKeyRegisterOp, }; use crate::chainstate::burn::ConsensusHash; - use crate::chainstate::stacks::index::TrieHashExtension; use crate::chainstate::stacks::StacksPublicKey; use crate::core::{StacksEpochExtension, *}; use crate::util_lib::db::Error as db_error; diff --git a/stackslib/src/chainstate/burn/distribution.rs b/stackslib/src/chainstate/burn/distribution.rs index ed01ae014b..59c335cd58 100644 --- a/stackslib/src/chainstate/burn/distribution.rs +++ b/stackslib/src/chainstate/burn/distribution.rs @@ -450,7 +450,6 @@ mod tests { }; use crate::chainstate::burn::ConsensusHash; use crate::chainstate::stacks::address::StacksAddressExtensions; - use crate::chainstate::stacks::index::TrieHashExtension; use crate::chainstate::stacks::StacksPublicKey; use crate::core::MINING_COMMITMENT_WINDOW; diff --git a/stackslib/src/chainstate/burn/mod.rs b/stackslib/src/chainstate/burn/mod.rs index be92c3088f..4552210f44 100644 --- a/stackslib/src/chainstate/burn/mod.rs +++ b/stackslib/src/chainstate/burn/mod.rs @@ -432,7 +432,6 @@ mod tests { use crate::burnchains::bitcoin::address::BitcoinAddress; use crate::burnchains::bitcoin::keys::BitcoinPublicKey; use crate::chainstate::burn::db::sortdb::*; - use crate::chainstate::stacks::index::TrieHashExtension; use crate::util_lib::db::Error as db_error; #[test] diff --git a/stackslib/src/chainstate/burn/operations/leader_block_commit.rs b/stackslib/src/chainstate/burn/operations/leader_block_commit.rs index 910315f082..31643a33d9 100644 --- a/stackslib/src/chainstate/burn/operations/leader_block_commit.rs +++ b/stackslib/src/chainstate/burn/operations/leader_block_commit.rs @@ -1172,7 +1172,6 @@ mod tests { use crate::chainstate::burn::operations::*; use crate::chainstate::burn::{ConsensusHash, *}; use crate::chainstate::stacks::address::StacksAddressExtensions; - use crate::chainstate::stacks::index::TrieHashExtension; use crate::chainstate::stacks::StacksPublicKey; use crate::core::{ StacksEpoch, StacksEpochExtension, StacksEpochId, PEER_VERSION_EPOCH_1_0, diff --git a/stackslib/src/chainstate/burn/operations/leader_key_register.rs b/stackslib/src/chainstate/burn/operations/leader_key_register.rs index 44402adc0c..5608b6739d 100644 --- a/stackslib/src/chainstate/burn/operations/leader_key_register.rs +++ b/stackslib/src/chainstate/burn/operations/leader_key_register.rs @@ -253,7 +253,6 @@ pub mod tests { }; use crate::chainstate::burn::{BlockSnapshot, ConsensusHash, OpsHash, SortitionHash}; use crate::chainstate::stacks::address::StacksAddressExtensions; - use crate::chainstate::stacks::index::TrieHashExtension; use crate::core::StacksEpochId; pub struct OpFixture { diff --git a/stackslib/src/chainstate/burn/sortition.rs b/stackslib/src/chainstate/burn/sortition.rs index b0221f1439..63a7b7feb3 100644 --- a/stackslib/src/chainstate/burn/sortition.rs +++ b/stackslib/src/chainstate/burn/sortition.rs @@ -40,7 +40,7 @@ use crate::chainstate::burn::{ SortitionHash, }; use crate::chainstate::stacks::db::StacksChainState; -use crate::chainstate::stacks::index::{ClarityMarfTrieId, MarfTrieId, TrieHashExtension}; +use crate::chainstate::stacks::index::{ClarityMarfTrieId, MarfTrieId}; use crate::core::*; use crate::util_lib::db::Error as db_error; diff --git a/stackslib/src/chainstate/stacks/index/bits.rs b/stackslib/src/chainstate/stacks/index/bits.rs index e212b03299..6397cee3a3 100644 --- a/stackslib/src/chainstate/stacks/index/bits.rs +++ b/stackslib/src/chainstate/stacks/index/bits.rs @@ -29,7 +29,7 @@ use stacks_common::util::macros::is_trace; use crate::chainstate::stacks::index::node::{ clear_backptr, ConsensusSerializable, TrieNode, TrieNode16, TrieNode256, TrieNode4, TrieNode48, - TrieNodeID, TrieNodeType, TriePtr, TRIEPATH_MAX_LEN, TRIEPTR_SIZE, + TrieNodeID, TrieNodeType, TriePtr, TRIEPTR_SIZE, }; use crate::chainstate::stacks::index::storage::{TrieFileStorage, TrieStorageConnection}; use crate::chainstate::stacks::index::{BlockMap, Error, MarfTrieId, TrieLeaf}; @@ -55,15 +55,15 @@ pub fn path_from_bytes(r: &mut R) -> Result, Error> { } })?; - if lenbuf[0] as usize > TRIEPATH_MAX_LEN { + if lenbuf[0] as usize > TRIEHASH_ENCODED_SIZE { trace!( "Path length is {} (expected <= {})", lenbuf[0], - TRIEPATH_MAX_LEN + TRIEHASH_ENCODED_SIZE ); return Err(Error::CorruptionError(format!( "Node path is longer than {} bytes (got {})", - TRIEPATH_MAX_LEN, lenbuf[0] + TRIEHASH_ENCODED_SIZE, lenbuf[0] ))); } @@ -326,7 +326,7 @@ pub fn read_nodetype_at_head_nohash( /// node hash id ptrs & ptr data path /// /// X is fixed and determined by the TrieNodeType variant. -/// Y is variable, but no more than TriePath::len(). +/// Y is variable, but no more than TrieHash::len(). /// /// If `read_hash` is false, then the contents of the node hash are undefined. fn inner_read_nodetype_at_head( diff --git a/stackslib/src/chainstate/stacks/index/cache.rs b/stackslib/src/chainstate/stacks/index/cache.rs index 7f92efdd8b..7547fd6d80 100644 --- a/stackslib/src/chainstate/stacks/index/cache.rs +++ b/stackslib/src/chainstate/stacks/index/cache.rs @@ -40,7 +40,7 @@ use crate::chainstate::stacks::index::bits::{ }; use crate::chainstate::stacks::index::node::{ clear_backptr, is_backptr, set_backptr, TrieNode, TrieNode16, TrieNode256, TrieNode4, - TrieNode48, TrieNodeID, TrieNodeType, TriePath, TriePtr, + TrieNode48, TrieNodeID, TrieNodeType, TriePtr, }; use crate::chainstate::stacks::index::{trie_sql, ClarityMarfTrieId, Error, MarfTrieId, TrieLeaf}; use crate::util_lib::db::{ @@ -420,7 +420,7 @@ pub mod test { } } else { for (key, value) in block_data.iter() { - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); let leaf = TrieLeaf::from_value(&vec![], value.clone()); marf.insert_raw(path, leaf).unwrap(); } @@ -443,7 +443,7 @@ pub mod test { for (i, block_data) in data.iter().enumerate() { test_debug!("Read block {}", i); for (key, value) in block_data.iter() { - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); let marf_leaf = TrieLeaf::from_value(&vec![], value.clone()); let read_time = SystemTime::now(); diff --git a/stackslib/src/chainstate/stacks/index/file.rs b/stackslib/src/chainstate/stacks/index/file.rs index 4123b1310a..5a7da69e52 100644 --- a/stackslib/src/chainstate/stacks/index/file.rs +++ b/stackslib/src/chainstate/stacks/index/file.rs @@ -42,7 +42,7 @@ use crate::chainstate::stacks::index::bits::{ }; use crate::chainstate::stacks::index::node::{ clear_backptr, is_backptr, set_backptr, TrieNode, TrieNode16, TrieNode256, TrieNode4, - TrieNode48, TrieNodeID, TrieNodeType, TriePath, TriePtr, + TrieNode48, TrieNodeID, TrieNodeType, TriePtr, }; use crate::chainstate::stacks::index::storage::{NodeHashReader, TrieStorageConnection}; use crate::chainstate::stacks::index::{trie_sql, ClarityMarfTrieId, Error, MarfTrieId, TrieLeaf}; diff --git a/stackslib/src/chainstate/stacks/index/marf.rs b/stackslib/src/chainstate/stacks/index/marf.rs index 81506a08c3..427cde29fc 100644 --- a/stackslib/src/chainstate/stacks/index/marf.rs +++ b/stackslib/src/chainstate/stacks/index/marf.rs @@ -28,14 +28,14 @@ use stacks_common::util::log; use crate::chainstate::stacks::index::bits::{get_leaf_hash, get_node_hash, read_root_hash}; use crate::chainstate::stacks::index::node::{ clear_backptr, is_backptr, set_backptr, CursorError, TrieCursor, TrieNode, TrieNode16, - TrieNode256, TrieNode4, TrieNode48, TrieNodeID, TrieNodeType, TriePath, TriePtr, TRIEPTR_SIZE, + TrieNode256, TrieNode4, TrieNode48, TrieNodeID, TrieNodeType, TriePtr, TRIEPTR_SIZE, }; use crate::chainstate::stacks::index::storage::{ TrieFileStorage, TrieHashCalculationMode, TrieStorageConnection, TrieStorageTransaction, }; use crate::chainstate::stacks::index::trie::Trie; use crate::chainstate::stacks::index::{ - ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieHashExtension, TrieLeaf, TrieMerkleProof, + ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieLeaf, TrieMerkleProof, }; use crate::util_lib::db::Error as db_error; @@ -126,6 +126,11 @@ pub trait MarfConnection { fn get(&mut self, block_hash: &T, key: &str) -> Result, Error> { self.with_conn(|c| MARF::get_by_key(c, block_hash, key)) } + + /// Resolve a TrieHash from the MARF to a MARFValue with respect to the given block height. + fn get_from_hash(&mut self, block_hash: &T, th: &TrieHash) -> Result, Error> { + self.with_conn(|c| MARF::get_by_hash(c, block_hash, th)) + } fn get_with_proof( &mut self, @@ -148,12 +153,11 @@ pub trait MarfConnection { hash: &TrieHash, ) -> Result)>, Error> { self.with_conn(|conn| { - let path = TriePath::from_bytes(hash.as_bytes()).ok_or(Error::BadSeekValue)?; - let marf_value = match MARF::get_by_path(conn, block_hash, &path)? { + let marf_value = match MARF::get_by_path(conn, block_hash, hash)? { None => return Ok(None), Some(x) => x, }; - let proof = TrieMerkleProof::from_path(conn, &path, &marf_value, block_hash)?; + let proof = TrieMerkleProof::from_path(conn, hash, &marf_value, block_hash)?; Ok(Some((marf_value, proof))) }) } @@ -797,7 +801,7 @@ impl MARF { fn walk_cow( storage: &mut TrieStorageTransaction, block_hash: &T, - path: &TriePath, + path: &TrieHash, ) -> Result, Error> { let block_id = storage.get_block_identifier(block_hash); MARF::extend_trie(storage, block_hash)?; @@ -902,7 +906,7 @@ impl MARF { fn walk( storage: &mut TrieStorageConnection, block_hash: &T, - path: &TriePath, + path: &TrieHash, ) -> Result<(TrieCursor, TrieNodeType), Error> { storage.open_block(block_hash)?; @@ -1010,7 +1014,7 @@ impl MARF { pub fn get_path( storage: &mut TrieStorageConnection, block_hash: &T, - path: &TriePath, + path: &TrieHash, ) -> Result, Error> { trace!("MARF::get_path({:?}) {:?}", block_hash, path); @@ -1061,7 +1065,7 @@ impl MARF { fn do_insert_leaf( storage: &mut TrieStorageTransaction, block_hash: &T, - path: &TriePath, + path: &TrieHash, leaf_value: &TrieLeaf, update_skiplist: bool, ) -> Result<(), Error> { @@ -1092,7 +1096,7 @@ impl MARF { pub fn insert_leaf( storage: &mut TrieStorageTransaction, block_hash: &T, - path: &TriePath, + path: &TrieHash, value: &TrieLeaf, ) -> Result<(), Error> { if storage.readonly() { @@ -1105,7 +1109,7 @@ impl MARF { pub fn insert_leaf_in_batch( storage: &mut TrieStorageTransaction, block_hash: &T, - path: &TriePath, + path: &TrieHash, value: &TrieLeaf, ) -> Result<(), Error> { if storage.readonly() { @@ -1142,7 +1146,7 @@ impl MARF { pub fn get_by_path( storage: &mut TrieStorageConnection, block_hash: &T, - path: &TriePath, + path: &TrieHash, ) -> Result, Error> { let (cur_block_hash, cur_block_id) = storage.get_cur_block_and_id(); @@ -1166,6 +1170,8 @@ impl MARF { result.map(|option_result| option_result.map(|leaf| leaf.data)) } + /// Load up a MARF value by key, given a handle to the storage connection and a tip to work off + /// of. pub fn get_by_key( storage: &mut TrieStorageConnection, block_hash: &T, @@ -1173,7 +1179,7 @@ impl MARF { ) -> Result, Error> { let (cur_block_hash, cur_block_id) = storage.get_cur_block_and_id(); - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); let result = MARF::get_path(storage, block_hash, &path).or_else(|e| match e { Error::NotFoundError => Ok(None), @@ -1195,6 +1201,35 @@ impl MARF { result.map(|option_result| option_result.map(|leaf| leaf.data)) } + /// Load up a MARF value by TrieHash, given a handle to the storage connection and a tip to + /// work off of. + pub fn get_by_hash( + storage: &mut TrieStorageConnection, + block_hash: &T, + path: &TrieHash, + ) -> Result, Error> { + let (cur_block_hash, cur_block_id) = storage.get_cur_block_and_id(); + + let result = MARF::get_path(storage, block_hash, &path).or_else(|e| match e { + Error::NotFoundError => Ok(None), + _ => Err(e), + }); + + // restore + storage + .open_block_maybe_id(&cur_block_hash, cur_block_id) + .map_err(|e| { + warn!( + "Failed to re-open {} {:?}: {:?}", + &cur_block_hash, cur_block_id, &e + ); + warn!("Result of failed hash lookup '{}': {:?}", path, &result); + e + })?; + + result.map(|option_result| option_result.map(|leaf| leaf.data)) + } + pub fn get_block_height_miner_tip( storage: &mut TrieStorageConnection, block_hash: &T, @@ -1305,7 +1340,7 @@ impl MARF { .zip(values[0..last].iter()) .try_for_each(|((index, key), value)| { let marf_leaf = TrieLeaf::from_value(&[], value.clone()); - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); if eta_enabled { let updated_progress = 100 * index / last; @@ -1323,7 +1358,7 @@ impl MARF { if result.is_ok() { // last insert updates the root with the skiplist hash let marf_leaf = TrieLeaf::from_value(&[], values[last].clone()); - let path = TriePath::from_key(&keys[last]); + let path = TrieHash::from_key(&keys[last]); result = MARF::insert_leaf(conn, block_hash, &path, &marf_leaf); } @@ -1362,14 +1397,13 @@ impl MARF { let proof = TrieMerkleProof::from_raw_entry(&mut conn, key, &marf_value, block_hash)?; Ok(Some((marf_value, proof))) } - + pub fn get_with_proof_from_hash( &mut self, block_hash: &T, - hash: &TrieHash, + path: &TrieHash, ) -> Result)>, Error> { let mut conn = self.storage.connection(); - let path = TriePath::from_bytes(hash.as_bytes()).ok_or(Error::BadSeekValue)?; let marf_value = match MARF::get_by_path(&mut conn, block_hash, &path)? { None => return Ok(None), Some(x) => x, @@ -1414,14 +1448,14 @@ impl MARF { return Err(Error::ReadOnlyError); } let marf_leaf = TrieLeaf::from_value(&[], value); - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); self.insert_raw(path, marf_leaf) } /// Insert the given (key, value) pair into the MARF. Inserting the same key twice silently /// overwrites the existing key. Succeeds if there are no storage errors. /// Must be called after a call to .begin() (will fail otherwise) - pub fn insert_raw(&mut self, path: TriePath, marf_leaf: TrieLeaf) -> Result<(), Error> { + pub fn insert_raw(&mut self, path: TrieHash, marf_leaf: TrieLeaf) -> Result<(), Error> { if self.storage.readonly() { return Err(Error::ReadOnlyError); } diff --git a/stackslib/src/chainstate/stacks/index/mod.rs b/stackslib/src/chainstate/stacks/index/mod.rs index eb082747c5..9fee7ab2d6 100644 --- a/stackslib/src/chainstate/stacks/index/mod.rs +++ b/stackslib/src/chainstate/stacks/index/mod.rs @@ -151,71 +151,6 @@ impl MarfTrieId for BurnchainHeaderHash {} #[cfg(test)] impl MarfTrieId for BlockHeaderHash {} -pub trait TrieHashExtension { - fn from_empty_data() -> TrieHash; - fn from_data(data: &[u8]) -> TrieHash; - fn from_data_array>(data: &[B]) -> TrieHash; - fn to_string(&self) -> String; -} - -impl TrieHashExtension for TrieHash { - /// TrieHash of zero bytes - fn from_empty_data() -> TrieHash { - // sha2-512/256 hash of empty string. - // this is used so frequently it helps performance if we just have a constant for it. - TrieHash([ - 0xc6, 0x72, 0xb8, 0xd1, 0xef, 0x56, 0xed, 0x28, 0xab, 0x87, 0xc3, 0x62, 0x2c, 0x51, - 0x14, 0x06, 0x9b, 0xdd, 0x3a, 0xd7, 0xb8, 0xf9, 0x73, 0x74, 0x98, 0xd0, 0xc0, 0x1e, - 0xce, 0xf0, 0x96, 0x7a, - ]) - } - - /// TrieHash from bytes - fn from_data(data: &[u8]) -> TrieHash { - if data.len() == 0 { - return TrieHash::from_empty_data(); - } - - let mut tmp = [0u8; 32]; - - let mut hasher = TrieHasher::new(); - hasher.update(data); - tmp.copy_from_slice(hasher.finalize().as_slice()); - - TrieHash(tmp) - } - - fn from_data_array>(data: &[B]) -> TrieHash { - if data.len() == 0 { - return TrieHash::from_empty_data(); - } - - let mut tmp = [0u8; 32]; - - let mut hasher = TrieHasher::new(); - - for item in data.iter() { - hasher.update(item); - } - tmp.copy_from_slice(hasher.finalize().as_slice()); - TrieHash(tmp) - } - - /// Convert to a String that can be used in e.g. sqlite - fn to_string(&self) -> String { - let s = format!("{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", - self.0[0], self.0[1], self.0[2], self.0[3], - self.0[4], self.0[5], self.0[6], self.0[7], - self.0[8], self.0[9], self.0[10], self.0[11], - self.0[12], self.0[13], self.0[14], self.0[15], - self.0[16], self.0[17], self.0[18], self.0[19], - self.0[20], self.0[21], self.0[22], self.0[23], - self.0[24], self.0[25], self.0[26], self.0[27], - self.0[28], self.0[29], self.0[30], self.0[31]); - s - } -} - /// Structure that holds the actual data in a MARF leaf node. /// It only stores the hash of some value string, but we add 8 extra bytes for future extensions. /// If not used (the rule today), then they should all be 0. diff --git a/stackslib/src/chainstate/stacks/index/node.rs b/stackslib/src/chainstate/stacks/index/node.rs index 19e8aa327f..4436d7f239 100644 --- a/stackslib/src/chainstate/stacks/index/node.rs +++ b/stackslib/src/chainstate/stacks/index/node.rs @@ -32,7 +32,7 @@ use crate::chainstate::stacks::index::bits::{ get_path_byte_len, get_ptrs_byte_len, path_from_bytes, ptrs_from_bytes, write_path_to_bytes, }; use crate::chainstate::stacks::index::{ - BlockMap, ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieHashExtension, TrieHasher, + BlockMap, ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieHasher, TrieLeaf, MARF_VALUE_ENCODED_SIZE, }; @@ -106,23 +106,6 @@ fn ptrs_consensus_hash( Ok(()) } -/// A path in the Trie is the SHA2-512/256 hash of its key. -pub struct TriePath([u8; 32]); -impl_array_newtype!(TriePath, u8, 32); -impl_array_hexstring_fmt!(TriePath); -impl_byte_array_newtype!(TriePath, u8, 32); - -pub const TRIEPATH_MAX_LEN: usize = 32; - -impl TriePath { - pub fn from_key(k: &str) -> TriePath { - let h = TrieHash::from_data(k.as_bytes()); - let mut hb = [0u8; TRIEPATH_MAX_LEN]; - hb.copy_from_slice(h.as_bytes()); - TriePath(hb) - } -} - /// All Trie nodes implement the following methods: pub trait TrieNode { /// Node ID for encoding/decoding @@ -339,7 +322,7 @@ impl TriePtr { /// nodes to visit when updating the root node hash. #[derive(Debug, Clone, PartialEq)] pub struct TrieCursor { - pub path: TriePath, // the path to walk + pub path: TrieHash, // the path to walk pub index: usize, // index into the path pub node_path_index: usize, // index into the currently-visited node's compressed path pub nodes: Vec, // list of nodes this cursor visits @@ -349,7 +332,7 @@ pub struct TrieCursor { } impl TrieCursor { - pub fn new(path: &TriePath, root_ptr: TriePtr) -> TrieCursor { + pub fn new(path: &TrieHash, root_ptr: TriePtr) -> TrieCursor { TrieCursor { path: path.clone(), index: 0, diff --git a/stackslib/src/chainstate/stacks/index/proofs.rs b/stackslib/src/chainstate/stacks/index/proofs.rs index 815def9c91..aae802334c 100644 --- a/stackslib/src/chainstate/stacks/index/proofs.rs +++ b/stackslib/src/chainstate/stacks/index/proofs.rs @@ -35,14 +35,14 @@ use crate::chainstate::stacks::index::bits::{ use crate::chainstate::stacks::index::marf::MARF; use crate::chainstate::stacks::index::node::{ clear_backptr, is_backptr, set_backptr, ConsensusSerializable, CursorError, TrieCursor, - TrieNode, TrieNode16, TrieNode256, TrieNode4, TrieNode48, TrieNodeID, TrieNodeType, TriePath, + TrieNode, TrieNode16, TrieNode256, TrieNode4, TrieNode48, TrieNodeID, TrieNodeType, TriePtr, }; use crate::chainstate::stacks::index::storage::{TrieFileStorage, TrieStorageConnection}; use crate::chainstate::stacks::index::trie::Trie; use crate::chainstate::stacks::index::{ BlockMap, ClarityMarfTrieId, Error, MARFValue, MarfTrieId, ProofTrieNode, ProofTriePtr, - TrieHashExtension, TrieLeaf, TrieMerkleProof, TrieMerkleProofType, + TrieLeaf, TrieMerkleProof, TrieMerkleProofType, }; impl ConsensusSerializable<()> for ProofTrieNode { @@ -1004,7 +1004,7 @@ impl TrieMerkleProof { /// * segment proof i+1 must be a prefix of segment proof i /// * segment proof 0 must end in a leaf /// * all segment proofs must end in a Node256 (a root) - fn is_proof_well_formed(proof: &Vec>, expected_path: &TriePath) -> bool { + fn is_proof_well_formed(proof: &Vec>, expected_path: &TrieHash) -> bool { if proof.len() == 0 { trace!("Proof is empty"); return false; @@ -1048,7 +1048,7 @@ impl TrieMerkleProof { } }; - // first path bytes must be the expected TriePath + // first path bytes must be the expected TrieHash if expected_path.as_bytes().to_vec() != path_bytes { trace!( "Invalid proof -- path bytes {:?} differs from the expected path {:?}", @@ -1121,7 +1121,7 @@ impl TrieMerkleProof { /// NOTE: Trie root hashes are globally unique by design, even if they represent the same contents, so the root_to_block map is bijective with high probability. pub fn verify_proof( proof: &Vec>, - path: &TriePath, + path: &TrieHash, value: &MARFValue, root_hash: &TrieHash, root_to_block: &HashMap, @@ -1351,7 +1351,7 @@ impl TrieMerkleProof { /// Verify this proof pub fn verify( &self, - path: &TriePath, + path: &TrieHash, marf_value: &MARFValue, root_hash: &TrieHash, root_to_block: &HashMap, @@ -1362,7 +1362,7 @@ impl TrieMerkleProof { /// Walk down the trie pointed to by s until we reach a backptr or a leaf fn walk_to_leaf_or_backptr( storage: &mut TrieStorageConnection, - path: &TriePath, + path: &TrieHash, ) -> Result<(TrieCursor, TrieNodeType, TriePtr), Error> { trace!( "Walk path {:?} from {:?} to the first backptr", @@ -1438,7 +1438,7 @@ impl TrieMerkleProof { /// If the path doesn't resolve, return an error (NotFoundError) pub fn from_path( storage: &mut TrieStorageConnection, - path: &TriePath, + path: &TrieHash, expected_value: &MARFValue, root_block_header: &T, ) -> Result, Error> { @@ -1562,7 +1562,7 @@ impl TrieMerkleProof { root_block_header: &T, ) -> Result, Error> { let marf_value = MARFValue::from_value(value); - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); TrieMerkleProof::from_path(storage, &path, &marf_value, root_block_header) } @@ -1572,7 +1572,7 @@ impl TrieMerkleProof { value: &MARFValue, root_block_header: &T, ) -> Result, Error> { - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); TrieMerkleProof::from_path(storage, &path, value, root_block_header) } } diff --git a/stackslib/src/chainstate/stacks/index/storage.rs b/stackslib/src/chainstate/stacks/index/storage.rs index 6994c7ad05..170430c74c 100644 --- a/stackslib/src/chainstate/stacks/index/storage.rs +++ b/stackslib/src/chainstate/stacks/index/storage.rs @@ -46,12 +46,12 @@ use crate::chainstate::stacks::index::file::{TrieFile, TrieFileNodeHashReader}; use crate::chainstate::stacks::index::marf::MARFOpenOpts; use crate::chainstate::stacks::index::node::{ clear_backptr, is_backptr, set_backptr, TrieNode, TrieNode16, TrieNode256, TrieNode4, - TrieNode48, TrieNodeID, TrieNodeType, TriePath, TriePtr, + TrieNode48, TrieNodeID, TrieNodeType, TriePtr, }; use crate::chainstate::stacks::index::profile::TrieBenchmark; use crate::chainstate::stacks::index::trie::Trie; use crate::chainstate::stacks::index::{ - trie_sql, BlockMap, ClarityMarfTrieId, Error, MarfTrieId, TrieHashExtension, TrieHasher, + trie_sql, BlockMap, ClarityMarfTrieId, Error, MarfTrieId, TrieHasher, TrieLeaf, }; use crate::util_lib::db::{ diff --git a/stackslib/src/chainstate/stacks/index/test/cache.rs b/stackslib/src/chainstate/stacks/index/test/cache.rs index 5a0bc41d00..1abd0e741a 100644 --- a/stackslib/src/chainstate/stacks/index/test/cache.rs +++ b/stackslib/src/chainstate/stacks/index/test/cache.rs @@ -105,7 +105,7 @@ fn test_marf_with_cache( } } else { for (key, value) in block_data.iter() { - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); let leaf = TrieLeaf::from_value(&vec![], value.clone()); marf.insert_raw(path, leaf).unwrap(); } @@ -128,7 +128,7 @@ fn test_marf_with_cache( for (i, block_data) in data.iter().enumerate() { test_debug!("Read block {}", i); for (key, value) in block_data.iter() { - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); let marf_leaf = TrieLeaf::from_value(&vec![], value.clone()); let read_time = SystemTime::now(); diff --git a/stackslib/src/chainstate/stacks/index/test/file.rs b/stackslib/src/chainstate/stacks/index/test/file.rs index 499198aca5..19ac5e60e4 100644 --- a/stackslib/src/chainstate/stacks/index/test/file.rs +++ b/stackslib/src/chainstate/stacks/index/test/file.rs @@ -106,7 +106,7 @@ fn test_migrate_existing_trie_blobs() { marf.begin(&last_block_header, &block_header).unwrap(); for (key, value) in block_data.iter() { - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); let leaf = TrieLeaf::from_value(&vec![], value.clone()); marf.insert_raw(path, leaf).unwrap(); } @@ -147,7 +147,7 @@ fn test_migrate_existing_trie_blobs() { // verify that we can read everything from the blobs for (i, block_data) in data.iter().enumerate() { for (key, value) in block_data.iter() { - let path = TriePath::from_key(key); + let path = TrieHash::from_key(key); let marf_leaf = TrieLeaf::from_value(&vec![], value.clone()); let leaf = MARF::get_path( diff --git a/stackslib/src/chainstate/stacks/index/test/marf.rs b/stackslib/src/chainstate/stacks/index/test/marf.rs index b66fc4dd8a..f4f4dd0de0 100644 --- a/stackslib/src/chainstate/stacks/index/test/marf.rs +++ b/stackslib/src/chainstate/stacks/index/test/marf.rs @@ -33,7 +33,7 @@ use crate::chainstate::stacks::index::storage::*; use crate::chainstate::stacks::index::test::*; use crate::chainstate::stacks::index::trie::*; use crate::chainstate::stacks::index::{ - ClarityMarfTrieId, Error, MARFValue, TrieHashExtension, TrieLeaf, + ClarityMarfTrieId, Error, MARFValue, TrieLeaf, }; #[test] @@ -52,7 +52,7 @@ fn marf_insert_different_leaf_same_block_100() { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); for i in 0..100 { let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); @@ -117,7 +117,7 @@ fn marf_insert_different_leaf_different_path_different_block_100() { marf.commit().unwrap(); marf.begin(&BlockHeaderHash::sentinel(), &block_header) .unwrap(); - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); marf.insert_raw(path, value).unwrap(); } @@ -140,7 +140,7 @@ fn marf_insert_different_leaf_different_path_different_block_100() { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, i as u8, ]; - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); let leaf = MARF::get_path(&mut marf.borrow_storage_backend(), &block_header, &path) @@ -189,7 +189,7 @@ fn marf_insert_same_leaf_different_block_100() { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); for i in 0..100 { let next_block_header = BlockHeaderHash::from_bytes(&[i + 1 as u8; 32]).unwrap(); @@ -197,7 +197,7 @@ fn marf_insert_same_leaf_different_block_100() { marf.commit().unwrap(); marf.begin(&BlockHeaderHash::sentinel(), &next_block_header) .unwrap(); - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); marf.insert_raw(path, value).unwrap(); } @@ -271,7 +271,7 @@ fn marf_insert_leaf_sequence_2() { i as u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let prior_block_header = BlockHeaderHash::from_bytes(&[i as u8; 32]).unwrap(); let next_block_header = BlockHeaderHash::from_bytes(&[i + 1 as u8; 32]).unwrap(); marf.commit().unwrap(); @@ -294,7 +294,7 @@ fn marf_insert_leaf_sequence_2() { i as u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); let leaf = MARF::get_path( @@ -348,7 +348,7 @@ fn marf_insert_leaf_sequence_100() { i as u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); marf.commit().unwrap(); let next_block_header = BlockHeaderHash::from_bytes(&[i as u8; 32]).unwrap(); @@ -372,7 +372,7 @@ fn marf_insert_leaf_sequence_100() { i as u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); eprintln!("Finding value inserted at {}", &next_block_header); @@ -567,7 +567,7 @@ where let next_path = path_gen(i, path.clone()); - let triepath = TriePath::from_bytes(&next_path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&next_path[..]).unwrap(); let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); debug!("----------------"); @@ -582,7 +582,7 @@ where let read_value = MARF::get_path( &mut marf.borrow_storage_backend(), &next_block_header, - &TriePath::from_bytes(&next_path[..]).unwrap(), + &TrieHash::from_bytes(&next_path[..]).unwrap(), ) .unwrap() .unwrap(); @@ -603,7 +603,7 @@ where let read_value = MARF::get_path( &mut marf.borrow_storage_backend(), &next_block_header, - &TriePath::from_bytes(&prev_path[..]).unwrap(), + &TrieHash::from_bytes(&prev_path[..]).unwrap(), ) .unwrap() .unwrap(); @@ -675,7 +675,7 @@ where // add a leaf at the end of the path let next_path = path_gen(i, path.clone()); - let triepath = TriePath::from_bytes(&next_path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&next_path[..]).unwrap(); let value = MARFValue([i as u8; 40]); assert_eq!( @@ -847,7 +847,7 @@ fn marf_merkle_verify_backptrs() { marf.commit().unwrap(); marf.begin(&block_header_1, &block_header_2).unwrap(); marf.insert_raw( - TriePath::from_bytes(&path_2[..]).unwrap(), + TrieHash::from_bytes(&path_2[..]).unwrap(), TrieLeaf::new(&vec![], &[20 as u8; 40].to_vec()), ) .unwrap(); @@ -865,7 +865,7 @@ fn marf_merkle_verify_backptrs() { marf.commit().unwrap(); marf.begin(&block_header_2, &block_header_3).unwrap(); marf.insert_raw( - TriePath::from_bytes(&path_3[..]).unwrap(), + TrieHash::from_bytes(&path_3[..]).unwrap(), TrieLeaf::new(&vec![], &[21 as u8; 40].to_vec()), ) .unwrap(); @@ -922,7 +922,7 @@ where let (path, next_block_header) = path_gen(i); - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let value = TrieLeaf::new( &vec![], &[ @@ -944,7 +944,7 @@ where let read_value = MARF::get_path( &mut marf.borrow_storage_backend(), &block_header, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ) .unwrap() .unwrap(); @@ -998,7 +998,7 @@ where let i1 = i % 256; let (path, _next_block_header) = path_gen(i); - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let value = TrieLeaf::new( &vec![], &[ @@ -1011,7 +1011,7 @@ where let read_value = MARF::get_path( &mut marf.borrow_storage_backend(), &block_header, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ) .unwrap() .unwrap(); @@ -1139,7 +1139,7 @@ fn marf_split_leaf_path() { .unwrap(); let path = [0u8; 32]; - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let value = TrieLeaf::new(&vec![], &[0u8; 40].to_vec()); debug!("----------------"); @@ -1161,7 +1161,7 @@ fn marf_split_leaf_path() { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ]; - let triepath_2 = TriePath::from_bytes(&path_2[..]).unwrap(); + let triepath_2 = TrieHash::from_bytes(&path_2[..]).unwrap(); let value_2 = TrieLeaf::new(&vec![], &[1u8; 40].to_vec()); debug!("----------------"); @@ -1602,7 +1602,7 @@ fn marf_read_random_1048576_4096_file_storage() { let path = TrieHash::from_data(&seed[..]).as_bytes()[0..32].to_vec(); seed = path.clone(); - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let value = TrieLeaf::new( &vec![], &[ @@ -1615,7 +1615,7 @@ fn marf_read_random_1048576_4096_file_storage() { let read_value = MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ) .unwrap() .unwrap(); @@ -1896,7 +1896,7 @@ fn marf_insert_flush_to_different_block() { None }; - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let value = TrieLeaf::new( &vec![], &[ @@ -1919,7 +1919,7 @@ fn marf_insert_flush_to_different_block() { let read_value = MARF::get_path( &mut marf.borrow_storage_backend(), &target_block, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ) .unwrap() .unwrap(); @@ -2017,7 +2017,7 @@ fn marf_insert_flush_to_different_block() { 24, 25, 26, 27, 28, 29, i0 as u8, i1 as u8, ]; - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let value = TrieLeaf::new( &vec![], &[ @@ -2037,7 +2037,7 @@ fn marf_insert_flush_to_different_block() { let read_value = MARF::get_path( &mut marf.borrow_storage_backend(), &read_from_block, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ) .unwrap() .unwrap(); @@ -2074,7 +2074,7 @@ fn test_marf_read_only() { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let leaf = TrieLeaf::new( &vec![], &[ @@ -2138,13 +2138,13 @@ fn test_marf_begin_from_sentinel_twice() { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let triepath_1 = TriePath::from_bytes(&path_1[..]).unwrap(); + let triepath_1 = TrieHash::from_bytes(&path_1[..]).unwrap(); let path_2 = [ 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let triepath_2 = TriePath::from_bytes(&path_2[..]).unwrap(); + let triepath_2 = TrieHash::from_bytes(&path_2[..]).unwrap(); let value_1 = TrieLeaf::new(&vec![], &vec![1u8; 40]); let value_2 = TrieLeaf::new(&vec![], &vec![2u8; 40]); @@ -2210,14 +2210,14 @@ fn test_marf_unconfirmed() { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let triepath_1 = TriePath::from_bytes(&path_1[..]).unwrap(); + let triepath_1 = TrieHash::from_bytes(&path_1[..]).unwrap(); let value_1 = TrieLeaf::new(&vec![], &vec![1u8; 40]); let path_2 = [ 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]; - let triepath_2 = TriePath::from_bytes(&path_2[..]).unwrap(); + let triepath_2 = TrieHash::from_bytes(&path_2[..]).unwrap(); let value_2 = TrieLeaf::new(&vec![], &vec![2u8; 40]); let block_header = StacksBlockId([0x33u8; 32]); diff --git a/stackslib/src/chainstate/stacks/index/test/mod.rs b/stackslib/src/chainstate/stacks/index/test/mod.rs index 2c3b04698c..9dbaa0959d 100644 --- a/stackslib/src/chainstate/stacks/index/test/mod.rs +++ b/stackslib/src/chainstate/stacks/index/test/mod.rs @@ -32,7 +32,7 @@ use crate::chainstate::stacks::index::proofs::*; use crate::chainstate::stacks::index::storage::*; use crate::chainstate::stacks::index::trie::*; use crate::chainstate::stacks::index::{ - MARFValue, MarfTrieId, TrieHashExtension, TrieLeaf, TrieMerkleProof, + MARFValue, MarfTrieId, TrieLeaf, TrieMerkleProof, }; use crate::chainstate::stacks::{BlockHeaderHash, TrieHash}; @@ -108,7 +108,7 @@ pub fn merkle_test( value: &Vec, ) -> () { let (_, root_hash) = Trie::read_root(s).unwrap(); - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let block_header = BlockHeaderHash([0u8; 32]); s.open_block(&block_header).unwrap(); @@ -147,7 +147,7 @@ pub fn merkle_test_marf( s.open_block(header).unwrap(); let (_, root_hash) = Trie::read_root(s).unwrap(); - let triepath = TriePath::from_bytes(&path[..]).unwrap(); + let triepath = TrieHash::from_bytes(&path[..]).unwrap(); let mut marf_value = [0u8; 40]; marf_value.copy_from_slice(&value[0..40]); @@ -199,7 +199,7 @@ pub fn merkle_test_marf_key_value( test_debug!("---------"); let root_to_block = root_to_block.unwrap_or_else(|| s.read_root_to_block_table().unwrap()); - let triepath = TriePath::from_key(key); + let triepath = TrieHash::from_key(key); let marf_value = MARFValue::from_value(value); assert!(proof.verify(&triepath, &marf_value, &root_hash, &root_to_block)); diff --git a/stackslib/src/chainstate/stacks/index/test/node.rs b/stackslib/src/chainstate/stacks/index/test/node.rs index a98491595d..227adda439 100644 --- a/stackslib/src/chainstate/stacks/index/test/node.rs +++ b/stackslib/src/chainstate/stacks/index/test/node.rs @@ -4215,7 +4215,7 @@ fn trie_cursor_walk_full() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4313,7 +4313,7 @@ fn trie_cursor_walk_1() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4406,7 +4406,7 @@ fn trie_cursor_walk_2() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4496,7 +4496,7 @@ fn trie_cursor_walk_3() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4585,7 +4585,7 @@ fn trie_cursor_walk_4() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4673,7 +4673,7 @@ fn trie_cursor_walk_5() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4760,7 +4760,7 @@ fn trie_cursor_walk_6() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4845,7 +4845,7 @@ fn trie_cursor_walk_10() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -4937,7 +4937,7 @@ fn trie_cursor_walk_20() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let mut walk_point = nodes[0].clone(); @@ -5028,7 +5028,7 @@ fn trie_cursor_walk_32() { // walk down the trie let mut c = TrieCursor::new( - &TriePath::from_bytes(&path).unwrap(), + &TrieHash::from_bytes(&path).unwrap(), trie_io.root_trieptr(), ); let walk_point = nodes[0].clone(); diff --git a/stackslib/src/chainstate/stacks/index/test/proofs.rs b/stackslib/src/chainstate/stacks/index/test/proofs.rs index 9642bfcdc5..9bd24af548 100644 --- a/stackslib/src/chainstate/stacks/index/test/proofs.rs +++ b/stackslib/src/chainstate/stacks/index/test/proofs.rs @@ -59,7 +59,7 @@ fn verifier_catches_stale_proof() { let new_value = m.get(&block_2, &k1).unwrap().unwrap(); test_debug!("NEW: {:?}", new_value); - let path = TriePath::from_key(&k1); + let path = TrieHash::from_key(&k1); merkle_test_marf_key_value(&mut m.borrow_storage_backend(), &block_2, &k1, &new_v, None); @@ -75,7 +75,7 @@ fn verifier_catches_stale_proof() { .unwrap(); // the verifier should not allow a proof from k1 to old_v from block_2 - let triepath_2 = TriePath::from_key(&k1); + let triepath_2 = TrieHash::from_key(&k1); let marf_value_2 = MARFValue::from_value(&old_v); assert!(!proof_2.verify(&triepath_2, &marf_value_2, &root_hash_2, &root_to_block)); @@ -86,7 +86,7 @@ fn verifier_catches_stale_proof() { .unwrap(); // the verifier should allow a proof from k1 to old_v from block_1 - let triepath_1 = TriePath::from_key(&k1); + let triepath_1 = TrieHash::from_key(&k1); let marf_value_1 = MARFValue::from_value(&old_v); assert!(proof_1.verify(&triepath_1, &marf_value_1, &root_hash_1, &root_to_block)); } @@ -169,7 +169,7 @@ fn ncc_verifier_catches_stale_proof() { TrieMerkleProof::from_entry(&mut m.borrow_storage_backend(), &k1, &another_v, &block_5) .unwrap(); - let triepath_4 = TriePath::from_key(&k1); + let triepath_4 = TrieHash::from_key(&k1); let marf_value_4 = MARFValue::from_value(&another_v); let root_to_block = { m.borrow_storage_backend() @@ -186,7 +186,7 @@ fn ncc_verifier_catches_stale_proof() { TrieMerkleProof::from_entry(&mut m.borrow_storage_backend(), &k1, &old_v, &block_2) .unwrap(); - let triepath_4 = TriePath::from_key(&k1); + let triepath_4 = TrieHash::from_key(&k1); let marf_value_4 = MARFValue::from_value(&old_v); let root_to_block = { m.borrow_storage_backend() diff --git a/stackslib/src/chainstate/stacks/index/test/storage.rs b/stackslib/src/chainstate/stacks/index/test/storage.rs index a996bc7186..fdd3e30191 100644 --- a/stackslib/src/chainstate/stacks/index/test/storage.rs +++ b/stackslib/src/chainstate/stacks/index/test/storage.rs @@ -164,7 +164,7 @@ fn load_store_trie_m_n_same(m: u64, n: u64, same: bool) { ]; path_bytes[24..32].copy_from_slice(&i.to_be_bytes()); - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let value = TrieLeaf::new(&vec![], &[i as u8; 40].to_vec()); confirmed_marf.insert_raw(path.clone(), value).unwrap(); } @@ -213,7 +213,7 @@ fn load_store_trie_m_n_same(m: u64, n: u64, same: bool) { ]; path_bytes[24..32].copy_from_slice(&i.to_be_bytes()); - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); // NOTE: may have been overwritten; just check for presence assert!( @@ -235,7 +235,7 @@ fn load_store_trie_m_n_same(m: u64, n: u64, same: bool) { path_bytes[16..24].copy_from_slice(&j.to_be_bytes()); } - let path = TriePath::from_bytes(&path_bytes).unwrap(); + let path = TrieHash::from_bytes(&path_bytes).unwrap(); let value = TrieLeaf::new(&vec![], &[(i + 128) as u8; 40].to_vec()); new_inserted.push((path.clone(), value.clone())); diff --git a/stackslib/src/chainstate/stacks/index/test/trie.rs b/stackslib/src/chainstate/stacks/index/test/trie.rs index ca2c0ced65..9bac45508c 100644 --- a/stackslib/src/chainstate/stacks/index/test/trie.rs +++ b/stackslib/src/chainstate/stacks/index/test/trie.rs @@ -137,7 +137,7 @@ fn trie_cursor_try_attach_leaf() { path[i] = 32; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); // end of path -- cursor points to the insertion point. @@ -164,7 +164,7 @@ fn trie_cursor_try_attach_leaf() { let leaf_opt_res = MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ); assert!(leaf_opt_res.is_ok()); @@ -194,7 +194,7 @@ fn trie_cursor_try_attach_leaf() { let leaf_opt_res = MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ); assert!(leaf_opt_res.is_ok()); @@ -250,7 +250,7 @@ fn trie_cursor_promote_leaf_to_node4() { // add a single leaf let mut c = TrieCursor::new( - &TriePath::from_bytes(&[ + &TrieHash::from_bytes(&[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ]) @@ -275,7 +275,7 @@ fn trie_cursor_promote_leaf_to_node4() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&[ + &TrieHash::from_bytes(&[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ]) @@ -317,7 +317,7 @@ fn trie_cursor_promote_leaf_to_node4() { path[i] = 32; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, node, node_hash) = walk_to_insertion_point(&mut f, &mut c); // end of path -- cursor points to the insertion point @@ -342,7 +342,7 @@ fn trie_cursor_promote_leaf_to_node4() { let leaf_opt_res = MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ); assert!(leaf_opt_res.is_ok()); @@ -372,7 +372,7 @@ fn trie_cursor_promote_leaf_to_node4() { let leaf_opt_res = MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap(), + &TrieHash::from_bytes(&path[..]).unwrap(), ); assert!(leaf_opt_res.is_ok()); @@ -467,7 +467,7 @@ fn trie_cursor_promote_node4_to_node16() { path[k] = j + 32; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); f.open_block(&block_header).unwrap(); @@ -486,7 +486,7 @@ fn trie_cursor_promote_node4_to_node16() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -515,7 +515,7 @@ fn trie_cursor_promote_node4_to_node16() { path[k] = 128; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -536,7 +536,7 @@ fn trie_cursor_promote_node4_to_node16() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -627,7 +627,7 @@ fn trie_cursor_promote_node16_to_node48() { path[k] = j + 32; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -648,7 +648,7 @@ fn trie_cursor_promote_node16_to_node48() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -677,7 +677,7 @@ fn trie_cursor_promote_node16_to_node48() { path[k] = 128; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -698,7 +698,7 @@ fn trie_cursor_promote_node16_to_node48() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -734,7 +734,7 @@ fn trie_cursor_promote_node16_to_node48() { path[k] = j + 40; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -755,7 +755,7 @@ fn trie_cursor_promote_node16_to_node48() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -784,7 +784,7 @@ fn trie_cursor_promote_node16_to_node48() { path[k] = 129; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -806,7 +806,7 @@ fn trie_cursor_promote_node16_to_node48() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -897,7 +897,7 @@ fn trie_cursor_promote_node48_to_node256() { path[k] = j + 32; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -918,7 +918,7 @@ fn trie_cursor_promote_node48_to_node256() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -947,7 +947,7 @@ fn trie_cursor_promote_node48_to_node256() { path[k] = 128; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -968,7 +968,7 @@ fn trie_cursor_promote_node48_to_node256() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -1004,7 +1004,7 @@ fn trie_cursor_promote_node48_to_node256() { path[k] = j + 40; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -1024,7 +1024,7 @@ fn trie_cursor_promote_node48_to_node256() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -1053,7 +1053,7 @@ fn trie_cursor_promote_node48_to_node256() { path[k] = 129; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -1074,7 +1074,7 @@ fn trie_cursor_promote_node48_to_node256() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -1110,7 +1110,7 @@ fn trie_cursor_promote_node48_to_node256() { path[k] = j + 90; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -1131,7 +1131,7 @@ fn trie_cursor_promote_node48_to_node256() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -1160,7 +1160,7 @@ fn trie_cursor_promote_node48_to_node256() { path[k] = 130; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -1181,7 +1181,7 @@ fn trie_cursor_promote_node48_to_node256() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -1256,7 +1256,7 @@ fn trie_cursor_splice_leaf_4() { path[5 * k + 2] = 32; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); test_debug!("Start splice-insert at {:?}", &c); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -1283,7 +1283,7 @@ fn trie_cursor_splice_leaf_4() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -1349,7 +1349,7 @@ fn trie_cursor_splice_leaf_2() { path[3 * k + 1] = 32; let mut c = - TrieCursor::new(&TriePath::from_bytes(&path[..]).unwrap(), f.root_trieptr()); + TrieCursor::new(&TrieHash::from_bytes(&path[..]).unwrap(), f.root_trieptr()); test_debug!("Start splice-insert at {:?}", &c); let (nodeptr, mut node, node_hash) = walk_to_insertion_point(&mut f, &mut c); @@ -1372,7 +1372,7 @@ fn trie_cursor_splice_leaf_2() { MARF::get_path( &mut f, &block_header, - &TriePath::from_bytes(&path[..]).unwrap() + &TrieHash::from_bytes(&path[..]).unwrap() ) .unwrap() .unwrap(), @@ -1413,7 +1413,7 @@ where for i in 0..count { eprintln!("{}", i); let path = path_gen(i); - let triepath = TriePath::from_bytes(&path).unwrap(); + let triepath = TrieHash::from_bytes(&path).unwrap(); let value = TrieLeaf::new( &vec![], &[ @@ -1519,7 +1519,7 @@ where for i in 0..count { let path = path_gen(i); - let triepath = TriePath::from_bytes(&path).unwrap(); + let triepath = TrieHash::from_bytes(&path).unwrap(); let value = MARF::get_path(&mut marf.borrow_storage_backend(), &block_header, &triepath) .unwrap() diff --git a/stackslib/src/chainstate/stacks/index/trie.rs b/stackslib/src/chainstate/stacks/index/trie.rs index 6c7cc7a08a..5c6f53ab65 100644 --- a/stackslib/src/chainstate/stacks/index/trie.rs +++ b/stackslib/src/chainstate/stacks/index/trie.rs @@ -40,7 +40,7 @@ use crate::chainstate::stacks::index::storage::{ TrieFileStorage, TrieHashCalculationMode, TrieStorageConnection, }; use crate::chainstate::stacks::index::{ - Error, MarfTrieId, TrieHashExtension, TrieHasher, TrieLeaf, + Error, MarfTrieId, TrieHasher, TrieLeaf, }; /// We don't actually instantiate a Trie, but we still need to pass a type parameter for the diff --git a/stackslib/src/chainstate/stacks/index/trie_sql.rs b/stackslib/src/chainstate/stacks/index/trie_sql.rs index c9d3b40dce..8134db9d44 100644 --- a/stackslib/src/chainstate/stacks/index/trie_sql.rs +++ b/stackslib/src/chainstate/stacks/index/trie_sql.rs @@ -45,7 +45,7 @@ use crate::chainstate::stacks::index::bits::{ use crate::chainstate::stacks::index::file::TrieFile; use crate::chainstate::stacks::index::node::{ clear_backptr, is_backptr, set_backptr, TrieNode, TrieNode16, TrieNode256, TrieNode4, - TrieNode48, TrieNodeID, TrieNodeType, TriePath, TriePtr, + TrieNode48, TrieNodeID, TrieNodeType, TriePtr, }; use crate::chainstate::stacks::index::storage::{TrieFileStorage, TrieStorageConnection}; use crate::chainstate::stacks::index::{trie_sql, BlockMap, Error, MarfTrieId, TrieLeaf}; diff --git a/stackslib/src/core/tests/mod.rs b/stackslib/src/core/tests/mod.rs index 01fcac9e89..7945a9331d 100644 --- a/stackslib/src/core/tests/mod.rs +++ b/stackslib/src/core/tests/mod.rs @@ -48,7 +48,7 @@ use crate::chainstate::stacks::db::test::{ }; use crate::chainstate::stacks::db::{StacksChainState, StacksHeaderInfo}; use crate::chainstate::stacks::events::StacksTransactionReceipt; -use crate::chainstate::stacks::index::{MarfTrieId, TrieHashExtension}; +use crate::chainstate::stacks::index::{MarfTrieId}; use crate::chainstate::stacks::miner::TransactionResult; use crate::chainstate::stacks::test::codec_all_transactions; use crate::chainstate::stacks::{ From 2cdf090ced19ec74b7c479051d2ee99ea0f3f89f Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 8 Nov 2024 15:08:40 -0500 Subject: [PATCH 08/21] chore: provide access to MARF'ed data via `MARF::get_by_hash` and `ClarityBackingStore::get_data_from_path` --- clarity/src/vm/database/clarity_db.rs | 15 +++++ clarity/src/vm/database/clarity_store.rs | 6 ++ clarity/src/vm/database/key_value_wrapper.rs | 30 ++++++++++ clarity/src/vm/database/sqlite.rs | 4 ++ stackslib/src/clarity_vm/database/marf.rs | 61 +++++++++++++++++++- stackslib/src/clarity_vm/database/mod.rs | 7 ++- 6 files changed, 120 insertions(+), 3 deletions(-) diff --git a/clarity/src/vm/database/clarity_db.rs b/clarity/src/vm/database/clarity_db.rs index 50715fd98f..993746b8b8 100644 --- a/clarity/src/vm/database/clarity_db.rs +++ b/clarity/src/vm/database/clarity_db.rs @@ -26,6 +26,7 @@ use stacks_common::types::chainstate::{ VRFSeed, }; use stacks_common::types::{Address, StacksEpoch as GenericStacksEpoch, StacksEpochId}; +use stacks_common::types::chainstate::TrieHash; use stacks_common::util::hash::{to_hex, Hash160, Sha256Sum, Sha512Trunc256Sum}; use super::clarity_store::SpecialCaseHandler; @@ -464,6 +465,13 @@ impl<'a> ClarityDatabase<'a> { { self.store.get_data::(key) } + + pub fn get_data_by_hash(&mut self, hash: &TrieHash) -> Result> + where + T: ClarityDeserializable, + { + self.store.get_data_by_hash::(hash) + } pub fn put_value(&mut self, key: &str, value: Value, epoch: &StacksEpochId) -> Result<()> { self.put_value_with_size(key, value, epoch)?; @@ -521,6 +529,13 @@ impl<'a> ClarityDatabase<'a> { { self.store.get_data_with_proof(key) } + + pub fn get_data_with_proof_by_hash(&mut self, hash: &TrieHash) -> Result)>> + where + T: ClarityDeserializable, + { + self.store.get_data_with_proof_by_hash(hash) + } pub fn make_key_for_trip( contract_identifier: &QualifiedContractIdentifier, diff --git a/clarity/src/vm/database/clarity_store.rs b/clarity/src/vm/database/clarity_store.rs index 6e8f878f6e..68c788626b 100644 --- a/clarity/src/vm/database/clarity_store.rs +++ b/clarity/src/vm/database/clarity_store.rs @@ -64,6 +64,8 @@ pub trait ClarityBackingStore { fn put_all_data(&mut self, items: Vec<(String, String)>) -> Result<()>; /// fetch K-V out of the committed datastore fn get_data(&mut self, key: &str) -> Result>; + /// fetch Hash(K)-V out of the commmitted datastore + fn get_data_from_path(&mut self, hash: &TrieHash) -> Result>; /// fetch K-V out of the committed datastore, along with the byte representation /// of the Merkle proof for that key-value pair fn get_data_with_proof(&mut self, key: &str) -> Result)>>; @@ -212,6 +214,10 @@ impl ClarityBackingStore for NullBackingStore { fn get_data(&mut self, _key: &str) -> Result> { panic!("NullBackingStore can't retrieve data") } + + fn get_data_from_path(&mut self, _hash: &TrieHash) -> Result> { + panic!("NullBackingStore can't retrieve data") + } fn get_data_with_proof(&mut self, _key: &str) -> Result)>> { panic!("NullBackingStore can't retrieve data") diff --git a/clarity/src/vm/database/key_value_wrapper.rs b/clarity/src/vm/database/key_value_wrapper.rs index 3fd845f92f..d319603d73 100644 --- a/clarity/src/vm/database/key_value_wrapper.rs +++ b/clarity/src/vm/database/key_value_wrapper.rs @@ -18,6 +18,7 @@ use std::hash::Hash; use hashbrown::HashMap; use stacks_common::types::chainstate::StacksBlockId; +use stacks_common::types::chainstate::TrieHash; use stacks_common::types::StacksEpochId; use stacks_common::util::hash::Sha512Trunc256Sum; @@ -368,6 +369,18 @@ impl<'a> RollbackWrapper<'a> { .map(|(value, proof)| Ok((T::deserialize(&value)?, proof))) .transpose() } + + /// this function will only return commitment proofs for values _already_ materialized + /// in the underlying store. otherwise it returns None. + pub fn get_data_with_proof_by_hash(&mut self, hash: &TrieHash) -> InterpreterResult)>> + where + T: ClarityDeserializable, + { + self.store + .get_data_with_proof_from_path(hash)? + .map(|(value, proof)| Ok((T::deserialize(&value)?, proof))) + .transpose() + } pub fn get_data(&mut self, key: &str) -> InterpreterResult> where @@ -391,6 +404,23 @@ impl<'a> RollbackWrapper<'a> { .map(|x| T::deserialize(&x)) .transpose() } + + /// DO NOT USE IN CONSENSUS CODE. + /// + /// Load data directly from the underlying store, given its trie hash. The lookup map will not + /// be used. + /// + /// This should never be called from within the Clarity VM, or via block-processing. It's only + /// meant to be used by the RPC system. + pub fn get_data_by_hash(&mut self, hash: &TrieHash) -> InterpreterResult> + where + T: ClarityDeserializable, + { + self.store + .get_data_from_path(hash)? + .map(|x| T::deserialize(&x)) + .transpose() + } pub fn deserialize_value( value_hex: &str, diff --git a/clarity/src/vm/database/sqlite.rs b/clarity/src/vm/database/sqlite.rs index 05c1939444..664bfddceb 100644 --- a/clarity/src/vm/database/sqlite.rs +++ b/clarity/src/vm/database/sqlite.rs @@ -324,6 +324,10 @@ impl ClarityBackingStore for MemoryBackingStore { SqliteConnection::get(self.get_side_store(), key) } + fn get_data_from_path(&mut self, hash: &TrieHash) -> Result> { + SqliteConnection::get(self.get_side_store(), hash.to_string().as_str()) + } + fn get_data_with_proof(&mut self, key: &str) -> Result)>> { Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![]))) } diff --git a/stackslib/src/clarity_vm/database/marf.rs b/stackslib/src/clarity_vm/database/marf.rs index be0f60ff56..38537002f8 100644 --- a/stackslib/src/clarity_vm/database/marf.rs +++ b/stackslib/src/clarity_vm/database/marf.rs @@ -20,7 +20,6 @@ use stacks_common::codec::StacksMessageCodec; use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash}; use crate::chainstate::stacks::index::marf::{MARFOpenOpts, MarfConnection, MarfTransaction, MARF}; -use crate::chainstate::stacks::index::node::TriePath; use crate::chainstate::stacks::index::{ ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieMerkleProof, }; @@ -477,6 +476,36 @@ impl<'a> ClarityBackingStore for ReadOnlyMarfStore<'a> { }) .transpose() } + + fn get_data_from_path(&mut self, hash: &TrieHash) -> InterpreterResult> { + trace!("MarfedKV get_from_hash: {:?} tip={}", hash, &self.chain_tip); + self.marf + .get_from_hash(&self.chain_tip, hash) + .or_else(|e| match e { + Error::NotFoundError => { + trace!( + "MarfedKV get {:?} off of {:?}: not found", + hash, + &self.chain_tip + ); + Ok(None) + } + _ => Err(e), + }) + .map_err(|_| InterpreterError::Expect("ERROR: Unexpected MARF Failure on GET".into()))? + .map(|marf_value| { + let side_key = marf_value.to_hex(); + trace!("MarfedKV get side-key for {:?}: {:?}", hash, &side_key); + SqliteConnection::get(self.get_side_store(), &side_key)?.ok_or_else(|| { + InterpreterError::Expect(format!( + "ERROR: MARF contained value_hash not found in side storage: {}", + side_key + )) + .into() + }) + }) + .transpose() + } fn put_all_data(&mut self, _items: Vec<(String, String)>) -> InterpreterResult<()> { error!("Attempted to commit changes to read-only MARF"); @@ -656,6 +685,36 @@ impl<'a> ClarityBackingStore for WritableMarfStore<'a> { }) .transpose() } + + fn get_data_from_path(&mut self, hash: &TrieHash) -> InterpreterResult> { + trace!("MarfedKV get_from_hash: {:?} tip={}", hash, &self.chain_tip); + self.marf + .get_from_hash(&self.chain_tip, hash) + .or_else(|e| match e { + Error::NotFoundError => { + trace!( + "MarfedKV get {:?} off of {:?}: not found", + hash, + &self.chain_tip + ); + Ok(None) + } + _ => Err(e), + }) + .map_err(|_| InterpreterError::Expect("ERROR: Unexpected MARF Failure on GET".into()))? + .map(|marf_value| { + let side_key = marf_value.to_hex(); + trace!("MarfedKV get side-key for {:?}: {:?}", hash, &side_key); + SqliteConnection::get(self.marf.sqlite_tx(), &side_key)?.ok_or_else(|| { + InterpreterError::Expect(format!( + "ERROR: MARF contained value_hash not found in side storage: {}", + side_key + )) + .into() + }) + }) + .transpose() + } fn get_data_with_proof(&mut self, key: &str) -> InterpreterResult)>> { self.marf diff --git a/stackslib/src/clarity_vm/database/mod.rs b/stackslib/src/clarity_vm/database/mod.rs index 51bfc57690..81c132f6fa 100644 --- a/stackslib/src/clarity_vm/database/mod.rs +++ b/stackslib/src/clarity_vm/database/mod.rs @@ -1232,6 +1232,10 @@ impl ClarityBackingStore for MemoryBackingStore { fn get_data(&mut self, key: &str) -> InterpreterResult> { SqliteConnection::get(self.get_side_store(), key) } + + fn get_data_from_path(&mut self, hash: &TrieHash) -> InterpreterResult> { + SqliteConnection::get(self.get_side_store(), hash.to_string().as_str()) + } fn get_data_with_proof(&mut self, key: &str) -> InterpreterResult)>> { Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![]))) @@ -1241,8 +1245,7 @@ impl ClarityBackingStore for MemoryBackingStore { &mut self, key: &TrieHash, ) -> InterpreterResult)>> { - // Ok(SqliteConnection::get(self.get_side_store(), )?.map(|x| (x, vec![]))) - Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![]))) + Ok(SqliteConnection::get(self.get_side_store(), key.to_string().as_str())?.map(|x| (x, vec![]))) } fn get_side_store(&mut self) -> &Connection { From 4dacdd4ef0f9272f964fb0b66e6ee90cb49a3bef Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 8 Nov 2024 15:09:20 -0500 Subject: [PATCH 09/21] chore: use new ClarityDB `get_data_from_path()` to load MARF data by key hash, instead of by key --- stackslib/src/net/api/getclaritymarfvalue.rs | 49 +++++++------------ .../src/net/api/tests/getclaritymarfvalue.rs | 22 ++++----- 2 files changed, 29 insertions(+), 42 deletions(-) diff --git a/stackslib/src/net/api/getclaritymarfvalue.rs b/stackslib/src/net/api/getclaritymarfvalue.rs index ff584a0ccf..c1d6146157 100644 --- a/stackslib/src/net/api/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/getclaritymarfvalue.rs @@ -19,6 +19,7 @@ use clarity::vm::representations::CONTRACT_PRINCIPAL_REGEX_STRING; use lazy_static::lazy_static; use regex::{Captures, Regex}; use stacks_common::types::net::PeerHost; +use stacks_common::types::chainstate::TrieHash; use stacks_common::util::hash::to_hex; use crate::net::http::{ @@ -31,17 +32,6 @@ use crate::net::httpcore::{ }; use crate::net::{Error as NetError, StacksNodeState, TipRequest}; -lazy_static! { - static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = - "[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?".into(); - static ref MARF_KEY_FOR_TRIP_REGEX_STRING: String = format!( - r"vm::{}::\d+::({})", - *CONTRACT_PRINCIPAL_REGEX_STRING, *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, - ); - static ref MARF_KEY_FOR_QUAD_REGEX_STRING: String = - format!(r"{}::[0-9a-fA-F]+", *MARF_KEY_FOR_TRIP_REGEX_STRING,); -} - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ClarityMarfResponse { pub data: String, @@ -53,12 +43,12 @@ pub struct ClarityMarfResponse { #[derive(Clone)] pub struct RPCGetClarityMarfRequestHandler { - pub clarity_marf_key: Option, + pub marf_key_hash: Option, } impl RPCGetClarityMarfRequestHandler { pub fn new() -> Self { Self { - clarity_marf_key: None, + marf_key_hash: None, } } } @@ -70,15 +60,11 @@ impl HttpRequest for RPCGetClarityMarfRequestHandler { } fn path_regex(&self) -> Regex { - Regex::new(&format!( - r"^/v2/clarity/marf/(?P(vm-epoch::epoch-version)|({})|({}))$", - *MARF_KEY_FOR_TRIP_REGEX_STRING, *MARF_KEY_FOR_QUAD_REGEX_STRING - )) - .unwrap() + Regex::new(r#"^/v2/clarity/marf/(?P[0-9a-f]{64})$"#).unwrap() } fn metrics_identifier(&self) -> &str { - "/v2/clarity/marf/:clarity_marf_key" + "/v2/clarity/marf/:marf_key_hash" } /// Try to decode this request. @@ -96,13 +82,14 @@ impl HttpRequest for RPCGetClarityMarfRequestHandler { )); } - let marf_key = if let Some(key_str) = captures.name("clarity_marf_key") { - key_str.as_str().to_string() + let marf_key = if let Some(key_str) = captures.name("marf_key_hash") { + TrieHash::from_hex(key_str.as_str()) + .map_err(|e| Error::Http(400, format!("Invalid hash string: {e:?}")))? } else { - return Err(Error::Http(404, "Missing `clarity_marf_key`".to_string())); + return Err(Error::Http(404, "Missing `marf_key_hash`".to_string())); }; - self.clarity_marf_key = Some(marf_key); + self.marf_key_hash = Some(marf_key); let contents = HttpRequestContents::new().query_string(query); Ok(contents) @@ -113,7 +100,7 @@ impl HttpRequest for RPCGetClarityMarfRequestHandler { impl RPCRequestHandler for RPCGetClarityMarfRequestHandler { /// Reset internal state fn restart(&mut self) { - self.clarity_marf_key = None; + self.marf_key_hash = None; } /// Make the response @@ -123,8 +110,8 @@ impl RPCRequestHandler for RPCGetClarityMarfRequestHandler { contents: HttpRequestContents, node: &mut StacksNodeState, ) -> Result<(HttpResponsePreamble, HttpResponseContents), NetError> { - let clarity_marf_key = self.clarity_marf_key.take().ok_or(NetError::SendError( - "`clarity_marf_key` not set".to_string(), + let marf_key_hash = self.marf_key_hash.take().ok_or(NetError::SendError( + "`marf_key_hash` not set".to_string(), ))?; let tip = match node.load_stacks_chain_tip(&preamble, &contents) { @@ -144,13 +131,13 @@ impl RPCRequestHandler for RPCGetClarityMarfRequestHandler { clarity_tx.with_clarity_db_readonly(|clarity_db| { let (value_hex, marf_proof): (String, _) = if with_proof { clarity_db - .get_data_with_proof(&clarity_marf_key) + .get_data_with_proof_by_hash(&marf_key_hash) .ok() .flatten() .map(|(a, b)| (a, Some(format!("0x{}", to_hex(&b)))))? } else { clarity_db - .get_data(&clarity_marf_key) + .get_data_by_hash(&marf_key_hash) .ok() .flatten() .map(|a| (a, None))? @@ -168,7 +155,7 @@ impl RPCRequestHandler for RPCGetClarityMarfRequestHandler { Ok(Some(None)) => { return StacksHttpResponse::new_error( &preamble, - &HttpNotFound::new("Marf key not found".to_string()), + &HttpNotFound::new("Marf key hash not found".to_string()), ) .try_into_contents() .map_err(NetError::from); @@ -205,14 +192,14 @@ impl HttpResponse for RPCGetClarityMarfRequestHandler { impl StacksHttpRequest { pub fn new_getclaritymarf( host: PeerHost, - clarity_marf_key: String, + marf_key_hash: TrieHash, tip_req: TipRequest, with_proof: bool, ) -> StacksHttpRequest { StacksHttpRequest::new_for_peer( host, "GET".into(), - format!("/v2/clarity/marf/{}", &clarity_marf_key), + format!("/v2/clarity/marf/{}", &marf_key_hash), HttpRequestContents::new() .for_tip(tip_req) .query_arg("proof".into(), if with_proof { "1" } else { "0" }.into()), diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index ce342b7442..f1e47fa377 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -20,6 +20,7 @@ use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions}; use clarity::vm::{ClarityName, ContractName}; use stacks_common::codec::StacksMessageCodec; use stacks_common::types::chainstate::StacksAddress; +use stacks_common::types::chainstate::TrieHash; use stacks_common::types::net::PeerHost; use stacks_common::types::Address; @@ -37,15 +38,15 @@ fn test_try_parse_request() { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); let mut http = StacksHttp::new(addr.clone(), &ConnectionOptions::default()); - let vm_key_epoch = "vm-epoch::epoch-version"; - let vm_key_trip = "vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::1::count"; - let vm_key_quad = "vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::0::data::1234"; + let vm_key_epoch = TrieHash::from_key("vm-epoch::epoch-version"); + let vm_key_trip = TrieHash::from_key("vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::1::count"); + let vm_key_quad = TrieHash::from_key("vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::0::data::1234"); let valid_keys = [vm_key_epoch, vm_key_trip, vm_key_quad]; for key in valid_keys { let request = StacksHttpRequest::new_getclaritymarf( addr.into(), - key.to_string(), + key, TipRequest::SpecificTip(StacksBlockId([0x22; 32])), true, ); @@ -72,12 +73,12 @@ fn test_try_parse_request() { let (preamble, contents) = parsed_request.destruct(); // consumed path args - assert_eq!(handler.clarity_marf_key, Some(key.to_string())); + assert_eq!(handler.marf_key_hash, Some(key.clone())); assert_eq!(&preamble, request.preamble()); handler.restart(); - assert!(handler.clarity_marf_key.is_none()); + assert!(handler.marf_key_hash.is_none()); } } @@ -90,7 +91,7 @@ fn test_try_make_response() { // query existing let request = StacksHttpRequest::new_getclaritymarf( addr.into(), - "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::bar".to_string(), + TrieHash::from_key("vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::bar"), TipRequest::UseLatestAnchoredTip, true, ); @@ -99,8 +100,7 @@ fn test_try_make_response() { // query existing unconfirmed let request = StacksHttpRequest::new_getclaritymarf( addr.into(), - "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world-unconfirmed::1::bar-unconfirmed" - .to_string(), + TrieHash::from_key("vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world-unconfirmed::1::bar-unconfirmed"), TipRequest::UseLatestUnconfirmedTip, true, ); @@ -109,7 +109,7 @@ fn test_try_make_response() { // query non-existant var let request = StacksHttpRequest::new_getclaritymarf( addr.into(), - "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::does-not-exist".to_string(), + TrieHash::from_key("vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::does-not-exist"), TipRequest::UseLatestAnchoredTip, true, ); @@ -118,7 +118,7 @@ fn test_try_make_response() { // query non-existant contract let request = StacksHttpRequest::new_getclaritymarf( addr.into(), - "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.does-not-exist::1::bar".to_string(), + TrieHash::from_key("vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.does-not-exist::1::bar"), TipRequest::UseLatestAnchoredTip, true, ); From 4a7f9e982307f379125080d2746ef60e7b2d0aed Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 8 Nov 2024 15:10:49 -0500 Subject: [PATCH 10/21] chore: cargo fmt --- clarity/src/vm/database/clarity_db.rs | 12 +++++++----- clarity/src/vm/database/clarity_store.rs | 2 +- clarity/src/vm/database/key_value_wrapper.rs | 12 +++++++----- clarity/src/vm/database/sqlite.rs | 2 +- stackslib/src/chainstate/stacks/index/marf.rs | 4 ++-- stackslib/src/chainstate/stacks/index/node.rs | 4 ++-- stackslib/src/chainstate/stacks/index/proofs.rs | 3 +-- stackslib/src/chainstate/stacks/index/storage.rs | 3 +-- stackslib/src/chainstate/stacks/index/test/marf.rs | 4 +--- stackslib/src/chainstate/stacks/index/test/mod.rs | 4 +--- stackslib/src/chainstate/stacks/index/trie.rs | 4 +--- stackslib/src/clarity_vm/database/marf.rs | 4 ++-- stackslib/src/clarity_vm/database/mod.rs | 7 +++++-- stackslib/src/core/tests/mod.rs | 2 +- stackslib/src/net/api/getclaritymarfvalue.rs | 9 +++++---- stackslib/src/net/api/tests/getclaritymarfvalue.rs | 13 ++++++++----- 16 files changed, 46 insertions(+), 43 deletions(-) diff --git a/clarity/src/vm/database/clarity_db.rs b/clarity/src/vm/database/clarity_db.rs index 993746b8b8..ff9bdfaf1a 100644 --- a/clarity/src/vm/database/clarity_db.rs +++ b/clarity/src/vm/database/clarity_db.rs @@ -23,10 +23,9 @@ use stacks_common::consts::{ }; use stacks_common::types::chainstate::{ BlockHeaderHash, BurnchainHeaderHash, ConsensusHash, SortitionId, StacksAddress, StacksBlockId, - VRFSeed, + TrieHash, VRFSeed, }; use stacks_common::types::{Address, StacksEpoch as GenericStacksEpoch, StacksEpochId}; -use stacks_common::types::chainstate::TrieHash; use stacks_common::util::hash::{to_hex, Hash160, Sha256Sum, Sha512Trunc256Sum}; use super::clarity_store::SpecialCaseHandler; @@ -465,7 +464,7 @@ impl<'a> ClarityDatabase<'a> { { self.store.get_data::(key) } - + pub fn get_data_by_hash(&mut self, hash: &TrieHash) -> Result> where T: ClarityDeserializable, @@ -529,8 +528,11 @@ impl<'a> ClarityDatabase<'a> { { self.store.get_data_with_proof(key) } - - pub fn get_data_with_proof_by_hash(&mut self, hash: &TrieHash) -> Result)>> + + pub fn get_data_with_proof_by_hash( + &mut self, + hash: &TrieHash, + ) -> Result)>> where T: ClarityDeserializable, { diff --git a/clarity/src/vm/database/clarity_store.rs b/clarity/src/vm/database/clarity_store.rs index 68c788626b..07d48c9504 100644 --- a/clarity/src/vm/database/clarity_store.rs +++ b/clarity/src/vm/database/clarity_store.rs @@ -214,7 +214,7 @@ impl ClarityBackingStore for NullBackingStore { fn get_data(&mut self, _key: &str) -> Result> { panic!("NullBackingStore can't retrieve data") } - + fn get_data_from_path(&mut self, _hash: &TrieHash) -> Result> { panic!("NullBackingStore can't retrieve data") } diff --git a/clarity/src/vm/database/key_value_wrapper.rs b/clarity/src/vm/database/key_value_wrapper.rs index d319603d73..c444aa553e 100644 --- a/clarity/src/vm/database/key_value_wrapper.rs +++ b/clarity/src/vm/database/key_value_wrapper.rs @@ -17,8 +17,7 @@ use std::hash::Hash; use hashbrown::HashMap; -use stacks_common::types::chainstate::StacksBlockId; -use stacks_common::types::chainstate::TrieHash; +use stacks_common::types::chainstate::{StacksBlockId, TrieHash}; use stacks_common::types::StacksEpochId; use stacks_common::util::hash::Sha512Trunc256Sum; @@ -369,10 +368,13 @@ impl<'a> RollbackWrapper<'a> { .map(|(value, proof)| Ok((T::deserialize(&value)?, proof))) .transpose() } - + /// this function will only return commitment proofs for values _already_ materialized /// in the underlying store. otherwise it returns None. - pub fn get_data_with_proof_by_hash(&mut self, hash: &TrieHash) -> InterpreterResult)>> + pub fn get_data_with_proof_by_hash( + &mut self, + hash: &TrieHash, + ) -> InterpreterResult)>> where T: ClarityDeserializable, { @@ -404,7 +406,7 @@ impl<'a> RollbackWrapper<'a> { .map(|x| T::deserialize(&x)) .transpose() } - + /// DO NOT USE IN CONSENSUS CODE. /// /// Load data directly from the underlying store, given its trie hash. The lookup map will not diff --git a/clarity/src/vm/database/sqlite.rs b/clarity/src/vm/database/sqlite.rs index 664bfddceb..65b4dfaea5 100644 --- a/clarity/src/vm/database/sqlite.rs +++ b/clarity/src/vm/database/sqlite.rs @@ -326,7 +326,7 @@ impl ClarityBackingStore for MemoryBackingStore { fn get_data_from_path(&mut self, hash: &TrieHash) -> Result> { SqliteConnection::get(self.get_side_store(), hash.to_string().as_str()) - } + } fn get_data_with_proof(&mut self, key: &str) -> Result)>> { Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![]))) diff --git a/stackslib/src/chainstate/stacks/index/marf.rs b/stackslib/src/chainstate/stacks/index/marf.rs index 427cde29fc..ffe9af2174 100644 --- a/stackslib/src/chainstate/stacks/index/marf.rs +++ b/stackslib/src/chainstate/stacks/index/marf.rs @@ -126,7 +126,7 @@ pub trait MarfConnection { fn get(&mut self, block_hash: &T, key: &str) -> Result, Error> { self.with_conn(|c| MARF::get_by_key(c, block_hash, key)) } - + /// Resolve a TrieHash from the MARF to a MARFValue with respect to the given block height. fn get_from_hash(&mut self, block_hash: &T, th: &TrieHash) -> Result, Error> { self.with_conn(|c| MARF::get_by_hash(c, block_hash, th)) @@ -1397,7 +1397,7 @@ impl MARF { let proof = TrieMerkleProof::from_raw_entry(&mut conn, key, &marf_value, block_hash)?; Ok(Some((marf_value, proof))) } - + pub fn get_with_proof_from_hash( &mut self, block_hash: &T, diff --git a/stackslib/src/chainstate/stacks/index/node.rs b/stackslib/src/chainstate/stacks/index/node.rs index 4436d7f239..da9fc8bbd2 100644 --- a/stackslib/src/chainstate/stacks/index/node.rs +++ b/stackslib/src/chainstate/stacks/index/node.rs @@ -32,8 +32,8 @@ use crate::chainstate::stacks::index::bits::{ get_path_byte_len, get_ptrs_byte_len, path_from_bytes, ptrs_from_bytes, write_path_to_bytes, }; use crate::chainstate::stacks::index::{ - BlockMap, ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieHasher, - TrieLeaf, MARF_VALUE_ENCODED_SIZE, + BlockMap, ClarityMarfTrieId, Error, MARFValue, MarfTrieId, TrieHasher, TrieLeaf, + MARF_VALUE_ENCODED_SIZE, }; #[derive(Debug, Clone, PartialEq)] diff --git a/stackslib/src/chainstate/stacks/index/proofs.rs b/stackslib/src/chainstate/stacks/index/proofs.rs index aae802334c..85e91ebefb 100644 --- a/stackslib/src/chainstate/stacks/index/proofs.rs +++ b/stackslib/src/chainstate/stacks/index/proofs.rs @@ -35,8 +35,7 @@ use crate::chainstate::stacks::index::bits::{ use crate::chainstate::stacks::index::marf::MARF; use crate::chainstate::stacks::index::node::{ clear_backptr, is_backptr, set_backptr, ConsensusSerializable, CursorError, TrieCursor, - TrieNode, TrieNode16, TrieNode256, TrieNode4, TrieNode48, TrieNodeID, TrieNodeType, - TriePtr, + TrieNode, TrieNode16, TrieNode256, TrieNode4, TrieNode48, TrieNodeID, TrieNodeType, TriePtr, }; use crate::chainstate::stacks::index::storage::{TrieFileStorage, TrieStorageConnection}; use crate::chainstate::stacks::index::trie::Trie; diff --git a/stackslib/src/chainstate/stacks/index/storage.rs b/stackslib/src/chainstate/stacks/index/storage.rs index 170430c74c..6e7ca815c9 100644 --- a/stackslib/src/chainstate/stacks/index/storage.rs +++ b/stackslib/src/chainstate/stacks/index/storage.rs @@ -51,8 +51,7 @@ use crate::chainstate::stacks::index::node::{ use crate::chainstate::stacks::index::profile::TrieBenchmark; use crate::chainstate::stacks::index::trie::Trie; use crate::chainstate::stacks::index::{ - trie_sql, BlockMap, ClarityMarfTrieId, Error, MarfTrieId, TrieHasher, - TrieLeaf, + trie_sql, BlockMap, ClarityMarfTrieId, Error, MarfTrieId, TrieHasher, TrieLeaf, }; use crate::util_lib::db::{ sql_pragma, sqlite_open, tx_begin_immediate, tx_busy_handler, Error as db_error, diff --git a/stackslib/src/chainstate/stacks/index/test/marf.rs b/stackslib/src/chainstate/stacks/index/test/marf.rs index f4f4dd0de0..e7535e9553 100644 --- a/stackslib/src/chainstate/stacks/index/test/marf.rs +++ b/stackslib/src/chainstate/stacks/index/test/marf.rs @@ -32,9 +32,7 @@ use crate::chainstate::stacks::index::proofs::*; use crate::chainstate::stacks::index::storage::*; use crate::chainstate::stacks::index::test::*; use crate::chainstate::stacks::index::trie::*; -use crate::chainstate::stacks::index::{ - ClarityMarfTrieId, Error, MARFValue, TrieLeaf, -}; +use crate::chainstate::stacks::index::{ClarityMarfTrieId, Error, MARFValue, TrieLeaf}; #[test] fn marf_insert_different_leaf_same_block_100() { diff --git a/stackslib/src/chainstate/stacks/index/test/mod.rs b/stackslib/src/chainstate/stacks/index/test/mod.rs index 9dbaa0959d..0ccdffa78b 100644 --- a/stackslib/src/chainstate/stacks/index/test/mod.rs +++ b/stackslib/src/chainstate/stacks/index/test/mod.rs @@ -31,9 +31,7 @@ use crate::chainstate::stacks::index::node::*; use crate::chainstate::stacks::index::proofs::*; use crate::chainstate::stacks::index::storage::*; use crate::chainstate::stacks::index::trie::*; -use crate::chainstate::stacks::index::{ - MARFValue, MarfTrieId, TrieLeaf, TrieMerkleProof, -}; +use crate::chainstate::stacks::index::{MARFValue, MarfTrieId, TrieLeaf, TrieMerkleProof}; use crate::chainstate::stacks::{BlockHeaderHash, TrieHash}; pub mod cache; diff --git a/stackslib/src/chainstate/stacks/index/trie.rs b/stackslib/src/chainstate/stacks/index/trie.rs index 5c6f53ab65..65e41cf3ed 100644 --- a/stackslib/src/chainstate/stacks/index/trie.rs +++ b/stackslib/src/chainstate/stacks/index/trie.rs @@ -39,9 +39,7 @@ use crate::chainstate::stacks::index::node::{ use crate::chainstate::stacks::index::storage::{ TrieFileStorage, TrieHashCalculationMode, TrieStorageConnection, }; -use crate::chainstate::stacks::index::{ - Error, MarfTrieId, TrieHasher, TrieLeaf, -}; +use crate::chainstate::stacks::index::{Error, MarfTrieId, TrieHasher, TrieLeaf}; /// We don't actually instantiate a Trie, but we still need to pass a type parameter for the /// storage implementation. diff --git a/stackslib/src/clarity_vm/database/marf.rs b/stackslib/src/clarity_vm/database/marf.rs index 38537002f8..3a8636b3b5 100644 --- a/stackslib/src/clarity_vm/database/marf.rs +++ b/stackslib/src/clarity_vm/database/marf.rs @@ -476,7 +476,7 @@ impl<'a> ClarityBackingStore for ReadOnlyMarfStore<'a> { }) .transpose() } - + fn get_data_from_path(&mut self, hash: &TrieHash) -> InterpreterResult> { trace!("MarfedKV get_from_hash: {:?} tip={}", hash, &self.chain_tip); self.marf @@ -685,7 +685,7 @@ impl<'a> ClarityBackingStore for WritableMarfStore<'a> { }) .transpose() } - + fn get_data_from_path(&mut self, hash: &TrieHash) -> InterpreterResult> { trace!("MarfedKV get_from_hash: {:?} tip={}", hash, &self.chain_tip); self.marf diff --git a/stackslib/src/clarity_vm/database/mod.rs b/stackslib/src/clarity_vm/database/mod.rs index 81c132f6fa..0bce54dcfb 100644 --- a/stackslib/src/clarity_vm/database/mod.rs +++ b/stackslib/src/clarity_vm/database/mod.rs @@ -1232,7 +1232,7 @@ impl ClarityBackingStore for MemoryBackingStore { fn get_data(&mut self, key: &str) -> InterpreterResult> { SqliteConnection::get(self.get_side_store(), key) } - + fn get_data_from_path(&mut self, hash: &TrieHash) -> InterpreterResult> { SqliteConnection::get(self.get_side_store(), hash.to_string().as_str()) } @@ -1245,7 +1245,10 @@ impl ClarityBackingStore for MemoryBackingStore { &mut self, key: &TrieHash, ) -> InterpreterResult)>> { - Ok(SqliteConnection::get(self.get_side_store(), key.to_string().as_str())?.map(|x| (x, vec![]))) + Ok( + SqliteConnection::get(self.get_side_store(), key.to_string().as_str())? + .map(|x| (x, vec![])), + ) } fn get_side_store(&mut self) -> &Connection { diff --git a/stackslib/src/core/tests/mod.rs b/stackslib/src/core/tests/mod.rs index 7945a9331d..03447e9bf4 100644 --- a/stackslib/src/core/tests/mod.rs +++ b/stackslib/src/core/tests/mod.rs @@ -48,7 +48,7 @@ use crate::chainstate::stacks::db::test::{ }; use crate::chainstate::stacks::db::{StacksChainState, StacksHeaderInfo}; use crate::chainstate::stacks::events::StacksTransactionReceipt; -use crate::chainstate::stacks::index::{MarfTrieId}; +use crate::chainstate::stacks::index::MarfTrieId; use crate::chainstate::stacks::miner::TransactionResult; use crate::chainstate::stacks::test::codec_all_transactions; use crate::chainstate::stacks::{ diff --git a/stackslib/src/net/api/getclaritymarfvalue.rs b/stackslib/src/net/api/getclaritymarfvalue.rs index c1d6146157..44d2e4dc7f 100644 --- a/stackslib/src/net/api/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/getclaritymarfvalue.rs @@ -18,8 +18,8 @@ use clarity::vm::clarity::ClarityConnection; use clarity::vm::representations::CONTRACT_PRINCIPAL_REGEX_STRING; use lazy_static::lazy_static; use regex::{Captures, Regex}; -use stacks_common::types::net::PeerHost; use stacks_common::types::chainstate::TrieHash; +use stacks_common::types::net::PeerHost; use stacks_common::util::hash::to_hex; use crate::net::http::{ @@ -110,9 +110,10 @@ impl RPCRequestHandler for RPCGetClarityMarfRequestHandler { contents: HttpRequestContents, node: &mut StacksNodeState, ) -> Result<(HttpResponsePreamble, HttpResponseContents), NetError> { - let marf_key_hash = self.marf_key_hash.take().ok_or(NetError::SendError( - "`marf_key_hash` not set".to_string(), - ))?; + let marf_key_hash = self + .marf_key_hash + .take() + .ok_or(NetError::SendError("`marf_key_hash` not set".to_string()))?; let tip = match node.load_stacks_chain_tip(&preamble, &contents) { Ok(tip) => tip, diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index f1e47fa377..3b1453c212 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -19,8 +19,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions}; use clarity::vm::{ClarityName, ContractName}; use stacks_common::codec::StacksMessageCodec; -use stacks_common::types::chainstate::StacksAddress; -use stacks_common::types::chainstate::TrieHash; +use stacks_common::types::chainstate::{StacksAddress, TrieHash}; use stacks_common::types::net::PeerHost; use stacks_common::types::Address; @@ -39,8 +38,10 @@ fn test_try_parse_request() { let mut http = StacksHttp::new(addr.clone(), &ConnectionOptions::default()); let vm_key_epoch = TrieHash::from_key("vm-epoch::epoch-version"); - let vm_key_trip = TrieHash::from_key("vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::1::count"); - let vm_key_quad = TrieHash::from_key("vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::0::data::1234"); + let vm_key_trip = + TrieHash::from_key("vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::1::count"); + let vm_key_quad = + TrieHash::from_key("vm::ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5.counter::0::data::1234"); let valid_keys = [vm_key_epoch, vm_key_trip, vm_key_quad]; for key in valid_keys { @@ -109,7 +110,9 @@ fn test_try_make_response() { // query non-existant var let request = StacksHttpRequest::new_getclaritymarf( addr.into(), - TrieHash::from_key("vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::does-not-exist"), + TrieHash::from_key( + "vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::does-not-exist", + ), TipRequest::UseLatestAnchoredTip, true, ); From 48a03bbff9ae308df6d7f3d38f946553d09257fb Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 8 Nov 2024 15:41:49 -0500 Subject: [PATCH 11/21] chore: test get_by_hash() by loading both the value in get_by_key() and the same value by get_by_hash(hash(key)) --- stackslib/src/chainstate/stacks/index/marf.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/stackslib/src/chainstate/stacks/index/marf.rs b/stackslib/src/chainstate/stacks/index/marf.rs index ffe9af2174..a4082627fd 100644 --- a/stackslib/src/chainstate/stacks/index/marf.rs +++ b/stackslib/src/chainstate/stacks/index/marf.rs @@ -122,8 +122,31 @@ pub trait MarfConnection { fn sqlite_conn(&self) -> &Connection; + /// Get and check a value against get_from_hash + /// (test only) + #[cfg(test)] + fn get_and_check_with_hash(&mut self, block_hash: &T, key: &str) { + let res = self.with_conn(|c| MARF::get_by_key(c, block_hash, key)); + let res_with_hash = + self.with_conn(|c| MARF::get_by_hash(c, block_hash, &TrieHash::from_key(key))); + match (res, res_with_hash) { + (Ok(Some(x)), Ok(Some(y))) => { + assert_eq!(x, y); + } + (Ok(None), Ok(None)) => {} + (Err(_), Err(_)) => {} + (x, y) => { + panic!("Inconsistency: {x:?} != {y:?}"); + } + } + } + + #[cfg(not(test))] + fn get_and_check_with_hash(&mut self, _block_hash: &T, _key: &str) {} + /// Resolve a key from the MARF to a MARFValue with respect to the given block height. fn get(&mut self, block_hash: &T, key: &str) -> Result, Error> { + self.get_and_check_with_hash(block_hash, key); self.with_conn(|c| MARF::get_by_key(c, block_hash, key)) } From 878a8a308f021ac900ae0750999d38f8243f5681 Mon Sep 17 00:00:00 2001 From: Hugo CAILLARD <911307+hugocaillard@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:56:20 +0100 Subject: [PATCH 12/21] feat: validate get clarity metadata key format --- clarity/src/vm/database/clarity_db.rs | 112 ++++++++++++++++-- stackslib/src/net/api/getclaritymetadata.rs | 50 ++++++-- .../src/net/api/tests/getclaritymarfvalue.rs | 2 +- .../src/net/api/tests/getclaritymetadata.rs | 73 +++++++++++- 4 files changed, 215 insertions(+), 22 deletions(-) diff --git a/clarity/src/vm/database/clarity_db.rs b/clarity/src/vm/database/clarity_db.rs index ff9bdfaf1a..4f6f3f7781 100644 --- a/clarity/src/vm/database/clarity_db.rs +++ b/clarity/src/vm/database/clarity_db.rs @@ -76,6 +76,68 @@ pub enum StoreType { PoxUnlockHeight = 0x15, } +impl TryFrom<&str> for StoreType { + type Error = String; + + fn try_from(value: &str) -> core::result::Result { + use self::StoreType::*; + + let hex_value = u8::from_str_radix(value, 10).map_err(|e| e.to_string())?; + match hex_value { + 0x00 => Ok(DataMap), + 0x01 => Ok(Variable), + 0x02 => Ok(FungibleToken), + 0x03 => Ok(CirculatingSupply), + 0x04 => Ok(NonFungibleToken), + 0x05 => Ok(DataMapMeta), + 0x06 => Ok(VariableMeta), + 0x07 => Ok(FungibleTokenMeta), + 0x08 => Ok(NonFungibleTokenMeta), + 0x09 => Ok(Contract), + 0x10 => Ok(SimmedBlock), + 0x11 => Ok(SimmedBlockHeight), + 0x12 => Ok(Nonce), + 0x13 => Ok(STXBalance), + 0x14 => Ok(PoxSTXLockup), + 0x15 => Ok(PoxUnlockHeight), + _ => Err("Invalid StoreType".into()), + } + } +} + +pub enum ContractDataVarName { + Contract, + ContractSize, + ContractSrc, + ContractDataSize, +} + +impl ContractDataVarName { + pub fn as_str(&self) -> &str { + match self { + Self::Contract => "contract", + Self::ContractSize => "contract-size", + Self::ContractSrc => "contract-src", + Self::ContractDataSize => "contract-data-size", + } + } +} + +impl TryFrom<&str> for ContractDataVarName { + type Error = String; + + fn try_from(value: &str) -> core::result::Result { + use self::ContractDataVarName::*; + match value { + "contract" => Ok(Contract), + "contract-size" => Ok(ContractSize), + "contract-src" => Ok(ContractSrc), + "contract-data-size" => Ok(ContractDataSize), + _ => Err("Invalid ContractDataVarName".into()), + } + } +} + pub struct ClarityDatabase<'a> { pub store: RollbackWrapper<'a>, headers_db: &'a dyn HeadersDB, @@ -576,12 +638,18 @@ impl<'a> ClarityDatabase<'a> { self.store .prepare_for_contract_metadata(contract_identifier, hash)?; // insert contract-size - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::ContractSize.as_str(), + ); self.insert_metadata(contract_identifier, &key, &(contract_content.len() as u64))?; // insert contract-src if STORE_CONTRACT_SRC_INTERFACE { - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::ContractSrc.as_str(), + ); self.insert_metadata(contract_identifier, &key, &contract_content.to_string())?; } Ok(()) @@ -591,7 +659,10 @@ impl<'a> ClarityDatabase<'a> { &mut self, contract_identifier: &QualifiedContractIdentifier, ) -> Option { - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::ContractSrc.as_str(), + ); self.fetch_metadata(contract_identifier, &key) .ok() .flatten() @@ -700,7 +771,10 @@ impl<'a> ClarityDatabase<'a> { &mut self, contract_identifier: &QualifiedContractIdentifier, ) -> Result { - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::ContractSize.as_str(), + ); let contract_size: u64 = self.fetch_metadata(contract_identifier, &key)? .ok_or_else(|| { @@ -708,7 +782,10 @@ impl<'a> ClarityDatabase<'a> { "Failed to read non-consensus contract metadata, even though contract exists in MARF." .into()) })?; - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::ContractDataSize.as_str(), + ); let data_size: u64 = self .fetch_metadata(contract_identifier, &key)? .ok_or_else(|| { @@ -727,7 +804,10 @@ impl<'a> ClarityDatabase<'a> { contract_identifier: &QualifiedContractIdentifier, data_size: u64, ) -> Result<()> { - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::ContractSize.as_str(), + ); let contract_size: u64 = self.fetch_metadata(contract_identifier, &key)? .ok_or_else(|| { @@ -737,7 +817,10 @@ impl<'a> ClarityDatabase<'a> { })?; contract_size.cost_overflow_add(data_size)?; - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::ContractDataSize.as_str(), + ); self.insert_metadata(contract_identifier, &key, &data_size)?; Ok(()) } @@ -747,13 +830,19 @@ impl<'a> ClarityDatabase<'a> { contract_identifier: &QualifiedContractIdentifier, contract: Contract, ) -> Result<()> { - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::Contract.as_str(), + ); self.insert_metadata(contract_identifier, &key, &contract)?; Ok(()) } pub fn has_contract(&mut self, contract_identifier: &QualifiedContractIdentifier) -> bool { - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::Contract.as_str(), + ); self.store.has_metadata_entry(contract_identifier, &key) } @@ -761,7 +850,10 @@ impl<'a> ClarityDatabase<'a> { &mut self, contract_identifier: &QualifiedContractIdentifier, ) -> Result { - let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract"); + let key = ClarityDatabase::make_metadata_key( + StoreType::Contract, + ContractDataVarName::Contract.as_str(), + ); let mut data: Contract = self.fetch_metadata(contract_identifier, &key)? .ok_or_else(|| InterpreterError::Expect( "Failed to read non-consensus contract metadata, even though contract exists in MARF." diff --git a/stackslib/src/net/api/getclaritymetadata.rs b/stackslib/src/net/api/getclaritymetadata.rs index 5ef3feee6e..5700e87af9 100644 --- a/stackslib/src/net/api/getclaritymetadata.rs +++ b/stackslib/src/net/api/getclaritymetadata.rs @@ -15,6 +15,8 @@ // along with this program. If not, see . use clarity::vm::clarity::ClarityConnection; +use clarity::vm::database::clarity_db::ContractDataVarName; +use clarity::vm::database::StoreType; use clarity::vm::representations::{CONTRACT_NAME_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING}; use clarity::vm::types::QualifiedContractIdentifier; use clarity::vm::ContractName; @@ -37,7 +39,7 @@ lazy_static! { static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = "[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?".into(); static ref METADATA_KEY_REGEX_STRING: String = format!( - r"vm-metadata::\d+::(contract|contract-size|contract-src|contract-data-size|({}))", + r"vm-metadata::(?P(\d{{1,2}}))::(?P(contract|contract-size|contract-src|contract-data-size|({})))", *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, ); } @@ -81,8 +83,6 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { "/v2/clarity/metadata/:principal/:contract_name/:clarity_metadata_key" } - /// Try to decode this request. - /// There's nothing to load here, so just make sure the request is well-formed. fn try_parse_request( &mut self, preamble: &HttpRequestPreamble, @@ -98,13 +98,43 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { let contract_identifier = request::get_contract_address(captures, "address", "contract")?; - let metadata_key = if let Some(key_str) = captures.name("clarity_metadata_key") { - key_str.as_str().to_string() - } else { - return Err(Error::Http( - 404, - "Missing `clarity_metadata_key`".to_string(), - )); + // Validate that the metadata key is well-formed. It must be of data type: + // DataMapMeta (5) | VariableMeta (6) | FungibleTokenMeta (7) | NonFungibleTokenMeta (8) + // or Contract (9) followed by a valid contract metadata name + match captures + .name("data_type") + .and_then(|data_type| StoreType::try_from(data_type.as_str()).ok()) + { + Some(data_type) => match data_type { + StoreType::DataMapMeta + | StoreType::VariableMeta + | StoreType::FungibleTokenMeta + | StoreType::NonFungibleTokenMeta => {} + StoreType::Contract => { + if captures + .name("var_name") + .and_then(|var_name| ContractDataVarName::try_from(var_name.as_str()).ok()) + .is_none() + { + return Err(Error::DecodeError("Invalid metadata var name".to_string())); + } + } + _ => { + return Err(Error::DecodeError("Invalid metadata type".to_string())); + } + }, + None => { + return Err(Error::DecodeError("Invalid metadata type".to_string())); + } + } + + let metadata_key = match captures.name("clarity_metadata_key") { + Some(key_str) => key_str.as_str().to_string(), + None => { + return Err(Error::DecodeError( + "Missing `clarity_metadata_key`".to_string(), + )); + } }; self.contract_identifier = Some(contract_identifier); diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index 3b1453c212..e360b7a72c 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -89,7 +89,7 @@ fn test_try_make_response() { let mut requests = vec![]; - // query existing + // query existing marf value let request = StacksHttpRequest::new_getclaritymarf( addr.into(), TrieHash::from_key("vm::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world::1::bar"), diff --git a/stackslib/src/net/api/tests/getclaritymetadata.rs b/stackslib/src/net/api/tests/getclaritymetadata.rs index 3de5949a87..57baf705ce 100644 --- a/stackslib/src/net/api/tests/getclaritymetadata.rs +++ b/stackslib/src/net/api/tests/getclaritymetadata.rs @@ -26,11 +26,12 @@ use stacks_common::types::Address; use super::test_rpc; use crate::net::api::*; use crate::net::connection::ConnectionOptions; +use crate::net::http::Error as HttpError; use crate::net::httpcore::{ HttpPreambleExtensions, HttpRequestContentsExtensions, RPCRequestHandler, StacksHttp, StacksHttpRequest, }; -use crate::net::{ProtocolFamily, TipRequest}; +use crate::net::{Error as NetError, ProtocolFamily, TipRequest}; #[test] fn test_try_parse_request() { @@ -85,6 +86,76 @@ fn test_try_parse_request() { assert!(handler.clarity_metadata_key.is_none()); } +#[test] +fn test_try_parse_invalid_store_type() { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); + let mut http = StacksHttp::new(addr.clone(), &ConnectionOptions::default()); + + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::2::contract-size".to_string(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])), + ); + assert_eq!( + request.contents().tip_request(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])) + ); + let bytes = request.try_serialize().unwrap(); + + let (parsed_preamble, offset) = http.read_preamble(&bytes).unwrap(); + let mut handler = getclaritymetadata::RPCGetClarityMetadataRequestHandler::new(); + let parsed_request_err = http + .handle_try_parse_request( + &mut handler, + &parsed_preamble.expect_request(), + &bytes[offset..], + ) + .unwrap_err(); + + assert_eq!( + parsed_request_err, + HttpError::DecodeError("Invalid metadata type".to_string()).into() + ); + handler.restart(); +} + +#[test] +fn test_try_parse_invalid_contract_metadata_var_name() { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); + let mut http = StacksHttp::new(addr.clone(), &ConnectionOptions::default()); + + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::9::contract-invalid-key".to_string(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])), + ); + assert_eq!( + request.contents().tip_request(), + TipRequest::SpecificTip(StacksBlockId([0x22; 32])) + ); + let bytes = request.try_serialize().unwrap(); + + let (parsed_preamble, offset) = http.read_preamble(&bytes).unwrap(); + let mut handler = getclaritymetadata::RPCGetClarityMetadataRequestHandler::new(); + let parsed_request_err = http + .handle_try_parse_request( + &mut handler, + &parsed_preamble.expect_request(), + &bytes[offset..], + ) + .unwrap_err(); + + assert_eq!( + parsed_request_err, + HttpError::DecodeError("Invalid metadata var name".to_string()).into() + ); + handler.restart(); +} + #[test] fn test_try_parse_request_for_analysis() { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 33333); From f27c84fee83cd744ec963b9b7606046a774cd457 Mon Sep 17 00:00:00 2001 From: Hugo CAILLARD <911307+hugocaillard@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:00:31 +0100 Subject: [PATCH 13/21] docs: update openapi.yaml and rpc-endpoint.md --- docs/rpc-endpoints.md | 4 ++-- docs/rpc/openapi.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/rpc-endpoints.md b/docs/rpc-endpoints.md index 973e3478d4..63a086ca1b 100644 --- a/docs/rpc-endpoints.md +++ b/docs/rpc-endpoints.md @@ -172,7 +172,7 @@ Where data is the hex serialization of the variable value. This endpoint also accepts a querystring parameter `?proof=` which when supplied `0`, will return the JSON object _without_ the `proof` field. -### GET /v2/clarity_marf_value/[Clarity MARF Key] +### GET /v2/clarity/marf/[Clarity MARF Key] Attempt to fetch the value of a MARF key. The key is identified with [Clarity MARF Key]. Returns JSON data in the form: @@ -186,7 +186,7 @@ Returns JSON data in the form: Where data is the hex serialization of the value. -### GET /v2/clarity_metadata/[Stacks Address]/[Contract Name]/[Clarity Metadata Key] +### GET /v2/clarity/metadata/[Stacks Address]/[Contract Name]/[Clarity Metadata Key] Attempt to fetch the metadata of a contract. The contract is identified with [Stacks Address] and [Contract Name] in the URL path. The metadata key is identified with [Clarity Metadata Key]. diff --git a/docs/rpc/openapi.yaml b/docs/rpc/openapi.yaml index 2b73198511..c4b1ec7b63 100644 --- a/docs/rpc/openapi.yaml +++ b/docs/rpc/openapi.yaml @@ -486,7 +486,7 @@ paths: If tip == "latest", the query will be run from the latest known tip (includes unconfirmed state). If the tip is left unspecified, the stacks chain tip will be selected (only includes confirmed state). - /v2/clarity_marf_value/{clarity_marf_key}: + /v2/clarity/marf/{clarity_marf_key}: post: summary: Get the MARF value for a given key tags: @@ -526,7 +526,7 @@ paths: description: The Stacks chain tip to query from. If tip == latest, the query will be run from the latest known tip (includes unconfirmed state). - /v2/clarity_metadata/{contract_address}/{contract_name}/{clarity_metadata_key}: + /v2/clarity/metadata/{contract_address}/{contract_name}/{clarity_metadata_key}: post: summary: Get the contract metadata for the metadata key tags: From 7f86352f185fefca963f20b4d3567d98bcefb2e9 Mon Sep 17 00:00:00 2001 From: Hugo CAILLARD <911307+hugocaillard@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:07:02 +0100 Subject: [PATCH 14/21] fix: metadata key validation for cotnract analysis --- stackslib/src/net/api/getclaritymetadata.rs | 66 +++++++++++---------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/stackslib/src/net/api/getclaritymetadata.rs b/stackslib/src/net/api/getclaritymetadata.rs index 5700e87af9..29ba1d6f4e 100644 --- a/stackslib/src/net/api/getclaritymetadata.rs +++ b/stackslib/src/net/api/getclaritymetadata.rs @@ -98,36 +98,6 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { let contract_identifier = request::get_contract_address(captures, "address", "contract")?; - // Validate that the metadata key is well-formed. It must be of data type: - // DataMapMeta (5) | VariableMeta (6) | FungibleTokenMeta (7) | NonFungibleTokenMeta (8) - // or Contract (9) followed by a valid contract metadata name - match captures - .name("data_type") - .and_then(|data_type| StoreType::try_from(data_type.as_str()).ok()) - { - Some(data_type) => match data_type { - StoreType::DataMapMeta - | StoreType::VariableMeta - | StoreType::FungibleTokenMeta - | StoreType::NonFungibleTokenMeta => {} - StoreType::Contract => { - if captures - .name("var_name") - .and_then(|var_name| ContractDataVarName::try_from(var_name.as_str()).ok()) - .is_none() - { - return Err(Error::DecodeError("Invalid metadata var name".to_string())); - } - } - _ => { - return Err(Error::DecodeError("Invalid metadata type".to_string())); - } - }, - None => { - return Err(Error::DecodeError("Invalid metadata type".to_string())); - } - } - let metadata_key = match captures.name("clarity_metadata_key") { Some(key_str) => key_str.as_str().to_string(), None => { @@ -137,6 +107,42 @@ impl HttpRequest for RPCGetClarityMetadataRequestHandler { } }; + if metadata_key != "analysis" { + // Validate that the metadata key is well-formed. It must be of data type: + // DataMapMeta (5) | VariableMeta (6) | FungibleTokenMeta (7) | NonFungibleTokenMeta (8) + // or Contract (9) followed by a valid contract metadata name + match captures + .name("data_type") + .and_then(|data_type| StoreType::try_from(data_type.as_str()).ok()) + { + Some(data_type) => match data_type { + StoreType::DataMapMeta + | StoreType::VariableMeta + | StoreType::FungibleTokenMeta + | StoreType::NonFungibleTokenMeta => {} + StoreType::Contract => { + if captures + .name("var_name") + .and_then(|var_name| { + ContractDataVarName::try_from(var_name.as_str()).ok() + }) + .is_none() + { + return Err(Error::DecodeError( + "Invalid metadata var name".to_string(), + )); + } + } + _ => { + return Err(Error::DecodeError("Invalid metadata type".to_string())); + } + }, + None => { + return Err(Error::DecodeError("Invalid metadata type".to_string())); + } + } + } + self.contract_identifier = Some(contract_identifier); self.clarity_metadata_key = Some(metadata_key); From 9429c8e309a09de689e895f1271b533d030d6abf Mon Sep 17 00:00:00 2001 From: Hugo Caillard <911307+hugocaillard@users.noreply.github.com> Date: Fri, 15 Nov 2024 22:40:19 +0100 Subject: [PATCH 15/21] docs: address review --- docs/rpc/openapi.yaml | 4 ++-- stackslib/src/net/api/getclaritymarfvalue.rs | 3 +-- stackslib/src/net/api/getclaritymetadata.rs | 3 +-- stackslib/src/net/api/tests/getclaritymarfvalue.rs | 3 +-- stackslib/src/net/api/tests/getclaritymetadata.rs | 3 +-- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/rpc/openapi.yaml b/docs/rpc/openapi.yaml index c4b1ec7b63..606db3a453 100644 --- a/docs/rpc/openapi.yaml +++ b/docs/rpc/openapi.yaml @@ -493,7 +493,7 @@ paths: - Smart Contracts operationId: get_clarity_marf_value description: | - Attempt to fetch the value of a MARF key. The key is identified with [Clarity MARF Key]. + Attempt to fetch the value of a MARF key. In the response, `data` is the hex serialization of the value. responses: @@ -533,7 +533,7 @@ paths: - Smart Contracts operationId: get_clarity_metadata_key description: | - Attempt to fetch the metadata of a contract. The contract is identified with [Stacks Address] and [Contract Name] in the URL path. The metadata key is identified with [Clarity Metadata Key]. + Attempt to fetch the metadata of a contract. The contract is identified with [Contract Address] and [Contract Name] in the URL path. The metadata key is identified with [Clarity Metadata Key]. In the response, `data` is formatted as JSON. responses: diff --git a/stackslib/src/net/api/getclaritymarfvalue.rs b/stackslib/src/net/api/getclaritymarfvalue.rs index 44d2e4dc7f..678d4fa46b 100644 --- a/stackslib/src/net/api/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/getclaritymarfvalue.rs @@ -1,5 +1,4 @@ -// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020-2024 Stacks Open Internet Foundation +// Copyright (C) 2024 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/stackslib/src/net/api/getclaritymetadata.rs b/stackslib/src/net/api/getclaritymetadata.rs index 29ba1d6f4e..a6606fe62a 100644 --- a/stackslib/src/net/api/getclaritymetadata.rs +++ b/stackslib/src/net/api/getclaritymetadata.rs @@ -1,5 +1,4 @@ -// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020-2024 Stacks Open Internet Foundation +// Copyright (C) 2024 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index e360b7a72c..e36b13ca74 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -1,5 +1,4 @@ -// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020-2024 Stacks Open Internet Foundation +// Copyright (C) 2024 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/stackslib/src/net/api/tests/getclaritymetadata.rs b/stackslib/src/net/api/tests/getclaritymetadata.rs index 57baf705ce..de52b27f2d 100644 --- a/stackslib/src/net/api/tests/getclaritymetadata.rs +++ b/stackslib/src/net/api/tests/getclaritymetadata.rs @@ -1,5 +1,4 @@ -// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020-2024 Stacks Open Internet Foundation +// Copyright (C) 2024 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by From 74397d0841f8b842af55972d7b5e77d8611e3e34 Mon Sep 17 00:00:00 2001 From: Hugo CAILLARD <911307+hugocaillard@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:43:37 +0100 Subject: [PATCH 16/21] refactor: fix unbounded regex and add tests --- stackslib/src/net/api/getclaritymetadata.rs | 12 +++-- .../src/net/api/tests/getclaritymetadata.rs | 47 +++++++++++++++++-- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/stackslib/src/net/api/getclaritymetadata.rs b/stackslib/src/net/api/getclaritymetadata.rs index a6606fe62a..ee6ec96567 100644 --- a/stackslib/src/net/api/getclaritymetadata.rs +++ b/stackslib/src/net/api/getclaritymetadata.rs @@ -16,7 +16,9 @@ use clarity::vm::clarity::ClarityConnection; use clarity::vm::database::clarity_db::ContractDataVarName; use clarity::vm::database::StoreType; -use clarity::vm::representations::{CONTRACT_NAME_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING}; +use clarity::vm::representations::{ + CONTRACT_NAME_REGEX_STRING, MAX_STRING_LEN, STANDARD_PRINCIPAL_REGEX_STRING, +}; use clarity::vm::types::QualifiedContractIdentifier; use clarity::vm::ContractName; use lazy_static::lazy_static; @@ -35,10 +37,12 @@ use crate::net::httpcore::{ use crate::net::{Error as NetError, StacksNodeState, TipRequest}; lazy_static! { - static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = - "[a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?".into(); + static ref CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING: String = format!( + "([a-zA-Z]([a-zA-Z0-9]|[-_!?+<>=/*])*|[-+=/*]|[<>]=?){{1,{}}}", + MAX_STRING_LEN + ); static ref METADATA_KEY_REGEX_STRING: String = format!( - r"vm-metadata::(?P(\d{{1,2}}))::(?P(contract|contract-size|contract-src|contract-data-size|({})))", + r"vm-metadata::(?P(\d{{1,2}}))::(?P(contract|contract-size|contract-src|contract-data-size|{}))", *CLARITY_NAME_NO_BOUNDARIES_REGEX_STRING, ); } diff --git a/stackslib/src/net/api/tests/getclaritymetadata.rs b/stackslib/src/net/api/tests/getclaritymetadata.rs index de52b27f2d..47249eb290 100644 --- a/stackslib/src/net/api/tests/getclaritymetadata.rs +++ b/stackslib/src/net/api/tests/getclaritymetadata.rs @@ -15,8 +15,10 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions}; +use clarity::vm::database::{ClaritySerializable, DataMapMetadata, DataVariableMetadata}; +use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions, TypeSignature}; use clarity::vm::{ClarityName, ContractName}; +use serde_json::json; use stacks_common::codec::StacksMessageCodec; use stacks_common::types::chainstate::StacksAddress; use stacks_common::types::net::PeerHost; @@ -211,7 +213,7 @@ fn test_try_make_response() { let mut requests = vec![]; - // query existing + // query existing contract size metadata let request = StacksHttpRequest::new_getclaritymetadata( addr.into(), StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), @@ -221,16 +223,51 @@ fn test_try_make_response() { ); requests.push(request); + // query existing data var metadata + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::5::test-map".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + + // query existing data map metadata + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::6::bar".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + let mut responses = test_rpc(function_name!(), requests); - // latest data + // contract size metadata let response = responses.remove(0); - assert_eq!( response.preamble().get_canonical_stacks_tip_height(), Some(1) ); - let resp = response.decode_clarity_metadata_response().unwrap(); assert_eq!(resp.data, "1432"); + + // data map metadata + let response = responses.remove(0); + let resp = response.decode_clarity_metadata_response().unwrap(); + let expected = DataMapMetadata { + key_type: TypeSignature::UIntType, + value_type: TypeSignature::UIntType, + }; + assert_eq!(resp.data, expected.serialize()); + + // data var metadata + let response = responses.remove(0); + let resp = response.decode_clarity_metadata_response().unwrap(); + let expected = DataVariableMetadata { + value_type: TypeSignature::IntType, + }; + assert_eq!(resp.data, expected.serialize()); } From a94abfda195ca653b6344924eaa62fba7bf2e250 Mon Sep 17 00:00:00 2001 From: Hugo CAILLARD <911307+hugocaillard@users.noreply.github.com> Date: Mon, 18 Nov 2024 19:30:39 +0100 Subject: [PATCH 17/21] test: add getclaritymetadata test case for errors --- .../src/net/api/tests/getclaritymetadata.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/stackslib/src/net/api/tests/getclaritymetadata.rs b/stackslib/src/net/api/tests/getclaritymetadata.rs index 47249eb290..3bac8daf5d 100644 --- a/stackslib/src/net/api/tests/getclaritymetadata.rs +++ b/stackslib/src/net/api/tests/getclaritymetadata.rs @@ -243,6 +243,26 @@ fn test_try_make_response() { ); requests.push(request); + // query undeclared var metadata + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::6::non-existing-var".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + + // query invalid metadata key (wrong store type) + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::2::bar".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + let mut responses = test_rpc(function_name!(), requests); // contract size metadata @@ -270,4 +290,14 @@ fn test_try_make_response() { value_type: TypeSignature::IntType, }; assert_eq!(resp.data, expected.serialize()); + + // invalid metadata key + let response = responses.remove(0); + let (preamble, body) = response.destruct(); + assert_eq!(preamble.status_code, 404); + + // unknwnon data var + let response = responses.remove(0); + let (preamble, body) = response.destruct(); + assert_eq!(preamble.status_code, 400); } From c9d5295f1e668405609eb1c77ab3c363867c0581 Mon Sep 17 00:00:00 2001 From: Hugo CAILLARD <911307+hugocaillard@users.noreply.github.com> Date: Mon, 18 Nov 2024 20:50:06 +0100 Subject: [PATCH 18/21] tests: add failing case --- stackslib/src/net/api/tests/getclaritymetadata.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stackslib/src/net/api/tests/getclaritymetadata.rs b/stackslib/src/net/api/tests/getclaritymetadata.rs index 3bac8daf5d..9ef7e678d2 100644 --- a/stackslib/src/net/api/tests/getclaritymetadata.rs +++ b/stackslib/src/net/api/tests/getclaritymetadata.rs @@ -263,6 +263,16 @@ fn test_try_make_response() { ); requests.push(request); + // query existing contract size metadata + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::9::contract-size".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + let mut responses = test_rpc(function_name!(), requests); // contract size metadata From 8e8fd17efc14359100b9cb2aab8091485af57ee6 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 21 Nov 2024 12:50:43 -0500 Subject: [PATCH 19/21] chore: expand test coverage to include errors --- .../src/net/api/tests/getclaritymetadata.rs | 72 +++++++++++++++++-- stackslib/src/net/api/tests/mod.rs | 58 +++++++++------ 2 files changed, 103 insertions(+), 27 deletions(-) diff --git a/stackslib/src/net/api/tests/getclaritymetadata.rs b/stackslib/src/net/api/tests/getclaritymetadata.rs index 9ef7e678d2..495bbb514f 100644 --- a/stackslib/src/net/api/tests/getclaritymetadata.rs +++ b/stackslib/src/net/api/tests/getclaritymetadata.rs @@ -213,6 +213,16 @@ fn test_try_make_response() { let mut requests = vec![]; + // query invalid metadata key (wrong store type) + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::2::bar".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + // query existing contract size metadata let request = StacksHttpRequest::new_getclaritymetadata( addr.into(), @@ -223,7 +233,7 @@ fn test_try_make_response() { ); requests.push(request); - // query existing data var metadata + // query existing data map metadata let request = StacksHttpRequest::new_getclaritymetadata( addr.into(), StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), @@ -233,7 +243,7 @@ fn test_try_make_response() { ); requests.push(request); - // query existing data map metadata + // query existing data var metadata let request = StacksHttpRequest::new_getclaritymetadata( addr.into(), StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), @@ -243,22 +253,32 @@ fn test_try_make_response() { ); requests.push(request); - // query undeclared var metadata + // query existing data var metadata let request = StacksHttpRequest::new_getclaritymetadata( addr.into(), StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), "hello-world".try_into().unwrap(), - "vm-metadata::6::non-existing-var".to_string(), + "vm-metadata::6::bar".to_string(), TipRequest::UseLatestAnchoredTip, ); requests.push(request); - // query invalid metadata key (wrong store type) + // query existing data var metadata let request = StacksHttpRequest::new_getclaritymetadata( addr.into(), StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), "hello-world".try_into().unwrap(), - "vm-metadata::2::bar".to_string(), + "vm-metadata::6::bar".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + + // query undeclared var metadata + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::6::non-existing-var".to_string(), TipRequest::UseLatestAnchoredTip, ); requests.push(request); @@ -273,8 +293,23 @@ fn test_try_make_response() { ); requests.push(request); + // query invalid metadata key (wrong store type) + let request = StacksHttpRequest::new_getclaritymetadata( + addr.into(), + StacksAddress::from_string("ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R").unwrap(), + "hello-world".try_into().unwrap(), + "vm-metadata::2::bar".to_string(), + TipRequest::UseLatestAnchoredTip, + ); + requests.push(request); + let mut responses = test_rpc(function_name!(), requests); + // unknwnon data var + let response = responses.remove(0); + let (preamble, body) = response.destruct(); + assert_eq!(preamble.status_code, 400); + // contract size metadata let response = responses.remove(0); assert_eq!( @@ -301,11 +336,36 @@ fn test_try_make_response() { }; assert_eq!(resp.data, expected.serialize()); + // data var metadata + let response = responses.remove(0); + let resp = response.decode_clarity_metadata_response().unwrap(); + let expected = DataVariableMetadata { + value_type: TypeSignature::IntType, + }; + assert_eq!(resp.data, expected.serialize()); + + // data var metadata + let response = responses.remove(0); + let resp = response.decode_clarity_metadata_response().unwrap(); + let expected = DataVariableMetadata { + value_type: TypeSignature::IntType, + }; + assert_eq!(resp.data, expected.serialize()); + // invalid metadata key let response = responses.remove(0); let (preamble, body) = response.destruct(); assert_eq!(preamble.status_code, 404); + // contract size metadata + let response = responses.remove(0); + assert_eq!( + response.preamble().get_canonical_stacks_tip_height(), + Some(1) + ); + let resp = response.decode_clarity_metadata_response().unwrap(); + assert_eq!(resp.data, "1432"); + // unknwnon data var let response = responses.remove(0); let (preamble, body) = response.destruct(); diff --git a/stackslib/src/net/api/tests/mod.rs b/stackslib/src/net/api/tests/mod.rs index 7c191ca674..d63bec8c4d 100644 --- a/stackslib/src/net/api/tests/mod.rs +++ b/stackslib/src/net/api/tests/mod.rs @@ -1024,7 +1024,7 @@ impl<'a> TestRPC<'a> { peer_2.sortdb = Some(peer_2_sortdb); peer_2.stacks_node = Some(peer_2_stacks_node); - let mut peer_1_mempool = peer_1.mempool.take().unwrap(); + peer_2.mempool = Some(peer_2_mempool); convo_send_recv(&mut convo_2, &mut convo_1); @@ -1033,8 +1033,6 @@ impl<'a> TestRPC<'a> { // hack around the borrow-checker convo_send_recv(&mut convo_1, &mut convo_2); - peer_2.mempool = Some(peer_2_mempool); - let peer_1_sortdb = peer_1.sortdb.take().unwrap(); let mut peer_1_stacks_node = peer_1.stacks_node.take().unwrap(); @@ -1056,27 +1054,45 @@ impl<'a> TestRPC<'a> { .unwrap(); } - { - let rpc_args = RPCHandlerArgs::default(); - let mut node_state = StacksNodeState::new( - &mut peer_1.network, - &peer_1_sortdb, - &mut peer_1_stacks_node.chainstate, - &mut peer_1_mempool, - &rpc_args, - false, - ); - convo_1.chat(&mut node_state).unwrap(); - } - - convo_1.try_flush().unwrap(); - peer_1.sortdb = Some(peer_1_sortdb); peer_1.stacks_node = Some(peer_1_stacks_node); - peer_1.mempool = Some(peer_1_mempool); - // should have gotten a reply - let resp_opt = convo_1.try_get_response(); + let resp_opt = loop { + debug!("Peer 1 try get response"); + convo_send_recv(&mut convo_1, &mut convo_2); + { + let peer_1_sortdb = peer_1.sortdb.take().unwrap(); + let mut peer_1_stacks_node = peer_1.stacks_node.take().unwrap(); + let mut peer_1_mempool = peer_1.mempool.take().unwrap(); + + let rpc_args = RPCHandlerArgs::default(); + let mut node_state = StacksNodeState::new( + &mut peer_1.network, + &peer_1_sortdb, + &mut peer_1_stacks_node.chainstate, + &mut peer_1_mempool, + &rpc_args, + false, + ); + + convo_1.chat(&mut node_state).unwrap(); + + peer_1.sortdb = Some(peer_1_sortdb); + peer_1.stacks_node = Some(peer_1_stacks_node); + peer_1.mempool = Some(peer_1_mempool); + } + + convo_1.try_flush().unwrap(); + + info!("Try get response from request {:?}", &request); + + // should have gotten a reply + let resp_opt = convo_1.try_get_response(); + if resp_opt.is_some() { + break resp_opt; + } + }; + assert!(resp_opt.is_some()); let resp = resp_opt.unwrap(); From f3bd41de2dc388577cc045bacf4bf21d4bb75677 Mon Sep 17 00:00:00 2001 From: Hugo CAILLARD <911307+hugocaillard@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:36:42 +0100 Subject: [PATCH 20/21] tests: add rpc get marf value vm-account test case --- .../src/net/api/tests/getclaritymarfvalue.rs | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index e36b13ca74..8536fd563e 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -15,8 +15,8 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions}; -use clarity::vm::{ClarityName, ContractName}; +use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions, TypeSignature}; +use clarity::vm::{ClarityName, ContractName, Value}; use stacks_common::codec::StacksMessageCodec; use stacks_common::types::chainstate::{StacksAddress, TrieHash}; use stacks_common::types::net::PeerHost; @@ -126,6 +126,15 @@ fn test_try_make_response() { ); requests.push(request); + // query vm-account balance + let request = StacksHttpRequest::new_getclaritymarf( + addr.into(), + TrieHash::from_key("vm-account::ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R::19"), + TipRequest::UseLatestAnchoredTip, + true, + ); + requests.push(request); + let mut responses = test_rpc(function_name!(), requests); // existing data @@ -179,4 +188,16 @@ fn test_try_make_response() { let (preamble, body) = response.destruct(); assert_eq!(preamble.status_code, 404); + + // vm-account blaance + let response = responses.remove(0); + debug!( + "Response:\n{}\n", + std::str::from_utf8(&response.try_serialize().unwrap()).unwrap() + ); + + let resp = response.decode_data_var_response().unwrap(); + let balance = Value::try_deserialize_hex(&resp.data[2..], &TypeSignature::IntType, false); + assert_eq!(balance, Ok(Value::Int(256_000_000_000))); + assert!(resp.marf_proof.is_some()); } From b2bfb5ee3ec5e0a39330b57b7e704d38c0af7242 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 26 Nov 2024 13:30:58 -0500 Subject: [PATCH 21/21] chore: use STXBalance --- .../src/net/api/tests/getclaritymarfvalue.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/stackslib/src/net/api/tests/getclaritymarfvalue.rs b/stackslib/src/net/api/tests/getclaritymarfvalue.rs index 8536fd563e..7255d1ee99 100644 --- a/stackslib/src/net/api/tests/getclaritymarfvalue.rs +++ b/stackslib/src/net/api/tests/getclaritymarfvalue.rs @@ -15,6 +15,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use clarity::vm::database::{ClarityDeserializable, STXBalance}; use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions, TypeSignature}; use clarity::vm::{ClarityName, ContractName, Value}; use stacks_common::codec::StacksMessageCodec; @@ -149,7 +150,7 @@ fn test_try_make_response() { Some(1) ); - let resp = response.decode_data_var_response().unwrap(); + let resp = response.decode_clarity_marf_response().unwrap(); assert_eq!(resp.data, "0x0000000000000000000000000000000000"); assert!(resp.marf_proof.is_some()); @@ -165,7 +166,7 @@ fn test_try_make_response() { Some(1) ); - let resp = response.decode_data_var_response().unwrap(); + let resp = response.decode_clarity_marf_response().unwrap(); assert_eq!(resp.data, "0x0100000000000000000000000000000001"); assert!(resp.marf_proof.is_some()); @@ -189,15 +190,16 @@ fn test_try_make_response() { let (preamble, body) = response.destruct(); assert_eq!(preamble.status_code, 404); - // vm-account blaance + // vm-account balance let response = responses.remove(0); debug!( "Response:\n{}\n", std::str::from_utf8(&response.try_serialize().unwrap()).unwrap() ); - let resp = response.decode_data_var_response().unwrap(); - let balance = Value::try_deserialize_hex(&resp.data[2..], &TypeSignature::IntType, false); - assert_eq!(balance, Ok(Value::Int(256_000_000_000))); - assert!(resp.marf_proof.is_some()); + let resp = response.decode_clarity_marf_response().unwrap(); + let balance = STXBalance::deserialize(&resp.data[2..]).unwrap(); + + assert_eq!(balance.amount_unlocked(), 1_000_000_000); + assert_eq!(balance.amount_locked(), 0); }