From a45bf1ae574d93c280f14b1f1d76bf090f30c49b Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Wed, 22 Jan 2025 10:20:24 +0400 Subject: [PATCH 01/16] Add metadata extraction --- core/Cargo.lock | 1 + core/Cargo.toml | 3 +- core/lib/contract_verifier/Cargo.toml | 1 + .../src/contract_identifier.rs | 314 ++++++++++++++++++ core/lib/contract_verifier/src/lib.rs | 1 + 5 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 core/lib/contract_verifier/src/contract_identifier.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index 4744b424cea0..085711793667 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -11685,6 +11685,7 @@ dependencies = [ "anyhow", "assert_matches", "chrono", + "ciborium", "ethabi", "futures-util", "hex", diff --git a/core/Cargo.toml b/core/Cargo.toml index 1c071c2839fa..b3f049a353c6 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -170,6 +170,7 @@ serde = "1" serde_json = "1" serde_with = "1" serde_yaml = "0.9" +ciborium = "0.2" sha2 = "0.10.8" sha3 = "0.10.8" sqlx = "0.8.1" @@ -231,7 +232,7 @@ tokio-stream = "0.1.16" circuit_encodings = "=0.150.19" circuit_sequencer_api = "=0.150.19" circuit_definitions = "=0.150.19" -crypto_codegen = { package = "zksync_solidity_vk_codegen",version = "=0.30.12" } +crypto_codegen = { package = "zksync_solidity_vk_codegen", version = "=0.30.12" } kzg = { package = "zksync_kzg", version = "=0.150.19" } zk_evm = { version = "=0.133.0" } zk_evm_1_3_1 = { package = "zk_evm", version = "0.131.0-rc.2" } diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index c2cf97826561..fad6adc88aed 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -26,6 +26,7 @@ ethabi.workspace = true vise.workspace = true hex.workspace = true serde = { workspace = true, features = ["derive"] } +ciborium.workspace = true tempfile.workspace = true regex.workspace = true reqwest.workspace = true diff --git a/core/lib/contract_verifier/src/contract_identifier.rs b/core/lib/contract_verifier/src/contract_identifier.rs new file mode 100644 index 000000000000..6e723f1e7cb8 --- /dev/null +++ b/core/lib/contract_verifier/src/contract_identifier.rs @@ -0,0 +1,314 @@ +use serde::{Deserialize, Serialize}; +use zksync_types::{bytecode::BytecodeMarker, web3::keccak256, H256}; + +/// Metadata detected in the contract bytecode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) enum DetectedMetadata { + /// No metadata detected. + #[default] + None, + /// Keccek256 hash of the metadata detected (only for EraVM). + Keccak256, + /// CBOR metadata detected. + Cbor, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ContractIdentifier { + /// Marker of the bytecode of the contract. + /// This might be important, since the metadata can be different for EVM and EraVM, + /// e.g. `zksolc` [supports][zksolc_keccak] keccak256 hash of the metadata as an alternative to CBOR. + /// + /// [zksolc_keccak]: https://matter-labs.github.io/era-compiler-solidity/latest/02-command-line-interface.html#--metadata-hash + pub bytecode_marker: BytecodeMarker, + /// SHA3 (keccak256) hash of the full contract bytecode. + /// Can be used as an identifier of precise contract compilation. + pub bytecode_sha3: H256, + /// SHA3 (keccak256) hash of the contract bytecode without metadata (e.g. with either + /// CBOR or keccak256 metadata hash being stripped). + /// Can be absent if the contract bytecode doesn't have metadata. + pub bytecode_without_metadata_sha3: Option, + /// Detected metadata in the contract bytecode. + pub detected_metadata: DetectedMetadata, +} + +/// Possible values for the metadata hashes structure. +/// Details can be found here: https://docs.soliditylang.org/en/latest/metadata.html +/// +/// We're not really interested in the values here, we just want to make sure that we +/// can deserialize the metadata. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct CborMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + ipfs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + bzzr1: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + bzzr0: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + experimental: Option, + #[serde(skip_serializing_if = "Option::is_none")] + solc: Option>, +} + +impl ContractIdentifier { + pub fn from_bytecode(bytecode_marker: BytecodeMarker, bytecode: &[u8]) -> Self { + // Calculate the hash for bytecode with metadata. + let bytecode_sha3 = H256::from_slice(&keccak256(bytecode)); + let mut self_ = Self { + bytecode_marker, + bytecode_sha3, + bytecode_without_metadata_sha3: None, + detected_metadata: DetectedMetadata::None, + }; + + // For EraVM, the default metadata is keccak256 hash of the metadata. + // Try to use it as a default value (to be overridden if CBOR metadata is detected). + if bytecode_marker == BytecodeMarker::EraVm { + // For metadata, we might have padding: it takes either 32 or 64 bytes depending + // on whether the amount of words in the contract is odd, so we need to check + // if there is padding. + if bytecode.len() > 64 { + let bytecode_without_metadata = + if bytecode[bytecode.len() - 64..bytecode.len() - 32] == [0u8; 32] { + // Padding is present, strip it. + &bytecode[..bytecode.len() - 64] + } else { + // No padding, strip metadata only. + &bytecode[..bytecode.len() - 32] + }; + let hash = H256::from_slice(&keccak256(bytecode_without_metadata)); + // This could be overridden if CBOR metadata is detected. + self_.bytecode_without_metadata_sha3 = Some(hash); + self_.detected_metadata = DetectedMetadata::Keccak256; + } + } + + // Try to detect CBOR metadata. + + // Last two bytes is the length of the metadata in big endian. + if bytecode.len() < 2 { + return self_; + } + let metadata_length = + u16::from_be_bytes([bytecode[bytecode.len() - 2], bytecode[bytecode.len() - 1]]) + as usize; + // Including size + let full_metadata_length = metadata_length + 2; + + // Get slice for the metadata. + if bytecode.len() < full_metadata_length { + return self_; + } + let raw_metadata = &bytecode[bytecode.len() - full_metadata_length..bytecode.len() - 2]; + // Try decoding. We are not interested in the actual value. + let _metadata: CborMetadata = match ciborium::from_reader(raw_metadata) { + Ok(metadata) => metadata, + Err(_) => return self_, + }; + + // Strip metadata and calculate hash. + let bytecode_without_metadata = match bytecode_marker { + BytecodeMarker::Evm => { + // On EVM, there is no padding. + &bytecode[..bytecode.len() - full_metadata_length] + } + BytecodeMarker::EraVm => { + // On EraVM, there is padding: + // 1. We must align the metadata length to 32 bytes. + // 2. We may need to add 32 bytes of padding. + let aligned_metadata_length = (metadata_length + 31) / 32 * 32; + let full_aligned_metadata_length = aligned_metadata_length + 32; + if bytecode.len() < full_aligned_metadata_length { + // This shouldn't normally happen (metadata was deserialized correctly), + // so we just disable partial matching just in case. + self_.bytecode_without_metadata_sha3 = None; + return self_; + } + // Check if padding was added. + if bytecode[bytecode.len() - full_aligned_metadata_length + ..bytecode.len() - aligned_metadata_length] + == [0u8; 32] + { + // Padding was added, strip it. + &bytecode[..bytecode.len() - full_aligned_metadata_length] + } else { + // Padding wasn't added, strip metadata only. + &bytecode[..bytecode.len() - aligned_metadata_length] + } + } + }; + self_.bytecode_without_metadata_sha3 = + Some(H256::from_slice(&keccak256(bytecode_without_metadata))); + + self_.detected_metadata = DetectedMetadata::Cbor; + self_ + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn eravm_cbor_without_padding() { + // Sample contract with no methods, compiled from the root of monorepo with: + // ./etc/zksolc-bin/v1.5.8/zksolc --solc ./etc/solc-bin/zkVM-0.8.28-1.0.1/solc --metadata-hash ipfs --codegen yul test.sol --bin + // (Use `zkstack contract-verifier init` to download compilers) + let data = hex::decode("0000008003000039000000400030043f0000000100200190000000110000c13d0000000900100198000000190000613d000000000101043b0000000a011001970000000b0010009c000000190000c13d0000000001000416000000000001004b000000190000c13d000000000100041a000000800010043f0000000c010000410000001c0001042e0000000001000416000000000001004b000000190000c13d00000020010000390000010000100443000001200000044300000008010000410000001c0001042e00000000010000190000001d000104300000001b000004320000001c0001042e0000001d0001043000000000000000000000000000000000000000020000000000000000000000000000004000000100000000000000000000000000000000000000000000000000fffffffc000000000000000000000000ffffffff000000000000000000000000000000000000000000000000000000006d4ce63c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000008000000000000000000000000000000000000000000000000000000000a16469706673582212208acf048570dcc1c3ff41bf8f20376049a42ae8a471f2b2ae8c14d8b356d86d79002a").unwrap(); + let sha3 = keccak256(&data); + let full_metadata_len = 64; // (CBOR metadata + len bytes) + let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + + let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); + assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); + assert_eq!(identifier.detected_metadata, DetectedMetadata::Cbor); + assert_eq!( + identifier.bytecode_sha3, + H256::from_slice(&sha3), + "Incorrect bytecode hash" + ); + assert_eq!( + identifier.bytecode_without_metadata_sha3, + Some(H256::from_slice(&sha3_without_metadata)), + "Incorrect bytecode without metadata hash" + ); + } + + #[test] + fn eravm_cbor_with_padding() { + // Same as `eravm_cbor_without_padding` but now bytecode has padding. + let data = hex::decode("00000001002001900000000c0000613d0000008001000039000000400010043f0000000001000416000000000001004b0000000c0000c13d00000020010000390000010000100443000001200000044300000005010000410000000f0001042e000000000100001900000010000104300000000e000004320000000f0001042e0000001000010430000000000000000000000000000000000000000000000000000000020000000000000000000000000000004000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a1646970667358221220d5be4da510b089bb58fa6c65f0a387eef966bcf48671a24fb2b1bc7190842978002a").unwrap(); + let sha3 = keccak256(&data); + let full_metadata_len = 64 + 32; // (CBOR metadata + len bytes + padding) + let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + + let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); + assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); + assert_eq!(identifier.detected_metadata, DetectedMetadata::Cbor); + assert_eq!( + identifier.bytecode_sha3, + H256::from_slice(&sha3), + "Incorrect bytecode hash" + ); + assert_eq!( + identifier.bytecode_without_metadata_sha3, + Some(H256::from_slice(&sha3_without_metadata)), + "Incorrect bytecode without metadata hash" + ); + } + + #[test] + fn eravm_keccak_without_padding() { + // Sample contract with no methods, compiled from the root of monorepo with: + // ./etc/zksolc-bin/v1.5.8/zksolc --solc ./etc/solc-bin/zkVM-0.8.28-1.0.1/solc --metadata-hash keccak256 --codegen yul test.sol --bin + // (Use `zkstack contract-verifier init` to download compilers) + let data = hex::decode("00000001002001900000000c0000613d0000008001000039000000400010043f0000000001000416000000000001004b0000000c0000c13d00000020010000390000010000100443000001200000044300000005010000410000000f0001042e000000000100001900000010000104300000000e000004320000000f0001042e000000100001043000000000000000000000000000000000000000000000000000000002000000000000000000000000000000400000010000000000000000000a00e4a5f19bb139176aa501024c7032404c065bc0012897fefd9ebc7e9a7677").unwrap(); + let sha3 = keccak256(&data); + let full_metadata_len = 32; // (keccak only) + let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + + let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); + assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); + assert_eq!(identifier.detected_metadata, DetectedMetadata::Keccak256); + assert_eq!( + identifier.bytecode_sha3, + H256::from_slice(&sha3), + "Incorrect bytecode hash" + ); + assert_eq!( + identifier.bytecode_without_metadata_sha3, + Some(H256::from_slice(&sha3_without_metadata)), + "Incorrect bytecode without metadata hash" + ); + } + + #[test] + fn eravm_keccak_with_padding() { + // Same as `eravm_keccak_without_padding`, but now bytecode has padding. + let data = hex::decode("0000008003000039000000400030043f0000000100200190000000110000c13d0000000900100198000000190000613d000000000101043b0000000a011001970000000b0010009c000000190000c13d0000000001000416000000000001004b000000190000c13d000000000100041a000000800010043f0000000c010000410000001c0001042e0000000001000416000000000001004b000000190000c13d00000020010000390000010000100443000001200000044300000008010000410000001c0001042e00000000010000190000001d000104300000001b000004320000001c0001042e0000001d0001043000000000000000000000000000000000000000020000000000000000000000000000004000000100000000000000000000000000000000000000000000000000fffffffc000000000000000000000000ffffffff000000000000000000000000000000000000000000000000000000006d4ce63c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000080000000000000000000000000000000000000000000000000000000000000000000000000000000009b1f0a6172ae84051eca37db231c0fa6249349f4ddaf86a87474a587c19d946d").unwrap(); + let sha3 = keccak256(&data); + let full_metadata_len = 64; // (keccak + padding) + let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + + let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); + assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); + assert_eq!(identifier.detected_metadata, DetectedMetadata::Keccak256); + assert_eq!( + identifier.bytecode_sha3, + H256::from_slice(&sha3), + "Incorrect bytecode hash" + ); + assert_eq!( + identifier.bytecode_without_metadata_sha3, + Some(H256::from_slice(&sha3_without_metadata)), + "Incorrect bytecode without metadata hash" + ); + } + + #[test] + fn evm_none() { + // Sample contract with no methods, compiled from the root of monorepo with: + // ./etc/solc-bin/0.8.28/solc test.sol --bin --no-cbor-metadata + // (Use `zkstack contract-verifier init` to download compilers) + let data = hex::decode("6080604052348015600e575f5ffd5b50607980601a5f395ff3fe6080604052348015600e575f5ffd5b50600436106026575f3560e01c80636d4ce63c14602a575b5f5ffd5b60306044565b604051603b91906062565b60405180910390f35b5f5f54905090565b5f819050919050565b605c81604c565b82525050565b5f60208201905060735f8301846055565b9291505056").unwrap(); + let sha3 = keccak256(&data); + + let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); + assert_eq!(identifier.bytecode_marker, BytecodeMarker::Evm); + assert_eq!(identifier.detected_metadata, DetectedMetadata::None); + assert_eq!( + identifier.bytecode_sha3, + H256::from_slice(&sha3), + "Incorrect bytecode hash" + ); + assert_eq!( + identifier.bytecode_without_metadata_sha3, None, + "Incorrect bytecode without metadata hash" + ); + } + + #[test] + fn evm_cbor() { + // ./etc/solc-bin/0.8.28/solc test.sol --bin --metadata-hash ipfs + let ipfs_bytecode = "6080604052348015600e575f5ffd5b5060af80601a5f395ff3fe6080604052348015600e575f5ffd5b50600436106026575f3560e01c80636d4ce63c14602a575b5f5ffd5b60306044565b604051603b91906062565b60405180910390f35b5f5f54905090565b5f819050919050565b605c81604c565b82525050565b5f60208201905060735f8301846055565b9291505056fea2646970667358221220bca846db362b62d2eb9891565b12433410e0f6a634657d2c7d1e7469447e8ab564736f6c634300081c0033"; + // ./etc/solc-bin/0.8.28/solc test.sol --bin --metadata-hash none + // Note that cbor will still be included but will only have solc version. + let none_bytecode = "6080604052348015600e575f5ffd5b50608680601a5f395ff3fe6080604052348015600e575f5ffd5b50600436106026575f3560e01c80636d4ce63c14602a575b5f5ffd5b60306044565b604051603b91906062565b60405180910390f35b5f5f54905090565b5f819050919050565b605c81604c565b82525050565b5f60208201905060735f8301846055565b9291505056fea164736f6c634300081c000a"; + // ./etc/solc-bin/0.8.28/solc test.sol --bin --metadata-hash swarm + let swarm_bytecode = "6080604052348015600e575f5ffd5b5060ae80601a5f395ff3fe6080604052348015600e575f5ffd5b50600436106026575f3560e01c80636d4ce63c14602a575b5f5ffd5b60306044565b604051603b91906062565b60405180910390f35b5f5f54905090565b5f819050919050565b605c81604c565b82525050565b5f60208201905060735f8301846055565b9291505056fea265627a7a72315820c0def30c57166e97d6a58290213f3b0d1f83532e7a0371c8e2b6dba826bae46164736f6c634300081c0032"; + + // Different variations of the same contract, compiled with different metadata options. + // Tuples of (label, bytecode, size of metadata (including length)). + // Size of metadata can be found using https://playground.sourcify.dev/ + let test_vector = [ + ("ipfs", ipfs_bytecode, 51usize + 2), + ("none", none_bytecode, 10 + 2), + ("swarm", swarm_bytecode, 50 + 2), + ]; + + for (label, bytecode, full_metadata_len) in test_vector { + let data = hex::decode(bytecode).unwrap(); + let sha3 = keccak256(&data); + let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + + let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); + assert_eq!(identifier.bytecode_marker, BytecodeMarker::Evm, "{label}"); + assert_eq!( + identifier.detected_metadata, + DetectedMetadata::Cbor, + "{label}" + ); + assert_eq!( + identifier.bytecode_sha3, + H256::from_slice(&sha3), + "{label}: Incorrect bytecode hash" + ); + assert_eq!( + identifier.bytecode_without_metadata_sha3, + Some(H256::from_slice(&sha3_without_metadata)), + "{label}: Incorrect bytecode without metadata hash" + ); + } + } +} diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 43da4127b809..0379e0549f7f 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -30,6 +30,7 @@ use crate::{ }; mod compilers; +mod contract_identifier; pub mod error; mod metrics; mod resolver; From 8364ca4303ed80ada2489eb215c7995be6c032cf Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Wed, 22 Jan 2025 13:48:59 +0400 Subject: [PATCH 02/16] Make it easier to run tests locally --- .../contract_verifier/src/compilers/solc.rs | 19 ++- .../contract_verifier/src/compilers/vyper.rs | 19 ++- .../contract_verifier/src/compilers/zksolc.rs | 25 +++- .../src/compilers/zkvyper.rs | 12 +- .../src/contract_identifier.rs | 70 +++++++---- core/lib/contract_verifier/src/lib.rs | 13 +- .../lib/contract_verifier/src/resolver/mod.rs | 3 +- core/lib/contract_verifier/src/tests/mod.rs | 87 ++++++++++---- core/lib/contract_verifier/src/tests/real.rs | 113 +++++++++++++----- 9 files changed, 255 insertions(+), 106 deletions(-) diff --git a/core/lib/contract_verifier/src/compilers/solc.rs b/core/lib/contract_verifier/src/compilers/solc.rs index 10adcad3542e..e421b1eaf16c 100644 --- a/core/lib/contract_verifier/src/compilers/solc.rs +++ b/core/lib/contract_verifier/src/compilers/solc.rs @@ -3,12 +3,17 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio}; use anyhow::Context; use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; -use zksync_types::contract_verification_api::{ - CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, +use zksync_types::{ + bytecode::BytecodeMarker, + contract_verification_api::{ + CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, + }, }; use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson}; -use crate::{error::ContractVerifierError, resolver::Compiler}; +use crate::{ + contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::Compiler, +}; // Here and below, fields are public for testing purposes. #[derive(Debug)] @@ -103,7 +108,7 @@ impl Compiler for Solc { async fn compile( self: Box, input: SolcInput, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let mut command = tokio::process::Command::new(&self.path); let mut child = command .arg("--standard-json") @@ -128,7 +133,11 @@ impl Compiler for Solc { if output.status.success() { let output = serde_json::from_slice(&output.stdout) .context("zksolc output is not valid JSON")?; - parse_standard_json_output(&output, input.contract_name, input.file_name, true) + let output = + parse_standard_json_output(&output, input.contract_name, input.file_name, true)?; + let id = + ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode()); + Ok((output, id)) } else { Err(ContractVerifierError::CompilerError( "solc", diff --git a/core/lib/contract_verifier/src/compilers/vyper.rs b/core/lib/contract_verifier/src/compilers/vyper.rs index 59b950f9f17f..59aba4f31d91 100644 --- a/core/lib/contract_verifier/src/compilers/vyper.rs +++ b/core/lib/contract_verifier/src/compilers/vyper.rs @@ -3,12 +3,17 @@ use std::{collections::HashMap, mem, path::PathBuf, process::Stdio}; use anyhow::Context; use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; -use zksync_types::contract_verification_api::{ - CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, +use zksync_types::{ + bytecode::BytecodeMarker, + contract_verification_api::{ + CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, + }, }; use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson}; -use crate::{error::ContractVerifierError, resolver::Compiler}; +use crate::{ + contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::Compiler, +}; #[derive(Debug)] pub(crate) struct VyperInput { @@ -76,7 +81,7 @@ impl Compiler for Vyper { async fn compile( self: Box, mut input: VyperInput, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let mut command = tokio::process::Command::new(&self.path); let mut child = command .arg("--standard-json") @@ -103,7 +108,11 @@ impl Compiler for Vyper { if output.status.success() { let output = serde_json::from_slice(&output.stdout).context("vyper output is not valid JSON")?; - parse_standard_json_output(&output, input.contract_name, input.file_name, true) + let output = + parse_standard_json_output(&output, input.contract_name, input.file_name, true)?; + let id = + ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode()); + Ok((output, id)) } else { Err(ContractVerifierError::CompilerError( "vyper", diff --git a/core/lib/contract_verifier/src/compilers/zksolc.rs b/core/lib/contract_verifier/src/compilers/zksolc.rs index ff435e96aeb6..d76387dc6f1f 100644 --- a/core/lib/contract_verifier/src/compilers/zksolc.rs +++ b/core/lib/contract_verifier/src/compilers/zksolc.rs @@ -6,12 +6,16 @@ use semver::Version; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; -use zksync_types::contract_verification_api::{ - CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, +use zksync_types::{ + bytecode::BytecodeMarker, + contract_verification_api::{ + CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, + }, }; use super::{parse_standard_json_output, process_contract_name, Source}; use crate::{ + contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::{Compiler, CompilerPaths}, }; @@ -179,7 +183,7 @@ impl Compiler for ZkSolc { async fn compile( self: Box, input: ZkSolcInput, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let mut command = tokio::process::Command::new(&self.paths.zk); command.stdout(Stdio::piped()).stderr(Stdio::piped()); @@ -238,7 +242,13 @@ impl Compiler for ZkSolc { if output.status.success() { let output = serde_json::from_slice(&output.stdout) .context("zksolc output is not valid JSON")?; - parse_standard_json_output(&output, contract_name, file_name, false) + let output = + parse_standard_json_output(&output, contract_name, file_name, false)?; + let id = ContractIdentifier::from_bytecode( + BytecodeMarker::EraVm, + output.deployed_bytecode(), + ); + Ok((output, id)) } else { Err(ContractVerifierError::CompilerError( "zksolc", @@ -267,7 +277,12 @@ impl Compiler for ZkSolc { if output.status.success() { let output = String::from_utf8(output.stdout).context("zksolc output is not UTF-8")?; - Self::parse_single_file_yul_output(&output) + let output = Self::parse_single_file_yul_output(&output)?; + let id = ContractIdentifier::from_bytecode( + BytecodeMarker::EraVm, + output.deployed_bytecode(), + ); + Ok((output, id)) } else { Err(ContractVerifierError::CompilerError( "zksolc", diff --git a/core/lib/contract_verifier/src/compilers/zkvyper.rs b/core/lib/contract_verifier/src/compilers/zkvyper.rs index 4f7c10214f8a..1de1d0e83231 100644 --- a/core/lib/contract_verifier/src/compilers/zkvyper.rs +++ b/core/lib/contract_verifier/src/compilers/zkvyper.rs @@ -3,10 +3,11 @@ use std::{ffi::OsString, path, path::Path, process::Stdio}; use anyhow::Context as _; use tokio::{fs, io::AsyncWriteExt}; use zksync_queued_job_processor::async_trait; -use zksync_types::contract_verification_api::CompilationArtifacts; +use zksync_types::{bytecode::BytecodeMarker, contract_verification_api::CompilationArtifacts}; use super::VyperInput; use crate::{ + contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::{Compiler, CompilerPaths}, }; @@ -95,7 +96,7 @@ impl Compiler for ZkVyper { async fn compile( self: Box, input: VyperInput, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let mut command = tokio::process::Command::new(&self.paths.zk); if let Some(o) = input.optimizer_mode.as_ref() { command.arg("-O").arg(o); @@ -123,7 +124,12 @@ impl Compiler for ZkVyper { if output.status.success() { let output = serde_json::from_slice(&output.stdout) .context("zkvyper output is not valid JSON")?; - Self::parse_output(&output, input.contract_name) + let output = Self::parse_output(&output, input.contract_name)?; + let id = ContractIdentifier::from_bytecode( + BytecodeMarker::EraVm, + output.deployed_bytecode(), + ); + Ok((output, id)) } else { Err(ContractVerifierError::CompilerError( "zkvyper", diff --git a/core/lib/contract_verifier/src/contract_identifier.rs b/core/lib/contract_verifier/src/contract_identifier.rs index 6e723f1e7cb8..14904819ef21 100644 --- a/core/lib/contract_verifier/src/contract_identifier.rs +++ b/core/lib/contract_verifier/src/contract_identifier.rs @@ -1,26 +1,21 @@ use serde::{Deserialize, Serialize}; use zksync_types::{bytecode::BytecodeMarker, web3::keccak256, H256}; -/// Metadata detected in the contract bytecode. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub(crate) enum DetectedMetadata { - /// No metadata detected. - #[default] - None, - /// Keccek256 hash of the metadata detected (only for EraVM). - Keccak256, - /// CBOR metadata detected. - Cbor, -} - +/// An identifier of the contract bytecode. +/// This identifier can be used to detect different contracts that share the same sources, +/// even if they differ in bytecode verbatim (e.g. if the contract metadata is different). +/// +/// Identifier depends on the marker of the bytecode of the contract. +/// This might be important, since the metadata can be different for EVM and EraVM, +/// e.g. `zksolc` [supports][zksolc_keccak] keccak256 hash of the metadata as an alternative to CBOR. +/// +/// [zksolc_keccak]: https://matter-labs.github.io/era-compiler-solidity/latest/02-command-line-interface.html#--metadata-hash +// Note: there are missing opportunities here, e.g. Etherscan is able to detect the contracts +// that differ in creation bytecode and/or constructor arguments (for partial match). This is +// less relevant for ZKsync, since there is no concept of creation bytecode there; although +// this may become needed if we will extend the EVM support. #[derive(Debug, Clone, Copy)] pub(crate) struct ContractIdentifier { - /// Marker of the bytecode of the contract. - /// This might be important, since the metadata can be different for EVM and EraVM, - /// e.g. `zksolc` [supports][zksolc_keccak] keccak256 hash of the metadata as an alternative to CBOR. - /// - /// [zksolc_keccak]: https://matter-labs.github.io/era-compiler-solidity/latest/02-command-line-interface.html#--metadata-hash - pub bytecode_marker: BytecodeMarker, /// SHA3 (keccak256) hash of the full contract bytecode. /// Can be used as an identifier of precise contract compilation. pub bytecode_sha3: H256, @@ -32,6 +27,18 @@ pub(crate) struct ContractIdentifier { pub detected_metadata: DetectedMetadata, } +/// Metadata detected in the contract bytecode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) enum DetectedMetadata { + /// No metadata detected. + #[default] + None, + /// Keccek256 hash of the metadata detected (only for EraVM). + Keccak256, + /// CBOR metadata detected. + Cbor, +} + /// Possible values for the metadata hashes structure. /// Details can be found here: https://docs.soliditylang.org/en/latest/metadata.html /// @@ -56,7 +63,6 @@ impl ContractIdentifier { // Calculate the hash for bytecode with metadata. let bytecode_sha3 = H256::from_slice(&keccak256(bytecode)); let mut self_ = Self { - bytecode_marker, bytecode_sha3, bytecode_without_metadata_sha3: None, detected_metadata: DetectedMetadata::None, @@ -161,7 +167,6 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); assert_eq!(identifier.detected_metadata, DetectedMetadata::Cbor); assert_eq!( identifier.bytecode_sha3, @@ -184,7 +189,6 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); assert_eq!(identifier.detected_metadata, DetectedMetadata::Cbor); assert_eq!( identifier.bytecode_sha3, @@ -209,7 +213,6 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); assert_eq!(identifier.detected_metadata, DetectedMetadata::Keccak256); assert_eq!( identifier.bytecode_sha3, @@ -232,7 +235,6 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.bytecode_marker, BytecodeMarker::EraVm); assert_eq!(identifier.detected_metadata, DetectedMetadata::Keccak256); assert_eq!( identifier.bytecode_sha3, @@ -246,6 +248,26 @@ mod tests { ); } + #[test] + fn eravm_too_short_bytecode() { + // Random short bytecode + let data = hex::decode("0000008003000039000000400030043f0000000100200190000000110000c13d") + .unwrap(); + let sha3 = keccak256(&data); + + let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); + assert_eq!(identifier.detected_metadata, DetectedMetadata::None); + assert_eq!( + identifier.bytecode_sha3, + H256::from_slice(&sha3), + "Incorrect bytecode hash" + ); + assert_eq!( + identifier.bytecode_without_metadata_sha3, None, + "Incorrect bytecode without metadata hash" + ); + } + #[test] fn evm_none() { // Sample contract with no methods, compiled from the root of monorepo with: @@ -255,7 +277,6 @@ mod tests { let sha3 = keccak256(&data); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); - assert_eq!(identifier.bytecode_marker, BytecodeMarker::Evm); assert_eq!(identifier.detected_metadata, DetectedMetadata::None); assert_eq!( identifier.bytecode_sha3, @@ -293,7 +314,6 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); - assert_eq!(identifier.bytecode_marker, BytecodeMarker::Evm, "{label}"); assert_eq!( identifier.detected_metadata, DetectedMetadata::Cbor, diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 0379e0549f7f..6c9ba57efa68 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -8,6 +8,7 @@ use std::{ use anyhow::Context as _; use chrono::Utc; +use contract_identifier::ContractIdentifier; use ethabi::{Contract, Token}; use resolver::{GitHubCompilerResolver, ResolverMultiplexer}; use tokio::time; @@ -245,7 +246,7 @@ impl ContractVerifier { let bytecode_marker = BytecodeMarker::new(deployed_contract.bytecode_hash) .context("unknown bytecode kind")?; - let artifacts = self.compile(request.req.clone(), bytecode_marker).await?; + let (artifacts, identifier) = self.compile(request.req.clone(), bytecode_marker).await?; let constructor_args = match bytecode_marker { BytecodeMarker::EraVm => self .decode_era_vm_constructor_args(&deployed_contract, request.req.contract_address)?, @@ -306,7 +307,7 @@ impl ContractVerifier { &self, version: &ZkCompilerVersions, req: VerificationIncomingRequest, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let zksolc = self.compiler_resolver.resolve_zksolc(version).await?; tracing::debug!(?zksolc, ?version, "resolved compiler"); let input = ZkSolc::build_input(req)?; @@ -320,7 +321,7 @@ impl ContractVerifier { &self, version: &ZkCompilerVersions, req: VerificationIncomingRequest, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let zkvyper = self.compiler_resolver.resolve_zkvyper(version).await?; tracing::debug!(?zkvyper, ?version, "resolved compiler"); let input = VyperInput::new(req)?; @@ -333,7 +334,7 @@ impl ContractVerifier { &self, version: &str, req: VerificationIncomingRequest, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let solc = self.compiler_resolver.resolve_solc(version).await?; tracing::debug!(?solc, ?req.compiler_versions, "resolved compiler"); let input = Solc::build_input(req)?; @@ -347,7 +348,7 @@ impl ContractVerifier { &self, version: &str, req: VerificationIncomingRequest, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let vyper = self.compiler_resolver.resolve_vyper(version).await?; tracing::debug!(?vyper, ?req.compiler_versions, "resolved compiler"); let input = VyperInput::new(req)?; @@ -362,7 +363,7 @@ impl ContractVerifier { &self, req: VerificationIncomingRequest, bytecode_marker: BytecodeMarker, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { let compiler_type = req.source_code_data.compiler_type(); let compiler_type_by_versions = req.compiler_versions.compiler_type(); if compiler_type != compiler_type_by_versions { diff --git a/core/lib/contract_verifier/src/resolver/mod.rs b/core/lib/contract_verifier/src/resolver/mod.rs index a9d2bcf9049d..4a93029a6de0 100644 --- a/core/lib/contract_verifier/src/resolver/mod.rs +++ b/core/lib/contract_verifier/src/resolver/mod.rs @@ -13,6 +13,7 @@ use zksync_types::contract_verification_api::CompilationArtifacts; pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver}; use crate::{ compilers::{SolcInput, VyperInput, ZkSolcInput}, + contract_identifier::ContractIdentifier, error::ContractVerifierError, ZkCompilerVersions, }; @@ -161,7 +162,7 @@ pub(crate) trait Compiler: Send + fmt::Debug { async fn compile( self: Box, input: In, - ) -> Result; + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError>; } #[derive(Debug)] diff --git a/core/lib/contract_verifier/src/tests/mod.rs b/core/lib/contract_verifier/src/tests/mod.rs index 2ffb51ceb30a..3178b518b8a2 100644 --- a/core/lib/contract_verifier/src/tests/mod.rs +++ b/core/lib/contract_verifier/src/tests/mod.rs @@ -208,8 +208,11 @@ async fn mock_deployment_inner( .unwrap(); } -type SharedMockFn = - Arc Result + Send + Sync>; +type SharedMockFn = Arc< + dyn Fn(In) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> + + Send + + Sync, +>; #[derive(Clone)] struct MockCompilerResolver { @@ -227,7 +230,10 @@ impl fmt::Debug for MockCompilerResolver { impl MockCompilerResolver { fn zksolc( - zksolc: impl Fn(ZkSolcInput) -> CompilationArtifacts + 'static + Send + Sync, + zksolc: impl Fn(ZkSolcInput) -> (CompilationArtifacts, ContractIdentifier) + + 'static + + Send + + Sync, ) -> Self { Self { zksolc: Arc::new(move |input| Ok(zksolc(input))), @@ -235,7 +241,9 @@ impl MockCompilerResolver { } } - fn solc(solc: impl Fn(SolcInput) -> CompilationArtifacts + 'static + Send + Sync) -> Self { + fn solc( + solc: impl Fn(SolcInput) -> (CompilationArtifacts, ContractIdentifier) + 'static + Send + Sync, + ) -> Self { Self { solc: Arc::new(move |input| Ok(solc(input))), zksolc: Arc::new(|input| panic!("unexpected zksolc call: {input:?}")), @@ -248,7 +256,7 @@ impl Compiler for MockCompilerResolver { async fn compile( self: Box, input: ZkSolcInput, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { (self.zksolc)(input) } } @@ -258,7 +266,7 @@ impl Compiler for MockCompilerResolver { async fn compile( self: Box, input: SolcInput, - ) -> Result { + ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { (self.solc)(input) } } @@ -404,11 +412,14 @@ async fn contract_verifier_basics(contract: TestContract) { let source = input.sources.values().next().unwrap(); assert!(source.content.contains("contract Counter"), "{source:?}"); - CompilationArtifacts { + let artifacts = CompilationArtifacts { bytecode: vec![0; 32], deployed_bytecode: None, abi: counter_contract_abi(), - } + }; + let contract_id = + ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, artifacts.deployed_bytecode()); + (artifacts, contract_id) }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), @@ -522,13 +533,15 @@ async fn verifying_evm_bytecode(contract: TestContract) { deployed_bytecode: Some(deployed_bytecode), abi: counter_contract_abi(), }; + let contract_id = + ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, artifacts.deployed_bytecode()); let mock_resolver = MockCompilerResolver::solc(move |input| { assert_eq!(input.standard_json.language, "Solidity"); assert_eq!(input.standard_json.sources.len(), 1); let source = input.standard_json.sources.values().next().unwrap(); assert!(source.content.contains("contract Counter"), "{source:?}"); - artifacts.clone() + (artifacts.clone(), contract_id) }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), @@ -559,10 +572,15 @@ async fn bytecode_mismatch_error() { .await .unwrap(); - let mock_resolver = MockCompilerResolver::zksolc(|_| CompilationArtifacts { - bytecode: vec![0; 32], - deployed_bytecode: None, - abi: counter_contract_abi(), + let mock_resolver = MockCompilerResolver::zksolc(|_| { + let artifacts = CompilationArtifacts { + bytecode: vec![0; 32], + deployed_bytecode: None, + abi: counter_contract_abi(), + }; + let contract_id = + ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, artifacts.deployed_bytecode()); + (artifacts, contract_id) }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), @@ -639,15 +657,29 @@ async fn args_mismatch_error(contract: TestContract, bytecode_kind: BytecodeMark .unwrap(); let mock_resolver = match bytecode_kind { - BytecodeMarker::EraVm => MockCompilerResolver::zksolc(move |_| CompilationArtifacts { - bytecode: bytecode.clone(), - deployed_bytecode: None, - abi: counter_contract_abi(), + BytecodeMarker::EraVm => MockCompilerResolver::zksolc(move |_| { + let artifacts = CompilationArtifacts { + bytecode: bytecode.clone(), + deployed_bytecode: None, + abi: counter_contract_abi(), + }; + let contract_id = ContractIdentifier::from_bytecode( + BytecodeMarker::EraVm, + artifacts.deployed_bytecode(), + ); + (artifacts, contract_id) }), - BytecodeMarker::Evm => MockCompilerResolver::solc(move |_| CompilationArtifacts { - bytecode: vec![3_u8; 48], - deployed_bytecode: Some(bytecode.clone()), - abi: counter_contract_abi(), + BytecodeMarker::Evm => MockCompilerResolver::solc(move |_| { + let artifacts = CompilationArtifacts { + bytecode: vec![3_u8; 48], + deployed_bytecode: Some(bytecode.clone()), + abi: counter_contract_abi(), + }; + let contract_id = ContractIdentifier::from_bytecode( + BytecodeMarker::Evm, + artifacts.deployed_bytecode(), + ); + (artifacts, contract_id) }), }; let verifier = ContractVerifier::with_resolver( @@ -708,10 +740,15 @@ async fn creation_bytecode_mismatch() { .await .unwrap(); - let mock_resolver = MockCompilerResolver::solc(move |_| CompilationArtifacts { - bytecode: vec![4; 20], // differs from `creation_bytecode` - deployed_bytecode: Some(deployed_bytecode.clone()), - abi: counter_contract_abi(), + let mock_resolver = MockCompilerResolver::solc(move |_| { + let artifacts = CompilationArtifacts { + bytecode: vec![4; 20], // differs from `creation_bytecode` + deployed_bytecode: Some(deployed_bytecode.clone()), + abi: counter_contract_abi(), + }; + let contract_id = + ContractIdentifier::from_bytecode(BytecodeMarker::Evm, artifacts.deployed_bytecode()); + (artifacts, contract_id) }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index ba7615528e15..4294e423a546 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -1,6 +1,11 @@ //! Tests using real compiler toolchains. Should be prepared by calling `zkstack contract-verifier init` //! with at least one `solc` and `zksolc` version. If there are no compilers, the tests will be ignored //! unless the `RUN_CONTRACT_VERIFICATION_TEST` env var is set to `true`, in which case the tests will fail. +//! +//! You can install the compilers to run these tests with the following command: +//! ``` +//! zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only +//! ``` use std::{env, sync::Arc, time::Duration}; @@ -19,38 +24,77 @@ impl Toolchain { const ALL: [Self; 2] = [Self::Solidity, Self::Vyper]; } +// The tests may expect specific compiler versions (e.g. contracts won't compile with Vyper 0.4.0), +// so we hardcode versions. +const ZKSOLC_VERSION: &str = "v1.5.3"; +const ERA_VM_SOLC_VERSION: &str = "0.8.26-1.0.1"; +const SOLC_VERSION: &str = "0.8.26"; +const VYPER_VERSION: &str = "v0.3.10"; +const ZKVYPER_VERSION: &str = "v1.5.4"; + #[derive(Debug, Clone)] struct TestCompilerVersions { solc: String, + eravm_solc: String, zksolc: String, vyper: String, zkvyper: String, } impl TestCompilerVersions { - fn new(versions: SupportedCompilerVersions) -> Option { - let solc = versions - .solc - .into_iter() - .find(|ver| !ver.starts_with("zkVM"))?; - Some(Self { - solc, - zksolc: versions.zksolc.into_iter().next()?, - vyper: versions.vyper.into_iter().next()?, - zkvyper: versions.zkvyper.into_iter().next()?, + fn new(versions: SupportedCompilerVersions) -> anyhow::Result { + // Stored compilers for our fork are prefixed with `zkVM-`. + let eravm_solc = format!("zkVM-{ERA_VM_SOLC_VERSION}"); + // Stored compilers for vyper do not have `v` prefix. + let vyper = VYPER_VERSION.strip_prefix("v").unwrap().to_owned(); + anyhow::ensure!( + versions.solc.contains(SOLC_VERSION), + "Expected solc version {} to be installed, but it is not", + SOLC_VERSION + ); + anyhow::ensure!( + versions.solc.contains(&eravm_solc), + "Expected era-vm solc version {} to be installed, but it is not", + ERA_VM_SOLC_VERSION + ); + anyhow::ensure!( + versions.zksolc.contains(ZKSOLC_VERSION), + "Expected zksolc version {} to be installed, but it is not", + ZKSOLC_VERSION + ); + anyhow::ensure!( + versions.vyper.contains(&vyper), + "Expected vyper version {} to be installed, but it is not", + VYPER_VERSION + ); + anyhow::ensure!( + versions.zkvyper.contains(ZKVYPER_VERSION), + "Expected zkvyper version {} to be installed, but it is not", + ZKVYPER_VERSION + ); + + Ok(Self { + solc: SOLC_VERSION.to_owned(), + eravm_solc, + zksolc: ZKSOLC_VERSION.to_owned(), + vyper, + zkvyper: ZKVYPER_VERSION.to_owned(), }) } fn zksolc(self) -> ZkCompilerVersions { ZkCompilerVersions { - base: self.solc, + base: self.eravm_solc, zk: self.zksolc, } } fn solc_for_api(self, bytecode_kind: BytecodeMarker) -> CompilerVersions { CompilerVersions::Solc { - compiler_solc_version: self.solc, + compiler_solc_version: match bytecode_kind { + BytecodeMarker::Evm => self.solc, + BytecodeMarker::EraVm => self.eravm_solc, + }, compiler_zksolc_version: match bytecode_kind { BytecodeMarker::Evm => None, BytecodeMarker::EraVm => Some(self.zksolc), @@ -76,32 +120,39 @@ impl TestCompilerVersions { } } -async fn checked_env_resolver() -> Option<(EnvCompilerResolver, TestCompilerVersions)> { +async fn checked_env_resolver() -> anyhow::Result<(EnvCompilerResolver, TestCompilerVersions)> { let compiler_resolver = EnvCompilerResolver::default(); - let supported_compilers = compiler_resolver.supported_versions().await.ok()?; - Some(( + let supported_compilers = compiler_resolver.supported_versions().await?; + Ok(( compiler_resolver, TestCompilerVersions::new(supported_compilers)?, )) } -fn assert_no_compilers_expected() { +fn assert_no_compilers_expected(err: anyhow::Error) { + let error_message = format!( + "Expected pre-installed compilers since `RUN_CONTRACT_VERIFICATION_TEST=true`, but at least one compiler is not installed.\n \ + Detail: {}\n\n \ + Use the following command to install compilers:\n \ + zkstack contract-verifier init --zksolc-version={} --zkvyper-version={} --solc-version={} --vyper-version={} --era-vm-solc-version={} --only", + err, ZKSOLC_VERSION, ZKVYPER_VERSION, SOLC_VERSION, VYPER_VERSION, ERA_VM_SOLC_VERSION + ); + assert_ne!( env::var("RUN_CONTRACT_VERIFICATION_TEST").ok().as_deref(), Some("true"), - "Expected pre-installed compilers since `RUN_CONTRACT_VERIFICATION_TEST=true`, but they are not installed. \ - Use `zkstack contract-verifier init` to install compilers" + "{error_message}" ); - println!("No compilers found, skipping the test"); + println!("At least one compiler is not found, skipping the test"); } /// Simplifies initializing real compiler resolver in tests. macro_rules! real_resolver { () => { match checked_env_resolver().await { - Some(resolver_and_versions) => resolver_and_versions, - None => { - assert_no_compilers_expected(); + Ok(resolver_and_versions) => resolver_and_versions, + Err(err) => { + assert_no_compilers_expected(err); return; } } @@ -126,7 +177,7 @@ async fn using_real_zksolc(specify_contract_file: bool) { } let input = ZkSolc::build_input(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); validate_bytecode(&output.bytecode).unwrap(); assert_eq!(output.abi, counter_contract_abi()); @@ -170,7 +221,7 @@ async fn using_standalone_solc(specify_contract_file: bool) { } let input = Solc::build_input(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, counter_contract_abi()); @@ -253,7 +304,7 @@ async fn compiling_yul_with_zksolc() { let compiler = compiler_resolver.resolve_zksolc(&version).await.unwrap(); let req = test_yul_request(supported_compilers.solc_for_api(BytecodeMarker::EraVm)); let input = ZkSolc::build_input(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); assert!(!output.bytecode.is_empty()); assert!(output.deployed_bytecode.is_none()); @@ -271,7 +322,7 @@ async fn compiling_standalone_yul() { compiler_zksolc_version: None, }); let input = Solc::build_input(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); assert!(!output.bytecode.is_empty()); assert_ne!(output.deployed_bytecode.unwrap(), output.bytecode); @@ -321,7 +372,7 @@ async fn using_real_zkvyper(specify_contract_file: bool) { BytecodeMarker::EraVm, ); let input = VyperInput::new(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); validate_bytecode(&output.bytecode).unwrap(); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -346,7 +397,7 @@ async fn using_standalone_vyper(specify_contract_file: bool) { BytecodeMarker::Evm, ); let input = VyperInput::new(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -366,7 +417,7 @@ async fn using_standalone_vyper_without_optimization() { ); req.optimization_used = false; let input = VyperInput::new(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -387,7 +438,7 @@ async fn using_standalone_vyper_with_code_size_optimization() { req.optimization_used = true; req.optimizer_mode = Some("codesize".to_owned()); let input = VyperInput::new(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let (output, _) = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -439,7 +490,7 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai }, }; let address = Address::repeat_byte(1); - let output = match (bytecode_kind, toolchain) { + let (output, _) = match (bytecode_kind, toolchain) { (BytecodeMarker::EraVm, Toolchain::Solidity) => { let compiler = compiler_resolver .resolve_zksolc(&supported_compilers.zksolc()) From 410b6645936d6a146a544a10e382ae7eb02dbe42 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 24 Jan 2025 10:49:00 +0400 Subject: [PATCH 03/16] Fetch partial matches from the DB --- core/Cargo.lock | 2 +- core/bin/verified_sources_fetcher/src/main.rs | 2 +- core/lib/contract_verifier/Cargo.toml | 1 - .../contract_verifier/src/compilers/mod.rs | 2 +- .../contract_verifier/src/compilers/solc.rs | 9 +- .../contract_verifier/src/compilers/vyper.rs | 9 +- .../contract_verifier/src/compilers/zksolc.rs | 6 +- .../src/compilers/zkvyper.rs | 6 +- core/lib/contract_verifier/src/lib.rs | 66 +++-- .../lib/contract_verifier/src/resolver/mod.rs | 5 +- core/lib/contract_verifier/src/tests/mod.rs | 2 +- ...cb2685cda1ae7ecca83062ede7320c3b4a427.json | 15 -- ...bffc78d92adb3c1e3ca60b12163e38c67047e.json | 22 -- ...3ed5f1e6eac5b71232c784abd4d4cd8677805.json | 17 ++ ...432d1e8cb5702857d9cbfa617f985e62eaf4e.json | 20 ++ ...072dcee6e6f8439e6b43eebd6df5563a4d0b9.json | 35 +++ ...9ddd85f5bfb18df281a36456f2d67db83be48.json | 20 ++ ...48743ffed494316750be3b0ffb10b2fc09e93.json | 22 ++ ...2800_contract-verifier-new-schema.down.sql | 3 + ...102800_contract-verifier-new-schema.up.sql | 13 + core/lib/dal/src/contract_verification_dal.rs | 231 +++++++++++++++--- .../models/storage_verification_request.rs | 2 +- core/lib/types/Cargo.toml | 1 + .../api.rs} | 16 ++ .../contract_identifier.rs | 118 ++++++--- .../types/src/contract_verification/mod.rs | 2 + core/lib/types/src/lib.rs | 2 +- .../src/api_impl.rs | 2 +- .../contract_verification_server/src/cache.rs | 2 +- .../contract_verification_server/src/tests.rs | 2 +- prover/Cargo.lock | 38 +++ zkstack_cli/Cargo.lock | 38 +++ 32 files changed, 587 insertions(+), 144 deletions(-) delete mode 100644 core/lib/dal/.sqlx/query-1823e1ac602ce4ba1db06543af9cb2685cda1ae7ecca83062ede7320c3b4a427.json delete mode 100644 core/lib/dal/.sqlx/query-2d0c2e9ec4187641baef8a33229bffc78d92adb3c1e3ca60b12163e38c67047e.json create mode 100644 core/lib/dal/.sqlx/query-349d41c8ce192e82152e9d254c23ed5f1e6eac5b71232c784abd4d4cd8677805.json create mode 100644 core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json create mode 100644 core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json create mode 100644 core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json create mode 100644 core/lib/dal/.sqlx/query-daa2ad4ebde17808e059aa6bcf148743ffed494316750be3b0ffb10b2fc09e93.json create mode 100644 core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.down.sql create mode 100644 core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql rename core/lib/types/src/{contract_verification_api.rs => contract_verification/api.rs} (95%) rename core/lib/{contract_verifier/src => types/src/contract_verification}/contract_identifier.rs (83%) create mode 100644 core/lib/types/src/contract_verification/mod.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index b449aae61b3f..cab261692009 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -11685,7 +11685,6 @@ dependencies = [ "anyhow", "assert_matches", "chrono", - "ciborium", "ethabi", "futures-util", "hex", @@ -13017,6 +13016,7 @@ dependencies = [ "bincode", "blake2 0.10.6", "chrono", + "ciborium", "derive_more 1.0.0", "hex", "itertools 0.10.5", diff --git a/core/bin/verified_sources_fetcher/src/main.rs b/core/bin/verified_sources_fetcher/src/main.rs index 981eebf4a706..5ddf65fd580d 100644 --- a/core/bin/verified_sources_fetcher/src/main.rs +++ b/core/bin/verified_sources_fetcher/src/main.rs @@ -3,7 +3,7 @@ use std::io::Write; use zksync_config::configs::DatabaseSecrets; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_env_config::FromEnv; -use zksync_types::contract_verification_api::SourceCodeData; +use zksync_types::contract_verification::api::SourceCodeData; #[tokio::main] async fn main() { diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index fad6adc88aed..c2cf97826561 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -26,7 +26,6 @@ ethabi.workspace = true vise.workspace = true hex.workspace = true serde = { workspace = true, features = ["derive"] } -ciborium.workspace = true tempfile.workspace = true regex.workspace = true reqwest.workspace = true diff --git a/core/lib/contract_verifier/src/compilers/mod.rs b/core/lib/contract_verifier/src/compilers/mod.rs index c82a6575ee4c..cbaf9d2225bc 100644 --- a/core/lib/contract_verifier/src/compilers/mod.rs +++ b/core/lib/contract_verifier/src/compilers/mod.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use anyhow::Context as _; use serde::{Deserialize, Serialize}; -use zksync_types::contract_verification_api::CompilationArtifacts; +use zksync_types::contract_verification::api::CompilationArtifacts; pub(crate) use self::{ solc::{Solc, SolcInput}, diff --git a/core/lib/contract_verifier/src/compilers/solc.rs b/core/lib/contract_verifier/src/compilers/solc.rs index e421b1eaf16c..fc165227924a 100644 --- a/core/lib/contract_verifier/src/compilers/solc.rs +++ b/core/lib/contract_verifier/src/compilers/solc.rs @@ -5,15 +5,14 @@ use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; use zksync_types::{ bytecode::BytecodeMarker, - contract_verification_api::{ - CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, + contract_verification::{ + api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest}, + contract_identifier::ContractIdentifier, }, }; use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson}; -use crate::{ - contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::Compiler, -}; +use crate::{error::ContractVerifierError, resolver::Compiler}; // Here and below, fields are public for testing purposes. #[derive(Debug)] diff --git a/core/lib/contract_verifier/src/compilers/vyper.rs b/core/lib/contract_verifier/src/compilers/vyper.rs index 59aba4f31d91..ea9500da1557 100644 --- a/core/lib/contract_verifier/src/compilers/vyper.rs +++ b/core/lib/contract_verifier/src/compilers/vyper.rs @@ -5,15 +5,14 @@ use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; use zksync_types::{ bytecode::BytecodeMarker, - contract_verification_api::{ - CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, + contract_verification::{ + api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest}, + contract_identifier::ContractIdentifier, }, }; use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson}; -use crate::{ - contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::Compiler, -}; +use crate::{error::ContractVerifierError, resolver::Compiler}; #[derive(Debug)] pub(crate) struct VyperInput { diff --git a/core/lib/contract_verifier/src/compilers/zksolc.rs b/core/lib/contract_verifier/src/compilers/zksolc.rs index d76387dc6f1f..4dc344a5a175 100644 --- a/core/lib/contract_verifier/src/compilers/zksolc.rs +++ b/core/lib/contract_verifier/src/compilers/zksolc.rs @@ -8,14 +8,14 @@ use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; use zksync_types::{ bytecode::BytecodeMarker, - contract_verification_api::{ - CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, + contract_verification::{ + api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest}, + contract_identifier::ContractIdentifier, }, }; use super::{parse_standard_json_output, process_contract_name, Source}; use crate::{ - contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::{Compiler, CompilerPaths}, }; diff --git a/core/lib/contract_verifier/src/compilers/zkvyper.rs b/core/lib/contract_verifier/src/compilers/zkvyper.rs index 1de1d0e83231..2a24c7cd2b02 100644 --- a/core/lib/contract_verifier/src/compilers/zkvyper.rs +++ b/core/lib/contract_verifier/src/compilers/zkvyper.rs @@ -3,11 +3,13 @@ use std::{ffi::OsString, path, path::Path, process::Stdio}; use anyhow::Context as _; use tokio::{fs, io::AsyncWriteExt}; use zksync_queued_job_processor::async_trait; -use zksync_types::{bytecode::BytecodeMarker, contract_verification_api::CompilationArtifacts}; +use zksync_types::{ + bytecode::BytecodeMarker, + contract_verification::{api::CompilationArtifacts, contract_identifier::ContractIdentifier}, +}; use super::VyperInput; use crate::{ - contract_identifier::ContractIdentifier, error::ContractVerifierError, resolver::{Compiler, CompilerPaths}, }; diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 6c9ba57efa68..4626897b264a 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -8,7 +8,6 @@ use std::{ use anyhow::Context as _; use chrono::Utc; -use contract_identifier::ContractIdentifier; use ethabi::{Contract, Token}; use resolver::{GitHubCompilerResolver, ResolverMultiplexer}; use tokio::time; @@ -16,9 +15,12 @@ use zksync_dal::{contract_verification_dal::DeployedContractData, ConnectionPool use zksync_queued_job_processor::{async_trait, JobProcessor}; use zksync_types::{ bytecode::{trim_padded_evm_bytecode, BytecodeHash, BytecodeMarker}, - contract_verification_api::{ - self as api, CompilationArtifacts, VerificationIncomingRequest, VerificationInfo, - VerificationRequest, + contract_verification::{ + api::{ + self as api, CompilationArtifacts, VerificationIncomingRequest, VerificationInfo, + VerificationProblem, VerificationRequest, + }, + contract_identifier::{ContractIdentifier, Match}, }, Address, CONTRACT_DEPLOYER_ADDRESS, }; @@ -31,7 +33,6 @@ use crate::{ }; mod compilers; -mod contract_identifier; pub mod error; mod metrics; mod resolver; @@ -226,7 +227,7 @@ impl ContractVerifier { async fn verify( &self, mut request: VerificationRequest, - ) -> Result { + ) -> Result<(VerificationInfo, ContractIdentifier), ContractVerifierError> { // Bytecode should be present because it is checked when accepting request. let mut storage = self .connection_pool @@ -267,14 +268,28 @@ impl ContractVerifier { .context("invalid stored EVM bytecode")?, }; - if artifacts.deployed_bytecode() != deployed_bytecode { - tracing::info!( - request_id = request.id, - deployed = hex::encode(deployed_bytecode), - compiled = hex::encode(artifacts.deployed_bytecode()), - "Deployed (runtime) bytecode mismatch", - ); - return Err(ContractVerifierError::BytecodeMismatch); + let mut verification_problems = Vec::new(); + + match identifier.matches(deployed_bytecode) { + Match::Full => {} + Match::Partial => { + tracing::trace!( + request_id = request.id, + deployed = hex::encode(deployed_bytecode), + compiled = hex::encode(artifacts.deployed_bytecode()), + "Partial bytecode match", + ); + verification_problems.push(VerificationProblem::IncorrectMetadata); + } + Match::None => { + tracing::trace!( + request_id = request.id, + deployed = hex::encode(deployed_bytecode), + compiled = hex::encode(artifacts.deployed_bytecode()), + "Deployed (runtime) bytecode mismatch", + ); + return Err(ContractVerifierError::BytecodeMismatch); + } } match constructor_args { @@ -286,6 +301,11 @@ impl ContractVerifier { hex::encode(&args), hex::encode(provided_constructor_args) ); + // We could, in theory, accept this contract and mark it as partially verified, + // but in during verification it is always possible to reconstruct the + // constructor arguments, so there is no reason for that. + // Mismatching constructor arguments are only needed for "similar bytecodes" + // (e.g. displayed contract as verified without a direct verification request). return Err(ContractVerifierError::IncorrectConstructorArguments); } } @@ -296,11 +316,13 @@ impl ContractVerifier { let verified_at = Utc::now(); tracing::trace!(%verified_at, "verified request"); - Ok(VerificationInfo { + let info = VerificationInfo { request, artifacts, verified_at, - }) + verification_problems, + }; + Ok((info, identifier)) } async fn compile_zksolc( @@ -546,17 +568,23 @@ impl ContractVerifier { async fn process_result( &self, request_id: usize, - verification_result: Result, + verification_result: Result<(VerificationInfo, ContractIdentifier), ContractVerifierError>, ) -> anyhow::Result<()> { let mut storage = self .connection_pool .connection_tagged("contract_verifier") .await?; match verification_result { - Ok(info) => { + Ok((info, identifier)) => { storage .contract_verification_dal() - .save_verification_info(info) + .save_verification_info( + info, + identifier.bytecode_sha3, + identifier + .bytecode_without_metadata_sha3 + .map(|hash| hash.hash()), + ) .await?; tracing::info!("Successfully processed request with id = {request_id}"); } diff --git a/core/lib/contract_verifier/src/resolver/mod.rs b/core/lib/contract_verifier/src/resolver/mod.rs index 4a93029a6de0..f71bf72ebb09 100644 --- a/core/lib/contract_verifier/src/resolver/mod.rs +++ b/core/lib/contract_verifier/src/resolver/mod.rs @@ -8,12 +8,13 @@ use std::{ use anyhow::Context as _; use tokio::fs; use zksync_queued_job_processor::async_trait; -use zksync_types::contract_verification_api::CompilationArtifacts; +use zksync_types::contract_verification::{ + api::CompilationArtifacts, contract_identifier::ContractIdentifier, +}; pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver}; use crate::{ compilers::{SolcInput, VyperInput, ZkSolcInput}, - contract_identifier::ContractIdentifier, error::ContractVerifierError, ZkCompilerVersions, }; diff --git a/core/lib/contract_verifier/src/tests/mod.rs b/core/lib/contract_verifier/src/tests/mod.rs index 3178b518b8a2..e87a7bcf7b09 100644 --- a/core/lib/contract_verifier/src/tests/mod.rs +++ b/core/lib/contract_verifier/src/tests/mod.rs @@ -9,7 +9,7 @@ use zksync_node_test_utils::{create_l1_batch, create_l2_block}; use zksync_types::{ address_to_h256, bytecode::{pad_evm_bytecode, BytecodeHash}, - contract_verification_api::{CompilerVersions, SourceCodeData, VerificationIncomingRequest}, + contract_verification::api::{CompilerVersions, SourceCodeData, VerificationIncomingRequest}, get_code_key, get_known_code_key, l2::L2Tx, tx::IncludedTxLocation, diff --git a/core/lib/dal/.sqlx/query-1823e1ac602ce4ba1db06543af9cb2685cda1ae7ecca83062ede7320c3b4a427.json b/core/lib/dal/.sqlx/query-1823e1ac602ce4ba1db06543af9cb2685cda1ae7ecca83062ede7320c3b4a427.json deleted file mode 100644 index 1e20a9151b98..000000000000 --- a/core/lib/dal/.sqlx/query-1823e1ac602ce4ba1db06543af9cb2685cda1ae7ecca83062ede7320c3b4a427.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n contracts_verification_info (address, verification_info)\n VALUES\n ($1, $2)\n ON CONFLICT (address) DO\n UPDATE\n SET\n verification_info = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Bytea", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "1823e1ac602ce4ba1db06543af9cb2685cda1ae7ecca83062ede7320c3b4a427" -} diff --git a/core/lib/dal/.sqlx/query-2d0c2e9ec4187641baef8a33229bffc78d92adb3c1e3ca60b12163e38c67047e.json b/core/lib/dal/.sqlx/query-2d0c2e9ec4187641baef8a33229bffc78d92adb3c1e3ca60b12163e38c67047e.json deleted file mode 100644 index f61f39e3b0b0..000000000000 --- a/core/lib/dal/.sqlx/query-2d0c2e9ec4187641baef8a33229bffc78d92adb3c1e3ca60b12163e38c67047e.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n COUNT(*) AS \"count!\"\n FROM\n contracts_verification_info\n WHERE\n address = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Bytea" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2d0c2e9ec4187641baef8a33229bffc78d92adb3c1e3ca60b12163e38c67047e" -} diff --git a/core/lib/dal/.sqlx/query-349d41c8ce192e82152e9d254c23ed5f1e6eac5b71232c784abd4d4cd8677805.json b/core/lib/dal/.sqlx/query-349d41c8ce192e82152e9d254c23ed5f1e6eac5b71232c784abd4d4cd8677805.json new file mode 100644 index 000000000000..2de71e3d1ab5 --- /dev/null +++ b/core/lib/dal/.sqlx/query-349d41c8ce192e82152e9d254c23ed5f1e6eac5b71232c784abd4d4cd8677805.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n contract_verification_info_v2 (\n initial_contract_addr,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n VALUES\n ($1, $2, $3, $4)\n ON CONFLICT (initial_contract_addr) DO\n UPDATE\n SET\n bytecode_keccak256 = $2,\n bytecode_without_metadata_keccak256 = $3,\n verification_info = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bytea", + "Bytea", + "Bytea", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "349d41c8ce192e82152e9d254c23ed5f1e6eac5b71232c784abd4d4cd8677805" +} diff --git a/core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json b/core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json new file mode 100644 index 000000000000..ad67acf09d7c --- /dev/null +++ b/core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n COUNT(*)\n FROM\n contracts_verification_info\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e" +} diff --git a/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json b/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json new file mode 100644 index 000000000000..45c11f60d9cf --- /dev/null +++ b/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n verification_info,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256\n FROM\n contract_verification_info_v2\n WHERE\n bytecode_keccak256 = $1\n OR\n (\n bytecode_without_metadata_keccak256 IS NOT null\n AND bytecode_without_metadata_keccak256 = $2\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "verification_info", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "bytecode_keccak256", + "type_info": "Bytea" + }, + { + "ordinal": 2, + "name": "bytecode_without_metadata_keccak256", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Bytea", + "Bytea" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9" +} diff --git a/core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json b/core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json new file mode 100644 index 000000000000..7e293cb033aa --- /dev/null +++ b/core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n COUNT(*)\n FROM\n contract_verification_info_v2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48" +} diff --git a/core/lib/dal/.sqlx/query-daa2ad4ebde17808e059aa6bcf148743ffed494316750be3b0ffb10b2fc09e93.json b/core/lib/dal/.sqlx/query-daa2ad4ebde17808e059aa6bcf148743ffed494316750be3b0ffb10b2fc09e93.json new file mode 100644 index 000000000000..fd279a570be9 --- /dev/null +++ b/core/lib/dal/.sqlx/query-daa2ad4ebde17808e059aa6bcf148743ffed494316750be3b0ffb10b2fc09e93.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n verification_info\n FROM\n contract_verification_info_v2\n WHERE\n initial_contract_addr = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "verification_info", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Bytea" + ] + }, + "nullable": [ + false + ] + }, + "hash": "daa2ad4ebde17808e059aa6bcf148743ffed494316750be3b0ffb10b2fc09e93" +} diff --git a/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.down.sql b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.down.sql new file mode 100644 index 000000000000..03c94eab5f61 --- /dev/null +++ b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS contract_verification_info_v2_bytecode_keccak256_idx; +DROP INDEX IF EXISTS contract_verification_info_v2_bytecode_without_metadata_keccak256_idx; +DROP TABLE IF EXISTS contract_verification_info_v2; diff --git a/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql new file mode 100644 index 000000000000..1e760d122b07 --- /dev/null +++ b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS contract_verification_info_v2 ( + initial_contract_addr BYTEA NOT NULL PRIMARY KEY, + bytecode_keccak256 BYTEA NOT NULL, + bytecode_without_metadata_keccak256 BYTEA, + verification_info JSONB NOT NULL, + + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Add hash indexes for hash columns +CREATE INDEX IF NOT EXISTS contract_verification_info_v2_bytecode_keccak256_idx ON contract_verification_info_v2 USING HASH (bytecode_keccak256); +CREATE INDEX IF NOT EXISTS contract_verification_info_v2_bytecode_without_metadata_keccak256_idx ON contract_verification_info_v2 USING HASH (bytecode_without_metadata_keccak256); diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 57bea5392cf8..b70c19e6370b 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -5,13 +5,18 @@ use std::{ time::Duration, }; +use anyhow::Context as _; use sqlx::postgres::types::PgInterval; use zksync_db_connection::{error::SqlxContext, instrument::InstrumentExt}; use zksync_types::{ address_to_h256, - contract_verification_api::{ - VerificationIncomingRequest, VerificationInfo, VerificationRequest, - VerificationRequestStatus, + bytecode::{trim_padded_evm_bytecode, BytecodeHash, BytecodeMarker}, + contract_verification::{ + api::{ + VerificationIncomingRequest, VerificationInfo, VerificationProblem, + VerificationRequest, VerificationRequestStatus, + }, + contract_identifier::ContractIdentifier, }, web3, Address, CONTRACT_DEPLOYER_ADDRESS, H256, }; @@ -188,6 +193,8 @@ impl ContractVerificationDal<'_, '_> { pub async fn save_verification_info( &mut self, verification_info: VerificationInfo, + bytecode_keccak256: H256, + bytecode_without_metadata_keccak256: Option, ) -> DalResult<()> { let mut transaction = self.storage.start_transaction().await?; let id = verification_info.request.id; @@ -213,18 +220,32 @@ impl ContractVerificationDal<'_, '_> { // Serialization should always succeed. let verification_info_json = serde_json::to_value(verification_info) .expect("Failed to serialize verification info into serde_json"); + let bytecode_without_metadata_keccak256: Option<&[u8]> = + match &bytecode_without_metadata_keccak256 { + Some(h) => Some(h.as_bytes()), + None => None, + }; sqlx::query!( r#" INSERT INTO - contracts_verification_info (address, verification_info) + contract_verification_info_v2 ( + initial_contract_addr, + bytecode_keccak256, + bytecode_without_metadata_keccak256, + verification_info + ) VALUES - ($1, $2) - ON CONFLICT (address) DO + ($1, $2, $3, $4) + ON CONFLICT (initial_contract_addr) DO UPDATE SET - verification_info = $2 + bytecode_keccak256 = $2, + bytecode_without_metadata_keccak256 = $3, + verification_info = $4 "#, address.as_bytes(), + bytecode_keccak256.as_bytes(), + bytecode_without_metadata_keccak256, &verification_info_json ) .instrument("save_verification_info#insert") @@ -376,27 +397,6 @@ impl ContractVerificationDal<'_, '_> { .await } - /// Returns true if the contract has a stored contracts_verification_info. - pub async fn is_contract_verified(&mut self, address: Address) -> DalResult { - let count = sqlx::query!( - r#" - SELECT - COUNT(*) AS "count!" - FROM - contracts_verification_info - WHERE - address = $1 - "#, - address.as_bytes() - ) - .instrument("is_contract_verified") - .with_arg("address", &address) - .fetch_one(self.storage) - .await? - .count; - Ok(count > 0) - } - async fn get_compiler_versions(&mut self, compiler: Compiler) -> DalResult> { let compiler = format!("{compiler}"); let versions: Vec<_> = sqlx::query!( @@ -537,6 +537,29 @@ impl ContractVerificationDal<'_, '_> { pub async fn get_contract_verification_info( &mut self, address: Address, + ) -> anyhow::Result> { + // Do everything in a read-only transaction for a consistent view. + let mut transaction = self + .storage + .transaction_builder()? + .set_readonly() + .build() + .await?; + + let mut dal = ContractVerificationDal { + storage: &mut transaction, + }; + let info = if dal.is_verification_info_migration_performed().await? { + dal.get_contract_verification_info_v2(address).await? + } else { + dal.get_contract_verification_info_v1(address).await? + }; + Ok(info) + } + + async fn get_contract_verification_info_v1( + &mut self, + address: Address, ) -> DalResult> { Ok(sqlx::query!( r#" @@ -560,6 +583,156 @@ impl ContractVerificationDal<'_, '_> { .await? .flatten()) } + + async fn get_contract_verification_info_v2( + &mut self, + address: Address, + ) -> anyhow::Result> { + // Try perfect match + if let Some(info) = sqlx::query!( + r#" + SELECT + verification_info + FROM + contract_verification_info_v2 + WHERE + initial_contract_addr = $1 + "#, + address.as_bytes(), + ) + .try_map(|row| { + serde_json::from_value(row.verification_info).decode_column("verification_info") + }) + .instrument("get_contract_verification_info_v2#perfect_match") + .with_arg("address", &address) + .fetch_optional(self.storage) + .await? + .flatten() + { + return Ok(Some(info)); + } + + // Try partial match + let Some(deployed_contract) = self.get_contract_info_for_verification(address).await? + else { + return Ok(None); + }; + + let bytecode_marker = BytecodeMarker::new(deployed_contract.bytecode_hash) + .context("unknown bytecode kind")?; + let deployed_bytecode = match bytecode_marker { + BytecodeMarker::EraVm => deployed_contract.bytecode.as_slice(), + BytecodeMarker::Evm => trim_padded_evm_bytecode( + BytecodeHash::try_from(deployed_contract.bytecode_hash) + .context("Invalid bytecode hash")?, + &deployed_contract.bytecode, + ) + .context("invalid stored EVM bytecode")?, + }; + + let identifier = ContractIdentifier::from_bytecode(bytecode_marker, deployed_bytecode); + // `unwrap_or_default` is safe, since we're checking that `bytecode_without_metadata_keccak256` is not null. + let bytecode_without_metadata_sha3 = identifier + .bytecode_without_metadata_sha3 + .as_ref() + .map(|h| h.hash()) + .unwrap_or_default(); + + let Some((mut info, bytecode_keccak256, bytecode_without_metadata_keccak256)) = + sqlx::query!( + r#" + SELECT + verification_info, + bytecode_keccak256, + bytecode_without_metadata_keccak256 + FROM + contract_verification_info_v2 + WHERE + bytecode_keccak256 = $1 + OR + ( + bytecode_without_metadata_keccak256 IS NOT null + AND bytecode_without_metadata_keccak256 = $2 + ) + "#, + identifier.bytecode_sha3.as_bytes(), + bytecode_without_metadata_sha3.as_bytes() + ) + .try_map(|row| { + let info = serde_json::from_value::(row.verification_info) + .decode_column("verification_info")?; + let bytecode_keccak256 = H256::from_slice(&row.bytecode_keccak256); + let bytecode_without_metadata_keccak256 = row + .bytecode_without_metadata_keccak256 + .map(|h| H256::from_slice(&h)); + Ok(( + info, + bytecode_keccak256, + bytecode_without_metadata_keccak256, + )) + }) + .instrument("get_contract_verification_info_v2#perfect_match") + .with_arg("address", &address) + .fetch_optional(self.storage) + .await? + else { + return Ok(None); + }; + if identifier.bytecode_sha3 != bytecode_keccak256 { + // Sanity check + if bytecode_without_metadata_keccak256.is_none() + || identifier.bytecode_without_metadata_sha3.map(|h| h.hash()) + != bytecode_without_metadata_keccak256 + { + tracing::error!( + contract_address = %address, + identifier = ?identifier, + bytecode_keccak256 = %bytecode_keccak256, + info = ?info, + "Bogus verification info fetched for contract", + ); + anyhow::bail!("Internal error: bogus verification info detected"); + } + + // Mark the contract as partial match (regardless of other issues). + info.verification_problems = vec![VerificationProblem::IncorrectMetadata]; + } + Ok(Some(info)) + } + + /// Checks if migration from `contracts_verification_info` to `contract_verification_info_v2` is performed + /// by checking if the latter has more or equal number of rows. + pub async fn is_verification_info_migration_performed(&mut self) -> DalResult { + let count_v1 = sqlx::query!( + r#" + SELECT + COUNT(*) + FROM + contracts_verification_info + "#, + ) + .instrument("is_verification_info_migration_performed#count_v1") + .fetch_one(self.storage) + .await? + .count + .unwrap() as usize; + + let count_v2 = sqlx::query!( + r#" + SELECT + COUNT(*) + FROM + contract_verification_info_v2 + "#, + ) + .instrument("is_verification_info_migration_performed#count_v2") + .fetch_one(self.storage) + .await? + .count + .unwrap() as usize; + + Ok(count_v2 >= count_v1) + } } #[cfg(test)] @@ -568,7 +741,7 @@ mod tests { use zksync_types::{ bytecode::BytecodeHash, - contract_verification_api::{CompilerVersions, SourceCodeData}, + contract_verification::api::{CompilerVersions, SourceCodeData}, tx::IncludedTxLocation, Execute, L1BatchNumber, L2BlockNumber, ProtocolVersion, }; diff --git a/core/lib/dal/src/models/storage_verification_request.rs b/core/lib/dal/src/models/storage_verification_request.rs index ae4718e41290..1ea70ed38129 100644 --- a/core/lib/dal/src/models/storage_verification_request.rs +++ b/core/lib/dal/src/models/storage_verification_request.rs @@ -1,5 +1,5 @@ use zksync_types::{ - contract_verification_api::{ + contract_verification::api::{ CompilerType, CompilerVersions, SourceCodeData, VerificationIncomingRequest, VerificationRequest, }, diff --git a/core/lib/types/Cargo.toml b/core/lib/types/Cargo.toml index 6af0e39d14f0..f4eeebfce038 100644 --- a/core/lib/types/Cargo.toml +++ b/core/lib/types/Cargo.toml @@ -29,6 +29,7 @@ rlp.workspace = true serde.workspace = true serde_json.workspace = true serde_with = { workspace = true, features = ["hex"] } +ciborium.workspace = true bigdecimal.workspace = true strum = { workspace = true, features = ["derive"] } thiserror.workspace = true diff --git a/core/lib/types/src/contract_verification_api.rs b/core/lib/types/src/contract_verification/api.rs similarity index 95% rename from core/lib/types/src/contract_verification_api.rs rename to core/lib/types/src/contract_verification/api.rs index cca5ae5a83a0..7ab6f2effaa7 100644 --- a/core/lib/types/src/contract_verification_api.rs +++ b/core/lib/types/src/contract_verification/api.rs @@ -232,12 +232,28 @@ impl CompilationArtifacts { } } +/// Non-critical issues detected during verification. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum VerificationProblem { + /// The bytecode is correct, but metadata hash is different. + IncorrectMetadata, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VerificationInfo { pub request: VerificationRequest, pub artifacts: CompilationArtifacts, pub verified_at: DateTime, + #[serde(default)] + pub verification_problems: Vec, +} + +impl VerificationInfo { + pub fn is_perfect_match(&self) -> bool { + self.verification_problems.is_empty() + } } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/core/lib/contract_verifier/src/contract_identifier.rs b/core/lib/types/src/contract_verification/contract_identifier.rs similarity index 83% rename from core/lib/contract_verifier/src/contract_identifier.rs rename to core/lib/types/src/contract_verification/contract_identifier.rs index 14904819ef21..6f64f8e2cf85 100644 --- a/core/lib/contract_verifier/src/contract_identifier.rs +++ b/core/lib/types/src/contract_verification/contract_identifier.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; -use zksync_types::{bytecode::BytecodeMarker, web3::keccak256, H256}; + +use crate::{bytecode::BytecodeMarker, web3::keccak256, H256}; /// An identifier of the contract bytecode. /// This identifier can be used to detect different contracts that share the same sources, @@ -15,28 +16,45 @@ use zksync_types::{bytecode::BytecodeMarker, web3::keccak256, H256}; // less relevant for ZKsync, since there is no concept of creation bytecode there; although // this may become needed if we will extend the EVM support. #[derive(Debug, Clone, Copy)] -pub(crate) struct ContractIdentifier { +pub struct ContractIdentifier { + /// Marker of the bytecode of the contract. + pub bytecode_marker: BytecodeMarker, /// SHA3 (keccak256) hash of the full contract bytecode. /// Can be used as an identifier of precise contract compilation. pub bytecode_sha3: H256, /// SHA3 (keccak256) hash of the contract bytecode without metadata (e.g. with either /// CBOR or keccak256 metadata hash being stripped). /// Can be absent if the contract bytecode doesn't have metadata. - pub bytecode_without_metadata_sha3: Option, - /// Detected metadata in the contract bytecode. - pub detected_metadata: DetectedMetadata, + pub bytecode_without_metadata_sha3: Option, + /// Size of metadata in the bytecode. + pub metadata_size: usize, } -/// Metadata detected in the contract bytecode. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub(crate) enum DetectedMetadata { - /// No metadata detected. - #[default] +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Match { + /// Contracts are identical. + Full, + /// Metadata is different. + Partial, + /// No match. None, +} + +/// Metadata detected in the contract bytecode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DetectedMetadata { /// Keccek256 hash of the metadata detected (only for EraVM). - Keccak256, + Keccak256(H256), /// CBOR metadata detected. - Cbor, + Cbor(H256), +} + +impl DetectedMetadata { + pub fn hash(&self) -> H256 { + match self { + Self::Keccak256(hash) | Self::Cbor(hash) => *hash, + } + } } /// Possible values for the metadata hashes structure. @@ -63,9 +81,10 @@ impl ContractIdentifier { // Calculate the hash for bytecode with metadata. let bytecode_sha3 = H256::from_slice(&keccak256(bytecode)); let mut self_ = Self { + bytecode_marker, bytecode_sha3, bytecode_without_metadata_sha3: None, - detected_metadata: DetectedMetadata::None, + metadata_size: 0, }; // For EraVM, the default metadata is keccak256 hash of the metadata. @@ -78,15 +97,16 @@ impl ContractIdentifier { let bytecode_without_metadata = if bytecode[bytecode.len() - 64..bytecode.len() - 32] == [0u8; 32] { // Padding is present, strip it. + self_.metadata_size = 64; &bytecode[..bytecode.len() - 64] } else { // No padding, strip metadata only. + self_.metadata_size = 32; &bytecode[..bytecode.len() - 32] }; let hash = H256::from_slice(&keccak256(bytecode_without_metadata)); // This could be overridden if CBOR metadata is detected. - self_.bytecode_without_metadata_sha3 = Some(hash); - self_.detected_metadata = DetectedMetadata::Keccak256; + self_.bytecode_without_metadata_sha3 = Some(DetectedMetadata::Keccak256(hash)); } } @@ -117,6 +137,7 @@ impl ContractIdentifier { let bytecode_without_metadata = match bytecode_marker { BytecodeMarker::Evm => { // On EVM, there is no padding. + self_.metadata_size = full_metadata_length; &bytecode[..bytecode.len() - full_metadata_length] } BytecodeMarker::EraVm => { @@ -129,6 +150,7 @@ impl ContractIdentifier { // This shouldn't normally happen (metadata was deserialized correctly), // so we just disable partial matching just in case. self_.bytecode_without_metadata_sha3 = None; + self_.metadata_size = 0; return self_; } // Check if padding was added. @@ -137,19 +159,42 @@ impl ContractIdentifier { == [0u8; 32] { // Padding was added, strip it. + self_.metadata_size = full_aligned_metadata_length; &bytecode[..bytecode.len() - full_aligned_metadata_length] } else { // Padding wasn't added, strip metadata only. + self_.metadata_size = aligned_metadata_length; &bytecode[..bytecode.len() - aligned_metadata_length] } } }; - self_.bytecode_without_metadata_sha3 = - Some(H256::from_slice(&keccak256(bytecode_without_metadata))); + let hash = H256::from_slice(&keccak256(bytecode_without_metadata)); + self_.bytecode_without_metadata_sha3 = Some(DetectedMetadata::Cbor(hash)); - self_.detected_metadata = DetectedMetadata::Cbor; self_ } + + pub fn matches(&self, other: &[u8]) -> Match { + let other_identifier = Self::from_bytecode(self.bytecode_marker, other); + + if self.bytecode_sha3 == other_identifier.bytecode_sha3 { + return Match::Full; + } + + // Check if metadata is different. + // Note that here we do not handle "complex" cases, e.g. lack of metadata in one contract + // and presence in another, or different kinds of metadata. This is OK: partial + // match is needed mostly when you cannot reproduce the original metadata, but one always + // can submit the contract with the same metadata kind. + if self.bytecode_without_metadata_sha3.is_some() + && self.bytecode_without_metadata_sha3 + == other_identifier.bytecode_without_metadata_sha3 + { + return Match::Partial; + } + + Match::None + } } #[cfg(test)] @@ -167,7 +212,7 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.detected_metadata, DetectedMetadata::Cbor); + assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_sha3, H256::from_slice(&sha3), @@ -175,7 +220,9 @@ mod tests { ); assert_eq!( identifier.bytecode_without_metadata_sha3, - Some(H256::from_slice(&sha3_without_metadata)), + Some(DetectedMetadata::Cbor(H256::from_slice( + &sha3_without_metadata + ))), "Incorrect bytecode without metadata hash" ); } @@ -189,7 +236,7 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.detected_metadata, DetectedMetadata::Cbor); + assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_sha3, H256::from_slice(&sha3), @@ -197,7 +244,9 @@ mod tests { ); assert_eq!( identifier.bytecode_without_metadata_sha3, - Some(H256::from_slice(&sha3_without_metadata)), + Some(DetectedMetadata::Cbor(H256::from_slice( + &sha3_without_metadata + ))), "Incorrect bytecode without metadata hash" ); } @@ -213,7 +262,7 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.detected_metadata, DetectedMetadata::Keccak256); + assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_sha3, H256::from_slice(&sha3), @@ -221,7 +270,9 @@ mod tests { ); assert_eq!( identifier.bytecode_without_metadata_sha3, - Some(H256::from_slice(&sha3_without_metadata)), + Some(DetectedMetadata::Keccak256(H256::from_slice( + &sha3_without_metadata + ))), "Incorrect bytecode without metadata hash" ); } @@ -235,7 +286,7 @@ mod tests { let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.detected_metadata, DetectedMetadata::Keccak256); + assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_sha3, H256::from_slice(&sha3), @@ -243,7 +294,9 @@ mod tests { ); assert_eq!( identifier.bytecode_without_metadata_sha3, - Some(H256::from_slice(&sha3_without_metadata)), + Some(DetectedMetadata::Keccak256(H256::from_slice( + &sha3_without_metadata + ))), "Incorrect bytecode without metadata hash" ); } @@ -256,7 +309,7 @@ mod tests { let sha3 = keccak256(&data); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.detected_metadata, DetectedMetadata::None); + assert_eq!(identifier.metadata_size, 0); assert_eq!( identifier.bytecode_sha3, H256::from_slice(&sha3), @@ -277,7 +330,7 @@ mod tests { let sha3 = keccak256(&data); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); - assert_eq!(identifier.detected_metadata, DetectedMetadata::None); + assert_eq!(identifier.metadata_size, 0); assert_eq!( identifier.bytecode_sha3, H256::from_slice(&sha3), @@ -315,9 +368,8 @@ mod tests { let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); assert_eq!( - identifier.detected_metadata, - DetectedMetadata::Cbor, - "{label}" + identifier.metadata_size, full_metadata_len, + "{label}: Wrong metadata length" ); assert_eq!( identifier.bytecode_sha3, @@ -326,7 +378,9 @@ mod tests { ); assert_eq!( identifier.bytecode_without_metadata_sha3, - Some(H256::from_slice(&sha3_without_metadata)), + Some(DetectedMetadata::Cbor(H256::from_slice( + &sha3_without_metadata + ))), "{label}: Incorrect bytecode without metadata hash" ); } diff --git a/core/lib/types/src/contract_verification/mod.rs b/core/lib/types/src/contract_verification/mod.rs new file mode 100644 index 000000000000..7f8686941edb --- /dev/null +++ b/core/lib/types/src/contract_verification/mod.rs @@ -0,0 +1,2 @@ +pub mod api; +pub mod contract_identifier; diff --git a/core/lib/types/src/lib.rs b/core/lib/types/src/lib.rs index 8ec98ec0571e..606de9b9e798 100644 --- a/core/lib/types/src/lib.rs +++ b/core/lib/types/src/lib.rs @@ -32,7 +32,7 @@ pub mod aggregated_operations; pub mod blob; pub mod block; pub mod commitment; -pub mod contract_verification_api; +pub mod contract_verification; pub mod debug_flat_call; pub mod fee; pub mod fee_model; diff --git a/core/node/contract_verification_server/src/api_impl.rs b/core/node/contract_verification_server/src/api_impl.rs index b0336fd284b6..1cf20befab02 100644 --- a/core/node/contract_verification_server/src/api_impl.rs +++ b/core/node/contract_verification_server/src/api_impl.rs @@ -10,7 +10,7 @@ use axum::{ use zksync_dal::{CoreDal, DalError}; use zksync_types::{ bytecode::BytecodeMarker, - contract_verification_api::{ + contract_verification::api::{ CompilerVersions, VerificationIncomingRequest, VerificationInfo, VerificationRequestStatus, }, Address, diff --git a/core/node/contract_verification_server/src/cache.rs b/core/node/contract_verification_server/src/cache.rs index c8e367515287..f7ba10c2bf92 100644 --- a/core/node/contract_verification_server/src/cache.rs +++ b/core/node/contract_verification_server/src/cache.rs @@ -5,7 +5,7 @@ use std::{ use tokio::sync::RwLock; use zksync_dal::{Connection, ConnectionPool, Core, CoreDal, DalError}; -use zksync_types::contract_verification_api::CompilerVersions; +use zksync_types::contract_verification::api::CompilerVersions; /// Compiler versions supported by the contract verifier. #[derive(Debug, Clone)] diff --git a/core/node/contract_verification_server/src/tests.rs b/core/node/contract_verification_server/src/tests.rs index 88b14db68733..3eb27056b960 100644 --- a/core/node/contract_verification_server/src/tests.rs +++ b/core/node/contract_verification_server/src/tests.rs @@ -13,7 +13,7 @@ use zksync_dal::{Connection, Core, CoreDal}; use zksync_node_test_utils::create_l2_block; use zksync_types::{ bytecode::{BytecodeHash, BytecodeMarker}, - contract_verification_api::CompilerVersions, + contract_verification::api::CompilerVersions, get_code_key, Address, L2BlockNumber, ProtocolVersion, StorageLog, }; diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 761c9e398cb4..4e4b542fbc02 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -803,6 +803,33 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "circuit_definitions" version = "0.150.19" @@ -2370,6 +2397,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "handlebars" version = "3.5.5" @@ -8899,6 +8936,7 @@ dependencies = [ "bigdecimal", "blake2 0.10.6", "chrono", + "ciborium", "derive_more", "hex", "itertools 0.10.5", diff --git a/zkstack_cli/Cargo.lock b/zkstack_cli/Cargo.lock index cfc16535db50..8859b9cfd4eb 100644 --- a/zkstack_cli/Cargo.lock +++ b/zkstack_cli/Cargo.lock @@ -628,6 +628,33 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -2143,6 +2170,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -7413,6 +7450,7 @@ dependencies = [ "bigdecimal", "blake2", "chrono", + "ciborium", "derive_more", "hex", "itertools 0.10.5", From 300fdc15ee7763dedbed36f4aa92c7d670f737e4 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 24 Jan 2025 13:55:43 +0400 Subject: [PATCH 04/16] Add tests for partial verification --- .github/workflows/ci-core-reusable.yml | 4 +- .../contract_verifier/src/compilers/zksolc.rs | 24 +- .../src/contract_identifier.rs | 0 core/lib/contract_verifier/src/tests/mod.rs | 10 +- core/lib/contract_verifier/src/tests/real.rs | 217 +++++++++++++++++- core/lib/dal/src/contract_verification_dal.rs | 28 +-- .../types/src/contract_verification/api.rs | 4 +- 7 files changed, 253 insertions(+), 34 deletions(-) create mode 100644 core/lib/contract_verifier/src/contract_identifier.rs diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index f44b1f54dc02..d4fcbe90e505 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -72,7 +72,7 @@ jobs: ci_run zkstack dev contracts - name: Download compilers for contract verifier tests - run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era + run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era - name: Rust unit tests run: | @@ -431,7 +431,7 @@ jobs: - name: Initialize Contract verifier run: | - ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era + ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log & ci_run zkstack contract-verifier wait --chain era --verbose diff --git a/core/lib/contract_verifier/src/compilers/zksolc.rs b/core/lib/contract_verifier/src/compilers/zksolc.rs index 4dc344a5a175..55fb2ca57f59 100644 --- a/core/lib/contract_verifier/src/compilers/zksolc.rs +++ b/core/lib/contract_verifier/src/compilers/zksolc.rs @@ -69,6 +69,7 @@ pub(crate) struct Optimizer { /// Whether the optimizer is enabled. pub enabled: bool, /// The optimization mode string. + #[serde(skip_serializing_if = "Option::is_none")] pub mode: Option, } @@ -148,12 +149,24 @@ impl ZkSolc { fn parse_single_file_yul_output( output: &str, ) -> Result { - let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap(); - let cap = re - .captures(output) - .context("Yul output doesn't match regex")?; + let cap = if output.contains("Binary:\n") { + // Format of the new output + // ======= /tmp/input.yul:Empty ======= + // Binary: + // 00000001002 <..> + let re = Regex::new(r"Binary:\n([\da-f]+)").unwrap(); + re.captures(output) + .with_context(|| format!("Yul output doesn't match regex. Output: {output}"))? + } else { + // Old compiler versions + let re_old = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap(); + re_old + .captures(output) + .with_context(|| format!("Yul output doesn't match regex. Output: {output}"))? + }; let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str(); let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?; + Ok(CompilationArtifacts { bytecode, deployed_bytecode: None, @@ -265,6 +278,9 @@ impl Compiler for ZkSolc { .context("cannot create temporary Yul file")?; file.write_all(source_code.as_bytes()) .context("failed writing Yul file")?; + + // TODO: `zksolc` support standard JSON for `yul` since 1.5.0, so we don't have + // to parse `--bin` output. let child = command .arg(file.path().to_str().unwrap()) .arg("--optimization") diff --git a/core/lib/contract_verifier/src/contract_identifier.rs b/core/lib/contract_verifier/src/contract_identifier.rs new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/core/lib/contract_verifier/src/tests/mod.rs b/core/lib/contract_verifier/src/tests/mod.rs index e87a7bcf7b09..ebd6469084da 100644 --- a/core/lib/contract_verifier/src/tests/mod.rs +++ b/core/lib/contract_verifier/src/tests/mod.rs @@ -446,7 +446,7 @@ async fn contract_verifier_basics(contract: TestContract) { let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); - assert_request_success(&mut storage, request_id, address, &expected_bytecode).await; + assert_request_success(&mut storage, request_id, address, &expected_bytecode, &[]).await; } async fn assert_request_success( @@ -454,6 +454,7 @@ async fn assert_request_success( request_id: usize, address: Address, expected_bytecode: &[u8], + verification_problems: &[VerificationProblem], ) -> VerificationInfo { let status = storage .contract_verification_dal() @@ -476,6 +477,11 @@ async fn assert_request_success( without_internal_types(verification_info.artifacts.abi.clone()), without_internal_types(counter_contract_abi()) ); + assert_eq!( + &verification_info.verification_problems, + verification_problems + ); + verification_info } @@ -554,7 +560,7 @@ async fn verifying_evm_bytecode(contract: TestContract) { let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); - assert_request_success(&mut storage, request_id, address, &creation_bytecode).await; + assert_request_success(&mut storage, request_id, address, &creation_bytecode, &[]).await; } #[tokio::test] diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index 4294e423a546..2d5b6758a3f2 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -4,13 +4,15 @@ //! //! You can install the compilers to run these tests with the following command: //! ``` -//! zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only +//! zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only //! ``` use std::{env, sync::Arc, time::Duration}; use assert_matches::assert_matches; -use zksync_types::bytecode::validate_bytecode; +use zksync_types::{ + bytecode::validate_bytecode, contract_verification::contract_identifier::DetectedMetadata, +}; use super::*; @@ -26,7 +28,7 @@ impl Toolchain { // The tests may expect specific compiler versions (e.g. contracts won't compile with Vyper 0.4.0), // so we hardcode versions. -const ZKSOLC_VERSION: &str = "v1.5.3"; +const ZKSOLC_VERSION: &str = "v1.5.10"; const ERA_VM_SOLC_VERSION: &str = "0.8.26-1.0.1"; const SOLC_VERSION: &str = "0.8.26"; const VYPER_VERSION: &str = "v0.3.10"; @@ -304,11 +306,15 @@ async fn compiling_yul_with_zksolc() { let compiler = compiler_resolver.resolve_zksolc(&version).await.unwrap(); let req = test_yul_request(supported_compilers.solc_for_api(BytecodeMarker::EraVm)); let input = ZkSolc::build_input(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let (output, identifier) = compiler.compile(input).await.unwrap(); assert!(!output.bytecode.is_empty()); assert!(output.deployed_bytecode.is_none()); assert_eq!(output.abi, serde_json::json!([])); + assert_matches!( + identifier.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Keccak256(_)) + ); } #[tokio::test] @@ -322,11 +328,16 @@ async fn compiling_standalone_yul() { compiler_zksolc_version: None, }); let input = Solc::build_input(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let (output, identifier) = compiler.compile(input).await.unwrap(); assert!(!output.bytecode.is_empty()); assert_ne!(output.deployed_bytecode.unwrap(), output.bytecode); assert_eq!(output.abi, serde_json::json!([])); + assert_matches!( + identifier.bytecode_without_metadata_sha3, + None, + "No metadata for compiler yul for EVM" + ); } fn test_vyper_request( @@ -372,10 +383,14 @@ async fn using_real_zkvyper(specify_contract_file: bool) { BytecodeMarker::EraVm, ); let input = VyperInput::new(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let (output, identifier) = compiler.compile(input).await.unwrap(); validate_bytecode(&output.bytecode).unwrap(); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); + assert_matches!( + identifier.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Keccak256(_)) + ); } #[test_casing(2, [false, true])] @@ -397,10 +412,12 @@ async fn using_standalone_vyper(specify_contract_file: bool) { BytecodeMarker::Evm, ); let input = VyperInput::new(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let (output, identifier) = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); + // Vyper does not provide metadata for bytecode. + assert_matches!(identifier.bytecode_without_metadata_sha3, None); } #[tokio::test] @@ -417,10 +434,12 @@ async fn using_standalone_vyper_without_optimization() { ); req.optimization_used = false; let input = VyperInput::new(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let (output, identifier) = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); + // Vyper does not provide metadata for bytecode. + assert_matches!(identifier.bytecode_without_metadata_sha3, None); } #[tokio::test] @@ -490,7 +509,7 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai }, }; let address = Address::repeat_byte(1); - let (output, _) = match (bytecode_kind, toolchain) { + let (output, identifier) = match (bytecode_kind, toolchain) { (BytecodeMarker::EraVm, Toolchain::Solidity) => { let compiler = compiler_resolver .resolve_zksolc(&supported_compilers.zksolc()) @@ -521,6 +540,29 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai } }; + match (bytecode_kind, toolchain) { + (BytecodeMarker::Evm, Toolchain::Vyper) => { + assert!( + identifier.bytecode_without_metadata_sha3.is_none(), + "No metadata for EVM Vyper" + ); + } + (BytecodeMarker::Evm, Toolchain::Solidity) => { + assert_matches!( + identifier.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Cbor(_)), + "Cbor metadata for EVM Solidity by default" + ); + } + (BytecodeMarker::EraVm, _) => { + assert_matches!( + identifier.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Keccak256(_)), + "Keccak256 metadata for EraVM by default" + ); + } + } + let pool = ConnectionPool::test_pool().await; let mut storage = pool.connection().await.unwrap(); prepare_storage(&mut storage).await; @@ -556,7 +598,162 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); - assert_request_success(&mut storage, request_id, address, &output.bytecode).await; + assert_request_success(&mut storage, request_id, address, &output.bytecode, &[]).await; +} + +#[test_casing(2, [false, true])] +#[tokio::test] +async fn using_zksolc_partial_match(use_cbor: bool) { + let (compiler_resolver, supported_compilers) = real_resolver!(); + + let mut req: VerificationIncomingRequest = VerificationIncomingRequest { + compiler_versions: supported_compilers + .clone() + .solc_for_api(BytecodeMarker::EraVm), + ..test_request(Address::repeat_byte(1), COUNTER_CONTRACT) + }; + let hash_type = if use_cbor { "ipfs" } else { "keccak256" }; + // We need to manually construct the input, since `SolSingleFile` doesn't let us specify metadata hash type. + // Note: prior to 1.5.7 field was named `bytecodeHash`. + req.source_code_data = SourceCodeData::StandardJsonInput( + serde_json::json!({ + "language": "Solidity", + "sources": { + "Counter.sol": { + "content": COUNTER_CONTRACT, + }, + },"settings": { + "outputSelection": { + "*": { + "": [ + "abi" + ], + "*": [ + "abi" + ] + } + }, + "isSystem": false, + "forceEvmla": false, + "metadata": { + "hashType": hash_type + }, + "optimizer": { + "enabled": true + } + } + }) + .as_object() + .unwrap() + .clone(), + ); + let contract_name = req.contract_name.clone(); + let address = Address::repeat_byte(1); + let compiler = compiler_resolver + .resolve_zksolc(&supported_compilers.clone().zksolc()) + .await + .unwrap(); + let input_for_request = ZkSolc::build_input(req.clone()).unwrap(); + + let (output_for_request, identifier_for_request) = + compiler.compile(input_for_request).await.unwrap(); + + // Now prepare data for contract verification storage (with different metadata). + let compiler = compiler_resolver + .resolve_zksolc(&supported_compilers.zksolc()) + .await + .unwrap(); + let mut input_for_storage = ZkSolc::build_input(req.clone()).unwrap(); + // Change the source file name. + if let ZkSolcInput::StandardJson { + input, file_name, .. + } = &mut input_for_storage + { + let source = input + .sources + .remove(&format!("{contract_name}.sol")) + .unwrap(); + let new_file_name = "random_name.sol".to_owned(); + input.sources.insert(new_file_name.clone(), source); + *file_name = new_file_name; + if use_cbor { + input.settings.other.as_object_mut().unwrap().insert( + "metadata".to_string(), + serde_json::json!({ "hashType": "ipfs"}), + ); + } + } else { + panic!("unexpected input: {input_for_storage:?}"); + } + + let (output_for_storage, identifier_for_storage) = + compiler.compile(input_for_storage).await.unwrap(); + + assert_eq!( + identifier_for_request.matches(output_for_storage.deployed_bytecode()), + Match::Partial, + "must be a partial match (1)" + ); + assert_eq!( + identifier_for_storage.matches(output_for_request.deployed_bytecode()), + Match::Partial, + "must be a partial match (2)" + ); + if use_cbor { + assert_matches!( + identifier_for_request.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Cbor(_)) + ); + assert_matches!( + identifier_for_storage.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Cbor(_)) + ); + } else { + assert_matches!( + identifier_for_request.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Keccak256(_)) + ); + assert_matches!( + identifier_for_storage.bytecode_without_metadata_sha3, + Some(DetectedMetadata::Keccak256(_)) + ); + } + + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + prepare_storage(&mut storage).await; + mock_deployment( + &mut storage, + address, + output_for_storage.bytecode.clone(), + &[], + ) + .await; + let request_id = storage + .contract_verification_dal() + .add_contract_verification_request(&req) + .await + .unwrap(); + + let verifier = ContractVerifier::with_resolver( + Duration::from_secs(60), + pool.clone(), + Arc::new(compiler_resolver), + ) + .await + .unwrap(); + + let (_stop_sender, stop_receiver) = watch::channel(false); + verifier.run(stop_receiver, Some(1)).await.unwrap(); + + assert_request_success( + &mut storage, + request_id, + address, + &output_for_request.bytecode, + &[VerificationProblem::IncorrectMetadata], + ) + .await; } #[test_casing(2, BYTECODE_KINDS)] diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index b70c19e6370b..2687f0df0054 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -641,20 +641,20 @@ impl ContractVerificationDal<'_, '_> { let Some((mut info, bytecode_keccak256, bytecode_without_metadata_keccak256)) = sqlx::query!( r#" - SELECT - verification_info, - bytecode_keccak256, - bytecode_without_metadata_keccak256 - FROM - contract_verification_info_v2 - WHERE - bytecode_keccak256 = $1 - OR - ( - bytecode_without_metadata_keccak256 IS NOT null - AND bytecode_without_metadata_keccak256 = $2 - ) - "#, + SELECT + verification_info, + bytecode_keccak256, + bytecode_without_metadata_keccak256 + FROM + contract_verification_info_v2 + WHERE + bytecode_keccak256 = $1 + OR + ( + bytecode_without_metadata_keccak256 IS NOT null + AND bytecode_without_metadata_keccak256 = $2 + ) + "#, identifier.bytecode_sha3.as_bytes(), bytecode_without_metadata_sha3.as_bytes() ) diff --git a/core/lib/types/src/contract_verification/api.rs b/core/lib/types/src/contract_verification/api.rs index 7ab6f2effaa7..92692511770c 100644 --- a/core/lib/types/src/contract_verification/api.rs +++ b/core/lib/types/src/contract_verification/api.rs @@ -233,7 +233,7 @@ impl CompilationArtifacts { } /// Non-critical issues detected during verification. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub enum VerificationProblem { /// The bytecode is correct, but metadata hash is different. @@ -246,7 +246,7 @@ pub struct VerificationInfo { pub request: VerificationRequest, pub artifacts: CompilationArtifacts, pub verified_at: DateTime, - #[serde(default)] + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub verification_problems: Vec, } From b6d2bf2cf5a56439b7801a5a6dabbd606f85e9ad Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 24 Jan 2025 14:03:12 +0400 Subject: [PATCH 05/16] Post-merge cleanup --- ...a6dfb560d7f79406dc8c0b50c155c6b7cc887.json | 35 +++ ...072dcee6e6f8439e6b43eebd6df5563a4d0b9.json | 35 --- prover/Cargo.lock | 257 +++++++++--------- 3 files changed, 168 insertions(+), 159 deletions(-) create mode 100644 core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json delete mode 100644 core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json diff --git a/core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json b/core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json new file mode 100644 index 000000000000..4b05ddce9694 --- /dev/null +++ b/core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n verification_info,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256\n FROM\n contract_verification_info_v2\n WHERE\n bytecode_keccak256 = $1\n OR\n (\n bytecode_without_metadata_keccak256 IS NOT null\n AND bytecode_without_metadata_keccak256 = $2\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "verification_info", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "bytecode_keccak256", + "type_info": "Bytea" + }, + { + "ordinal": 2, + "name": "bytecode_without_metadata_keccak256", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Bytea", + "Bytea" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887" +} diff --git a/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json b/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json deleted file mode 100644 index 45c11f60d9cf..000000000000 --- a/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n verification_info,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256\n FROM\n contract_verification_info_v2\n WHERE\n bytecode_keccak256 = $1\n OR\n (\n bytecode_without_metadata_keccak256 IS NOT null\n AND bytecode_without_metadata_keccak256 = $2\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "verification_info", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "bytecode_keccak256", - "type_info": "Bytea" - }, - { - "ordinal": 2, - "name": "bytecode_without_metadata_keccak256", - "type_info": "Bytea" - } - ], - "parameters": { - "Left": [ - "Bytea", - "Bytea" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9" -} diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 4e4b542fbc02..8748b27a52ca 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -625,9 +625,9 @@ dependencies = [ [[package]] name = "boojum" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14bd053feb7173130679a2119e105b5e78af7eb6b0e752de6793e4ee63d8e899" +checksum = "d689807d79092f8f7cfcb72a2313a43da77d56314e41324810566f385875c185" dependencies = [ "arrayvec 0.7.6", "bincode", @@ -657,9 +657,9 @@ dependencies = [ [[package]] name = "boojum-cuda" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd43bc7fc457920cb3b823e4f95ffbbf180b2c48b8d643125cd121325cdd8db" +checksum = "896aeb550e4b92e6c96858c1d0aa8413c00f97fb91f321a2bf3ed912942870f8" dependencies = [ "boojum", "cmake", @@ -832,9 +832,9 @@ dependencies = [ [[package]] name = "circuit_definitions" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10ebc81d5c2f6ee8de436c242f6466fb315fe25afcbc81aa1c47dfca39a55403" +checksum = "1f04f9c7c6b39255199aaba49802c5f40f95bcff24f5a456446a912d254f4bb1" dependencies = [ "circuit_encodings", "crossbeam", @@ -846,26 +846,26 @@ dependencies = [ [[package]] name = "circuit_encodings" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33375d2448a78c1aed9b8755f7939a6b6f19e2fa80f44f4930a5b4c2bb7cbb44" +checksum = "fc3399f1981164c3c687ea15b1eedd35a16f28069c845a24530de21f996f3fdd" dependencies = [ "derivative", "serde", - "zk_evm 0.150.19", + "zk_evm 0.150.20", "zkevm_circuits", ] [[package]] name = "circuit_sequencer_api" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2fec5c28e5a9f085279e70e13b2eebb63a95ee0bfb99d58095ac01c1c7b256" +checksum = "b5583037ec61607ac481b0c887b7fb4f860e65c92c6f3f7be74f6bab7c40c3ce" dependencies = [ "derivative", "rayon", "serde", - "zk_evm 0.150.19", + "zk_evm 0.150.20", "zksync_bellman", ] @@ -1733,9 +1733,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "era_cudart" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f881cf689ba889bb0fa04c0e71aba701acd7fafd3fa545e3f2782f2a8c0ba0" +checksum = "6ff6fc4fba6bf756cdebd6750161d280af2859d217dad89bfb2823ac760bf0e8" dependencies = [ "bitflags 2.6.0", "era_cudart_sys", @@ -1744,11 +1744,11 @@ dependencies = [ [[package]] name = "era_cudart_sys" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f29cbd9e6d97fc1f05b484f960e921fe69548b4773a361b2e403e4cb9d6d575" +checksum = "7e0daeb39d2111a868a50e0bd7d90fa355f93022038088c0dd865bbdda1113ef" dependencies = [ - "serde_json", + "regex-lite", ] [[package]] @@ -1865,9 +1865,9 @@ dependencies = [ [[package]] name = "fflonk" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e63d70f1cbf9e572ccaf22ca1dfce4b93ff48b9a5e8dd70de50d87edb960d173" +checksum = "b36c5fa909ab71b7eb4b8f7fd092f72ed83b93f2615e42f245ca808d8f308917" dependencies = [ "bincode", "byteorder", @@ -1882,9 +1882,9 @@ dependencies = [ [[package]] name = "fflonk-cuda" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b008e6158c95747b3b135adbd7f6d563c406849a10c00abfef109b4d0442a589" +checksum = "890b635123fe176814ddbda1fbe006c55ca02375e5dde83539018f283219a8ba" dependencies = [ "bincode", "byteorder", @@ -2002,9 +2002,9 @@ dependencies = [ [[package]] name = "franklin-crypto" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d7b8e5864df7f3747e5e64a5b87b4a57aa2a4a20c55c9e96a3a305a8143c45" +checksum = "8309d8fc22fc389d831390473b0ee9fe94e85f19a8b9229b9aec8aa73f5bcee3" dependencies = [ "arr_macro", "bit-vec 0.6.3", @@ -4587,9 +4587,9 @@ dependencies = [ [[package]] name = "proof-compression" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40624617ed1535349cf31671a7091703d3a31e64d6a7760a5e952c68ee1f4f0e" +checksum = "de4c8014afcd29bfe93ac3bd1ea0d9f2da06fa9895337bead3f3d0d904080e36" dependencies = [ "bincode", "byteorder", @@ -4599,6 +4599,7 @@ dependencies = [ "serde", "serde_json", "shivini", + "zksync-gpu-prover", ] [[package]] @@ -4742,7 +4743,7 @@ dependencies = [ [[package]] name = "prover_cli" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "assert_cmd", @@ -4775,7 +4776,7 @@ dependencies = [ [[package]] name = "prover_version" -version = "0.1.0" +version = "17.1.1" dependencies = [ "zksync_prover_fri_types", ] @@ -4982,6 +4983,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -5107,9 +5114,9 @@ dependencies = [ [[package]] name = "rescue_poseidon" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c250446885c257bee70bc0f2600229ce72f03073b87fb8f5dd278dba16b11f30" +checksum = "5e631fd184b6d2f2c04f9dc75405289d99fd0d6612d8dfbb478c01bfbab648fb" dependencies = [ "addchain", "arrayvec 0.7.6", @@ -5897,9 +5904,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "shivini" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c336213e4ec7651d2984e892326d09c195ee166493c192b0d8aad36288e5c5f" +checksum = "8937f1fe25a1ea33a40bdf560847b934fe68322c40cd54dd77ab433128022cce" dependencies = [ "bincode", "boojum", @@ -5988,9 +5995,9 @@ dependencies = [ [[package]] name = "snark_wrapper" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f361c2c47b71ee43f62954ce69f7730e14acb7fb3b0f2c697da02f97327c569" +checksum = "eddb498315057210abd25e2fbe2ea30ab69a07ca0c166406a3e7c056ec8fbbfd" dependencies = [ "derivative", "rand 0.4.6", @@ -7876,9 +7883,9 @@ dependencies = [ [[package]] name = "zk_evm" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ee848aa90ae045457795b1c0afeb388fbd9fa1e57aa0e8791b28f405e7cc2c" +checksum = "f11d0310228af78e804e5e7deccd1ad6797fce1c44c3b8016722ab78dc183c4a" dependencies = [ "anyhow", "lazy_static", @@ -7886,7 +7893,7 @@ dependencies = [ "serde", "serde_json", "static_assertions", - "zk_evm_abstractions 0.150.19", + "zk_evm_abstractions 0.150.20", ] [[package]] @@ -7917,22 +7924,22 @@ dependencies = [ [[package]] name = "zk_evm_abstractions" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f08feaa3e3d99e1e57234fe6ba2aa062609492c6499b2344121c4a699292ab7" +checksum = "d7616edbdeeeb214211e9bdc4346b6a62c6c6118c3d2b83b7db24c01f65f6e25" dependencies = [ "anyhow", "num_enum 0.6.1", "serde", "static_assertions", - "zkevm_opcode_defs 0.150.19", + "zkevm_opcode_defs 0.150.20", ] [[package]] name = "zkevm-assembly" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd4bc83f3a711d820829dccce24fa59ab4c588c2745203ec6a6ad8c871362b7" +checksum = "c2dc9539ce7f550231934e6b1faae23387fd132f1ac053b8e674d30968158bff" dependencies = [ "env_logger 0.9.3", "hex", @@ -7945,14 +7952,14 @@ dependencies = [ "smallvec", "structopt", "thiserror 1.0.69", - "zkevm_opcode_defs 0.150.19", + "zkevm_opcode_defs 0.150.20", ] [[package]] name = "zkevm_circuits" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760cfbbce18f42bbecd2565de9bf658234cac2431cce9b0c1df08e9df645d467" +checksum = "6f36004572f5086c513715e11f38230e2538c159d4f5d90dc518833c6fc78293" dependencies = [ "arrayvec 0.7.6", "boojum", @@ -7964,7 +7971,7 @@ dependencies = [ "seq-macro", "serde", "smallvec", - "zkevm_opcode_defs 0.150.19", + "zkevm_opcode_defs 0.150.20", "zksync_cs_derive", ] @@ -8012,9 +8019,9 @@ dependencies = [ [[package]] name = "zkevm_opcode_defs" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2bd8ef52c8f9911dd034b91d29f087ab52f80a80f9d996deb881abbb953793" +checksum = "ce6b4a47c0e7f95b51d29ca336821321cec4bbba0acdd412c3a209270a0d37fe" dependencies = [ "bitflags 2.6.0", "blake2 0.10.6", @@ -8029,9 +8036,9 @@ dependencies = [ [[package]] name = "zkevm_test_harness" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24b28c5b855e4e28d85455b48f346e9d46a00c6af84d5bbc38e5b5f7410b5cb" +checksum = "36ed8dd80455d90a51a6618a5bc07685beaad582cabca71ccef25866cd73993b" dependencies = [ "bincode", "circuit_definitions", @@ -8057,9 +8064,9 @@ dependencies = [ [[package]] name = "zksync-gpu-ffi" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f6a84e4e361977a2dc5dbe783e3856e40b1050dc1b9bb3e9833a5e59c20697" +checksum = "6c3fbd4c8df140131d28b05581b19418bc5e561beb21dec6f24ca2f34343399c" dependencies = [ "cmake", "crossbeam", @@ -8072,9 +8079,9 @@ dependencies = [ [[package]] name = "zksync-gpu-prover" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba58bbaf4920c635553d3dfb7796636223f55e75ae6512eb9c98f48f0a03215" +checksum = "0256175ceb3ea675d4c0ebcd690fdd45138bab1c5bc298b2e0db320e5abc0bdb" dependencies = [ "bit-vec 0.6.3", "cfg-if", @@ -8089,9 +8096,9 @@ dependencies = [ [[package]] name = "zksync-wrapper-prover" -version = "0.152.10" +version = "0.152.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4bcf41946f95a1e64ce99cde1d54966a04c5ef2c89d9a87f0fa61e39987510b" +checksum = "390d8f99cf47fade7f2fe38925f9787b3d27641a878887ae980e4ab5f6731ac0" dependencies = [ "circuit_definitions", "zkevm_test_harness", @@ -8100,7 +8107,7 @@ dependencies = [ [[package]] name = "zksync_basic_types" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "chrono", @@ -8121,9 +8128,9 @@ dependencies = [ [[package]] name = "zksync_bellman" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06d424f7e3862d7a6715179bafffbe7a5dce17129f95ac4124502ab9f1edfb8" +checksum = "78fc3c598daf718b6fc791bfbb01c4634199e479ea9b2c82d06cd108b967d441" dependencies = [ "arrayvec 0.7.6", "bit-vec 0.6.3", @@ -8144,7 +8151,7 @@ dependencies = [ [[package]] name = "zksync_circuit_prover" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8174,7 +8181,7 @@ dependencies = [ [[package]] name = "zksync_circuit_prover_service" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8194,9 +8201,9 @@ dependencies = [ [[package]] name = "zksync_concurrency" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8312ab73d3caa55775bd531795b507fa8f76bd9dabfaeb0954fe43e8fc1323b" +checksum = "cec98400a9e8ba02bfd029eacfe7d6fb7b85b8ef00de59d6bb119d29cc9f7442" dependencies = [ "anyhow", "once_cell", @@ -8213,7 +8220,7 @@ dependencies = [ [[package]] name = "zksync_config" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "rand 0.8.5", @@ -8229,9 +8236,9 @@ dependencies = [ [[package]] name = "zksync_consensus_crypto" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b539960de98df3c3bd27d2d9b97de862027686bbb3bdfc5aaad5b74bb929a1" +checksum = "c04840825dfbe3b9f708d245c87618d5dcf28f29d7b58922971351068a0b8231" dependencies = [ "anyhow", "blst", @@ -8250,9 +8257,9 @@ dependencies = [ [[package]] name = "zksync_consensus_roles" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c49949546895a10431b9daec6ec4208ef0917ace006446d304b51f5b234ba462" +checksum = "05498eab1de26869028b5822cfa4490cac625508d427d59668dc73e8162de65f" dependencies = [ "anyhow", "bit-vec 0.6.3", @@ -8272,9 +8279,9 @@ dependencies = [ [[package]] name = "zksync_consensus_storage" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb0d6a54e7d8d2adeee4ba38662161e9309180ad497299092e5641db9fb1c1e" +checksum = "b20eb99fdd0e171a370214d2b7c99b5d4e8c11b9828a6b5705423bf653849a70" dependencies = [ "anyhow", "async-trait", @@ -8292,9 +8299,9 @@ dependencies = [ [[package]] name = "zksync_consensus_utils" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723e2a4b056cc5af192a83163c89a6951ee75c098cc5c4a4cdc435f4232d88bd" +checksum = "f2f9fa69ef68e6a1955a1d7b33077103fb6d106b560fec0d599c6de268f5be03" dependencies = [ "anyhow", "rand 0.8.5", @@ -8304,7 +8311,7 @@ dependencies = [ [[package]] name = "zksync_contracts" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "envy", "hex", @@ -8317,7 +8324,7 @@ dependencies = [ [[package]] name = "zksync_core_leftovers" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "ctrlc", @@ -8331,7 +8338,7 @@ dependencies = [ [[package]] name = "zksync_crypto_primitives" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "blake2 0.10.6", @@ -8347,9 +8354,9 @@ dependencies = [ [[package]] name = "zksync_cs_derive" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23237b019a469bfa59c11108beff84a63a43f52fa3afbf1b461527031fc47644" +checksum = "97ab7469afcd9e1cb220fe17b3c9f2abe031648b94add97da37065c58be08554" dependencies = [ "proc-macro-error", "proc-macro2 1.0.92", @@ -8359,7 +8366,7 @@ dependencies = [ [[package]] name = "zksync_dal" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "bigdecimal", @@ -8394,7 +8401,7 @@ dependencies = [ [[package]] name = "zksync_db_connection" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "rand 0.8.5", @@ -8410,7 +8417,7 @@ dependencies = [ [[package]] name = "zksync_env_config" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "envy", @@ -8421,7 +8428,7 @@ dependencies = [ [[package]] name = "zksync_eth_client" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "async-trait", "jsonrpsee", @@ -8438,7 +8445,7 @@ dependencies = [ [[package]] name = "zksync_eth_signer" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "async-trait", "rlp", @@ -8449,9 +8456,9 @@ dependencies = [ [[package]] name = "zksync_ff" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5aa518ed0ea7ef737d50de02025f5a593dbb11104b3c1bf5a00f39581b47dc" +checksum = "6583c2db6dc787600879d27ec98d2eb628a757ee41831e54f8be1dae4acc599f" dependencies = [ "byteorder", "hex", @@ -8462,9 +8469,9 @@ dependencies = [ [[package]] name = "zksync_ff_derive" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b43100a1278e2f64820368db8751c2441860ea74ab5749074cf8f864647af" +checksum = "8f62e93dde881d8dd44d1864c7682394dde6d18e582fc5af78768221a1766fdf" dependencies = [ "num-bigint 0.4.6", "num-integer", @@ -8477,9 +8484,9 @@ dependencies = [ [[package]] name = "zksync_kzg" -version = "0.150.19" +version = "0.150.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9da880b8282a97d9dfd6ac9f0189d310c0602059a8de20aa66a883979d6adba" +checksum = "174f82592590901cbcf2b298059c89f817b404299ffbd050a3915ea72357f545" dependencies = [ "boojum", "derivative", @@ -8494,7 +8501,7 @@ dependencies = [ [[package]] name = "zksync_l1_contract_interface" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "circuit_definitions", @@ -8511,7 +8518,7 @@ dependencies = [ [[package]] name = "zksync_mini_merkle_tree" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "once_cell", "zksync_basic_types", @@ -8520,7 +8527,7 @@ dependencies = [ [[package]] name = "zksync_multivm" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "circuit_sequencer_api", @@ -8535,7 +8542,7 @@ dependencies = [ "zk_evm 0.133.0", "zk_evm 0.140.0", "zk_evm 0.141.0", - "zk_evm 0.150.19", + "zk_evm 0.150.20", "zksync_contracts", "zksync_mini_merkle_tree", "zksync_system_constants", @@ -8546,7 +8553,7 @@ dependencies = [ [[package]] name = "zksync_object_store" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "async-trait", @@ -8569,9 +8576,9 @@ dependencies = [ [[package]] name = "zksync_pairing" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f0d96f3e386f3b4c76a614d73b71714d6712e917d462bf8053b8af352da0b3" +checksum = "baafdd03ca7a48dc9b6808be3630f2d8a003aa425d71946e9158d8c0aeb1cc79" dependencies = [ "byteorder", "cfg-if", @@ -8582,7 +8589,7 @@ dependencies = [ [[package]] name = "zksync_proof_fri_compressor" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8619,9 +8626,9 @@ dependencies = [ [[package]] name = "zksync_protobuf" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8986ad796f8e00d8999fee72effba1a21bce40f5f877d681ac9cd89a94834d8" +checksum = "d9032e12528c2466293b206d6edb53b7e900e4a4cc4573e4d075ac2dc00e1b55" dependencies = [ "anyhow", "bit-vec 0.6.3", @@ -8640,9 +8647,9 @@ dependencies = [ [[package]] name = "zksync_protobuf_build" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d870b31995e3acb8e47afeb68ebeeffcf6121e70020e65b3d5d31692115d236" +checksum = "7c644fc8ef3c4d343ea42cebd5551e3562933f15dd9b0e68a52c2657603eb0f5" dependencies = [ "anyhow", "heck 0.5.0", @@ -8657,7 +8664,7 @@ dependencies = [ [[package]] name = "zksync_protobuf_config" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "hex", @@ -8677,7 +8684,7 @@ dependencies = [ [[package]] name = "zksync_prover_autoscaler" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8714,7 +8721,7 @@ dependencies = [ [[package]] name = "zksync_prover_dal" -version = "0.1.0" +version = "17.1.1" dependencies = [ "sqlx", "strum", @@ -8724,7 +8731,7 @@ dependencies = [ [[package]] name = "zksync_prover_fri" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8758,7 +8765,7 @@ dependencies = [ [[package]] name = "zksync_prover_fri_gateway" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8784,7 +8791,7 @@ dependencies = [ [[package]] name = "zksync_prover_fri_types" -version = "0.1.0" +version = "17.1.1" dependencies = [ "circuit_definitions", "serde", @@ -8794,7 +8801,7 @@ dependencies = [ [[package]] name = "zksync_prover_fri_utils" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "regex", @@ -8812,7 +8819,7 @@ dependencies = [ [[package]] name = "zksync_prover_interface" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "chrono", "circuit_definitions", @@ -8821,6 +8828,7 @@ dependencies = [ "serde", "serde_with", "strum", + "zksync_bellman", "zksync_object_store", "zksync_types", "zksync_vm_interface", @@ -8828,7 +8836,7 @@ dependencies = [ [[package]] name = "zksync_prover_job_monitor" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8850,7 +8858,7 @@ dependencies = [ [[package]] name = "zksync_prover_job_processor" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -8865,7 +8873,7 @@ dependencies = [ [[package]] name = "zksync_prover_keystore" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "bincode", @@ -8878,6 +8886,7 @@ dependencies = [ "hex", "md5", "once_cell", + "proof-compression", "serde", "serde_json", "sha3 0.10.8", @@ -8892,7 +8901,7 @@ dependencies = [ [[package]] name = "zksync_queued_job_processor" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "async-trait", @@ -8904,9 +8913,9 @@ dependencies = [ [[package]] name = "zksync_solidity_vk_codegen" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb10f377dcc24fe2268cc5f530c16af1c879a791570d8fe64064b58ba143c7cc" +checksum = "bb05a12f5552d7947427f755e29f548ce94733851f1fa16edaf8b75c28033e73" dependencies = [ "ethereum-types", "franklin-crypto", @@ -8921,7 +8930,7 @@ dependencies = [ [[package]] name = "zksync_system_constants" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "once_cell", "zksync_basic_types", @@ -8929,7 +8938,7 @@ dependencies = [ [[package]] name = "zksync_types" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "async-trait", @@ -8962,7 +8971,7 @@ dependencies = [ [[package]] name = "zksync_utils" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "futures 0.3.31", @@ -8977,7 +8986,7 @@ dependencies = [ [[package]] name = "zksync_vk_setup_data_generator_server_fri" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "bincode", @@ -9001,7 +9010,7 @@ dependencies = [ [[package]] name = "zksync_vlog" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "chrono", @@ -9031,8 +9040,8 @@ source = "git+https://github.com/matter-labs/vm2.git?rev=457d8a7eea9093af9440662 dependencies = [ "enum_dispatch", "primitive-types", - "zk_evm_abstractions 0.150.19", - "zkevm_opcode_defs 0.150.19", + "zk_evm_abstractions 0.150.20", + "zkevm_opcode_defs 0.150.20", "zksync_vm2_interface", ] @@ -9046,7 +9055,7 @@ dependencies = [ [[package]] name = "zksync_vm_interface" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "async-trait", @@ -9062,7 +9071,7 @@ dependencies = [ [[package]] name = "zksync_web3_decl" -version = "0.1.0" +version = "26.1.0-non-semver-compat" dependencies = [ "anyhow", "async-trait", @@ -9083,7 +9092,7 @@ dependencies = [ [[package]] name = "zksync_witness_generator" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", @@ -9121,7 +9130,7 @@ dependencies = [ [[package]] name = "zksync_witness_vector_generator" -version = "0.1.0" +version = "17.1.1" dependencies = [ "anyhow", "async-trait", From 514900fcc1c76e6689de7d35fde527af334376f4 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 24 Jan 2025 14:54:42 +0400 Subject: [PATCH 06/16] Add storage migration --- core/bin/contract-verifier/src/main.rs | 30 +++- .../src/contract_identifier.rs | 0 ...535cb2b88fe85d7cff6514e331124be845ec0.json | 29 ++++ ...7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json | 17 +++ core/lib/dal/src/contract_verification_dal.rs | 129 ++++++++++++++++++ core/tests/ts-integration/hardhat.config.ts | 2 +- .../ts-integration/scripts/compile-yul.ts | 2 +- .../tests/api/contract-verification.test.ts | 2 +- .../ts-integration/tests/api/debug.test.ts | 2 +- 9 files changed, 208 insertions(+), 5 deletions(-) delete mode 100644 core/lib/contract_verifier/src/contract_identifier.rs create mode 100644 core/lib/dal/.sqlx/query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json create mode 100644 core/lib/dal/.sqlx/query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index ab86c147977d..2bd32b2c4561 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -6,7 +6,7 @@ use tokio::sync::watch; use zksync_config::configs::PrometheusConfig; use zksync_contract_verifier_lib::ContractVerifier; use zksync_core_leftovers::temp_config_store::{load_database_secrets, load_general_config}; -use zksync_dal::{ConnectionPool, Core}; +use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::ManagedTasks; use zksync_vlog::prometheus::PrometheusExporterConfig; @@ -25,6 +25,32 @@ struct Opt { secrets_path: Option, } +async fn perform_storage_migration(pool: &ConnectionPool) -> anyhow::Result<()> { + const BATCH_SIZE: usize = 1000; + + // Make it possible to override just in case. + let batch_size = std::env::var("CONTRACT_VERIFIER_MIGRATION_BATCH_SIZE") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(BATCH_SIZE); + + let mut storage = pool.connection().await?; + let migration_performed = storage + .contract_verification_dal() + .is_verification_info_migration_performed() + .await?; + if !migration_performed { + tracing::info!("Running the storage migration for the contract verifier table"); + storage + .contract_verification_dal() + .perform_verification_info_migration(batch_size) + .await?; + } else { + tracing::info!("Storage migration is not needed"); + } + Ok(()) +} + #[tokio::main] async fn main() -> anyhow::Result<()> { let opt = Opt::parse(); @@ -51,6 +77,8 @@ async fn main() -> anyhow::Result<()> { .build() .await?; + perform_storage_migration(&pool).await?; + let (stop_sender, stop_receiver) = watch::channel(false); let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool) .await diff --git a/core/lib/contract_verifier/src/contract_identifier.rs b/core/lib/contract_verifier/src/contract_identifier.rs deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/core/lib/dal/.sqlx/query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json b/core/lib/dal/.sqlx/query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json new file mode 100644 index 000000000000..8cdbd1f9ce27 --- /dev/null +++ b/core/lib/dal/.sqlx/query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n address,\n verification_info\n FROM\n contracts_verification_info\n ORDER BY\n address\n OFFSET $1\n LIMIT $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "address", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "verification_info", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0" +} diff --git a/core/lib/dal/.sqlx/query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json b/core/lib/dal/.sqlx/query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json new file mode 100644 index 000000000000..64af462c70a1 --- /dev/null +++ b/core/lib/dal/.sqlx/query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n contract_verification_info_v2 (\n initial_contract_addr,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n SELECT\n u.address,\n u.bytecode_keccak256,\n u.bytecode_without_metadata_keccak256,\n u.verification_info\n FROM\n UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSON []) AS u (\n address,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n ON CONFLICT (initial_contract_addr) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "ByteaArray", + "ByteaArray", + "ByteaArray", + "JsonArray" + ] + }, + "nullable": [] + }, + "hash": "92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee" +} diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 2687f0df0054..8e73ad572835 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -733,6 +733,135 @@ impl ContractVerificationDal<'_, '_> { Ok(count_v2 >= count_v1) } + + pub async fn perform_verification_info_migration( + &mut self, + batch_size: usize, + ) -> anyhow::Result<()> { + // Offset is a number of already migrated contracts. + let mut offset = sqlx::query!( + r#" + SELECT + COUNT(*) + FROM + contract_verification_info_v2 + "#, + ) + .instrument("perform_verification_info_migration#count") + .fetch_one(self.storage) + .await? + .count + .unwrap() as usize; + + loop { + let mut transaction = self.storage.start_transaction().await?; + let contracts: Vec<(Vec, serde_json::Value)> = sqlx::query!( + r#" + SELECT + address, + verification_info + FROM + contracts_verification_info + ORDER BY + address + OFFSET $1 + LIMIT $2 + "#, + offset as i64, + batch_size as i64, + ) + .instrument("perform_verification_info_migration#select") + .with_arg("offset", &offset) + .with_arg("batch_size", &batch_size) + .fetch_all(&mut transaction) + .await? + .into_iter() + .filter_map(|row| row.verification_info.map(|info| (row.address, info))) + .collect(); + + if contracts.is_empty() { + tracing::info!("No more contracts to process"); + break; + } + + tracing::info!( + "Processing {} contracts; offset {}", + contracts.len(), + offset + ); + + let mut addresses = Vec::with_capacity(batch_size); + let mut verification_infos = Vec::with_capacity(batch_size); + let mut bytecode_keccak256s = Vec::with_capacity(batch_size); + let mut bytecode_without_metadata_keccak256s = Vec::with_capacity(batch_size); + + for (address, info_json) in contracts { + let verification_info = + serde_json::from_value::(info_json.clone()) + .context("Failed to deserialize verification info")?; + let bytecode_marker = if verification_info.artifacts.deployed_bytecode.is_some() { + BytecodeMarker::Evm + } else { + BytecodeMarker::EraVm + }; + let identifier = ContractIdentifier::from_bytecode( + bytecode_marker, + verification_info.artifacts.deployed_bytecode(), + ); + + addresses.push(address); + verification_infos.push(info_json); + bytecode_keccak256s.push(identifier.bytecode_sha3.as_bytes().to_vec()); + bytecode_without_metadata_keccak256s.push( + identifier + .bytecode_without_metadata_sha3 + .as_ref() + .map(|h| h.hash().as_bytes().to_vec()) + .unwrap_or_default(), + ); + } + + // Insert all the values + sqlx::query!( + r#" + INSERT INTO + contract_verification_info_v2 ( + initial_contract_addr, + bytecode_keccak256, + bytecode_without_metadata_keccak256, + verification_info + ) + SELECT + u.address, + u.bytecode_keccak256, + u.bytecode_without_metadata_keccak256, + u.verification_info + FROM + UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSON []) AS u ( + address, + bytecode_keccak256, + bytecode_without_metadata_keccak256, + verification_info + ) + ON CONFLICT (initial_contract_addr) DO NOTHING + "#, + &addresses as _, + &bytecode_keccak256s as _, + &bytecode_without_metadata_keccak256s as _, + &verification_infos as _, + ) + .instrument("perform_verification_info_migration#insert") + .with_arg("offset", &offset) + .with_arg("batch_size", &batch_size) + .execute(&mut transaction) + .await?; + + offset += batch_size; + transaction.commit().await?; + } + + Ok(()) + } } #[cfg(test)] diff --git a/core/tests/ts-integration/hardhat.config.ts b/core/tests/ts-integration/hardhat.config.ts index a96a83ca3ee3..20f3ecd4f4f7 100644 --- a/core/tests/ts-integration/hardhat.config.ts +++ b/core/tests/ts-integration/hardhat.config.ts @@ -4,7 +4,7 @@ import '@matterlabs/hardhat-zksync-vyper'; export default { zksolc: { - version: '1.5.3', + version: '1.5.10', compilerSource: 'binary', settings: { enableEraVMExtensions: true diff --git a/core/tests/ts-integration/scripts/compile-yul.ts b/core/tests/ts-integration/scripts/compile-yul.ts index 876caacdfab3..868f7d10ae6f 100644 --- a/core/tests/ts-integration/scripts/compile-yul.ts +++ b/core/tests/ts-integration/scripts/compile-yul.ts @@ -7,7 +7,7 @@ import { getZksolcUrl, saltFromUrl } from '@matterlabs/hardhat-zksync-solc'; import { getCompilersDir } from 'hardhat/internal/util/global-dir'; import path from 'path'; -const COMPILER_VERSION = '1.5.3'; +const COMPILER_VERSION = '1.5.10'; const IS_COMPILER_PRE_RELEASE = false; async function compilerLocation(): Promise { diff --git a/core/tests/ts-integration/tests/api/contract-verification.test.ts b/core/tests/ts-integration/tests/api/contract-verification.test.ts index 8f8830ce7516..21657bec9950 100644 --- a/core/tests/ts-integration/tests/api/contract-verification.test.ts +++ b/core/tests/ts-integration/tests/api/contract-verification.test.ts @@ -10,7 +10,7 @@ import { NodeMode } from '../../src/types'; // Regular expression to match ISO dates. const DATE_REGEX = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?/; -const ZKSOLC_VERSION = 'v1.5.3'; +const ZKSOLC_VERSION = 'v1.5.10'; const SOLC_VERSION = '0.8.26'; const ZK_VM_SOLC_VERSION = 'zkVM-0.8.26-1.0.1'; diff --git a/core/tests/ts-integration/tests/api/debug.test.ts b/core/tests/ts-integration/tests/api/debug.test.ts index 2af18c8438b8..8cde65ac2555 100644 --- a/core/tests/ts-integration/tests/api/debug.test.ts +++ b/core/tests/ts-integration/tests/api/debug.test.ts @@ -29,7 +29,7 @@ describe('Debug methods', () => { test('Should not fail for infinity recursion', async () => { const bytecodePath = `${ testMaster.environment().pathToHome - }/core/tests/ts-integration/contracts/zkasm/artifacts/deep_stak.zkasm/zkasm/deep_stak.zkasm.zbin`; + }/core/tests/ts-integration/contracts/zkasm/artifacts/deep_stak.zkasm/deep_stak.zkasm/deep_stak.zkasm.zbin`; const bytecode = fs.readFileSync(bytecodePath, 'utf-8'); const contractFactory = new zksync.ContractFactory([], bytecode, testMaster.mainAccount()); From ca16c776f9cf3673c1fa9110a14e92ce37f14279 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 28 Jan 2025 10:52:58 +0400 Subject: [PATCH 07/16] Speed up the migration --- core/Cargo.lock | 1 + ...083912044f8f5328a867e9ed1a063eb1f528.json} | 6 +- ...85d93fdc96b6cd8fdf10663ea891e135ee13.json} | 8 +- core/lib/dal/Cargo.toml | 1 + core/lib/dal/src/contract_verification_dal.rs | 98 +++++++++---------- prover/Cargo.lock | 1 + 6 files changed, 58 insertions(+), 57 deletions(-) rename core/lib/dal/.sqlx/{query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json => query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json} (65%) rename core/lib/dal/.sqlx/{query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json => query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json} (52%) diff --git a/core/Cargo.lock b/core/Cargo.lock index 2da5202db00b..d17074daa84e 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -11848,6 +11848,7 @@ dependencies = [ "itertools 0.10.5", "prost 0.12.6", "rand 0.8.5", + "rayon", "serde", "serde_json", "sqlx", diff --git a/core/lib/dal/.sqlx/query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json b/core/lib/dal/.sqlx/query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json similarity index 65% rename from core/lib/dal/.sqlx/query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json rename to core/lib/dal/.sqlx/query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json index 64af462c70a1..e7eb33ccfa63 100644 --- a/core/lib/dal/.sqlx/query-92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee.json +++ b/core/lib/dal/.sqlx/query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n contract_verification_info_v2 (\n initial_contract_addr,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n SELECT\n u.address,\n u.bytecode_keccak256,\n u.bytecode_without_metadata_keccak256,\n u.verification_info\n FROM\n UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSON []) AS u (\n address,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n ON CONFLICT (initial_contract_addr) DO NOTHING\n ", + "query": "\n INSERT INTO\n contract_verification_info_v2 (\n initial_contract_addr,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n SELECT\n u.address,\n u.bytecode_keccak256,\n u.bytecode_without_metadata_keccak256,\n u.verification_info\n FROM\n UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSONB []) AS u (\n address,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n ON CONFLICT (initial_contract_addr) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -8,10 +8,10 @@ "ByteaArray", "ByteaArray", "ByteaArray", - "JsonArray" + "JsonbArray" ] }, "nullable": [] }, - "hash": "92bcaf1ae3762e92f92d5b49c7b7c4b74c8e29e9595c8e4e15ca88a81b7d17ee" + "hash": "2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528" } diff --git a/core/lib/dal/.sqlx/query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json b/core/lib/dal/.sqlx/query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json similarity index 52% rename from core/lib/dal/.sqlx/query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json rename to core/lib/dal/.sqlx/query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json index 8cdbd1f9ce27..e42bcd9d53c5 100644 --- a/core/lib/dal/.sqlx/query-5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0.json +++ b/core/lib/dal/.sqlx/query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n address,\n verification_info\n FROM\n contracts_verification_info\n ORDER BY\n address\n OFFSET $1\n LIMIT $2\n ", + "query": "\n SELECT\n address,\n verification_info::text as verification_info\n FROM\n contracts_verification_info\n ORDER BY\n address\n OFFSET $1\n LIMIT $2\n ", "describe": { "columns": [ { @@ -11,7 +11,7 @@ { "ordinal": 1, "name": "verification_info", - "type_info": "Jsonb" + "type_info": "Text" } ], "parameters": { @@ -22,8 +22,8 @@ }, "nullable": [ false, - true + null ] }, - "hash": "5dc32dc8aab39492d24b9638ef5535cb2b88fe85d7cff6514e331124be845ec0" + "hash": "c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13" } diff --git a/core/lib/dal/Cargo.toml b/core/lib/dal/Cargo.toml index 4b093dd181bb..80cd955ff298 100644 --- a/core/lib/dal/Cargo.toml +++ b/core/lib/dal/Cargo.toml @@ -53,6 +53,7 @@ hex.workspace = true strum = { workspace = true, features = ["derive"] } tracing.workspace = true chrono = { workspace = true, features = ["serde"] } +rayon.workspace = true [dev-dependencies] zksync_test_contracts.workspace = true diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 8e73ad572835..ac1930987ade 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -6,6 +6,7 @@ use std::{ }; use anyhow::Context as _; +use rayon::prelude::*; use sqlx::postgres::types::PgInterval; use zksync_db_connection::{error::SqlxContext, instrument::InstrumentExt}; use zksync_types::{ @@ -738,28 +739,19 @@ impl ContractVerificationDal<'_, '_> { &mut self, batch_size: usize, ) -> anyhow::Result<()> { - // Offset is a number of already migrated contracts. - let mut offset = sqlx::query!( - r#" - SELECT - COUNT(*) - FROM - contract_verification_info_v2 - "#, - ) - .instrument("perform_verification_info_migration#count") - .fetch_one(self.storage) - .await? - .count - .unwrap() as usize; + // We use a long-running transaction, since the migration is one-time and during it + // no writes are expected to the tables, so locked rows are not a problem. + let mut transaction = self.storage.start_transaction().await?; + // Offset is a number of already migrated contracts. + let mut offset = 0usize; loop { - let mut transaction = self.storage.start_transaction().await?; - let contracts: Vec<(Vec, serde_json::Value)> = sqlx::query!( + // Fetch JSON as text to avoid roundtrip through `serde_json::Value`, as it's super slow. + let (addresses, verification_infos): (Vec>, Vec) = sqlx::query!( r#" SELECT address, - verification_info + verification_info::text as verification_info FROM contracts_verification_info ORDER BY @@ -779,47 +771,53 @@ impl ContractVerificationDal<'_, '_> { .filter_map(|row| row.verification_info.map(|info| (row.address, info))) .collect(); - if contracts.is_empty() { + if addresses.is_empty() { tracing::info!("No more contracts to process"); break; } tracing::info!( "Processing {} contracts; offset {}", - contracts.len(), + addresses.len(), offset ); - let mut addresses = Vec::with_capacity(batch_size); - let mut verification_infos = Vec::with_capacity(batch_size); - let mut bytecode_keccak256s = Vec::with_capacity(batch_size); - let mut bytecode_without_metadata_keccak256s = Vec::with_capacity(batch_size); - - for (address, info_json) in contracts { - let verification_info = - serde_json::from_value::(info_json.clone()) - .context("Failed to deserialize verification info")?; - let bytecode_marker = if verification_info.artifacts.deployed_bytecode.is_some() { - BytecodeMarker::Evm - } else { - BytecodeMarker::EraVm - }; - let identifier = ContractIdentifier::from_bytecode( - bytecode_marker, - verification_info.artifacts.deployed_bytecode(), - ); + let (bytecode_keccak256s, bytecode_without_metadata_keccak256s): ( + Vec>, + Vec>, + ) = (0..addresses.len()) + .into_par_iter() + .map(|idx| { + let address = &addresses[idx]; + let info_json = &verification_infos[idx]; + let verification_info = serde_json::from_str::(info_json) + .unwrap_or_else(|err| { + panic!( + "Malformed data in DB, address {}, data: {info_json}, error: {err}", + hex::encode(address) + ); + }); + let bytecode_marker = if verification_info.artifacts.deployed_bytecode.is_some() + { + BytecodeMarker::Evm + } else { + BytecodeMarker::EraVm + }; + let identifier = ContractIdentifier::from_bytecode( + bytecode_marker, + verification_info.artifacts.deployed_bytecode(), + ); - addresses.push(address); - verification_infos.push(info_json); - bytecode_keccak256s.push(identifier.bytecode_sha3.as_bytes().to_vec()); - bytecode_without_metadata_keccak256s.push( - identifier - .bytecode_without_metadata_sha3 - .as_ref() - .map(|h| h.hash().as_bytes().to_vec()) - .unwrap_or_default(), - ); - } + ( + identifier.bytecode_sha3.as_bytes().to_vec(), + identifier + .bytecode_without_metadata_sha3 + .as_ref() + .map(|h| h.hash().as_bytes().to_vec()) + .unwrap_or_default(), + ) + }) + .unzip(); // Insert all the values sqlx::query!( @@ -837,7 +835,7 @@ impl ContractVerificationDal<'_, '_> { u.bytecode_without_metadata_keccak256, u.verification_info FROM - UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSON []) AS u ( + UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSONB []) AS u ( address, bytecode_keccak256, bytecode_without_metadata_keccak256, @@ -857,9 +855,9 @@ impl ContractVerificationDal<'_, '_> { .await?; offset += batch_size; - transaction.commit().await?; } + transaction.commit().await?; Ok(()) } } diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 8748b27a52ca..f266f880201b 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -8376,6 +8376,7 @@ dependencies = [ "itertools 0.10.5", "prost 0.12.6", "rand 0.8.5", + "rayon", "serde", "serde_json", "sqlx", From e138d6b0ccf9ece4dff2eca1a3f7158f0490b7da Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 28 Jan 2025 11:16:16 +0400 Subject: [PATCH 08/16] Simplify Compiler interface --- .../contract_verifier/src/compilers/solc.rs | 14 +--- .../contract_verifier/src/compilers/vyper.rs | 14 +--- .../contract_verifier/src/compilers/zksolc.rs | 22 ++--- .../src/compilers/zkvyper.rs | 13 +-- core/lib/contract_verifier/src/lib.rs | 14 ++-- .../lib/contract_verifier/src/resolver/mod.rs | 6 +- core/lib/contract_verifier/src/tests/mod.rs | 81 ++++++------------- core/lib/contract_verifier/src/tests/real.rs | 43 +++++++--- 8 files changed, 79 insertions(+), 128 deletions(-) diff --git a/core/lib/contract_verifier/src/compilers/solc.rs b/core/lib/contract_verifier/src/compilers/solc.rs index fc165227924a..7c6fb5100e57 100644 --- a/core/lib/contract_verifier/src/compilers/solc.rs +++ b/core/lib/contract_verifier/src/compilers/solc.rs @@ -3,12 +3,8 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio}; use anyhow::Context; use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; -use zksync_types::{ - bytecode::BytecodeMarker, - contract_verification::{ - api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest}, - contract_identifier::ContractIdentifier, - }, +use zksync_types::contract_verification::api::{ + CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, }; use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson}; @@ -107,7 +103,7 @@ impl Compiler for Solc { async fn compile( self: Box, input: SolcInput, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let mut command = tokio::process::Command::new(&self.path); let mut child = command .arg("--standard-json") @@ -134,9 +130,7 @@ impl Compiler for Solc { .context("zksolc output is not valid JSON")?; let output = parse_standard_json_output(&output, input.contract_name, input.file_name, true)?; - let id = - ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode()); - Ok((output, id)) + Ok(output) } else { Err(ContractVerifierError::CompilerError( "solc", diff --git a/core/lib/contract_verifier/src/compilers/vyper.rs b/core/lib/contract_verifier/src/compilers/vyper.rs index ea9500da1557..50d73ac259cb 100644 --- a/core/lib/contract_verifier/src/compilers/vyper.rs +++ b/core/lib/contract_verifier/src/compilers/vyper.rs @@ -3,12 +3,8 @@ use std::{collections::HashMap, mem, path::PathBuf, process::Stdio}; use anyhow::Context; use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; -use zksync_types::{ - bytecode::BytecodeMarker, - contract_verification::{ - api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest}, - contract_identifier::ContractIdentifier, - }, +use zksync_types::contract_verification::api::{ + CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, }; use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson}; @@ -80,7 +76,7 @@ impl Compiler for Vyper { async fn compile( self: Box, mut input: VyperInput, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let mut command = tokio::process::Command::new(&self.path); let mut child = command .arg("--standard-json") @@ -109,9 +105,7 @@ impl Compiler for Vyper { serde_json::from_slice(&output.stdout).context("vyper output is not valid JSON")?; let output = parse_standard_json_output(&output, input.contract_name, input.file_name, true)?; - let id = - ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode()); - Ok((output, id)) + Ok(output) } else { Err(ContractVerifierError::CompilerError( "vyper", diff --git a/core/lib/contract_verifier/src/compilers/zksolc.rs b/core/lib/contract_verifier/src/compilers/zksolc.rs index 55fb2ca57f59..02114832874f 100644 --- a/core/lib/contract_verifier/src/compilers/zksolc.rs +++ b/core/lib/contract_verifier/src/compilers/zksolc.rs @@ -6,12 +6,8 @@ use semver::Version; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use zksync_queued_job_processor::async_trait; -use zksync_types::{ - bytecode::BytecodeMarker, - contract_verification::{ - api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest}, - contract_identifier::ContractIdentifier, - }, +use zksync_types::contract_verification::api::{ + CompilationArtifacts, SourceCodeData, VerificationIncomingRequest, }; use super::{parse_standard_json_output, process_contract_name, Source}; @@ -196,7 +192,7 @@ impl Compiler for ZkSolc { async fn compile( self: Box, input: ZkSolcInput, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let mut command = tokio::process::Command::new(&self.paths.zk); command.stdout(Stdio::piped()).stderr(Stdio::piped()); @@ -257,11 +253,7 @@ impl Compiler for ZkSolc { .context("zksolc output is not valid JSON")?; let output = parse_standard_json_output(&output, contract_name, file_name, false)?; - let id = ContractIdentifier::from_bytecode( - BytecodeMarker::EraVm, - output.deployed_bytecode(), - ); - Ok((output, id)) + Ok(output) } else { Err(ContractVerifierError::CompilerError( "zksolc", @@ -294,11 +286,7 @@ impl Compiler for ZkSolc { let output = String::from_utf8(output.stdout).context("zksolc output is not UTF-8")?; let output = Self::parse_single_file_yul_output(&output)?; - let id = ContractIdentifier::from_bytecode( - BytecodeMarker::EraVm, - output.deployed_bytecode(), - ); - Ok((output, id)) + Ok(output) } else { Err(ContractVerifierError::CompilerError( "zksolc", diff --git a/core/lib/contract_verifier/src/compilers/zkvyper.rs b/core/lib/contract_verifier/src/compilers/zkvyper.rs index 2a24c7cd2b02..a5dfbaadcee3 100644 --- a/core/lib/contract_verifier/src/compilers/zkvyper.rs +++ b/core/lib/contract_verifier/src/compilers/zkvyper.rs @@ -3,10 +3,7 @@ use std::{ffi::OsString, path, path::Path, process::Stdio}; use anyhow::Context as _; use tokio::{fs, io::AsyncWriteExt}; use zksync_queued_job_processor::async_trait; -use zksync_types::{ - bytecode::BytecodeMarker, - contract_verification::{api::CompilationArtifacts, contract_identifier::ContractIdentifier}, -}; +use zksync_types::contract_verification::api::CompilationArtifacts; use super::VyperInput; use crate::{ @@ -98,7 +95,7 @@ impl Compiler for ZkVyper { async fn compile( self: Box, input: VyperInput, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let mut command = tokio::process::Command::new(&self.paths.zk); if let Some(o) = input.optimizer_mode.as_ref() { command.arg("-O").arg(o); @@ -127,11 +124,7 @@ impl Compiler for ZkVyper { let output = serde_json::from_slice(&output.stdout) .context("zkvyper output is not valid JSON")?; let output = Self::parse_output(&output, input.contract_name)?; - let id = ContractIdentifier::from_bytecode( - BytecodeMarker::EraVm, - output.deployed_bytecode(), - ); - Ok((output, id)) + Ok(output) } else { Err(ContractVerifierError::CompilerError( "zkvyper", diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 4626897b264a..2178aa332239 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -247,7 +247,9 @@ impl ContractVerifier { let bytecode_marker = BytecodeMarker::new(deployed_contract.bytecode_hash) .context("unknown bytecode kind")?; - let (artifacts, identifier) = self.compile(request.req.clone(), bytecode_marker).await?; + let artifacts = self.compile(request.req.clone(), bytecode_marker).await?; + let identifier = + ContractIdentifier::from_bytecode(bytecode_marker, artifacts.deployed_bytecode()); let constructor_args = match bytecode_marker { BytecodeMarker::EraVm => self .decode_era_vm_constructor_args(&deployed_contract, request.req.contract_address)?, @@ -329,7 +331,7 @@ impl ContractVerifier { &self, version: &ZkCompilerVersions, req: VerificationIncomingRequest, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let zksolc = self.compiler_resolver.resolve_zksolc(version).await?; tracing::debug!(?zksolc, ?version, "resolved compiler"); let input = ZkSolc::build_input(req)?; @@ -343,7 +345,7 @@ impl ContractVerifier { &self, version: &ZkCompilerVersions, req: VerificationIncomingRequest, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let zkvyper = self.compiler_resolver.resolve_zkvyper(version).await?; tracing::debug!(?zkvyper, ?version, "resolved compiler"); let input = VyperInput::new(req)?; @@ -356,7 +358,7 @@ impl ContractVerifier { &self, version: &str, req: VerificationIncomingRequest, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let solc = self.compiler_resolver.resolve_solc(version).await?; tracing::debug!(?solc, ?req.compiler_versions, "resolved compiler"); let input = Solc::build_input(req)?; @@ -370,7 +372,7 @@ impl ContractVerifier { &self, version: &str, req: VerificationIncomingRequest, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let vyper = self.compiler_resolver.resolve_vyper(version).await?; tracing::debug!(?vyper, ?req.compiler_versions, "resolved compiler"); let input = VyperInput::new(req)?; @@ -385,7 +387,7 @@ impl ContractVerifier { &self, req: VerificationIncomingRequest, bytecode_marker: BytecodeMarker, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { let compiler_type = req.source_code_data.compiler_type(); let compiler_type_by_versions = req.compiler_versions.compiler_type(); if compiler_type != compiler_type_by_versions { diff --git a/core/lib/contract_verifier/src/resolver/mod.rs b/core/lib/contract_verifier/src/resolver/mod.rs index f71bf72ebb09..b2bd659408d3 100644 --- a/core/lib/contract_verifier/src/resolver/mod.rs +++ b/core/lib/contract_verifier/src/resolver/mod.rs @@ -8,9 +8,7 @@ use std::{ use anyhow::Context as _; use tokio::fs; use zksync_queued_job_processor::async_trait; -use zksync_types::contract_verification::{ - api::CompilationArtifacts, contract_identifier::ContractIdentifier, -}; +use zksync_types::contract_verification::api::CompilationArtifacts; pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver}; use crate::{ @@ -163,7 +161,7 @@ pub(crate) trait Compiler: Send + fmt::Debug { async fn compile( self: Box, input: In, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError>; + ) -> Result; } #[derive(Debug)] diff --git a/core/lib/contract_verifier/src/tests/mod.rs b/core/lib/contract_verifier/src/tests/mod.rs index ebd6469084da..31cec03e138e 100644 --- a/core/lib/contract_verifier/src/tests/mod.rs +++ b/core/lib/contract_verifier/src/tests/mod.rs @@ -208,11 +208,8 @@ async fn mock_deployment_inner( .unwrap(); } -type SharedMockFn = Arc< - dyn Fn(In) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> - + Send - + Sync, ->; +type SharedMockFn = + Arc Result + Send + Sync>; #[derive(Clone)] struct MockCompilerResolver { @@ -230,10 +227,7 @@ impl fmt::Debug for MockCompilerResolver { impl MockCompilerResolver { fn zksolc( - zksolc: impl Fn(ZkSolcInput) -> (CompilationArtifacts, ContractIdentifier) - + 'static - + Send - + Sync, + zksolc: impl Fn(ZkSolcInput) -> CompilationArtifacts + 'static + Send + Sync, ) -> Self { Self { zksolc: Arc::new(move |input| Ok(zksolc(input))), @@ -241,9 +235,7 @@ impl MockCompilerResolver { } } - fn solc( - solc: impl Fn(SolcInput) -> (CompilationArtifacts, ContractIdentifier) + 'static + Send + Sync, - ) -> Self { + fn solc(solc: impl Fn(SolcInput) -> CompilationArtifacts + 'static + Send + Sync) -> Self { Self { solc: Arc::new(move |input| Ok(solc(input))), zksolc: Arc::new(|input| panic!("unexpected zksolc call: {input:?}")), @@ -256,7 +248,7 @@ impl Compiler for MockCompilerResolver { async fn compile( self: Box, input: ZkSolcInput, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { (self.zksolc)(input) } } @@ -266,7 +258,7 @@ impl Compiler for MockCompilerResolver { async fn compile( self: Box, input: SolcInput, - ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> { + ) -> Result { (self.solc)(input) } } @@ -412,14 +404,11 @@ async fn contract_verifier_basics(contract: TestContract) { let source = input.sources.values().next().unwrap(); assert!(source.content.contains("contract Counter"), "{source:?}"); - let artifacts = CompilationArtifacts { + CompilationArtifacts { bytecode: vec![0; 32], deployed_bytecode: None, abi: counter_contract_abi(), - }; - let contract_id = - ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, artifacts.deployed_bytecode()); - (artifacts, contract_id) + } }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), @@ -539,15 +528,13 @@ async fn verifying_evm_bytecode(contract: TestContract) { deployed_bytecode: Some(deployed_bytecode), abi: counter_contract_abi(), }; - let contract_id = - ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, artifacts.deployed_bytecode()); let mock_resolver = MockCompilerResolver::solc(move |input| { assert_eq!(input.standard_json.language, "Solidity"); assert_eq!(input.standard_json.sources.len(), 1); let source = input.standard_json.sources.values().next().unwrap(); assert!(source.content.contains("contract Counter"), "{source:?}"); - (artifacts.clone(), contract_id) + artifacts.clone() }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), @@ -578,15 +565,10 @@ async fn bytecode_mismatch_error() { .await .unwrap(); - let mock_resolver = MockCompilerResolver::zksolc(|_| { - let artifacts = CompilationArtifacts { - bytecode: vec![0; 32], - deployed_bytecode: None, - abi: counter_contract_abi(), - }; - let contract_id = - ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, artifacts.deployed_bytecode()); - (artifacts, contract_id) + let mock_resolver = MockCompilerResolver::zksolc(|_| CompilationArtifacts { + bytecode: vec![0; 32], + deployed_bytecode: None, + abi: counter_contract_abi(), }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), @@ -663,29 +645,15 @@ async fn args_mismatch_error(contract: TestContract, bytecode_kind: BytecodeMark .unwrap(); let mock_resolver = match bytecode_kind { - BytecodeMarker::EraVm => MockCompilerResolver::zksolc(move |_| { - let artifacts = CompilationArtifacts { - bytecode: bytecode.clone(), - deployed_bytecode: None, - abi: counter_contract_abi(), - }; - let contract_id = ContractIdentifier::from_bytecode( - BytecodeMarker::EraVm, - artifacts.deployed_bytecode(), - ); - (artifacts, contract_id) + BytecodeMarker::EraVm => MockCompilerResolver::zksolc(move |_| CompilationArtifacts { + bytecode: bytecode.clone(), + deployed_bytecode: None, + abi: counter_contract_abi(), }), - BytecodeMarker::Evm => MockCompilerResolver::solc(move |_| { - let artifacts = CompilationArtifacts { - bytecode: vec![3_u8; 48], - deployed_bytecode: Some(bytecode.clone()), - abi: counter_contract_abi(), - }; - let contract_id = ContractIdentifier::from_bytecode( - BytecodeMarker::Evm, - artifacts.deployed_bytecode(), - ); - (artifacts, contract_id) + BytecodeMarker::Evm => MockCompilerResolver::solc(move |_| CompilationArtifacts { + bytecode: vec![3_u8; 48], + deployed_bytecode: Some(bytecode.clone()), + abi: counter_contract_abi(), }), }; let verifier = ContractVerifier::with_resolver( @@ -747,14 +715,11 @@ async fn creation_bytecode_mismatch() { .unwrap(); let mock_resolver = MockCompilerResolver::solc(move |_| { - let artifacts = CompilationArtifacts { + CompilationArtifacts { bytecode: vec![4; 20], // differs from `creation_bytecode` deployed_bytecode: Some(deployed_bytecode.clone()), abi: counter_contract_abi(), - }; - let contract_id = - ContractIdentifier::from_bytecode(BytecodeMarker::Evm, artifacts.deployed_bytecode()); - (artifacts, contract_id) + } }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index 2d5b6758a3f2..8a54f8c2f53f 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -179,7 +179,7 @@ async fn using_real_zksolc(specify_contract_file: bool) { } let input = ZkSolc::build_input(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); validate_bytecode(&output.bytecode).unwrap(); assert_eq!(output.abi, counter_contract_abi()); @@ -223,7 +223,7 @@ async fn using_standalone_solc(specify_contract_file: bool) { } let input = Solc::build_input(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, counter_contract_abi()); @@ -306,7 +306,9 @@ async fn compiling_yul_with_zksolc() { let compiler = compiler_resolver.resolve_zksolc(&version).await.unwrap(); let req = test_yul_request(supported_compilers.solc_for_api(BytecodeMarker::EraVm)); let input = ZkSolc::build_input(req).unwrap(); - let (output, identifier) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); + let identifier = + ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, output.deployed_bytecode()); assert!(!output.bytecode.is_empty()); assert!(output.deployed_bytecode.is_none()); @@ -328,7 +330,9 @@ async fn compiling_standalone_yul() { compiler_zksolc_version: None, }); let input = Solc::build_input(req).unwrap(); - let (output, identifier) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); + let identifier = + ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode()); assert!(!output.bytecode.is_empty()); assert_ne!(output.deployed_bytecode.unwrap(), output.bytecode); @@ -383,7 +387,9 @@ async fn using_real_zkvyper(specify_contract_file: bool) { BytecodeMarker::EraVm, ); let input = VyperInput::new(req).unwrap(); - let (output, identifier) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); + let identifier = + ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, output.deployed_bytecode()); validate_bytecode(&output.bytecode).unwrap(); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -412,7 +418,9 @@ async fn using_standalone_vyper(specify_contract_file: bool) { BytecodeMarker::Evm, ); let input = VyperInput::new(req).unwrap(); - let (output, identifier) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); + let identifier = + ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode()); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -434,7 +442,9 @@ async fn using_standalone_vyper_without_optimization() { ); req.optimization_used = false; let input = VyperInput::new(req).unwrap(); - let (output, identifier) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); + let identifier = + ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode()); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -457,7 +467,7 @@ async fn using_standalone_vyper_with_code_size_optimization() { req.optimization_used = true; req.optimizer_mode = Some("codesize".to_owned()); let input = VyperInput::new(req).unwrap(); - let (output, _) = compiler.compile(input).await.unwrap(); + let output = compiler.compile(input).await.unwrap(); assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); @@ -509,7 +519,7 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai }, }; let address = Address::repeat_byte(1); - let (output, identifier) = match (bytecode_kind, toolchain) { + let output = match (bytecode_kind, toolchain) { (BytecodeMarker::EraVm, Toolchain::Solidity) => { let compiler = compiler_resolver .resolve_zksolc(&supported_compilers.zksolc()) @@ -539,6 +549,7 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai compiler.compile(input).await.unwrap() } }; + let identifier = ContractIdentifier::from_bytecode(bytecode_kind, output.deployed_bytecode()); match (bytecode_kind, toolchain) { (BytecodeMarker::Evm, Toolchain::Vyper) => { @@ -655,8 +666,11 @@ async fn using_zksolc_partial_match(use_cbor: bool) { .unwrap(); let input_for_request = ZkSolc::build_input(req.clone()).unwrap(); - let (output_for_request, identifier_for_request) = - compiler.compile(input_for_request).await.unwrap(); + let output_for_request = compiler.compile(input_for_request).await.unwrap(); + let identifier_for_request = ContractIdentifier::from_bytecode( + BytecodeMarker::EraVm, + output_for_request.deployed_bytecode(), + ); // Now prepare data for contract verification storage (with different metadata). let compiler = compiler_resolver @@ -686,8 +700,11 @@ async fn using_zksolc_partial_match(use_cbor: bool) { panic!("unexpected input: {input_for_storage:?}"); } - let (output_for_storage, identifier_for_storage) = - compiler.compile(input_for_storage).await.unwrap(); + let output_for_storage = compiler.compile(input_for_storage).await.unwrap(); + let identifier_for_storage = ContractIdentifier::from_bytecode( + BytecodeMarker::EraVm, + output_for_storage.deployed_bytecode(), + ); assert_eq!( identifier_for_request.matches(output_for_storage.deployed_bytecode()), From d04ed976fc1ee0e356d390fe1f6da6d50ba0825f Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 28 Jan 2025 11:23:52 +0400 Subject: [PATCH 09/16] Replace use of sha3 with keccak256 --- core/lib/contract_verifier/src/lib.rs | 4 +- core/lib/contract_verifier/src/tests/real.rs | 24 ++-- core/lib/dal/src/contract_verification_dal.rs | 18 +-- .../contract_identifier.rs | 120 ++++++++---------- 4 files changed, 80 insertions(+), 86 deletions(-) diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 2178aa332239..2f5ac0f46ba3 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -582,9 +582,9 @@ impl ContractVerifier { .contract_verification_dal() .save_verification_info( info, - identifier.bytecode_sha3, + identifier.bytecode_keccak256, identifier - .bytecode_without_metadata_sha3 + .bytecode_without_metadata_keccak256 .map(|hash| hash.hash()), ) .await?; diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index 8a54f8c2f53f..d6957908268a 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -314,7 +314,7 @@ async fn compiling_yul_with_zksolc() { assert!(output.deployed_bytecode.is_none()); assert_eq!(output.abi, serde_json::json!([])); assert_matches!( - identifier.bytecode_without_metadata_sha3, + identifier.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Keccak256(_)) ); } @@ -338,7 +338,7 @@ async fn compiling_standalone_yul() { assert_ne!(output.deployed_bytecode.unwrap(), output.bytecode); assert_eq!(output.abi, serde_json::json!([])); assert_matches!( - identifier.bytecode_without_metadata_sha3, + identifier.bytecode_without_metadata_keccak256, None, "No metadata for compiler yul for EVM" ); @@ -394,7 +394,7 @@ async fn using_real_zkvyper(specify_contract_file: bool) { validate_bytecode(&output.bytecode).unwrap(); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); assert_matches!( - identifier.bytecode_without_metadata_sha3, + identifier.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Keccak256(_)) ); } @@ -425,7 +425,7 @@ async fn using_standalone_vyper(specify_contract_file: bool) { assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); // Vyper does not provide metadata for bytecode. - assert_matches!(identifier.bytecode_without_metadata_sha3, None); + assert_matches!(identifier.bytecode_without_metadata_keccak256, None); } #[tokio::test] @@ -449,7 +449,7 @@ async fn using_standalone_vyper_without_optimization() { assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); // Vyper does not provide metadata for bytecode. - assert_matches!(identifier.bytecode_without_metadata_sha3, None); + assert_matches!(identifier.bytecode_without_metadata_keccak256, None); } #[tokio::test] @@ -554,20 +554,20 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai match (bytecode_kind, toolchain) { (BytecodeMarker::Evm, Toolchain::Vyper) => { assert!( - identifier.bytecode_without_metadata_sha3.is_none(), + identifier.bytecode_without_metadata_keccak256.is_none(), "No metadata for EVM Vyper" ); } (BytecodeMarker::Evm, Toolchain::Solidity) => { assert_matches!( - identifier.bytecode_without_metadata_sha3, + identifier.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Cbor(_)), "Cbor metadata for EVM Solidity by default" ); } (BytecodeMarker::EraVm, _) => { assert_matches!( - identifier.bytecode_without_metadata_sha3, + identifier.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Keccak256(_)), "Keccak256 metadata for EraVM by default" ); @@ -718,20 +718,20 @@ async fn using_zksolc_partial_match(use_cbor: bool) { ); if use_cbor { assert_matches!( - identifier_for_request.bytecode_without_metadata_sha3, + identifier_for_request.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Cbor(_)) ); assert_matches!( - identifier_for_storage.bytecode_without_metadata_sha3, + identifier_for_storage.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Cbor(_)) ); } else { assert_matches!( - identifier_for_request.bytecode_without_metadata_sha3, + identifier_for_request.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Keccak256(_)) ); assert_matches!( - identifier_for_storage.bytecode_without_metadata_sha3, + identifier_for_storage.bytecode_without_metadata_keccak256, Some(DetectedMetadata::Keccak256(_)) ); } diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index ac1930987ade..13315f6c341c 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -633,8 +633,8 @@ impl ContractVerificationDal<'_, '_> { let identifier = ContractIdentifier::from_bytecode(bytecode_marker, deployed_bytecode); // `unwrap_or_default` is safe, since we're checking that `bytecode_without_metadata_keccak256` is not null. - let bytecode_without_metadata_sha3 = identifier - .bytecode_without_metadata_sha3 + let bytecode_without_metadata_keccak256 = identifier + .bytecode_without_metadata_keccak256 .as_ref() .map(|h| h.hash()) .unwrap_or_default(); @@ -656,8 +656,8 @@ impl ContractVerificationDal<'_, '_> { AND bytecode_without_metadata_keccak256 = $2 ) "#, - identifier.bytecode_sha3.as_bytes(), - bytecode_without_metadata_sha3.as_bytes() + identifier.bytecode_keccak256.as_bytes(), + bytecode_without_metadata_keccak256.as_bytes() ) .try_map(|row| { let info = serde_json::from_value::(row.verification_info) @@ -679,10 +679,12 @@ impl ContractVerificationDal<'_, '_> { else { return Ok(None); }; - if identifier.bytecode_sha3 != bytecode_keccak256 { + if identifier.bytecode_keccak256 != bytecode_keccak256 { // Sanity check if bytecode_without_metadata_keccak256.is_none() - || identifier.bytecode_without_metadata_sha3.map(|h| h.hash()) + || identifier + .bytecode_without_metadata_keccak256 + .map(|h| h.hash()) != bytecode_without_metadata_keccak256 { tracing::error!( @@ -809,9 +811,9 @@ impl ContractVerificationDal<'_, '_> { ); ( - identifier.bytecode_sha3.as_bytes().to_vec(), + identifier.bytecode_keccak256.as_bytes().to_vec(), identifier - .bytecode_without_metadata_sha3 + .bytecode_without_metadata_keccak256 .as_ref() .map(|h| h.hash().as_bytes().to_vec()) .unwrap_or_default(), diff --git a/core/lib/types/src/contract_verification/contract_identifier.rs b/core/lib/types/src/contract_verification/contract_identifier.rs index 6f64f8e2cf85..4310923b1394 100644 --- a/core/lib/types/src/contract_verification/contract_identifier.rs +++ b/core/lib/types/src/contract_verification/contract_identifier.rs @@ -19,13 +19,13 @@ use crate::{bytecode::BytecodeMarker, web3::keccak256, H256}; pub struct ContractIdentifier { /// Marker of the bytecode of the contract. pub bytecode_marker: BytecodeMarker, - /// SHA3 (keccak256) hash of the full contract bytecode. + /// keccak256 hash of the full contract bytecode. /// Can be used as an identifier of precise contract compilation. - pub bytecode_sha3: H256, - /// SHA3 (keccak256) hash of the contract bytecode without metadata (e.g. with either + pub bytecode_keccak256: H256, + /// keccak256 hash of the contract bytecode without metadata (e.g. with either /// CBOR or keccak256 metadata hash being stripped). /// Can be absent if the contract bytecode doesn't have metadata. - pub bytecode_without_metadata_sha3: Option, + pub bytecode_without_metadata_keccak256: Option, /// Size of metadata in the bytecode. pub metadata_size: usize, } @@ -43,7 +43,7 @@ pub enum Match { /// Metadata detected in the contract bytecode. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DetectedMetadata { - /// Keccek256 hash of the metadata detected (only for EraVM). + /// keccak256 hash of the metadata detected (only for EraVM). Keccak256(H256), /// CBOR metadata detected. Cbor(H256), @@ -79,11 +79,11 @@ struct CborMetadata { impl ContractIdentifier { pub fn from_bytecode(bytecode_marker: BytecodeMarker, bytecode: &[u8]) -> Self { // Calculate the hash for bytecode with metadata. - let bytecode_sha3 = H256::from_slice(&keccak256(bytecode)); + let bytecode_keccak256 = H256(keccak256(bytecode)); let mut self_ = Self { bytecode_marker, - bytecode_sha3, - bytecode_without_metadata_sha3: None, + bytecode_keccak256, + bytecode_without_metadata_keccak256: None, metadata_size: 0, }; @@ -106,7 +106,7 @@ impl ContractIdentifier { }; let hash = H256::from_slice(&keccak256(bytecode_without_metadata)); // This could be overridden if CBOR metadata is detected. - self_.bytecode_without_metadata_sha3 = Some(DetectedMetadata::Keccak256(hash)); + self_.bytecode_without_metadata_keccak256 = Some(DetectedMetadata::Keccak256(hash)); } } @@ -149,7 +149,7 @@ impl ContractIdentifier { if bytecode.len() < full_aligned_metadata_length { // This shouldn't normally happen (metadata was deserialized correctly), // so we just disable partial matching just in case. - self_.bytecode_without_metadata_sha3 = None; + self_.bytecode_without_metadata_keccak256 = None; self_.metadata_size = 0; return self_; } @@ -168,8 +168,8 @@ impl ContractIdentifier { } } }; - let hash = H256::from_slice(&keccak256(bytecode_without_metadata)); - self_.bytecode_without_metadata_sha3 = Some(DetectedMetadata::Cbor(hash)); + let hash = H256(keccak256(bytecode_without_metadata)); + self_.bytecode_without_metadata_keccak256 = Some(DetectedMetadata::Cbor(hash)); self_ } @@ -177,7 +177,7 @@ impl ContractIdentifier { pub fn matches(&self, other: &[u8]) -> Match { let other_identifier = Self::from_bytecode(self.bytecode_marker, other); - if self.bytecode_sha3 == other_identifier.bytecode_sha3 { + if self.bytecode_keccak256 == other_identifier.bytecode_keccak256 { return Match::Full; } @@ -186,9 +186,9 @@ impl ContractIdentifier { // and presence in another, or different kinds of metadata. This is OK: partial // match is needed mostly when you cannot reproduce the original metadata, but one always // can submit the contract with the same metadata kind. - if self.bytecode_without_metadata_sha3.is_some() - && self.bytecode_without_metadata_sha3 - == other_identifier.bytecode_without_metadata_sha3 + if self.bytecode_without_metadata_keccak256.is_some() + && self.bytecode_without_metadata_keccak256 + == other_identifier.bytecode_without_metadata_keccak256 { return Match::Partial; } @@ -207,22 +207,20 @@ mod tests { // ./etc/zksolc-bin/v1.5.8/zksolc --solc ./etc/solc-bin/zkVM-0.8.28-1.0.1/solc --metadata-hash ipfs --codegen yul test.sol --bin // (Use `zkstack contract-verifier init` to download compilers) let data = hex::decode("0000008003000039000000400030043f0000000100200190000000110000c13d0000000900100198000000190000613d000000000101043b0000000a011001970000000b0010009c000000190000c13d0000000001000416000000000001004b000000190000c13d000000000100041a000000800010043f0000000c010000410000001c0001042e0000000001000416000000000001004b000000190000c13d00000020010000390000010000100443000001200000044300000008010000410000001c0001042e00000000010000190000001d000104300000001b000004320000001c0001042e0000001d0001043000000000000000000000000000000000000000020000000000000000000000000000004000000100000000000000000000000000000000000000000000000000fffffffc000000000000000000000000ffffffff000000000000000000000000000000000000000000000000000000006d4ce63c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000008000000000000000000000000000000000000000000000000000000000a16469706673582212208acf048570dcc1c3ff41bf8f20376049a42ae8a471f2b2ae8c14d8b356d86d79002a").unwrap(); - let sha3 = keccak256(&data); + let bytecode_keccak256 = H256(keccak256(&data)); let full_metadata_len = 64; // (CBOR metadata + len bytes) - let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + let bytecode_without_metadata_keccak256 = + H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( - identifier.bytecode_sha3, - H256::from_slice(&sha3), + identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_sha3, - Some(DetectedMetadata::Cbor(H256::from_slice( - &sha3_without_metadata - ))), + identifier.bytecode_without_metadata_keccak256, + Some(DetectedMetadata::Cbor(bytecode_without_metadata_keccak256)), "Incorrect bytecode without metadata hash" ); } @@ -231,22 +229,20 @@ mod tests { fn eravm_cbor_with_padding() { // Same as `eravm_cbor_without_padding` but now bytecode has padding. let data = hex::decode("00000001002001900000000c0000613d0000008001000039000000400010043f0000000001000416000000000001004b0000000c0000c13d00000020010000390000010000100443000001200000044300000005010000410000000f0001042e000000000100001900000010000104300000000e000004320000000f0001042e0000001000010430000000000000000000000000000000000000000000000000000000020000000000000000000000000000004000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a1646970667358221220d5be4da510b089bb58fa6c65f0a387eef966bcf48671a24fb2b1bc7190842978002a").unwrap(); - let sha3 = keccak256(&data); + let bytecode_keccak256 = H256(keccak256(&data)); let full_metadata_len = 64 + 32; // (CBOR metadata + len bytes + padding) - let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + let bytecode_without_metadata_keccak256 = + H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( - identifier.bytecode_sha3, - H256::from_slice(&sha3), + identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_sha3, - Some(DetectedMetadata::Cbor(H256::from_slice( - &sha3_without_metadata - ))), + identifier.bytecode_without_metadata_keccak256, + Some(DetectedMetadata::Cbor(bytecode_without_metadata_keccak256)), "Incorrect bytecode without metadata hash" ); } @@ -257,22 +253,22 @@ mod tests { // ./etc/zksolc-bin/v1.5.8/zksolc --solc ./etc/solc-bin/zkVM-0.8.28-1.0.1/solc --metadata-hash keccak256 --codegen yul test.sol --bin // (Use `zkstack contract-verifier init` to download compilers) let data = hex::decode("00000001002001900000000c0000613d0000008001000039000000400010043f0000000001000416000000000001004b0000000c0000c13d00000020010000390000010000100443000001200000044300000005010000410000000f0001042e000000000100001900000010000104300000000e000004320000000f0001042e000000100001043000000000000000000000000000000000000000000000000000000002000000000000000000000000000000400000010000000000000000000a00e4a5f19bb139176aa501024c7032404c065bc0012897fefd9ebc7e9a7677").unwrap(); - let sha3 = keccak256(&data); + let bytecode_keccak256 = H256(keccak256(&data)); let full_metadata_len = 32; // (keccak only) - let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + let bytecode_without_metadata_keccak256 = + H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( - identifier.bytecode_sha3, - H256::from_slice(&sha3), + identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_sha3, - Some(DetectedMetadata::Keccak256(H256::from_slice( - &sha3_without_metadata - ))), + identifier.bytecode_without_metadata_keccak256, + Some(DetectedMetadata::Keccak256( + bytecode_without_metadata_keccak256 + )), "Incorrect bytecode without metadata hash" ); } @@ -281,22 +277,22 @@ mod tests { fn eravm_keccak_with_padding() { // Same as `eravm_keccak_without_padding`, but now bytecode has padding. let data = hex::decode("0000008003000039000000400030043f0000000100200190000000110000c13d0000000900100198000000190000613d000000000101043b0000000a011001970000000b0010009c000000190000c13d0000000001000416000000000001004b000000190000c13d000000000100041a000000800010043f0000000c010000410000001c0001042e0000000001000416000000000001004b000000190000c13d00000020010000390000010000100443000001200000044300000008010000410000001c0001042e00000000010000190000001d000104300000001b000004320000001c0001042e0000001d0001043000000000000000000000000000000000000000020000000000000000000000000000004000000100000000000000000000000000000000000000000000000000fffffffc000000000000000000000000ffffffff000000000000000000000000000000000000000000000000000000006d4ce63c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000080000000000000000000000000000000000000000000000000000000000000000000000000000000009b1f0a6172ae84051eca37db231c0fa6249349f4ddaf86a87474a587c19d946d").unwrap(); - let sha3 = keccak256(&data); + let bytecode_keccak256 = H256(keccak256(&data)); let full_metadata_len = 64; // (keccak + padding) - let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + let bytecode_without_metadata_keccak256 = + H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( - identifier.bytecode_sha3, - H256::from_slice(&sha3), + identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_sha3, - Some(DetectedMetadata::Keccak256(H256::from_slice( - &sha3_without_metadata - ))), + identifier.bytecode_without_metadata_keccak256, + Some(DetectedMetadata::Keccak256( + bytecode_without_metadata_keccak256 + )), "Incorrect bytecode without metadata hash" ); } @@ -306,17 +302,16 @@ mod tests { // Random short bytecode let data = hex::decode("0000008003000039000000400030043f0000000100200190000000110000c13d") .unwrap(); - let sha3 = keccak256(&data); + let bytecode_keccak256 = H256(keccak256(&data)); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); assert_eq!(identifier.metadata_size, 0); assert_eq!( - identifier.bytecode_sha3, - H256::from_slice(&sha3), + identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_sha3, None, + identifier.bytecode_without_metadata_keccak256, None, "Incorrect bytecode without metadata hash" ); } @@ -327,17 +322,16 @@ mod tests { // ./etc/solc-bin/0.8.28/solc test.sol --bin --no-cbor-metadata // (Use `zkstack contract-verifier init` to download compilers) let data = hex::decode("6080604052348015600e575f5ffd5b50607980601a5f395ff3fe6080604052348015600e575f5ffd5b50600436106026575f3560e01c80636d4ce63c14602a575b5f5ffd5b60306044565b604051603b91906062565b60405180910390f35b5f5f54905090565b5f819050919050565b605c81604c565b82525050565b5f60208201905060735f8301846055565b9291505056").unwrap(); - let sha3 = keccak256(&data); + let bytecode_keccak256 = H256(keccak256(&data)); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); assert_eq!(identifier.metadata_size, 0); assert_eq!( - identifier.bytecode_sha3, - H256::from_slice(&sha3), + identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_sha3, None, + identifier.bytecode_without_metadata_keccak256, None, "Incorrect bytecode without metadata hash" ); } @@ -363,8 +357,9 @@ mod tests { for (label, bytecode, full_metadata_len) in test_vector { let data = hex::decode(bytecode).unwrap(); - let sha3 = keccak256(&data); - let sha3_without_metadata = keccak256(&data[..data.len() - full_metadata_len]); + let bytecode_keccak256 = H256(keccak256(&data)); + let bytecode_without_metadata_keccak256 = + H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); assert_eq!( @@ -372,15 +367,12 @@ mod tests { "{label}: Wrong metadata length" ); assert_eq!( - identifier.bytecode_sha3, - H256::from_slice(&sha3), + identifier.bytecode_keccak256, bytecode_keccak256, "{label}: Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_sha3, - Some(DetectedMetadata::Cbor(H256::from_slice( - &sha3_without_metadata - ))), + identifier.bytecode_without_metadata_keccak256, + Some(DetectedMetadata::Cbor(bytecode_without_metadata_keccak256)), "{label}: Incorrect bytecode without metadata hash" ); } From 1d224fbcea02eae0ec8be5b6f7f34d1efbdc4efb Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 28 Jan 2025 11:54:27 +0400 Subject: [PATCH 10/16] Rework metadata detection --- .../contract_identifier.rs | 130 +++++++++--------- 1 file changed, 62 insertions(+), 68 deletions(-) diff --git a/core/lib/types/src/contract_verification/contract_identifier.rs b/core/lib/types/src/contract_verification/contract_identifier.rs index 4310923b1394..234d5db2785b 100644 --- a/core/lib/types/src/contract_verification/contract_identifier.rs +++ b/core/lib/types/src/contract_verification/contract_identifier.rs @@ -26,8 +26,6 @@ pub struct ContractIdentifier { /// CBOR or keccak256 metadata hash being stripped). /// Can be absent if the contract bytecode doesn't have metadata. pub bytecode_without_metadata_keccak256: Option, - /// Size of metadata in the bytecode. - pub metadata_size: usize, } #[derive(Debug, Clone, Copy, PartialEq)] @@ -80,100 +78,106 @@ impl ContractIdentifier { pub fn from_bytecode(bytecode_marker: BytecodeMarker, bytecode: &[u8]) -> Self { // Calculate the hash for bytecode with metadata. let bytecode_keccak256 = H256(keccak256(bytecode)); - let mut self_ = Self { + + // Try to detect metadata. + // CBOR takes precedence (since keccak doesn't have direct markers, so it's partially a + // fallback). + let bytecode_without_metadata_keccak256 = + Self::detect_cbor_metadata(bytecode_marker, bytecode) + .or_else(|| Self::detect_keccak_metadata(bytecode_marker, bytecode)); + + Self { bytecode_marker, bytecode_keccak256, - bytecode_without_metadata_keccak256: None, - metadata_size: 0, - }; + bytecode_without_metadata_keccak256, + } + } - // For EraVM, the default metadata is keccak256 hash of the metadata. - // Try to use it as a default value (to be overridden if CBOR metadata is detected). + /// Will try to detect keccak256 metadata hash (only for EraVM) + fn detect_keccak_metadata( + bytecode_marker: BytecodeMarker, + bytecode: &[u8], + ) -> Option { + // For EraVM, the one option for metadata hash is keccak256 hash of the metadata. if bytecode_marker == BytecodeMarker::EraVm { // For metadata, we might have padding: it takes either 32 or 64 bytes depending // on whether the amount of words in the contract is odd, so we need to check // if there is padding. - if bytecode.len() > 64 { - let bytecode_without_metadata = - if bytecode[bytecode.len() - 64..bytecode.len() - 32] == [0u8; 32] { - // Padding is present, strip it. - self_.metadata_size = 64; - &bytecode[..bytecode.len() - 64] - } else { - // No padding, strip metadata only. - self_.metadata_size = 32; - &bytecode[..bytecode.len() - 32] - }; - let hash = H256::from_slice(&keccak256(bytecode_without_metadata)); - // This could be overridden if CBOR metadata is detected. - self_.bytecode_without_metadata_keccak256 = Some(DetectedMetadata::Keccak256(hash)); - } + let bytecode_without_metadata = Self::strip_padding(bytecode, 32)?; + let hash = H256(keccak256(bytecode_without_metadata)); + Some(DetectedMetadata::Keccak256(hash)) + } else { + None } + } - // Try to detect CBOR metadata. + /// Will try to detect CBOR metadata. + fn detect_cbor_metadata( + bytecode_marker: BytecodeMarker, + bytecode: &[u8], + ) -> Option { + let length = bytecode.len(); // Last two bytes is the length of the metadata in big endian. - if bytecode.len() < 2 { - return self_; + if length < 2 { + return None; } let metadata_length = - u16::from_be_bytes([bytecode[bytecode.len() - 2], bytecode[bytecode.len() - 1]]) - as usize; + u16::from_be_bytes([bytecode[length - 2], bytecode[length - 1]]) as usize; // Including size let full_metadata_length = metadata_length + 2; // Get slice for the metadata. - if bytecode.len() < full_metadata_length { - return self_; + if length < full_metadata_length { + return None; } - let raw_metadata = &bytecode[bytecode.len() - full_metadata_length..bytecode.len() - 2]; + let raw_metadata = &bytecode[length - full_metadata_length..length - 2]; // Try decoding. We are not interested in the actual value. let _metadata: CborMetadata = match ciborium::from_reader(raw_metadata) { Ok(metadata) => metadata, - Err(_) => return self_, + Err(_) => return None, }; // Strip metadata and calculate hash. let bytecode_without_metadata = match bytecode_marker { BytecodeMarker::Evm => { // On EVM, there is no padding. - self_.metadata_size = full_metadata_length; - &bytecode[..bytecode.len() - full_metadata_length] + &bytecode[..length - full_metadata_length] } BytecodeMarker::EraVm => { // On EraVM, there is padding: // 1. We must align the metadata length to 32 bytes. // 2. We may need to add 32 bytes of padding. - let aligned_metadata_length = (metadata_length + 31) / 32 * 32; - let full_aligned_metadata_length = aligned_metadata_length + 32; - if bytecode.len() < full_aligned_metadata_length { - // This shouldn't normally happen (metadata was deserialized correctly), - // so we just disable partial matching just in case. - self_.bytecode_without_metadata_keccak256 = None; - self_.metadata_size = 0; - return self_; - } - // Check if padding was added. - if bytecode[bytecode.len() - full_aligned_metadata_length - ..bytecode.len() - aligned_metadata_length] - == [0u8; 32] - { - // Padding was added, strip it. - self_.metadata_size = full_aligned_metadata_length; - &bytecode[..bytecode.len() - full_aligned_metadata_length] - } else { - // Padding wasn't added, strip metadata only. - self_.metadata_size = aligned_metadata_length; - &bytecode[..bytecode.len() - aligned_metadata_length] - } + let aligned_metadata_length = metadata_length.div_ceil(32) * 32; + Self::strip_padding(bytecode, aligned_metadata_length)? } }; let hash = H256(keccak256(bytecode_without_metadata)); - self_.bytecode_without_metadata_keccak256 = Some(DetectedMetadata::Cbor(hash)); + Some(DetectedMetadata::Cbor(hash)) + } + + /// Adds one word to the metadata length and check if it's a padding word. + /// If it is, strips the padding. + /// Returns `None` if `metadata_length` + padding won't fit into the bytecode. + fn strip_padding(bytecode: &[u8], metadata_length: usize) -> Option<&[u8]> { + const PADDING_WORD: [u8; 32] = [0u8; 32]; - self_ + let length = bytecode.len(); + let metadata_with_padding_length = metadata_length + 32; + if length < metadata_with_padding_length { + return None; + } + if bytecode[length - metadata_with_padding_length..length - metadata_length] == PADDING_WORD + { + // Padding was added, strip it. + Some(&bytecode[..length - metadata_with_padding_length]) + } else { + // Padding wasn't added, strip metadata only. + Some(&bytecode[..length - metadata_length]) + } } + /// Checks the kind of match between identifier and other bytecode. pub fn matches(&self, other: &[u8]) -> Match { let other_identifier = Self::from_bytecode(self.bytecode_marker, other); @@ -213,7 +217,6 @@ mod tests { H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" @@ -235,7 +238,6 @@ mod tests { H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" @@ -259,7 +261,6 @@ mod tests { H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" @@ -283,7 +284,6 @@ mod tests { H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.metadata_size, full_metadata_len); assert_eq!( identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" @@ -305,7 +305,6 @@ mod tests { let bytecode_keccak256 = H256(keccak256(&data)); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::EraVm, &data); - assert_eq!(identifier.metadata_size, 0); assert_eq!( identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" @@ -325,7 +324,6 @@ mod tests { let bytecode_keccak256 = H256(keccak256(&data)); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); - assert_eq!(identifier.metadata_size, 0); assert_eq!( identifier.bytecode_keccak256, bytecode_keccak256, "Incorrect bytecode hash" @@ -362,10 +360,6 @@ mod tests { H256(keccak256(&data[..data.len() - full_metadata_len])); let identifier = ContractIdentifier::from_bytecode(BytecodeMarker::Evm, &data); - assert_eq!( - identifier.metadata_size, full_metadata_len, - "{label}: Wrong metadata length" - ); assert_eq!( identifier.bytecode_keccak256, bytecode_keccak256, "{label}: Incorrect bytecode hash" From 10ddeca563a88c227097da13e12ef193884f8d1d Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 28 Jan 2025 12:05:44 +0400 Subject: [PATCH 11/16] Review comments --- core/lib/contract_verifier/src/tests/real.rs | 32 ++++++---------- ...432d1e8cb5702857d9cbfa617f985e62eaf4e.json | 20 ---------- ...046b7820e39b63ed4f98bcaa3b3f305cbe576.json | 26 +++++++++++++ ...9ddd85f5bfb18df281a36456f2d67db83be48.json | 20 ---------- core/lib/dal/src/contract_verification_dal.rs | 37 ++++--------------- .../types/src/contract_verification/api.rs | 10 +++++ 6 files changed, 55 insertions(+), 90 deletions(-) delete mode 100644 core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json create mode 100644 core/lib/dal/.sqlx/query-668cf72b78c6071340143ba9498046b7820e39b63ed4f98bcaa3b3f305cbe576.json delete mode 100644 core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index d6957908268a..524b7f818c3b 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -51,28 +51,23 @@ impl TestCompilerVersions { let vyper = VYPER_VERSION.strip_prefix("v").unwrap().to_owned(); anyhow::ensure!( versions.solc.contains(SOLC_VERSION), - "Expected solc version {} to be installed, but it is not", - SOLC_VERSION + "Expected solc version {SOLC_VERSION} to be installed, but it is not" ); anyhow::ensure!( versions.solc.contains(&eravm_solc), - "Expected era-vm solc version {} to be installed, but it is not", - ERA_VM_SOLC_VERSION + "Expected era-vm solc version {ERA_VM_SOLC_VERSION} to be installed, but it is not" ); anyhow::ensure!( versions.zksolc.contains(ZKSOLC_VERSION), - "Expected zksolc version {} to be installed, but it is not", - ZKSOLC_VERSION + "Expected zksolc version {ZKSOLC_VERSION} to be installed, but it is not" ); anyhow::ensure!( versions.vyper.contains(&vyper), - "Expected vyper version {} to be installed, but it is not", - VYPER_VERSION + "Expected vyper version {VYPER_VERSION} to be installed, but it is not" ); anyhow::ensure!( versions.zkvyper.contains(ZKVYPER_VERSION), - "Expected zkvyper version {} to be installed, but it is not", - ZKVYPER_VERSION + "Expected zkvyper version {ZKVYPER_VERSION} to be installed, but it is not" ); Ok(Self { @@ -633,16 +628,13 @@ async fn using_zksolc_partial_match(use_cbor: bool) { "Counter.sol": { "content": COUNTER_CONTRACT, }, - },"settings": { + }, + "settings": { "outputSelection": { - "*": { - "": [ - "abi" - ], - "*": [ - "abi" - ] - } + "*": { + "": [ "abi" ], + "*": [ "abi" ] + } }, "isSystem": false, "forceEvmla": false, @@ -650,7 +642,7 @@ async fn using_zksolc_partial_match(use_cbor: bool) { "hashType": hash_type }, "optimizer": { - "enabled": true + "enabled": true } } }) diff --git a/core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json b/core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json deleted file mode 100644 index ad67acf09d7c..000000000000 --- a/core/lib/dal/.sqlx/query-60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n COUNT(*)\n FROM\n contracts_verification_info\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "60e8c9a2c028cd8e99a7cafd067432d1e8cb5702857d9cbfa617f985e62eaf4e" -} diff --git a/core/lib/dal/.sqlx/query-668cf72b78c6071340143ba9498046b7820e39b63ed4f98bcaa3b3f305cbe576.json b/core/lib/dal/.sqlx/query-668cf72b78c6071340143ba9498046b7820e39b63ed4f98bcaa3b3f305cbe576.json new file mode 100644 index 000000000000..9dffecdc4c19 --- /dev/null +++ b/core/lib/dal/.sqlx/query-668cf72b78c6071340143ba9498046b7820e39b63ed4f98bcaa3b3f305cbe576.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n (SELECT COUNT(*) FROM contracts_verification_info) AS count_v1,\n (SELECT COUNT(*) FROM contract_verification_info_v2) AS count_v2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count_v1", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "count_v2", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "668cf72b78c6071340143ba9498046b7820e39b63ed4f98bcaa3b3f305cbe576" +} diff --git a/core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json b/core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json deleted file mode 100644 index 7e293cb033aa..000000000000 --- a/core/lib/dal/.sqlx/query-8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n COUNT(*)\n FROM\n contract_verification_info_v2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "8b5c41c99fb3bd1c9fc99a194639ddd85f5bfb18df281a36456f2d67db83be48" -} diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 13315f6c341c..7baf1c781f10 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -706,35 +706,18 @@ impl ContractVerificationDal<'_, '_> { /// Checks if migration from `contracts_verification_info` to `contract_verification_info_v2` is performed /// by checking if the latter has more or equal number of rows. pub async fn is_verification_info_migration_performed(&mut self) -> DalResult { - let count_v1 = sqlx::query!( + let row = sqlx::query!( r#" SELECT - COUNT(*) - FROM - contracts_verification_info - "#, - ) - .instrument("is_verification_info_migration_performed#count_v1") - .fetch_one(self.storage) - .await? - .count - .unwrap() as usize; - - let count_v2 = sqlx::query!( - r#" - SELECT - COUNT(*) - FROM - contract_verification_info_v2 + (SELECT COUNT(*) FROM contracts_verification_info) AS count_v1, + (SELECT COUNT(*) FROM contract_verification_info_v2) AS count_v2 "#, ) - .instrument("is_verification_info_migration_performed#count_v2") + .instrument("is_verification_info_migration_performed") .fetch_one(self.storage) - .await? - .count - .unwrap() as usize; + .await?; - Ok(count_v2 >= count_v1) + Ok(row.count_v2 >= row.count_v1) } pub async fn perform_verification_info_migration( @@ -799,14 +782,8 @@ impl ContractVerificationDal<'_, '_> { hex::encode(address) ); }); - let bytecode_marker = if verification_info.artifacts.deployed_bytecode.is_some() - { - BytecodeMarker::Evm - } else { - BytecodeMarker::EraVm - }; let identifier = ContractIdentifier::from_bytecode( - bytecode_marker, + verification_info.bytecode_marker(), verification_info.artifacts.deployed_bytecode(), ); diff --git a/core/lib/types/src/contract_verification/api.rs b/core/lib/types/src/contract_verification/api.rs index 92692511770c..594596070d9f 100644 --- a/core/lib/types/src/contract_verification/api.rs +++ b/core/lib/types/src/contract_verification/api.rs @@ -5,6 +5,7 @@ use serde::{ de::{Deserializer, Error, MapAccess, Unexpected, Visitor}, Deserialize, Serialize, }; +use zksync_basic_types::bytecode::BytecodeMarker; pub use crate::Execute as ExecuteData; use crate::{web3::Bytes, Address}; @@ -254,6 +255,15 @@ impl VerificationInfo { pub fn is_perfect_match(&self) -> bool { self.verification_problems.is_empty() } + + pub fn bytecode_marker(&self) -> BytecodeMarker { + // Deployed bytecode is only present for EVM contracts. + if self.artifacts.deployed_bytecode.is_some() { + BytecodeMarker::Evm + } else { + BytecodeMarker::EraVm + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] From aab12d4dd522c1d7167ec353ac82c15a8d62e3ee Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 28 Jan 2025 14:55:53 +0400 Subject: [PATCH 12/16] Speed up the migration --- ...3083912044f8f5328a867e9ed1a063eb1f528.json | 17 --- ...89ff52100e6a93537c35961e62268d7cd26e.json} | 6 +- ...80a651e28abb3dd3582211ceb6b2bb8009258.json | 20 +++ core/lib/dal/src/contract_verification_dal.rs | 137 +++++++++++------- 4 files changed, 105 insertions(+), 75 deletions(-) delete mode 100644 core/lib/dal/.sqlx/query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json rename core/lib/dal/.sqlx/{query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json => query-a331b209eafd82595ad75e24135989ff52100e6a93537c35961e62268d7cd26e.json} (62%) create mode 100644 core/lib/dal/.sqlx/query-ca2c63a0e25406eec4e92a7cfda80a651e28abb3dd3582211ceb6b2bb8009258.json diff --git a/core/lib/dal/.sqlx/query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json b/core/lib/dal/.sqlx/query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json deleted file mode 100644 index e7eb33ccfa63..000000000000 --- a/core/lib/dal/.sqlx/query-2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n contract_verification_info_v2 (\n initial_contract_addr,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n SELECT\n u.address,\n u.bytecode_keccak256,\n u.bytecode_without_metadata_keccak256,\n u.verification_info\n FROM\n UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSONB []) AS u (\n address,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256,\n verification_info\n )\n ON CONFLICT (initial_contract_addr) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "ByteaArray", - "ByteaArray", - "ByteaArray", - "JsonbArray" - ] - }, - "nullable": [] - }, - "hash": "2ff3f234acc5b33c663383424b13083912044f8f5328a867e9ed1a063eb1f528" -} diff --git a/core/lib/dal/.sqlx/query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json b/core/lib/dal/.sqlx/query-a331b209eafd82595ad75e24135989ff52100e6a93537c35961e62268d7cd26e.json similarity index 62% rename from core/lib/dal/.sqlx/query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json rename to core/lib/dal/.sqlx/query-a331b209eafd82595ad75e24135989ff52100e6a93537c35961e62268d7cd26e.json index e42bcd9d53c5..7d2121d708f3 100644 --- a/core/lib/dal/.sqlx/query-c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13.json +++ b/core/lib/dal/.sqlx/query-a331b209eafd82595ad75e24135989ff52100e6a93537c35961e62268d7cd26e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n address,\n verification_info::text as verification_info\n FROM\n contracts_verification_info\n ORDER BY\n address\n OFFSET $1\n LIMIT $2\n ", + "query": "\n SELECT\n address,\n verification_info::text AS verification_info\n FROM\n contracts_verification_info\n WHERE address > $1\n ORDER BY\n address\n LIMIT $2\n ", "describe": { "columns": [ { @@ -16,7 +16,7 @@ ], "parameters": { "Left": [ - "Int8", + "Bytea", "Int8" ] }, @@ -25,5 +25,5 @@ null ] }, - "hash": "c7a3811f4a9b6a1bf87cf266d3b685d93fdc96b6cd8fdf10663ea891e135ee13" + "hash": "a331b209eafd82595ad75e24135989ff52100e6a93537c35961e62268d7cd26e" } diff --git a/core/lib/dal/.sqlx/query-ca2c63a0e25406eec4e92a7cfda80a651e28abb3dd3582211ceb6b2bb8009258.json b/core/lib/dal/.sqlx/query-ca2c63a0e25406eec4e92a7cfda80a651e28abb3dd3582211ceb6b2bb8009258.json new file mode 100644 index 000000000000..2708c5b78701 --- /dev/null +++ b/core/lib/dal/.sqlx/query-ca2c63a0e25406eec4e92a7cfda80a651e28abb3dd3582211ceb6b2bb8009258.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n COUNT(*)\n FROM\n contract_verification_info_v2 v2\n JOIN contracts_verification_info v1 ON initial_contract_addr = address\n WHERE v1.verification_info::text != v2.verification_info::text\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "ca2c63a0e25406eec4e92a7cfda80a651e28abb3dd3582211ceb6b2bb8009258" +} diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 7baf1c781f10..adf3555a42eb 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -8,7 +8,10 @@ use std::{ use anyhow::Context as _; use rayon::prelude::*; use sqlx::postgres::types::PgInterval; -use zksync_db_connection::{error::SqlxContext, instrument::InstrumentExt}; +use zksync_db_connection::{ + error::SqlxContext, + instrument::{CopyStatement, InstrumentExt}, +}; use zksync_types::{ address_to_h256, bytecode::{trim_padded_evm_bytecode, BytecodeHash, BytecodeMarker}, @@ -730,25 +733,28 @@ impl ContractVerificationDal<'_, '_> { // Offset is a number of already migrated contracts. let mut offset = 0usize; + let mut cursor = vec![]; loop { + let cursor_str = format!("0x{}", hex::encode(&cursor)); + // Fetch JSON as text to avoid roundtrip through `serde_json::Value`, as it's super slow. let (addresses, verification_infos): (Vec>, Vec) = sqlx::query!( r#" SELECT address, - verification_info::text as verification_info + verification_info::text AS verification_info FROM contracts_verification_info + WHERE address > $1 ORDER BY address - OFFSET $1 LIMIT $2 "#, - offset as i64, + &cursor, batch_size as i64, ) .instrument("perform_verification_info_migration#select") - .with_arg("offset", &offset) + .with_arg("cursor", &cursor_str) .with_arg("batch_size", &batch_size) .fetch_all(&mut transaction) .await? @@ -762,15 +768,11 @@ impl ContractVerificationDal<'_, '_> { } tracing::info!( - "Processing {} contracts; offset {}", - addresses.len(), - offset + "Processing {} contracts (processed: {offset}); cursor {cursor_str}", + addresses.len() ); - let (bytecode_keccak256s, bytecode_without_metadata_keccak256s): ( - Vec>, - Vec>, - ) = (0..addresses.len()) + let ids: Vec = (0..addresses.len()) .into_par_iter() .map(|idx| { let address = &addresses[idx]; @@ -782,60 +784,85 @@ impl ContractVerificationDal<'_, '_> { hex::encode(address) ); }); - let identifier = ContractIdentifier::from_bytecode( + ContractIdentifier::from_bytecode( verification_info.bytecode_marker(), verification_info.artifacts.deployed_bytecode(), - ); - - ( - identifier.bytecode_keccak256.as_bytes().to_vec(), - identifier - .bytecode_without_metadata_keccak256 - .as_ref() - .map(|h| h.hash().as_bytes().to_vec()) - .unwrap_or_default(), ) }) - .unzip(); + .collect(); + + let now = chrono::Utc::now().naive_utc().to_string(); + let mut buffer = String::new(); + for idx in 0..addresses.len() { + let address = hex::encode(&addresses[idx]); + let bytecode_keccak256 = hex::encode(ids[idx].bytecode_keccak256); + let bytecode_without_metadata_keccak256 = ids[idx] + .bytecode_without_metadata_keccak256 + .map(|h| format!(r#"\\x{}"#, hex::encode(h.hash()))) + .unwrap_or_else(|| "null".to_owned()); + let verification_info = verification_infos[idx].replace('"', r#""""#); + + let row = format!( + r#"\\x{initial_contract_addr},\\x{bytecode_keccak256},{bytecode_without_metadata_keccak256},"{verification_info}",{created_at},{updated_at}"#, + initial_contract_addr = address, + bytecode_keccak256 = bytecode_keccak256, + bytecode_without_metadata_keccak256 = bytecode_without_metadata_keccak256, + verification_info = verification_info, + created_at = now, + updated_at = now + ); + buffer.push_str(&row); + buffer.push('\n'); + } - // Insert all the values - sqlx::query!( - r#" - INSERT INTO - contract_verification_info_v2 ( - initial_contract_addr, - bytecode_keccak256, - bytecode_without_metadata_keccak256, - verification_info - ) - SELECT - u.address, - u.bytecode_keccak256, - u.bytecode_without_metadata_keccak256, - u.verification_info - FROM - UNNEST($1::BYTEA [], $2::BYTEA [], $3::BYTEA [], $4::JSONB []) AS u ( - address, - bytecode_keccak256, - bytecode_without_metadata_keccak256, - verification_info - ) - ON CONFLICT (initial_contract_addr) DO NOTHING - "#, - &addresses as _, - &bytecode_keccak256s as _, - &bytecode_without_metadata_keccak256s as _, - &verification_infos as _, + let contracts_len = addresses.len(); + let copy = CopyStatement::new( + "COPY contract_verification_info_v2( + initial_contract_addr, + bytecode_keccak256, + bytecode_without_metadata_keccak256, + verification_info, + created_at, + updated_at + ) FROM STDIN (FORMAT CSV, NULL 'null', DELIMITER ',')", ) - .instrument("perform_verification_info_migration#insert") - .with_arg("offset", &offset) - .with_arg("batch_size", &batch_size) - .execute(&mut transaction) + .instrument("perform_verification_info_migration#copy") + .with_arg("cursor", &cursor_str) + .with_arg("contracts.len", &contracts_len) + .start(&mut transaction) .await?; + copy.send(buffer.as_bytes()).await?; + offset += batch_size; + cursor = addresses.last().unwrap().clone(); + } + + // Sanity check. + tracing::info!("All the rows are migrated, verifying the migration"); + let count_inequal = sqlx::query!( + r#" + SELECT + COUNT(*) + FROM + contract_verification_info_v2 v2 + JOIN contracts_verification_info v1 ON initial_contract_addr = address + WHERE v1.verification_info::text != v2.verification_info::text + "#, + ) + .instrument("is_verification_info_migration_performed") + .fetch_one(&mut transaction) + .await? + .count + .unwrap(); + if count_inequal > 0 { + anyhow::bail!( + "Migration failed: {} rows have different data in the new table", + count_inequal + ); } + tracing::info!("Migration is successful, committing the transaction"); transaction.commit().await?; Ok(()) } From b1e4ba41fc4d57d4d2bf2f19ff038883d888fa46 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 28 Jan 2025 17:22:04 +0400 Subject: [PATCH 13/16] Change the index type --- .../20250122102800_contract-verifier-new-schema.up.sql | 4 ++-- core/lib/dal/src/contract_verification_dal.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql index 1e760d122b07..974f8b2e2717 100644 --- a/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql +++ b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql @@ -9,5 +9,5 @@ CREATE TABLE IF NOT EXISTS contract_verification_info_v2 ( ); -- Add hash indexes for hash columns -CREATE INDEX IF NOT EXISTS contract_verification_info_v2_bytecode_keccak256_idx ON contract_verification_info_v2 USING HASH (bytecode_keccak256); -CREATE INDEX IF NOT EXISTS contract_verification_info_v2_bytecode_without_metadata_keccak256_idx ON contract_verification_info_v2 USING HASH (bytecode_without_metadata_keccak256); +CREATE INDEX IF NOT EXISTS contract_verification_info_v2_bytecode_keccak256_idx ON contract_verification_info_v2 (bytecode_keccak256); +CREATE INDEX IF NOT EXISTS contract_verification_info_v2_bytecode_without_metadata_keccak256_idx ON contract_verification_info_v2 (bytecode_without_metadata_keccak256); diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index adf3555a42eb..92e0c98098cb 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -607,7 +607,7 @@ impl ContractVerificationDal<'_, '_> { .try_map(|row| { serde_json::from_value(row.verification_info).decode_column("verification_info") }) - .instrument("get_contract_verification_info_v2#perfect_match") + .instrument("get_contract_verification_info_v2#direct") .with_arg("address", &address) .fetch_optional(self.storage) .await? @@ -675,7 +675,7 @@ impl ContractVerificationDal<'_, '_> { bytecode_without_metadata_keccak256, )) }) - .instrument("get_contract_verification_info_v2#perfect_match") + .instrument("get_contract_verification_info_v2#similar") .with_arg("address", &address) .fetch_optional(self.storage) .await? From cc206ef895c9ce02334c10225ce404320820a962 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Wed, 29 Jan 2025 11:08:18 +0400 Subject: [PATCH 14/16] Nits --- core/bin/contract-verifier/src/main.rs | 2 +- core/lib/contract_verifier/src/compilers/solc.rs | 4 +--- core/lib/contract_verifier/src/compilers/vyper.rs | 4 +--- core/lib/contract_verifier/src/compilers/zksolc.rs | 7 ++----- core/lib/contract_verifier/src/compilers/zkvyper.rs | 3 +-- core/lib/dal/src/contract_verification_dal.rs | 13 +++++++++---- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 2bd32b2c4561..93f23816c67d 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -40,7 +40,7 @@ async fn perform_storage_migration(pool: &ConnectionPool) -> anyhow::Resul .is_verification_info_migration_performed() .await?; if !migration_performed { - tracing::info!("Running the storage migration for the contract verifier table"); + tracing::info!(batch_size = %batch_size, "Running the storage migration for the contract verifier table"); storage .contract_verification_dal() .perform_verification_info_migration(batch_size) diff --git a/core/lib/contract_verifier/src/compilers/solc.rs b/core/lib/contract_verifier/src/compilers/solc.rs index 7c6fb5100e57..4224e2a3dbcc 100644 --- a/core/lib/contract_verifier/src/compilers/solc.rs +++ b/core/lib/contract_verifier/src/compilers/solc.rs @@ -128,9 +128,7 @@ impl Compiler for Solc { if output.status.success() { let output = serde_json::from_slice(&output.stdout) .context("zksolc output is not valid JSON")?; - let output = - parse_standard_json_output(&output, input.contract_name, input.file_name, true)?; - Ok(output) + parse_standard_json_output(&output, input.contract_name, input.file_name, true) } else { Err(ContractVerifierError::CompilerError( "solc", diff --git a/core/lib/contract_verifier/src/compilers/vyper.rs b/core/lib/contract_verifier/src/compilers/vyper.rs index 50d73ac259cb..46034a62e0a3 100644 --- a/core/lib/contract_verifier/src/compilers/vyper.rs +++ b/core/lib/contract_verifier/src/compilers/vyper.rs @@ -103,9 +103,7 @@ impl Compiler for Vyper { if output.status.success() { let output = serde_json::from_slice(&output.stdout).context("vyper output is not valid JSON")?; - let output = - parse_standard_json_output(&output, input.contract_name, input.file_name, true)?; - Ok(output) + parse_standard_json_output(&output, input.contract_name, input.file_name, true) } else { Err(ContractVerifierError::CompilerError( "vyper", diff --git a/core/lib/contract_verifier/src/compilers/zksolc.rs b/core/lib/contract_verifier/src/compilers/zksolc.rs index 02114832874f..1b20f6d5d242 100644 --- a/core/lib/contract_verifier/src/compilers/zksolc.rs +++ b/core/lib/contract_verifier/src/compilers/zksolc.rs @@ -251,9 +251,7 @@ impl Compiler for ZkSolc { if output.status.success() { let output = serde_json::from_slice(&output.stdout) .context("zksolc output is not valid JSON")?; - let output = - parse_standard_json_output(&output, contract_name, file_name, false)?; - Ok(output) + parse_standard_json_output(&output, contract_name, file_name, false) } else { Err(ContractVerifierError::CompilerError( "zksolc", @@ -285,8 +283,7 @@ impl Compiler for ZkSolc { if output.status.success() { let output = String::from_utf8(output.stdout).context("zksolc output is not UTF-8")?; - let output = Self::parse_single_file_yul_output(&output)?; - Ok(output) + Self::parse_single_file_yul_output(&output) } else { Err(ContractVerifierError::CompilerError( "zksolc", diff --git a/core/lib/contract_verifier/src/compilers/zkvyper.rs b/core/lib/contract_verifier/src/compilers/zkvyper.rs index a5dfbaadcee3..4056736547f6 100644 --- a/core/lib/contract_verifier/src/compilers/zkvyper.rs +++ b/core/lib/contract_verifier/src/compilers/zkvyper.rs @@ -123,8 +123,7 @@ impl Compiler for ZkVyper { if output.status.success() { let output = serde_json::from_slice(&output.stdout) .context("zkvyper output is not valid JSON")?; - let output = Self::parse_output(&output, input.contract_name)?; - Ok(output) + Self::parse_output(&output, input.contract_name) } else { Err(ContractVerifierError::CompilerError( "zkvyper", diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 92e0c98098cb..4afeeb5e2b21 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -676,7 +676,12 @@ impl ContractVerificationDal<'_, '_> { )) }) .instrument("get_contract_verification_info_v2#similar") - .with_arg("address", &address) + .with_arg("address", &address) // Can be useful for debugging + .with_arg("bytecode_keccak256", &identifier.bytecode_keccak256) + .with_arg( + "bytecode_without_metadata_keccak256", + &bytecode_without_metadata_keccak256, + ) .fetch_optional(self.storage) .await? else { @@ -840,7 +845,7 @@ impl ContractVerificationDal<'_, '_> { // Sanity check. tracing::info!("All the rows are migrated, verifying the migration"); - let count_inequal = sqlx::query!( + let count_unequal = sqlx::query!( r#" SELECT COUNT(*) @@ -855,10 +860,10 @@ impl ContractVerificationDal<'_, '_> { .await? .count .unwrap(); - if count_inequal > 0 { + if count_unequal > 0 { anyhow::bail!( "Migration failed: {} rows have different data in the new table", - count_inequal + count_unequal ); } From 74bfd2ac1762e1067b329897a4dcc644bece7814 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Wed, 29 Jan 2025 12:23:24 +0400 Subject: [PATCH 15/16] Move partial matching logic to API --- core/lib/basic_types/src/bytecode.rs | 8 + ...a6dfb560d7f79406dc8c0b50c155c6b7cc887.json | 35 ---- ...072dcee6e6f8439e6b43eebd6df5563a4d0b9.json | 35 ++++ core/lib/dal/src/contract_verification_dal.rs | 158 ++++++------------ .../src/api_impl.rs | 89 ++++++++-- 5 files changed, 176 insertions(+), 149 deletions(-) delete mode 100644 core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json create mode 100644 core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json diff --git a/core/lib/basic_types/src/bytecode.rs b/core/lib/basic_types/src/bytecode.rs index 12b4df69a6c6..a96c5b2b10cd 100644 --- a/core/lib/basic_types/src/bytecode.rs +++ b/core/lib/basic_types/src/bytecode.rs @@ -168,6 +168,14 @@ impl BytecodeMarker { } } +/// Removes padding from the bytecode, if necessary. +pub fn trim_bytecode(bytecode_hash: BytecodeHash, raw: &[u8]) -> anyhow::Result<&[u8]> { + match bytecode_hash.marker() { + BytecodeMarker::EraVm => Ok(raw), + BytecodeMarker::Evm => trim_padded_evm_bytecode(bytecode_hash, raw), + } +} + /// Removes padding from an EVM bytecode, returning the original EVM bytecode. pub fn trim_padded_evm_bytecode(bytecode_hash: BytecodeHash, raw: &[u8]) -> anyhow::Result<&[u8]> { if bytecode_hash.marker() != BytecodeMarker::Evm { diff --git a/core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json b/core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json deleted file mode 100644 index 4b05ddce9694..000000000000 --- a/core/lib/dal/.sqlx/query-220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n verification_info,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256\n FROM\n contract_verification_info_v2\n WHERE\n bytecode_keccak256 = $1\n OR\n (\n bytecode_without_metadata_keccak256 IS NOT null\n AND bytecode_without_metadata_keccak256 = $2\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "verification_info", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "bytecode_keccak256", - "type_info": "Bytea" - }, - { - "ordinal": 2, - "name": "bytecode_without_metadata_keccak256", - "type_info": "Bytea" - } - ], - "parameters": { - "Left": [ - "Bytea", - "Bytea" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "220f29792139f7dfc19b886e343a6dfb560d7f79406dc8c0b50c155c6b7cc887" -} diff --git a/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json b/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json new file mode 100644 index 000000000000..45c11f60d9cf --- /dev/null +++ b/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n verification_info,\n bytecode_keccak256,\n bytecode_without_metadata_keccak256\n FROM\n contract_verification_info_v2\n WHERE\n bytecode_keccak256 = $1\n OR\n (\n bytecode_without_metadata_keccak256 IS NOT null\n AND bytecode_without_metadata_keccak256 = $2\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "verification_info", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "bytecode_keccak256", + "type_info": "Bytea" + }, + { + "ordinal": 2, + "name": "bytecode_without_metadata_keccak256", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Bytea", + "Bytea" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9" +} diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 4afeeb5e2b21..e8f260c396ea 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -5,7 +5,6 @@ use std::{ time::Duration, }; -use anyhow::Context as _; use rayon::prelude::*; use sqlx::postgres::types::PgInterval; use zksync_db_connection::{ @@ -14,11 +13,10 @@ use zksync_db_connection::{ }; use zksync_types::{ address_to_h256, - bytecode::{trim_padded_evm_bytecode, BytecodeHash, BytecodeMarker}, contract_verification::{ api::{ - VerificationIncomingRequest, VerificationInfo, VerificationProblem, - VerificationRequest, VerificationRequestStatus, + VerificationIncomingRequest, VerificationInfo, VerificationRequest, + VerificationRequestStatus, }, contract_identifier::ContractIdentifier, }, @@ -592,8 +590,7 @@ impl ContractVerificationDal<'_, '_> { &mut self, address: Address, ) -> anyhow::Result> { - // Try perfect match - if let Some(info) = sqlx::query!( + Ok(sqlx::query!( r#" SELECT verification_info @@ -607,108 +604,61 @@ impl ContractVerificationDal<'_, '_> { .try_map(|row| { serde_json::from_value(row.verification_info).decode_column("verification_info") }) - .instrument("get_contract_verification_info_v2#direct") + .instrument("get_contract_verification_info_v2") .with_arg("address", &address) .fetch_optional(self.storage) .await? - .flatten() - { - return Ok(Some(info)); - } - - // Try partial match - let Some(deployed_contract) = self.get_contract_info_for_verification(address).await? - else { - return Ok(None); - }; - - let bytecode_marker = BytecodeMarker::new(deployed_contract.bytecode_hash) - .context("unknown bytecode kind")?; - let deployed_bytecode = match bytecode_marker { - BytecodeMarker::EraVm => deployed_contract.bytecode.as_slice(), - BytecodeMarker::Evm => trim_padded_evm_bytecode( - BytecodeHash::try_from(deployed_contract.bytecode_hash) - .context("Invalid bytecode hash")?, - &deployed_contract.bytecode, - ) - .context("invalid stored EVM bytecode")?, - }; - - let identifier = ContractIdentifier::from_bytecode(bytecode_marker, deployed_bytecode); - // `unwrap_or_default` is safe, since we're checking that `bytecode_without_metadata_keccak256` is not null. - let bytecode_without_metadata_keccak256 = identifier - .bytecode_without_metadata_keccak256 - .as_ref() - .map(|h| h.hash()) - .unwrap_or_default(); - - let Some((mut info, bytecode_keccak256, bytecode_without_metadata_keccak256)) = - sqlx::query!( - r#" - SELECT - verification_info, - bytecode_keccak256, - bytecode_without_metadata_keccak256 - FROM - contract_verification_info_v2 - WHERE - bytecode_keccak256 = $1 - OR - ( - bytecode_without_metadata_keccak256 IS NOT null - AND bytecode_without_metadata_keccak256 = $2 - ) - "#, - identifier.bytecode_keccak256.as_bytes(), - bytecode_without_metadata_keccak256.as_bytes() - ) - .try_map(|row| { - let info = serde_json::from_value::(row.verification_info) - .decode_column("verification_info")?; - let bytecode_keccak256 = H256::from_slice(&row.bytecode_keccak256); - let bytecode_without_metadata_keccak256 = row - .bytecode_without_metadata_keccak256 - .map(|h| H256::from_slice(&h)); - Ok(( - info, - bytecode_keccak256, - bytecode_without_metadata_keccak256, - )) - }) - .instrument("get_contract_verification_info_v2#similar") - .with_arg("address", &address) // Can be useful for debugging - .with_arg("bytecode_keccak256", &identifier.bytecode_keccak256) - .with_arg( - "bytecode_without_metadata_keccak256", - &bytecode_without_metadata_keccak256, - ) - .fetch_optional(self.storage) - .await? - else { - return Ok(None); - }; - if identifier.bytecode_keccak256 != bytecode_keccak256 { - // Sanity check - if bytecode_without_metadata_keccak256.is_none() - || identifier - .bytecode_without_metadata_keccak256 - .map(|h| h.hash()) - != bytecode_without_metadata_keccak256 - { - tracing::error!( - contract_address = %address, - identifier = ?identifier, - bytecode_keccak256 = %bytecode_keccak256, - info = ?info, - "Bogus verification info fetched for contract", - ); - anyhow::bail!("Internal error: bogus verification info detected"); - } + .flatten()) + } - // Mark the contract as partial match (regardless of other issues). - info.verification_problems = vec![VerificationProblem::IncorrectMetadata]; - } - Ok(Some(info)) + pub async fn get_partial_match_verification_info( + &mut self, + bytecode_keccak256: H256, + bytecode_without_metadata_keccak256: Option, + ) -> DalResult)>> { + // `unwrap_or_default` is OK here, since we're checking that `bytecode_without_metadata_keccak256` is not null. + let bytecode_without_metadata_keccak256 = + bytecode_without_metadata_keccak256.unwrap_or_default(); + sqlx::query!( + r#" + SELECT + verification_info, + bytecode_keccak256, + bytecode_without_metadata_keccak256 + FROM + contract_verification_info_v2 + WHERE + bytecode_keccak256 = $1 + OR + ( + bytecode_without_metadata_keccak256 IS NOT null + AND bytecode_without_metadata_keccak256 = $2 + ) + "#, + bytecode_keccak256.as_bytes(), + bytecode_without_metadata_keccak256.as_bytes() + ) + .try_map(|row| { + let info = serde_json::from_value::(row.verification_info) + .decode_column("verification_info")?; + let bytecode_keccak256 = H256::from_slice(&row.bytecode_keccak256); + let bytecode_without_metadata_keccak256 = row + .bytecode_without_metadata_keccak256 + .map(|h| H256::from_slice(&h)); + Ok(( + info, + bytecode_keccak256, + bytecode_without_metadata_keccak256, + )) + }) + .instrument("get_partial_match_verification_info") + .with_arg("bytecode_keccak256", &bytecode_keccak256) + .with_arg( + "bytecode_without_metadata_keccak256", + &bytecode_without_metadata_keccak256, + ) + .fetch_optional(self.storage) + .await } /// Checks if migration from `contracts_verification_info` to `contract_verification_info_v2` is performed diff --git a/core/node/contract_verification_server/src/api_impl.rs b/core/node/contract_verification_server/src/api_impl.rs index 1cf20befab02..547d7f942fc1 100644 --- a/core/node/contract_verification_server/src/api_impl.rs +++ b/core/node/contract_verification_server/src/api_impl.rs @@ -7,11 +7,15 @@ use axum::{ response::{IntoResponse, Response}, Json, }; -use zksync_dal::{CoreDal, DalError}; +use zksync_dal::{contract_verification_dal::ContractVerificationDal, CoreDal, DalError}; use zksync_types::{ - bytecode::BytecodeMarker, - contract_verification::api::{ - CompilerVersions, VerificationIncomingRequest, VerificationInfo, VerificationRequestStatus, + bytecode::{trim_bytecode, BytecodeHash, BytecodeMarker}, + contract_verification::{ + api::{ + CompilerVersions, VerificationIncomingRequest, VerificationInfo, VerificationProblem, + VerificationRequestStatus, + }, + contract_identifier::ContractIdentifier, }, Address, }; @@ -220,15 +224,80 @@ impl RestApi { address: Path
, ) -> ApiResult { let method_latency = METRICS.call[&"contract_verification_info"].start(); - let info = self_ + let mut conn = self_ .replica_connection_pool .connection_tagged("api") - .await? - .contract_verification_dal() - .get_contract_verification_info(*address) - .await? - .ok_or(ApiError::VerificationInfoNotFound)?; + .await?; + let mut dal = conn.contract_verification_dal(); + + let info = if let Some(info) = dal.get_contract_verification_info(*address).await? { + info + } else if let Some(partial_match) = + get_partial_match_verification_info(&mut dal, *address).await? + { + partial_match + } else { + return Err(ApiError::VerificationInfoNotFound); + }; method_latency.observe(); Ok(Json(info)) } } + +/// Tries to do a lookup for partial match verification info. +/// Should be called only if a perfect match is not found. +async fn get_partial_match_verification_info( + dal: &mut ContractVerificationDal<'_, '_>, + address: Address, +) -> anyhow::Result> { + let Some(deployed_contract) = dal.get_contract_info_for_verification(address).await? else { + return Ok(None); + }; + + let bytecode_hash = + BytecodeHash::try_from(deployed_contract.bytecode_hash).context("Invalid bytecode hash")?; + let deployed_bytecode = trim_bytecode(bytecode_hash, &deployed_contract.bytecode) + .context("Invalid deployed bytecode")?; + + let identifier = ContractIdentifier::from_bytecode(bytecode_hash.marker(), deployed_bytecode); + // `unwrap_or_default` is safe, since we're checking that `bytecode_without_metadata_keccak256` is not null. + let bytecode_without_metadata_keccak256 = identifier + .bytecode_without_metadata_keccak256 + .as_ref() + .map(|h| h.hash()); + + let Some((mut info, fetched_keccak256, fetched_keccak256_without_metadata)) = dal + .get_partial_match_verification_info( + identifier.bytecode_keccak256, + bytecode_without_metadata_keccak256, + ) + .await? + else { + return Ok(None); + }; + + if identifier.bytecode_keccak256 != fetched_keccak256 { + // Sanity check + let has_hash_without_metadata = fetched_keccak256_without_metadata.is_some(); + let hashes_without_metadata_match = identifier + .bytecode_without_metadata_keccak256 + .map(|h| h.hash()) + == bytecode_without_metadata_keccak256; + + if !has_hash_without_metadata || !hashes_without_metadata_match { + tracing::error!( + contract_address = %address, + identifier = ?identifier, + fetched_keccak256 = %fetched_keccak256, + info = ?info, + "Bogus verification info fetched for contract", + ); + anyhow::bail!("Internal error: bogus verification info detected"); + } + + // Mark the contract as partial match (regardless of other issues). + info.verification_problems = vec![VerificationProblem::IncorrectMetadata]; + } + + Ok(Some(info)) +} From 5a534b706ec24d175ddd07d86cc5323b40c228b1 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Thu, 30 Jan 2025 10:42:11 +0400 Subject: [PATCH 16/16] Moar comments --- core/lib/contract_verifier/src/lib.rs | 4 +- core/lib/contract_verifier/src/tests/real.rs | 40 +++--- ...072dcee6e6f8439e6b43eebd6df5563a4d0b9.json | 2 +- ...102800_contract-verifier-new-schema.up.sql | 2 +- core/lib/dal/src/contract_verification_dal.rs | 29 ++--- .../contract_identifier.rs | 114 +++++++++++------- .../src/api_impl.rs | 23 ++-- 7 files changed, 108 insertions(+), 106 deletions(-) diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 2f5ac0f46ba3..864c7b747d43 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -583,9 +583,7 @@ impl ContractVerifier { .save_verification_info( info, identifier.bytecode_keccak256, - identifier - .bytecode_without_metadata_keccak256 - .map(|hash| hash.hash()), + identifier.bytecode_without_metadata_keccak256, ) .await?; tracing::info!("Successfully processed request with id = {request_id}"); diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index 524b7f818c3b..d82edf7b0020 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -309,8 +309,8 @@ async fn compiling_yul_with_zksolc() { assert!(output.deployed_bytecode.is_none()); assert_eq!(output.abi, serde_json::json!([])); assert_matches!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Keccak256(_)) + identifier.detected_metadata, + Some(DetectedMetadata::Keccak256) ); } @@ -333,7 +333,7 @@ async fn compiling_standalone_yul() { assert_ne!(output.deployed_bytecode.unwrap(), output.bytecode); assert_eq!(output.abi, serde_json::json!([])); assert_matches!( - identifier.bytecode_without_metadata_keccak256, + identifier.detected_metadata, None, "No metadata for compiler yul for EVM" ); @@ -389,8 +389,8 @@ async fn using_real_zkvyper(specify_contract_file: bool) { validate_bytecode(&output.bytecode).unwrap(); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); assert_matches!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Keccak256(_)) + identifier.detected_metadata, + Some(DetectedMetadata::Keccak256) ); } @@ -420,7 +420,7 @@ async fn using_standalone_vyper(specify_contract_file: bool) { assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); // Vyper does not provide metadata for bytecode. - assert_matches!(identifier.bytecode_without_metadata_keccak256, None); + assert_matches!(identifier.detected_metadata, None); } #[tokio::test] @@ -444,7 +444,7 @@ async fn using_standalone_vyper_without_optimization() { assert!(output.deployed_bytecode.is_some()); assert_eq!(output.abi, without_internal_types(counter_contract_abi())); // Vyper does not provide metadata for bytecode. - assert_matches!(identifier.bytecode_without_metadata_keccak256, None); + assert_matches!(identifier.detected_metadata, None); } #[tokio::test] @@ -549,21 +549,21 @@ async fn using_real_compiler_in_verifier(bytecode_kind: BytecodeMarker, toolchai match (bytecode_kind, toolchain) { (BytecodeMarker::Evm, Toolchain::Vyper) => { assert!( - identifier.bytecode_without_metadata_keccak256.is_none(), + identifier.detected_metadata.is_none(), "No metadata for EVM Vyper" ); } (BytecodeMarker::Evm, Toolchain::Solidity) => { assert_matches!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Cbor(_)), + identifier.detected_metadata, + Some(DetectedMetadata::Cbor), "Cbor metadata for EVM Solidity by default" ); } (BytecodeMarker::EraVm, _) => { assert_matches!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Keccak256(_)), + identifier.detected_metadata, + Some(DetectedMetadata::Keccak256), "Keccak256 metadata for EraVM by default" ); } @@ -710,21 +710,21 @@ async fn using_zksolc_partial_match(use_cbor: bool) { ); if use_cbor { assert_matches!( - identifier_for_request.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Cbor(_)) + identifier_for_request.detected_metadata, + Some(DetectedMetadata::Cbor) ); assert_matches!( - identifier_for_storage.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Cbor(_)) + identifier_for_storage.detected_metadata, + Some(DetectedMetadata::Cbor) ); } else { assert_matches!( - identifier_for_request.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Keccak256(_)) + identifier_for_request.detected_metadata, + Some(DetectedMetadata::Keccak256) ); assert_matches!( - identifier_for_storage.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Keccak256(_)) + identifier_for_storage.detected_metadata, + Some(DetectedMetadata::Keccak256) ); } diff --git a/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json b/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json index 45c11f60d9cf..a78dcf480064 100644 --- a/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json +++ b/core/lib/dal/.sqlx/query-6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9.json @@ -28,7 +28,7 @@ "nullable": [ false, false, - true + false ] }, "hash": "6cb50a8fbe1341ba7ea496bb0f2072dcee6e6f8439e6b43eebd6df5563a4d0b9" diff --git a/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql index 974f8b2e2717..c7eee9063221 100644 --- a/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql +++ b/core/lib/dal/migrations/20250122102800_contract-verifier-new-schema.up.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS contract_verification_info_v2 ( initial_contract_addr BYTEA NOT NULL PRIMARY KEY, bytecode_keccak256 BYTEA NOT NULL, - bytecode_without_metadata_keccak256 BYTEA, + bytecode_without_metadata_keccak256 BYTEA NOT NULL, verification_info JSONB NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index e8f260c396ea..9125f972c55b 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -196,7 +196,7 @@ impl ContractVerificationDal<'_, '_> { &mut self, verification_info: VerificationInfo, bytecode_keccak256: H256, - bytecode_without_metadata_keccak256: Option, + bytecode_without_metadata_keccak256: H256, ) -> DalResult<()> { let mut transaction = self.storage.start_transaction().await?; let id = verification_info.request.id; @@ -222,11 +222,6 @@ impl ContractVerificationDal<'_, '_> { // Serialization should always succeed. let verification_info_json = serde_json::to_value(verification_info) .expect("Failed to serialize verification info into serde_json"); - let bytecode_without_metadata_keccak256: Option<&[u8]> = - match &bytecode_without_metadata_keccak256 { - Some(h) => Some(h.as_bytes()), - None => None, - }; sqlx::query!( r#" INSERT INTO @@ -247,7 +242,7 @@ impl ContractVerificationDal<'_, '_> { "#, address.as_bytes(), bytecode_keccak256.as_bytes(), - bytecode_without_metadata_keccak256, + bytecode_without_metadata_keccak256.as_bytes(), &verification_info_json ) .instrument("save_verification_info#insert") @@ -614,11 +609,8 @@ impl ContractVerificationDal<'_, '_> { pub async fn get_partial_match_verification_info( &mut self, bytecode_keccak256: H256, - bytecode_without_metadata_keccak256: Option, - ) -> DalResult)>> { - // `unwrap_or_default` is OK here, since we're checking that `bytecode_without_metadata_keccak256` is not null. - let bytecode_without_metadata_keccak256 = - bytecode_without_metadata_keccak256.unwrap_or_default(); + bytecode_without_metadata_keccak256: H256, + ) -> DalResult> { sqlx::query!( r#" SELECT @@ -642,9 +634,8 @@ impl ContractVerificationDal<'_, '_> { let info = serde_json::from_value::(row.verification_info) .decode_column("verification_info")?; let bytecode_keccak256 = H256::from_slice(&row.bytecode_keccak256); - let bytecode_without_metadata_keccak256 = row - .bytecode_without_metadata_keccak256 - .map(|h| H256::from_slice(&h)); + let bytecode_without_metadata_keccak256 = + H256::from_slice(&row.bytecode_without_metadata_keccak256); Ok(( info, bytecode_keccak256, @@ -751,14 +742,12 @@ impl ContractVerificationDal<'_, '_> { for idx in 0..addresses.len() { let address = hex::encode(&addresses[idx]); let bytecode_keccak256 = hex::encode(ids[idx].bytecode_keccak256); - let bytecode_without_metadata_keccak256 = ids[idx] - .bytecode_without_metadata_keccak256 - .map(|h| format!(r#"\\x{}"#, hex::encode(h.hash()))) - .unwrap_or_else(|| "null".to_owned()); + let bytecode_without_metadata_keccak256 = + hex::encode(ids[idx].bytecode_without_metadata_keccak256); let verification_info = verification_infos[idx].replace('"', r#""""#); let row = format!( - r#"\\x{initial_contract_addr},\\x{bytecode_keccak256},{bytecode_without_metadata_keccak256},"{verification_info}",{created_at},{updated_at}"#, + r#"\\x{initial_contract_addr},\\x{bytecode_keccak256},\\x{bytecode_without_metadata_keccak256},"{verification_info}",{created_at},{updated_at}"#, initial_contract_addr = address, bytecode_keccak256 = bytecode_keccak256, bytecode_without_metadata_keccak256 = bytecode_without_metadata_keccak256, diff --git a/core/lib/types/src/contract_verification/contract_identifier.rs b/core/lib/types/src/contract_verification/contract_identifier.rs index 234d5db2785b..354ddeaa0e29 100644 --- a/core/lib/types/src/contract_verification/contract_identifier.rs +++ b/core/lib/types/src/contract_verification/contract_identifier.rs @@ -24,8 +24,10 @@ pub struct ContractIdentifier { pub bytecode_keccak256: H256, /// keccak256 hash of the contract bytecode without metadata (e.g. with either /// CBOR or keccak256 metadata hash being stripped). - /// Can be absent if the contract bytecode doesn't have metadata. - pub bytecode_without_metadata_keccak256: Option, + /// If no metadata is detected, equal to `bytecode_keccak256`. + pub bytecode_without_metadata_keccak256: H256, + /// Kind of detected metadata. + pub detected_metadata: Option, } #[derive(Debug, Clone, Copy, PartialEq)] @@ -41,18 +43,10 @@ pub enum Match { /// Metadata detected in the contract bytecode. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DetectedMetadata { - /// keccak256 hash of the metadata detected (only for EraVM). - Keccak256(H256), - /// CBOR metadata detected. - Cbor(H256), -} - -impl DetectedMetadata { - pub fn hash(&self) -> H256 { - match self { - Self::Keccak256(hash) | Self::Cbor(hash) => *hash, - } - } + /// keccak256 metadata (only for EraVM) + Keccak256, + /// CBOR metadata + Cbor, } /// Possible values for the metadata hashes structure. @@ -82,22 +76,26 @@ impl ContractIdentifier { // Try to detect metadata. // CBOR takes precedence (since keccak doesn't have direct markers, so it's partially a // fallback). - let bytecode_without_metadata_keccak256 = - Self::detect_cbor_metadata(bytecode_marker, bytecode) - .or_else(|| Self::detect_keccak_metadata(bytecode_marker, bytecode)); + let (detected_metadata, bytecode_without_metadata_keccak256) = + if let Some(hash) = Self::detect_cbor_metadata(bytecode_marker, bytecode) { + (Some(DetectedMetadata::Cbor), hash) + } else if let Some(hash) = Self::detect_keccak_metadata(bytecode_marker, bytecode) { + (Some(DetectedMetadata::Keccak256), hash) + } else { + // Fallback + (None, bytecode_keccak256) + }; Self { bytecode_marker, bytecode_keccak256, bytecode_without_metadata_keccak256, + detected_metadata, } } /// Will try to detect keccak256 metadata hash (only for EraVM) - fn detect_keccak_metadata( - bytecode_marker: BytecodeMarker, - bytecode: &[u8], - ) -> Option { + fn detect_keccak_metadata(bytecode_marker: BytecodeMarker, bytecode: &[u8]) -> Option { // For EraVM, the one option for metadata hash is keccak256 hash of the metadata. if bytecode_marker == BytecodeMarker::EraVm { // For metadata, we might have padding: it takes either 32 or 64 bytes depending @@ -105,17 +103,14 @@ impl ContractIdentifier { // if there is padding. let bytecode_without_metadata = Self::strip_padding(bytecode, 32)?; let hash = H256(keccak256(bytecode_without_metadata)); - Some(DetectedMetadata::Keccak256(hash)) + Some(hash) } else { None } } /// Will try to detect CBOR metadata. - fn detect_cbor_metadata( - bytecode_marker: BytecodeMarker, - bytecode: &[u8], - ) -> Option { + fn detect_cbor_metadata(bytecode_marker: BytecodeMarker, bytecode: &[u8]) -> Option { let length = bytecode.len(); // Last two bytes is the length of the metadata in big endian. @@ -153,7 +148,7 @@ impl ContractIdentifier { } }; let hash = H256(keccak256(bytecode_without_metadata)); - Some(DetectedMetadata::Cbor(hash)) + Some(hash) } /// Adds one word to the metadata length and check if it's a padding word. @@ -190,9 +185,8 @@ impl ContractIdentifier { // and presence in another, or different kinds of metadata. This is OK: partial // match is needed mostly when you cannot reproduce the original metadata, but one always // can submit the contract with the same metadata kind. - if self.bytecode_without_metadata_keccak256.is_some() - && self.bytecode_without_metadata_keccak256 - == other_identifier.bytecode_without_metadata_keccak256 + if self.bytecode_without_metadata_keccak256 + == other_identifier.bytecode_without_metadata_keccak256 { return Match::Partial; } @@ -222,8 +216,12 @@ mod tests { "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Cbor(bytecode_without_metadata_keccak256)), + identifier.detected_metadata, + Some(DetectedMetadata::Cbor), + "Incorrect detected metadata" + ); + assert_eq!( + identifier.bytecode_without_metadata_keccak256, bytecode_without_metadata_keccak256, "Incorrect bytecode without metadata hash" ); } @@ -243,8 +241,12 @@ mod tests { "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Cbor(bytecode_without_metadata_keccak256)), + identifier.detected_metadata, + Some(DetectedMetadata::Cbor), + "Incorrect detected metadata" + ); + assert_eq!( + identifier.bytecode_without_metadata_keccak256, bytecode_without_metadata_keccak256, "Incorrect bytecode without metadata hash" ); } @@ -266,10 +268,12 @@ mod tests { "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Keccak256( - bytecode_without_metadata_keccak256 - )), + identifier.detected_metadata, + Some(DetectedMetadata::Keccak256), + "Incorrect detected metadata" + ); + assert_eq!( + identifier.bytecode_without_metadata_keccak256, bytecode_without_metadata_keccak256, "Incorrect bytecode without metadata hash" ); } @@ -289,10 +293,12 @@ mod tests { "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Keccak256( - bytecode_without_metadata_keccak256 - )), + identifier.detected_metadata, + Some(DetectedMetadata::Keccak256), + "Incorrect detected metadata" + ); + assert_eq!( + identifier.bytecode_without_metadata_keccak256, bytecode_without_metadata_keccak256, "Incorrect bytecode without metadata hash" ); } @@ -310,7 +316,13 @@ mod tests { "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_keccak256, None, + identifier.detected_metadata, None, + "Incorrect detected metadata" + ); + // When no metadata is detected, `bytecode_without_metadata_keccak256` is equal to + // `bytecode_keccak256`. + assert_eq!( + identifier.bytecode_without_metadata_keccak256, bytecode_keccak256, "Incorrect bytecode without metadata hash" ); } @@ -329,7 +341,13 @@ mod tests { "Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_keccak256, None, + identifier.detected_metadata, None, + "Incorrect detected metadata" + ); + // When no metadata is detected, `bytecode_without_metadata_keccak256` is equal to + // `bytecode_keccak256`. + assert_eq!( + identifier.bytecode_without_metadata_keccak256, bytecode_keccak256, "Incorrect bytecode without metadata hash" ); } @@ -365,8 +383,12 @@ mod tests { "{label}: Incorrect bytecode hash" ); assert_eq!( - identifier.bytecode_without_metadata_keccak256, - Some(DetectedMetadata::Cbor(bytecode_without_metadata_keccak256)), + identifier.detected_metadata, + Some(DetectedMetadata::Cbor), + "{label}: Incorrect detected metadata" + ); + assert_eq!( + identifier.bytecode_without_metadata_keccak256, bytecode_without_metadata_keccak256, "{label}: Incorrect bytecode without metadata hash" ); } diff --git a/core/node/contract_verification_server/src/api_impl.rs b/core/node/contract_verification_server/src/api_impl.rs index 547d7f942fc1..3e21c2f2cba8 100644 --- a/core/node/contract_verification_server/src/api_impl.rs +++ b/core/node/contract_verification_server/src/api_impl.rs @@ -260,16 +260,10 @@ async fn get_partial_match_verification_info( .context("Invalid deployed bytecode")?; let identifier = ContractIdentifier::from_bytecode(bytecode_hash.marker(), deployed_bytecode); - // `unwrap_or_default` is safe, since we're checking that `bytecode_without_metadata_keccak256` is not null. - let bytecode_without_metadata_keccak256 = identifier - .bytecode_without_metadata_keccak256 - .as_ref() - .map(|h| h.hash()); - let Some((mut info, fetched_keccak256, fetched_keccak256_without_metadata)) = dal .get_partial_match_verification_info( identifier.bytecode_keccak256, - bytecode_without_metadata_keccak256, + identifier.bytecode_without_metadata_keccak256, ) .await? else { @@ -278,17 +272,16 @@ async fn get_partial_match_verification_info( if identifier.bytecode_keccak256 != fetched_keccak256 { // Sanity check - let has_hash_without_metadata = fetched_keccak256_without_metadata.is_some(); - let hashes_without_metadata_match = identifier - .bytecode_without_metadata_keccak256 - .map(|h| h.hash()) - == bytecode_without_metadata_keccak256; + let has_metadata = identifier.detected_metadata.is_some(); + let hashes_without_metadata_match = + identifier.bytecode_without_metadata_keccak256 == fetched_keccak256_without_metadata; - if !has_hash_without_metadata || !hashes_without_metadata_match { + if !has_metadata || !hashes_without_metadata_match { tracing::error!( - contract_address = %address, + contract_address = ?address, identifier = ?identifier, - fetched_keccak256 = %fetched_keccak256, + fetched_keccak256 = ?fetched_keccak256, + fetched_keccak256_without_metadata = ?fetched_keccak256_without_metadata, info = ?info, "Bogus verification info fetched for contract", );