From 8835b31d76b2f7c45416eaf67a748d8df9dbc753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Bene=C5=A1?= Date: Fri, 25 Oct 2024 16:08:55 -0600 Subject: [PATCH] refactor: passing partial note logs through transient storage (#9356) --- .../encrypted_event_emission.nr | 30 ++- .../encrypted_logs/encrypted_note_emission.nr | 63 ++---- .../aztec/src/encrypted_logs/payload.nr | 33 ++- .../aztec-nr/aztec/src/macros/notes/mod.nr | 202 +++++++++++++++--- .../contracts/nft_contract/src/main.nr | 46 ++-- .../contracts/token_contract/src/main.nr | 115 ++++++---- .../crates/types/src/meta/mod.nr | 7 +- .../logs/l1_payload/encrypted_log_payload.ts | 12 ++ .../src/logs/l1_payload/l1_note_payload.ts | 131 ++++++++++-- .../end-to-end/src/e2e_block_building.test.ts | 5 +- yarn-project/end-to-end/src/e2e_nft.test.ts | 41 +++- .../foundation/src/collection/array.ts | 10 +- .../foundation/src/serialize/buffer_reader.ts | 8 + .../src/database/deferred_note_dao.test.ts | 22 +- .../pxe/src/database/deferred_note_dao.ts | 25 +-- .../pxe/src/database/incoming_note_dao.ts | 3 +- .../pxe/src/database/kv_pxe_database.ts | 2 +- .../pxe/src/database/outgoing_note_dao.ts | 3 +- .../src/note_processor/note_processor.test.ts | 66 +++--- .../pxe/src/note_processor/note_processor.ts | 136 ++++++------ .../utils/add_nullable_field_to_payload.ts | 67 ------ .../utils/add_public_values_to_payload.ts | 63 ++++++ .../utils/brute_force_note_info.ts | 14 +- .../note_processor/utils/produce_note_daos.ts | 12 +- .../utils/produce_note_daos_for_key.ts | 133 ++---------- .../src/public/enqueued_call_simulator.ts | 35 ++- 26 files changed, 763 insertions(+), 521 deletions(-) delete mode 100644 yarn-project/pxe/src/note_processor/utils/add_nullable_field_to_payload.ts create mode 100644 yarn-project/pxe/src/note_processor/utils/add_public_values_to_payload.ts diff --git a/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_event_emission.nr b/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_event_emission.nr index 5cccd74799b..a3efa97c6f5 100644 --- a/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_event_emission.nr +++ b/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_event_emission.nr @@ -1,5 +1,5 @@ use crate::{ - context::PrivateContext, encrypted_logs::payload::compute_encrypted_log, + context::PrivateContext, encrypted_logs::payload::compute_private_log_payload, event::event_interface::EventInterface, keys::getters::get_ovsk_app, oracle::random::random, }; use dep::protocol_types::{ @@ -8,7 +8,8 @@ use dep::protocol_types::{ public_keys::{IvpkM, OvpkM}, }; -fn compute_raw_event_log( +/// Computes private event log payload and a log hash +fn compute_payload_and_hash( context: PrivateContext, event: Event, randomness: Field, @@ -22,13 +23,22 @@ where { let contract_address: AztecAddress = context.this_address(); let plaintext = event.private_to_be_bytes(randomness); - let encrypted_log: [u8; 416 + N * 32] = - compute_encrypted_log(contract_address, ovsk_app, ovpk, ivpk, recipient, plaintext); + + // For event logs we never include public values prefix as there are never any public values + let encrypted_log: [u8; 416 + N * 32] = compute_private_log_payload( + contract_address, + ovsk_app, + ovpk, + ivpk, + recipient, + plaintext, + false, + ); let log_hash = sha256_to_field(encrypted_log); (encrypted_log, log_hash) } -unconstrained fn compute_raw_event_log_unconstrained( +unconstrained fn compute_payload_and_hash_unconstrained( context: PrivateContext, event: Event, randomness: Field, @@ -40,7 +50,7 @@ where Event: EventInterface, { let ovsk_app = get_ovsk_app(ovpk.hash()); - compute_raw_event_log(context, event, randomness, ovsk_app, ovpk, ivpk, recipient) + compute_payload_and_hash(context, event, randomness, ovsk_app, ovpk, ivpk, recipient) } pub fn encode_and_encrypt_event( @@ -60,7 +70,7 @@ where let randomness = unsafe { random() }; let ovsk_app: Field = context.request_ovsk_app(ovpk.hash()); let (encrypted_log, log_hash) = - compute_raw_event_log(*context, e, randomness, ovsk_app, ovpk, ivpk, recipient); + compute_payload_and_hash(*context, e, randomness, ovsk_app, ovpk, ivpk, recipient); context.emit_raw_event_log_with_masked_address(randomness, encrypted_log, log_hash); } } @@ -81,7 +91,7 @@ where // value generation. let randomness = unsafe { random() }; let (encrypted_log, log_hash) = unsafe { - compute_raw_event_log_unconstrained(*context, e, randomness, ovpk, ivpk, recipient) + compute_payload_and_hash_unconstrained(*context, e, randomness, ovpk, ivpk, recipient) }; context.emit_raw_event_log_with_masked_address(randomness, encrypted_log, log_hash); } @@ -103,7 +113,7 @@ where |e: Event| { let ovsk_app: Field = context.request_ovsk_app(ovpk.hash()); let (encrypted_log, log_hash) = - compute_raw_event_log(*context, e, randomness, ovsk_app, ovpk, ivpk, recipient); + compute_payload_and_hash(*context, e, randomness, ovsk_app, ovpk, ivpk, recipient); context.emit_raw_event_log_with_masked_address(randomness, encrypted_log, log_hash); } } @@ -134,7 +144,7 @@ where // return the log from this function to the app, otherwise it could try to do stuff with it and then that might // be wrong. let (encrypted_log, log_hash) = unsafe { - compute_raw_event_log_unconstrained(*context, e, randomness, ovpk, ivpk, recipient) + compute_payload_and_hash_unconstrained(*context, e, randomness, ovpk, ivpk, recipient) }; context.emit_raw_event_log_with_masked_address(randomness, encrypted_log, log_hash); } diff --git a/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_note_emission.nr b/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_note_emission.nr index 5239418d7b9..a5e9f30b82a 100644 --- a/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_note_emission.nr +++ b/noir-projects/aztec-nr/aztec/src/encrypted_logs/encrypted_note_emission.nr @@ -1,6 +1,6 @@ use crate::{ context::PrivateContext, - encrypted_logs::payload::compute_encrypted_log, + encrypted_logs::payload::compute_private_log_payload, keys::getters::get_ovsk_app, note::{note_emission::NoteEmission, note_interface::NoteInterface}, }; @@ -11,14 +11,15 @@ use dep::protocol_types::{ public_keys::{IvpkM, OvpkM, PublicKeys}, }; -fn compute_raw_note_log( +/// Computes private note log payload and a log hash +fn compute_payload_and_hash( context: PrivateContext, note: Note, ovsk_app: Field, ovpk: OvpkM, ivpk: IvpkM, recipient: AztecAddress, -) -> (u32, [u8; 416 + N * 32], Field) +) -> (u32, [u8; 417 + N * 32], Field) where Note: NoteInterface, { @@ -33,25 +34,34 @@ where let contract_address: AztecAddress = context.this_address(); let plaintext = note.to_be_bytes(storage_slot); - let encrypted_log: [u8; 416 + N * 32] = - compute_encrypted_log(contract_address, ovsk_app, ovpk, ivpk, recipient, plaintext); + + // For note logs we always include public values prefix + let encrypted_log: [u8; 417 + N * 32] = compute_private_log_payload( + contract_address, + ovsk_app, + ovpk, + ivpk, + recipient, + plaintext, + true, + ); let log_hash = sha256_to_field(encrypted_log); (note_hash_counter, encrypted_log, log_hash) } -unconstrained fn compute_raw_note_log_unconstrained( +unconstrained fn compute_payload_and_hash_unconstrained( context: PrivateContext, note: Note, ovpk: OvpkM, ivpk: IvpkM, recipient: AztecAddress, -) -> (u32, [u8; 416 + N * 32], Field) +) -> (u32, [u8; 417 + N * 32], Field) where Note: NoteInterface, { let ovsk_app = get_ovsk_app(ovpk.hash()); - compute_raw_note_log(context, note, ovsk_app, ovpk, ivpk, recipient) + compute_payload_and_hash(context, note, ovsk_app, ovpk, ivpk, recipient) } // This function seems to be affected by the following Noir bug: @@ -70,7 +80,7 @@ where let ovsk_app: Field = context.request_ovsk_app(ovpk.hash()); let (note_hash_counter, encrypted_log, log_hash) = - compute_raw_note_log(*context, e.note, ovsk_app, ovpk, ivpk, recipient); + compute_payload_and_hash(*context, e.note, ovsk_app, ovpk, ivpk, recipient); context.emit_raw_note_log(note_hash_counter, encrypted_log, log_hash); } } @@ -104,38 +114,9 @@ where // for the log to be deleted when it shouldn't have (which is fine - they can already make the content be // whatever), or cause for the log to not be deleted when it should have (which is also fine - it'll be a log // for a note that doesn't exist). - let (note_hash_counter, encrypted_log, log_hash) = - unsafe { compute_raw_note_log_unconstrained(*context, e.note, ovpk, ivpk, recipient) }; + let (note_hash_counter, encrypted_log, log_hash) = unsafe { + compute_payload_and_hash_unconstrained(*context, e.note, ovpk, ivpk, recipient) + }; context.emit_raw_note_log(note_hash_counter, encrypted_log, log_hash); } } - -/// Encrypts a partial log and emits it. Takes recipient keys on the input and encrypts both the outgoing and incoming -/// logs for the recipient. This is necessary because in the partial notes flow the outgoing always has to be the same -/// as the incoming to not leak any information (typically the `from` party finalizing the partial note in public does -/// not know who the recipient is). -pub fn encrypt_and_emit_partial_log( - context: &mut PrivateContext, - log_plaintext: [u8; M], - recipient_keys: PublicKeys, - recipient: AztecAddress, -) { - let ovsk_app: Field = context.request_ovsk_app(recipient_keys.ovpk_m.hash()); - - let encrypted_log: [u8; 352 + M] = compute_encrypted_log( - context.this_address(), - ovsk_app, - recipient_keys.ovpk_m, - recipient_keys.ivpk_m, - recipient, - log_plaintext, - ); - let log_hash = sha256_to_field(encrypted_log); - - // Unfortunately we need to push a dummy note hash to the context here because a note log requires having - // a counter that corresponds to a note hash in the same call. - let note_hash_counter = context.side_effect_counter; - context.push_note_hash(5); - - context.emit_raw_note_log(note_hash_counter, encrypted_log, log_hash); -} diff --git a/noir-projects/aztec-nr/aztec/src/encrypted_logs/payload.nr b/noir-projects/aztec-nr/aztec/src/encrypted_logs/payload.nr index 9831127a58c..c01e57b8ce9 100644 --- a/noir-projects/aztec-nr/aztec/src/encrypted_logs/payload.nr +++ b/noir-projects/aztec-nr/aztec/src/encrypted_logs/payload.nr @@ -17,13 +17,14 @@ use crate::{ utils::point::point_to_bytes, }; -pub fn compute_encrypted_log( +fn compute_private_log_payload( contract_address: AztecAddress, ovsk_app: Field, ovpk: OvpkM, ivpk: IvpkM, recipient: AztecAddress, plaintext: [u8; P], + include_public_values_prefix: bool, ) -> [u8; M] { let (eph_sk, eph_pk) = generate_ephemeral_key_pair(); @@ -41,27 +42,39 @@ pub fn compute_encrypted_log( eph_pk, ); + // If we include the prefix for number of public values, we need to add 1 byte to the offset + let mut offset = if include_public_values_prefix { 1 } else { 0 }; + let mut encrypted_bytes: [u8; M] = [0; M]; // @todo We ignore the tags for now + offset += 64; + let eph_pk_bytes = point_to_bytes(eph_pk); for i in 0..32 { - encrypted_bytes[64 + i] = eph_pk_bytes[i]; + encrypted_bytes[offset + i] = eph_pk_bytes[i]; } + + offset += 32; for i in 0..48 { - encrypted_bytes[96 + i] = incoming_header_ciphertext[i]; - encrypted_bytes[144 + i] = outgoing_header_ciphertext[i]; + encrypted_bytes[offset + i] = incoming_header_ciphertext[i]; + encrypted_bytes[offset + 48 + i] = outgoing_header_ciphertext[i]; } + + offset += 48 * 2; for i in 0..144 { - encrypted_bytes[192 + i] = outgoing_body_ciphertext[i]; + encrypted_bytes[offset + i] = outgoing_body_ciphertext[i]; } + + offset += 144; // Then we fill in the rest as the incoming body ciphertext - let size = M - 336; + let size = M - offset; assert_eq(size, incoming_body_ciphertext.len(), "ciphertext length mismatch"); for i in 0..size { - encrypted_bytes[336 + i] = incoming_body_ciphertext[i]; + encrypted_bytes[offset + i] = incoming_body_ciphertext[i]; } // Current unoptimized size of the encrypted log + // empty_prefix (1 byte) // incoming_tag (32 bytes) // outgoing_tag (32 bytes) // eph_pk (32 bytes) @@ -160,7 +173,8 @@ pub fn compute_outgoing_body_ciphertext( mod test { use crate::encrypted_logs::payload::{ - compute_encrypted_log, compute_incoming_body_ciphertext, compute_outgoing_body_ciphertext, + compute_incoming_body_ciphertext, compute_outgoing_body_ciphertext, + compute_private_log_payload, }; use dep::protocol_types::{ address::AztecAddress, @@ -210,13 +224,14 @@ mod test { 0x25afb798ea6d0b8c1618e50fdeafa463059415013d3b7c75d46abf5e242be70c, ); - let log: [u8; 448] = compute_encrypted_log( + let log: [u8; 448] = compute_private_log_payload( contract_address, ovsk_app, ovpk_m, ivpk_m, recipient, plaintext, + false, ); // The following value was generated by `tagged_log.test.ts` diff --git a/noir-projects/aztec-nr/aztec/src/macros/notes/mod.nr b/noir-projects/aztec-nr/aztec/src/macros/notes/mod.nr index 88820b32d0c..4cd35bbc3c3 100644 --- a/noir-projects/aztec-nr/aztec/src/macros/notes/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/macros/notes/mod.nr @@ -7,6 +7,8 @@ use std::{ }; comptime global NOTE_HEADER_TYPE = type_of(NoteHeader::empty()); +// The following is a fixed ciphertext overhead as defined by `compute_private_log_payload` +comptime global NOTE_CIPHERTEXT_OVERHEAD: u32 = 353; /// A map from note type to (note_struct_definition, serialized_note_length, note_type_id, fields). /// `fields` is an array of tuples where each tuple contains the name of the field/struct member (e.g. `amount` @@ -386,7 +388,21 @@ comptime fn generate_multi_scalar_mul( /// hiding_point /// } /// } -/// } +/// +/// fn encrypt_log(self, context: &mut PrivateContext, recipient_keys: aztec::protocol_types::public_keys::PublicKeys, recipient: aztec::protocol_types::address::AztecAddress) -> [Field; 17] { +/// let ovsk_app: Field = context.request_ovsk_app(recipient_keys.ovpk_m.hash()); +/// +/// let encrypted_log_bytes: [u8; 513] = aztec::encrypted_logs::payload::compute_private_log_payload( +/// context.this_address(), +/// ovsk_app, +/// recipient_keys.ovpk_m, +/// recipient_keys.ivpk_m, +/// recipient, +/// self.log_plaintext +/// ); +/// +/// aztec::utils::bytes::bytes_to_fields(encrypted_log_bytes) +/// } /// /// impl aztec::protocol_types::traits::Empty for TokenNoteSetupPayload { /// fn empty() -> Self { @@ -420,6 +436,12 @@ comptime fn generate_setup_payload( let setup_log_plaintext = get_setup_log_plaintext_body(s, log_plaintext_length, indexed_nullable_fields); + // Then we compute values for `encrypt_log(...)` function + let encrypted_log_byte_length = NOTE_CIPHERTEXT_OVERHEAD + log_plaintext_length; + // Each field contains 31 bytes so the length in fields is computed as ceil(encrypted_log_byte_length / 31) + // --> we achieve rouding by adding 30 and then dividing without remainder + let encrypted_log_field_length = (encrypted_log_byte_length + 30) / 31; + ( quote { struct $setup_payload_name { @@ -441,6 +463,22 @@ comptime fn generate_setup_payload( hiding_point } } + + fn encrypt_log(self, context: &mut PrivateContext, recipient_keys: aztec::protocol_types::public_keys::PublicKeys, recipient: aztec::protocol_types::address::AztecAddress) -> [Field; $encrypted_log_field_length] { + let ovsk_app: Field = context.request_ovsk_app(recipient_keys.ovpk_m.hash()); + + let encrypted_log_bytes: [u8; $encrypted_log_byte_length] = aztec::encrypted_logs::payload::compute_private_log_payload( + context.this_address(), + ovsk_app, + recipient_keys.ovpk_m, + recipient_keys.ivpk_m, + recipient, + self.log_plaintext, + true + ); + + aztec::utils::bytes::bytes_to_fields(encrypted_log_bytes) + } } impl aztec::protocol_types::traits::Empty for $setup_payload_name { @@ -508,30 +546,70 @@ comptime fn get_setup_log_plaintext_body( /// Example: /// ``` /// struct TokenNoteFinalizationPayload { -/// log: [Field; 2], -/// note_hash: Field +/// context: &mut aztec::prelude::PublicContext, +/// hiding_point_slot: Field, +/// setup_log_slot: Field, +/// public_values: [Field; 2] /// } /// /// impl TokenNoteFinalizationPayload { -/// fn new(mut self, hiding_point: aztec::protocol_types::point::Point, amount: U128) -> TokenNoteFinalizationPayload { -/// self.log = [amount.lo as Field, amount.hi as Field]; +/// fn new(mut self, context: &mut aztec::prelude::PublicContext, slot: Field, amount: U128) -> TokenNoteFinalizationPayload { +/// self.context = context; +/// self.hiding_point_slot = slot; +/// self.setup_log_slot = slot + aztec::protocol_types::point::POINT_LENGTH as Field; +/// self.public_values = [amount.lo as Field, amount.hi as Field]; +/// self +/// } /// -/// let finalization_hiding_point = std::embedded_curve_ops::multi_scalar_mul( -/// [aztec::generators::Ga3, aztec::generators::Ga4], -/// [ -/// std::hash::from_field_unsafe(amount.lo), -/// std::hash::from_field_unsafe(amount.hi) -/// ] -/// ) + hiding_point; +/// fn emit(self) { +/// self.emit_note_hash(); +/// self.emit_log(); +/// } /// -/// self.note_hash = finalization_hiding_point.x; -/// self +/// fn emit_note_hash(self) { +/// let hiding_point: aztec::prelude::Point = self.context.storage_read(self.hiding_point_slot); +/// assert(!aztec::protocol_types::traits::is_empty(hiding_point), "transfer not prepared"); +/// +/// let finalization_hiding_point = std::embedded_curve_ops::multi_scalar_mul([aztec::generators::Ga3, aztec::generators::Ga4], [std::hash::from_field_unsafe(self.public_values[0]), std::hash::from_field_unsafe(self.public_values[1])]) + hiding_point; +/// +/// let note_hash = finalization_hiding_point.x; +/// +/// self.context.push_note_hash(note_hash); +/// +/// // We reset public storage to zero to achieve the effect of transient storage - kernels will squash +/// // the writes +/// // self.context.storage_write(self.hiding_point_slot, [0; aztec::protocol_types::point::POINT_LENGTH]); +/// } +/// +/// fn emit_log(self) { +/// let setup_log_fields: [Field; 16] = self.context.storage_read(self.setup_log_slot); +/// +/// let setup_log: [u8; 481] = aztec::utils::bytes::fields_to_bytes(setup_log_fields); +/// +/// let mut finalization_log = [0; 513]; +/// +/// for i in 0..setup_log.len() { +/// finalization_log[i] = setup_log[i]; +/// } +/// +/// for i in 0..self.public_values.len() { +/// let public_value_bytes: [u8; 32] = self.public_values[i].to_be_bytes(); +/// for j in 0..public_value_bytes.len() { +/// finalization_log[160 + i * 32 + j] = public_value_bytes[j]; +/// } +/// } +/// +/// self.context.emit_unencrypted_log(finalization_log); +/// +/// // We reset public storage to zero to achieve the effect of transient storage - kernels will squash +/// // the writes +/// // self.context.storage_write(self.setup_log_slot, [0; 16]); /// } /// } /// /// impl aztec::protocol_types::traits::Empty for TokenNoteFinalizationPayload { /// fn empty() -> Self { -/// Self { log: [0; 2], note_hash: 0 } +/// Self { context: &mut aztec::prelude::PublicContext::empty(), hiding_point_slot: 0, setup_log_slot: 0, public_values: [0, 0] } /// } /// } /// ``` @@ -560,28 +638,63 @@ comptime fn generate_finalization_payload( }; // We compute the log length and we concatenate the fields into a single quote. - let log_length = fields_list.len(); + let public_values_length = fields_list.len(); let fields = fields_list.join(quote {,}); // Now we compute quotes relevant to the multi-scalar multiplication. - let (generators_list, scalars_list, args_list, msm_aux_vars) = + let (generators_list, _, args_list, msm_aux_vars) = generate_multi_scalar_mul(indexed_nullable_fields); + // We generate scalars_list manually as we need it to refer self.public_values + let mut scalars_list: [Quoted] = &[]; + for i in 0..public_values_length { + scalars_list = + scalars_list.push_back(quote { std::hash::from_field_unsafe(self.public_values[$i]) }); + } + let generators = generators_list.join(quote {,}); let scalars = scalars_list.join(quote {,}); let args = args_list.join(quote {,}); + // Then we compute values for `encrypt_log(...)` function + let setup_log_plaintext_length = indexed_fixed_fields.len() * 32 + 64; + let setup_log_byte_length = NOTE_CIPHERTEXT_OVERHEAD + setup_log_plaintext_length; + // Each field contains 31 bytes so the length in fields is computed as ceil(setup_log_byte_length / 31) + // --> we achieve rouding by adding 30 and then dividing without remainder + let setup_log_field_length = (setup_log_byte_length + 30) / 31; + let finalization_log_byte_length = setup_log_byte_length + public_values_length * 32; + ( quote { struct $finalization_payload_name { - log: [Field; $log_length], - note_hash: Field, + context: &mut aztec::prelude::PublicContext, + hiding_point_slot: Field, + setup_log_slot: Field, + public_values: [Field; $public_values_length], } impl $finalization_payload_name { - fn new(mut self, hiding_point: aztec::protocol_types::point::Point, $args) -> $finalization_payload_name { + fn new(mut self, context: &mut aztec::prelude::PublicContext, slot: Field, $args) -> $finalization_payload_name { + self.context = context; + + self.hiding_point_slot = slot; + self.setup_log_slot = slot + aztec::protocol_types::point::POINT_LENGTH as Field; + $aux_vars_for_serialization - self.log = [$fields]; + self.public_values = [$fields]; + + self + } + + fn emit(self) { + self.emit_note_hash(); + self.emit_log(); + } + + fn emit_note_hash(self) { + // Read the hiding point from "transient" storage and check it's not empty to ensure the transfer was prepared + let hiding_point: aztec::prelude::Point = self.context.storage_read(self.hiding_point_slot); + assert(!aztec::protocol_types::traits::is_empty(hiding_point), "transfer not prepared"); $msm_aux_vars let finalization_hiding_point = std::embedded_curve_ops::multi_scalar_mul( @@ -589,14 +702,55 @@ comptime fn generate_finalization_payload( [$scalars] ) + hiding_point; - self.note_hash = finalization_hiding_point.x; - self + let note_hash = finalization_hiding_point.x; + + self.context.push_note_hash(note_hash); + + // We reset public storage to zero to achieve the effect of transient storage - kernels will squash + // the writes + // TODO(#9376): Uncomment the following line. + // self.context.storage_write(self.hiding_point_slot, [0; aztec::protocol_types::point::POINT_LENGTH]); + } + + fn emit_log(self) { + // We load the setup log from storage + let setup_log_fields: [Field; $setup_log_field_length] = self.context.storage_read(self.setup_log_slot); + + // We convert the log from fields to bytes + let setup_log: [u8; $setup_log_byte_length] = aztec::utils::bytes::fields_to_bytes(setup_log_fields); + + // We append the public value to the log and emit it as unencrypted log + let mut finalization_log = [0; $finalization_log_byte_length]; + + // Iterate over the partial log and copy it to the final log + for i in 1..setup_log.len() { + finalization_log[i] = setup_log[i]; + } + + // Now we populate the first byte with number of public values + finalization_log[0] = $public_values_length; + + // Iterate over the public values and append them to the log + for i in 0..$public_values_length { + let public_value_bytes: [u8; 32] = self.public_values[i].to_be_bytes(); + for j in 0..public_value_bytes.len() { + finalization_log[$setup_log_byte_length + i * 32 + j] = public_value_bytes[j]; + } + } + + // We emit the finalization log via the unencrypted logs stream + self.context.emit_unencrypted_log(finalization_log); + + // We reset public storage to zero to achieve the effect of transient storage - kernels will squash + // the writes + // TODO(#9376): Uncomment the following line. + // self.context.storage_write(self.setup_log_slot, [0; $setup_log_field_length]); } } impl aztec::protocol_types::traits::Empty for $finalization_payload_name { fn empty() -> Self { - Self { log: [0; $log_length], note_hash: 0 } + Self { context: &mut aztec::prelude::PublicContext::empty(), public_values: [0; $public_values_length], hiding_point_slot: 0, setup_log_slot: 0 } } } }, diff --git a/noir-projects/noir-contracts/contracts/nft_contract/src/main.nr b/noir-projects/noir-contracts/contracts/nft_contract/src/main.nr index f5d83db36a4..27b6531598b 100644 --- a/noir-projects/noir-contracts/contracts/nft_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/nft_contract/src/main.nr @@ -13,10 +13,7 @@ contract NFT { compute_authwit_nullifier, }; use dep::aztec::{ - encrypted_logs::encrypted_note_emission::{ - encode_and_encrypt_note, encrypt_and_emit_partial_log, - }, - hash::pedersen_hash, + encrypted_logs::encrypted_note_emission::encode_and_encrypt_note, keys::getters::get_public_keys, macros::{ events::event, @@ -29,7 +26,7 @@ contract NFT { AztecAddress, Map, NoteGetterOptions, NoteViewerOptions, PrivateContext, PrivateSet, PublicContext, PublicMutable, SharedImmutable, }, - protocol_types::{point::Point, traits::{is_empty, Serialize}}, + protocol_types::{point::Point, traits::Serialize}, utils::comparison::Comparator, }; use dep::compressed_string::FieldCompressedString; @@ -192,8 +189,8 @@ contract NFT { let note_setup_payload = NFTNote::setup_payload().new(to_npk_m_hash, note_randomness, to_note_slot); - // We encrypt and emit the partial note log - encrypt_and_emit_partial_log(context, note_setup_payload.log_plaintext, to_keys, to); + // We encrypt the note log + let setup_log = note_setup_payload.encrypt_log(context, to_keys, to); // Using the x-coordinate as a hiding point slot is safe against someone else interfering with it because // we have a guarantee that the public functions of the transaction are executed right after the private ones @@ -212,19 +209,30 @@ contract NFT { // is zero because the slot is the x-coordinate of the hiding point and hence we could only overwrite // the value in the slot with the same value. This makes usage of the `unsafe` method safe. NFT::at(context.this_address()) - ._store_point_in_transient_storage_unsafe( + ._store_payload_in_transient_storage_unsafe( hiding_point_slot, note_setup_payload.hiding_point, + setup_log, ) .enqueue(context); hiding_point_slot } + // TODO(#9375): Having to define the note log length here is very unfortunate as it's basically impossible for + // users to derive manually. This will however go away once we have a real transient storage since we will not need + // the public call and instead we would do something like `context.transient_storage_write(slot, payload)` and that + // will allow us to use generics and hence user will not need to define it explicitly. We cannot use generics here + // as it is an entrypoint function. #[public] #[internal] - fn _store_point_in_transient_storage_unsafe(slot: Field, point: Point) { + fn _store_payload_in_transient_storage_unsafe( + slot: Field, + point: Point, + setup_log: [Field; 16], + ) { context.storage_write(slot, point); + context.storage_write(slot + aztec::protocol_types::point::POINT_LENGTH as Field, setup_log); } /// Finalizes a transfer of NFT with `token_id` from public balance of `from` to a private balance of `to`. @@ -250,32 +258,22 @@ contract NFT { fn _finalize_transfer_to_private( from: AztecAddress, token_id: Field, - hiding_point_slot: Field, + note_transient_storage_slot: Field, context: &mut PublicContext, storage: Storage<&mut PublicContext>, ) { let public_owners_storage = storage.public_owners.at(token_id); assert(public_owners_storage.read().eq(from), "invalid NFT owner"); - // Read the hiding point from "transient" storage and check it's not empty to ensure the transfer was prepared - let hiding_point: Point = context.storage_read(hiding_point_slot); - assert(!is_empty(hiding_point), "transfer not prepared"); - // Set the public NFT owner to zero public_owners_storage.write(AztecAddress::zero()); // Finalize the partial note with the `token_id` - let finalization_payload = NFTNote::finalization_payload().new(hiding_point, token_id); - - // We insert the finalization note hash - context.push_note_hash(finalization_payload.note_hash); - - // We emit the `token_id` as unencrypted event such that the `NoteProcessor` can use it to reconstruct the note - context.emit_unencrypted_log(finalization_payload.log); + let finalization_payload = + NFTNote::finalization_payload().new(context, note_transient_storage_slot, token_id); - // At last we reset public storage to zero to achieve the effect of transient storage - kernels will squash - // the writes - context.storage_write(hiding_point_slot, Point::empty()); + // At last we emit the note hash and the final log + finalization_payload.emit(); } /** diff --git a/noir-projects/noir-contracts/contracts/token_contract/src/main.nr b/noir-projects/noir-contracts/contracts/token_contract/src/main.nr index b38e513d7da..12aa4021a13 100644 --- a/noir-projects/noir-contracts/contracts/token_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/token_contract/src/main.nr @@ -24,7 +24,6 @@ contract Token { encrypted_event_emission::encode_and_encrypt_event_unconstrained, encrypted_note_emission::{ encode_and_encrypt_note, encode_and_encrypt_note_unconstrained, - encrypt_and_emit_partial_log, }, }, hash::compute_secret_hash, @@ -323,8 +322,15 @@ contract Token { } let from_keys = get_public_keys(from); + // TODO: constrain encryption below - we are using unconstrained here only becuase of the following Noir issue + // https://github.com/noir-lang/noir/issues/5771 storage.balances.at(from).sub(from_keys.npk_m, U128::from_integer(amount)).emit( - encode_and_encrypt_note(&mut context, from_keys.ovpk_m, from_keys.ivpk_m, from), + encode_and_encrypt_note_unconstrained( + &mut context, + from_keys.ovpk_m, + from_keys.ivpk_m, + from, + ), ); Token::at(context.this_address())._increase_public_balance(to, amount).enqueue(&mut context); } @@ -562,54 +568,74 @@ contract Token { storage.balances.at(user).set.storage_slot, ); - // 5. We encrypt and emit the partial note log - encrypt_and_emit_partial_log( - &mut context, - fee_payer_setup_payload.log_plaintext, - fee_payer_keys, - fee_payer, - ); - encrypt_and_emit_partial_log( - &mut context, - user_setup_payload.log_plaintext, - user_keys, - user, - ); - - // 6. We convert the hiding points to standard `Point` type as we cannot pass `TokenNoteHidingPoint` type - // as an argument to a function due to macro limitations (the `TokenNoteHidingPoint` type is macro generated - // and hence is not resolved soon enough by the compiler). - let fee_payer_point = fee_payer_setup_payload.hiding_point; - let user_point = user_setup_payload.hiding_point; + // 5. We get transient storage slots + // Using the x-coordinate as a hiding point slot is safe against someone else interfering with it because + // we have a guarantee that the public functions of the transaction are executed right after the private ones + // and for this reason the protocol guarantees that nobody can front-run us in consuming the hiding point. + // This guarantee would break if `finalize_transfer_to_private` was not called in the same transaction. This + // however is not the flow we are currently concerned with. To support the multi-transaction flow we could + // introduce a `from` function argument, hash the x-coordinate with it and then repeat the hashing in + // `finalize_transfer_to_private`. + // + // We can also be sure that the `hiding_point_slot` will not overwrite any other value in the storage because + // in our state variables we derive slots using a different hash function from multi scalar multiplication + // (MSM). + let fee_payer_point_slot = fee_payer_setup_payload.hiding_point.x; + let user_point_slot = user_setup_payload.hiding_point.x; + + // 6. We compute setup logs + let fee_payer_setup_log = + fee_payer_setup_payload.encrypt_log(&mut context, fee_payer_keys, fee_payer); + let user_setup_log = user_setup_payload.encrypt_log(&mut context, user_keys, user); + + // 7. We store the hiding points an logs in transients storage + Token::at(context.this_address()) + ._store_payload_in_transient_storage_unsafe( + fee_payer_point_slot, + fee_payer_setup_payload.hiding_point, + fee_payer_setup_log, + ) + .enqueue(&mut context); + Token::at(context.this_address()) + ._store_payload_in_transient_storage_unsafe( + user_point_slot, + user_setup_payload.hiding_point, + user_setup_log, + ) + .enqueue(&mut context); - // 7. Set the public teardown function to `complete_refund(...)`. Public teardown is the only time when a public + // 8. Set the public teardown function to `complete_refund(...)`. Public teardown is the only time when a public // function has access to the final transaction fee, which is needed to compute the actual refund amount. context.set_public_teardown_function( context.this_address(), - comptime { - FunctionSelector::from_signature( - "complete_refund((Field,Field,bool),(Field,Field,bool),Field)", - ) - }, - [ - fee_payer_point.x, - fee_payer_point.y, - fee_payer_point.is_infinite as Field, - user_point.x, - user_point.y, - user_point.is_infinite as Field, - funded_amount, - ], + comptime { FunctionSelector::from_signature("complete_refund(Field,Field,Field)") }, + [fee_payer_point_slot, user_point_slot, funded_amount], ); } // docs:end:setup_refund + // TODO(#9375): Having to define the note log length here is very unfortunate as it's basically impossible for + // users to derive manually. This will however go away once we have a real transient storage since we will not need + // the public call and instead we would do something like `context.transient_storage_write(slot, payload)` and that + // will allow us to use generics and hence user will not need to define it explicitly. We cannot use generics here + // as it is an entrypoint function. + #[public] + #[internal] + fn _store_payload_in_transient_storage_unsafe( + slot: Field, + point: Point, + setup_log: [Field; 16], + ) { + context.storage_write(slot, point); + context.storage_write(slot + aztec::protocol_types::point::POINT_LENGTH as Field, setup_log); + } + // TODO(#7728): even though the funded_amount should be a U128, we can't have that type in a contract interface due // to serialization issues. // docs:start:complete_refund #[public] #[internal] - fn complete_refund(fee_payer_point: Point, user_point: Point, funded_amount: Field) { + fn complete_refund(fee_payer_slot: Field, user_slot: Field, funded_amount: Field) { // TODO(#7728): Remove the next line let funded_amount = U128::from_integer(funded_amount); let tx_fee = U128::from_integer(context.transaction_fee()); @@ -624,18 +650,13 @@ contract Token { // 3. We construct the note finalization payloads with the correct amounts and hiding points to get the note // hashes and unencrypted logs. let fee_payer_finalization_payload = - TokenNote::finalization_payload().new(fee_payer_point, tx_fee); + TokenNote::finalization_payload().new(&mut context, fee_payer_slot, tx_fee); let user_finalization_payload = - TokenNote::finalization_payload().new(user_point, refund_amount); - - // 4. We emit the `tx_fee` and `refund_amount` as unencrypted event such that the `NoteProcessor` can use it - // to reconstruct the note. - context.emit_unencrypted_log(fee_payer_finalization_payload.log); - context.emit_unencrypted_log(user_finalization_payload.log); + TokenNote::finalization_payload().new(&mut context, user_slot, refund_amount); - // 5. At last we emit the note hashes. - context.push_note_hash(fee_payer_finalization_payload.note_hash); - context.push_note_hash(user_finalization_payload.note_hash); + // 4. At last we emit the note hashes and the final note logs. + fee_payer_finalization_payload.emit(); + user_finalization_payload.emit(); // --> Once the tx is settled user and fee recipient can add the notes to their pixies. } // docs:end:complete_refund diff --git a/noir-projects/noir-protocol-circuits/crates/types/src/meta/mod.nr b/noir-projects/noir-protocol-circuits/crates/types/src/meta/mod.nr index 23de9e52177..156d1c2c414 100644 --- a/noir-projects/noir-protocol-circuits/crates/types/src/meta/mod.nr +++ b/noir-projects/noir-protocol-circuits/crates/types/src/meta/mod.nr @@ -97,8 +97,11 @@ pub comptime fn flatten_to_fields(name: Quoted, typ: Type, omit: [Quoted]) -> ([ let mut aux_vars = &[]; if omit.all(|to_omit| to_omit != name) { - if typ.is_field() | typ.as_integer().is_some() | typ.is_bool() { - // For field, integer and bool we just cast to Field and add the value to fields + if typ.is_field() { + // For field we just add the value to fields + fields = fields.push_back(name); + } else if typ.is_field() | typ.as_integer().is_some() | typ.is_bool() { + // For integer and bool we just cast to Field and add the value to fields fields = fields.push_back(quote { $name as Field }); } else if typ.as_struct().is_some() { // For struct we pref diff --git a/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts b/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts index 4deecd47968..e9231ce876e 100644 --- a/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts +++ b/yarn-project/circuit-types/src/logs/l1_payload/encrypted_log_payload.ts @@ -27,9 +27,21 @@ const OUTGOING_BODY_SIZE = 144; */ export class EncryptedLogPayload { constructor( + /** + * Note discovery tag used by the recipient of the log. + */ public readonly incomingTag: Fr, + /** + * Note discovery tag used by the sender of the log. + */ public readonly outgoingTag: Fr, + /** + * Address of a contract that emitted the log. + */ public readonly contractAddress: AztecAddress, + /** + * Decrypted incoming body. + */ public readonly incomingBodyPlaintext: Buffer, ) {} diff --git a/yarn-project/circuit-types/src/logs/l1_payload/l1_note_payload.ts b/yarn-project/circuit-types/src/logs/l1_payload/l1_note_payload.ts index c3f20cc0b10..122f81a07ec 100644 --- a/yarn-project/circuit-types/src/logs/l1_payload/l1_note_payload.ts +++ b/yarn-project/circuit-types/src/logs/l1_payload/l1_note_payload.ts @@ -1,11 +1,11 @@ -import { AztecAddress } from '@aztec/circuits.js'; +import { AztecAddress, Vector } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; +import { randomInt } from '@aztec/foundation/crypto'; import { type Fq, Fr } from '@aztec/foundation/fields'; import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; -import { type EncryptedL2NoteLog } from '../encrypted_l2_note_log.js'; +import { EncryptedL2NoteLog } from '../encrypted_l2_note_log.js'; import { EncryptedLogPayload } from './encrypted_log_payload.js'; -import { Note } from './payload.js'; /** * A class which wraps note data which is pushed on L1. @@ -14,10 +14,6 @@ import { Note } from './payload.js'; */ export class L1NotePayload { constructor( - /** - * A note as emitted from Noir contract. Can be used along with private key to compute nullifier. - */ - public note: Note, /** * Address of the contract this tx is interacting with. */ @@ -30,11 +26,24 @@ export class L1NotePayload { * Type identifier for the underlying note, required to determine how to compute its hash and nullifier. */ public noteTypeId: NoteSelector, + /** + * Note values delivered encrypted. + * @dev Note that to recreate the correct note we need to merge privateNoteValues and publicNoteValues. To do that + * we need access to the contract ABI (that is done in the NoteProcessor). + */ + public privateNoteValues: Fr[], + /** + * Note values delivered in plaintext. + * @dev Note that to recreate the correct note we need to merge privateNoteValues and publicNoteValues. To do that + * we need access to the contract ABI (that is done in the NoteProcessor). + */ + public publicNoteValues: Fr[], ) {} - static fromIncomingBodyPlaintextAndContractAddress( + static fromIncomingBodyPlaintextContractAndPublicValues( plaintext: Buffer, contractAddress: AztecAddress, + publicNoteValues: Fr[], ): L1NotePayload | undefined { try { const reader = BufferReader.asReader(plaintext); @@ -43,35 +52,39 @@ export class L1NotePayload { const storageSlot = fields[0]; const noteTypeId = NoteSelector.fromField(fields[1]); - const note = new Note(fields.slice(2)); + const privateNoteValues = fields.slice(2); - return new L1NotePayload(note, contractAddress, storageSlot, noteTypeId); + return new L1NotePayload(contractAddress, storageSlot, noteTypeId, privateNoteValues, publicNoteValues); } catch (e) { return undefined; } } - static decryptAsIncoming(log: EncryptedL2NoteLog, sk: Fq): L1NotePayload | undefined { - const decryptedLog = EncryptedLogPayload.decryptAsIncoming(log.data, sk); + static decryptAsIncoming(log: Buffer, sk: Fq): L1NotePayload | undefined { + const { publicValues, encryptedLog } = parseLog(log); + const decryptedLog = EncryptedLogPayload.decryptAsIncoming(encryptedLog.data, sk); if (!decryptedLog) { return undefined; } - return this.fromIncomingBodyPlaintextAndContractAddress( + return this.fromIncomingBodyPlaintextContractAndPublicValues( decryptedLog.incomingBodyPlaintext, decryptedLog.contractAddress, + publicValues, ); } - static decryptAsOutgoing(log: EncryptedL2NoteLog, sk: Fq): L1NotePayload | undefined { - const decryptedLog = EncryptedLogPayload.decryptAsOutgoing(log.data, sk); + static decryptAsOutgoing(log: Buffer, sk: Fq): L1NotePayload | undefined { + const { publicValues, encryptedLog } = parseLog(log); + const decryptedLog = EncryptedLogPayload.decryptAsOutgoing(encryptedLog.data, sk); if (!decryptedLog) { return undefined; } - return this.fromIncomingBodyPlaintextAndContractAddress( + return this.fromIncomingBodyPlaintextContractAndPublicValues( decryptedLog.incomingBodyPlaintext, decryptedLog.contractAddress, + publicValues, ); } @@ -80,7 +93,7 @@ export class L1NotePayload { * @returns Buffer representation of the L1NotePayload object. */ toIncomingBodyPlaintext() { - const fields = [this.storageSlot, this.noteTypeId.toField(), ...this.note.items]; + const fields = [this.storageSlot, this.noteTypeId.toField(), ...this.privateNoteValues]; return serializeToBuffer(fields); } @@ -90,15 +103,93 @@ export class L1NotePayload { * @returns A random L1NotePayload object. */ static random(contract = AztecAddress.random()) { - return new L1NotePayload(Note.random(), contract, Fr.random(), NoteSelector.random()); + const numPrivateNoteValues = randomInt(10) + 1; + const privateNoteValues = Array.from({ length: numPrivateNoteValues }, () => Fr.random()); + + const numPublicNoteValues = randomInt(10) + 1; + const publicNoteValues = Array.from({ length: numPublicNoteValues }, () => Fr.random()); + + return new L1NotePayload(contract, Fr.random(), NoteSelector.random(), privateNoteValues, publicNoteValues); } public equals(other: L1NotePayload) { return ( - this.note.equals(other.note) && this.contractAddress.equals(other.contractAddress) && this.storageSlot.equals(other.storageSlot) && - this.noteTypeId.equals(other.noteTypeId) + this.noteTypeId.equals(other.noteTypeId) && + this.privateNoteValues.every((value, index) => value.equals(other.privateNoteValues[index])) && + this.publicNoteValues.every((value, index) => value.equals(other.publicNoteValues[index])) ); } + + toBuffer() { + return serializeToBuffer( + this.contractAddress, + this.storageSlot, + this.noteTypeId, + new Vector(this.privateNoteValues), + new Vector(this.publicNoteValues), + ); + } + + static fromBuffer(buffer: Buffer | BufferReader) { + const reader = BufferReader.asReader(buffer); + return new L1NotePayload( + reader.readObject(AztecAddress), + reader.readObject(Fr), + reader.readObject(NoteSelector), + reader.readVector(Fr), + reader.readVector(Fr), + ); + } +} + +/** + * Parse the given log into an array of public values and an encrypted log. + * + * @param log - Log to be parsed. + * @returns An object containing the public values and the encrypted log. + */ +function parseLog(log: Buffer) { + // First we remove padding bytes + const processedLog = removePaddingBytes(log); + + const reader = new BufferReader(processedLog); + + // Then we extract public values from the log + const numPublicValues = reader.readUInt8(); + + const publicValuesLength = numPublicValues * Fr.SIZE_IN_BYTES; + const encryptedLogLength = reader.remainingBytes() - publicValuesLength; + + // Now we get the buffer corresponding to the encrypted log + const encryptedLog = new EncryptedL2NoteLog(reader.readBytes(encryptedLogLength)); + + // At last we load the public values + const publicValues = reader.readArray(numPublicValues, Fr); + + return { publicValues, encryptedLog }; +} + +/** + * When a log is emitted via the unencrypted log channel each field contains only 1 byte. OTOH when a log is emitted + * via the encrypted log channel there are no empty bytes. This function removes the padding bytes. + * @param unprocessedLog - Log to be processed. + * @returns Log with padding bytes removed. + */ +function removePaddingBytes(unprocessedLog: Buffer) { + // Determine whether first 31 bytes of each 32 bytes block of bytes are 0 + const is1FieldPerByte = unprocessedLog.every((byte, index) => index % 32 === 31 || byte === 0); + + if (is1FieldPerByte) { + // We take every 32nd byte from the log and return the result + const processedLog = Buffer.alloc(unprocessedLog.length / 32); + for (let i = 0; i < processedLog.length; i++) { + processedLog[i] = unprocessedLog[31 + i * 32]; + } + + return processedLog; + } + + return unprocessedLog; } diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index d93f42e3ea0..8fd6fc85953 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -299,8 +299,9 @@ describe('e2e_block_building', () => { // compare logs expect(rct.status).toEqual('success'); const noteValues = tx.noteEncryptedLogs.unrollLogs().map(l => { - const notePayload = L1NotePayload.decryptAsIncoming(l, thisWallet.getEncryptionSecret()); - return notePayload?.note.items[0]; + const notePayload = L1NotePayload.decryptAsIncoming(l.data, thisWallet.getEncryptionSecret()); + // In this test we care only about the privately delivered values + return notePayload?.privateNoteValues[0]; }); expect(noteValues[0]).toEqual(new Fr(10)); expect(noteValues[1]).toEqual(new Fr(11)); diff --git a/yarn-project/end-to-end/src/e2e_nft.test.ts b/yarn-project/end-to-end/src/e2e_nft.test.ts index 1f7c5eb3008..e79ffaed49f 100644 --- a/yarn-project/end-to-end/src/e2e_nft.test.ts +++ b/yarn-project/end-to-end/src/e2e_nft.test.ts @@ -62,21 +62,40 @@ describe('NFT', () => { // the sender would be the AMM contract. const recipient = user2Wallet.getAddress(); - const { debugInfo } = await nftContractAsUser1.methods - .transfer_to_private(recipient, TOKEN_ID) - .send() - .wait({ debug: true }); + await nftContractAsUser1.methods.transfer_to_private(recipient, TOKEN_ID).send().wait(); const publicOwnerAfter = await nftContractAsUser1.methods.owner_of(TOKEN_ID).simulate(); expect(publicOwnerAfter).toEqual(AztecAddress.ZERO); - // We should get 4 data writes setting values to 0 - 3 for note hiding point and 1 for public owner (we transfer - // to private so public owner is set to 0). Ideally we would have here only 1 data write as the 4 values change - // from zero to non-zero to zero in the tx and hence no write could be committed. This makes public writes - // squashing too expensive for transient storage. This however probably does not matter as I assume we will want - // to implement a real transient storage anyway. (Informed Leila about the potential optimization.) - const publicDataWritesValues = debugInfo!.publicDataWrites!.map(write => write.newValue.toBigInt()); - expect(publicDataWritesValues).toEqual([0n, 0n, 0n, 0n]); + // We should get 20 data writes setting values to 0 - 3 for note hiding point, 16 for partial log and 1 for public + // owner (we transfer to private so public owner is set to 0). Ideally we would have here only 1 data write as the + // 4 values change from zero to non-zero to zero in the tx and hence no write could be committed. This makes public + // writes squashing too expensive for transient storage. This however probably does not matter as I assume we will + // want to implement a real transient storage anyway. (Informed Leila about the potential optimization.) + // TODO(#9376): Re-enable the following check. + // const publicDataWritesValues = debugInfo!.publicDataWrites!.map(write => write.newValue.toBigInt()); + // expect(publicDataWritesValues).toEqual([ + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // 0n, + // ]); }); it('transfers in private', async () => { diff --git a/yarn-project/foundation/src/collection/array.ts b/yarn-project/foundation/src/collection/array.ts index b703e66a119..ea97385aaba 100644 --- a/yarn-project/foundation/src/collection/array.ts +++ b/yarn-project/foundation/src/collection/array.ts @@ -5,11 +5,17 @@ import { type Tuple } from '../serialize/types.js'; * @param arr - Array with elements to pad. * @param elem - Element to use for padding. * @param length - Target length. + * @param errorMsg - Error message to throw if target length exceeds the input array length. * @returns A new padded array. */ -export function padArrayEnd(arr: T[], elem: T, length: N): Tuple { +export function padArrayEnd( + arr: T[], + elem: T, + length: N, + errorMsg = 'Array size exceeds target length', +): Tuple { if (arr.length > length) { - throw new Error(`Array size exceeds target length`); + throw new Error(errorMsg); } // Since typescript cannot always deduce that something is a tuple, we cast return [...arr, ...Array(length - arr.length).fill(elem)] as Tuple; diff --git a/yarn-project/foundation/src/serialize/buffer_reader.ts b/yarn-project/foundation/src/serialize/buffer_reader.ts index d183e8fd39c..7abe3f59336 100644 --- a/yarn-project/foundation/src/serialize/buffer_reader.ts +++ b/yarn-project/foundation/src/serialize/buffer_reader.ts @@ -340,6 +340,14 @@ export class BufferReader { return this.buffer.length; } + /** + * Gets bytes remaining to be read from the buffer. + * @returns Bytes remaining to be read from the buffer. + */ + public remainingBytes(): number { + return this.buffer.length - this.index; + } + #rangeCheck(numBytes: number) { if (this.index + numBytes > this.buffer.length) { throw new Error( diff --git a/yarn-project/pxe/src/database/deferred_note_dao.test.ts b/yarn-project/pxe/src/database/deferred_note_dao.test.ts index 90da662aa5c..79250c687b8 100644 --- a/yarn-project/pxe/src/database/deferred_note_dao.test.ts +++ b/yarn-project/pxe/src/database/deferred_note_dao.test.ts @@ -1,32 +1,18 @@ -import { Note, UnencryptedTxL2Logs, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; +import { L1NotePayload, UnencryptedTxL2Logs, randomTxHash } from '@aztec/circuit-types'; +import { Fr, Point } from '@aztec/circuits.js'; import { randomInt } from '@aztec/foundation/crypto'; import { DeferredNoteDao } from './deferred_note_dao.js'; export const randomDeferredNoteDao = ({ publicKey = Point.random(), - note = Note.random(), - contractAddress = AztecAddress.random(), + payload = L1NotePayload.random(), txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), noteHashes = [Fr.random(), Fr.random()], dataStartIndexForTx = randomInt(100), unencryptedLogs = UnencryptedTxL2Logs.random(1, 1), }: Partial = {}) => { - return new DeferredNoteDao( - publicKey, - note, - contractAddress, - storageSlot, - noteTypeId, - txHash, - noteHashes, - dataStartIndexForTx, - unencryptedLogs, - ); + return new DeferredNoteDao(publicKey, payload, txHash, noteHashes, dataStartIndexForTx, unencryptedLogs); }; describe('Deferred Note DAO', () => { diff --git a/yarn-project/pxe/src/database/deferred_note_dao.ts b/yarn-project/pxe/src/database/deferred_note_dao.ts index 1351b5b3b2d..d5cafe85f89 100644 --- a/yarn-project/pxe/src/database/deferred_note_dao.ts +++ b/yarn-project/pxe/src/database/deferred_note_dao.ts @@ -1,6 +1,5 @@ -import { Note, TxHash, UnencryptedTxL2Logs } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point, type PublicKey, Vector } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; +import { L1NotePayload, TxHash, UnencryptedTxL2Logs } from '@aztec/circuit-types'; +import { Fr, Point, type PublicKey, Vector } from '@aztec/circuits.js'; import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; /** @@ -12,14 +11,8 @@ export class DeferredNoteDao { constructor( /** IvpkM or OvpkM (depending on if incoming or outgoing) the note was encrypted with. */ public publicKey: PublicKey, - /** The note as emitted from the Noir contract. */ - public note: Note, - /** The contract address this note is created in. */ - public contractAddress: AztecAddress, - /** The specific storage location of the note on the contract. */ - public storageSlot: Fr, - /** The type ID of the note on the contract. */ - public noteTypeId: NoteSelector, + /** The note payload delivered via L1. */ + public payload: L1NotePayload, /** The hash of the tx the note was created in. Equal to the first nullifier */ public txHash: TxHash, /** New note hashes in this transaction, one of which belongs to this note */ @@ -33,10 +26,7 @@ export class DeferredNoteDao { toBuffer(): Buffer { return serializeToBuffer( this.publicKey, - this.note, - this.contractAddress, - this.storageSlot, - this.noteTypeId, + this.payload, this.txHash, new Vector(this.noteHashes), this.dataStartIndexForTx, @@ -47,10 +37,7 @@ export class DeferredNoteDao { const reader = BufferReader.asReader(buffer); return new DeferredNoteDao( reader.readObject(Point), - reader.readObject(Note), - reader.readObject(AztecAddress), - reader.readObject(Fr), - reader.readObject(NoteSelector), + reader.readObject(L1NotePayload), reader.readObject(TxHash), reader.readVector(Fr), reader.readNumber(), diff --git a/yarn-project/pxe/src/database/incoming_note_dao.ts b/yarn-project/pxe/src/database/incoming_note_dao.ts index 88961240638..0bd0c6a992c 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.ts @@ -41,6 +41,7 @@ export class IncomingNoteDao implements NoteData { ) {} static fromPayloadAndNoteInfo( + note: Note, payload: L1NotePayload, noteInfo: NoteInfo, dataStartIndexForTx: number, @@ -48,7 +49,7 @@ export class IncomingNoteDao implements NoteData { ) { const noteHashIndexInTheWholeTree = BigInt(dataStartIndexForTx + noteInfo.noteHashIndex); return new IncomingNoteDao( - payload.note, + note, payload.contractAddress, payload.storageSlot, payload.noteTypeId, diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 280bc679fd7..412aacf4fa5 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -215,7 +215,7 @@ export class KVPxeDatabase implements PxeDatabase { const newLength = await this.#deferredNotes.push(...deferredNotes.map(note => note.toBuffer())); for (const [index, note] of deferredNotes.entries()) { const noteId = newLength - deferredNotes.length + index; - await this.#deferredNotesByContract.set(note.contractAddress.toString(), noteId); + await this.#deferredNotesByContract.set(note.payload.contractAddress.toString(), noteId); } } diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.ts b/yarn-project/pxe/src/database/outgoing_note_dao.ts index a2efe0375d4..048bef1ad3b 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.ts @@ -35,6 +35,7 @@ export class OutgoingNoteDao { ) {} static fromPayloadAndNoteInfo( + note: Note, payload: L1NotePayload, noteInfo: NoteInfo, dataStartIndexForTx: number, @@ -42,7 +43,7 @@ export class OutgoingNoteDao { ) { const noteHashIndexInTheWholeTree = BigInt(dataStartIndexForTx + noteInfo.noteHashIndex); return new OutgoingNoteDao( - payload.note, + note, payload.contractAddress, payload.storageSlot, payload.noteTypeId, diff --git a/yarn-project/pxe/src/note_processor/note_processor.test.ts b/yarn-project/pxe/src/note_processor/note_processor.test.ts index 8b231c67bf5..0d145b91b99 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.test.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.test.ts @@ -1,4 +1,11 @@ -import { type AztecNode, EncryptedL2NoteLog, EncryptedLogPayload, L1NotePayload, L2Block } from '@aztec/circuit-types'; +import { + type AztecNode, + EncryptedL2NoteLog, + EncryptedLogPayload, + L1NotePayload, + L2Block, + Note, +} from '@aztec/circuit-types'; import { AztecAddress, CompleteAddress, @@ -59,7 +66,9 @@ class MockNoteRequest { encrypt(): EncryptedL2NoteLog { const ephSk = GrumpkinScalar.random(); const recipient = AztecAddress.random(); - const log = this.logPayload.encrypt(ephSk, recipient, this.ivpk, this.ovKeys); + const logWithoutNumPublicValues = this.logPayload.encrypt(ephSk, recipient, this.ivpk, this.ovKeys); + // We prefix the log with an empty byte indicating there are 0 public values. + const log = Buffer.concat([Buffer.alloc(1), logWithoutNumPublicValues]); return new EncryptedL2NoteLog(log); } @@ -69,11 +78,18 @@ class MockNoteRequest { ); } - get notePayload(): L1NotePayload | undefined { - return L1NotePayload.fromIncomingBodyPlaintextAndContractAddress( + get snippetOfNoteDao() { + const payload = L1NotePayload.fromIncomingBodyPlaintextContractAndPublicValues( this.logPayload.incomingBodyPlaintext, this.logPayload.contractAddress, - ); + [], + )!; + return { + note: new Note(payload.privateNoteValues), + contractAddress: payload.contractAddress, + storageSlot: payload.storageSlot, + noteTypeId: payload.noteTypeId, + }; } } @@ -111,8 +127,8 @@ describe('Note Processor', () => { // Then we update the relevant note hashes to match the note requests for (const request of noteRequestsForBlock) { - const notePayload = request.notePayload; - const noteHash = pedersenHash(notePayload!.note.items); + const note = request.snippetOfNoteDao.note; + const noteHash = pedersenHash(note.items); block.body.txEffects[request.txIndex].noteHashes[request.noteHashIndex] = noteHash; // Now we populate the log - to simplify we say that there is only 1 function invocation in each tx @@ -196,7 +212,7 @@ describe('Note Processor', () => { expect(addNotesSpy).toHaveBeenCalledWith( [ expect.objectContaining({ - ...request.notePayload, + ...request.snippetOfNoteDao, index: request.indexWithinNoteHashTree, }), ], @@ -213,7 +229,7 @@ describe('Note Processor', () => { expect(addNotesSpy).toHaveBeenCalledTimes(1); // For outgoing notes, the resulting DAO does not contain index. - expect(addNotesSpy).toHaveBeenCalledWith([], [expect.objectContaining(request.notePayload)], account.address); + expect(addNotesSpy).toHaveBeenCalledWith([], [expect.objectContaining(request.snippetOfNoteDao)], account.address); }, 25_000); it('should store multiple notes that belong to us', async () => { @@ -240,23 +256,23 @@ describe('Note Processor', () => { // Incoming should contain notes from requests 0, 2, 4 because in those requests we set owner ivpk. [ expect.objectContaining({ - ...requests[0].notePayload, + ...requests[0].snippetOfNoteDao, index: requests[0].indexWithinNoteHashTree, }), expect.objectContaining({ - ...requests[2].notePayload, + ...requests[2].snippetOfNoteDao, index: requests[2].indexWithinNoteHashTree, }), expect.objectContaining({ - ...requests[4].notePayload, + ...requests[4].snippetOfNoteDao, index: requests[4].indexWithinNoteHashTree, }), ], // Outgoing should contain notes from requests 0, 1, 4 because in those requests we set owner ovKeys. [ - expect.objectContaining(requests[0].notePayload), - expect.objectContaining(requests[1].notePayload), - expect.objectContaining(requests[4].notePayload), + expect.objectContaining(requests[0].snippetOfNoteDao), + expect.objectContaining(requests[1].snippetOfNoteDao), + expect.objectContaining(requests[4].snippetOfNoteDao), ], account.address, ); @@ -292,11 +308,11 @@ describe('Note Processor', () => { { const addedIncoming: IncomingNoteDao[] = addNotesSpy.mock.calls[0][0]; expect(addedIncoming.map(dao => dao)).toEqual([ - expect.objectContaining({ ...requests[0].notePayload, index: requests[0].indexWithinNoteHashTree }), - expect.objectContaining({ ...requests[1].notePayload, index: requests[1].indexWithinNoteHashTree }), - expect.objectContaining({ ...requests[2].notePayload, index: requests[2].indexWithinNoteHashTree }), - expect.objectContaining({ ...requests[3].notePayload, index: requests[3].indexWithinNoteHashTree }), - expect.objectContaining({ ...requests[4].notePayload, index: requests[4].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[0].snippetOfNoteDao, index: requests[0].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[1].snippetOfNoteDao, index: requests[1].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[2].snippetOfNoteDao, index: requests[2].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[3].snippetOfNoteDao, index: requests[3].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[4].snippetOfNoteDao, index: requests[4].indexWithinNoteHashTree }), ]); // Check that every note has a different nonce. @@ -309,11 +325,11 @@ describe('Note Processor', () => { { const addedOutgoing: OutgoingNoteDao[] = addNotesSpy.mock.calls[0][1]; expect(addedOutgoing.map(dao => dao)).toEqual([ - expect.objectContaining(requests[0].notePayload), - expect.objectContaining(requests[1].notePayload), - expect.objectContaining(requests[2].notePayload), - expect.objectContaining(requests[3].notePayload), - expect.objectContaining(requests[4].notePayload), + expect.objectContaining(requests[0].snippetOfNoteDao), + expect.objectContaining(requests[1].snippetOfNoteDao), + expect.objectContaining(requests[2].snippetOfNoteDao), + expect.objectContaining(requests[3].snippetOfNoteDao), + expect.objectContaining(requests[4].snippetOfNoteDao), ]); // Outgoing note daos do not have a nonce so we don't check it. diff --git a/yarn-project/pxe/src/note_processor/note_processor.ts b/yarn-project/pxe/src/note_processor/note_processor.ts index b3bfa94e53e..9e4fcfae5cb 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.ts @@ -128,7 +128,9 @@ export class NoteProcessor { // Iterate over both blocks and encrypted logs. for (const block of blocks) { this.stats.blocks++; - const { txLogs } = block.body.noteEncryptedLogs; + const { txLogs: encryptedTxLogs } = block.body.noteEncryptedLogs; + const { txLogs: unencryptedTxLogs } = block.body.unencryptedLogs; + const dataStartIndexForBlock = block.header.state.partial.noteHashTree.nextAvailableLeafIndex - block.body.numberOfTxsIncludingPadded * MAX_NOTE_HASHES_PER_TX; @@ -139,65 +141,72 @@ export class NoteProcessor { const outgoingNotes: OutgoingNoteDao[] = []; // Iterate over all the encrypted logs and try decrypting them. If successful, store the note. - for (let indexOfTxInABlock = 0; indexOfTxInABlock < txLogs.length; ++indexOfTxInABlock) { + for (let indexOfTxInABlock = 0; indexOfTxInABlock < encryptedTxLogs.length; ++indexOfTxInABlock) { this.stats.txs++; const dataStartIndexForTx = dataStartIndexForBlock + indexOfTxInABlock * MAX_NOTE_HASHES_PER_TX; const noteHashes = block.body.txEffects[indexOfTxInABlock].noteHashes; // Note: Each tx generates a `TxL2Logs` object and for this reason we can rely on its index corresponding // to the index of a tx in a block. - const txFunctionLogs = txLogs[indexOfTxInABlock].functionLogs; + const encryptedTxFunctionLogs = encryptedTxLogs[indexOfTxInABlock].functionLogs; + const unencryptedTxFunctionLogs = unencryptedTxLogs[indexOfTxInABlock].functionLogs; const excludedIndices: Set = new Set(); - for (const functionLogs of txFunctionLogs) { - for (const log of functionLogs.logs) { - this.stats.seen++; - const incomingNotePayload = L1NotePayload.decryptAsIncoming(log, addressSecret); - const outgoingNotePayload = L1NotePayload.decryptAsOutgoing(log, ovskM); - - if (incomingNotePayload || outgoingNotePayload) { - if (incomingNotePayload && outgoingNotePayload && !incomingNotePayload.equals(outgoingNotePayload)) { - throw new Error( - `Incoming and outgoing note payloads do not match. Incoming: ${JSON.stringify( - incomingNotePayload, - )}, Outgoing: ${JSON.stringify(outgoingNotePayload)}`, - ); - } - - const payload = incomingNotePayload || outgoingNotePayload; - - const txEffect = block.body.txEffects[indexOfTxInABlock]; - const { incomingNote, outgoingNote, incomingDeferredNote, outgoingDeferredNote } = await produceNoteDaos( - this.simulator, - this.db, - incomingNotePayload ? this.ivpkM : undefined, - outgoingNotePayload ? this.ovpkM : undefined, - payload!, - txEffect.txHash, - noteHashes, - dataStartIndexForTx, - excludedIndices, - this.log, - txEffect.unencryptedLogs, - ); - - if (incomingNote) { - incomingNotes.push(incomingNote); - this.stats.decryptedIncoming++; - } - if (outgoingNote) { - outgoingNotes.push(outgoingNote); - this.stats.decryptedOutgoing++; - } - if (incomingDeferredNote) { - deferredIncomingNotes.push(incomingDeferredNote); - this.stats.deferredIncoming++; - } - if (outgoingDeferredNote) { - deferredOutgoingNotes.push(outgoingDeferredNote); - this.stats.deferredOutgoing++; - } - if (incomingNote == undefined && outgoingNote == undefined && incomingDeferredNote == undefined) { - this.stats.failed++; + // We iterate over both encrypted and unencrypted logs to decrypt the notes since partial notes are passed + // via the unencrypted logs stream. + for (const txFunctionLogs of [encryptedTxFunctionLogs, unencryptedTxFunctionLogs]) { + for (const functionLogs of txFunctionLogs) { + for (const unprocessedLog of functionLogs.logs) { + this.stats.seen++; + const incomingNotePayload = L1NotePayload.decryptAsIncoming(unprocessedLog.data, addressSecret); + const outgoingNotePayload = L1NotePayload.decryptAsOutgoing(unprocessedLog.data, ovskM); + + if (incomingNotePayload || outgoingNotePayload) { + if (incomingNotePayload && outgoingNotePayload && !incomingNotePayload.equals(outgoingNotePayload)) { + throw new Error( + `Incoming and outgoing note payloads do not match. Incoming: ${JSON.stringify( + incomingNotePayload, + )}, Outgoing: ${JSON.stringify(outgoingNotePayload)}`, + ); + } + + const payload = incomingNotePayload || outgoingNotePayload; + + const txEffect = block.body.txEffects[indexOfTxInABlock]; + const { incomingNote, outgoingNote, incomingDeferredNote, outgoingDeferredNote } = + await produceNoteDaos( + this.simulator, + this.db, + incomingNotePayload ? this.ivpkM : undefined, + outgoingNotePayload ? this.ovpkM : undefined, + payload!, + txEffect.txHash, + noteHashes, + dataStartIndexForTx, + excludedIndices, + this.log, + txEffect.unencryptedLogs, + ); + + if (incomingNote) { + incomingNotes.push(incomingNote); + this.stats.decryptedIncoming++; + } + if (outgoingNote) { + outgoingNotes.push(outgoingNote); + this.stats.decryptedOutgoing++; + } + if (incomingDeferredNote) { + deferredIncomingNotes.push(incomingDeferredNote); + this.stats.deferredIncoming++; + } + if (outgoingDeferredNote) { + deferredOutgoingNotes.push(outgoingDeferredNote); + this.stats.deferredOutgoing++; + } + + if (incomingNote == undefined && outgoingNote == undefined && incomingDeferredNote == undefined) { + this.stats.failed++; + } } } } @@ -273,15 +282,15 @@ export class NoteProcessor { await this.db.addDeferredNotes([...deferredIncomingNotes, ...deferredOutgoingNotes]); deferredIncomingNotes.forEach(noteDao => { this.log.verbose( - `Deferred incoming note for contract ${noteDao.contractAddress} at slot ${ - noteDao.storageSlot + `Deferred incoming note for contract ${noteDao.payload.contractAddress} at slot ${ + noteDao.payload.storageSlot } in tx ${noteDao.txHash.toString()}`, ); }); deferredOutgoingNotes.forEach(noteDao => { this.log.verbose( - `Deferred outgoing note for contract ${noteDao.contractAddress} at slot ${ - noteDao.storageSlot + `Deferred outgoing note for contract ${noteDao.payload.contractAddress} at slot ${ + noteDao.payload.storageSlot } in tx ${noteDao.txHash.toString()}`, ); }); @@ -306,18 +315,7 @@ export class NoteProcessor { const outgoingNotes: OutgoingNoteDao[] = []; for (const deferredNote of deferredNoteDaos) { - const { - publicKey, - note, - contractAddress, - storageSlot, - noteTypeId, - txHash, - noteHashes, - dataStartIndexForTx, - unencryptedLogs, - } = deferredNote; - const payload = new L1NotePayload(note, contractAddress, storageSlot, noteTypeId); + const { publicKey, payload, txHash, noteHashes, dataStartIndexForTx, unencryptedLogs } = deferredNote; const isIncoming = publicKey.equals(this.ivpkM); const isOutgoing = publicKey.equals(this.ovpkM); diff --git a/yarn-project/pxe/src/note_processor/utils/add_nullable_field_to_payload.ts b/yarn-project/pxe/src/note_processor/utils/add_nullable_field_to_payload.ts deleted file mode 100644 index 478702f6e00..00000000000 --- a/yarn-project/pxe/src/note_processor/utils/add_nullable_field_to_payload.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { L1NotePayload, Note } from '@aztec/circuit-types'; -import { type Fr } from '@aztec/foundation/fields'; -import { ContractNotFoundError } from '@aztec/simulator'; - -import { type PxeDatabase } from '../../database/pxe_database.js'; - -/** - * Inserts publicly delivered nullable fields into the note payload. - * @param db - PXE database used to fetch contract instance and artifact. - * @param payload - Note payload to which nullable fields should be added. - * @param nullableFields - List of nullable fields to be added to the note payload. - * @returns Note payload with nullable fields added. - */ -export async function addNullableFieldsToPayload( - db: PxeDatabase, - payload: L1NotePayload, - nullableFields: Fr[], -): Promise { - const instance = await db.getContractInstance(payload.contractAddress); - if (!instance) { - throw new ContractNotFoundError( - `Could not find instance for ${payload.contractAddress.toString()}. This should never happen here as the partial notes flow should be triggered only for non-deferred notes.`, - ); - } - - const artifact = await db.getContractArtifact(instance.contractClassId); - if (!artifact) { - throw new Error( - `Could not find artifact for contract class ${instance.contractClassId.toString()}. This should never happen here as the partial notes flow should be triggered only for non-deferred notes.`, - ); - } - - const noteFields = Object.values(artifact.notes).find(note => note.id.equals(payload.noteTypeId))?.fields; - - if (!noteFields) { - throw new Error(`Could not find note fields for note type ${payload.noteTypeId.toString()}.`); - } - - // We sort note fields by index so that we can iterate over them in order. - noteFields.sort((a, b) => a.index - b.index); - - // Now we insert the nullable fields into the note based on its indices defined in the ABI. - const modifiedNoteItems = [...payload.note.items]; - let indexInNullable = 0; - for (let i = 0; i < noteFields.length; i++) { - const noteField = noteFields[i]; - if (noteField.nullable) { - if (i == noteFields.length - 1) { - // We are processing the last field so we simply insert the rest of the nullable fields at the end - modifiedNoteItems.push(...nullableFields.slice(indexInNullable)); - } else { - const noteFieldLength = noteFields[i + 1].index - noteField.index; - const nullableFieldsToInsert = nullableFields.slice(indexInNullable, indexInNullable + noteFieldLength); - indexInNullable += noteFieldLength; - // Now we insert the nullable fields at the note field index - modifiedNoteItems.splice(noteField.index, 0, ...nullableFieldsToInsert); - } - } - } - - return new L1NotePayload( - new Note(modifiedNoteItems), - payload.contractAddress, - payload.storageSlot, - payload.noteTypeId, - ); -} diff --git a/yarn-project/pxe/src/note_processor/utils/add_public_values_to_payload.ts b/yarn-project/pxe/src/note_processor/utils/add_public_values_to_payload.ts new file mode 100644 index 00000000000..8a249ceab6d --- /dev/null +++ b/yarn-project/pxe/src/note_processor/utils/add_public_values_to_payload.ts @@ -0,0 +1,63 @@ +import { type L1NotePayload, Note } from '@aztec/circuit-types'; +import { ContractNotFoundError } from '@aztec/simulator'; + +import { type PxeDatabase } from '../../database/pxe_database.js'; + +/** + * Merges privately and publicly delivered note values. + * @param db - PXE database used to fetch contract instance and artifact. + * @param payload - Payload corresponding to the note. + * @returns Note payload with public fields added. + */ +export async function getOrderedNoteItems( + db: PxeDatabase, + { contractAddress, noteTypeId, privateNoteValues, publicNoteValues }: L1NotePayload, +): Promise { + if (publicNoteValues.length === 0) { + return new Note(privateNoteValues); + } + + const instance = await db.getContractInstance(contractAddress); + if (!instance) { + throw new ContractNotFoundError( + `Could not find instance for ${contractAddress.toString()}. This should never happen here as the partial notes flow should be triggered only for non-deferred notes.`, + ); + } + + const artifact = await db.getContractArtifact(instance.contractClassId); + if (!artifact) { + throw new Error( + `Could not find artifact for contract class ${instance.contractClassId.toString()}. This should never happen here as the partial notes flow should be triggered only for non-deferred notes.`, + ); + } + + const noteFields = Object.values(artifact.notes).find(note => note.id.equals(noteTypeId))?.fields; + + if (!noteFields) { + throw new Error(`Could not find note fields for note type ${noteTypeId.toString()}.`); + } + + // We sort note fields by index so that we can iterate over them in order. + noteFields.sort((a, b) => a.index - b.index); + + // Now we insert the public fields into the note based on its indices defined in the ABI. + const modifiedNoteItems = privateNoteValues; + let indexInPublicValues = 0; + for (let i = 0; i < noteFields.length; i++) { + const noteField = noteFields[i]; + if (noteField.nullable) { + if (i == noteFields.length - 1) { + // We are processing the last field so we simply insert the rest of the public fields at the end + modifiedNoteItems.push(...publicNoteValues.slice(indexInPublicValues)); + } else { + const noteFieldLength = noteFields[i + 1].index - noteField.index; + const publicValuesToInsert = publicNoteValues.slice(indexInPublicValues, indexInPublicValues + noteFieldLength); + indexInPublicValues += noteFieldLength; + // Now we insert the public fields at the note field index + modifiedNoteItems.splice(noteField.index, 0, ...publicValuesToInsert); + } + } + } + + return new Note(modifiedNoteItems); +} diff --git a/yarn-project/pxe/src/note_processor/utils/brute_force_note_info.ts b/yarn-project/pxe/src/note_processor/utils/brute_force_note_info.ts index f7515a61b25..cde87260e71 100644 --- a/yarn-project/pxe/src/note_processor/utils/brute_force_note_info.ts +++ b/yarn-project/pxe/src/note_processor/utils/brute_force_note_info.ts @@ -1,5 +1,7 @@ -import { type L1NotePayload, type TxHash } from '@aztec/circuit-types'; +import { type Note, type TxHash } from '@aztec/circuit-types'; +import { type AztecAddress } from '@aztec/circuits.js'; import { computeNoteHashNonce, siloNullifier } from '@aztec/circuits.js/hash'; +import { type NoteSelector } from '@aztec/foundation/abi'; import { Fr } from '@aztec/foundation/fields'; import { type AcirSimulator } from '@aztec/simulator'; @@ -18,7 +20,10 @@ export interface NoteInfo { * @remarks This method assists in identifying spent notes in the note hash tree. * @param siloedNoteHashes - Note hashes in the tx. One of them should correspond to the note we are looking for * @param txHash - Hash of a tx the note was emitted in. - * @param l1NotePayload - The note payload. + * @param contractAddress - Address of the contract the note was emitted in. + * @param storageSlot - Storage slot of the note. + * @param noteTypeId - Type of the note. + * @param note - Note items. * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same * l1NotePayload. We need to find a different index for each replicate. * @param computeNullifier - A flag indicating whether to compute the nullifier or just return 0. @@ -29,7 +34,10 @@ export async function bruteForceNoteInfo( simulator: AcirSimulator, siloedNoteHashes: Fr[], txHash: TxHash, - { contractAddress, storageSlot, noteTypeId, note }: L1NotePayload, + contractAddress: AztecAddress, + storageSlot: Fr, + noteTypeId: NoteSelector, + note: Note, excludedIndices: Set, computeNullifier: boolean, ): Promise { diff --git a/yarn-project/pxe/src/note_processor/utils/produce_note_daos.ts b/yarn-project/pxe/src/note_processor/utils/produce_note_daos.ts index 5198664efe2..15127d36cd9 100644 --- a/yarn-project/pxe/src/note_processor/utils/produce_note_daos.ts +++ b/yarn-project/pxe/src/note_processor/utils/produce_note_daos.ts @@ -46,8 +46,6 @@ export async function produceNoteDaos( incomingDeferredNote: DeferredNoteDao | undefined; outgoingDeferredNote: DeferredNoteDao | undefined; }> { - // WARNING: This code is full of tech debt and will be refactored once we have final design of partial notes - // delivery. if (!ivpkM && !ovpkM) { throw new Error('Both ivpkM and ovpkM are undefined. Cannot create note.'); } @@ -78,11 +76,11 @@ export async function produceNoteDaos( // Incoming note is defined meaning that this PXE has both the incoming and outgoing keys. We can skip computing // note hash and note index since we already have them in the incoming note. outgoingNote = new OutgoingNoteDao( - payload.note, - payload.contractAddress, - payload.storageSlot, - payload.noteTypeId, - txHash, + incomingNote.note, + incomingNote.contractAddress, + incomingNote.storageSlot, + incomingNote.noteTypeId, + incomingNote.txHash, incomingNote.nonce, incomingNote.noteHash, incomingNote.index, diff --git a/yarn-project/pxe/src/note_processor/utils/produce_note_daos_for_key.ts b/yarn-project/pxe/src/note_processor/utils/produce_note_daos_for_key.ts index 1b3acd032ad..42b04fc3c13 100644 --- a/yarn-project/pxe/src/note_processor/utils/produce_note_daos_for_key.ts +++ b/yarn-project/pxe/src/note_processor/utils/produce_note_daos_for_key.ts @@ -1,11 +1,11 @@ -import { type L1NotePayload, type TxHash, UnencryptedTxL2Logs } from '@aztec/circuit-types'; -import { Fr, type PublicKey } from '@aztec/circuits.js'; +import { type L1NotePayload, type Note, type TxHash, type UnencryptedTxL2Logs } from '@aztec/circuit-types'; +import { type Fr, type PublicKey } from '@aztec/circuits.js'; import { type Logger } from '@aztec/foundation/log'; import { type AcirSimulator, ContractNotFoundError } from '@aztec/simulator'; import { DeferredNoteDao } from '../../database/deferred_note_dao.js'; import { type PxeDatabase } from '../../database/pxe_database.js'; -import { addNullableFieldsToPayload } from './add_nullable_field_to_payload.js'; +import { getOrderedNoteItems } from './add_public_values_to_payload.js'; import { type NoteInfo, bruteForceNoteInfo } from './brute_force_note_info.js'; export async function produceNoteDaosForKey( @@ -19,61 +19,40 @@ export async function produceNoteDaosForKey( excludedIndices: Set, logger: Logger, unencryptedLogs: UnencryptedTxL2Logs, - daoConstructor: (payload: L1NotePayload, noteInfo: NoteInfo, dataStartIndexForTx: number, pkM: PublicKey) => T, + daoConstructor: ( + note: Note, + payload: L1NotePayload, + noteInfo: NoteInfo, + dataStartIndexForTx: number, + pkM: PublicKey, + ) => T, ): Promise<[T | undefined, DeferredNoteDao | undefined]> { let noteDao: T | undefined; let deferredNoteDao: DeferredNoteDao | undefined; try { + // We get the note by merging publicly and privately delivered note values. + const note = await getOrderedNoteItems(db, payload); + const noteInfo = await bruteForceNoteInfo( simulator, noteHashes, txHash, - payload, + payload.contractAddress, + payload.storageSlot, + payload.noteTypeId, + note, excludedIndices, true, // For incoming we compute a nullifier (recipient of incoming is the party that nullifies). ); excludedIndices?.add(noteInfo.noteHashIndex); - noteDao = daoConstructor(payload, noteInfo, dataStartIndexForTx, pkM); + noteDao = daoConstructor(note, payload, noteInfo, dataStartIndexForTx, pkM); } catch (e) { if (e instanceof ContractNotFoundError) { logger.warn(e.message); - deferredNoteDao = new DeferredNoteDao( - pkM, - payload.note, - payload.contractAddress, - payload.storageSlot, - payload.noteTypeId, - txHash, - noteHashes, - dataStartIndexForTx, - unencryptedLogs, - ); - } else if ( - (e as any).message.includes('failed to solve blackbox function: embedded_curve_add') || - (e as any).message.includes('Could not find key prefix.') - ) { - // TODO(#8769): This branch is a temporary partial notes delivery solution that should be eventually replaced. - // Both error messages above occur only when we are dealing with a partial note and are thrown when calling - // `note.compute_note_hash()` or `note.compute_nullifier_without_context()` - // in `compute_note_hash_and_optionally_a_nullifier` function. It occurs with partial notes because in the - // partial flow we receive a note log of a note that is missing some fields here and then we try to compute - // the note hash with MSM while some of the fields are zeroed out (or get a nsk for zero npk_m_hash). - noteDao = await handlePartialNote( - simulator, - db, - pkM, - payload, - txHash, - noteHashes, - dataStartIndexForTx, - excludedIndices, - logger, - unencryptedLogs, - daoConstructor, - ); + deferredNoteDao = new DeferredNoteDao(pkM, payload, txHash, noteHashes, dataStartIndexForTx, unencryptedLogs); } else { logger.error(`Could not process note because of "${e}". Discarding note...`); } @@ -81,77 +60,3 @@ export async function produceNoteDaosForKey( return [noteDao, deferredNoteDao]; } - -async function handlePartialNote( - simulator: AcirSimulator, - db: PxeDatabase, - pkM: PublicKey, - payload: L1NotePayload, - txHash: TxHash, - noteHashes: Fr[], - dataStartIndexForTx: number, - excludedIndices: Set, - logger: Logger, - unencryptedLogs: UnencryptedTxL2Logs, - daoConstructor: (payload: L1NotePayload, noteInfo: NoteInfo, dataStartIndexForTx: number, pkM: PublicKey) => T, -): Promise { - let noteDao: T | undefined; - - for (const functionLogs of unencryptedLogs.functionLogs) { - for (const log of functionLogs.logs) { - const { data } = log; - // It is the expectation that partial notes will have the corresponding unencrypted log be multiple - // of Fr.SIZE_IN_BYTES as the nullable fields should be simply concatenated. - if (data.length % Fr.SIZE_IN_BYTES === 0) { - const nullableFields = []; - for (let i = 0; i < data.length; i += Fr.SIZE_IN_BYTES) { - const chunk = data.subarray(i, i + Fr.SIZE_IN_BYTES); - nullableFields.push(Fr.fromBuffer(chunk)); - } - - // We insert the nullable fields into the note and then we try to produce the note dao again - const payloadWithNullableFields = await addNullableFieldsToPayload(db, payload, nullableFields); - - let deferredNoteDao: DeferredNoteDao | undefined; - try { - [noteDao, deferredNoteDao] = await produceNoteDaosForKey( - simulator, - db, - pkM, - payloadWithNullableFields, - txHash, - noteHashes, - dataStartIndexForTx, - excludedIndices, - logger, - UnencryptedTxL2Logs.empty(), // We set unencrypted logs to empty to prevent infinite recursion. - daoConstructor, - ); - } catch (e) { - // We ignore the key prefix error because that is expected to be triggered when an incorrect value - // is inserted at the position of `npk_m_hash`. This happens commonly because we are brute forcing - // the unencrypted logs. - if (!(e as any).message.includes('Could not find key prefix.')) { - throw e; - } - } - - if (deferredNoteDao) { - // This should not happen as we should first get contract not found error before the blackbox func error. - throw new Error('Partial notes should never be deferred.'); - } - - if (noteDao) { - // We managed to complete the partial note so we terminate the search. - break; - } - } - } - } - - if (!noteDao) { - logger.error(`Partial note note found. Discarding note...`); - } - - return noteDao; -} diff --git a/yarn-project/simulator/src/public/enqueued_call_simulator.ts b/yarn-project/simulator/src/public/enqueued_call_simulator.ts index 85e63579cf3..166014d1f8b 100644 --- a/yarn-project/simulator/src/public/enqueued_call_simulator.ts +++ b/yarn-project/simulator/src/public/enqueued_call_simulator.ts @@ -283,9 +283,24 @@ export class EnqueuedCallSimulator { callContext: result.executionRequest.callContext, proverAddress: AztecAddress.ZERO, argsHash: computeVarArgsHash(result.executionRequest.args), - noteHashes: padArrayEnd(result.noteHashes, NoteHash.empty(), MAX_NOTE_HASHES_PER_CALL), - nullifiers: padArrayEnd(result.nullifiers, Nullifier.empty(), MAX_NULLIFIERS_PER_CALL), - l2ToL1Msgs: padArrayEnd(result.l2ToL1Messages, L2ToL1Message.empty(), MAX_L2_TO_L1_MSGS_PER_CALL), + noteHashes: padArrayEnd( + result.noteHashes, + NoteHash.empty(), + MAX_NOTE_HASHES_PER_CALL, + `Too many note hashes. Got ${result.noteHashes.length} with max being ${MAX_NOTE_HASHES_PER_CALL}`, + ), + nullifiers: padArrayEnd( + result.nullifiers, + Nullifier.empty(), + MAX_NULLIFIERS_PER_CALL, + `Too many nullifiers. Got ${result.nullifiers.length} with max being ${MAX_NULLIFIERS_PER_CALL}`, + ), + l2ToL1Msgs: padArrayEnd( + result.l2ToL1Messages, + L2ToL1Message.empty(), + MAX_L2_TO_L1_MSGS_PER_CALL, + `Too many L2 to L1 messages. Got ${result.l2ToL1Messages.length} with max being ${MAX_L2_TO_L1_MSGS_PER_CALL}`, + ), startSideEffectCounter: result.startSideEffectCounter, endSideEffectCounter: result.endSideEffectCounter, returnsHash: computeVarArgsHash(result.returnValues), @@ -293,38 +308,50 @@ export class EnqueuedCallSimulator { result.noteHashReadRequests, TreeLeafReadRequest.empty(), MAX_NOTE_HASH_READ_REQUESTS_PER_CALL, + `Too many note hash read requests. Got ${result.noteHashReadRequests.length} with max being ${MAX_NOTE_HASH_READ_REQUESTS_PER_CALL}`, ), nullifierReadRequests: padArrayEnd( result.nullifierReadRequests, ReadRequest.empty(), MAX_NULLIFIER_READ_REQUESTS_PER_CALL, + `Too many nullifier read requests. Got ${result.nullifierReadRequests.length} with max being ${MAX_NULLIFIER_READ_REQUESTS_PER_CALL}`, ), nullifierNonExistentReadRequests: padArrayEnd( result.nullifierNonExistentReadRequests, ReadRequest.empty(), MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL, + `Too many nullifier non-existent read requests. Got ${result.nullifierNonExistentReadRequests.length} with max being ${MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL}`, ), l1ToL2MsgReadRequests: padArrayEnd( result.l1ToL2MsgReadRequests, TreeLeafReadRequest.empty(), MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL, + `Too many L1 to L2 message read requests. Got ${result.l1ToL2MsgReadRequests.length} with max being ${MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL}`, ), contractStorageReads: padArrayEnd( result.contractStorageReads, ContractStorageRead.empty(), MAX_PUBLIC_DATA_READS_PER_CALL, + `Too many public data reads. Got ${result.contractStorageReads.length} with max being ${MAX_PUBLIC_DATA_READS_PER_CALL}`, ), contractStorageUpdateRequests: padArrayEnd( result.contractStorageUpdateRequests, ContractStorageUpdateRequest.empty(), MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL, + `Too many public data update requests. Got ${result.contractStorageUpdateRequests.length} with max being ${MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL}`, ), publicCallRequests: padArrayEnd( result.publicCallRequests, PublicInnerCallRequest.empty(), MAX_PUBLIC_CALL_STACK_LENGTH_PER_CALL, + `Too many public call requests. Got ${result.publicCallRequests.length} with max being ${MAX_PUBLIC_CALL_STACK_LENGTH_PER_CALL}`, + ), + unencryptedLogsHashes: padArrayEnd( + result.unencryptedLogsHashes, + LogHash.empty(), + MAX_UNENCRYPTED_LOGS_PER_CALL, + `Too many unencrypted logs. Got ${result.unencryptedLogsHashes.length} with max being ${MAX_UNENCRYPTED_LOGS_PER_CALL}`, ), - unencryptedLogsHashes: padArrayEnd(result.unencryptedLogsHashes, LogHash.empty(), MAX_UNENCRYPTED_LOGS_PER_CALL), historicalHeader: this.historicalHeader, globalVariables: this.globalVariables, startGasLeft: Gas.from(result.startGasLeft),