Merge pull request #2204 from AleoHQ/feat/last-committed-ids

Encodes the last election certificate IDs in the batch header and subdag

ledger/narwhal/batch-header/src/bytes.rs

@@ -20,7 +20,8 @@ impl<N: Network> FromBytes for BatchHeader<N> {
         // Read the version.
         let version = u8::read_le(&mut reader)?;
         // Ensure the version is valid.
-        if version != 1 {
+        // TODO (howardwu): For mainnet - Change the version back to 1.
+        if version != 1 && version != 2 {
             return Err(error("Invalid batch header version"));
         }
 
@@ -65,12 +66,44 @@ impl<N: Network> FromBytes for BatchHeader<N> {
             previous_certificate_ids.insert(Field::read_le(&mut reader)?);
         }
 
+        // TODO (howardwu): For mainnet - Change this to always encode the number of committed certificate IDs.
+        //  We currently only encode the size and certificates in the new version, for backwards compatibility.
+        let num_last_election_certificate_ids = if version == 2 {
+            // Read the number of last election certificate IDs.
+            u16::read_le(&mut reader)?
+        } else {
+            // Set the number of last election certificate IDs to zero.
+            0
+        };
+        // Ensure the number of last election certificate IDs is within bounds.
+        if num_last_election_certificate_ids as usize > Self::MAX_CERTIFICATES {
+            return Err(error(format!(
+                "Number of last election certificate IDs ({num_last_election_certificate_ids}) exceeds the maximum ({})",
+                Self::MAX_CERTIFICATES
+            )));
+        }
+        // Read the last election certificate IDs.
+        let mut last_election_certificate_ids = IndexSet::new();
+        for _ in 0..num_last_election_certificate_ids {
+            // Read the certificate ID.
+            last_election_certificate_ids.insert(Field::read_le(&mut reader)?);
+        }
+
         // Read the signature.
         let signature = Signature::read_le(&mut reader)?;
 
         // Construct the batch.
-        let batch = Self::from(author, round, timestamp, transmission_ids, previous_certificate_ids, signature)
-            .map_err(|e| error(e.to_string()))?;
+        let batch = Self::from(
+            version,
+            author,
+            round,
+            timestamp,
+            transmission_ids,
+            previous_certificate_ids,
+            last_election_certificate_ids,
+            signature,
+        )
+        .map_err(|e| error(e.to_string()))?;
 
         // Return the batch.
         match batch.batch_id == batch_id {
@@ -84,7 +117,8 @@ impl<N: Network> ToBytes for BatchHeader<N> {
     /// Writes the batch header to the buffer.
     fn write_le<W: Write>(&self, mut writer: W) -> IoResult<()> {
         // Write the version.
-        1u8.write_le(&mut writer)?;
+        // TODO (howardwu): For mainnet - Change this back to '1u8.write_le(&mut writer)?';
+        self.version.write_le(&mut writer)?;
         // Write the batch ID.
         self.batch_id.write_le(&mut writer)?;
         // Write the author.
@@ -107,6 +141,19 @@ impl<N: Network> ToBytes for BatchHeader<N> {
             // Write the certificate ID.
             certificate_id.write_le(&mut writer)?;
         }
+        // TODO (howardwu): For mainnet - Change this to always encode the number of committed certificate IDs.
+        //  We currently only encode the size and certificates in the new version, for backwards compatibility.
+        if self.version != 1 {
+            // Write the number of last election certificate IDs.
+            u16::try_from(self.last_election_certificate_ids.len())
+                .map_err(|e| error(e.to_string()))?
+                .write_le(&mut writer)?;
+            // Write the last election certificate IDs.
+            for certificate_id in &self.last_election_certificate_ids {
+                // Write the certificate ID.
+                certificate_id.write_le(&mut writer)?;
+            }
+        }
         // Write the signature.
         self.signature.write_le(&mut writer)
     }

ledger/narwhal/batch-header/src/lib.rs

@@ -14,6 +14,7 @@
 
 #![forbid(unsafe_code)]
 #![warn(clippy::cast_possible_truncation)]
+#![allow(clippy::too_many_arguments)]
 
 mod bytes;
 mod serialize;
@@ -30,6 +31,10 @@ use narwhal_transmission_id::TransmissionID;
 
 #[derive(Clone, PartialEq, Eq)]
 pub struct BatchHeader<N: Network> {
+    /// The version of the batch header.
+    /// TODO (howardwu): For mainnet - Remove this version from the struct, we only use it here for backwards compatibility.
+    ///  NOTE: You must keep the version encoding in the byte serialization, just remove it from the struct in memory.
+    version: u8,
     /// The batch ID, defined as the hash of the round number, timestamp, transmission IDs, and previous batch certificate IDs.
     batch_id: Field<N>,
     /// The author of the batch.
@@ -42,6 +47,8 @@ pub struct BatchHeader<N: Network> {
     transmission_ids: IndexSet<TransmissionID<N>>,
     /// The batch certificate IDs of the previous round.
     previous_certificate_ids: IndexSet<Field<N>>,
+    /// The last election batch certificate IDs.
+    last_election_certificate_ids: IndexSet<Field<N>>,
     /// The signature of the batch ID from the creator.
     signature: Signature<N>,
 }
@@ -65,47 +72,99 @@ impl<N: Network> BatchHeader<N> {
         timestamp: i64,
         transmission_ids: IndexSet<TransmissionID<N>>,
         previous_certificate_ids: IndexSet<Field<N>>,
+        last_election_certificate_ids: IndexSet<Field<N>>,
         rng: &mut R,
     ) -> Result<Self> {
+        // Set the version.
+        // TODO (howardwu): For mainnet - Remove this version from the struct, we only use it here for backwards compatibility.
+        //  NOTE: You must keep the version encoding in the byte serialization, just remove it from the struct in memory.
+        let version = 2u8;
+
         match round {
-            // If the round is zero or one, then there should be no previous certificate IDs.
-            0 | 1 => ensure!(previous_certificate_ids.is_empty(), "Invalid round number, must not have certificates"),
+            0 | 1 => {
+                // If the round is zero or one, then there should be no previous certificate IDs.
+                ensure!(previous_certificate_ids.is_empty(), "Invalid round number, must not have certificates");
+                // If the round is zero or one, then there should be no last election certificate IDs.
+                ensure!(last_election_certificate_ids.is_empty(), "Invalid batch, contains election certificates");
+            }
             // If the round is not zero and not one, then there should be at least one previous certificate ID.
             _ => ensure!(!previous_certificate_ids.is_empty(), "Invalid round number, must have certificates"),
         }
         // Retrieve the address.
         let author = Address::try_from(private_key)?;
         // Compute the batch ID.
-        let batch_id = Self::compute_batch_id(author, round, timestamp, &transmission_ids, &previous_certificate_ids)?;
+        let batch_id = Self::compute_batch_id(
+            version,
+            author,
+            round,
+            timestamp,
+            &transmission_ids,
+            &previous_certificate_ids,
+            &last_election_certificate_ids,
+        )?;
         // Sign the preimage.
         let signature = private_key.sign(&[batch_id], rng)?;
         // Return the batch header.
-        Ok(Self { author, batch_id, round, timestamp, transmission_ids, previous_certificate_ids, signature })
+        Ok(Self {
+            version,
+            author,
+            batch_id,
+            round,
+            timestamp,
+            transmission_ids,
+            previous_certificate_ids,
+            last_election_certificate_ids,
+            signature,
+        })
     }
 
     /// Initializes a new batch header.
     pub fn from(
+        version: u8,
         author: Address<N>,
         round: u64,
         timestamp: i64,
         transmission_ids: IndexSet<TransmissionID<N>>,
         previous_certificate_ids: IndexSet<Field<N>>,
+        last_election_certificate_ids: IndexSet<Field<N>>,
         signature: Signature<N>,
     ) -> Result<Self> {
         match round {
-            // If the round is zero or one, then there should be no previous certificate IDs.
-            0 | 1 => ensure!(previous_certificate_ids.is_empty(), "Invalid round number, must not have certificates"),
+            0 | 1 => {
+                // If the round is zero or one, then there should be no previous certificate IDs.
+                ensure!(previous_certificate_ids.is_empty(), "Invalid round number, must not have certificates");
+                // If the round is zero or one, then there should be no last election certificate IDs.
+                ensure!(last_election_certificate_ids.is_empty(), "Invalid batch, contains election certificates");
+            }
             // If the round is not zero and not one, then there should be at least one previous certificate ID.
             _ => ensure!(!previous_certificate_ids.is_empty(), "Invalid round number, must have certificates"),
         }
         // Compute the batch ID.
-        let batch_id = Self::compute_batch_id(author, round, timestamp, &transmission_ids, &previous_certificate_ids)?;
+        let batch_id = Self::compute_batch_id(
+            version,
+            author,
+            round,
+            timestamp,
+            &transmission_ids,
+            &previous_certificate_ids,
+            &last_election_certificate_ids,
+        )?;
         // Verify the signature.
         if !signature.verify(&author, &[batch_id]) {
             bail!("Invalid signature for the batch header");
         }
         // Return the batch header.
-        Ok(Self { author, batch_id, round, timestamp, transmission_ids, previous_certificate_ids, signature })
+        Ok(Self {
+            version,
+            author,
+            batch_id,
+            round,
+            timestamp,
+            transmission_ids,
+            previous_certificate_ids,
+            last_election_certificate_ids,
+            signature,
+        })
     }
 }
 
@@ -140,6 +199,11 @@ impl<N: Network> BatchHeader<N> {
         &self.previous_certificate_ids
     }
 
+    /// Returns the last election batch certificate IDs.
+    pub const fn last_election_certificate_ids(&self) -> &IndexSet<Field<N>> {
+        &self.last_election_certificate_ids
+    }
+
     /// Returns the signature.
     pub const fn signature(&self) -> &Signature<N> {
         &self.signature
@@ -198,8 +262,19 @@ pub mod test_helpers {
             narwhal_transmission_id::test_helpers::sample_transmission_ids(rng).into_iter().collect::<IndexSet<_>>();
         // Checkpoint the timestamp for the batch.
         let timestamp = OffsetDateTime::now_utc().unix_timestamp();
+        // Sample the last election certificate IDs.
+        let last_election_certificate_ids = (0..5).map(|_| Field::<CurrentNetwork>::rand(rng)).collect::<IndexSet<_>>();
         // Return the batch header.
-        BatchHeader::new(&private_key, round, timestamp, transmission_ids, previous_certificate_ids, rng).unwrap()
+        BatchHeader::new(
+            &private_key,
+            round,
+            timestamp,
+            transmission_ids,
+            previous_certificate_ids,
+            last_election_certificate_ids,
+            rng,
+        )
+        .unwrap()
     }
 
     /// Returns a list of sample batch headers, sampled at random.

ledger/narwhal/batch-header/src/serialize.rs

@@ -19,13 +19,16 @@ impl<N: Network> Serialize for BatchHeader<N> {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
         match serializer.is_human_readable() {
             true => {
-                let mut header = serializer.serialize_struct("BatchHeader", 7)?;
+                let mut header = serializer.serialize_struct("BatchHeader", 9)?;
+                // TODO (howardwu): For mainnet - Remove the version field, and update the 'len' above to 8.
+                header.serialize_field("version", &self.version)?;
                 header.serialize_field("batch_id", &self.batch_id)?;
                 header.serialize_field("author", &self.author)?;
                 header.serialize_field("round", &self.round)?;
                 header.serialize_field("timestamp", &self.timestamp)?;
                 header.serialize_field("transmission_ids", &self.transmission_ids)?;
                 header.serialize_field("previous_certificate_ids", &self.previous_certificate_ids)?;
+                header.serialize_field("last_election_certificate_ids", &self.last_election_certificate_ids)?;
                 header.serialize_field("signature", &self.signature)?;
                 header.end()
             }
@@ -42,13 +45,31 @@ impl<'de, N: Network> Deserialize<'de> for BatchHeader<N> {
                 let mut header = serde_json::Value::deserialize(deserializer)?;
                 let batch_id: Field<N> = DeserializeExt::take_from_value::<D>(&mut header, "batch_id")?;
 
+                // TODO (howardwu): For mainnet - Remove the version parsing.
+                // If the version field is present, then parse the version.
+                let version = DeserializeExt::take_from_value::<D>(&mut header, "version").unwrap_or(1);
+                // TODO (howardwu): For mainnet - Remove the version checking.
+                // Ensure the version is valid.
+                if version != 1 && version != 2 {
+                    return Err(error("Invalid batch header version")).map_err(de::Error::custom);
+                }
+                // TODO (howardwu): For mainnet - Always take from the 'header', no need to use this match case anymore.
+                // If the version is not 1, then parse the last election certificate IDs.
+                let last_election_certificate_ids = match version {
+                    1 => IndexSet::new(),
+                    2 => DeserializeExt::take_from_value::<D>(&mut header, "last_election_certificate_ids")?,
+                    _ => unreachable!(),
+                };
+
                 // Recover the header.
                 let batch_header = Self::from(
+                    version,
                     DeserializeExt::take_from_value::<D>(&mut header, "author")?,
                     DeserializeExt::take_from_value::<D>(&mut header, "round")?,
                     DeserializeExt::take_from_value::<D>(&mut header, "timestamp")?,
                     DeserializeExt::take_from_value::<D>(&mut header, "transmission_ids")?,
                     DeserializeExt::take_from_value::<D>(&mut header, "previous_certificate_ids")?,
+                    last_election_certificate_ids,
                     DeserializeExt::take_from_value::<D>(&mut header, "signature")?,
                 )
                 .map_err(de::Error::custom)?;

ledger/narwhal/batch-header/src/to_id.rs

@@ -18,23 +18,27 @@ impl<N: Network> BatchHeader<N> {
     /// Returns the batch ID.
     pub fn to_id(&self) -> Result<Field<N>> {
         Self::compute_batch_id(
+            self.version,
             self.author,
             self.round,
             self.timestamp,
             &self.transmission_ids,
             &self.previous_certificate_ids,
+            &self.last_election_certificate_ids,
         )
     }
 }
 
 impl<N: Network> BatchHeader<N> {
     /// Returns the batch ID.
     pub fn compute_batch_id(
+        version: u8,
         author: Address<N>,
         round: u64,
         timestamp: i64,
         transmission_ids: &IndexSet<TransmissionID<N>>,
         previous_certificate_ids: &IndexSet<Field<N>>,
+        last_election_certificate_ids: &IndexSet<Field<N>>,
     ) -> Result<Field<N>> {
         let mut preimage = Vec::new();
         // Insert the author.
@@ -56,6 +60,17 @@ impl<N: Network> BatchHeader<N> {
             // Insert the certificate ID.
             certificate_id.write_le(&mut preimage)?;
         }
+        // TODO (howardwu): For mainnet - Change this to always encode the number of committed certificate IDs.
+        //  We currently only encode the size and certificates only in the new version, for backwards compatibility.
+        if version != 1 {
+            // Insert the number of last election certificate IDs.
+            u32::try_from(last_election_certificate_ids.len())?.write_le(&mut preimage)?;
+            // Insert the last election certificate IDs.
+            for certificate_id in last_election_certificate_ids {
+                // Insert the certificate ID.
+                certificate_id.write_le(&mut preimage)?;
+            }
+        }
         // Hash the preimage.
         N::hash_bhp1024(&preimage.to_bits_le())
     }