refactor(storage): use flatbuffer to instead of bincode in storage (p…

…art 4)

BREAKING CHANGE: the data format in storage is changed

protos/schemas/common.fbs

@@ -158,3 +158,8 @@ table TransactionMeta {
     bits:           Bytes;
     len:            uint32;
 }
+
+struct CellMeta {
+    capacity:       uint64;
+    data_hash:      Bytes32;
+}

protos/schemas/storage.fbs

@@ -51,3 +51,7 @@ table StoredProposalShortIds {
 table StoredEpochExt {
     data: EpochExt;
 }
+
+table StoredCellMeta {
+    data: CellMeta;
+}

protos/src/convert/from_common.rs

@@ -312,3 +312,11 @@ impl<'a> From<protos::TransactionMeta<'a>> for TransactionMeta {
             .build()
     }
 }
+
+impl From<&protos::CellMeta> for (Capacity, H256) {
+    fn from(proto: &protos::CellMeta) -> Self {
+        let capacity = Capacity::shannons(proto.capacity());
+        let data_hash = proto.data_hash().into();
+        (capacity, data_hash)
+    }
+}

protos/src/convert/from_storage.rs

@@ -5,6 +5,7 @@ use ckb_core::{
     header::Header,
     transaction::{CellOutput, ProposalShortId, Transaction},
     uncle::UncleBlock,
+    Capacity,
 };
 
 use crate as protos;
@@ -108,3 +109,9 @@ impl<'a> From<protos::StoredEpochExt<'a>> for EpochExt {
         cast!(proto.data()).into()
     }
 }
+
+impl<'a> From<protos::StoredCellMeta<'a>> for (Capacity, H256) {
+    fn from(proto: protos::StoredCellMeta<'a>) -> Self {
+        cast!(proto.data()).into()
+    }
+}

protos/src/convert/to_common.rs

@@ -10,7 +10,7 @@ use ckb_core::{
     transaction::{CellInput, CellOutput, OutPoint, ProposalShortId, Transaction},
     transaction_meta::TransactionMeta,
     uncle::UncleBlock,
-    Bytes,
+    Bytes, Capacity,
 };
 
 use crate as protos;
@@ -410,3 +410,11 @@ impl<'a> protos::TransactionMeta<'a> {
         builder.finish()
     }
 }
+
+impl From<&(Capacity, H256)> for protos::CellMeta {
+    fn from(meta: &(Capacity, H256)) -> Self {
+        let capacity = meta.0.as_u64();
+        let data_hash = (&meta.1).into();
+        Self::new(capacity, &data_hash)
+    }
+}

protos/src/convert/to_storage.rs

@@ -1,11 +1,13 @@
 use flatbuffers::{FlatBufferBuilder, WIPOffset};
+use numext_fixed_hash::H256;
 
 use ckb_core::{
     block::Block,
     extras::{EpochExt, TransactionInfo},
     header::Header,
     transaction::{ProposalShortId, Transaction},
     uncle::UncleBlock,
+    Capacity,
 };
 
 use crate as protos;
@@ -190,3 +192,15 @@ impl<'a> protos::StoredEpochExt<'a> {
         builder.finish()
     }
 }
+
+impl<'a> protos::StoredCellMeta<'a> {
+    pub fn build<'b>(
+        fbb: &mut FlatBufferBuilder<'b>,
+        meta: &(Capacity, H256),
+    ) -> WIPOffset<protos::StoredCellMeta<'b>> {
+        let data = meta.into();
+        let mut builder = protos::StoredCellMetaBuilder::new(fbb);
+        builder.add_data(&data);
+        builder.finish()
+    }
+}

protos/src/generated/common.rs

@@ -586,6 +586,65 @@ impl EpochExt {
     }
 }
 
+// struct CellMeta, aligned to 8
+#[repr(C, align(8))]
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct CellMeta {
+    capacity_: u64,
+    data_hash_: Bytes32,
+} // pub struct CellMeta
+impl flatbuffers::SafeSliceAccess for CellMeta {}
+impl<'a> flatbuffers::Follow<'a> for CellMeta {
+    type Inner = &'a CellMeta;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        <&'a CellMeta>::follow(buf, loc)
+    }
+}
+impl<'a> flatbuffers::Follow<'a> for &'a CellMeta {
+    type Inner = &'a CellMeta;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        flatbuffers::follow_cast_ref::<CellMeta>(buf, loc)
+    }
+}
+impl<'b> flatbuffers::Push for CellMeta {
+    type Output = CellMeta;
+    #[inline]
+    fn push(&self, dst: &mut [u8], _rest: &[u8]) {
+        let src = unsafe {
+            ::std::slice::from_raw_parts(self as *const CellMeta as *const u8, Self::size())
+        };
+        dst.copy_from_slice(src);
+    }
+}
+impl<'b> flatbuffers::Push for &'b CellMeta {
+    type Output = CellMeta;
+
+    #[inline]
+    fn push(&self, dst: &mut [u8], _rest: &[u8]) {
+        let src = unsafe {
+            ::std::slice::from_raw_parts(*self as *const CellMeta as *const u8, Self::size())
+        };
+        dst.copy_from_slice(src);
+    }
+}
+
+impl CellMeta {
+    pub fn new<'a>(_capacity: u64, _data_hash: &'a Bytes32) -> Self {
+        CellMeta {
+            capacity_: _capacity.to_little_endian(),
+            data_hash_: *_data_hash,
+        }
+    }
+    pub fn capacity<'a>(&'a self) -> u64 {
+        self.capacity_.from_little_endian()
+    }
+    pub fn data_hash<'a>(&'a self) -> &'a Bytes32 {
+        &self.data_hash_
+    }
+}
+
 pub enum BytesOffset {}
 #[derive(Copy, Clone, Debug, PartialEq)]
 

protos/src/generated/storage.rs

@@ -1097,3 +1097,79 @@ impl<'a: 'b, 'b> StoredEpochExtBuilder<'a, 'b> {
         flatbuffers::WIPOffset::new(o.value())
     }
 }
+
+pub enum StoredCellMetaOffset {}
+#[derive(Copy, Clone, Debug, PartialEq)]
+
+pub struct StoredCellMeta<'a> {
+    pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for StoredCellMeta<'a> {
+    type Inner = StoredCellMeta<'a>;
+    #[inline]
+    fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+        Self {
+            _tab: flatbuffers::Table { buf: buf, loc: loc },
+        }
+    }
+}
+
+impl<'a> StoredCellMeta<'a> {
+    #[inline]
+    pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+        StoredCellMeta { _tab: table }
+    }
+    #[allow(unused_mut)]
+    pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
+        _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
+        args: &'args StoredCellMetaArgs<'args>,
+    ) -> flatbuffers::WIPOffset<StoredCellMeta<'bldr>> {
+        let mut builder = StoredCellMetaBuilder::new(_fbb);
+        if let Some(x) = args.data {
+            builder.add_data(x);
+        }
+        builder.finish()
+    }
+
+    pub const VT_DATA: flatbuffers::VOffsetT = 4;
+
+    #[inline]
+    pub fn data(&self) -> Option<&'a CellMeta> {
+        self._tab.get::<CellMeta>(StoredCellMeta::VT_DATA, None)
+    }
+}
+
+pub struct StoredCellMetaArgs<'a> {
+    pub data: Option<&'a CellMeta>,
+}
+impl<'a> Default for StoredCellMetaArgs<'a> {
+    #[inline]
+    fn default() -> Self {
+        StoredCellMetaArgs { data: None }
+    }
+}
+pub struct StoredCellMetaBuilder<'a: 'b, 'b> {
+    fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>,
+    start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b> StoredCellMetaBuilder<'a, 'b> {
+    #[inline]
+    pub fn add_data(&mut self, data: &'b CellMeta) {
+        self.fbb_
+            .push_slot_always::<&CellMeta>(StoredCellMeta::VT_DATA, data);
+    }
+    #[inline]
+    pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> StoredCellMetaBuilder<'a, 'b> {
+        let start = _fbb.start_table();
+        StoredCellMetaBuilder {
+            fbb_: _fbb,
+            start_: start,
+        }
+    }
+    #[inline]
+    pub fn finish(self) -> flatbuffers::WIPOffset<StoredCellMeta<'a>> {
+        let o = self.fbb_.end_table(self.start_);
+        flatbuffers::WIPOffset::new(o.value())
+    }
+}

store/src/store.rs

@@ -383,42 +383,39 @@ impl<T: KeyValueDB> ChainStore for ChainKVStore<T> {
     }
 
     fn get_cell_meta(&self, tx_hash: &H256, index: u32) -> Option<CellMeta> {
-        self.get_cell_output(tx_hash, index)
-            .and_then(|cell_output| {
-                self.get(
-                    COLUMN_CELL_META,
-                    CellKey::calculate(tx_hash, index).as_ref(),
-                )
-                .map(|raw| {
-                    let data_hash = H256::from_slice(&raw[..]).expect("db safe access");
-                    (cell_output, data_hash)
-                })
-            })
-            .and_then(|(cell_output, data_hash)| {
-                self.get_transaction_info(tx_hash)
-                    .map(|tx_info| (tx_info, cell_output, data_hash))
-            })
-            .map(|(tx_info, cell_output, data_hash)| {
-                let out_point = CellOutPoint {
-                    tx_hash: tx_hash.to_owned(),
-                    index: index as u32,
-                };
-                let capacity = cell_output.capacity;
-                let cellbase = tx_info.index == 0;
-                let block_info = BlockInfo {
-                    number: tx_info.block_number,
-                    epoch: tx_info.block_epoch,
-                };
-                CellMeta {
-                    //cell_output: Some(cell_output),
-                    cell_output: None,
-                    out_point,
-                    block_info: Some(block_info),
-                    cellbase,
-                    capacity,
-                    data_hash: Some(data_hash),
-                }
-            })
+        self.process_get(
+            COLUMN_CELL_META,
+            CellKey::calculate(tx_hash, index).as_ref(),
+            |slice| {
+                let meta: (Capacity, H256) =
+                    flatbuffers::get_root::<ckb_protos::StoredCellMeta>(&slice).into();
+                Ok(Some(meta))
+            },
+        )
+        .and_then(|meta| {
+            self.get_transaction_info(tx_hash)
+                .map(|tx_info| (tx_info, meta))
+        })
+        .map(|(tx_info, meta)| {
+            let out_point = CellOutPoint {
+                tx_hash: tx_hash.to_owned(),
+                index: index as u32,
+            };
+            let cellbase = tx_info.index == 0;
+            let block_info = BlockInfo {
+                number: tx_info.block_number,
+                epoch: tx_info.block_epoch,
+            };
+            let (capacity, data_hash) = meta;
+            CellMeta {
+                cell_output: None,
+                out_point,
+                block_info: Some(block_info),
+                cellbase,
+                capacity,
+                data_hash: Some(data_hash),
+            }
+        })
     }
 
     fn get_cellbase(&self, hash: &H256) -> Option<Transaction> {
@@ -563,8 +560,13 @@ impl<B: DbBatch> StoreBatch for DefaultStoreBatch<B> {
                     index: cell_index as u32,
                 };
                 let store_key = out_point.cell_key();
-                let data_hash = output.data_hash();
-                self.insert_raw(COLUMN_CELL_META, store_key.as_ref(), data_hash.as_bytes())?;
+                insert_flatbuffers!(
+                    self,
+                    COLUMN_CELL_META,
+                    store_key.as_ref(),
+                    StoredCellMeta,
+                    &(output.capacity, output.data_hash())
+                );
             }
         }