Add incremental snapshot utils

core/src/accounts_hash_verifier.rs

@@ -99,6 +99,7 @@ impl AccountsHashVerifier {
         let accounts_package = solana_runtime::snapshot_utils::process_accounts_package_pre(
             accounts_package,
             thread_pool,
+            None,
         );
         Self::process_accounts_package(
             accounts_package,

core/src/snapshot_packager_service.rs

@@ -156,7 +156,7 @@ mod tests {
         }
 
         // Create a packageable snapshot
-        let output_tar_path = snapshot_utils::build_snapshot_archive_path(
+        let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
             snapshot_package_output_path,
             42,
             &Hash::default(),
@@ -177,7 +177,7 @@ mod tests {
         // Make tarball from packageable snapshot
         snapshot_utils::archive_snapshot_package(
             &snapshot_package,
-            snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         )
         .unwrap();
 

core/src/test_validator.rs

@@ -13,7 +13,9 @@ use {
         genesis_utils::create_genesis_config_with_leader_ex,
         hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         snapshot_config::SnapshotConfig,
-        snapshot_utils::{ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
+        snapshot_utils::{
+            ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        },
     },
     solana_sdk::{
         account::{Account, AccountSharedData},
@@ -492,7 +494,7 @@ impl TestValidator {
                 snapshot_package_output_path: ledger_path.to_path_buf(),
                 archive_format: ArchiveFormat::Tar,
                 snapshot_version: SnapshotVersion::default(),
-                maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+                maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             }),
             enforce_ulimit_nofile: false,
             warp_slot: config.warp_slot,

core/src/validator.rs

@@ -1187,7 +1187,7 @@ fn new_banks_from_ledger(
         );
         leader_schedule_cache.set_root(&bank_forks.root_bank());
 
-        let archive_file = solana_runtime::snapshot_utils::bank_to_snapshot_archive(
+        let archive_file = solana_runtime::snapshot_utils::bank_to_full_snapshot_archive(
             ledger_path,
             &bank_forks.root_bank(),
             None,

core/tests/snapshots.rs

@@ -49,7 +49,9 @@ mod tests {
         bank_forks::BankForks,
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         snapshot_config::SnapshotConfig,
-        snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
+        snapshot_utils::{
+            self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        },
         status_cache::MAX_CACHE_ENTRIES,
     };
     use solana_sdk::{
@@ -119,7 +121,7 @@ mod tests {
                 snapshot_path: PathBuf::from(snapshot_dir.path()),
                 archive_format: ArchiveFormat::TarBzip2,
                 snapshot_version,
-                maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+                maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             };
             bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
             SnapshotTestConfig {
@@ -148,20 +150,21 @@ mod tests {
         let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
 
         let check_hash_calculation = false;
-        let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archive(
+        let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archives(
             account_paths,
             &[],
             &old_bank_forks
                 .snapshot_config
                 .as_ref()
                 .unwrap()
                 .snapshot_path,
-            snapshot_utils::build_snapshot_archive_path(
+            snapshot_utils::build_full_snapshot_archive_path(
                 snapshot_package_output_path.to_path_buf(),
                 old_last_bank.slot(),
                 &old_last_bank.get_accounts_hash(),
                 ArchiveFormat::TarBzip2,
             ),
+            None,
             ArchiveFormat::TarBzip2,
             old_genesis_config,
             None,
@@ -181,10 +184,10 @@ mod tests {
             .clone();
         assert_eq!(*bank, deserialized_bank);
 
-        let slot_snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshot_path);
+        let bank_snapshot_infos = snapshot_utils::get_bank_snapshots(&snapshot_path);
 
-        for p in slot_snapshot_paths {
-            snapshot_utils::remove_snapshot(p.slot, &snapshot_path).unwrap();
+        for p in bank_snapshot_infos {
+            snapshot_utils::remove_bank_snapshot(p.slot, &snapshot_path).unwrap();
         }
     }
 
@@ -235,12 +238,11 @@ mod tests {
         let last_bank = bank_forks.get(last_slot).unwrap();
         let snapshot_config = &snapshot_test_config.snapshot_config;
         let snapshot_path = &snapshot_config.snapshot_path;
-        let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
-            .pop()
+        let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path)
             .expect("no snapshots found in path");
-        let snapshot_package = snapshot_utils::package_snapshot(
+        let snapshot_package = snapshot_utils::package_full_snapshot(
             last_bank,
-            &last_slot_snapshot_path,
+            &last_bank_snapshot_info,
             snapshot_path,
             last_bank.src.slot_deltas(&last_bank.src.roots()),
             &snapshot_config.snapshot_package_output_path,
@@ -253,10 +255,11 @@ mod tests {
         let snapshot_package = snapshot_utils::process_accounts_package_pre(
             snapshot_package,
             Some(last_bank.get_thread_pool()),
+            None,
         );
         snapshot_utils::archive_snapshot_package(
             &snapshot_package,
-            DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         )
         .unwrap();
 
@@ -325,7 +328,8 @@ mod tests {
         // Take snapshot of zeroth bank
         let bank0 = bank_forks.get(0).unwrap();
         let storages = bank0.get_snapshot_storages();
-        snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
+        snapshot_utils::add_bank_snapshot(snapshot_path, bank0, &storages, snapshot_version)
+            .unwrap();
 
         // Set up snapshotting channels
         let (sender, receiver) = channel();
@@ -343,7 +347,7 @@ mod tests {
         let saved_slot = 4;
         let mut saved_archive_path = None;
 
-        for forks in 0..snapshot_utils::MAX_SNAPSHOTS + 2 {
+        for forks in 0..snapshot_utils::MAX_BANK_SNAPSHOTS + 2 {
             let bank = Bank::new_from_parent(
                 &bank_forks[forks as u64],
                 &Pubkey::default(),
@@ -420,7 +424,7 @@ mod tests {
                 let options = CopyOptions::new();
                 fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
 
-                saved_archive_path = Some(snapshot_utils::build_snapshot_archive_path(
+                saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
                     snapshot_package_output_path.to_path_buf(),
                     slot,
                     &accounts_hash,
@@ -431,11 +435,14 @@ mod tests {
 
         // Purge all the outdated snapshots, including the ones needed to generate the package
         // currently sitting in the channel
-        snapshot_utils::purge_old_snapshots(snapshot_path);
-        assert!(snapshot_utils::get_snapshot_paths(&snapshots_dir)
+        snapshot_utils::purge_old_bank_snapshots(snapshot_path);
+
+        let mut bank_snapshot_infos = snapshot_utils::get_bank_snapshots(&snapshots_dir);
+        bank_snapshot_infos.sort_unstable();
+        assert!(bank_snapshot_infos
             .into_iter()
             .map(|path| path.slot)
-            .eq(3..=snapshot_utils::MAX_SNAPSHOTS as u64 + 2));
+            .eq(3..=snapshot_utils::MAX_BANK_SNAPSHOTS as u64 + 2));
 
         // Create a SnapshotPackagerService to create tarballs from all the pending
         // SnapshotPackage's on the channel. By the time this service starts, we have already
@@ -453,7 +460,7 @@ mod tests {
             None,
             &exit,
             &cluster_info,
-            DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         );
 
         let thread_pool = accounts_db::make_min_priority_thread_pool();
@@ -471,6 +478,7 @@ mod tests {
                         solana_runtime::snapshot_utils::process_accounts_package_pre(
                             snapshot_package,
                             Some(&thread_pool),
+                            None,
                         );
                     *pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
                 }

download-utils/src/lib.rs

@@ -246,22 +246,22 @@ pub fn download_genesis_if_missing(
 
 pub fn download_snapshot<'a, 'b>(
     rpc_addr: &SocketAddr,
-    snapshot_output_dir: &Path,
+    snapshot_archives_dir: &Path,
     desired_snapshot_hash: (Slot, Hash),
     use_progress_bar: bool,
     maximum_snapshots_to_retain: usize,
     progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
 ) -> Result<(), String> {
-    snapshot_utils::purge_old_snapshot_archives(snapshot_output_dir, maximum_snapshots_to_retain);
+    snapshot_utils::purge_old_snapshot_archives(snapshot_archives_dir, maximum_snapshots_to_retain);
 
     for compression in &[
         ArchiveFormat::TarZstd,
         ArchiveFormat::TarGzip,
         ArchiveFormat::TarBzip2,
         ArchiveFormat::Tar, // `solana-test-validator` creates uncompressed snapshots
     ] {
-        let desired_snapshot_package = snapshot_utils::build_snapshot_archive_path(
-            snapshot_output_dir.to_path_buf(),
+        let desired_snapshot_package = snapshot_utils::build_full_snapshot_archive_path(
+            snapshot_archives_dir.to_path_buf(),
             desired_snapshot_hash.0,
             &desired_snapshot_hash.1,
             *compression,

ledger-tool/src/main.rs

@@ -30,7 +30,9 @@ use solana_runtime::{
     bank_forks::BankForks,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     snapshot_config::SnapshotConfig,
-    snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
+    snapshot_utils::{
+        self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+    },
 };
 use solana_sdk::{
     account::{AccountSharedData, ReadableAccount, WritableAccount},
@@ -697,7 +699,7 @@ fn load_bank_forks(
             snapshot_path,
             archive_format: ArchiveFormat::TarBzip2,
             snapshot_version: SnapshotVersion::default(),
-            maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         })
     };
     let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
@@ -904,7 +906,7 @@ fn main() {
         .default_value(SnapshotVersion::default().into())
         .help("Output snapshot version");
 
-    let default_max_snapshot_to_retain = &DEFAULT_MAX_SNAPSHOTS_TO_RETAIN.to_string();
+    let default_max_snapshot_to_retain = &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string();
     let maximum_snapshots_to_retain_arg = Arg::with_name("maximum_snapshots_to_retain")
         .long("maximum-snapshots-to-retain")
         .value_name("NUMBER")
@@ -2196,7 +2198,7 @@ fn main() {
                         bank.slot(),
                     );
 
-                    let archive_file = snapshot_utils::bank_to_snapshot_archive(
+                    let archive_file = snapshot_utils::bank_to_full_snapshot_archive(
                         ledger_path,
                         &bank,
                         Some(snapshot_version),

ledger/src/bank_forks_utils.rs

@@ -11,7 +11,7 @@ use solana_entry::entry::VerifyRecyclers;
 use solana_runtime::{
     bank_forks::BankForks,
     snapshot_config::SnapshotConfig,
-    snapshot_utils::{self, SnapshotArchiveInfo},
+    snapshot_utils::{self, FullSnapshotArchiveInfo},
 };
 use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash};
 use std::{fs, path::PathBuf, process, result};
@@ -53,9 +53,11 @@ pub fn load(
         fs::create_dir_all(&snapshot_config.snapshot_path)
             .expect("Couldn't create snapshot directory");
 
-        if let Some(snapshot_archive_info) = snapshot_utils::get_highest_snapshot_archive_info(
-            &snapshot_config.snapshot_package_output_path,
-        ) {
+        if let Some(full_snapshot_archive_info) =
+            snapshot_utils::get_highest_full_snapshot_archive_info(
+                &snapshot_config.snapshot_package_output_path,
+            )
+        {
             return load_from_snapshot(
                 genesis_config,
                 blockstore,
@@ -65,7 +67,7 @@ pub fn load(
                 process_options,
                 transaction_status_sender,
                 cache_block_meta_sender,
-                &snapshot_archive_info,
+                &full_snapshot_archive_info,
             );
         } else {
             info!("No snapshot package available; will load from genesis");
@@ -113,11 +115,11 @@ fn load_from_snapshot(
     process_options: ProcessOptions,
     transaction_status_sender: Option<&TransactionStatusSender>,
     cache_block_meta_sender: Option<&CacheBlockMetaSender>,
-    snapshot_archive_info: &SnapshotArchiveInfo,
+    full_snapshot_archive_info: &FullSnapshotArchiveInfo,
 ) -> LoadResult {
     info!(
         "Loading snapshot package: {:?}",
-        &snapshot_archive_info.path
+        full_snapshot_archive_info.path()
     );
 
     // Fail hard here if snapshot fails to load, don't silently continue
@@ -126,12 +128,13 @@ fn load_from_snapshot(
         process::exit(1);
     }
 
-    let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archive(
+    let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archives(
         &account_paths,
         &process_options.frozen_accounts,
         &snapshot_config.snapshot_path,
-        &snapshot_archive_info.path,
-        snapshot_archive_info.archive_format,
+        full_snapshot_archive_info.path(),
+        None,
+        *full_snapshot_archive_info.archive_format(),
         genesis_config,
         process_options.debug_keys.clone(),
         Some(&crate::builtins::get(process_options.bpf_jit)),
@@ -152,10 +155,18 @@ fn load_from_snapshot(
         deserialized_bank.get_accounts_hash(),
     );
 
-    if deserialized_bank_slot_and_hash != (snapshot_archive_info.slot, snapshot_archive_info.hash) {
+    if deserialized_bank_slot_and_hash
+        != (
+            *full_snapshot_archive_info.slot(),
+            *full_snapshot_archive_info.hash(),
+        )
+    {
         error!(
             "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}",
-            (snapshot_archive_info.slot, snapshot_archive_info.hash),
+            (
+                full_snapshot_archive_info.slot(),
+                full_snapshot_archive_info.hash()
+            ),
             deserialized_bank_slot_and_hash
         );
         process::exit(1);

ledger/src/blockstore_processor.rs

@@ -567,7 +567,16 @@ fn do_process_blockstore_from_root(
         ("slot", bank_forks.root(), i64),
         ("forks", initial_forks.len(), i64),
         ("calculate_capitalization_us", time_cap.as_us(), i64),
-        ("untar_us", timings.untar_us, i64),
+        (
+            "full_snapshot_untar_us",
+            timings.full_snapshot_untar_us,
+            i64
+        ),
+        (
+            "incremental_snapshot_untar_us",
+            timings.incremental_snapshot_untar_us,
+            i64
+        ),
         (
             "rebuild_bank_from_snapshots_us",
             timings.rebuild_bank_from_snapshots_us,