From 01b9286c3f26753856f23de4cbbff0a5b6614aa4 Mon Sep 17 00:00:00 2001
From: garyschulte <garyschulte@gmail.com>
Date: Fri, 3 May 2024 16:52:55 -0700
Subject: [PATCH] Default bonsai to fully flat db and code storage by codehash 
 (#6894)

* change to full flat db and code stored by code hash by default
* deprecate --Xsnapsync-synchronizer-flat-db-healing-enabled, add DataStorageOption for --Xbonsai-full-flat-db-enabled

Signed-off-by: garyschulte <garyschulte@gmail.com>
---
 CHANGELOG.md                                  |  1 +
 .../org/hyperledger/besu/cli/BesuCommand.java |  4 +-
 .../options/stable/DataStorageOptions.java    | 16 +++++++
 .../options/unstable/SynchronizerOptions.java | 43 ++++-------------
 .../hyperledger/besu/cli/BesuCommandTest.java | 37 ++++++++++----
 .../cli/options/SynchronizerOptionsTest.java  |  1 +
 .../src/test/resources/everything_config.toml |  2 +
 .../storage/flat/FlatDbStrategyProvider.java  | 30 +++++++++++-
 .../worldstate/DataStorageConfiguration.java  | 18 ++++++-
 .../bonsai/AbstractIsolationTests.java        |  5 +-
 .../BonsaiWorldStateKeyValueStorageTest.java  | 48 ++++++++-----------
 .../flat/FlatDbStrategyProviderTest.java      | 10 ++--
 .../sync/snapsync/SnapSyncConfiguration.java  |  7 ---
 .../snapsync/SnapWorldStateDownloader.java    | 10 ----
 .../snapsync/SnapWorldDownloadStateTest.java  |  2 +-
 15 files changed, 135 insertions(+), 99 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f64e16851d7..a9909ef9d6c6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,7 @@
 - Allow users to specify which plugins are registered [#6700](https://github.com/hyperledger/besu/pull/6700)
 - Layered txpool tuning for blob transactions [#6940](https://github.com/hyperledger/besu/pull/6940)
 - Update Gradle to 7.6.4 [#7030](https://github.com/hyperledger/besu/pull/7030)
+- Default bonsai to use full-flat db and code-storage-by-code-hash [#6984](https://github.com/hyperledger/besu/pull/6894)
 
 ### Bug fixes
 - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665)
diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
index e3419f594c49..07058883bb0e 100644
--- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
+++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
@@ -1706,8 +1706,8 @@ && isOptionSet(commandLine, "--sync-min-peers")) {
 
     CommandLineUtils.failIfOptionDoesntMeetRequirement(
         commandLine,
-        "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true",
-        unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled(),
+        "--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true",
+        dataStorageOptions.toDomainObject().getUnstable().getBonsaiFullFlatDbEnabled(),
         asList(
             "--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
             "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request"));
diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
index c380fb6c2cf1..779a88115ca0 100644
--- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
+++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
@@ -17,6 +17,7 @@
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
@@ -93,6 +94,18 @@ public static class Unstable {
             "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
     private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
 
+    // TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future
+    // release
+    @CommandLine.Option(
+        hidden = true,
+        names = {
+          "--Xbonsai-full-flat-db-enabled",
+          "--Xsnapsync-synchronizer-flat-db-healing-enabled"
+        },
+        arity = "1",
+        description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})")
+    private Boolean bonsaiFullFlatDbEnabled = DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
+
     @CommandLine.Option(
         hidden = true,
         names = {"--Xbonsai-code-using-code-hash-enabled"},
@@ -161,6 +174,8 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai
         domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled();
     dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize =
         domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize();
+    dataStorageOptions.unstableOptions.bonsaiFullFlatDbEnabled =
+        domainObject.getUnstable().getBonsaiFullFlatDbEnabled();
     dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled =
         domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();
 
@@ -177,6 +192,7 @@ public DataStorageConfiguration toDomainObject() {
             ImmutableDataStorageConfiguration.Unstable.builder()
                 .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled)
                 .bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize)
+                .bonsaiFullFlatDbEnabled(unstableOptions.bonsaiFullFlatDbEnabled)
                 .bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled)
                 .build())
         .build();
diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java
index 1adf7080582e..e52839cceb4c 100644
--- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java
+++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java
@@ -81,9 +81,6 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
   private static final String SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG =
       "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request";
 
-  private static final String SNAP_FLAT_DB_HEALING_ENABLED_FLAG =
-      "--Xsnapsync-synchronizer-flat-db-healing-enabled";
-
   private static final String SNAP_SERVER_ENABLED_FLAG = "--Xsnapsync-server-enabled";
 
   private static final String CHECKPOINT_POST_MERGE_FLAG = "--Xcheckpoint-post-merge-enabled";
@@ -292,18 +289,11 @@ public void parseBlockPropagationRange(final String arg) {
   private int snapsyncFlatStorageHealedCountPerRequest =
       SnapSyncConfiguration.DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST;
 
-  @CommandLine.Option(
-      names = SNAP_FLAT_DB_HEALING_ENABLED_FLAG,
-      hidden = true,
-      paramLabel = "<Boolean>",
-      description = "Snap sync flat db healing enabled (default: ${DEFAULT-VALUE})")
-  private Boolean snapsyncFlatDbHealingEnabled =
-      SnapSyncConfiguration.DEFAULT_IS_FLAT_DB_HEALING_ENABLED;
-
   @CommandLine.Option(
       names = SNAP_SERVER_ENABLED_FLAG,
       hidden = true,
       paramLabel = "<Boolean>",
+      arity = "0..1",
       description = "Snap sync server enabled (default: ${DEFAULT-VALUE})")
   private Boolean snapsyncServerEnabled = SnapSyncConfiguration.DEFAULT_SNAP_SERVER_ENABLED;
 
@@ -316,15 +306,6 @@ public void parseBlockPropagationRange(final String arg) {
 
   private SynchronizerOptions() {}
 
-  /**
-   * Flag to know whether the flat db healing feature is enabled or disabled.
-   *
-   * @return true is the flat db healing is enabled
-   */
-  public boolean isSnapsyncFlatDbHealingEnabled() {
-    return snapsyncFlatDbHealingEnabled;
-  }
-
   /**
    * Flag to know whether the Snap sync server feature is enabled or disabled.
    *
@@ -382,9 +363,8 @@ public static SynchronizerOptions fromConfig(final SynchronizerConfiguration con
         config.getSnapSyncConfiguration().getLocalFlatAccountCountToHealPerRequest();
     options.snapsyncFlatStorageHealedCountPerRequest =
         config.getSnapSyncConfiguration().getLocalFlatStorageCountToHealPerRequest();
-    options.snapsyncFlatDbHealingEnabled =
-        config.getSnapSyncConfiguration().isFlatDbHealingEnabled();
     options.checkpointPostMergeSyncEnabled = config.isCheckpointPostMergeEnabled();
+    options.snapsyncServerEnabled = config.getSnapSyncConfiguration().isSnapServerEnabled();
     return options;
   }
 
@@ -416,7 +396,6 @@ public SynchronizerConfiguration.Builder toDomainObject() {
             .trienodeCountPerRequest(snapsyncTrieNodeCountPerRequest)
             .localFlatAccountCountToHealPerRequest(snapsyncFlatAccountHealedCountPerRequest)
             .localFlatStorageCountToHealPerRequest(snapsyncFlatStorageHealedCountPerRequest)
-            .isFlatDbHealingEnabled(snapsyncFlatDbHealingEnabled)
             .isSnapServerEnabled(snapsyncServerEnabled)
             .build());
     builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled);
@@ -469,17 +448,13 @@ public List<String> getCLIOptions() {
             SNAP_BYTECODE_COUNT_PER_REQUEST_FLAG,
             OptionParser.format(snapsyncBytecodeCountPerRequest),
             SNAP_TRIENODE_COUNT_PER_REQUEST_FLAG,
-            OptionParser.format(snapsyncTrieNodeCountPerRequest));
-    if (isSnapsyncFlatDbHealingEnabled()) {
-      value.addAll(
-          Arrays.asList(
-              SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
-              OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
-              SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
-              OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
-              SNAP_SERVER_ENABLED_FLAG,
-              OptionParser.format(snapsyncServerEnabled)));
-    }
+            OptionParser.format(snapsyncTrieNodeCountPerRequest),
+            SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
+            OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
+            SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
+            OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
+            SNAP_SERVER_ENABLED_FLAG,
+            OptionParser.format(snapsyncServerEnabled));
     return value;
   }
 }
diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
index 4353fd7d5feb..1b8083fabe56 100644
--- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
+++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
@@ -2338,29 +2338,48 @@ public void kzgTrustedSetupFileLoadedWithCustomGenesisFile()
   }
 
   @Test
-  public void snapsyncHealingOptionShouldBeDisabledByDefault() {
+  public void bonsaiFlatDbShouldBeEnabledByDefault() {
     final TestBesuCommand besuCommand = parseCommand();
-    assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isFalse();
+    assertThat(
+            besuCommand
+                .getDataStorageOptions()
+                .toDomainObject()
+                .getUnstable()
+                .getBonsaiFullFlatDbEnabled())
+        .isTrue();
   }
 
   @Test
   public void snapsyncHealingOptionShouldWork() {
-    final TestBesuCommand besuCommand =
-        parseCommand("--Xsnapsync-synchronizer-flat-db-healing-enabled", "true");
-    assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isTrue();
+    final TestBesuCommand besuCommand = parseCommand("--Xbonsai-full-flat-db-enabled", "false");
+    assertThat(
+            besuCommand
+                .dataStorageOptions
+                .toDomainObject()
+                .getUnstable()
+                .getBonsaiFullFlatDbEnabled())
+        .isFalse();
   }
 
   @Test
   public void snapsyncForHealingFeaturesShouldFailWhenHealingIsNotEnabled() {
-    parseCommand("--Xsnapsync-synchronizer-flat-account-healed-count-per-request", "100");
+    parseCommand(
+        "--Xbonsai-full-flat-db-enabled",
+        "false",
+        "--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
+        "100");
     assertThat(commandErrorOutput.toString(UTF_8))
         .contains(
-            "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
+            "--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
 
-    parseCommand("--Xsnapsync-synchronizer-flat-slot-healed-count-per-request", "100");
+    parseCommand(
+        "--Xbonsai-full-flat-db-enabled",
+        "false",
+        "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request",
+        "100");
     assertThat(commandErrorOutput.toString(UTF_8))
         .contains(
-            "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
+            "--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
   }
 
   @Test
diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java
index ce2a4d0d55f7..3dcd38809e46 100644
--- a/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java
+++ b/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java
@@ -78,6 +78,7 @@ protected SynchronizerConfiguration.Builder createCustomizedDomainObject() {
                 .storageCountPerRequest(SnapSyncConfiguration.DEFAULT_STORAGE_COUNT_PER_REQUEST + 2)
                 .bytecodeCountPerRequest(
                     SnapSyncConfiguration.DEFAULT_BYTECODE_COUNT_PER_REQUEST + 2)
+                .isSnapServerEnabled(Boolean.TRUE)
                 .build());
   }
 
diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml
index a92eac34e5cc..d1cdaa324634 100644
--- a/besu/src/test/resources/everything_config.toml
+++ b/besu/src/test/resources/everything_config.toml
@@ -218,6 +218,8 @@ receipt-compaction-enabled=true
 # feature flags
 Xsecp256k1-native-enabled=false
 Xaltbn128-native-enabled=false
+Xsnapsync-server-enabled=true
+Xbonsai-full-flat-db-enabled=true
 
 # compatibility flags
 compatibility-eth64-forkid-enabled=false
diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java
index 17a340fe5d09..3ce51e793a48 100644
--- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java
+++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java
@@ -16,6 +16,7 @@
 
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
+import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
 
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy;
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy;
@@ -70,12 +71,36 @@ public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldState
 
   @VisibleForTesting
   FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) {
+    final FlatDbMode requestedFlatDbMode =
+        dataStorageConfiguration.getUnstable().getBonsaiFullFlatDbEnabled()
+            ? FlatDbMode.FULL
+            : FlatDbMode.PARTIAL;
+
+    final var existingTrieData =
+        composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).isPresent();
+
     var flatDbMode =
         FlatDbMode.fromVersion(
             composedWorldStateStorage
                 .get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE)
                 .map(Bytes::wrap)
-                .orElse(FlatDbMode.PARTIAL.getVersion()));
+                .orElseGet(
+                    () -> {
+                      // if we do not have a db-supplied config for flatdb, derive it:
+                      // default to partial if trie data exists, but the flat config does not,
+                      // and default to the storage config otherwise
+                      var flatDbModeVal =
+                          existingTrieData
+                              ? FlatDbMode.PARTIAL.getVersion()
+                              : requestedFlatDbMode.getVersion();
+                      // persist this config in the db
+                      var setDbModeTx = composedWorldStateStorage.startTransaction();
+                      setDbModeTx.put(
+                          TRIE_BRANCH_STORAGE, FLAT_DB_MODE, flatDbModeVal.toArrayUnsafe());
+                      setDbModeTx.commit();
+
+                      return flatDbModeVal;
+                    }));
     LOG.info("Bonsai flat db mode found {}", flatDbMode);
 
     return flatDbMode;
@@ -123,7 +148,7 @@ public FlatDbStrategy getFlatDbStrategy(
   public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
     final SegmentedKeyValueStorageTransaction transaction =
         composedWorldStateStorage.startTransaction();
-    // TODO: consider ARCHIVE mode
+    LOG.info("setting FlatDbStrategy to FULL");
     transaction.put(
         TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
     transaction.commit();
@@ -134,6 +159,7 @@ public void downgradeToPartialFlatDbMode(
       final SegmentedKeyValueStorage composedWorldStateStorage) {
     final SegmentedKeyValueStorageTransaction transaction =
         composedWorldStateStorage.startTransaction();
+    LOG.info("setting FlatDbStrategy to PARTIAL");
     transaction.put(
         TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe());
     transaction.commit();
diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java
index 12be8fc471db..a14f86bbb5f6 100644
--- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java
+++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java
@@ -38,6 +38,13 @@ public interface DataStorageConfiguration {
           .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
           .build();
 
+  DataStorageConfiguration DEFAULT_BONSAI_PARTIAL_DB_CONFIG =
+      ImmutableDataStorageConfiguration.builder()
+          .dataStorageFormat(DataStorageFormat.BONSAI)
+          .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
+          .unstable(Unstable.DEFAULT_PARTIAL)
+          .build();
+
   DataStorageConfiguration DEFAULT_FOREST_CONFIG =
       ImmutableDataStorageConfiguration.builder()
           .dataStorageFormat(DataStorageFormat.FOREST)
@@ -65,11 +72,15 @@ interface Unstable {
     boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false;
     long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
     int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000;
-    boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = false;
+    boolean DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED = true;
+    boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = true;
 
     DataStorageConfiguration.Unstable DEFAULT =
         ImmutableDataStorageConfiguration.Unstable.builder().build();
 
+    DataStorageConfiguration.Unstable DEFAULT_PARTIAL =
+        ImmutableDataStorageConfiguration.Unstable.builder().bonsaiFullFlatDbEnabled(false).build();
+
     @Value.Default
     default boolean getBonsaiLimitTrieLogsEnabled() {
       return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
@@ -80,6 +91,11 @@ default int getBonsaiTrieLogPruningWindowSize() {
       return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
     }
 
+    @Value.Default
+    default boolean getBonsaiFullFlatDbEnabled() {
+      return DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
+    }
+
     @Value.Default
     default boolean getBonsaiCodeStoredByCodeHashEnabled() {
       return DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java
index 2e1dd913a4dd..17fab89676a0 100644
--- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java
+++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java
@@ -152,8 +152,11 @@ public abstract class AbstractIsolationTests {
   @BeforeEach
   public void createStorage() {
     worldStateKeyValueStorage =
+        // FYI: BonsaiSnapshoIsolationTests  work with frozen/cached worldstates, using PARTIAL
+        // flat db strategy allows the tests to make account assertions based on trie
+        // (whereas a full db strategy will not, since the worldstates are frozen/cached)
         createKeyValueStorageProvider()
-            .createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_CONFIG);
+            .createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_PARTIAL_DB_CONFIG);
     archive =
         new BonsaiWorldStateProvider(
             (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage,
diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java
index c3d7da9fe772..02818407b465 100644
--- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java
+++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java
@@ -45,7 +45,6 @@
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
-import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -79,18 +78,19 @@ public static Collection<Object[]> flatDbModeAndCodeStorageMode() {
 
   BonsaiWorldStateKeyValueStorage storage;
 
-  public void setUp(final FlatDbMode flatDbMode) {
-    storage = emptyStorage();
-    if (flatDbMode.equals(FlatDbMode.FULL)) {
-      storage.upgradeToFullFlatDbMode();
-    }
+  public BonsaiWorldStateKeyValueStorage setUp(final FlatDbMode flatDbMode) {
+    return setUp(flatDbMode, false);
   }
 
-  public void setUp(final FlatDbMode flatDbMode, final boolean useCodeHashStorage) {
+  public BonsaiWorldStateKeyValueStorage setUp(
+      final FlatDbMode flatDbMode, final boolean useCodeHashStorage) {
     storage = emptyStorage(useCodeHashStorage);
     if (flatDbMode.equals(FlatDbMode.FULL)) {
       storage.upgradeToFullFlatDbMode();
+    } else if (flatDbMode.equals(FlatDbMode.PARTIAL)) {
+      storage.downgradeToPartialFlatDbMode();
     }
+    return storage;
   }
 
   @ParameterizedTest
@@ -215,9 +215,8 @@ void getAccountStorageTrieNode_saveAndGetRegularValue(final FlatDbMode flatDbMod
   @ParameterizedTest
   @MethodSource("flatDbMode")
   void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode(final FlatDbMode flatDbMode) {
-    setUp(flatDbMode);
     Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL);
-    final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
+    final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
     final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
     final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
 
@@ -233,8 +232,8 @@ void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode(final FlatDbMode flatD
     updater.commit();
 
     // remove flat database
+    storage.downgradeToPartialFlatDbMode();
     storage.clearFlatDatabase();
-
     storage.upgradeToFullFlatDbMode();
 
     Mockito.reset(storage);
@@ -247,9 +246,8 @@ void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode(final FlatDbMode flatD
   @ParameterizedTest
   @MethodSource("flatDbMode")
   void getAccount_loadFromTrieWhenEmptyAndFlatDbPartialMode(final FlatDbMode flatDbMode) {
-    setUp(flatDbMode);
     Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL);
-    final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
+    final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
     final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
     final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
     final TreeMap<Bytes32, Bytes> accounts =
@@ -277,9 +275,8 @@ void getAccount_loadFromTrieWhenEmptyAndFlatDbPartialMode(final FlatDbMode flatD
   @ParameterizedTest
   @MethodSource("flatDbMode")
   void shouldUsePartialDBStrategyAfterDowngradingMode(final FlatDbMode flatDbMode) {
-    setUp(flatDbMode);
     Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL);
-    final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
+    final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
     final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
     final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
     final TreeMap<Bytes32, Bytes> accounts =
@@ -309,9 +306,9 @@ void shouldUsePartialDBStrategyAfterDowngradingMode(final FlatDbMode flatDbMode)
   @ParameterizedTest
   @MethodSource("flatDbMode")
   void getStorage_loadFromTrieWhenEmptyWithPartialMode(final FlatDbMode flatDbMode) {
-    setUp(flatDbMode);
+    final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
     Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL);
-    final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
+
     final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
     final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
     final TreeMap<Bytes32, Bytes> accounts =
@@ -359,9 +356,8 @@ void getStorage_loadFromTrieWhenEmptyWithPartialMode(final FlatDbMode flatDbMode
   @ParameterizedTest
   @MethodSource("flatDbMode")
   void getStorage_loadFromTrieWhenEmptyWithFullMode(final FlatDbMode flatDbMode) {
-    setUp(flatDbMode);
     Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL);
-    final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
+    final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
     storage.upgradeToFullFlatDbMode();
     final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage);
     final MerkleTrie<Bytes, Bytes> trie = TrieGenerator.generateTrie(coordinator, 1);
@@ -380,8 +376,7 @@ void getStorage_loadFromTrieWhenEmptyWithFullMode(final FlatDbMode flatDbMode) {
   @ParameterizedTest
   @MethodSource("flatDbMode")
   void clear_reloadFlatDbStrategy(final FlatDbMode flatDbMode) {
-    setUp(flatDbMode);
-    final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage());
+    final BonsaiWorldStateKeyValueStorage storage = spy(setUp(flatDbMode));
 
     // save world state root hash
     final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater();
@@ -489,7 +484,7 @@ private BonsaiWorldStateKeyValueStorage emptyStorage(final boolean useCodeHashSt
   void successfulPruneReturnsTrue() {
     final KeyValueStorage mockTrieLogStorage = mock(KeyValueStorage.class);
     when(mockTrieLogStorage.tryDelete(any())).thenReturn(true);
-    final BonsaiWorldStateKeyValueStorage storage = setupMockStorage(mockTrieLogStorage);
+    final BonsaiWorldStateKeyValueStorage storage = setupSpyStorage(mockTrieLogStorage);
     assertThat(storage.pruneTrieLog(Hash.ZERO)).isTrue();
   }
 
@@ -497,7 +492,7 @@ void successfulPruneReturnsTrue() {
   void failedPruneReturnsFalse() {
     final KeyValueStorage mockTrieLogStorage = mock(KeyValueStorage.class);
     when(mockTrieLogStorage.tryDelete(any())).thenReturn(false);
-    final BonsaiWorldStateKeyValueStorage storage = setupMockStorage(mockTrieLogStorage);
+    final BonsaiWorldStateKeyValueStorage storage = setupSpyStorage(mockTrieLogStorage);
     assertThat(storage.pruneTrieLog(Hash.ZERO)).isFalse();
   }
 
@@ -505,18 +500,17 @@ void failedPruneReturnsFalse() {
   void exceptionalPruneReturnsFalse() {
     final KeyValueStorage mockTrieLogStorage = mock(KeyValueStorage.class);
     when(mockTrieLogStorage.tryDelete(any())).thenThrow(new RuntimeException("test exception"));
-    final BonsaiWorldStateKeyValueStorage storage = setupMockStorage(mockTrieLogStorage);
+    final BonsaiWorldStateKeyValueStorage storage = setupSpyStorage(mockTrieLogStorage);
     assertThat(storage.pruneTrieLog(Hash.ZERO)).isFalse();
   }
 
-  private BonsaiWorldStateKeyValueStorage setupMockStorage(
+  private BonsaiWorldStateKeyValueStorage setupSpyStorage(
       final KeyValueStorage mockTrieLogStorage) {
-    final StorageProvider mockStorageProvider = mock(StorageProvider.class);
+    final StorageProvider mockStorageProvider = spy(new InMemoryKeyValueStorageProvider());
     when(mockStorageProvider.getStorageBySegmentIdentifier(
             KeyValueSegmentIdentifier.TRIE_LOG_STORAGE))
         .thenReturn(mockTrieLogStorage);
-    when(mockStorageProvider.getStorageBySegmentIdentifiers(any()))
-        .thenReturn(mock(SegmentedKeyValueStorage.class));
+
     return new BonsaiWorldStateKeyValueStorage(
         mockStorageProvider,
         new NoOpMetricsSystem(),
diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java
index 7e0d27250685..b3c709b89f2c 100644
--- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java
+++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java
@@ -63,7 +63,7 @@ void loadsFlatDbStrategyForStoredFlatDbMode(final FlatDbMode flatDbMode) {
   @Test
   void loadsPartialFlatDbStrategyWhenNoFlatDbModeStored() {
     flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
-    assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(FlatDbMode.PARTIAL);
+    assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(FlatDbMode.FULL);
   }
 
   @Test
@@ -76,7 +76,7 @@ void upgradesFlatDbStrategyToFullFlatDbMode() {
     assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage))
         .isInstanceOf(FullFlatDbStrategy.class);
     assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
-        .isInstanceOf(AccountHashCodeStorageStrategy.class);
+        .isInstanceOf(CodeHashCodeStorageStrategy.class);
   }
 
   @ParameterizedTest
@@ -99,7 +99,7 @@ void emptyDbCreatesFlatDbStrategyUsingCodeByHashConfig(final boolean codeByHashE
         codeByHashEnabled
             ? CodeHashCodeStorageStrategy.class
             : AccountHashCodeStorageStrategy.class;
-    assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL);
+    assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL);
     assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
         .isInstanceOf(expectedCodeStorageClass);
   }
@@ -129,7 +129,7 @@ void existingAccountHashDbUsesAccountHash(final boolean codeByHashEnabled) {
     transaction.commit();
 
     flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
-    assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL);
+    assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL);
     assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
         .isInstanceOf(AccountHashCodeStorageStrategy.class);
   }
@@ -158,7 +158,7 @@ void existingCodeHashDbUsesCodeHash(final boolean codeByHashEnabled) {
     transaction.commit();
 
     flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
-    assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL);
+    assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL);
     assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
         .isInstanceOf(CodeHashCodeStorageStrategy.class);
   }
diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java
index 5d96a447376a..669287cfef2a 100644
--- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java
+++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java
@@ -36,8 +36,6 @@ public class SnapSyncConfiguration {
   public static final int DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST =
       1024; // The default number of flat slots entries to verify and heal per request.
 
-  public static final Boolean DEFAULT_IS_FLAT_DB_HEALING_ENABLED = Boolean.FALSE;
-
   public static final Boolean DEFAULT_SNAP_SERVER_ENABLED = Boolean.FALSE;
 
   public static SnapSyncConfiguration getDefault() {
@@ -79,11 +77,6 @@ public int getLocalFlatStorageCountToHealPerRequest() {
     return DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST;
   }
 
-  @Value.Default
-  public Boolean isFlatDbHealingEnabled() {
-    return DEFAULT_IS_FLAT_DB_HEALING_ENABLED;
-  }
-
   @Value.Default
   public Boolean isSnapServerEnabled() {
     return DEFAULT_SNAP_SERVER_ENABLED;
diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java
index 9513f80bb174..9ae49b93c77e 100644
--- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java
+++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java
@@ -184,16 +184,6 @@ public CompletableFuture<Void> run(
       } else {
         // start from scratch
         worldStateStorageCoordinator.clear();
-        // we have to upgrade to full flat db mode if we are in bonsai mode
-        if (snapSyncConfiguration.isFlatDbHealingEnabled()) {
-          worldStateStorageCoordinator.applyOnMatchingStrategy(
-              DataStorageFormat.BONSAI,
-              strategy -> {
-                BonsaiWorldStateKeyValueStorage onBonsai =
-                    (BonsaiWorldStateKeyValueStorage) strategy;
-                onBonsai.upgradeToFullFlatDbMode();
-              });
-        }
         ranges.forEach(
             (key, value) ->
                 newDownloadState.enqueueRequest(
diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java
index e0442d16940c..28298acb9c36 100644
--- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java
+++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java
@@ -118,7 +118,7 @@ public void setUp(final DataStorageFormat storageFormat) {
           new BonsaiWorldStateKeyValueStorage(
               new InMemoryKeyValueStorageProvider(),
               new NoOpMetricsSystem(),
-              DataStorageConfiguration.DEFAULT_BONSAI_CONFIG);
+              DataStorageConfiguration.DEFAULT_BONSAI_PARTIAL_DB_CONFIG);
     } else {
       worldStateKeyValueStorage =
           new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage());