Change write json files to disk strategy

We have written all files each time at each new block which accumulated to about
1 GB of data, this led to stability issues and very high disk IO for the explorer nodes.

Now we write only the data of the new block. We also remove the deletion of all
files at startup.
The dao state is still written in a monolithic file so that cannot be optimized
but we added a new directly where we add each block named by block height.
Looking up the latest written file tells us our last persisted block.
So all that data is equivalent to the monolithic daoState data which should
be removed once the webapp devs have implemented the change to use the blocks directory.

common/src/main/java/bisq/common/file/JsonFileManager.java

@@ -24,13 +24,26 @@
 import java.io.File;
 import java.io.PrintWriter;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.ThreadPoolExecutor;
 
 import lombok.extern.slf4j.Slf4j;
 
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
 @Slf4j
 public class JsonFileManager {
-    private final ThreadPoolExecutor executor;
+    private final static List<JsonFileManager> INSTANCES = new ArrayList<>();
+
+    public static void shutDownAllInstances() {
+        INSTANCES.forEach(JsonFileManager::shutDown);
+    }
+
+
+    @Nullable
+    private ThreadPoolExecutor executor;
     private final File dir;
 
 
@@ -41,54 +54,62 @@ public class JsonFileManager {
     public JsonFileManager(File dir) {
         this.dir = dir;
 
-        this.executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60);
+        if (!dir.exists() && !dir.mkdir()) {
+            log.warn("make dir failed");
+        }
 
-        if (!dir.exists())
-            if (!dir.mkdir())
-                log.warn("make dir failed");
+        INSTANCES.add(this);
+    }
 
-        Runtime.getRuntime().addShutdownHook(new Thread(JsonFileManager.this::shutDown,
-                "JsonFileManager.ShutDownHook"));
+    @NotNull
+    protected ThreadPoolExecutor getExecutor() {
+        if (executor == null) {
+            executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60);
+        }
+        return executor;
     }
 
     public void shutDown() {
-        executor.shutdown();
+        if (executor != null) {
+            executor.shutdown();
+        }
     }
 
-    public void writeToDisc(String json, String fileName) {
-        executor.execute(() -> {
-            File jsonFile = new File(Paths.get(dir.getAbsolutePath(), fileName + ".json").toString());
-            File tempFile = null;
-            PrintWriter printWriter = null;
-            try {
-                tempFile = File.createTempFile("temp", null, dir);
-                if (!executor.isShutdown() && !executor.isTerminated() && !executor.isTerminating())
-                    tempFile.deleteOnExit();
-
-                printWriter = new PrintWriter(tempFile);
-                printWriter.println(json);
-
-                // This close call and comment is borrowed from FileManager. Not 100% sure it that is really needed but
-                // seems that had fixed in the past and we got reported issues on Windows so that fix might be still
-                // required.
-                // Close resources before replacing file with temp file because otherwise it causes problems on windows
-                // when rename temp file
-                printWriter.close();
+    public void writeToDiscThreaded(String json, String fileName) {
+        getExecutor().execute(() -> writeToDisc(json, fileName));
+    }
 
-                FileUtil.renameFile(tempFile, jsonFile);
-            } catch (Throwable t) {
-                log.error("storageFile " + jsonFile.toString());
-                t.printStackTrace();
-            } finally {
-                if (tempFile != null && tempFile.exists()) {
-                    log.warn("Temp file still exists after failed save. We will delete it now. storageFile=" + fileName);
-                    if (!tempFile.delete())
-                        log.error("Cannot delete temp file.");
-                }
-
-                if (printWriter != null)
-                    printWriter.close();
+    public void writeToDisc(String json, String fileName) {
+        File jsonFile = new File(Paths.get(dir.getAbsolutePath(), fileName + ".json").toString());
+        File tempFile = null;
+        PrintWriter printWriter = null;
+        try {
+            tempFile = File.createTempFile("temp", null, dir);
+            tempFile.deleteOnExit();
+
+            printWriter = new PrintWriter(tempFile);
+            printWriter.println(json);
+
+            // This close call and comment is borrowed from FileManager. Not 100% sure it that is really needed but
+            // seems that had fixed in the past and we got reported issues on Windows so that fix might be still
+            // required.
+            // Close resources before replacing file with temp file because otherwise it causes problems on windows
+            // when rename temp file
+            printWriter.close();
+
+            FileUtil.renameFile(tempFile, jsonFile);
+        } catch (Throwable t) {
+            log.error("storageFile " + jsonFile.toString());
+            t.printStackTrace();
+        } finally {
+            if (tempFile != null && tempFile.exists()) {
+                log.warn("Temp file still exists after failed save. We will delete it now. storageFile=" + fileName);
+                if (!tempFile.delete())
+                    log.error("Cannot delete temp file.");
             }
-        });
+
+            if (printWriter != null)
+                printWriter.close();
+        }
     }
 }

core/src/main/java/bisq/core/app/misc/ExecutableForAppWithP2p.java

@@ -21,6 +21,7 @@
 import bisq.core.btc.setup.WalletsSetup;
 import bisq.core.btc.wallet.BsqWalletService;
 import bisq.core.btc.wallet.BtcWalletService;
+import bisq.core.dao.DaoSetup;
 import bisq.core.offer.OpenOfferManager;
 import bisq.core.support.dispute.arbitration.arbitrator.ArbitratorManager;
 
@@ -31,6 +32,7 @@
 import bisq.common.UserThread;
 import bisq.common.app.DevEnv;
 import bisq.common.config.Config;
+import bisq.common.file.JsonFileManager;
 import bisq.common.handlers.ResultHandler;
 import bisq.common.persistence.PersistenceManager;
 import bisq.common.setup.GracefulShutDownHandler;
@@ -83,6 +85,8 @@ public void gracefulShutDown(ResultHandler resultHandler) {
         log.info("gracefulShutDown");
         try {
             if (injector != null) {
+                JsonFileManager.shutDownAllInstances();
+                injector.getInstance(DaoSetup.class).shutDown();
                 injector.getInstance(ArbitratorManager.class).shutDown();
                 injector.getInstance(OpenOfferManager.class).shutDown(() -> injector.getInstance(P2PService.class).shutDown(() -> {
                     injector.getInstance(WalletsSetup.class).shutDownComplete.addListener((ov, o, n) -> {

core/src/main/java/bisq/core/dao/node/BsqNode.java

@@ -209,7 +209,7 @@ protected void onParseBlockChainComplete() {
         parseBlockchainComplete = true;
         daoStateService.onParseBlockChainComplete();
 
-        maybeExportToJson();
+        exportJsonFilesService.onParseBlockChainComplete();
     }
 
     @SuppressWarnings("WeakerAccess")
@@ -291,7 +291,7 @@ protected Optional<Block> doParseBlock(RawBlock rawBlock) throws RequiredReorgFr
         return Optional.empty();
     }
 
-    protected void maybeExportToJson() {
-        exportJsonFilesService.maybeExportToJson();
+    protected void maybeExportNewBlockToJson(Block block) {
+        exportJsonFilesService.onNewBlock(block);
     }
 }