Start implementation of CSV result compression

dice-group · Jan 22, 2025 · 3a293d5 · 3a293d5
1 parent 0c171e4
commit 3a293d5
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 2 deletions.
diff --git a/src/main/java/org/aksw/iguana/cc/storage/impl/CSVStorage.java b/src/main/java/org/aksw/iguana/cc/storage/impl/CSVStorage.java
@@ -32,8 +32,12 @@ public class CSVStorage implements Storage {
     /** This private record is used to store information about the connections used in a task. */
     private record ConnectionInfo(String connection, String version, String dataset) {}
 
-    public record Config(String directory) implements StorageConfig {
+    public record Config(String directory, Integer compressionLevel) implements StorageConfig {
         public Config(String directory) {
+            this(directory, null);
+        }
+
+        public Config(String directory, Integer compressionLevel) {
             if (directory == null) {
                 directory = "results";
             }
@@ -42,6 +46,7 @@ public Config(String directory) {
                 throw new IllegalArgumentException("The given path is not a directory.");
             }
             this.directory = directory;
+            this.compressionLevel = 3;
         }
     }
 
@@ -58,8 +63,14 @@ public Config(String directory) {
     private Resource taskRes;
     List<ConnectionInfo> connections;
 
+    private Integer compressionLevel = 3;
+
     public CSVStorage(Config config, List<Metric> metrics, String suiteID) {
         this(config.directory(), metrics, suiteID);
+        if (config.compressionLevel() != null && (config.compressionLevel() < 1 || config.compressionLevel() > 19)) {
+            throw new IllegalArgumentException("Compression level must be between 1 and 19.");
+        }
+        this.compressionLevel = 3;
     }
 
     public CSVStorage(String folderPath, List<Metric> metrics, String suiteID) {
@@ -230,6 +241,29 @@ public void storeData(Storable data) {
         }
     }
 
+    @Override
+    public void close() {
+        final var temp = this.suiteFolder.getParent().relativize(this.suiteFolder);
+        LOGGER.info("Compressing the suite folder.");
+        LOGGER.info(String.join(" ", "tar", "-I", String.format("\"zstd -%d\"", compressionLevel), "-cf", temp + ".tar.zst", temp.toString()));
+        LOGGER.info(this.suiteFolder.getParent().toFile().toString());
+        if (this.compressionLevel != null) {
+            try {
+                final var process = new ProcessBuilder("tar", "-I", String.format("\"zstd -%d\"", compressionLevel), "-cf", temp + ".tar.zst", temp.toString())
+                        .directory(this.suiteFolder.getParent().toFile().getAbsoluteFile())
+                        .start();
+                try {
+                    LOGGER.info("Waiting for the compression process to finish.");
+                    process.waitFor();
+                    LOGGER.info(Arrays.toString(process.getInputStream().readAllBytes()));
+                    LOGGER.info("Compression process finished. Fuck you");
+                } catch (InterruptedException ignored) {} // ignore interruption
+            } catch (IOException e) {
+                LOGGER.error("Error while compressing the suite folder.", e);
+            }
+        }
+    }
+
     /**
      * This method sets the objects attributes by querying the given model.
      *

diff --git a/src/main/java/org/aksw/iguana/cc/storage/impl/RDFFileStorage.java b/src/main/java/org/aksw/iguana/cc/storage/impl/RDFFileStorage.java
@@ -131,7 +131,7 @@ private OutputStream getFileOutputstream() throws IOException {
             return outputStream;
         }
 
-        if (compression) {
+        if (compression) { // TODO: check if process closes properly
             final var process = new ProcessBuilder("zstd", "-o", path.toString() + ".zstd", "-T0", "-" + compressionLevel, "-q", "-").start();
             outputStream = process.getOutputStream();
             return outputStream;