Merge pull request #6 from soundvibe/bloom-filter-io

Metadata write performance improvements

README.md

@@ -97,12 +97,12 @@ PalDB is available on Maven Central, hence just add the following dependency:
 <dependency>
     <groupId>net.soundvibe</groupId>
     <artifactId>paldb</artifactId>
-    <version>2.1.1</version>
+    <version>2.1.2</version>
 </dependency>
 ```
 Scala SBT
 ```
-libraryDependencies += "net.soundvibe" % "paldb" % "2.1.1"
+libraryDependencies += "net.soundvibe" % "paldb" % "2.1.2"
 ```
 
 

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>net.soundvibe</groupId>
     <artifactId>paldb</artifactId>
-    <version>2.1.1</version>
+    <version>2.1.2</version>
     <packaging>jar</packaging>
     <name>paldb</name>
     <description>Embeddable persistent key-value store</description>
@@ -69,7 +69,7 @@
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
-            <version>5.4.2</version>
+            <version>5.6.0</version>
             <scope>test</scope>
         </dependency>
 
@@ -147,7 +147,7 @@
 
             <plugin>
                 <artifactId>maven-source-plugin</artifactId>
-                <version>3.1.0</version>
+                <version>3.2.1</version>
                 <executions>
                     <execution>
                         <id>attach-sources</id>

src/main/java/com/linkedin/paldb/impl/StorageWriter.java

@@ -187,8 +187,7 @@ public void close() throws IOException {
       //Write metadata file
       File metadataFile = new File(tempFolder, "metadata.dat");
       metadataFile.deleteOnExit();
-
-      try (var metadataOutputStream = new RandomAccessFile(metadataFile, "rw")) {
+      try (var metadataOutputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(metadataFile)))) {
         writeMetadata(metadataOutputStream, bloomFilter);
       }
 
@@ -213,7 +212,7 @@ public void close() throws IOException {
     }
   }
 
-  private void writeMetadata(RandomAccessFile dataOutputStream, BloomFilter bloomFilter) throws IOException {
+  private void writeMetadata(DataOutputStream dataOutputStream, BloomFilter bloomFilter) throws IOException {
     //Write format version
     dataOutputStream.writeUTF(FormatVersion.getLatestVersion().name());
 
@@ -227,19 +226,24 @@ private void writeMetadata(RandomAccessFile dataOutputStream, BloomFilter bloomF
     //Write size (number of keys)
     dataOutputStream.writeLong(keyCount);
 
-    //write bloom filter bit size
-    dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.bitSize() : 0);
-    //write bloom filter long array size
-    dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.bits().length : 0);
-    //write bloom filter hash functions
-    dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.hashFunctions() : 0);
-    //write bloom filter bits
     if (bloomFilter != null) {
+      //write bloom filter bit size
+      dataOutputStream.writeInt(bloomFilter.bitSize());
+      //write bloom filter bits array size
+      dataOutputStream.writeInt(bloomFilter.bits().length);
+      //write bloom filter hash functions
+      dataOutputStream.writeInt(bloomFilter.hashFunctions());
+      //write bloom filter bits
       for (final long bit : bloomFilter.bits()) {
         dataOutputStream.writeLong(bit);
       }
+    } else {
+      dataOutputStream.writeInt(0);
+      //write bloom filter long array size
+      dataOutputStream.writeInt(0);
+      //write bloom filter hash functions
+      dataOutputStream.writeInt(0);
     }
-
     //Write the number of different key length
     dataOutputStream.writeInt(keyLengthCount);
 
@@ -280,7 +284,7 @@ private void writeMetadata(RandomAccessFile dataOutputStream, BloomFilter bloomF
     }
 
     //Write the position of the index and the data
-    long indexOffset = dataOutputStream.getFilePointer() + (Long.SIZE / Byte.SIZE) + (Long.SIZE / Byte.SIZE);
+    long indexOffset = (long) dataOutputStream.size() + (Long.SIZE / Byte.SIZE) + (Long.SIZE / Byte.SIZE);
     dataOutputStream.writeLong(indexOffset);
     //data offset
     dataOutputStream.writeLong(indexOffset + indexesLength);

src/test/java/com/linkedin/paldb/impl/TestStore.java

@@ -631,6 +631,22 @@ void should_not_find_when_bloom_filter_enabled() {
     }
   }
 
+  @Test
+  void should_write_huge_bloom_filter() {
+    var config = PalDBConfigBuilder.<String,byte[]>create()
+            .withEnableBloomFilter(true).build();
+    try (StoreWriter<String,byte[]> writer = PalDB.createWriter(storeFile, config)) {
+      for (int i = 0; i < 10_000_000; i++) {
+        writer.put(String.valueOf(i), EMPTY_VALUE);
+      }
+    }
+
+    try (StoreReader<String,byte[]> reader = PalDB.createReader(storeFile, config)) {
+      assertNull(reader.get("foo"));
+      assertArrayEquals(EMPTY_VALUE, reader.get("100"));
+    }
+  }
+
   private static final byte[] EMPTY_VALUE = new byte[0];
 
   @Test