diff --git a/README.md b/README.md index 8071d7b..e6bce9c 100644 --- a/README.md +++ b/README.md @@ -97,12 +97,12 @@ PalDB is available on Maven Central, hence just add the following dependency: net.soundvibe paldb - 2.1.1 + 2.1.2 ``` Scala SBT ``` -libraryDependencies += "net.soundvibe" % "paldb" % "2.1.1" +libraryDependencies += "net.soundvibe" % "paldb" % "2.1.2" ``` diff --git a/pom.xml b/pom.xml index c7f8a9c..7e7b878 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ net.soundvibe paldb - 2.1.1 + 2.1.2 jar paldb Embeddable persistent key-value store @@ -69,7 +69,7 @@ org.junit.jupiter junit-jupiter - 5.4.2 + 5.6.0 test @@ -147,7 +147,7 @@ maven-source-plugin - 3.1.0 + 3.2.1 attach-sources diff --git a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index 0ac4b46..ffa2ec4 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -187,8 +187,7 @@ public void close() throws IOException { //Write metadata file File metadataFile = new File(tempFolder, "metadata.dat"); metadataFile.deleteOnExit(); - - try (var metadataOutputStream = new RandomAccessFile(metadataFile, "rw")) { + try (var metadataOutputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(metadataFile)))) { writeMetadata(metadataOutputStream, bloomFilter); } @@ -213,7 +212,7 @@ public void close() throws IOException { } } - private void writeMetadata(RandomAccessFile dataOutputStream, BloomFilter bloomFilter) throws IOException { + private void writeMetadata(DataOutputStream dataOutputStream, BloomFilter bloomFilter) throws IOException { //Write format version dataOutputStream.writeUTF(FormatVersion.getLatestVersion().name()); @@ -227,19 +226,24 @@ private void writeMetadata(RandomAccessFile dataOutputStream, BloomFilter bloomF //Write size (number of keys) dataOutputStream.writeLong(keyCount); - //write bloom filter bit size - dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.bitSize() : 0); - //write bloom filter long array size - dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.bits().length : 0); - //write bloom filter hash functions - dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.hashFunctions() : 0); - //write bloom filter bits if (bloomFilter != null) { + //write bloom filter bit size + dataOutputStream.writeInt(bloomFilter.bitSize()); + //write bloom filter bits array size + dataOutputStream.writeInt(bloomFilter.bits().length); + //write bloom filter hash functions + dataOutputStream.writeInt(bloomFilter.hashFunctions()); + //write bloom filter bits for (final long bit : bloomFilter.bits()) { dataOutputStream.writeLong(bit); } + } else { + dataOutputStream.writeInt(0); + //write bloom filter long array size + dataOutputStream.writeInt(0); + //write bloom filter hash functions + dataOutputStream.writeInt(0); } - //Write the number of different key length dataOutputStream.writeInt(keyLengthCount); @@ -280,7 +284,7 @@ private void writeMetadata(RandomAccessFile dataOutputStream, BloomFilter bloomF } //Write the position of the index and the data - long indexOffset = dataOutputStream.getFilePointer() + (Long.SIZE / Byte.SIZE) + (Long.SIZE / Byte.SIZE); + long indexOffset = (long) dataOutputStream.size() + (Long.SIZE / Byte.SIZE) + (Long.SIZE / Byte.SIZE); dataOutputStream.writeLong(indexOffset); //data offset dataOutputStream.writeLong(indexOffset + indexesLength); diff --git a/src/test/java/com/linkedin/paldb/impl/TestStore.java b/src/test/java/com/linkedin/paldb/impl/TestStore.java index 0edf053..ff8a00c 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -631,6 +631,22 @@ void should_not_find_when_bloom_filter_enabled() { } } + @Test + void should_write_huge_bloom_filter() { + var config = PalDBConfigBuilder.create() + .withEnableBloomFilter(true).build(); + try (StoreWriter writer = PalDB.createWriter(storeFile, config)) { + for (int i = 0; i < 10_000_000; i++) { + writer.put(String.valueOf(i), EMPTY_VALUE); + } + } + + try (StoreReader reader = PalDB.createReader(storeFile, config)) { + assertNull(reader.get("foo")); + assertArrayEquals(EMPTY_VALUE, reader.get("100")); + } + } + private static final byte[] EMPTY_VALUE = new byte[0]; @Test