From a14e410a50d2ebd9f219d765b4f6dc89495ec392 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Fri, 22 Nov 2019 11:29:36 +0200 Subject: [PATCH 1/5] Index now supports adding more keys than max integer value. --- pom.xml | 2 +- .../linkedin/paldb/impl/StorageReader.java | 91 +++++++--- .../linkedin/paldb/impl/StorageWriter.java | 161 +++++++++++++----- .../com/linkedin/paldb/utils/BloomFilter.java | 4 +- .../com/linkedin/paldb/impl/TestStore.java | 46 +++++ 5 files changed, 234 insertions(+), 70 deletions(-) diff --git a/pom.xml b/pom.xml index 53621ed..2045845 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,7 @@ org.rocksdb rocksdbjni - 4.0 + 6.2.2 test diff --git a/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/src/main/java/com/linkedin/paldb/impl/StorageReader.java index 933c9db..cf96363 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -37,15 +37,15 @@ public class StorageReader implements Iterable> { // Buffer segment size private final long segmentSize; // Number of keys in the index - private final int keyCount; + private final long keyCount; // Key count for each key length - private final int[] keyCounts; + private final long[] keyCounts; // Slot size for each key length private final int[] slotSizes; // Number of slots for each key length - private final int[] slots; + private final long[] slots; // Offset of the index for different key length - private final int[] indexOffsets; + private final long[] indexOffsets; // Offset of the data in the channel private final long dataOffset; // Offset of the data for different key length @@ -59,7 +59,7 @@ public class StorageReader implements Iterable> { private final boolean mMapData; // Buffers private final BloomFilter bloomFilter; - private MappedByteBuffer indexBuffer; + private MappedByteBuffer[] indexBuffers; private MappedByteBuffer[] dataBuffers; StorageReader(Configuration configuration, File file) throws IOException { @@ -84,7 +84,7 @@ public class StorageReader implements Iterable> { FormatVersion formatVersion = null; // Offset of the index in the channel - int indexOffset; + long indexOffset; int bloomFilterBitSize = 0; int bloomFilterHashFunctions = 0; @@ -120,7 +120,7 @@ public class StorageReader implements Iterable> { createdAt = dataInputStream.readLong(); //Metadata counters - keyCount = dataInputStream.readInt(); + keyCount = dataInputStream.readLong(); //read bloom filter bit size bloomFilterBitSize = dataInputStream.readInt(); @@ -145,26 +145,26 @@ public class StorageReader implements Iterable> { final int maxKeyLength = dataInputStream.readInt(); //Read offset counts and keys - indexOffsets = new int[maxKeyLength + 1]; + indexOffsets = new long[maxKeyLength + 1]; dataOffsets = new long[maxKeyLength + 1]; - keyCounts = new int[maxKeyLength + 1]; - slots = new int[maxKeyLength + 1]; + keyCounts = new long[maxKeyLength + 1]; + slots = new long[maxKeyLength + 1]; slotSizes = new int[maxKeyLength + 1]; int mSlotSize = 0; for (int i = 0; i < keyLengthCount; i++) { int keyLength = dataInputStream.readInt(); - keyCounts[keyLength] = dataInputStream.readInt(); - slots[keyLength] = dataInputStream.readInt(); + keyCounts[keyLength] = dataInputStream.readLong(); + slots[keyLength] = dataInputStream.readLong(); slotSizes[keyLength] = dataInputStream.readInt(); - indexOffsets[keyLength] = dataInputStream.readInt(); + indexOffsets[keyLength] = dataInputStream.readLong(); dataOffsets[keyLength] = dataInputStream.readLong(); mSlotSize = Math.max(mSlotSize, slotSizes[keyLength]); } //Read index and data offset - indexOffset = dataInputStream.readInt() + ignoredBytes; + indexOffset = dataInputStream.readLong() + ignoredBytes; dataOffset = dataInputStream.readLong() + ignoredBytes; } //Close metadata @@ -179,7 +179,7 @@ public class StorageReader implements Iterable> { //Check if data size fits in memory map limit mMapData = configuration.getBoolean(Configuration.MMAP_DATA_ENABLED); - indexBuffer = initIndexBuffer(channel, indexOffset); + indexBuffers = initIndexBuffers(channel, indexOffset); dataBuffers = initDataBuffers(channel); //logging @@ -202,9 +202,18 @@ public class StorageReader implements Iterable> { } } - private MappedByteBuffer initIndexBuffer(FileChannel channel, int indexOffset) { + private MappedByteBuffer[] initIndexBuffers(FileChannel channel, long indexOffset) { + long indexSize = dataOffset - indexOffset; + int bufIdx = 0; + int bufArraySize = (int) (indexSize / segmentSize) + ((indexSize % segmentSize != 0) ? 1 : 0); + var result = new MappedByteBuffer[bufArraySize]; try { - return channel.map(FileChannel.MapMode.READ_ONLY, indexOffset, dataOffset - indexOffset); + for (long offset = 0; offset < indexSize; offset += segmentSize) { + long remainingFileSize = indexSize - offset; + long thisSegmentSize = Math.min(segmentSize, remainingFileSize); + result[bufIdx++] = channel.map(FileChannel.MapMode.READ_ONLY, indexOffset + offset, thisSegmentSize); + } + return result; } catch (IOException e) { throw new UncheckedIOException(e); } @@ -237,15 +246,15 @@ public byte[] get(byte[] key) throws IOException { return null; } - int numSlots = slots[keyLength]; + long numSlots = slots[keyLength]; int slotSize = slotSizes[keyLength]; - int ixOffset = indexOffsets[keyLength]; + long ixOffset = indexOffsets[keyLength]; long dtOffset = dataOffsets[keyLength]; long hash = Murmur3.hash(key); var slotBuffer = new byte[slotSize]; - for (int probe = 0; probe < numSlots; probe++) { - int slot = (int) ((hash + probe) % numSlots); - indexBuffer.get(ixOffset + slot * slotSize, slotBuffer, 0, slotSize); + for (long probe = 0; probe < numSlots; probe++) { + long slot = ((hash + probe) % numSlots); + getFromIndexBuffer(ixOffset + slot * slotSize, slotBuffer, slotSize); long offset = LongPacker.unpackLong(slotBuffer, keyLength); if (offset == 0L) { return null; @@ -257,6 +266,36 @@ public byte[] get(byte[] key) throws IOException { return null; } + private void getFromIndexBuffer(long offset, byte[] slotBuffer, int slotSize) { + int bufferIndex = (int) (offset / segmentSize); + var buf = indexBuffers[bufferIndex]; + var pos = (int) (offset % segmentSize); + + int remaining = remaining(buf, pos); + if (remaining < slotSize) { + int splitOffset = 0; + buf.get(pos, slotBuffer, 0, remaining); + buf = indexBuffers[++bufferIndex]; + int bytesLeft = slotSize - remaining; + splitOffset += remaining; + remaining = remaining(buf, 0); + + while (remaining < bytesLeft) { + buf.get(0, slotBuffer, splitOffset, remaining); + buf = indexBuffers[++bufferIndex]; + splitOffset += remaining; + bytesLeft -= remaining; + remaining = remaining(buf, 0); + } + + if (remaining > 0 && bytesLeft > 0) { + buf.get(0, slotBuffer, splitOffset, bytesLeft); + } + } else { + buf.get(pos, slotBuffer, 0, slotSize); + } + } + private boolean isKey(byte[] slotBuffer, byte[] key) { return Arrays.compare(slotBuffer, 0, key.length, key, 0, key.length) == 0; } @@ -265,12 +304,12 @@ private boolean isKey(byte[] slotBuffer, byte[] key) { public void close() throws IOException { channel.close(); mappedFile.close(); - indexBuffer = null; + indexBuffers = null; dataBuffers = null; System.gc(); //need to call gc because otherwise memory mapped file buffers won't close. See: https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4715154 } - public int getKeyCount() { + public long getKeyCount() { return keyCount; } @@ -389,7 +428,7 @@ private class StorageIterator implements Iterator> { private long keyIndex; private long keyLimit; private long currentDataOffset; - private int currentIndexOffset; + private long currentIndexOffset; public StorageIterator(boolean value) { @@ -421,7 +460,7 @@ public FastEntry next() { try { long offset = 0; while (offset == 0) { - indexBuffer.get(currentIndexOffset, currentSlotBuffer); + getFromIndexBuffer(currentIndexOffset, currentSlotBuffer, currentSlotBuffer.length); offset = LongPacker.unpackLong(currentSlotBuffer, currentKeyLength); currentIndexOffset += currentSlotBuffer.length; } diff --git a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index bb0701a..cfb421e 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -20,7 +20,7 @@ import org.slf4j.*; import java.io.*; -import java.nio.MappedByteBuffer; +import java.nio.*; import java.nio.channels.FileChannel; import java.text.DecimalFormat; import java.util.*; @@ -53,13 +53,15 @@ public class StorageWriter { // Max offset length private int[] maxOffsetLengths; // Number of keys - private int keyCount; - private int[] keyCounts; + private long keyCount; + private long[] keyCounts; // Number of values - private int valueCount; + private long valueCount; // Number of collisions private int collisions; + private final long segmentSize; + StorageWriter(Configuration configuration, OutputStream stream) { config = configuration; loadFactor = config.getDouble(Configuration.LOAD_FACTOR); @@ -80,11 +82,11 @@ public class StorageWriter { lastValuesLength = new int[0]; dataLengths = new long[0]; maxOffsetLengths = new int[0]; - keyCounts = new int[0]; + keyCounts = new long[0]; + segmentSize = config.getLong(Configuration.MMAP_SEGMENT_SIZE); } - public void put(byte[] key, byte[] value) - throws IOException { + public void put(byte[] key, byte[] value) throws IOException { int keyLength = key.length; //Get the Output stream for that keyLength, each key length has its own file @@ -104,8 +106,8 @@ public void put(byte[] key, byte[] value) } // Write offset and record max offset length - int offsetLength = LongPacker.packLong(indexStream, dataLength); - maxOffsetLengths[keyLength] = Math.max(offsetLength, maxOffsetLengths[keyLength]); + int offsetDataLength = LongPacker.packLong(indexStream, dataLength); + maxOffsetLengths[keyLength] = Math.max(offsetDataLength, maxOffsetLengths[keyLength]); // Write if data is not the same if (!sameValue) { @@ -137,7 +139,8 @@ public void close() throws IOException { dos.close(); } } - for (DataOutputStream dos : indexStreams) { + + for (var dos : indexStreams) { if (dos != null) { dos.close(); } @@ -165,9 +168,9 @@ public void close() throws IOException { //Write metadata file File metadataFile = new File(tempFolder, "metadata.dat"); metadataFile.deleteOnExit(); - try (FileOutputStream metadataOutputStream = new FileOutputStream(metadataFile); - DataOutputStream metadataDataOutputStream = new DataOutputStream(metadataOutputStream)) { - writeMetadata(metadataDataOutputStream, bloomFilter); + + try (var metadataOutputStream = new RandomAccessFile(metadataFile, "rw")) { + writeMetadata(metadataOutputStream, bloomFilter); } filesToMerge.add(0, metadataFile); @@ -192,7 +195,7 @@ public void close() throws IOException { } } - private void writeMetadata(DataOutputStream dataOutputStream, BloomFilter bloomFilter) throws IOException { + private void writeMetadata(RandomAccessFile dataOutputStream, BloomFilter bloomFilter) throws IOException { //Write format version dataOutputStream.writeUTF(FormatVersion.getLatestVersion().name()); @@ -204,7 +207,7 @@ private void writeMetadata(DataOutputStream dataOutputStream, BloomFilter bloomF int maxKeyLength = keyCounts.length - 1; //Write size (number of keys) - dataOutputStream.writeInt(keyCount); + dataOutputStream.writeLong(keyCount); //write bloom filter bit size dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.bitSize() : 0); @@ -233,18 +236,18 @@ private void writeMetadata(DataOutputStream dataOutputStream, BloomFilter bloomF dataOutputStream.writeInt(i); // Write key count - dataOutputStream.writeInt(keyCounts[i]); + dataOutputStream.writeLong(keyCounts[i]); // Write slot count - int slots = (int) Math.round(keyCounts[i] / loadFactor); - dataOutputStream.writeInt(slots); + long slots = Math.round(keyCounts[i] / loadFactor); + dataOutputStream.writeLong(slots); // Write slot size int offsetLength = maxOffsetLengths[i]; dataOutputStream.writeInt(i + offsetLength); // Write index offset - dataOutputStream.writeInt((int) indexesLength); + dataOutputStream.writeLong(indexesLength); // Increment index length indexesLength += (i + offsetLength) * slots; @@ -258,24 +261,104 @@ private void writeMetadata(DataOutputStream dataOutputStream, BloomFilter bloomF } //Write the position of the index and the data - int indexOffset = dataOutputStream.size() + (Integer.SIZE / Byte.SIZE) + (Long.SIZE / Byte.SIZE); - dataOutputStream.writeInt(indexOffset); + long indexOffset = dataOutputStream.getFilePointer() + (Long.SIZE / Byte.SIZE) + (Long.SIZE / Byte.SIZE); + dataOutputStream.writeLong(indexOffset); + //data offset dataOutputStream.writeLong(indexOffset + indexesLength); } + private MappedByteBuffer[] initIndexBuffers(FileChannel channel, long indexSize) { + int bufIdx = 0; + int bufArraySize = (int) (indexSize / segmentSize) + ((indexSize % segmentSize != 0) ? 1 : 0); + var result = new MappedByteBuffer[bufArraySize]; + try { + for (long offset = 0; offset < indexSize; offset += segmentSize) { + long remainingFileSize = indexSize - offset; + long thisSegmentSize = Math.min(segmentSize, remainingFileSize); + result[bufIdx++] = channel.map(FileChannel.MapMode.READ_WRITE, offset, thisSegmentSize); + } + return result; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void getFromIndexBuffer(MappedByteBuffer[] indexBuffers, long offset, byte[] slotBuffer, int slotSize) { + int bufferIndex = (int) (offset / segmentSize); + var buf = indexBuffers[bufferIndex]; + var pos = (int) (offset % segmentSize); + + int remaining = remaining(buf, pos); + if (remaining < slotSize) { + int splitOffset = 0; + buf.get(pos, slotBuffer, 0, remaining); + buf = indexBuffers[++bufferIndex]; + int bytesLeft = slotSize - remaining; + splitOffset += remaining; + remaining = remaining(buf, 0); + + while (remaining < bytesLeft) { + buf.get(0, slotBuffer, splitOffset, remaining); + buf = indexBuffers[++bufferIndex]; + splitOffset += remaining; + bytesLeft -= remaining; + remaining = remaining(buf, 0); + } + + if (remaining > 0 && bytesLeft > 0) { + buf.get(0, slotBuffer, splitOffset, bytesLeft); + } + } else { + buf.get(pos, slotBuffer, 0, slotSize); + } + } + + private void putIntoIndexBuffer(MappedByteBuffer[] indexBuffers, long offset, byte[] slotBuffer, int slotSize) { + int bufferIndex = (int) (offset / segmentSize); + var buf = indexBuffers[bufferIndex]; + var pos = (int) (offset % segmentSize); + + int remaining = remaining(buf, pos); + if (remaining < slotSize) { + int splitOffset = 0; + buf.put(pos, slotBuffer, 0, remaining); + buf = indexBuffers[++bufferIndex]; + int bytesLeft = slotSize - remaining; + splitOffset += remaining; + remaining = remaining(buf, 0); + + while (remaining < bytesLeft) { + buf.put(0, slotBuffer, splitOffset, remaining); + buf = indexBuffers[++bufferIndex]; + splitOffset += remaining; + bytesLeft -= remaining; + remaining = remaining(buf, 0); + } + + if (remaining > 0 && bytesLeft > 0) { + buf.put(0, slotBuffer, splitOffset, bytesLeft); + } + } else { + buf.put(pos, slotBuffer, 0, slotSize); + } + } + + private static int remaining(ByteBuffer buffer, int pos) { + return buffer.limit() - pos; + } + private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOException { long count = keyCounts[keyLength]; - int slots = (int) Math.round(count / loadFactor); + long numSlots = Math.round(count / loadFactor); int offsetLength = maxOffsetLengths[keyLength]; int slotSize = keyLength + offsetLength; // Init index File indexFile = new File(tempFolder, "index" + keyLength + ".dat"); try (RandomAccessFile indexAccessFile = new RandomAccessFile(indexFile, "rw")) { - indexAccessFile.setLength((long) slots * slotSize); + indexAccessFile.setLength(numSlots * slotSize); try (FileChannel indexChannel = indexAccessFile.getChannel()) { - MappedByteBuffer byteBuffer = indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexAccessFile.length()); - + var indexBuffers = initIndexBuffers(indexChannel, indexAccessFile.length()); // Init reading stream File tempIndexFile = indexFiles[keyLength]; try (DataInputStream tempIndexStream = new DataInputStream(new BufferedInputStream(new FileInputStream(tempIndexFile)))) { @@ -284,12 +367,12 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti byte[] offsetBuffer = new byte[offsetLength]; // Read all keys - for (int i = 0; i < count; i++) { + for (long i = 0; i < count; i++) { // Read key tempIndexStream.readFully(keyBuffer); // Read offset - long offset = LongPacker.unpackLong(tempIndexStream); + long offsetData = LongPacker.unpackLong(tempIndexStream); // Hash long hash = Murmur3.hash(keyBuffer); @@ -298,18 +381,15 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti } boolean collision = false; - for (int probe = 0; probe < count; probe++) { - int slot = (int) ((hash + probe) % slots); - byteBuffer.position(slot * slotSize); - byteBuffer.get(slotBuffer); - + for (long probe = 0; probe < count; probe++) { + long slot = ((hash + probe) % numSlots); + getFromIndexBuffer(indexBuffers, slot * slotSize, slotBuffer, slotSize); long found = LongPacker.unpackLong(slotBuffer, keyLength); if (found == 0) { // The spot is empty use it - byteBuffer.position(slot * slotSize); - byteBuffer.put(keyBuffer); - int pos = LongPacker.packLong(offsetBuffer, offset); - byteBuffer.put(offsetBuffer, 0, pos); + putIntoIndexBuffer(indexBuffers, slot * slotSize, keyBuffer, keyBuffer.length); + int pos = LongPacker.packLong(offsetBuffer, offsetData); + putIntoIndexBuffer(indexBuffers, (slot * slotSize) + keyBuffer.length, offsetBuffer, pos); break; } else { collision = true; @@ -332,6 +412,7 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti log.info("Built index file {} \n{}", indexFile.getName(), msg); } finally { + indexBuffers = null; if (tempIndexFile.delete()) { log.info("Temporary index file {} has been deleted", tempIndexFile.getName()); } @@ -439,11 +520,9 @@ private DataOutputStream getIndexStream(int keyLength) throws IOException { if (dos == null) { File file = new File(tempFolder, "temp_index" + keyLength + ".dat"); file.deleteOnExit(); - indexFiles[keyLength] = file; - dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file))); + indexFiles[keyLength] = file; indexStreams[keyLength] = dos; - dataLengths[keyLength]++; } return dos; @@ -451,8 +530,8 @@ private DataOutputStream getIndexStream(int keyLength) throws IOException { private int getNumKeyCount() { int res = 0; - for (final int count : keyCounts) { - if (count != 0) { + for (final long count : keyCounts) { + if (count != 0L) { res++; } } diff --git a/src/main/java/com/linkedin/paldb/utils/BloomFilter.java b/src/main/java/com/linkedin/paldb/utils/BloomFilter.java index 78c12d9..f483789 100644 --- a/src/main/java/com/linkedin/paldb/utils/BloomFilter.java +++ b/src/main/java/com/linkedin/paldb/utils/BloomFilter.java @@ -11,13 +11,13 @@ public class BloomFilter { private static final double LN2 = 0.6931471805599453; // ln(2) private final int sizeInBits; - public BloomFilter(int elements, int sizeInBits) { + public BloomFilter(long elements, int sizeInBits) { this.sizeInBits = sizeInBits; this.hashFunctions = Math.max(1, (int) Math.round(LN2 * sizeInBits / elements)); this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / BITS_IN_LONG))]; } - public BloomFilter(int expectedElements, double errorRate) { + public BloomFilter(long expectedElements, double errorRate) { this.sizeInBits = Math.max(BITS_IN_LONG, (int) Math.ceil( (-1 * expectedElements * log(errorRate)) / (LN2 * LN2))); this.hashFunctions = Math.max(1, (int) Math.round(((double) sizeInBits / expectedElements) * LN2)); this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / BITS_IN_LONG))]; diff --git a/src/test/java/com/linkedin/paldb/impl/TestStore.java b/src/test/java/com/linkedin/paldb/impl/TestStore.java index 923efba..5a7e91f 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -330,6 +330,28 @@ public void testDataOnTwoBuffers() throws IOException { } } + @Test + public void testIndexOnManyIndexBuffers() throws IOException { + var keys = new String[]{GenerateTestData.generateStringData(100), GenerateTestData + .generateStringData(10000), GenerateTestData.generateStringData(100)}; + var values = new Integer[]{1, 2, 3}; + + StorageSerialization serialization = new StorageSerialization(new Configuration()); + int byteSize = serialization.serialize(keys[0]).length + serialization.serialize(keys[1]).length; + + //Write + writeStore(storeFile, keys, values); + + //Read + Configuration configuration = new Configuration(); + configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(32)); + try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { + for (int i = 0; i < keys.length; i++) { + assertEquals(reader.get(keys[i], null), values[i]); + } + } + } + @Test public void testDataSizeOnTwoBuffers() throws IOException { Integer[] keys = new Integer[]{1, 2, 3}; @@ -499,6 +521,30 @@ void should_write_file_close_file_and_delete_it() throws IOException { assertEquals(0L, DirectoryUtils.folderSize(tempDir.toFile())); } + private static final byte[] EMPTY_VALUE = new byte[0]; + + @Test + @Disabled + void should_add_more_than_int_max_size_keys() { + long keysCount = (long) Integer.MAX_VALUE + 100L; + var from = System.currentTimeMillis(); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + System.out.println(String.format("Adding %d keys...", keysCount)); + for (long i = 0; i < keysCount; i++) { + writer.put(i, EMPTY_VALUE); + } + } + System.out.println(String.format("%d keys were added successfully in %d ms", keysCount, System.currentTimeMillis() - from)); + + from = System.currentTimeMillis(); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + for (long i = 0; i < keysCount; i++) { + assertArrayEquals(EMPTY_VALUE, reader.get(i)); + } + } + System.out.println(String.format("%d keys were read in %d ms", keysCount, System.currentTimeMillis() - from)); + } + // UTILITY private void testReadKeyToString(K[] keys) { From 40bb54d27aacf95728ecb93832722bfc26234d9e Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Fri, 22 Nov 2019 16:52:40 +0200 Subject: [PATCH 2/5] Added configuration optiona to allow duplicates. --- README.md | 8 ++++---- .../com/linkedin/paldb/api/Configuration.java | 3 +++ .../linkedin/paldb/api/PalDBConfigBuilder.java | 12 ++++++++++++ .../com/linkedin/paldb/impl/StorageReader.java | 2 +- .../com/linkedin/paldb/impl/StorageWriter.java | 18 +++++++++++++++--- .../paldb/api/PalDBConfigBuilderTest.java | 2 ++ .../com/linkedin/paldb/impl/TestStore.java | 17 +++++++++++++++++ 7 files changed, 54 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 419ed54..a46e269 100644 --- a/README.md +++ b/README.md @@ -74,12 +74,12 @@ PalDB is available on Maven Central, hence just add the following dependency: net.soundvibe paldb - 2.0.1 + 2.0.2 ``` Scala SBT ``` -libraryDependencies += "net.soundvibe" % "paldb" % "2.0.1" +libraryDependencies += "net.soundvibe" % "paldb" % "2.0.2" ``` @@ -92,7 +92,7 @@ No, the final binary file is created when `StoreWriter.close()` is called. **Are duplicate keys allowed?** -No, duplicate keys aren't allowed and an exception will be thrown. +Duplicates are not allowed by default. But it could be changed in writer configuration if needed. **Do keys have an order when iterating?** @@ -131,6 +131,7 @@ Write parameters: + `compression.enabled`, enable compression (boolean) [default: false] + `bloom.filter.enabled`, enable bloom filter (boolean) [default: false] + `bloom.filter.error.factor`, bloom filter error rate (double) [default: 0.01] ++ `duplicates.enabled`, allow duplicates (boolean) [default: false] Read parameters: @@ -204,7 +205,6 @@ Machine-learning applications often have complex binary model files created in t Limitations ----------- + PalDB is optimal in replacing the usage of large in-memory data storage but still use memory (off-heap, yet much less) to do its job. Disabling memory mapping and relying on seeks is possible but is not what PalDB has been optimized for. -+ The size of the index is limited to 2GB. There's no limitation in the data size however. + PalDB reader is thread-safe but writer is not thread-safe at the moment so synchronization should be done externally if multi-threaded. Contributions diff --git a/src/main/java/com/linkedin/paldb/api/Configuration.java b/src/main/java/com/linkedin/paldb/api/Configuration.java index 86ff5cd..e01bc27 100644 --- a/src/main/java/com/linkedin/paldb/api/Configuration.java +++ b/src/main/java/com/linkedin/paldb/api/Configuration.java @@ -51,6 +51,8 @@ public class Configuration { public static final String BLOOM_FILTER_ENABLED = "bloom.filter.enabled"; //Bloom filter error rate public static final String BLOOM_FILTER_ERROR_FACTOR = "bloom.filter.error.factor"; + //Enable duplicates (keep last key) + public static final String ALLOW_DUPLICATES = "duplicates.enabled"; // Property map private final Map properties = new HashMap<>(); @@ -72,6 +74,7 @@ public Configuration() { putWithSystemPropertyDefault(COMPRESSION_ENABLED, "false"); putWithSystemPropertyDefault(BLOOM_FILTER_ENABLED, "false"); putWithSystemPropertyDefault(BLOOM_FILTER_ERROR_FACTOR, "0.01"); + putWithSystemPropertyDefault(ALLOW_DUPLICATES, "false"); //Serializers serializers = new Serializers(); diff --git a/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java b/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java index 1727256..310c8aa 100644 --- a/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java +++ b/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java @@ -60,6 +60,18 @@ public PalDBConfigBuilder withEnableCompression(boolean enabled) { return this; } + /** + * PalDB configuration property. + *

+ * duplicates.enabled - allow duplicates [default: false] + * @param enabled flag + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withEnableDuplicates(boolean enabled) { + palDbConfiguration.set(Configuration.ALLOW_DUPLICATES, String.valueOf(enabled)); + return this; + } + /** * PalDB configuration property. *

diff --git a/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/src/main/java/com/linkedin/paldb/impl/StorageReader.java index cf96363..142a2f3 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -296,7 +296,7 @@ private void getFromIndexBuffer(long offset, byte[] slotBuffer, int slotSize) { } } - private boolean isKey(byte[] slotBuffer, byte[] key) { + private static boolean isKey(byte[] slotBuffer, byte[] key) { return Arrays.compare(slotBuffer, 0, key.length, key, 0, key.length) == 0; } diff --git a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index cfb421e..024342d 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -61,6 +61,7 @@ public class StorageWriter { private int collisions; private final long segmentSize; + private final boolean duplicatesEnabled; StorageWriter(Configuration configuration, OutputStream stream) { config = configuration; @@ -84,6 +85,7 @@ public class StorageWriter { maxOffsetLengths = new int[0]; keyCounts = new long[0]; segmentSize = config.getLong(Configuration.MMAP_SEGMENT_SIZE); + duplicatesEnabled = config.getBoolean(Configuration.ALLOW_DUPLICATES); } public void put(byte[] key, byte[] value) throws IOException { @@ -394,9 +396,15 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti } else { collision = true; // Check for duplicates - if (Arrays.equals(keyBuffer, Arrays.copyOf(slotBuffer, keyLength))) { - throw new DuplicateKeyException( - String.format("A duplicate key has been found for key bytes %s", Arrays.toString(keyBuffer))); + if (isKey(slotBuffer, keyBuffer)) { + if (duplicatesEnabled) { + putIntoIndexBuffer(indexBuffers, slot * slotSize, keyBuffer, keyBuffer.length); + int pos = LongPacker.packLong(offsetBuffer, offsetData); + putIntoIndexBuffer(indexBuffers, (slot * slotSize) + keyBuffer.length, offsetBuffer, pos); + } else { + throw new DuplicateKeyException( + String.format("A duplicate key has been found for key bytes %s", Arrays.toString(keyBuffer))); + } } } } @@ -422,6 +430,10 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti return indexFile; } + private static boolean isKey(byte[] slotBuffer, byte[] key) { + return Arrays.compare(slotBuffer, 0, key.length, key, 0, key.length) == 0; + } + //Fail if the size of the expected store file exceed 2/3rd of the free disk space private void checkFreeDiskSpace(List inputFiles) { //Check for free space diff --git a/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java b/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java index 5cdf09c..055aaf1 100644 --- a/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java +++ b/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java @@ -15,6 +15,7 @@ public void testAllPropertiesSet() { .withEnableCompression(true) .withEnableBloomFilter(true) .withBloomFilterErrorFactor(0.01) + .withEnableDuplicates(true) .build(); assertEquals(500, config.getInt(Configuration.MMAP_SEGMENT_SIZE)); @@ -23,5 +24,6 @@ public void testAllPropertiesSet() { assertTrue(config.getBoolean(Configuration.COMPRESSION_ENABLED)); assertTrue(config.getBoolean(Configuration.BLOOM_FILTER_ENABLED)); assertEquals(0.01, config.getDouble(Configuration.BLOOM_FILTER_ERROR_FACTOR)); + assertTrue(config.getBoolean(Configuration.ALLOW_DUPLICATES)); } } \ No newline at end of file diff --git a/src/test/java/com/linkedin/paldb/impl/TestStore.java b/src/test/java/com/linkedin/paldb/impl/TestStore.java index 5a7e91f..cccc991 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -521,6 +521,23 @@ void should_write_file_close_file_and_delete_it() throws IOException { assertEquals(0L, DirectoryUtils.folderSize(tempDir.toFile())); } + @Test + void should_allow_duplicates() { + var config = new Configuration(); + config.set(Configuration.ALLOW_DUPLICATES, String.valueOf(true)); + + try (StoreWriter writer = PalDB.createWriter(storeFile, config)) { + writer.put("foobar", EMPTY_VALUE); + writer.put("foobar", "test".getBytes()); + writer.put("any data", "test2".getBytes()); + } + + try (StoreReader reader = PalDB.createReader(storeFile, config)) { + assertArrayEquals("test".getBytes(), reader.get("foobar")); + assertArrayEquals("test2".getBytes(), reader.get("any data")); + } + } + private static final byte[] EMPTY_VALUE = new byte[0]; @Test From 376b0e78506a08c3fffad6d6e9fc5a1eb5261ade Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Fri, 22 Nov 2019 18:01:09 +0200 Subject: [PATCH 3/5] Performance improvements for DataInputOutput. More test coverage. --- .../com/linkedin/paldb/api/Configuration.java | 2 +- .../com/linkedin/paldb/utils/BloomFilter.java | 15 ++++---- .../linkedin/paldb/utils/DataInputOutput.java | 37 ++++++++----------- .../com/linkedin/paldb/utils/TempUtils.java | 19 +++------- .../linkedin/paldb/api/TestConfiguration.java | 3 ++ .../com/linkedin/paldb/impl/TestStore.java | 33 +++++++++-------- .../linkedin/paldb/utils/BloomFilterTest.java | 14 +++++++ .../paldb/utils/DataInputOutputTest.java | 25 +++++++++++++ 8 files changed, 88 insertions(+), 60 deletions(-) create mode 100644 src/test/java/com/linkedin/paldb/utils/DataInputOutputTest.java diff --git a/src/main/java/com/linkedin/paldb/api/Configuration.java b/src/main/java/com/linkedin/paldb/api/Configuration.java index e01bc27..3fe9e39 100644 --- a/src/main/java/com/linkedin/paldb/api/Configuration.java +++ b/src/main/java/com/linkedin/paldb/api/Configuration.java @@ -414,7 +414,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = properties != null ? properties.hashCode() : 0; + int result = properties.hashCode(); result = 31 * result + (serializers != null ? serializers.hashCode() : 0); return result; } diff --git a/src/main/java/com/linkedin/paldb/utils/BloomFilter.java b/src/main/java/com/linkedin/paldb/utils/BloomFilter.java index f483789..741c543 100644 --- a/src/main/java/com/linkedin/paldb/utils/BloomFilter.java +++ b/src/main/java/com/linkedin/paldb/utils/BloomFilter.java @@ -5,7 +5,6 @@ import static java.lang.Math.log; public class BloomFilter { - private static final int BITS_IN_LONG = 64; private final long[] bits; private final int hashFunctions; // Number of hash functions private static final double LN2 = 0.6931471805599453; // ln(2) @@ -14,13 +13,13 @@ public class BloomFilter { public BloomFilter(long elements, int sizeInBits) { this.sizeInBits = sizeInBits; this.hashFunctions = Math.max(1, (int) Math.round(LN2 * sizeInBits / elements)); - this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / BITS_IN_LONG))]; + this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / Long.SIZE))]; } public BloomFilter(long expectedElements, double errorRate) { - this.sizeInBits = Math.max(BITS_IN_LONG, (int) Math.ceil( (-1 * expectedElements * log(errorRate)) / (LN2 * LN2))); + this.sizeInBits = Math.max(Long.SIZE, (int) Math.ceil( (-1 * expectedElements * log(errorRate)) / (LN2 * LN2))); this.hashFunctions = Math.max(1, (int) Math.round(((double) sizeInBits / expectedElements) * LN2)); - this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / BITS_IN_LONG))]; + this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / Long.SIZE))]; } public BloomFilter(int hashFunctions, int bitSize, long[] bits) { @@ -49,14 +48,14 @@ public long[] bits() { } private void setBit(int position) { - int flagIndex = position / BITS_IN_LONG; - int bitIndexInFlag = position % BITS_IN_LONG; + int flagIndex = position / Long.SIZE; + int bitIndexInFlag = position % Long.SIZE; bits[flagIndex] |= (1L << bitIndexInFlag); } private boolean getBit(int position) { - int flagIndex = position / BITS_IN_LONG; - int bitIndexInFlag = position % BITS_IN_LONG; + int flagIndex = position / Long.SIZE; + int bitIndexInFlag = position % Long.SIZE; return ((bits[flagIndex] >> bitIndexInFlag) & 1L) == 1; } diff --git a/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java b/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java index a6c6669..09de579 100644 --- a/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java +++ b/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java @@ -19,6 +19,8 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.lang.invoke.*; +import java.nio.ByteOrder; import java.util.Arrays; @@ -54,11 +56,6 @@ public DataInputOutput reset() { return this; } - public void resetForReading() { - count = pos; - pos = 0; - } - public DataInputOutput reset(byte[] b) { pos = 0; buf = b; @@ -126,15 +123,19 @@ public char readChar() { @Override public int readInt() { - return (((buf[pos++] & 0xff) << 24) | ((buf[pos++] & 0xff) << 16) | ((buf[pos++] & 0xff) << 8) | ( - (buf[pos++] & 0xff) << 0)); + int result = (int)INT_HANDLE.get(buf, pos); + pos += Integer.BYTES; + return result; } + private static final VarHandle INT_HANDLE = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.BIG_ENDIAN); + private static final VarHandle LONG_HANDLE = MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.BIG_ENDIAN); + @Override public long readLong() { - return (((long) (buf[pos++] & 0xff) << 56) | ((long) (buf[pos++] & 0xff) << 48) | ((long) (buf[pos++] & 0xff) << 40) - | ((long) (buf[pos++] & 0xff) << 32) | ((long) (buf[pos++] & 0xff) << 24) | ((long) (buf[pos++] & 0xff) << 16) - | ((long) (buf[pos++] & 0xff) << 8) | ((long) (buf[pos++] & 0xff) << 0)); + long result = (long)LONG_HANDLE.get(buf, pos); + pos += Long.BYTES; + return result; } @Override @@ -220,23 +221,15 @@ public void writeChar(int v) { @Override public void writeInt(int v) { ensureAvail(4); - buf[pos++] = (byte) (0xff & (v >> 24)); - buf[pos++] = (byte) (0xff & (v >> 16)); - buf[pos++] = (byte) (0xff & (v >> 8)); - buf[pos++] = (byte) (0xff & (v >> 0)); + INT_HANDLE.set(buf, pos, v); + pos += Integer.BYTES; } @Override public void writeLong(long v) { ensureAvail(8); - buf[pos++] = (byte) (0xff & (v >> 56)); - buf[pos++] = (byte) (0xff & (v >> 48)); - buf[pos++] = (byte) (0xff & (v >> 40)); - buf[pos++] = (byte) (0xff & (v >> 32)); - buf[pos++] = (byte) (0xff & (v >> 24)); - buf[pos++] = (byte) (0xff & (v >> 16)); - buf[pos++] = (byte) (0xff & (v >> 8)); - buf[pos++] = (byte) (0xff & (v >> 0)); + LONG_HANDLE.set(buf, pos, v); + pos += Long.BYTES; } @Override diff --git a/src/main/java/com/linkedin/paldb/utils/TempUtils.java b/src/main/java/com/linkedin/paldb/utils/TempUtils.java index b1ddaee..58a4d86 100644 --- a/src/main/java/com/linkedin/paldb/utils/TempUtils.java +++ b/src/main/java/com/linkedin/paldb/utils/TempUtils.java @@ -15,6 +15,7 @@ package com.linkedin.paldb.utils; import java.io.*; +import java.nio.file.Files; /** @@ -34,18 +35,11 @@ private TempUtils() { * @return temporary folder */ public static File createTempDir(String prefix) { - File baseDir = new File(System.getProperty("java.io.tmpdir")); - String baseName = prefix + System.currentTimeMillis() + "-"; - - for (int counter = 0; counter < 10000; counter++) { - File tempDir = new File(baseDir, baseName + counter); - if (tempDir.mkdir()) { - return tempDir; - } + try { + return Files.createTempDirectory(prefix).toFile(); + } catch (IOException e) { + throw new UncheckedIOException(e); } - throw new IllegalStateException( - "Failed to create directory within " + 10000 + " attempts (tried " + baseName + "0 to " + baseName + (10000 - 1) - + ')'); } /** @@ -56,8 +50,7 @@ public static File createTempDir(String prefix) { * @return temporary file * @throws IOException if an IO error occurs */ - public static File copyIntoTempFile(String fileName, InputStream inputStream) - throws IOException { + public static File copyIntoTempFile(String fileName, InputStream inputStream) throws IOException { File destFile = File.createTempFile(fileName, null); destFile.deleteOnExit(); try (BufferedInputStream bufferedStream = inputStream instanceof BufferedInputStream ? (BufferedInputStream) inputStream diff --git a/src/test/java/com/linkedin/paldb/api/TestConfiguration.java b/src/test/java/com/linkedin/paldb/api/TestConfiguration.java index 0988470..75313d4 100644 --- a/src/test/java/com/linkedin/paldb/api/TestConfiguration.java +++ b/src/test/java/com/linkedin/paldb/api/TestConfiguration.java @@ -75,7 +75,10 @@ public void testEquals() { c3.set("foo", "notbar"); assertEquals(c1, c2); + assertEquals(c1, c1); + assertNotEquals(c1, "any value"); assertNotEquals(c1, c3); + assertEquals(c1.hashCode(), c2.hashCode()); } @Test diff --git a/src/test/java/com/linkedin/paldb/impl/TestStore.java b/src/test/java/com/linkedin/paldb/impl/TestStore.java index cccc991..160c7be 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -317,12 +317,12 @@ public void testDataOnTwoBuffers() throws IOException { StorageSerialization serialization = new StorageSerialization(new Configuration()); int byteSize = serialization.serialize(values[0]).length + serialization.serialize(values[1]).length; + Configuration configuration = new Configuration(); + configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize - 100)); //Write - writeStore(storeFile, keys, values); + writeStore(storeFile, keys, values, configuration); //Read - Configuration configuration = new Configuration(); - configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize - 100)); try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { for (int i = 0; i < keys.length; i++) { assertEquals(reader.get((Integer) keys[i], null), values[i]); @@ -332,19 +332,17 @@ public void testDataOnTwoBuffers() throws IOException { @Test public void testIndexOnManyIndexBuffers() throws IOException { + Configuration configuration = new Configuration(); + configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(32)); + var keys = new String[]{GenerateTestData.generateStringData(100), GenerateTestData .generateStringData(10000), GenerateTestData.generateStringData(100)}; var values = new Integer[]{1, 2, 3}; - StorageSerialization serialization = new StorageSerialization(new Configuration()); - int byteSize = serialization.serialize(keys[0]).length + serialization.serialize(keys[1]).length; - //Write - writeStore(storeFile, keys, values); + writeStore(storeFile, keys, values, configuration); //Read - Configuration configuration = new Configuration(); - configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(32)); try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { for (int i = 0; i < keys.length; i++) { assertEquals(reader.get(keys[i], null), values[i]); @@ -356,21 +354,20 @@ public void testIndexOnManyIndexBuffers() throws IOException { public void testDataSizeOnTwoBuffers() throws IOException { Integer[] keys = new Integer[]{1, 2, 3}; String[] values = new String[]{GenerateTestData.generateStringData(100), GenerateTestData - .generateStringData(10000), GenerateTestData.generateStringData(100)}; + .generateStringData(10000), GenerateTestData.generateStringData(100)}; StorageSerialization serialization = new StorageSerialization(new Configuration()); byte[] b1 = serialization.serialize(values[0]); byte[] b2 = serialization.serialize(values[1]); int byteSize = b1.length + b2.length; int sizeSize = - LongPacker.packInt(new DataInputOutput(), b1.length) + LongPacker.packInt(new DataInputOutput(), b2.length); + LongPacker.packInt(new DataInputOutput(), b1.length) + LongPacker.packInt(new DataInputOutput(), b2.length); + Configuration configuration = new Configuration(); + configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize + sizeSize + 3)); //Write - writeStore(storeFile, keys, values); - + writeStore(storeFile, keys, values, configuration); //Read - Configuration configuration = new Configuration(); - configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize + sizeSize + 3)); try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { for (int i = 0; i < keys.length; i++) { assertEquals(reader.get(keys[i], null), values[i]); @@ -645,7 +642,11 @@ private void testReadKeyToIntArray(K[] keys) { } private void writeStore(File location, K[] keys, V[] values) { - try (StoreWriter writer = PalDB.createWriter(location, new Configuration())) { + writeStore(location, keys, values, new Configuration()); + } + + private void writeStore(File location, K[] keys, V[] values, Configuration configuration) { + try (StoreWriter writer = PalDB.createWriter(location, configuration)) { writer.putAll(keys, values); } } diff --git a/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java index b35b771..132d965 100644 --- a/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java +++ b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java @@ -63,6 +63,20 @@ void should_test_multi_thread_get() { futures.forEach(CompletableFuture::join); } + @Test + void should_be_equal() { + var bits = new long[Math.max(1, (int) Math.ceil((double) 6235225 / Long.SIZE))]; + var sut1 = new BloomFilter(4, 6235225, bits); + var sut2 = new BloomFilter(4, 6235225, bits); + + for (int i = 0; i < 100; i++) { + sut1.add((i + "foo").getBytes()); + sut2.add((i + "foo").getBytes()); + } + + assertEquals(sut1, sut2); + } + @Test void correctness() { System.out.println("Testing correctness.\n"+ diff --git a/src/test/java/com/linkedin/paldb/utils/DataInputOutputTest.java b/src/test/java/com/linkedin/paldb/utils/DataInputOutputTest.java new file mode 100644 index 0000000..71bd317 --- /dev/null +++ b/src/test/java/com/linkedin/paldb/utils/DataInputOutputTest.java @@ -0,0 +1,25 @@ +package com.linkedin.paldb.utils; + +import org.junit.jupiter.api.*; + +import java.util.stream.LongStream; + +import static org.junit.jupiter.api.Assertions.*; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class DataInputOutputTest { + + @Test + void should_write_and_read_long() { + var sut = new DataInputOutput(); + long max = 10_000_000L; + + LongStream.range(0L, max) + .forEach(sut::writeLong); + + sut.reset(); + + LongStream.range(0L, max) + .forEach(value -> assertEquals(value, sut.readLong())); + } +} \ No newline at end of file From e9fab4f1feb2cc3122b95d4aa8b0e019e696ab9c Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Fri, 22 Nov 2019 23:25:30 +0200 Subject: [PATCH 4/5] Cleaned up unit tests. Cleaned up some code. --- .../api/errors/DuplicateKeyException.java | 2 +- .../paldb/api/errors/MissingClass.java | 7 + .../paldb/api/errors/MissingSerializer.java | 7 + .../paldb/api/errors/OutOfDiskSpace.java | 2 +- .../paldb/api/errors/PalDBException.java | 8 + .../api/errors/UnsupportedTypeException.java | 2 +- .../paldb/api/errors/VersionMismatch.java | 8 + .../com/linkedin/paldb/impl/ReaderImpl.java | 2 - .../linkedin/paldb/impl/StorageReader.java | 43 +-- .../paldb/impl/StorageSerialization.java | 343 +++++++++--------- .../linkedin/paldb/impl/StorageWriter.java | 108 ++---- .../linkedin/paldb/utils/DataInputOutput.java | 72 +++- .../com/linkedin/paldb/utils/TempUtils.java | 21 ++ .../paldb/api/PalDBConfigBuilderTest.java | 4 +- .../linkedin/paldb/api/TestConfiguration.java | 131 +++---- .../linkedin/paldb/impl/GenerateTestData.java | 6 +- .../linkedin/paldb/impl/TestSerializers.java | 26 +- .../paldb/impl/TestStorageSerialization.java | 281 +++++++------- .../com/linkedin/paldb/impl/TestStore.java | 120 +++--- .../linkedin/paldb/impl/TestStoreReader.java | 217 ++++++----- .../performance/TestMemoryUsageHashMap.java | 17 +- .../paldb/performance/TestReadThroughput.java | 17 +- .../TestReadThroughputLevelDB.java | 39 +- .../performance/TestReadThrouputHashMap.java | 17 +- .../performance/TestReadThrouputRocksDB.java | 64 ++-- .../paldb/performance/TestStoreSize.java | 10 +- .../performance/utils/DirectoryUtils.java | 11 +- .../paldb/performance/utils/NanoBench.java | 62 ++-- .../linkedin/paldb/utils/BloomFilterTest.java | 2 +- .../paldb/utils/TestFormatVersion.java | 14 +- .../linkedin/paldb/utils/TestLongPacker.java | 53 ++- .../com/linkedin/paldb/utils/TestMurmur3.java | 8 +- .../linkedin/paldb/utils/TestTempUtils.java | 40 +- 33 files changed, 866 insertions(+), 898 deletions(-) create mode 100644 src/main/java/com/linkedin/paldb/api/errors/MissingClass.java create mode 100644 src/main/java/com/linkedin/paldb/api/errors/MissingSerializer.java create mode 100644 src/main/java/com/linkedin/paldb/api/errors/PalDBException.java create mode 100644 src/main/java/com/linkedin/paldb/api/errors/VersionMismatch.java diff --git a/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java b/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java index 32a7466..216addb 100644 --- a/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java +++ b/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java @@ -1,6 +1,6 @@ package com.linkedin.paldb.api.errors; -public class DuplicateKeyException extends RuntimeException { +public class DuplicateKeyException extends PalDBException { public DuplicateKeyException(String message) { super(message); diff --git a/src/main/java/com/linkedin/paldb/api/errors/MissingClass.java b/src/main/java/com/linkedin/paldb/api/errors/MissingClass.java new file mode 100644 index 0000000..36c99e0 --- /dev/null +++ b/src/main/java/com/linkedin/paldb/api/errors/MissingClass.java @@ -0,0 +1,7 @@ +package com.linkedin.paldb.api.errors; + +public class MissingClass extends PalDBException { + public MissingClass(String message) { + super(message); + } +} diff --git a/src/main/java/com/linkedin/paldb/api/errors/MissingSerializer.java b/src/main/java/com/linkedin/paldb/api/errors/MissingSerializer.java new file mode 100644 index 0000000..d321800 --- /dev/null +++ b/src/main/java/com/linkedin/paldb/api/errors/MissingSerializer.java @@ -0,0 +1,7 @@ +package com.linkedin.paldb.api.errors; + +public class MissingSerializer extends PalDBException { + public MissingSerializer(String message) { + super(message); + } +} diff --git a/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java b/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java index 2b9d15e..76506a4 100644 --- a/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java +++ b/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java @@ -1,6 +1,6 @@ package com.linkedin.paldb.api.errors; -public class OutOfDiskSpace extends RuntimeException { +public class OutOfDiskSpace extends PalDBException { public OutOfDiskSpace(String message) { super(message); diff --git a/src/main/java/com/linkedin/paldb/api/errors/PalDBException.java b/src/main/java/com/linkedin/paldb/api/errors/PalDBException.java new file mode 100644 index 0000000..2fdcb79 --- /dev/null +++ b/src/main/java/com/linkedin/paldb/api/errors/PalDBException.java @@ -0,0 +1,8 @@ +package com.linkedin.paldb.api.errors; + +public abstract class PalDBException extends RuntimeException { + + public PalDBException(String message) { + super(message); + } +} diff --git a/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java b/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java index 282589d..35ad942 100644 --- a/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java +++ b/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java @@ -21,7 +21,7 @@ * * @see StoreReader */ -public class UnsupportedTypeException extends RuntimeException { +public class UnsupportedTypeException extends PalDBException { public UnsupportedTypeException(Object obj) { super("The type '" + obj.getClass() + "' isn't supported"); diff --git a/src/main/java/com/linkedin/paldb/api/errors/VersionMismatch.java b/src/main/java/com/linkedin/paldb/api/errors/VersionMismatch.java new file mode 100644 index 0000000..385de82 --- /dev/null +++ b/src/main/java/com/linkedin/paldb/api/errors/VersionMismatch.java @@ -0,0 +1,8 @@ +package com.linkedin.paldb.api.errors; + +public class VersionMismatch extends PalDBException { + + public VersionMismatch(String message) { + super(message); + } +} diff --git a/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java index af7ab00..038c13b 100644 --- a/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java +++ b/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java @@ -110,8 +110,6 @@ public V get(K key, V defaultValue) { } } catch (IOException ex) { throw new UncheckedIOException(ex); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); } } diff --git a/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/src/main/java/com/linkedin/paldb/impl/StorageReader.java index 142a2f3..2f3f0b9 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -15,6 +15,7 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.Configuration; +import com.linkedin.paldb.api.errors.VersionMismatch; import com.linkedin.paldb.utils.*; import org.slf4j.*; @@ -26,6 +27,8 @@ import java.time.Instant; import java.util.*; +import static com.linkedin.paldb.utils.DataInputOutput.*; + /** * Internal read implementation. @@ -111,7 +114,7 @@ public class StorageReader implements Iterable> { formatVersion = FormatVersion.fromBytes(versionFound); if (formatVersion == null || !formatVersion.is(FormatVersion.getLatestVersion())) { - throw new RuntimeException( + throw new VersionMismatch( "Version mismatch, expected was '" + FormatVersion.getLatestVersion() + "' and found '" + formatVersion + "'"); } @@ -254,7 +257,7 @@ public byte[] get(byte[] key) throws IOException { var slotBuffer = new byte[slotSize]; for (long probe = 0; probe < numSlots; probe++) { long slot = ((hash + probe) % numSlots); - getFromIndexBuffer(ixOffset + slot * slotSize, slotBuffer, slotSize); + getFromBuffers(indexBuffers, ixOffset + slot * slotSize, slotBuffer, slotSize, segmentSize); long offset = LongPacker.unpackLong(slotBuffer, keyLength); if (offset == 0L) { return null; @@ -266,36 +269,6 @@ public byte[] get(byte[] key) throws IOException { return null; } - private void getFromIndexBuffer(long offset, byte[] slotBuffer, int slotSize) { - int bufferIndex = (int) (offset / segmentSize); - var buf = indexBuffers[bufferIndex]; - var pos = (int) (offset % segmentSize); - - int remaining = remaining(buf, pos); - if (remaining < slotSize) { - int splitOffset = 0; - buf.get(pos, slotBuffer, 0, remaining); - buf = indexBuffers[++bufferIndex]; - int bytesLeft = slotSize - remaining; - splitOffset += remaining; - remaining = remaining(buf, 0); - - while (remaining < bytesLeft) { - buf.get(0, slotBuffer, splitOffset, remaining); - buf = indexBuffers[++bufferIndex]; - splitOffset += remaining; - bytesLeft -= remaining; - remaining = remaining(buf, 0); - } - - if (remaining > 0 && bytesLeft > 0) { - buf.get(0, slotBuffer, splitOffset, bytesLeft); - } - } else { - buf.get(pos, slotBuffer, 0, slotSize); - } - } - private static boolean isKey(byte[] slotBuffer, byte[] key) { return Arrays.compare(slotBuffer, 0, key.length, key, 0, key.length) == 0; } @@ -313,10 +286,6 @@ public long getKeyCount() { return keyCount; } - private static int remaining(ByteBuffer buffer, int pos) { - return buffer.limit() - pos; - } - //Read the data at the given offset, the data can be spread over multiple data buffers private byte[] getMMapBytes(long offset) throws IOException { var buf = dataBuffers[(int) (offset / segmentSize)]; @@ -460,7 +429,7 @@ public FastEntry next() { try { long offset = 0; while (offset == 0) { - getFromIndexBuffer(currentIndexOffset, currentSlotBuffer, currentSlotBuffer.length); + getFromBuffers(indexBuffers, currentIndexOffset, currentSlotBuffer, currentSlotBuffer.length, segmentSize); offset = LongPacker.unpackLong(currentSlotBuffer, currentKeyLength); currentIndexOffset += currentSlotBuffer.length; } diff --git a/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java index a369569..fabf940 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java @@ -15,7 +15,7 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.*; -import com.linkedin.paldb.api.errors.UnsupportedTypeException; +import com.linkedin.paldb.api.errors.*; import com.linkedin.paldb.utils.*; import org.xerial.snappy.Snappy; @@ -26,7 +26,7 @@ /** * Internal serialization implementation. */ -public final class StorageSerialization { +final class StorageSerialization { //Buffer private final DataInputOutput dataInputOutput = new DataInputOutput(); @@ -43,7 +43,7 @@ public final class StorageSerialization { * * @param config configuration */ - public StorageSerialization(Configuration config) { + StorageSerialization(Configuration config) { this.compression = config.getBoolean(Configuration.COMPRESSION_ENABLED); this.serializers = config.getSerializers(); } @@ -56,7 +56,7 @@ public StorageSerialization(Configuration config) { * @return key as byte array * @throws IOException if an io error occurs */ - public byte[] serializeKey(K key) throws IOException { + byte[] serializeKey(K key) throws IOException { if (key == null) { throw new NullPointerException(); } @@ -72,7 +72,7 @@ public byte[] serializeKey(K key) throws IOException { * @param dataOutput data output * @throws IOException if an io error occurs */ - public void serializeKey(Object key, DataOutput dataOutput) throws IOException { + void serializeKey(Object key, DataOutput dataOutput) throws IOException { serializeObject(key, dataOutput, false); } @@ -83,7 +83,7 @@ public void serializeKey(Object key, DataOutput dataOutput) throws IOException { * @return value as byte array * @throws IOException if an io error occurs */ - public byte[] serializeValue(Object value) throws IOException { + byte[] serializeValue(Object value) throws IOException { serializeObject(value, dataInputOutput.reset(), compression); return dataInputOutput.toByteArray(); @@ -96,7 +96,7 @@ public byte[] serializeValue(Object value) throws IOException { * @param dataOutput data output * @throws IOException if an io error occurs */ - public void serializeValue(Object value, DataOutput dataOutput) throws IOException { + void serializeValue(Object value, DataOutput dataOutput) throws IOException { serializeObject(value, dataOutput, compression); } @@ -111,37 +111,37 @@ private void serializeObject(Object obj, DataOutput dataOutput, boolean useCompr //Cast to primitive arrays if necessary if (obj != null && obj.getClass().isArray()) { if (obj instanceof Integer[]) { - obj = (int[]) getPrimitiveArray((Integer[]) obj); + obj = getPrimitiveArray((Integer[]) obj); } else if (obj instanceof Boolean[]) { - obj = (boolean[]) getPrimitiveArray((Object[]) obj); + obj = getPrimitiveArray((Object[]) obj); } else if (obj instanceof Byte[]) { - obj = (byte[]) getPrimitiveArray((Object[]) obj); + obj = getPrimitiveArray((Object[]) obj); } else if (obj instanceof Character[]) { - obj = (char[]) getPrimitiveArray((Object[]) obj); + obj = getPrimitiveArray((Object[]) obj); } else if (obj instanceof Double[]) { - obj = (double[]) getPrimitiveArray((Object[]) obj); + obj = getPrimitiveArray((Object[]) obj); } else if (obj instanceof Float[]) { - obj = (float[]) getPrimitiveArray((Object[]) obj); + obj = getPrimitiveArray((Object[]) obj); } else if (obj instanceof Long[]) { - obj = (long[]) getPrimitiveArray((Object[]) obj); + obj = getPrimitiveArray((Object[]) obj); } else if (obj instanceof Short[]) { - obj = (short[]) getPrimitiveArray((Object[]) obj); + obj = getPrimitiveArray((Object[]) obj); } else if (obj instanceof Integer[][]) { - obj = (int[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } else if (obj instanceof Boolean[][]) { - obj = (boolean[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } else if (obj instanceof Byte[][]) { - obj = (byte[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } else if (obj instanceof Character[][]) { - obj = (char[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } else if (obj instanceof Double[][]) { - obj = (double[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } else if (obj instanceof Float[][]) { - obj = (float[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } else if (obj instanceof Long[][]) { - obj = (long[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } else if (obj instanceof Short[][]) { - obj = (short[][]) getPrimitiveArray((Object[][]) obj); + obj = getPrimitiveArray((Object[][]) obj); } } @@ -153,7 +153,7 @@ private void serializeObject(Object obj, DataOutput dataOutput, boolean useCompr * * @return true if enabled, false otherwise */ - public boolean isCompressionEnabled() { + boolean isCompressionEnabled() { return compression; } @@ -231,94 +231,94 @@ private static Class getPrimitiveType(Class type) { // SERIALIZATION - final static int NULL_ID = -1; - final static int NULL = 0; - final static int BOOLEAN_TRUE = 2; - final static int BOOLEAN_FALSE = 3; - final static int INTEGER_MINUS_1 = 4; - final static int INTEGER_0 = 5; - final static int INTEGER_1 = 6; - final static int INTEGER_2 = 7; - final static int INTEGER_3 = 8; - final static int INTEGER_4 = 9; - final static int INTEGER_5 = 10; - final static int INTEGER_6 = 11; - final static int INTEGER_7 = 12; - final static int INTEGER_8 = 13; - final static int INTEGER_255 = 14; - final static int INTEGER_PACK_NEG = 15; - final static int INTEGER_PACK = 16; - final static int LONG_MINUS_1 = 17; - final static int LONG_0 = 18; - final static int LONG_1 = 19; - final static int LONG_2 = 20; - final static int LONG_3 = 21; - final static int LONG_4 = 22; - final static int LONG_5 = 23; - final static int LONG_6 = 24; - final static int LONG_7 = 25; - final static int LONG_8 = 26; - final static int LONG_PACK_NEG = 27; - final static int LONG_PACK = 28; - final static int LONG_255 = 29; - final static int LONG_MINUS_MAX = 30; - final static int SHORT_MINUS_1 = 31; - final static int SHORT_0 = 32; - final static int SHORT_1 = 33; - final static int SHORT_255 = 34; - final static int SHORT_FULL = 35; - final static int BYTE_MINUS_1 = 36; - final static int BYTE_0 = 37; - final static int BYTE_1 = 38; - final static int BYTE_FULL = 39; - final static int CHAR = 40; - final static int FLOAT_MINUS_1 = 41; - final static int FLOAT_0 = 42; - final static int FLOAT_1 = 43; - final static int FLOAT_255 = 44; - final static int FLOAT_SHORT = 45; - final static int FLOAT_FULL = 46; - final static int DOUBLE_MINUS_1 = 47; - final static int DOUBLE_0 = 48; - final static int DOUBLE_1 = 49; - final static int DOUBLE_255 = 50; - final static int DOUBLE_SHORT = 51; - final static int DOUBLE_FULL = 52; - final static int DOUBLE_ARRAY = 53; - final static int BIGDECIMAL = 54; - final static int BIGINTEGER = 55; - final static int FLOAT_ARRAY = 56; - final static int INTEGER_MINUS_MAX = 57; - final static int SHORT_ARRAY = 58; - final static int BOOLEAN_ARRAY = 59; - final static int ARRAY_INT_B = 60; - final static int ARRAY_INT_S = 61; - final static int ARRAY_INT_I = 62; - final static int ARRAY_INT_PACKED = 63; - final static int ARRAY_LONG_B = 64; - final static int ARRAY_LONG_S = 65; - final static int ARRAY_LONG_I = 66; - final static int ARRAY_LONG_L = 67; - final static int ARRAY_LONG_PACKED = 68; - final static int CHAR_ARRAY = 69; - final static int BYTE_ARRAY = 70; - final static int STRING_ARRAY = 71; - final static int ARRAY_OBJECT = 72; - final static int STRING_EMPTY = 101; - final static int NOTUSED_STRING_C = 102; - final static int STRING = 103; - final static int ARRAY_INT_C = 104; - final static int ARRAY_LONG_C = 105; - final static int DOUBLE_ARRAY_C = 106; - final static int FLOAT_ARRAY_C = 107; - final static int CHAR_ARRAY_C = 108; - final static int BYTE_ARRAY_C = 109; - final static int SHORT_ARRAY_C = 110; - final static int INT_INT_ARRAY = 111; - final static int LONG_LONG_ARRAY = 112; - final static int CLASS = 113; - final static int CUSTOM = 114; - final static String EMPTY_STRING = ""; + static final int NULL_ID = -1; + private static final int NULL = 0; + private static final int BOOLEAN_TRUE = 2; + private static final int BOOLEAN_FALSE = 3; + private static final int INTEGER_MINUS_1 = 4; + private static final int INTEGER_0 = 5; + private static final int INTEGER_1 = 6; + private static final int INTEGER_2 = 7; + private static final int INTEGER_3 = 8; + private static final int INTEGER_4 = 9; + private static final int INTEGER_5 = 10; + private static final int INTEGER_6 = 11; + private static final int INTEGER_7 = 12; + private static final int INTEGER_8 = 13; + private static final int INTEGER_255 = 14; + private static final int INTEGER_PACK_NEG = 15; + private static final int INTEGER_PACK = 16; + private static final int LONG_MINUS_1 = 17; + private static final int LONG_0 = 18; + private static final int LONG_1 = 19; + private static final int LONG_2 = 20; + private static final int LONG_3 = 21; + private static final int LONG_4 = 22; + private static final int LONG_5 = 23; + private static final int LONG_6 = 24; + private static final int LONG_7 = 25; + private static final int LONG_8 = 26; + private static final int LONG_PACK_NEG = 27; + private static final int LONG_PACK = 28; + private static final int LONG_255 = 29; + private static final int LONG_MINUS_MAX = 30; + private static final int SHORT_MINUS_1 = 31; + private static final int SHORT_0 = 32; + private static final int SHORT_1 = 33; + private static final int SHORT_255 = 34; + private static final int SHORT_FULL = 35; + private static final int BYTE_MINUS_1 = 36; + private static final int BYTE_0 = 37; + private static final int BYTE_1 = 38; + private static final int BYTE_FULL = 39; + private static final int CHAR = 40; + private static final int FLOAT_MINUS_1 = 41; + private static final int FLOAT_0 = 42; + private static final int FLOAT_1 = 43; + private static final int FLOAT_255 = 44; + private static final int FLOAT_SHORT = 45; + private static final int FLOAT_FULL = 46; + private static final int DOUBLE_MINUS_1 = 47; + private static final int DOUBLE_0 = 48; + private static final int DOUBLE_1 = 49; + private static final int DOUBLE_255 = 50; + private static final int DOUBLE_SHORT = 51; + private static final int DOUBLE_FULL = 52; + private static final int DOUBLE_ARRAY = 53; + private static final int BIGDECIMAL = 54; + private static final int BIGINTEGER = 55; + private static final int FLOAT_ARRAY = 56; + private static final int INTEGER_MINUS_MAX = 57; + private static final int SHORT_ARRAY = 58; + private static final int BOOLEAN_ARRAY = 59; + private static final int ARRAY_INT_B = 60; + private static final int ARRAY_INT_S = 61; + private static final int ARRAY_INT_I = 62; + private static final int ARRAY_INT_PACKED = 63; + private static final int ARRAY_LONG_B = 64; + private static final int ARRAY_LONG_S = 65; + private static final int ARRAY_LONG_I = 66; + private static final int ARRAY_LONG_L = 67; + private static final int ARRAY_LONG_PACKED = 68; + private static final int CHAR_ARRAY = 69; + private static final int BYTE_ARRAY = 70; + private static final int STRING_ARRAY = 71; + private static final int ARRAY_OBJECT = 72; + private static final int STRING_EMPTY = 101; + static final int NOTUSED_STRING_C = 102; + private static final int STRING = 103; + private static final int ARRAY_INT_C = 104; + private static final int ARRAY_LONG_C = 105; + private static final int DOUBLE_ARRAY_C = 106; + private static final int FLOAT_ARRAY_C = 107; + private static final int CHAR_ARRAY_C = 108; + private static final int BYTE_ARRAY_C = 109; + private static final int SHORT_ARRAY_C = 110; + private static final int INT_INT_ARRAY = 111; + private static final int LONG_LONG_ARRAY = 112; + private static final int CLASS = 113; + private static final int CUSTOM = 114; + private static final String EMPTY_STRING = ""; byte[] serialize(Object obj) throws IOException { return serialize(obj, false); @@ -557,7 +557,7 @@ private static void serializeString(final DataOutput out, final String val) thro final int len = val.length(); LongPacker.packInt(out, len); for (int i = 0; i < len; i++) { - int c = (int) val.charAt(i); //TODO investigate if c could be negative here + int c = val.charAt(i); //TODO investigate if c could be negative here LongPacker.packInt(out, c); } } @@ -781,7 +781,7 @@ private void serializeObjectArray(final DataOutput out, final Object[] val) thro } } - public Object deserialize(byte[] buf) throws ClassNotFoundException, IOException { + Object deserialize(byte[] buf) throws IOException { DataInputOutput bs = new DataInputOutput(buf); Object ret = deserialize(bs); if (bs.available() != 0) { @@ -791,7 +791,7 @@ public Object deserialize(byte[] buf) throws ClassNotFoundException, IOException return ret; } - public Object deserialize(DataInput is) throws IOException, ClassNotFoundException { + Object deserialize(DataInput is) throws IOException { Object ret = null; final int head = is.readUnsignedByte(); @@ -800,7 +800,7 @@ public Object deserialize(DataInput is) throws IOException, ClassNotFoundExcepti var className = is.readUTF(); Serializer serializer = serializers.getSerializer(className); if (serializer == null) { - throw new ClassNotFoundException("Serializer not registered: " + className); + throw new MissingSerializer("Serializer not registered: " + className); } ret = serializer.read(is); } else { @@ -814,115 +814,115 @@ public Object deserialize(DataInput is) throws IOException, ClassNotFoundExcepti ret = Boolean.FALSE; break; case INTEGER_MINUS_1: - ret = Integer.valueOf(-1); + ret = -1; break; case INTEGER_0: - ret = Integer.valueOf(0); + ret = 0; break; case INTEGER_1: - ret = Integer.valueOf(1); + ret = 1; break; case INTEGER_2: - ret = Integer.valueOf(2); + ret = 2; break; case INTEGER_3: - ret = Integer.valueOf(3); + ret = 3; break; case INTEGER_4: - ret = Integer.valueOf(4); + ret = 4; break; case INTEGER_5: - ret = Integer.valueOf(5); + ret = 5; break; case INTEGER_6: - ret = Integer.valueOf(6); + ret = 6; break; case INTEGER_7: - ret = Integer.valueOf(7); + ret = 7; break; case INTEGER_8: - ret = Integer.valueOf(8); + ret = 8; break; case INTEGER_MINUS_MAX: - ret = Integer.valueOf(Integer.MIN_VALUE); + ret = Integer.MIN_VALUE; break; case INTEGER_255: - ret = Integer.valueOf(is.readUnsignedByte()); + ret = is.readUnsignedByte(); break; case INTEGER_PACK_NEG: - ret = Integer.valueOf(-LongPacker.unpackInt(is)); + ret = -LongPacker.unpackInt(is); break; case INTEGER_PACK: - ret = Integer.valueOf(LongPacker.unpackInt(is)); + ret = LongPacker.unpackInt(is); break; case LONG_MINUS_1: - ret = Long.valueOf(-1); + ret = (long) -1; break; case LONG_0: - ret = Long.valueOf(0); + ret = 0L; break; case LONG_1: - ret = Long.valueOf(1); + ret = 1L; break; case LONG_2: - ret = Long.valueOf(2); + ret = 2L; break; case LONG_3: - ret = Long.valueOf(3); + ret = 3L; break; case LONG_4: - ret = Long.valueOf(4); + ret = 4L; break; case LONG_5: - ret = Long.valueOf(5); + ret = 5L; break; case LONG_6: - ret = Long.valueOf(6); + ret = 6L; break; case LONG_7: - ret = Long.valueOf(7); + ret = 7L; break; case LONG_8: - ret = Long.valueOf(8); + ret = 8L; break; case LONG_255: - ret = Long.valueOf(is.readUnsignedByte()); + ret = (long) is.readUnsignedByte(); break; case LONG_PACK_NEG: - ret = Long.valueOf(-LongPacker.unpackLong(is)); + ret = -LongPacker.unpackLong(is); break; case LONG_PACK: - ret = Long.valueOf(LongPacker.unpackLong(is)); + ret = LongPacker.unpackLong(is); break; case LONG_MINUS_MAX: - ret = Long.valueOf(Long.MIN_VALUE); + ret = Long.MIN_VALUE; break; case SHORT_MINUS_1: - ret = Short.valueOf((short) -1); + ret = (short) -1; break; case SHORT_0: - ret = Short.valueOf((short) 0); + ret = (short) 0; break; case SHORT_1: - ret = Short.valueOf((short) 1); + ret = (short) 1; break; case SHORT_255: - ret = Short.valueOf((short) is.readUnsignedByte()); + ret = (short) is.readUnsignedByte(); break; case SHORT_FULL: - ret = Short.valueOf(is.readShort()); + ret = is.readShort(); break; case BYTE_MINUS_1: - ret = Byte.valueOf((byte) -1); + ret = (byte) -1; break; case BYTE_0: - ret = Byte.valueOf((byte) 0); + ret = (byte) 0; break; case BYTE_1: - ret = Byte.valueOf((byte) 1); + ret = (byte) 1; break; case BYTE_FULL: - ret = Byte.valueOf(is.readByte()); + ret = is.readByte(); break; case SHORT_ARRAY: ret = deserializeShortArray(is); @@ -952,43 +952,43 @@ public Object deserialize(DataInput is) throws IOException, ClassNotFoundExcepti ret = deserializeCharCompressedArray(is); break; case CHAR: - ret = Character.valueOf(is.readChar()); + ret = is.readChar(); break; case FLOAT_MINUS_1: - ret = Float.valueOf(-1); + ret = (float) -1; break; case FLOAT_0: - ret = Float.valueOf(0); + ret = (float) 0; break; case FLOAT_1: - ret = Float.valueOf(1); + ret = 1f; break; case FLOAT_255: - ret = Float.valueOf(is.readUnsignedByte()); + ret = (float) is.readUnsignedByte(); break; case FLOAT_SHORT: - ret = Float.valueOf(is.readShort()); + ret = (float) is.readShort(); break; case FLOAT_FULL: - ret = Float.valueOf(is.readFloat()); + ret = is.readFloat(); break; case DOUBLE_MINUS_1: - ret = Double.valueOf(-1); + ret = (double) -1; break; case DOUBLE_0: - ret = Double.valueOf(0); + ret = (double) 0; break; case DOUBLE_1: - ret = Double.valueOf(1); + ret = 1d; break; case DOUBLE_255: - ret = Double.valueOf(is.readUnsignedByte()); + ret = (double) is.readUnsignedByte(); break; case DOUBLE_SHORT: - ret = Double.valueOf(is.readShort()); + ret = (double) is.readShort(); break; case DOUBLE_FULL: - ret = Double.valueOf(is.readDouble()); + ret = is.readDouble(); break; case BIGINTEGER: ret = new BigInteger((byte[]) deserialize(is)); @@ -1074,10 +1074,14 @@ private static String deserializeString(DataInput buf) return new String(b); } - private static Class deserializeClass(DataInput is) throws IOException, ClassNotFoundException { + private static Class deserializeClass(DataInput is) throws IOException { is.readByte(); String className = deserializeString(is); - return Class.forName(className); + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new MissingClass("Class is missing: " + className); + } } private static short[] deserializeShortArray(DataInput is) throws IOException { @@ -1192,8 +1196,7 @@ private static byte[] deserializeByteCompressedArray(DataInput is) throws IOExce return Snappy.uncompress(b); } - private Object[] deserializeArrayObject(DataInput is) - throws IOException, ClassNotFoundException { + private Object[] deserializeArrayObject(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); Object[] s = (Object[]) Array.newInstance(Object.class, size); diff --git a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index 024342d..c23c17d 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -22,9 +22,12 @@ import java.io.*; import java.nio.*; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.text.DecimalFormat; import java.util.*; +import static com.linkedin.paldb.utils.DataInputOutput.*; + /** * Internal write implementation. */ @@ -285,70 +288,6 @@ private MappedByteBuffer[] initIndexBuffers(FileChannel channel, long indexSize) } } - private void getFromIndexBuffer(MappedByteBuffer[] indexBuffers, long offset, byte[] slotBuffer, int slotSize) { - int bufferIndex = (int) (offset / segmentSize); - var buf = indexBuffers[bufferIndex]; - var pos = (int) (offset % segmentSize); - - int remaining = remaining(buf, pos); - if (remaining < slotSize) { - int splitOffset = 0; - buf.get(pos, slotBuffer, 0, remaining); - buf = indexBuffers[++bufferIndex]; - int bytesLeft = slotSize - remaining; - splitOffset += remaining; - remaining = remaining(buf, 0); - - while (remaining < bytesLeft) { - buf.get(0, slotBuffer, splitOffset, remaining); - buf = indexBuffers[++bufferIndex]; - splitOffset += remaining; - bytesLeft -= remaining; - remaining = remaining(buf, 0); - } - - if (remaining > 0 && bytesLeft > 0) { - buf.get(0, slotBuffer, splitOffset, bytesLeft); - } - } else { - buf.get(pos, slotBuffer, 0, slotSize); - } - } - - private void putIntoIndexBuffer(MappedByteBuffer[] indexBuffers, long offset, byte[] slotBuffer, int slotSize) { - int bufferIndex = (int) (offset / segmentSize); - var buf = indexBuffers[bufferIndex]; - var pos = (int) (offset % segmentSize); - - int remaining = remaining(buf, pos); - if (remaining < slotSize) { - int splitOffset = 0; - buf.put(pos, slotBuffer, 0, remaining); - buf = indexBuffers[++bufferIndex]; - int bytesLeft = slotSize - remaining; - splitOffset += remaining; - remaining = remaining(buf, 0); - - while (remaining < bytesLeft) { - buf.put(0, slotBuffer, splitOffset, remaining); - buf = indexBuffers[++bufferIndex]; - splitOffset += remaining; - bytesLeft -= remaining; - remaining = remaining(buf, 0); - } - - if (remaining > 0 && bytesLeft > 0) { - buf.put(0, slotBuffer, splitOffset, bytesLeft); - } - } else { - buf.put(pos, slotBuffer, 0, slotSize); - } - } - - private static int remaining(ByteBuffer buffer, int pos) { - return buffer.limit() - pos; - } - private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOException { long count = keyCounts[keyLength]; long numSlots = Math.round(count / loadFactor); @@ -385,22 +324,22 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti boolean collision = false; for (long probe = 0; probe < count; probe++) { long slot = ((hash + probe) % numSlots); - getFromIndexBuffer(indexBuffers, slot * slotSize, slotBuffer, slotSize); + getFromBuffers(indexBuffers, slot * slotSize, slotBuffer, slotSize, segmentSize); long found = LongPacker.unpackLong(slotBuffer, keyLength); if (found == 0) { // The spot is empty use it - putIntoIndexBuffer(indexBuffers, slot * slotSize, keyBuffer, keyBuffer.length); + putIntoBuffers(indexBuffers, slot * slotSize, keyBuffer, keyBuffer.length, segmentSize); int pos = LongPacker.packLong(offsetBuffer, offsetData); - putIntoIndexBuffer(indexBuffers, (slot * slotSize) + keyBuffer.length, offsetBuffer, pos); + putIntoBuffers(indexBuffers, (slot * slotSize) + keyBuffer.length, offsetBuffer, pos, segmentSize); break; } else { collision = true; // Check for duplicates if (isKey(slotBuffer, keyBuffer)) { if (duplicatesEnabled) { - putIntoIndexBuffer(indexBuffers, slot * slotSize, keyBuffer, keyBuffer.length); + putIntoBuffers(indexBuffers, slot * slotSize, keyBuffer, keyBuffer.length, segmentSize); int pos = LongPacker.packLong(offsetBuffer, offsetData); - putIntoIndexBuffer(indexBuffers, (slot * slotSize) + keyBuffer.length, offsetBuffer, pos); + putIntoBuffers(indexBuffers, (slot * slotSize) + keyBuffer.length, offsetBuffer, pos, segmentSize); } else { throw new DuplicateKeyException( String.format("A duplicate key has been found for key bytes %s", Arrays.toString(keyBuffer))); @@ -420,12 +359,18 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti log.info("Built index file {} \n{}", indexFile.getName(), msg); } finally { + Arrays.fill(indexBuffers, null); indexBuffers = null; + System.gc(); if (tempIndexFile.delete()) { log.info("Temporary index file {} has been deleted", tempIndexFile.getName()); } } } + } catch (Exception e) { + System.gc(); + Files.deleteIfExists(indexFile.toPath()); + throw e; } return indexFile; } @@ -457,20 +402,11 @@ private void checkFreeDiskSpace(List inputFiles) { //Merge files to the provided fileChannel private void mergeFiles(List inputFiles, OutputStream outputStream) throws IOException { long startTime = System.nanoTime(); - //Merge files for (File f : inputFiles) { if (f.exists()) { - try (FileInputStream fileInputStream = new FileInputStream(f); - BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { - log.info("Merging {} size={}", f.getName(), f.length()); - - byte[] buffer = new byte[8192]; - int length; - while ((length = bufferedInputStream.read(buffer)) > 0) { - outputStream.write(buffer, 0, length); - } - } + log.info("Merging {} size={}", f.getName(), f.length()); + Files.copy(f.toPath(), outputStream); } else { log.info("Skip merging file {} because it doesn't exist", f.getName()); } @@ -478,14 +414,18 @@ private void mergeFiles(List inputFiles, OutputStream outputStream) throws log.debug("Time to merge {} s", ((System.nanoTime() - startTime) / 1000000000.0)); } - //Cleanup files private void cleanup(List inputFiles) { for (File f : inputFiles) { - if (f.exists() && f.delete()) { - log.info("Deleted temporary file {}", f.getName()); + try { + if (Files.deleteIfExists(f.toPath())) { + log.info("Deleted temporary file {}", f.getName()); + } + } catch (IOException e) { + log.error("Cannot delete file " + f, e); } } - if (tempFolder.delete()) { + + if (TempUtils.deleteDirectory(tempFolder)) { log.info("Deleted temporary folder at {}", tempFolder.getAbsolutePath()); } } diff --git a/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java b/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java index 09de579..14ca93e 100644 --- a/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java +++ b/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java @@ -20,7 +20,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.lang.invoke.*; -import java.nio.ByteOrder; +import java.nio.*; import java.util.Arrays; @@ -69,6 +69,70 @@ public byte[] toByteArray() { return d; } + public static void getFromBuffers(MappedByteBuffer[] buffers, long offset, byte[] slotBuffer, int slotSize, long segmentSize) { + int bufferIndex = (int) (offset / segmentSize); + var buf = buffers[bufferIndex]; + var pos = (int) (offset % segmentSize); + + int remaining = remaining(buf, pos); + if (remaining < slotSize) { + int splitOffset = 0; + buf.get(pos, slotBuffer, 0, remaining); + buf = buffers[++bufferIndex]; + int bytesLeft = slotSize - remaining; + splitOffset += remaining; + remaining = remaining(buf, 0); + + while (remaining < bytesLeft) { + buf.get(0, slotBuffer, splitOffset, remaining); + buf = buffers[++bufferIndex]; + splitOffset += remaining; + bytesLeft -= remaining; + remaining = remaining(buf, 0); + } + + if (remaining > 0 && bytesLeft > 0) { + buf.get(0, slotBuffer, splitOffset, bytesLeft); + } + } else { + buf.get(pos, slotBuffer, 0, slotSize); + } + } + + public static void putIntoBuffers(MappedByteBuffer[] buffers, long offset, byte[] slotBuffer, int slotSize, long segmentSize) { + int bufferIndex = (int) (offset / segmentSize); + var buf = buffers[bufferIndex]; + var pos = (int) (offset % segmentSize); + + int remaining = remaining(buf, pos); + if (remaining < slotSize) { + int splitOffset = 0; + buf.put(pos, slotBuffer, 0, remaining); + buf = buffers[++bufferIndex]; + int bytesLeft = slotSize - remaining; + splitOffset += remaining; + remaining = remaining(buf, 0); + + while (remaining < bytesLeft) { + buf.put(0, slotBuffer, splitOffset, remaining); + buf = buffers[++bufferIndex]; + splitOffset += remaining; + bytesLeft -= remaining; + remaining = remaining(buf, 0); + } + + if (remaining > 0 && bytesLeft > 0) { + buf.put(0, slotBuffer, splitOffset, bytesLeft); + } + } else { + buf.put(pos, slotBuffer, 0, slotSize); + } + } + + public static int remaining(ByteBuffer buffer, int pos) { + return buffer.limit() - pos; + } + @Override public int available() { return count - pos; @@ -108,12 +172,12 @@ public int readUnsignedByte() { @Override public short readShort() { - return (short) (((short) (buf[pos++] & 0xff) << 8) | ((short) (buf[pos++] & 0xff) << 0)); + return (short) (((short) (buf[pos++] & 0xff) << 8) | ((short) (buf[pos++] & 0xff))); } @Override public int readUnsignedShort() { - return (((int) (buf[pos++] & 0xff) << 8) | ((int) (buf[pos++] & 0xff) << 0)); + return (((int) (buf[pos++] & 0xff) << 8) | ((int) (buf[pos++] & 0xff))); } @Override @@ -210,7 +274,7 @@ public void writeByte(int v) { public void writeShort(int v) { ensureAvail(2); buf[pos++] = (byte) (0xff & (v >> 8)); - buf[pos++] = (byte) (0xff & (v >> 0)); + buf[pos++] = (byte) (0xff & (v)); } @Override diff --git a/src/main/java/com/linkedin/paldb/utils/TempUtils.java b/src/main/java/com/linkedin/paldb/utils/TempUtils.java index 58a4d86..fa55b2a 100644 --- a/src/main/java/com/linkedin/paldb/utils/TempUtils.java +++ b/src/main/java/com/linkedin/paldb/utils/TempUtils.java @@ -14,6 +14,8 @@ package com.linkedin.paldb.utils; +import org.slf4j.*; + import java.io.*; import java.nio.file.Files; @@ -23,11 +25,30 @@ */ public final class TempUtils { + private static final Logger log = LoggerFactory.getLogger(TempUtils.class); + // Default constructor private TempUtils() { } + public static boolean deleteDirectory(File directoryToBeDeleted) { + if (directoryToBeDeleted.isDirectory()) { + File[] allContents = directoryToBeDeleted.listFiles(); + if (allContents != null) { + for (File file : allContents) { + deleteDirectory(file); + } + } + } + try { + return Files.deleteIfExists(directoryToBeDeleted.toPath()); + } catch (IOException e) { + log.warn("Cannot delete directory: " + directoryToBeDeleted, e); + return false; + } + } + /** * Creates a temporary directory prefixed with prefix. * diff --git a/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java b/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java index 055aaf1..a2ceb57 100644 --- a/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java +++ b/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java @@ -4,10 +4,10 @@ import static org.junit.jupiter.api.Assertions.*; -public class PalDBConfigBuilderTest { +class PalDBConfigBuilderTest { @Test - public void testAllPropertiesSet() { + void testAllPropertiesSet() { var config = PalDBConfigBuilder.create() .withMemoryMapSegmentSize(500) .withMemoryMapDataEnabled(false) diff --git a/src/test/java/com/linkedin/paldb/api/TestConfiguration.java b/src/test/java/com/linkedin/paldb/api/TestConfiguration.java index 75313d4..66b82da 100644 --- a/src/test/java/com/linkedin/paldb/api/TestConfiguration.java +++ b/src/test/java/com/linkedin/paldb/api/TestConfiguration.java @@ -17,38 +17,36 @@ import org.junit.jupiter.api.Test; -import java.awt.*; -import java.io.*; -import java.util.*; +import java.util.Arrays; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.*; -public class TestConfiguration { +class TestConfiguration { @Test - public void testConfiguration() { + void testConfiguration() { Configuration c = new Configuration(); c.set("foo", "bar"); - assertEquals(c.get("foo", null), "bar"); - assertEquals(c.get("bar", "foo"), "foo"); + assertEquals("bar", c.get("foo", null)); + assertEquals("foo", c.get("bar", "foo")); } @Test - public void testConfigurationCopy() { + void testConfigurationCopy() { Configuration c = new Configuration(); c.set("foo", "bar"); Configuration r = new Configuration(c); - assertEquals(r.get("foo", null), "bar"); + assertEquals("bar", r.get("foo", null)); c.set("foo", ""); - assertEquals(r.get("foo", null), "bar"); + assertEquals("bar", r.get("foo", null)); } @Test - public void testConfigurationReadOnly() { + void testConfigurationReadOnly() { Configuration c = new Configuration(); c.set("foo", "bar"); @@ -59,12 +57,12 @@ public void testConfigurationReadOnly() { } @Test - public void testEqualsEmpty() { + void testEqualsEmpty() { assertEquals(new Configuration(), new Configuration()); } @Test - public void testEquals() { + void testEquals() { Configuration c1 = new Configuration(); c1.set("foo", "bar"); @@ -76,13 +74,13 @@ public void testEquals() { assertEquals(c1, c2); assertEquals(c1, c1); - assertNotEquals(c1, "any value"); + assertNotEquals("any value", c1); assertNotEquals(c1, c3); assertEquals(c1.hashCode(), c2.hashCode()); } @Test - public void testGetBoolean() { + void testGetBoolean() { Configuration c = new Configuration(); c.set("foo", "true"); c.set("bar", "false"); @@ -92,12 +90,12 @@ public void testGetBoolean() { } @Test - public void testGetBooleanMissing() { + void testGetBooleanMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getBoolean("foo")); } @Test - public void testGetBooleanDefault() { + void testGetBooleanDefault() { Configuration c = new Configuration(); c.set("foo", "true"); @@ -106,169 +104,148 @@ public void testGetBooleanDefault() { } @Test - public void testGetDouble() { + void testGetDouble() { Configuration c = new Configuration(); c.set("foo", "1.0"); - assertEquals(c.getDouble("foo"), 1.0); + assertEquals(1.0, c.getDouble("foo")); } @Test - public void testGetDoubleMissing() { + void testGetDoubleMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getDouble("foo")); } @Test - public void testGetDoubleDefault() { + void testGetDoubleDefault() { Configuration c = new Configuration(); c.set("foo", "1.0"); - assertEquals(c.getDouble("foo", 2.0), 1.0); - assertEquals(c.getDouble("bar", 2.0), 2.0); + assertEquals(1.0, c.getDouble("foo", 2.0)); + assertEquals(2.0, c.getDouble("bar", 2.0)); } @Test - public void testGetFloat() { + void testGetFloat() { Configuration c = new Configuration(); c.set("foo", "1.0"); - assertEquals(c.getFloat("foo"), 1f); + assertEquals(1f, c.getFloat("foo")); } @Test - public void testGetFloatMissing() { + void testGetFloatMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getFloat("foo")); } @Test - public void testGetFloatDefault() { + void testGetFloatDefault() { Configuration c = new Configuration(); c.set("foo", "1.0"); - assertEquals(c.getFloat("foo", 2f), 1f); - assertEquals(c.getFloat("bar", 2f), 2f); + assertEquals(1f, c.getFloat("foo", 2f)); + assertEquals(2f, c.getFloat("bar", 2f)); } @Test - public void testGetInt() { + void testGetInt() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getInt("foo"), 1); + assertEquals(1, c.getInt("foo")); } @Test - public void testGetIntMissing() { + void testGetIntMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getInt("foo")); } @Test - public void testGetIntDefault() { + void testGetIntDefault() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getInt("foo", 2), 1); - assertEquals(c.getInt("bar", 2), 2); + assertEquals(1, c.getInt("foo", 2)); + assertEquals(2, c.getInt("bar", 2)); } @Test - public void testGetShort() { + void testGetShort() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getShort("foo"), (short) 1); + assertEquals((short) 1, c.getShort("foo")); } @Test - public void testGetShortMissing() { + void testGetShortMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getShort("foo")); } @Test - public void testGetShortDefault() { + void testGetShortDefault() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getShort("foo", (short) 2), (short) 1); - assertEquals(c.getShort("bar", (short) 2), (short) 2); + assertEquals((short) 1, c.getShort("foo", (short) 2)); + assertEquals((short) 2, c.getShort("bar", (short) 2)); } @Test - public void testGetLongMissing() { + void testGetLongMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getLong("foo")); } @Test - public void testGetLong() { + void testGetLong() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getLong("foo"), 1L); + assertEquals(1L, c.getLong("foo")); } @Test - public void testGetLongDefault() { + void testGetLongDefault() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getLong("foo", 2L), 1L); - assertEquals(c.getLong("bar", 2L), 2L); + assertEquals(1L, c.getLong("foo", 2L)); + assertEquals(2L, c.getLong("bar", 2L)); } @Test - public void testGetClass() + void testGetClass() throws ClassNotFoundException { Configuration c = new Configuration(); c.set("foo", Integer.class.getName()); - assertEquals(c.getClass("foo"), Integer.class); + assertEquals(Integer.class, c.getClass("foo")); } @Test - public void testGetClassMissing() { + void testGetClassMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getClass("foo")); } @Test - public void testGetList() { + void testGetList() { Configuration c = new Configuration(); c.set("foo", "foo,bar"); - assertEquals(c.getList("foo"), Arrays.asList("foo", "bar")); + assertEquals(Arrays.asList("foo", "bar"), c.getList("foo")); } @Test - public void testGetListMissing() { + void testGetListMissing() { assertThrows(IllegalArgumentException.class, () -> new Configuration().getList("foo")); } @Test - public void testGetListDefault() { + void testGetListDefault() { Configuration c = new Configuration(); c.set("foo", "foo,bar"); - assertEquals(c.getList("foo", singletonList("that")), Arrays.asList("foo", "bar")); - assertEquals(c.getList("bar", singletonList("that")), singletonList("that")); - } - - // UTILITY - - public static class PointSerializer implements Serializer { - - @Override - public Point read(DataInput input) { - return null; - } - - @Override - public Class serializedClass() { - return Point.class; - } - - @Override - public void write(DataOutput output, Point input) { - - } - + assertEquals(Arrays.asList("foo", "bar"), c.getList("foo", singletonList("that"))); + assertEquals(singletonList("that"), c.getList("bar", singletonList("that"))); } } diff --git a/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java b/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java index 2f1a79c..a31b605 100644 --- a/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java +++ b/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java @@ -98,7 +98,7 @@ public static Object[] generateCompoundKeys(int count) { Object[] res = new Object[count]; Random random = new Random(345); for (int i = 0; i < count; i++) { - Object[] k = new Object[]{Byte.valueOf((byte) random.nextInt(10)), Integer.valueOf(i)}; + Object[] k = new Object[]{(byte) random.nextInt(10), i}; res[i] = k; } return res; @@ -125,7 +125,7 @@ public static String[] generateStringData(int count, int letters) { public static Integer[] generateIntData(int count) { Integer[] res = new Integer[count]; - Random random = new Random(count + 34593263544354353l); + Random random = new Random(count + 34593263544354353L); for (int i = 0; i < count; i++) { res[i] = random.nextInt(1000000); } @@ -134,7 +134,7 @@ public static Integer[] generateIntData(int count) { public static int[][] generateIntArrayData(int count, int size) { int[][] res = new int[count][]; - Random random = new Random(count + 34593263544354353l); + Random random = new Random(count + 34593263544354353L); for (int i = 0; i < count; i++) { int[] r = new int[size]; for (int j = 0; j < size; j++) { diff --git a/src/test/java/com/linkedin/paldb/impl/TestSerializers.java b/src/test/java/com/linkedin/paldb/impl/TestSerializers.java index 185d1f8..2d15146 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestSerializers.java +++ b/src/test/java/com/linkedin/paldb/impl/TestSerializers.java @@ -28,38 +28,38 @@ public class TestSerializers { private Serializers serializers; @BeforeEach - public void setUp() { + void setUp() { serializers = new Serializers(); } @Test - public void testRegister() { + void testRegister() { ColorSerializer i = new ColorSerializer(); serializers.registerSerializer(i); - assertSame(serializers.getSerializer(Color.class), i); + assertSame(i, serializers.getSerializer(Color.class)); } @Test - public void testRegisterTwice() { + void testRegisterTwice() { ColorSerializer i1 = new ColorSerializer(); ColorSerializer i2 = new ColorSerializer(); serializers.registerSerializer(i1); serializers.registerSerializer(i2); - assertSame(serializers.getSerializer(Color.class), i1); + assertSame(i1, serializers.getSerializer(Color.class)); } @Test - public void testRegisterTwo() { + void testRegisterTwo() { ColorSerializer i = new ColorSerializer(); PointSerializer f = new PointSerializer(); serializers.registerSerializer(i); serializers.registerSerializer(f); - assertSame(serializers.getSerializer(Color.class), i); - assertSame(serializers.getSerializer(Point.class), f); + assertSame(i, serializers.getSerializer(Color.class)); + assertSame(f, serializers.getSerializer(Point.class)); } @Test - public void testGetSerializer() { + void testGetSerializer() { ColorSerializer i = new ColorSerializer(); serializers.registerSerializer(i); assertNull(serializers.getSerializer(Point.class)); @@ -67,16 +67,16 @@ public void testGetSerializer() { } @Test - public void testSerialize() { + void testSerialize() { serializers.registerSerializer(new ColorSerializer()); assertNotNull(serializers.getSerializer(Color.class)); } @Test - public void testInterfaceType() { + void testInterfaceType() { SerializerWithInterface i = new SerializerWithInterface(); serializers.registerSerializer(i); - assertSame(serializers.getSerializer(AnInterface.class), i); + assertSame(i, serializers.getSerializer(AnInterface.class)); } // HELPER @@ -119,7 +119,7 @@ public void write(DataOutput output, Point input) { } - public interface AnInterface { + interface AnInterface { } diff --git a/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java b/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java index 505ee11..640f107 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java @@ -21,23 +21,22 @@ import java.awt.*; import java.io.*; import java.math.*; -import java.util.Arrays; import static org.junit.jupiter.api.Assertions.*; -public class TestStorageSerialization { +class TestStorageSerialization { private Configuration configuration; private StorageSerialization serialization; @BeforeEach - public void setUp() { + void setUp() { configuration = new Configuration(); serialization = new StorageSerialization(configuration); } @Test - public void testCompressionEnabled() { + void testCompressionEnabled() { assertFalse(serialization.isCompressionEnabled()); Configuration config = new Configuration(); config.set(Configuration.COMPRESSION_ENABLED, "true"); @@ -46,14 +45,14 @@ public void testCompressionEnabled() { } @Test - public void testSerializeKey() throws IOException, ClassNotFoundException { + void testSerializeKey() throws IOException { Integer l = 1; Object d = serialization.deserialize(serialization.serializeKey(l)); - assertEquals(d, l); + assertEquals(l, d); } @Test - public void testSerializeKeyDataOutput() throws IOException, ClassNotFoundException { + void testSerializeKeyDataOutput() throws IOException { Integer l = 1; ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); @@ -63,11 +62,11 @@ public void testSerializeKeyDataOutput() throws IOException, ClassNotFoundExcept ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInputStream dis = new DataInputStream(bis); - assertEquals(serialization.deserialize(dis), l); + assertEquals(l, serialization.deserialize(dis)); } @Test - public void testSerializeValueDataOutput() throws IOException, ClassNotFoundException { + void testSerializeValueDataOutput() throws IOException { Integer l = 1; ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); @@ -77,52 +76,47 @@ public void testSerializeValueDataOutput() throws IOException, ClassNotFoundExce ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInputStream dis = new DataInputStream(bis); - assertEquals(serialization.deserialize(dis), l); + assertEquals(l, serialization.deserialize(dis)); } @Test - public void testSerializeKeyNull() { + void testSerializeKeyNull() { assertThrows(NullPointerException.class, () -> serialization.serializeKey(null)); } @Test - public void testTransformValue() - throws ClassNotFoundException, IOException { + void testTransformValue() throws IOException { Integer l = 1; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - assertEquals(deserialize, l); + assertEquals(l, deserialize); } @Test - public void testTransformList() - throws ClassNotFoundException, IOException { + void testTransformList() throws IOException { Integer[] l = new Integer[]{1, 2}; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - assertEquals(deserialize.getClass(), int[].class); - assertArrayEquals((int[]) deserialize, new int[]{1, 2}); + assertEquals(int[].class, deserialize.getClass()); + assertArrayEquals(new int[]{1, 2}, (int[]) deserialize); } @Test - public void testTransformListWithNull() - throws ClassNotFoundException, IOException { + void testTransformListWithNull() throws IOException { Integer[] l = new Integer[]{1, null, 2}; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - assertEquals(deserialize.getClass(), int[].class); - assertArrayEquals((int[])deserialize, new int[]{1, 0, 2}); + assertEquals(int[].class, deserialize.getClass()); + assertArrayEquals(new int[]{1, 0, 2}, (int[])deserialize); } @Test - public void testTransformListOfList() - throws ClassNotFoundException, IOException { + void testTransformListOfList() throws IOException { Integer[][] l = new Integer[][]{{1}, {2}}; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - assertEquals(deserialize.getClass(), int[][].class); - assertArrayEquals((int[][])deserialize, new int[][]{{1}, {2}}); + assertEquals(int[][].class, deserialize.getClass()); + assertArrayEquals(new int[][]{{1}, {2}}, (int[][])deserialize); } @Test - public void testCustomSerializer() - throws Throwable { + void testCustomSerializer() throws Throwable { configuration.registerSerializer(new Serializer() { @Override public void write(DataOutput dataOutput, Point input) @@ -145,12 +139,11 @@ public Class serializedClass() { }); Point p = new Point(42, 9); byte[] buf = serialization.serialize(p); - assertEquals(serialization.deserialize(buf), p); + assertEquals(p, serialization.deserialize(buf)); } @Test - public void testCustomArraySerializer() - throws Throwable { + void testCustomArraySerializer() throws Throwable { configuration.registerSerializer(new Serializer() { @Override public void write(DataOutput dataOutput, Point[] input) @@ -173,11 +166,11 @@ public Class serializedClass() { }); Point[] p = new Point[]{new Point(42, 9)}; byte[] buf = serialization.serialize(p); - assertArrayEquals((Point[])serialization.deserialize(buf), p); + assertArrayEquals(p, (Point[])serialization.deserialize(buf)); } @Test - public void testInnerClassSerializer() throws Throwable { + void testInnerClassSerializer() throws Throwable { configuration.registerSerializer(new Serializer() { @Override @@ -198,11 +191,11 @@ public void write(DataOutput dataOutput, ImplementsA input) throws IOException { }); ImplementsA a = new ImplementsA(42); byte[] buf = serialization.serialize(a); - assertEquals(serialization.deserialize(buf), a); + assertEquals(a, serialization.deserialize(buf)); } @Test - public void testInheritedSerializer() throws Throwable { + void testInheritedSerializer() { configuration.registerSerializer(new Serializer() { @Override @@ -224,36 +217,34 @@ public void write(DataOutput dataOutput, A input) throws IOException { assertThrows(UnsupportedTypeException.class, () -> { ImplementsA a = new ImplementsA(42); byte[] buf = serialization.serialize(a); - assertEquals(serialization.deserialize(buf), a); + assertEquals(a, serialization.deserialize(buf)); }); } @Test - public void testNull() throws Throwable { + void testNull() throws Throwable { byte[] buf = serialization.serialize(null); assertNull(serialization.deserialize(buf)); } @Test - public void testByte() - throws Throwable { + void testByte() throws Throwable { byte[] vals = new byte[]{-1, 0, 1, 6}; for (byte val : vals) { byte[] buf = serialization.serialize(val); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Byte.class); - assertEquals(l2, val); + assertSame(Byte.class, l2.getClass()); + assertEquals(val, l2); } } @Test - public void testNotSupported() { + void testNotSupported() { assertThrows(UnsupportedTypeException.class, () -> serialization.serialize(new Color(0, 0, 0))); } @Test - public void testInt() - throws IOException, ClassNotFoundException { + void testInt() throws IOException { int[] vals = {Integer.MIN_VALUE, -Short.MIN_VALUE * 2, -Short.MIN_VALUE + 1, -Short.MIN_VALUE, -10, -9, -8, -7, -6, -5, -4, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 127, 254, 255, 256, Short.MAX_VALUE, @@ -261,64 +252,59 @@ public void testInt() for (int i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Integer.class); - assertEquals(l2, i); + assertSame(Integer.class, l2.getClass()); + assertEquals(i, l2); } } @Test - public void testShort() - throws IOException, ClassNotFoundException { + void testShort() throws IOException { short[] vals = {(short) (-Short.MIN_VALUE + 1), (short) -Short.MIN_VALUE, -10, -9, -8, -7, -6, -5, -4, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 127, 254, 255, 256, Short.MAX_VALUE, Short.MAX_VALUE - 1, Short.MAX_VALUE}; for (short i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Short.class); - assertEquals(l2, i); + assertSame(Short.class, l2.getClass()); + assertEquals(i, l2); } } @Test - public void testDouble() - throws IOException, ClassNotFoundException { + void testDouble() throws IOException { double[] vals = {1f, 0f, -1f, Math.PI, 255, 256, Short.MAX_VALUE, Short.MAX_VALUE + 1, -100}; for (double i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Double.class); - assertEquals(l2, i); + assertSame(Double.class, l2.getClass()); + assertEquals(i, l2); } } @Test - public void testFloat() - throws IOException, ClassNotFoundException { + void testFloat() throws IOException { float[] vals = {1f, 0f, -1f, (float) Math.PI, 255, 256, Short.MAX_VALUE, Short.MAX_VALUE + 1, -100}; for (float i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Float.class); - assertEquals(l2, i); + assertSame(Float.class, l2.getClass()); + assertEquals(i, l2); } } @Test - public void testChar() - throws IOException, ClassNotFoundException { + void testChar() throws IOException { char[] vals = {'a', ' '}; for (char i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Character.class); - assertEquals(l2, i); + assertSame(Character.class, l2.getClass()); + assertEquals(i, l2); } } @Test - public void testLong() - throws IOException, ClassNotFoundException { + void testLong() throws IOException { long[] vals = {Long.MIN_VALUE, Integer.MIN_VALUE, Integer.MIN_VALUE - 1, Integer.MIN_VALUE + 1, @@ -328,221 +314,208 @@ public void testLong() for (long i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Long.class); - assertEquals(l2, i); + assertSame(Long.class, l2.getClass()); + assertEquals(i, l2); } } @Test - public void testBoolean() - throws IOException, ClassNotFoundException { + void testBoolean() throws IOException { byte[] buf = serialization.serialize(true); Object l2 = serialization.deserialize(buf); - assertTrue(l2.getClass() == Boolean.class); - assertEquals(l2, true); + assertSame(Boolean.class, l2.getClass()); + assertEquals(true, l2); byte[] buf2 = serialization.serialize(false); Object l22 = serialization.deserialize(buf2); - assertTrue(l22.getClass() == Boolean.class); - assertEquals(l22, false); + assertSame(Boolean.class, l22.getClass()); + assertEquals(false, l22); } @Test - public void testString() - throws IOException, ClassNotFoundException { + void testString() throws IOException { byte[] buf = serialization.serialize("Abcd"); String l2 = (String) serialization.deserialize(buf); - assertEquals(l2, "Abcd"); + assertEquals("Abcd", l2); } @Test - public void testEmptyString() - throws IOException, ClassNotFoundException { + void testEmptyString() throws IOException { byte[] buf = serialization.serialize(""); String l2 = (String) serialization.deserialize(buf); - assertEquals(l2, ""); + assertEquals("", l2); } @Test - public void testBigString() - throws IOException, ClassNotFoundException { - String bigString = ""; + void testBigString() throws IOException { + var bigString = new StringBuilder(); for (int i = 0; i < 1e4; i++) { - bigString += i % 10; + bigString.append(i % 10); } - byte[] buf = serialization.serialize(bigString); + byte[] buf = serialization.serialize(bigString.toString()); String l2 = (String) serialization.deserialize(buf); - assertEquals(l2, bigString); + assertEquals(bigString.toString(), l2); } @Test - public void testClass() - throws IOException, ClassNotFoundException { + void testClass() throws IOException { byte[] buf = serialization.serialize(String.class); Class l2 = (Class) serialization.deserialize(buf); - assertEquals(l2, String.class); + assertEquals(String.class, l2); } @Test - public void testClass2() - throws IOException, ClassNotFoundException { + void testClass2() throws IOException { byte[] buf = serialization.serialize(long[].class); Class l2 = (Class) serialization.deserialize(buf); - assertEquals(l2, long[].class); + assertEquals(long[].class, l2); } @Test - public void testUnicodeString() - throws ClassNotFoundException, IOException { + void testUnicodeString() throws IOException { String s = "Ciudad Bolíva"; byte[] buf = serialization.serialize(s); Object l2 = serialization.deserialize(buf); - assertEquals(l2, s); + assertEquals(s, l2); } @Test - public void testStringArray() - throws ClassNotFoundException, IOException { + void testStringArray() throws IOException { String[] l = new String[]{"foo", "bar", ""}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (String[]) deserialize)); + assertArrayEquals(l, (String[]) deserialize); } @Test - public void testObjectArray() - throws ClassNotFoundException, IOException { + void testObjectArray() throws IOException { Object[] l = new Object[]{"foo", 2, Boolean.TRUE}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (Object[]) deserialize)); + assertArrayEquals(l, (Object[]) deserialize); } @Test - public void testBooleanArray() - throws ClassNotFoundException, IOException { + void testBooleanArray() throws IOException { boolean[] l = new boolean[]{true, false}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (boolean[]) deserialize)); + assertArrayEquals(l, (boolean[]) deserialize); } @Test - public void testDoubleArray() - throws ClassNotFoundException, IOException { + void testDoubleArray() throws IOException { double[] l = new double[]{Math.PI, 1D}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (double[]) deserialize)); + assertArrayEquals(l, (double[]) deserialize); } @Test - public void testFloatArray() - throws ClassNotFoundException, IOException { + void testFloatArray() throws IOException { float[] l = new float[]{1F, 1.234235F}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (float[]) deserialize)); + assertArrayEquals(l, (float[]) deserialize); } @Test - public void testByteArray() - throws ClassNotFoundException, IOException { + void testByteArray() throws IOException { byte[] l = new byte[]{1, 34, -5}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (byte[]) deserialize)); + assertArrayEquals(l, (byte[]) deserialize); } @Test - public void testShortArray() - throws ClassNotFoundException, IOException { + void testShortArray() + throws Exception { short[] l = new short[]{1, 345, -5000}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (short[]) deserialize)); + assertArrayEquals(l, (short[]) deserialize); } @Test - public void testCharArray() - throws ClassNotFoundException, IOException { + void testCharArray() + throws IOException { char[] l = new char[]{'1', 'a', '&'}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - assertTrue(Arrays.equals(l, (char[]) deserialize)); + assertArrayEquals(l, (char[]) deserialize); } @Test - public void testIntArray() - throws ClassNotFoundException, IOException { + void testIntArray() + throws IOException { int[][] l = new int[][]{{3, 5}, {-1200, 29999}, {3, 100000}, {-43999, 100000}}; for (int[] a : l) { Object deserialize = serialization.deserialize(serialization.serialize(a)); - assertTrue(Arrays.equals(a, (int[]) deserialize)); + assertArrayEquals(a, (int[]) deserialize); } } @Test - public void testLongArray() - throws ClassNotFoundException, IOException { - long[][] l = new long[][]{{3l, 5l}, {-1200l, 29999l}, {3l, 100000l}, {-43999l, 100000l}, {-123l, 12345678901234l}}; + void testLongArray() + throws IOException { + long[][] l = new long[][]{{3L, 5L}, {-1200L, 29999L}, {3L, 100000L}, {-43999L, 100000L}, {-123L, 12345678901234L}}; for (long[] a : l) { Object deserialize = serialization.deserialize(serialization.serialize(a)); - assertTrue(Arrays.equals(a, (long[]) deserialize)); + assertArrayEquals(a, (long[]) deserialize); } } @Test - public void testDoubleCompressedArray() - throws ClassNotFoundException, IOException { + void testDoubleCompressedArray() + throws IOException { double[] l = generateDoubleArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - assertTrue(Arrays.equals(l, (double[]) deserialize)); + assertArrayEquals(l, (double[]) deserialize); } @Test - public void testFloatCompressedArray() - throws ClassNotFoundException, IOException { + void testFloatCompressedArray() + throws IOException { float[] l = generateFloatArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - assertTrue(Arrays.equals(l, (float[]) deserialize)); + assertArrayEquals(l, (float[]) deserialize); } @Test - public void testByteCompressedArray() - throws ClassNotFoundException, IOException { + void testByteCompressedArray() + throws IOException { byte[] l = generateByteArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - assertTrue(Arrays.equals(l, (byte[]) deserialize)); + assertArrayEquals(l, (byte[]) deserialize); } @Test - public void testCharCompressedArray() - throws ClassNotFoundException, IOException { + void testCharCompressedArray() + throws IOException { char[] l = generateCharArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - assertTrue(Arrays.equals(l, (char[]) deserialize)); + assertArrayEquals(l, (char[]) deserialize); } @Test - public void testShortCompressedArray() - throws ClassNotFoundException, IOException { + void testShortCompressedArray() + throws IOException { short[] l = generateShortArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - assertTrue(Arrays.equals(l, (short[]) deserialize)); + assertArrayEquals(l, (short[]) deserialize); } @Test - public void testIntCompressedArray() - throws ClassNotFoundException, IOException { + void testIntCompressedArray() + throws IOException { int[] l = generateIntArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - assertTrue(Arrays.equals(l, (int[]) deserialize)); + assertArrayEquals(l, (int[]) deserialize); } @Test - public void testLongCompressedArray() - throws ClassNotFoundException, IOException { + void testLongCompressedArray() + throws IOException { long[] l = generateLongArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - assertTrue(Arrays.equals(l, (long[]) deserialize)); + assertArrayEquals(l, (long[]) deserialize); } @Test - public void testBigDecimal() - throws IOException, ClassNotFoundException { + void testBigDecimal() + throws IOException { BigDecimal d = new BigDecimal("445656.7889889895165654423236"); assertEquals(d, serialization.deserialize(serialization.serialize(d))); d = new BigDecimal("-53534534534534445656.7889889895165654423236"); @@ -550,8 +523,8 @@ public void testBigDecimal() } @Test - public void testBigInteger() - throws IOException, ClassNotFoundException { + void testBigInteger() + throws IOException { BigInteger d = new BigInteger("4456567889889895165654423236"); assertEquals(d, serialization.deserialize(serialization.serialize(d))); d = new BigInteger("-535345345345344456567889889895165654423236"); @@ -559,8 +532,8 @@ public void testBigInteger() } @Test - public void testMultiDimensionalIntArray() - throws IOException, ClassNotFoundException { + void testMultiDimensionalIntArray() + throws IOException { int[][] d = new int[2][]; d[0] = new int[]{1, 3}; d[1] = new int[]{-3, 1}; @@ -570,8 +543,8 @@ public void testMultiDimensionalIntArray() } @Test - public void testMultiDimensionalLongArray() - throws IOException, ClassNotFoundException { + void testMultiDimensionalLongArray() + throws IOException { long[][] d = new long[2][]; d[0] = new long[]{1, 3}; d[1] = new long[]{-3, 1}; @@ -654,7 +627,7 @@ private static class ImplementsA implements A { int val; - public ImplementsA(int val) { + ImplementsA(int val) { this.val = val; } diff --git a/src/test/java/com/linkedin/paldb/impl/TestStore.java b/src/test/java/com/linkedin/paldb/impl/TestStore.java index 160c7be..939bac5 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -24,40 +24,40 @@ import java.nio.file.*; import java.util.*; -import static com.linkedin.paldb.utils.TestTempUtils.deleteDirectory; +import static com.linkedin.paldb.utils.TempUtils.deleteDirectory; import static org.junit.jupiter.api.Assertions.*; -public class TestStore { +class TestStore { private Path tempDir; private File storeFile; @BeforeEach - public void setUp() throws IOException { + void setUp() throws IOException { tempDir = Files.createTempDirectory("tmp"); storeFile = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); } @AfterEach - public void cleanUp() { + void cleanUp() { deleteDirectory(tempDir.toFile()); } @Test - public void testEmpty() { + void testEmpty() { StoreWriter writer = PalDB.createWriter(storeFile, new Configuration()); writer.close(); assertTrue(storeFile.exists()); try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.size(), 0); + assertEquals(0, reader.size()); assertNull(reader.get(1, null)); } } @Test - public void testEmptyStream() { + void testEmptyStream() { ByteArrayOutputStream bos = new ByteArrayOutputStream(); StoreWriter writer = PalDB.createWriter(bos, new Configuration()); writer.close(); @@ -70,25 +70,25 @@ public void testEmptyStream() { } @Test - public void testEmptyDefaultConfig() { + void testEmptyDefaultConfig() { StoreWriter writer = PalDB.createWriter(storeFile); writer.close(); assertTrue(storeFile.exists()); try (StoreReader reader = PalDB.createReader(storeFile)) { - assertEquals(reader.size(), 0); + assertEquals(0, reader.size()); assertNull(reader.get(1, null)); } } @Test - public void testNewConfiguration() { + void testNewConfiguration() { assertNotNull(PalDB.newConfiguration()); } @Test - public void testNoFolder() { + void testNoFolder() { File file = new File("nofolder.store"); file.deleteOnExit(); StoreWriter writer = PalDB.createWriter(file, new Configuration()); @@ -98,27 +98,27 @@ public void testNoFolder() { } @Test - public void testReaderFileNotFound() { + void testReaderFileNotFound() { assertThrows(RuntimeException.class, () -> PalDB.createReader(new File("notfound"), PalDB.newConfiguration())); } @Test - public void testReaderNullFile() { + void testReaderNullFile() { assertThrows(NullPointerException.class, () -> PalDB.createReader((File) null, PalDB.newConfiguration())); } @Test - public void testReaderNullConfig() { + void testReaderNullConfig() { assertThrows(NullPointerException .class, () -> PalDB.createReader(new File("notfound"), null)); } @Test - public void testReaderNullStream() { + void testReaderNullStream() { assertThrows(NullPointerException.class, () -> PalDB.createReader((InputStream) null, PalDB.newConfiguration())); } @Test - public void testReaderNullConfigForStream() { + void testReaderNullConfigForStream() { assertThrows(NullPointerException.class, () -> { PalDB.createReader(new InputStream() { @Override @@ -130,22 +130,22 @@ public int read() { } @Test - public void testWriterNullFile() { + void testWriterNullFile() { assertThrows(NullPointerException.class, () -> PalDB.createWriter((File) null, PalDB.newConfiguration())); } @Test - public void testWriterNullConfig() { + void testWriterNullConfig() { assertThrows(NullPointerException.class, () -> PalDB.createWriter(new File("notfound"), null)); } @Test - public void testWriterNullStream() { + void testWriterNullStream() { assertThrows(NullPointerException.class, () -> PalDB.createWriter((OutputStream) null, PalDB.newConfiguration())); } @Test - public void testWriterNullConfigForStream() { + void testWriterNullConfigForStream() { assertThrows(NullPointerException.class, () -> PalDB.createWriter(new OutputStream() { @Override public void write(int i) { @@ -155,7 +155,7 @@ public void write(int i) { } @Test - public void testInvalidSegmentSize() { + void testInvalidSegmentSize() { StoreWriter writer = PalDB.createWriter(storeFile); writer.close(); @@ -165,7 +165,7 @@ public void testInvalidSegmentSize() { } @Test - public void testByteMarkEmpty() throws IOException { + void testByteMarkEmpty() throws IOException { try (FileOutputStream fos = new FileOutputStream(storeFile)) { fos.write(12345); fos.write(FormatVersion.getPrefixBytes()[0]); @@ -175,25 +175,25 @@ public void testByteMarkEmpty() throws IOException { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.size(), 0); + assertEquals(0, reader.size()); assertNull(reader.get(1, null)); } } @Test - public void testOneKey() { + void testOneKey() { try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { writer.put(1, "foo"); } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.size(), 1); - assertEquals(reader.get(1), "foo"); + assertEquals(1, reader.size()); + assertEquals("foo", reader.get(1)); } } @Test - public void testPutSerializedKey() throws IOException { + void testPutSerializedKey() throws IOException { StorageSerialization storageSerialization = new StorageSerialization(new Configuration()); byte[] serializedKey = storageSerialization.serializeKey(1); byte[] serializedValue = storageSerialization.serializeValue("foo"); @@ -203,13 +203,13 @@ public void testPutSerializedKey() throws IOException { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.size(), 1); - assertEquals(reader.get(1), "foo"); + assertEquals(1, reader.size()); + assertEquals("foo", reader.get(1)); } } @Test - public void testByteMarkOneKey() throws IOException { + void testByteMarkOneKey() throws IOException { try (FileOutputStream fos = new FileOutputStream(storeFile); StoreWriter writer = PalDB.createWriter(fos, new Configuration())) { fos.write(12345); @@ -220,13 +220,13 @@ public void testByteMarkOneKey() throws IOException { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.size(), 1); - assertEquals(reader.get(1), "foo"); + assertEquals(1, reader.size()); + assertEquals("foo", reader.get(1)); } } @Test - public void testTwoFirstKeyLength() { + void testTwoFirstKeyLength() { Integer key1 = 1; Integer key2 = 245; @@ -239,8 +239,8 @@ public void testTwoFirstKeyLength() { //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.get(key1).intValue(), 1); - assertEquals(reader.get(key2).intValue(), 6); + assertEquals(1, reader.get(key1).intValue()); + assertEquals(6, reader.get(key2).intValue()); assertNull(reader.get(0, null)); assertNull(reader.get(6, null)); assertNull(reader.get(244, null)); @@ -250,7 +250,7 @@ public void testTwoFirstKeyLength() { } @Test - public void testKeyLengthGap() { + void testKeyLengthGap() { Integer key1 = 1; Integer key2 = 2450; @@ -263,8 +263,8 @@ public void testKeyLengthGap() { //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.get(key1).intValue(), 1); - assertEquals(reader.get(key2).intValue(), 6); + assertEquals(1, reader.get(key1).intValue()); + assertEquals(6, reader.get(key2).intValue()); assertNull(reader.get(0, null)); assertNull(reader.get(6, null)); assertNull(reader.get(244, null)); @@ -276,7 +276,7 @@ public void testKeyLengthGap() { } @Test - public void testKeyLengthStartTwo() { + void testKeyLengthStartTwo() { Integer key1 = 245; Integer key2 = 2450; @@ -289,8 +289,8 @@ public void testKeyLengthStartTwo() { //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - assertEquals(reader.get(key1).intValue(), 1); - assertEquals(reader.get(key2).intValue(), 6); + assertEquals(1, reader.get(key1).intValue()); + assertEquals(6, reader.get(key2).intValue()); assertNull(reader.get(6, null)); assertNull(reader.get(244, null)); assertNull(reader.get(267, null)); @@ -301,7 +301,7 @@ public void testKeyLengthStartTwo() { } @Test - public void testDuplicateKeys() { + void testDuplicateKeys() { StoreWriter writer = PalDB.createWriter(storeFile, new Configuration()); writer.put(0, "ABC"); writer.put(0, "DGE"); @@ -309,7 +309,7 @@ public void testDuplicateKeys() { } @Test - public void testDataOnTwoBuffers() throws IOException { + void testDataOnTwoBuffers() throws IOException { Object[] keys = new Object[]{1, 2, 3}; Object[] values = new Object[]{GenerateTestData.generateStringData(100), GenerateTestData .generateStringData(10000), GenerateTestData.generateStringData(100)}; @@ -331,7 +331,7 @@ public void testDataOnTwoBuffers() throws IOException { } @Test - public void testIndexOnManyIndexBuffers() throws IOException { + void testIndexOnManyIndexBuffers() { Configuration configuration = new Configuration(); configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(32)); @@ -351,7 +351,7 @@ public void testIndexOnManyIndexBuffers() throws IOException { } @Test - public void testDataSizeOnTwoBuffers() throws IOException { + void testDataSizeOnTwoBuffers() throws IOException { Integer[] keys = new Integer[]{1, 2, 3}; String[] values = new String[]{GenerateTestData.generateStringData(100), GenerateTestData .generateStringData(10000), GenerateTestData.generateStringData(100)}; @@ -376,62 +376,62 @@ public void testDataSizeOnTwoBuffers() throws IOException { } @Test - public void testReadStringToString() { + void testReadStringToString() { testReadKeyToString(GenerateTestData.generateStringKeys(100)); } @Test - public void testReadIntToString() { + void testReadIntToString() { testReadKeyToString(GenerateTestData.generateIntKeys(100)); } @Test - public void testReadDoubleToString() { + void testReadDoubleToString() { testReadKeyToString(GenerateTestData.generateDoubleKeys(100)); } @Test - public void testReadLongToString() { + void testReadLongToString() { testReadKeyToString(GenerateTestData.generateLongKeys(100)); } @Test - public void testReadStringToInt() { + void testReadStringToInt() { testReadKeyToInt(GenerateTestData.generateStringKeys(100)); } @Test - public void testReadByteToInt() { + void testReadByteToInt() { testReadKeyToInt(GenerateTestData.generateByteKeys(100)); } @Test - public void testReadIntToInt() { + void testReadIntToInt() { testReadKeyToInt(GenerateTestData.generateIntKeys(100)); } @Test - public void testReadIntToIntArray() { + void testReadIntToIntArray() { testReadKeyToIntArray(GenerateTestData.generateIntKeys(100)); } @Test - public void testReadCompoundToString() { + void testReadCompoundToString() { testReadKeyToString(GenerateTestData.generateCompoundKeys(100)); } @Test - public void testReadCompoundByteToString() { + void testReadCompoundByteToString() { testReadKeyToString(new Object[]{GenerateTestData.generateCompoundByteKey()}); } @Test - public void testReadIntToNull() { + void testReadIntToNull() { testReadKeyToNull(GenerateTestData.generateIntKeys(100)); } @Test - public void testReadDisk() { + void testReadDisk() { Integer[] keys = GenerateTestData.generateIntKeys(10000); Configuration configuration = new Configuration(); @@ -456,7 +456,7 @@ public void testReadDisk() { } @Test - public void testIterate() { + void testIterate() { Integer[] keys = GenerateTestData.generateIntKeys(100); String[] values = GenerateTestData.generateStringData(keys.length, 12); @@ -653,12 +653,12 @@ private void writeStore(File location, K[] keys, V[] values, Configuration private void testKeyLength(Object key, int expectedLength) { StorageSerialization serializationImpl = new StorageSerialization(new Configuration()); - int keyLength = 0; + int keyLength; try { keyLength = serializationImpl.serializeKey(key).length; } catch (IOException e) { throw new RuntimeException(e); } - assertEquals(keyLength, expectedLength); + assertEquals(expectedLength, keyLength); } } diff --git a/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index 363a5c4..caf2143 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -25,22 +25,22 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; -import static com.linkedin.paldb.utils.TestTempUtils.deleteDirectory; +import static com.linkedin.paldb.utils.TempUtils.deleteDirectory; import static org.junit.jupiter.api.Assertions.*; -public class TestStoreReader { +class TestStoreReader { private Path tempDir; private File storeFile; @BeforeEach - public void setUp() throws IOException { + void setUp() throws IOException { tempDir = Files.createTempDirectory("tmp"); storeFile = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); } @AfterEach - public void cleanUp() { + void cleanUp() { deleteDirectory(tempDir.toFile()); } @@ -61,28 +61,28 @@ private StoreReader readerFor(V value) { } @Test - public void testFile() { + void testFile() { try (var reader = readerFor(true)) { assertEquals(reader.getFile(), storeFile); } } @Test - public void testSize() { + void testSize() { try (var reader = readerFor(true)) { - assertEquals(reader.size(), 1); + assertEquals(1, reader.size()); } } @Test - public void testStoreClosed() { + void testStoreClosed() { var reader = readerFor(true); reader.close(); assertThrows(IllegalStateException.class, () -> reader.get(0)); } @Test - public void testGetBoolean() { + void testGetBoolean() { try (var reader = readerFor(true)) { assertTrue(reader.get(0)); assertTrue(reader.get(0, false)); @@ -91,14 +91,14 @@ public void testGetBoolean() { } @Test - public void testGetBooleanMissing() { + void testGetBooleanMissing() { try (var reader = readerFor(true)) { assertNull(reader.get(-1)); } } @Test - public void testGetByte() { + void testGetByte() { try (var reader = readerFor((byte)1)) { assertEquals(reader.get(0).byteValue(), (byte) 1); assertEquals(reader.get(0, (byte) 5).byteValue(), (byte) 1); @@ -107,300 +107,300 @@ public void testGetByte() { } @Test - public void testGetByteMissing() { + void testGetByteMissing() { try (var reader = readerFor((byte)1)) { assertNull(reader.get(-1)); } } @Test - public void testGetChar() { + void testGetChar() { try (var reader = readerFor('a')) { - assertEquals(reader.get(0).charValue(), 'a'); - assertEquals(reader.get(0, 'b').charValue(), 'a'); - assertEquals(reader.get(-1, 'b').charValue(), 'b'); + assertEquals('a', reader.get(0).charValue()); + assertEquals('a', reader.get(0, 'b').charValue()); + assertEquals('b', reader.get(-1, 'b').charValue()); } } @Test - public void testGetCharMissing() { + void testGetCharMissing() { try (var reader = readerFor('a')) { assertNull(reader.get(-1)); } } @Test - public void testGetDouble() { + void testGetDouble() { try (var reader = readerFor(1.0)) { - assertEquals(reader.get(0).doubleValue(), 1.0); - assertEquals(reader.get(0, 2.0).doubleValue(), 1.0); - assertEquals(reader.get(-1, 2.0).doubleValue(), 2.0); + assertEquals(1.0, reader.get(0).doubleValue()); + assertEquals(1.0, reader.get(0, 2.0).doubleValue()); + assertEquals(2.0, reader.get(-1, 2.0).doubleValue()); } } @Test - public void testGetDoubleMissing() { + void testGetDoubleMissing() { try (var reader = readerFor(1.0)) { assertNull(reader.get(-1)); } } @Test - public void testGetFloat() { + void testGetFloat() { try (var reader = readerFor(1f)) { - assertEquals(reader.get(0).floatValue(), 1f); - assertEquals(reader.get(0, 2f).floatValue(), 1f); - assertEquals(reader.get(-1, 2f).floatValue(), 2f); + assertEquals(1f, reader.get(0).floatValue()); + assertEquals(1f, reader.get(0, 2f).floatValue()); + assertEquals(2f, reader.get(-1, 2f).floatValue()); } } @Test - public void testGetFloatMissing() { + void testGetFloatMissing() { try (var reader = readerFor(1.0)) { assertNull(reader.get(-1)); } } @Test - public void testGetShort() { + void testGetShort() { try (var reader = readerFor((short) 1)) { - assertEquals(reader.get(0).shortValue(), (short) 1); - assertEquals(reader.get(0, (short) 2).shortValue(), (short) 1); - assertEquals(reader.get(-1, (short) 2).shortValue(), (short) 2); + assertEquals((short) 1, reader.get(0).shortValue()); + assertEquals((short) 1, reader.get(0, (short) 2).shortValue()); + assertEquals((short) 2, reader.get(-1, (short) 2).shortValue()); } } @Test - public void testGetShortMissing() { + void testGetShortMissing() { try (var reader = readerFor((short) 1)) { assertNull(reader.get(-1)); } } @Test - public void testGetInt() { + void testGetInt() { try (var reader = readerFor(1)) { - assertEquals(reader.get(0).intValue(), 1); - assertEquals(reader.get(0, 2).intValue(), 1); - assertEquals(reader.get(-1, 2).intValue(), 2); + assertEquals(1, reader.get(0).intValue()); + assertEquals(1, reader.get(0, 2).intValue()); + assertEquals(2, reader.get(-1, 2).intValue()); } } @Test - public void testGetIntMissing() { + void testGetIntMissing() { try (var reader = readerFor(1)) { assertNull(reader.get(-1)); } } @Test - public void testGetLong() { + void testGetLong() { try (var reader = readerFor(1L)) { - assertEquals(reader.get(0).longValue(), 1L); - assertEquals(reader.get(0, 2L).longValue(), 1L); - assertEquals(reader.get(-1, 2L).longValue(), 2L); + assertEquals(1L, reader.get(0).longValue()); + assertEquals(1L, reader.get(0, 2L).longValue()); + assertEquals(2L, reader.get(-1, 2L).longValue()); } } @Test - public void testGetLongMissing() { + void testGetLongMissing() { try (var reader = readerFor(1L)) { assertNull(reader.get(-1)); } } @Test - public void testGetString() { + void testGetString() { try (var reader = readerFor("foo")) { - assertEquals(reader.get(0), "foo"); - assertEquals(reader.get(0, "bar"), "foo"); - assertEquals(reader.get(-1, "bar"), "bar"); + assertEquals("foo", reader.get(0)); + assertEquals("foo", reader.get(0, "bar")); + assertEquals("bar", reader.get(-1, "bar")); } } @Test - public void testGetStringMissing() { + void testGetStringMissing() { try (var reader = readerFor("foo")) { assertNull(reader.get(-1)); } } @Test - public void testGetBooleanArray() { + void testGetBooleanArray() { try (var reader = readerFor(new boolean[]{true})) { - assertArrayEquals(reader.get(0), new boolean[]{true}); - assertArrayEquals(reader.get(0, new boolean[]{false}), new boolean[]{true}); - assertArrayEquals(reader.get(-1, new boolean[]{false}), new boolean[]{false}); + assertArrayEquals(new boolean[]{true}, reader.get(0)); + assertArrayEquals(new boolean[]{true}, reader.get(0, new boolean[]{false})); + assertArrayEquals(new boolean[]{false}, reader.get(-1, new boolean[]{false})); } } @Test - public void testGetBooleanArrayMissing() { + void testGetBooleanArrayMissing() { try (var reader = readerFor(new boolean[]{true})) { assertNull(reader.get(-1)); } } @Test - public void testGetByteArray() { + void testGetByteArray() { try (var reader = readerFor(new byte[]{1})) { - assertArrayEquals(reader.get(0), new byte[]{1}); - assertArrayEquals(reader.get(0, new byte[]{2}), new byte[]{1}); - assertArrayEquals(reader.get(-1, new byte[]{2}), new byte[]{2}); + assertArrayEquals(new byte[]{1}, reader.get(0)); + assertArrayEquals(new byte[]{1}, reader.get(0, new byte[]{2})); + assertArrayEquals(new byte[]{2}, reader.get(-1, new byte[]{2})); } } @Test - public void testGetByteArrayMissing() { + void testGetByteArrayMissing() { try (var reader = readerFor(new byte[]{1})) { assertNull(reader.get(-1)); } } @Test - public void testGetCharArray() { + void testGetCharArray() { try (var reader = readerFor(new char[]{'a'})) { - assertArrayEquals(reader.get(0), new char[]{'a'}); - assertArrayEquals(reader.get(0, new char[]{'b'}), new char[]{'a'}); - assertArrayEquals(reader.get(-1, new char[]{'b'}), new char[]{'b'}); + assertArrayEquals(new char[]{'a'}, reader.get(0)); + assertArrayEquals(new char[]{'a'}, reader.get(0, new char[]{'b'})); + assertArrayEquals(new char[]{'b'}, reader.get(-1, new char[]{'b'})); } } @Test - public void testGetCharArrayMissing() { + void testGetCharArrayMissing() { try (var reader = readerFor(new char[]{'a'})) { assertNull(reader.get(-1)); } } @Test - public void testGetDoubleArray() { + void testGetDoubleArray() { try (var reader = readerFor(new double[]{1.0})) { - assertArrayEquals(reader.get(0), new double[]{1.0}); - assertArrayEquals(reader.get(0, new double[]{2.0}), new double[]{1.0}); - assertArrayEquals(reader.get(-1, new double[]{2.0}), new double[]{2.0}); + assertArrayEquals(new double[]{1.0}, reader.get(0)); + assertArrayEquals(new double[]{1.0}, reader.get(0, new double[]{2.0})); + assertArrayEquals(new double[]{2.0}, reader.get(-1, new double[]{2.0})); } } @Test - public void testGetDoubleArrayMissing() { + void testGetDoubleArrayMissing() { try (var reader = readerFor(new double[]{1.0})) { assertNull(reader.get(-1)); } } @Test - public void testGetFloatArray() { + void testGetFloatArray() { try (var reader = readerFor(new float[]{1f})) { - assertArrayEquals(reader.get(0), new float[]{1f}); - assertArrayEquals(reader.get(0, new float[]{2f}), new float[]{1f}); - assertArrayEquals(reader.get(-1, new float[]{2f}), new float[]{2f}); + assertArrayEquals(new float[]{1f}, reader.get(0)); + assertArrayEquals(new float[]{1f}, reader.get(0, new float[]{2f})); + assertArrayEquals(new float[]{2f}, reader.get(-1, new float[]{2f})); } } @Test - public void testGetFloatArrayMissing() { + void testGetFloatArrayMissing() { try (var reader = readerFor(new float[]{1f})) { assertNull(reader.get(-1)); } } @Test - public void testGetShortArray() { + void testGetShortArray() { try (var reader = readerFor(new short[]{1})) { - assertArrayEquals(reader.get(0), new short[]{1}); - assertArrayEquals(reader.get(0, new short[]{2}), new short[]{1}); - assertArrayEquals(reader.get(-1, new short[]{2}), new short[]{2}); + assertArrayEquals(new short[]{1}, reader.get(0)); + assertArrayEquals(new short[]{1}, reader.get(0, new short[]{2})); + assertArrayEquals(new short[]{2}, reader.get(-1, new short[]{2})); } } @Test - public void testGetShortArrayMissing() { + void testGetShortArrayMissing() { try (var reader = readerFor(new short[]{1})) { assertNull(reader.get(-1)); } } @Test - public void testGetIntArray() { + void testGetIntArray() { try (var reader = readerFor(new int[]{1})) { - assertArrayEquals(reader.get(0), new int[]{1}); - assertArrayEquals(reader.get(0, new int[]{2}), new int[]{1}); - assertArrayEquals(reader.get(-1, new int[]{2}), new int[]{2}); + assertArrayEquals(new int[]{1}, reader.get(0)); + assertArrayEquals(new int[]{1}, reader.get(0, new int[]{2})); + assertArrayEquals(new int[]{2}, reader.get(-1, new int[]{2})); } } @Test - public void testGetIntArrayMissing() { + void testGetIntArrayMissing() { try (var reader = readerFor(new int[]{1})) { assertNull(reader.get(-1)); } } @Test - public void testGetLongArray() { + void testGetLongArray() { try (var reader = readerFor(new long[]{1L})) { - assertArrayEquals(reader.get(0), new long[]{1L}); - assertArrayEquals(reader.get(0, new long[]{2L}), new long[]{1L}); - assertArrayEquals(reader.get(-1, new long[]{2L}), new long[]{2L}); + assertArrayEquals(new long[]{1L}, reader.get(0)); + assertArrayEquals(new long[]{1L}, reader.get(0, new long[]{2L})); + assertArrayEquals(new long[]{2L}, reader.get(-1, new long[]{2L})); } } @Test - public void testGetLongArrayMissing() { + void testGetLongArrayMissing() { try (var reader = readerFor(new long[]{1L})) { assertNull(reader.get(-1)); } } @Test - public void testGetStringArray() { + void testGetStringArray() { try (var reader = readerFor(new String[]{"foo"})) { - assertArrayEquals(reader.get(0), new String[]{"foo"}); - assertArrayEquals(reader.get(0, new String[]{"bar"}), new String[]{"foo"}); - assertArrayEquals(reader.get(-1, new String[]{"bar"}), new String[]{"bar"}); + assertArrayEquals(new String[]{"foo"}, reader.get(0)); + assertArrayEquals(new String[]{"foo"}, reader.get(0, new String[]{"bar"})); + assertArrayEquals(new String[]{"bar"}, reader.get(-1, new String[]{"bar"})); } } @Test - public void testGetStringArrayMissing() { + void testGetStringArrayMissing() { try (var reader = readerFor(new String[]{"foo"})) { assertNull(reader.get(-1)); } } @Test - public void testGetMissing() { + void testGetMissing() { try (var reader = readerFor("foo")) { assertNull(reader.get(-1)); } } @Test - public void testGetArray() { + void testGetArray() { try (var reader = readerFor(new Object[]{"foo"})) { - assertArrayEquals(reader.get(0), new Object[]{"foo"}); - assertArrayEquals(reader.get(0, new Object[]{"bar"}), new Object[]{"foo"}); - assertArrayEquals(reader.get(-1, new Object[]{"bar"}), new Object[]{"bar"}); + assertArrayEquals(new Object[]{"foo"}, reader.get(0)); + assertArrayEquals(new Object[]{"foo"}, reader.get(0, new Object[]{"bar"})); + assertArrayEquals(new Object[]{"bar"}, reader.get(-1, new Object[]{"bar"})); } } @Test - public void testGetArrayMissing() { + void testGetArrayMissing() { try (var reader = readerFor(new Object[]{"foo"})) { assertNull(reader.get(-1)); } } @Test - public void testGetPoint() { + void testGetPoint() { try (var reader = readerFor(new Point(4, 56))) { - assertEquals(reader.get(0), new Point(4, 56)); + assertEquals(new Point(4, 56), reader.get(0)); } } @Test - public void testIterator() { + void testIterator() { var values = List.of("foo", "bar"); try (var reader = readerForMany(values.get(0), values.get(1))) { var iter = reader.iterable(); @@ -417,18 +417,18 @@ public void testIterator() { } @Test - public void testIterate() { + void testIterate() { var values = List.of("foo", "bar"); try (var reader = readerForMany(values.get(0), values.get(1))) { for (var entry: reader) { var val = values.get(entry.getKey()); - assertEquals(entry.getValue(), val); + assertEquals(val, entry.getValue()); } } } @Test - public void testKeyIterator() { + void testKeyIterator() { var values = List.of("foo", "bar"); try (var reader = readerForMany(values.get(0), values.get(1))) { var iter = reader.keys(); @@ -444,12 +444,12 @@ public void testKeyIterator() { actual.add(k); expected.add(i); } - assertEquals(actual, expected); + assertEquals(expected, actual); } } @Test - public void testMultiThreadRead() throws InterruptedException { + void testMultiThreadRead() throws InterruptedException { int threadCount = 50; final CountDownLatch latch = new CountDownLatch(threadCount); final AtomicBoolean success = new AtomicBoolean(true); @@ -460,8 +460,8 @@ public void testMultiThreadRead() throws InterruptedException { try { for(int c = 0; c < 100000; c++) { if(!success.get())break; - assertEquals(reader.get(1), "any"); - assertEquals(reader.get(0), "foobar"); + assertEquals("any", reader.get(1)); + assertEquals("foobar", reader.get(0)); } } catch (Throwable error){ error.printStackTrace(); @@ -497,6 +497,5 @@ public void write(DataOutput output, Point input) output.writeInt(input.x); output.writeInt(input.y); } - } } diff --git a/src/test/java/com/linkedin/paldb/performance/TestMemoryUsageHashMap.java b/src/test/java/com/linkedin/paldb/performance/TestMemoryUsageHashMap.java index cc9ba3e..0aa77e5 100644 --- a/src/test/java/com/linkedin/paldb/performance/TestMemoryUsageHashMap.java +++ b/src/test/java/com/linkedin/paldb/performance/TestMemoryUsageHashMap.java @@ -21,12 +21,12 @@ @Disabled @Tag("performance") -public class TestMemoryUsageHashMap { +class TestMemoryUsageHashMap { private Set ref; @Test - public void testMemoryUsage() { + void testMemoryUsage() { System.out.println("MEMORY USAGE (HashSet int)\n\n"); System.out.println("KEYS;MB"); @@ -37,14 +37,11 @@ public void testMemoryUsage() { // Benchmark final int cnt = i; NanoBench nanoBench = NanoBench.create(); - nanoBench.memoryOnly().warmUps(2).measurements(10).measure(String.format("Measure memory for %d keys", i), new Runnable() { - @Override - public void run() { - Random setRandom = new Random(4532); - ref = new HashSet(cnt); - while (ref.size() < cnt) { - ref.add(setRandom.nextInt(Integer.MAX_VALUE)); - } + nanoBench.memoryOnly().warmUps(2).measurements(10).measure(String.format("Measure memory for %d keys", i), () -> { + Random setRandom = new Random(4532); + ref = new HashSet<>(cnt); + while (ref.size() < cnt) { + ref.add(setRandom.nextInt(Integer.MAX_VALUE)); } }); diff --git a/src/test/java/com/linkedin/paldb/performance/TestReadThroughput.java b/src/test/java/com/linkedin/paldb/performance/TestReadThroughput.java index 8610fb8..106908e 100644 --- a/src/test/java/com/linkedin/paldb/performance/TestReadThroughput.java +++ b/src/test/java/com/linkedin/paldb/performance/TestReadThroughput.java @@ -29,19 +29,20 @@ @Disabled @Tag("performance") -public class TestReadThroughput { +class TestReadThroughput { private File testFolder = createTempDir(); private static final int READS = 500000; + @SuppressWarnings("ResultOfMethodCallIgnored") @BeforeEach - public void setUp() { + void setUp() { DirectoryUtils.deleteDirectory(testFolder); testFolder.mkdir(); } @AfterEach - public void cleanUp() { + void cleanUp() { DirectoryUtils.deleteDirectory(testFolder); } @@ -54,7 +55,7 @@ private static File createTempDir() { } @Test - public void testReadThroughput() { + void testReadThroughput() { List measures = new ArrayList<>(); int max = 10000000; @@ -68,7 +69,7 @@ public void testReadThroughput() { } @Test - public void testReadThroughputMultiThread() { + void testReadThroughputMultiThread() { List measures = new ArrayList<>(); int max = 10000000; @@ -82,7 +83,7 @@ public void testReadThroughputMultiThread() { } @Test - public void testReadThroughputWithCache() { + void testReadThroughputWithCache() { List measures = new ArrayList<>(); int max = 10000000; @@ -96,7 +97,7 @@ public void testReadThroughputWithCache() { } @Test - public void testReadThroughputWithCacheRandomFinds() { + void testReadThroughputWithCacheRandomFinds() { List measures = new ArrayList<>(); int max = 10000000; @@ -110,7 +111,7 @@ public void testReadThroughputWithCacheRandomFinds() { } @Test - public void testReadThroughputWithCacheRandomFindsMultipleThreads() { + void testReadThroughputWithCacheRandomFindsMultipleThreads() { List measures = new ArrayList<>(); int max = 10000000; diff --git a/src/test/java/com/linkedin/paldb/performance/TestReadThroughputLevelDB.java b/src/test/java/com/linkedin/paldb/performance/TestReadThroughputLevelDB.java index 3c0dda8..706eb9f 100644 --- a/src/test/java/com/linkedin/paldb/performance/TestReadThroughputLevelDB.java +++ b/src/test/java/com/linkedin/paldb/performance/TestReadThroughputLevelDB.java @@ -27,25 +27,25 @@ @Disabled @Tag("performance") -public class TestReadThroughputLevelDB { +class TestReadThroughputLevelDB { private File TEST_FOLDER = new File("testreadthroughputleveldb"); private final int READS = 500000; @BeforeEach - public void setUp() { + void setUp() { DirectoryUtils.deleteDirectory(TEST_FOLDER); TEST_FOLDER.mkdir(); } @AfterEach - public void cleanUp() { + void cleanUp() { DirectoryUtils.deleteDirectory(TEST_FOLDER); } @Test - public void testReadThroughput() { - List measures = new ArrayList(); + void testReadThroughput() { + List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { @@ -72,7 +72,7 @@ private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, f options.compressionType(CompressionType.NONE); options.verifyChecksums(false); options.blockSize(1024); - options.cacheSize(0l); + options.cacheSize(0L); DB db = null; try { db = factory.open(file, options); @@ -104,21 +104,18 @@ private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, f // Measure nanoBench.cpuOnly().warmUps(5).measurements(20) - .measure("Measure %d reads for %d keys with cache", new Runnable() { - @Override - public void run() { - Random r = new Random(); - int length = keys.length; - for (int i = 0; i < READS; i++) { - int index; - if (i % 2 == 0 && frequentReads) { - index = r.nextInt(length / 10); - } else { - index = r.nextInt(length); - } - Integer key = keys[index]; - reader.get(bytes(key.toString())); + .measure("Measure %d reads for %d keys with cache", () -> { + Random r = new Random(); + int length = keys.length; + for (int i = 0; i < READS; i++) { + int index; + if (i % 2 == 0 && frequentReads) { + index = r.nextInt(length / 10); + } else { + index = r.nextInt(length); } + Integer key = keys[index]; + reader.get(bytes(key.toString())); } }); } catch (IOException e) { @@ -135,7 +132,7 @@ public void run() { // Return measure double rps = READS * nanoBench.getTps(); - return new Measure(DirectoryUtils.folderSize(TEST_FOLDER), rps, valueLength, 0l, keys.length); + return new Measure(DirectoryUtils.folderSize(TEST_FOLDER), rps, valueLength, 0L, keys.length); } private void report(String title, List measures) { diff --git a/src/test/java/com/linkedin/paldb/performance/TestReadThrouputHashMap.java b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputHashMap.java index 615b10b..6dc9362 100644 --- a/src/test/java/com/linkedin/paldb/performance/TestReadThrouputHashMap.java +++ b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputHashMap.java @@ -19,14 +19,16 @@ import java.util.*; +import static org.junit.jupiter.api.Assertions.assertTrue; + @Disabled @Tag("performance") -public class TestReadThrouputHashMap { +class TestReadThrouputHashMap { private final int READS = 500000; @Test - public void testReadThroughput() { + void testReadThroughput() { System.out.println("READ THROUGHPUT (HashSet int)\n\n"); System.out.println("KEYS;RPS"); @@ -36,7 +38,7 @@ public void testReadThroughput() { // Prepare set Random setRandom = new Random(4532); - final Set set = new HashSet(i); + final Set set = new HashSet<>(i); while (set.size() < i) { set.add(setRandom.nextInt(Integer.MAX_VALUE)); } @@ -45,12 +47,9 @@ public void testReadThroughput() { // Benchmark final Random random = new Random(i); NanoBench nanoBench = NanoBench.create(); - nanoBench.cpuOnly().measure(String.format("Measure %d reads for %d keys", READS, i), new Runnable() { - @Override - public void run() { - for (int i = 0; i < READS; i++) { - set.contains(keys[random.nextInt(keys.length)]); - } + nanoBench.cpuOnly().measure(String.format("Measure %d reads for %d keys", READS, i), () -> { + for (int i1 = 0; i1 < READS; i1++) { + assertTrue(set.contains(keys[random.nextInt(keys.length)])); } }); diff --git a/src/test/java/com/linkedin/paldb/performance/TestReadThrouputRocksDB.java b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputRocksDB.java index 0f01ca1..7155c61 100644 --- a/src/test/java/com/linkedin/paldb/performance/TestReadThrouputRocksDB.java +++ b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputRocksDB.java @@ -25,30 +25,30 @@ @Disabled @Tag("performance") -public class TestReadThrouputRocksDB { +class TestReadThrouputRocksDB { private File TEST_FOLDER = new File("testreadthroughputrocksdb"); private final int READS = 500000; @BeforeEach - public void loadLibrary() { + void loadLibrary() { RocksDB.loadLibrary(); } @BeforeEach - public void setUp() { + void setUp() { DirectoryUtils.deleteDirectory(TEST_FOLDER); TEST_FOLDER.mkdir(); } @AfterEach - public void cleanUp() { + void cleanUp() { DirectoryUtils.deleteDirectory(TEST_FOLDER); } @Test - public void testReadThroughput() { - List measures = new ArrayList(); + void testReadThroughput() { + List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { @@ -78,8 +78,14 @@ private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, f options.setCompactionStyle(CompactionStyle.LEVEL); options.setMaxOpenFiles(-1); + var wOptions = new WriteOptions(); + wOptions.setDisableWAL(true); + + var fOptions = new FlushOptions(); + fOptions.setWaitForFlush(true); + final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); - tableOptions.setFilter(new BloomFilter(10, false)); + tableOptions.setFilterPolicy(new BloomFilter(10, false)); options.setTableFormatConfig(tableOptions); RocksDB db = null; @@ -88,19 +94,22 @@ private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, f for (Integer key : keys) { if (valueLength == 0) { - db.put(key.toString().getBytes(), "1".getBytes()); + db.put(wOptions, key.toString().getBytes(), "1".getBytes()); } else { - db.put(key.toString().getBytes(), RandomStringUtils.randomAlphabetic(valueLength).getBytes()); + db.put(wOptions, key.toString().getBytes(), RandomStringUtils.randomAlphabetic(valueLength).getBytes()); } } + db.flush(fOptions); } catch (RocksDBException e) { throw new RuntimeException(e); } finally { + wOptions.close(); + fOptions.close(); if (db != null) { db.close(); } } - options.dispose(); + options.close(); final ReadOptions readOptions = new ReadOptions(); readOptions.setVerifyChecksums(false); @@ -121,24 +130,21 @@ private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, f // Measure nanoBench.cpuOnly().warmUps(5).measurements(20) - .measure("Measure %d reads for %d keys with cache", new Runnable() { - @Override - public void run() { - Random r = new Random(); - int length = keys.length; - for (int i = 0; i < READS; i++) { - int index; - if (i % 2 == 0 && frequentReads) { - index = r.nextInt(length / 10); - } else { - index = r.nextInt(length); - } - Integer key = keys[index]; - try { - reader.get(readOptions, key.toString().getBytes()); - } catch (RocksDBException e) { - - } + .measure("Measure %d reads for %d keys with cache", () -> { + Random r = new Random(); + int length = keys.length; + for (int i = 0; i < READS; i++) { + int index; + if (i % 2 == 0 && frequentReads) { + index = r.nextInt(length / 10); + } else { + index = r.nextInt(length); + } + Integer key = keys[index]; + try { + reader.get(readOptions, key.toString().getBytes()); + } catch (RocksDBException e) { + } } }); @@ -152,7 +158,7 @@ public void run() { // Return measure double rps = READS * nanoBench.getTps(); - return new Measure(DirectoryUtils.folderSize(TEST_FOLDER), rps, valueLength, 0l, keys.length); + return new Measure(DirectoryUtils.folderSize(TEST_FOLDER), rps, valueLength, 0L, keys.length); } private void report(String title, List measures) { diff --git a/src/test/java/com/linkedin/paldb/performance/TestStoreSize.java b/src/test/java/com/linkedin/paldb/performance/TestStoreSize.java index d2023d3..cc73a38 100644 --- a/src/test/java/com/linkedin/paldb/performance/TestStoreSize.java +++ b/src/test/java/com/linkedin/paldb/performance/TestStoreSize.java @@ -22,23 +22,23 @@ import java.io.File; @Disabled @Tag("performance") -public class TestStoreSize { +class TestStoreSize { private File TEST_FOLDER = new File("teststoresize"); @BeforeEach - public void setUp() { + void setUp() { DirectoryUtils.deleteDirectory(TEST_FOLDER); TEST_FOLDER.mkdir(); } @AfterEach - public void cleanUp() { + void cleanUp() { DirectoryUtils.deleteDirectory(TEST_FOLDER); } @Test - public void testStoreSizeSet() { + void testStoreSizeSet() { System.out.println("STORE SIZE (SET int -> boolean)\n\n"); System.out.println("FILE LENGTH;KEYS;BYTES_PER_KEY"); @@ -49,7 +49,7 @@ public void testStoreSizeSet() { // Prepare store final Integer[] keys = GenerateTestData.generateRandomIntKeys(i, Integer.MAX_VALUE); final File storeFile = new File(TEST_FOLDER, i + ".store"); - StoreWriter writer = PalDB.createWriter(storeFile, new Configuration()); + StoreWriter writer = PalDB.createWriter(storeFile, new Configuration()); for (Integer key : keys) { writer.put(key, Boolean.TRUE); } diff --git a/src/test/java/com/linkedin/paldb/performance/utils/DirectoryUtils.java b/src/test/java/com/linkedin/paldb/performance/utils/DirectoryUtils.java index 2a03013..05c2773 100644 --- a/src/test/java/com/linkedin/paldb/performance/utils/DirectoryUtils.java +++ b/src/test/java/com/linkedin/paldb/performance/utils/DirectoryUtils.java @@ -1,6 +1,7 @@ package com.linkedin.paldb.performance.utils; import java.io.File; +import java.util.Objects; public class DirectoryUtils { @@ -9,11 +10,11 @@ public static boolean deleteDirectory(File directory) { if (directory.exists()) { File[] files = directory.listFiles(); if (null != files) { - for (int i = 0; i < files.length; i++) { - if (files[i].isDirectory()) { - deleteDirectory(files[i]); + for (final File file : files) { + if (file.isDirectory()) { + deleteDirectory(file); } else { - files[i].delete(); + file.delete(); } } } @@ -23,7 +24,7 @@ public static boolean deleteDirectory(File directory) { public static long folderSize(File directory) { long length = 0; - for (File file : directory.listFiles()) { + for (File file : Objects.requireNonNull(directory.listFiles())) { if (file == null) break; if (file.isFile()) { length += file.length(); diff --git a/src/test/java/com/linkedin/paldb/performance/utils/NanoBench.java b/src/test/java/com/linkedin/paldb/performance/utils/NanoBench.java index 43b5637..8139e1a 100644 --- a/src/test/java/com/linkedin/paldb/performance/utils/NanoBench.java +++ b/src/test/java/com/linkedin/paldb/performance/utils/NanoBench.java @@ -23,8 +23,8 @@ public static NanoBench create() { private int numberOfWarmUp = 20; private List listeners; - public NanoBench() { - listeners = new ArrayList(2); + private NanoBench() { + listeners = new ArrayList<>(2); listeners.add(new CPUMeasure(logger)); listeners.add(new MemoryUsage(logger)); } @@ -40,14 +40,14 @@ public NanoBench warmUps(int numberOfWarmups) { } public NanoBench cpuAndMemory() { - listeners = new ArrayList(2); + listeners = new ArrayList<>(2); listeners.add(new CPUMeasure(logger)); listeners.add(new MemoryUsage(logger)); return this; } public NanoBench bytesOnly() { - listeners = new ArrayList(1); + listeners = new ArrayList<>(1); listeners.add(new BytesMeasure(logger)); return this; } @@ -61,13 +61,13 @@ public static Logger getLogger() { } public NanoBench cpuOnly() { - listeners = new ArrayList(1); + listeners = new ArrayList<>(1); listeners.add(new CPUMeasure(logger)); return this; } public NanoBench memoryOnly() { - listeners = new ArrayList(1); + listeners = new ArrayList<>(1); listeners.add(new MemoryUsage(logger)); return this; } @@ -133,7 +133,7 @@ public void measure(String label, Runnable task) { } } - static int[] arrayStress = new int[10000]; + private static int[] arrayStress = new int[10000]; private void stress() { int m = 0; @@ -172,7 +172,7 @@ private static class TimeMeasureProxy implements Runnable { private Runnable runnable; private List listeners; - public TimeMeasureProxy(MeasureState state, Runnable runnable, List listeners) { + TimeMeasureProxy(MeasureState state, Runnable runnable, List listeners) { super(); this.state = state; this.runnable = runnable; @@ -210,15 +210,15 @@ private interface MeasureListener { public static abstract class BytesRunnable implements Runnable { - protected int measure; + int measure; public void run() { measure = runMeasure(); } - public abstract int runMeasure(); + abstract int runMeasure(); - public int getMeasure() { + int getMeasure() { return measure; } } @@ -235,7 +235,7 @@ private static class MeasureState implements Comparable { private int measurement; private int bytesMeasure; - public MeasureState(String label, long index, int measurement) { + MeasureState(String label, long index, int measurement) { super(); this.label = label; this.measurement = measurement; @@ -246,7 +246,7 @@ public long getIndex() { return index; } - public String getLabel() { + String getLabel() { return label; } @@ -258,35 +258,29 @@ public long getEndTime() { return endTime; } - public long getMeasurements() { + long getMeasurements() { return measurement; } - public long getMeasureTime() { + long getMeasureTime() { return endTime - startTime; } - public void startNow() { + void startNow() { this.startTime = System.nanoTime(); } - public void endNow() { + void endNow() { this.endTime = System.nanoTime(); } - public int getBytesMeasure() { + int getBytesMeasure() { return bytesMeasure; } @Override public int compareTo(MeasureState another) { - if (this.startTime > another.startTime) { - return -1; - } else if (this.startTime < another.startTime) { - return 1; - } else { - return 0; - } + return Long.compare(another.startTime, this.startTime); } } @@ -311,7 +305,7 @@ public static class CPUMeasure implements MeasureListener { private double finalAvg; private double finalTotal; - public CPUMeasure(Logger logger) { + CPUMeasure(Logger logger) { this.log = logger; } @@ -343,15 +337,15 @@ private void outputMeasureInfo(MeasureState state) { } } - public double getFinalAvg() { + double getFinalAvg() { return finalAvg; } - public double getFinalTotal() { + double getFinalTotal() { return finalTotal; } - public double getFinalTps() { + double getFinalTps() { return finalTps; } @@ -367,7 +361,7 @@ private static class BytesMeasure implements MeasureListener { private int count = 0; private long bytesUsed = 0; - public BytesMeasure(Logger logger) { + BytesMeasure(Logger logger) { this.log = logger; } @@ -417,7 +411,7 @@ private static class MemoryUsage implements MeasureListener { // Final private long finalBytes; - public MemoryUsage(Logger logger) { + MemoryUsage(Logger logger) { this.log = logger; } @@ -446,7 +440,7 @@ private void outputMeasureInfo(MeasureState state) { } } - public long getFinalBytes() { + long getFinalBytes() { return finalBytes; } @@ -467,7 +461,7 @@ public static class MemoryUtil { /** * Call GC until no more memory can be freed */ - public static void restoreJvm() { + static void restoreJvm() { int maxRestoreJvmLoops = 10; long memUsedPrev = memoryUsed(); for (int i = 0; i < maxRestoreJvmLoops; i++) { @@ -490,7 +484,7 @@ public static void restoreJvm() { * * @return heap memory used in bytes */ - public static long memoryUsed() { + static long memoryUsed() { Runtime rt = Runtime.getRuntime(); return rt.totalMemory() - rt.freeMemory(); } diff --git a/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java index 132d965..16df3e2 100644 --- a/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java +++ b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java @@ -11,7 +11,7 @@ import static org.junit.jupiter.api.Assertions.*; -public class BloomFilterTest { +class BloomFilterTest { private int elements = 1_000_000; private int bitsize = 10_000_000; diff --git a/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java b/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java index 2742231..5700375 100644 --- a/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java +++ b/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java @@ -18,25 +18,25 @@ import static org.junit.jupiter.api.Assertions.*; -public class TestFormatVersion { +class TestFormatVersion { @Test - public void testIs() { + void testIs() { assertTrue(FormatVersion.PALDB_V1.is(FormatVersion.PALDB_V1)); } @Test - public void testBytes() { - assertEquals(FormatVersion.fromBytes(FormatVersion.PALDB_V1.getBytes()), FormatVersion.PALDB_V1); + void testBytes() { + assertEquals(FormatVersion.PALDB_V1, FormatVersion.fromBytes(FormatVersion.PALDB_V1.getBytes())); } @Test - public void testPrefixBytes() { - assertArrayEquals(FormatVersion.getPrefixBytes(), "PALDB".getBytes()); + void testPrefixBytes() { + assertArrayEquals("PALDB".getBytes(), FormatVersion.getPrefixBytes()); } @Test - public void testGetLastVersion() { + void testGetLastVersion() { assertEquals(FormatVersion.getLatestVersion(), FormatVersion.values()[FormatVersion.values().length - 1]); } } diff --git a/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java b/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java index 34b42db..58170af 100644 --- a/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java +++ b/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java @@ -22,95 +22,92 @@ import static org.junit.jupiter.api.Assertions.*; -public class TestLongPacker { +class TestLongPacker { @Test - public void testPackInt() + void testPackInt() throws IOException { DataInputOutput dio = new DataInputOutput(); LongPacker.packInt(dio.reset(), 42); - assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), 42); + assertEquals(42, LongPacker.unpackInt(dio.reset(dio.toByteArray()))); } @Test - public void testPackIntZero() + void testPackIntZero() throws IOException { DataInputOutput dio = new DataInputOutput(); LongPacker.packInt(dio.reset(), 0); - assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), 0); + assertEquals(0, LongPacker.unpackInt(dio.reset(dio.toByteArray()))); } @Test - public void testPackIntMax() + void testPackIntMax() throws IOException { DataInputOutput dio = new DataInputOutput(); LongPacker.packInt(dio.reset(), Integer.MAX_VALUE); - assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), Integer.MAX_VALUE); + assertEquals(Integer.MAX_VALUE, LongPacker.unpackInt(dio.reset(dio.toByteArray()))); } @Test - public void testPackIntNeg() { + void testPackIntNeg() { DataInputOutput dio = new DataInputOutput(); assertThrows(IllegalArgumentException.class, () -> LongPacker.packInt(dio.reset(), -42)); } @Test - public void testPackLong() + void testPackLong() throws IOException { DataInputOutput dio = new DataInputOutput(); - LongPacker.packLong(dio.reset(), 42l); - assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), 42); + LongPacker.packLong(dio.reset(), 42L); + assertEquals(42, LongPacker.unpackLong(dio.reset(dio.toByteArray()))); } @Test - public void testPackLongZero() + void testPackLongZero() throws IOException { DataInputOutput dio = new DataInputOutput(); - LongPacker.packLong(dio.reset(), 0l); - assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), 0l); + LongPacker.packLong(dio.reset(), 0L); + assertEquals(0L, LongPacker.unpackLong(dio.reset(dio.toByteArray()))); } @Test - public void testPackLongBytes() - throws IOException { + void testPackLongBytes() { byte[] buf = new byte[15]; - LongPacker.packLong(buf, 42l); - assertEquals(LongPacker.unpackLong(buf), 42l); + LongPacker.packLong(buf, 42L); + assertEquals(42L, LongPacker.unpackLong(buf)); } @Test - public void testPackLongMax() + void testPackLongMax() throws IOException { DataInputOutput dio = new DataInputOutput(); LongPacker.packLong(dio.reset(), Long.MAX_VALUE); - assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), Long.MAX_VALUE); + assertEquals(Long.MAX_VALUE, LongPacker.unpackLong(dio.reset(dio.toByteArray()))); } @Test - public void testPackLongBytesMax() - throws IOException { + void testPackLongBytesMax() { byte[] buf = new byte[15]; LongPacker.packLong(buf, Long.MAX_VALUE); - assertEquals(LongPacker.unpackLong(buf), Long.MAX_VALUE); + assertEquals(Long.MAX_VALUE, LongPacker.unpackLong(buf)); } @Test - public void testPackLongNeg() - throws IOException { + void testPackLongNeg() { DataInputOutput dio = new DataInputOutput(); assertThrows(IllegalArgumentException.class, () -> LongPacker.packLong(dio.reset(), -42L)); } @Test - public void testPackLongBytesNeg() { + void testPackLongBytesNeg() { assertThrows(IllegalArgumentException.class, () -> LongPacker.packLong(new byte[15], -42L)); } @Test - public void test() throws IOException { + void test() throws IOException { DataInputOutput dio = new DataInputOutput(); LongPacker.packInt(dio.reset(), 5); ByteBuffer bb = ByteBuffer.wrap(dio.getBuf()); - assertEquals(LongPacker.unpackInt(bb), 5); + assertEquals(5, LongPacker.unpackInt(bb)); } } diff --git a/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java b/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java index 25faa75..d0b4aea 100644 --- a/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java +++ b/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java @@ -19,20 +19,20 @@ import static org.junit.jupiter.api.Assertions.*; -public class TestMurmur3 { +class TestMurmur3 { @Test - public void testHashEquals() { + void testHashEquals() { assertEquals(Murmur3.hash("foo".getBytes()), Murmur3.hash("foo".getBytes())); } @Test - public void testEmpty() { + void testEmpty() { assertTrue(Murmur3.hash(new byte[0]) > 0); } @Test - public void testSameHash() { + void testSameHash() { var bytes = "foo".getBytes(); assertEquals(Murmur3.hash(bytes, 42), MurmurHash3.hash32(bytes, bytes.length, 42)); } diff --git a/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java b/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java index 53480cd..4baf6b2 100644 --- a/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java +++ b/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java @@ -17,42 +17,30 @@ import org.junit.jupiter.api.Test; import java.io.*; +import java.nio.file.*; import static org.junit.jupiter.api.Assertions.*; -public class TestTempUtils { - - public static boolean deleteDirectory(File directoryToBeDeleted) { - if (directoryToBeDeleted.isDirectory()) { - File[] allContents = directoryToBeDeleted.listFiles(); - if (allContents != null) { - for (File file : allContents) { - deleteDirectory(file); - } - } - } - return directoryToBeDeleted.delete(); - } +class TestTempUtils { @Test - public void testTempDir() { + void testTempDir() { File file = TempUtils.createTempDir("foo"); assertTrue(file.exists()); assertTrue(file.isDirectory()); assertTrue(file.getName().contains("foo")); - file.delete(); + assertTrue(file.delete()); } @Test - public void testCopyIntoTempFile() - throws IOException { + void testCopyIntoTempFile() throws IOException { ByteArrayInputStream bis = new ByteArrayInputStream("foo".getBytes()); File file = TempUtils.copyIntoTempFile("bar", bis); assertTrue(file.exists()); assertTrue(file.isFile()); assertTrue(file.getName().contains("bar")); - assertEquals(bis.available(), 0); + assertEquals(0, bis.available()); ByteArrayOutputStream bos = new ByteArrayOutputStream(); FileInputStream fis = new FileInputStream(file); @@ -63,7 +51,21 @@ public void testCopyIntoTempFile() } fis.close(); bos.close(); - assertArrayEquals(bos.toByteArray(), "foo".getBytes()); + assertArrayEquals("foo".getBytes(), bos.toByteArray()); + } + + @Test + void should_delete_files_and_dir() throws IOException { + var testDir = Files.createTempDirectory("testDir"); + + for (int i = 0; i < 10; i++) { + Files.createTempFile(testDir, "tmp1", ".tmp"); + } + + assertEquals(10, Files.list(testDir).count()); + + TempUtils.deleteDirectory(testDir.toFile()); + assertFalse(Files.exists(testDir )); } } From c0812579bcd2aadd520ff8edcfb39bde1a195976 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Mon, 25 Nov 2019 13:38:30 +0200 Subject: [PATCH 5/5] More test coverage. Updated readme --- README.md | 11 +++++++ pom.xml | 2 +- .../com/linkedin/paldb/api/Serializer.java | 10 +++--- .../linkedin/paldb/impl/StorageWriter.java | 1 - .../com/linkedin/paldb/impl/TestStore.java | 33 +++++++++++++++++++ .../linkedin/paldb/utils/BloomFilterTest.java | 8 +++++ 6 files changed, 58 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a46e269..fbdfd4f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,17 @@ PalDB is an embeddable persistent key-value store with very fast read performanc PalDB's JAR is only 65K and has a single dependency (snappy, which isn't mandatory). It's also very easy to use with just a few configuration parameters. +**This is separate evolution of original PalDB.** + +Improvements from PalDB 1.0.2 +------------------- +- StoreReader is now fully thread-safe without any performance decrease. +- StoreReader and StoreWriter can be used with generics (StoreReader and StoreWriter) +- Typesafe ``PalDBConfigBuilder`` for easier configuration +- There are no limits on how many keys you can store +- Duplicates could be optionally allowed +- Bloom filters could be enabled for even better read performance in some cases + Performance ----------- diff --git a/pom.xml b/pom.xml index 2045845..a049446 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ net.soundvibe paldb - 2.0.1 + 2.0.2 jar paldb Embeddable persistent write-once key-value store diff --git a/src/main/java/com/linkedin/paldb/api/Serializer.java b/src/main/java/com/linkedin/paldb/api/Serializer.java index 064e7c5..d10f32e 100644 --- a/src/main/java/com/linkedin/paldb/api/Serializer.java +++ b/src/main/java/com/linkedin/paldb/api/Serializer.java @@ -29,9 +29,9 @@ * configuration.registerSerializer(new PointSerializer()); * * - * @param class type + * @param class type */ -public interface Serializer extends Serializable { +public interface Serializer extends Serializable { /** * Writes the instance input to the data output. @@ -39,7 +39,7 @@ public interface Serializer extends Serializable { * @param input instance * @throws IOException if an io error occurs */ - void write(DataOutput dataOutput, K input) throws IOException; + void write(DataOutput dataOutput, T input) throws IOException; /** * Reads the data input and creates the instance. @@ -48,7 +48,7 @@ public interface Serializer extends Serializable { * @return new instance of type K. * @throws IOException if an io error occurs */ - K read(DataInput dataInput) throws IOException; + T read(DataInput dataInput) throws IOException; - Class serializedClass(); + Class serializedClass(); } diff --git a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index c23c17d..a890a5b 100644 --- a/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -361,7 +361,6 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti } finally { Arrays.fill(indexBuffers, null); indexBuffers = null; - System.gc(); if (tempIndexFile.delete()) { log.info("Temporary index file {} has been deleted", tempIndexFile.getName()); } diff --git a/src/test/java/com/linkedin/paldb/impl/TestStore.java b/src/test/java/com/linkedin/paldb/impl/TestStore.java index 939bac5..8aa3221 100644 --- a/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -535,6 +535,39 @@ void should_allow_duplicates() { } } + @Test + void should_not_allow_put_null_keys() { + try (StoreWriter writer = PalDB.createWriter(storeFile)) { + assertThrows(NullPointerException.class, () -> writer.put((String)null, EMPTY_VALUE)); + assertThrows(NullPointerException.class, () -> writer.putAll((String[])null, new byte[][]{})); + } + } + + @Test + void should_not_allow_getting_null_keys() { + try (StoreWriter writer = PalDB.createWriter(storeFile)) { + writer.put("any value", EMPTY_VALUE); + } + + try (StoreReader reader = PalDB.createReader(storeFile)) { + assertThrows(NullPointerException.class, () -> reader.get(null)); + } + } + + @Test + void should_not_find_when_bloom_filter_enabled() { + var config = PalDBConfigBuilder.create() + .withEnableBloomFilter(true).build(); + try (StoreWriter writer = PalDB.createWriter(storeFile, config)) { + writer.put("abc", EMPTY_VALUE); + } + + try (StoreReader reader = PalDB.createReader(storeFile, config)) { + assertNull(reader.get("foo")); + assertArrayEquals(EMPTY_VALUE, reader.get("abc")); + } + } + private static final byte[] EMPTY_VALUE = new byte[0]; @Test diff --git a/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java index 16df3e2..95bb56d 100644 --- a/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java +++ b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java @@ -77,6 +77,14 @@ void should_be_equal() { assertEquals(sut1, sut2); } + @Test + void should_return_same_bits() { + var bits = new long[Math.max(1, (int) Math.ceil((double) 6235225 / Long.SIZE))]; + var sut = new BloomFilter(4, 6235225, bits); + + assertArrayEquals(bits, sut.bits()); + } + @Test void correctness() { System.out.println("Testing correctness.\n"+