From 255082ad837f1810ec24bc4446dd6ed29200d435 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 20:38:31 +0200 Subject: [PATCH 01/20] Initial support for java 11. Replaced logging with sl4j. Used try-with-resources in most places. Support for gradle 5. Other code cleanups. --- .gitignore | 2 + .travis.yml | 12 +- build.gradle | 2 +- gradle.properties | 4 +- paldb/build.gradle | 27 +-- .../com/linkedin/paldb/impl/ReaderImpl.java | 21 +- .../linkedin/paldb/impl/ReaderIterable.java | 6 +- .../paldb/impl/ReaderKeyIterable.java | 5 +- .../com/linkedin/paldb/impl/Serializers.java | 50 ++--- .../com/linkedin/paldb/impl/StorageCache.java | 18 +- .../linkedin/paldb/impl/StorageReader.java | 73 +++---- .../paldb/impl/StorageSerialization.java | 20 +- .../linkedin/paldb/impl/StorageWriter.java | 195 +++++++----------- .../com/linkedin/paldb/impl/StoreImpl.java | 31 ++- .../com/linkedin/paldb/impl/WriterImpl.java | 26 +-- .../linkedin/paldb/utils/DataInputOutput.java | 96 +++------ .../linkedin/paldb/utils/FormatVersion.java | 23 +-- .../com/linkedin/paldb/utils/LongPacker.java | 24 +-- .../com/linkedin/paldb/utils/TempUtils.java | 31 +-- .../com/linkedin/paldb/impl/TestStore.java | 65 +++--- .../linkedin/paldb/impl/TestStoreReader.java | 80 +++++-- paldb/src/test/resources/logback-test.xml | 15 ++ 22 files changed, 384 insertions(+), 442 deletions(-) create mode 100644 paldb/src/test/resources/logback-test.xml diff --git a/.gitignore b/.gitignore index a3c393f..c4914ae 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ out/ /ligradle .DS_Store /doc +gradle/ +gradlew* diff --git a/.travis.yml b/.travis.yml index 420571e..4e45e0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,13 @@ language: java -jdk: - - oraclejdk8 - - oraclejdk7 + +sudo: false +dist: trusty + +matrix: + include: + - jdk: openjdk11 + - jdk: openjdk12 + - jdk: openjdk13 script: - gradle build diff --git a/build.gradle b/build.gradle index 05e851e..5c5e282 100644 --- a/build.gradle +++ b/build.gradle @@ -3,7 +3,7 @@ buildscript { mavenCentral() } dependencies { - classpath "org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.4.0" + classpath "org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.8.3" } } diff --git a/gradle.properties b/gradle.properties index a8c1fd0..a6f9c9b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,6 @@ org.gradle.configureondemand=true ide.recursive=true org.gradle.parallel=false -VERSION_NAME=1.2.0 -GROUP=com.linkedin.paldb +VERSION_NAME=1.2.1 +GROUP=net.soundvibe.paldb ARCHIVE_NAME=paldb diff --git a/paldb/build.gradle b/paldb/build.gradle index ecacd6e..80d28fa 100644 --- a/paldb/build.gradle +++ b/paldb/build.gradle @@ -4,7 +4,8 @@ apply plugin: 'com.github.kt3k.coveralls' apply plugin: 'signing' apply plugin: 'maven' -sourceCompatibility = 1.6 +sourceCompatibility = JavaVersion.VERSION_11 +targetCompatibility = JavaVersion.VERSION_11 group = GROUP version = VERSION_NAME @@ -28,7 +29,7 @@ jar { } task perfTest(type: Test) { - testClassesDir = sourceSets.perfTest.output.classesDir + testClassesDirs = sourceSets.perfTest.output.classesDirs classpath = sourceSets.perfTest.runtimeClasspath } @@ -39,10 +40,12 @@ perfTest { } dependencies { - compile 'org.xerial.snappy:snappy-java:1.0.5' + compile 'org.xerial.snappy:snappy-java:1.1.7.3' + compile 'org.slf4j:slf4j-api:1.7.28' - testCompile 'org.testng:testng:6.8.8' + testCompile 'org.testng:testng:7.0.0' testCompile 'commons-lang:commons-lang:2.6' + testCompile 'ch.qos.logback:logback-classic:1.2.3' perfTestCompile configurations.testCompile @@ -92,16 +95,16 @@ if (project.hasProperty('release')) { authentication(userName: ossrhUser, password: ossrhPassword) } pom.project { - group 'com.linkedin.paldb' + group 'net.soundvibe.paldb' name 'paldb' description 'Embeddable persistent write-once key-value store' packaging 'jar' - url 'https://github.com/linkedin/PalDB' + url 'https://github.com/soundvibe/PalDB' scm { - connection 'scm:git:https://github.com/linkedin/PalDB.git' - developerConnection 'scm:git@github.com:linkedin/PalDB.git' - url 'https://github.com/linkedin/PalDB' + connection 'scm:git@github.com:soundvibe/PalDB.git' + developerConnection 'scm:git@github.com:soundvibe/PalDB.git' + url 'https://github.com/soundvibe/PalDB' } licenses { @@ -114,9 +117,9 @@ if (project.hasProperty('release')) { developers { developer { - id 'mbastian' - name 'Mathieu Bastian' - email 'mbastian@apache.org' + id 'lnaginionis' + name 'Linas Naginionis' + email 'lnaginionis@gmail.com' } } } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java index ded364a..bd776ed 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java @@ -18,11 +18,10 @@ import com.linkedin.paldb.api.NotFoundException; import com.linkedin.paldb.api.StoreReader; import com.linkedin.paldb.utils.DataInputOutput; -import java.io.File; -import java.io.IOException; +import org.slf4j.*; + +import java.io.*; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; /** @@ -31,7 +30,7 @@ public final class ReaderImpl implements StoreReader { // Logger - private final static Logger LOGGER = Logger.getLogger(ReaderImpl.class.getName()); + private static final Logger log = LoggerFactory.getLogger(ReaderImpl.class); // Configuration private final Configuration config; // Buffer @@ -59,11 +58,11 @@ public final class ReaderImpl implements StoreReader { // Open storage try { - LOGGER.log(Level.INFO, "Opening reader storage"); + log.info("Opening reader storage"); serialization = new StorageSerialization(config); storage = new StorageReader(config, file); } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } opened = true; @@ -75,11 +74,11 @@ public final class ReaderImpl implements StoreReader { public void close() { checkOpen(); try { - LOGGER.log(Level.INFO, "Closing reader storage"); + log.info("Closing reader storage"); storage.close(); opened = false; } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } } @@ -404,13 +403,13 @@ public char[] getCharArray(Object key) @Override public Iterable> iterable() { checkOpen(); - return new ReaderIterable(storage, serialization); + return new ReaderIterable<>(storage, serialization); } @Override public Iterable keys() { checkOpen(); - return new ReaderKeyIterable(storage, serialization); + return new ReaderKeyIterable<>(storage, serialization); } // UTILITIES diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderIterable.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderIterable.java index 4497152..694e17c 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderIterable.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderIterable.java @@ -43,7 +43,7 @@ public final class ReaderIterable implements Iterable> { @Override public Iterator> iterator() { - return new ReaderIterator(byteIterable.iterator(), serialization); + return new ReaderIterator<>(byteIterable.iterator(), serialization); } /** @@ -54,7 +54,7 @@ public Iterator> iterator() { private static final class ReaderIterator implements Iterator> { // Reusable entry - private final FastEntry entry = new FastEntry(); + private final FastEntry entry = new FastEntry<>(); // Iterator private final Iterator> byteIterator; // Buffer @@ -125,7 +125,7 @@ public V getValue() { } @Override - public Object setValue(Object value) { + public V setValue(V value) { throw new UnsupportedOperationException("Not supported."); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java index 505615a..5e33d61 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java @@ -44,7 +44,7 @@ public final class ReaderKeyIterable implements Iterable { @Override public Iterator iterator() { - return new ReaderKeyIterator(byteIterable.iterator(), serialization); + return new ReaderKeyIterator<>(byteIterable.iterator(), serialization); } /** @@ -79,8 +79,7 @@ public boolean hasNext() { public K next() { Map.Entry byteEntry = byteIterator.next(); try { - K key = (K) serialization.deserialize(dataInputOutput.reset(byteEntry.getKey())); - return key; + return (K) serialization.deserialize(dataInputOutput.reset(byteEntry.getKey())); } catch (Exception ex) { throw new RuntimeException(ex); } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java index 4601a7a..7b85ff3 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java @@ -15,23 +15,12 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.Serializer; +import org.slf4j.*; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.lang.reflect.Array; -import java.lang.reflect.GenericArrayType; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.io.*; +import java.lang.reflect.*; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; /** @@ -40,8 +29,8 @@ public final class Serializers implements Serializable { // Logger - private final static Logger LOGGER = Logger.getLogger(Serializers.class.getName()); - private AtomicInteger COUNTER; + private static final Logger log = LoggerFactory.getLogger(Serializers.class); + private AtomicInteger counter; private Map serializers; private Serializer[] serializersArray; @@ -49,8 +38,8 @@ public final class Serializers implements Serializable { * Default constructor. */ public Serializers() { - COUNTER = new AtomicInteger(); - serializers = new HashMap(); + counter = new AtomicInteger(); + serializers = new HashMap<>(); serializersArray = new Serializer[0]; } @@ -62,16 +51,15 @@ public Serializers() { public synchronized void registerSerializer(Serializer serializer) { Class objClass = getSerializerType(serializer); if (!serializers.containsKey(objClass)) { - int index = COUNTER.getAndIncrement(); + int index = counter.getAndIncrement(); serializers.put(objClass, new SerializerWrapper(index, serializer)); if (serializersArray.length <= index) { serializersArray = Arrays.copyOf(serializersArray, index + 1); } serializersArray[index] = serializer; - LOGGER.info(String - .format("Registered new serializer '%s' %n for '%s' at index %d", serializer.getClass().getName(), - objClass.getName(), index)); + log.info("Registered new serializer '{}' for '{}' at index {}", serializer.getClass().getName(), + objClass.getName(), index); } } @@ -123,7 +111,7 @@ static void serialize(DataOutput out, Serializers serializers) msg.append(String.format("%n (%d) %s", index, name)); } - LOGGER.info(msg.toString()); + log.info(msg.toString()); } } @@ -137,8 +125,8 @@ static void serialize(DataOutput out, Serializers serializers) private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { // Init - COUNTER = new AtomicInteger(); - serializers = new HashMap(); + counter = new AtomicInteger(); + serializers = new HashMap<>(); serializersArray = new Serializer[0]; deserialize(in, this); @@ -176,12 +164,12 @@ static void deserialize(DataInput in, Serializers serializers) msg.append(String.format("%n (%d) %s", index, serializerClassName)); } catch (Exception ex) { - LOGGER.log(Level.WARNING, (String.format("Can't find the serializer '%s'", serializerClassName)), ex); + log.warn("Can't find the serializer '{}'", serializerClassName, ex); } } - serializers.COUNTER.set(max + 1); + serializers.counter.set(max + 1); - LOGGER.info(msg.toString()); + log.info(msg.toString()); } } @@ -189,11 +177,11 @@ static void deserialize(DataInput in, Serializers serializers) * Clear all serializers. */ void clear() { - LOGGER.info("Clear all serializers"); + log.info("Clear all serializers"); serializers.clear(); serializersArray = new Serializer[0]; - COUNTER.set(0); + counter.set(0); } /** diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java index d40ba39..7c2272d 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java @@ -14,13 +14,11 @@ package com.linkedin.paldb.impl; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.Serializer; +import com.linkedin.paldb.api.*; +import org.slf4j.*; + import java.text.DecimalFormat; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; +import java.util.*; /** @@ -41,7 +39,7 @@ public class StorageCache { protected static final Object NULL_VALUE = new Object(); //Logger - private final static Logger LOGGER = Logger.getLogger(StorageCache.class.getName()); + private static final Logger log = LoggerFactory.getLogger(StorageCache.class); /** * Factory to create and initialize the cache. @@ -62,7 +60,7 @@ static StorageCache initCache(Configuration configuration) { * 632 bytes for the LinkedHashMap * 24 bytes theoretical overhead per entry but more like 45 in practice */ - final static int OVERHEAD = 50; + static final int OVERHEAD = 50; private final LinkedHashMap cache; private final Configuration configuration; private long maxWeight; @@ -88,7 +86,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } }; maxWeight = config.getLong(Configuration.CACHE_BYTES); - LOGGER.log(Level.INFO, "Cache initialized with maximum {0} Mb usage", + log.info("Cache initialized with maximum {} Mb usage", new DecimalFormat("#,##0.00").format(maxWeight / (1024.0 * 1024.0))); configuration = config; } @@ -242,7 +240,7 @@ public long getWeight() { private static class DisabledCache extends StorageCache { DisabledCache() { - LOGGER.log(Level.INFO, "Cache disabled"); + log.info("Cache disabled"); } @Override diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index 2a1e22a..491773e 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -15,29 +15,14 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.utils.DataInputOutput; -import com.linkedin.paldb.utils.FormatVersion; -import com.linkedin.paldb.utils.HashUtils; -import com.linkedin.paldb.utils.LongPacker; -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; +import com.linkedin.paldb.utils.*; +import org.slf4j.*; + +import java.io.*; +import java.nio.*; import java.nio.channels.FileChannel; -import java.text.DecimalFormat; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Iterator; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; +import java.text.*; +import java.util.*; /** @@ -46,7 +31,7 @@ public class StorageReader implements Iterable> { // Logger - private final static Logger LOGGER = Logger.getLogger(StorageReader.class.getName()); + private static final Logger log = LoggerFactory.getLogger(StorageReader.class); // Configuration private final Configuration config; // File path @@ -96,7 +81,7 @@ public class StorageReader implements Iterable> { if (!file.exists()) { throw new FileNotFoundException("File " + file.getAbsolutePath() + " not found"); } - LOGGER.log(Level.INFO, "Opening file {0}", file.getName()); + log.info("Opening file {}", file.getName()); //Config segmentSize = config.getLong(Configuration.MMAP_SEGMENT_SIZE); @@ -219,24 +204,26 @@ public class StorageReader implements Iterable> { } //logging - DecimalFormat integerFormat = new DecimalFormat("#,##0.00"); - StringBuilder statMsg = new StringBuilder("Storage metadata\n"); - statMsg.append(" Created at: " + formatCreatedAt(createdAt) + "\n"); - statMsg.append(" Format version: " + formatVersion.name() + "\n"); - statMsg.append(" Key count: " + keyCount + "\n"); - for (int i = 0; i < keyCounts.length; i++) { - if (keyCounts[i] > 0) { - statMsg.append(" Key count for key length " + i + ": " + keyCounts[i] + "\n"); + if (log.isDebugEnabled()) { + DecimalFormat integerFormat = new DecimalFormat("#,##0.00"); + StringBuilder statMsg = new StringBuilder("Storage metadata\n"); + statMsg.append(" Created at: ").append(formatCreatedAt(createdAt)).append("\n"); + statMsg.append(" Format version: ").append(formatVersion.name()).append("\n"); + statMsg.append(" Key count: ").append(keyCount).append("\n"); + for (int i = 0; i < keyCounts.length; i++) { + if (keyCounts[i] > 0) { + statMsg.append(" Key count for key length ").append(i).append(": ").append(keyCounts[i]).append("\n"); + } } + statMsg.append(" Index size: ").append(integerFormat.format((dataOffset - indexOffset) / (1024.0 * 1024.0))).append(" Mb\n"); + statMsg.append(" Data size: ").append(integerFormat.format((fileSize - dataOffset) / (1024.0 * 1024.0))).append(" Mb\n"); + if (mMapData) { + statMsg.append(" Number of memory mapped data buffers: ").append(dataBuffers.length); + } else { + statMsg.append(" Memory mapped data disabled, using disk"); + } + log.debug(statMsg.toString()); } - statMsg.append(" Index size: " + integerFormat.format((dataOffset - indexOffset) / (1024.0 * 1024.0)) + " Mb\n"); - statMsg.append(" Data size: " + integerFormat.format((fileSize - dataOffset) / (1024.0 * 1024.0)) + " Mb\n"); - if (mMapData) { - statMsg.append(" Number of memory mapped data buffers: " + dataBuffers.length); - } else { - statMsg.append(" Memory mapped data disabled, using disk"); - } - LOGGER.info(statMsg.toString()); } //Get the value for the given key or null @@ -262,8 +249,7 @@ public byte[] get(byte[] key) return null; } if (isKey(slotBuffer, key)) { - byte[] value = mMapData ? getMMapBytes(dataOffset + offset) : getDiskBytes(dataOffset + offset); - return value; + return mMapData ? getMMapBytes(dataOffset + offset) : getDiskBytes(dataOffset + offset); } } return null; @@ -287,7 +273,6 @@ public void close() dataBuffers = null; mappedFile = null; channel = null; - System.gc(); } public int getKeyCount() { @@ -454,7 +439,7 @@ public FastEntry next() { } return entry; } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java index 97afdd3..d73e3b2 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java @@ -14,20 +14,14 @@ package com.linkedin.paldb.impl; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.Serializer; -import com.linkedin.paldb.api.UnsupportedTypeException; -import com.linkedin.paldb.utils.DataInputOutput; -import com.linkedin.paldb.utils.LongPacker; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.EOFException; -import java.io.IOException; -import java.lang.reflect.Array; -import java.math.BigDecimal; -import java.math.BigInteger; +import com.linkedin.paldb.api.*; +import com.linkedin.paldb.utils.*; import org.xerial.snappy.Snappy; +import java.io.*; +import java.lang.reflect.Array; +import java.math.*; + /** * Internal serialization implementation. */ @@ -1110,7 +1104,7 @@ private static String deserializeString(DataInput buf) private static Class deserializeClass(DataInput is) throws IOException, ClassNotFoundException { is.readByte(); - String className = (String) deserializeString(is); + String className = deserializeString(is); Class cls = Class.forName(className); return cls; } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index 69437ea..645138e 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -15,35 +15,21 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.utils.FormatVersion; -import com.linkedin.paldb.utils.HashUtils; -import com.linkedin.paldb.utils.LongPacker; -import com.linkedin.paldb.utils.TempUtils; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.RandomAccessFile; +import com.linkedin.paldb.utils.*; +import org.slf4j.*; + +import java.io.*; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; +import java.util.*; /** * Internal write implementation. */ public class StorageWriter { - private final static Logger LOGGER = Logger.getLogger(StorageWriter.class.getName()); + private static final Logger log = LoggerFactory.getLogger(StorageWriter.class); // Configuration private final Configuration config; private final double loadFactor; @@ -85,7 +71,7 @@ public class StorageWriter { // Create temp path folder tempFolder = TempUtils.createTempDir("paldbtempwriter"); tempFolder.deleteOnExit(); - LOGGER.log(Level.INFO, "Creating temporary folder at {0}", tempFolder.toString()); + log.info("Creating temporary folder at {}", tempFolder); outputStream = stream instanceof BufferedOutputStream ? stream : new BufferedOutputStream(stream); indexStreams = new DataOutputStream[0]; dataStreams = new DataOutputStream[0]; @@ -161,22 +147,22 @@ public void close() } // Stats - LOGGER.log(Level.INFO, "Number of keys: {0}", keyCount); - LOGGER.log(Level.INFO, "Number of values: {0}", valueCount); + log.info("Number of keys: {}", keyCount); + log.info("Number of values: {}", valueCount); // Prepare files to merge - List filesToMerge = new ArrayList(); + List filesToMerge = new ArrayList<>(); try { //Write metadata file File metadataFile = new File(tempFolder, "metadata.dat"); metadataFile.deleteOnExit(); - FileOutputStream metadataOututStream = new FileOutputStream(metadataFile); - DataOutputStream metadataDataOutputStream = new DataOutputStream(metadataOututStream); - writeMetadata(metadataDataOutputStream); - metadataDataOutputStream.close(); - metadataOututStream.close(); + try (FileOutputStream metadataOutputStream = new FileOutputStream(metadataFile); + DataOutputStream metadataDataOutputStream = new DataOutputStream(metadataOutputStream)) { + writeMetadata(metadataDataOutputStream); + } + filesToMerge.add(metadataFile); // Build index file @@ -187,7 +173,7 @@ public void close() } // Stats collisions - LOGGER.log(Level.INFO, "Number of collisions: {0}", collisions); + log.info("Number of collisions: {}", collisions); // Add data files for (File dataFile : dataFiles) { @@ -280,84 +266,70 @@ private File buildIndex(int keyLength) // Init index File indexFile = new File(tempFolder, "index" + keyLength + ".dat"); - RandomAccessFile indexAccessFile = new RandomAccessFile(indexFile, "rw"); - try { - indexAccessFile.setLength(slots * slotSize); - FileChannel indexChannel = indexAccessFile.getChannel(); - MappedByteBuffer byteBuffer = indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexAccessFile.length()); + try (RandomAccessFile indexAccessFile = new RandomAccessFile(indexFile, "rw")) { + indexAccessFile.setLength((long) slots * slotSize); + try (FileChannel indexChannel = indexAccessFile.getChannel()) { + MappedByteBuffer byteBuffer = indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexAccessFile.length()); // Init reading stream - File tempIndexFile = indexFiles[keyLength]; - DataInputStream tempIndexStream = new DataInputStream(new BufferedInputStream(new FileInputStream(tempIndexFile))); - try { - byte[] keyBuffer = new byte[keyLength]; - byte[] slotBuffer = new byte[slotSize]; - byte[] offsetBuffer = new byte[offsetLength]; - - // Read all keys - for (int i = 0; i < count; i++) { - // Read key - tempIndexStream.readFully(keyBuffer); - - // Read offset - long offset = LongPacker.unpackLong(tempIndexStream); - - // Hash - long hash = (long) hashUtils.hash(keyBuffer); - - boolean collision = false; - for (int probe = 0; probe < count; probe++) { - int slot = (int) ((hash + probe) % slots); - byteBuffer.position(slot * slotSize); - byteBuffer.get(slotBuffer); - - long found = LongPacker.unpackLong(slotBuffer, keyLength); - if (found == 0) { - // The spot is empty use it + File tempIndexFile = indexFiles[keyLength]; + try (DataInputStream tempIndexStream = new DataInputStream(new BufferedInputStream(new FileInputStream(tempIndexFile)))) { + byte[] keyBuffer = new byte[keyLength]; + byte[] slotBuffer = new byte[slotSize]; + byte[] offsetBuffer = new byte[offsetLength]; + + // Read all keys + for (int i = 0; i < count; i++) { + // Read key + tempIndexStream.readFully(keyBuffer); + + // Read offset + long offset = LongPacker.unpackLong(tempIndexStream); + + // Hash + long hash = hashUtils.hash(keyBuffer); + + boolean collision = false; + for (int probe = 0; probe < count; probe++) { + int slot = (int) ((hash + probe) % slots); byteBuffer.position(slot * slotSize); - byteBuffer.put(keyBuffer); - int pos = LongPacker.packLong(offsetBuffer, offset); - byteBuffer.put(offsetBuffer, 0, pos); - break; - } else { - collision = true; - // Check for duplicates - if (Arrays.equals(keyBuffer, Arrays.copyOf(slotBuffer, keyLength))) { - throw new RuntimeException( - String.format("A duplicate key has been found for for key bytes %s", Arrays.toString(keyBuffer))); + byteBuffer.get(slotBuffer); + + long found = LongPacker.unpackLong(slotBuffer, keyLength); + if (found == 0) { + // The spot is empty use it + byteBuffer.position(slot * slotSize); + byteBuffer.put(keyBuffer); + int pos = LongPacker.packLong(offsetBuffer, offset); + byteBuffer.put(offsetBuffer, 0, pos); + break; + } else { + collision = true; + // Check for duplicates + if (Arrays.equals(keyBuffer, Arrays.copyOf(slotBuffer, keyLength))) { + throw new RuntimeException( + String.format("A duplicate key has been found for for key bytes %s", Arrays.toString(keyBuffer))); + } } } - } - if (collision) { - collisions++; + if (collision) { + collisions++; + } } - } - - String msg = " Max offset length: " + offsetLength + " bytes" + - "\n Slot size: " + slotSize + " bytes"; - LOGGER.log(Level.INFO, "Built index file {0}\n" + msg, indexFile.getName()); - } finally { - // Close input - tempIndexStream.close(); + String msg = " Max offset length: " + offsetLength + " bytes" + + "\n Slot size: " + slotSize + " bytes"; - // Close index and make sure resources are liberated - indexChannel.close(); - indexChannel = null; - byteBuffer = null; + log.info("Built index file {} \n{}", indexFile.getName(), msg); - // Delete temp index file - if (tempIndexFile.delete()) { - LOGGER.log(Level.INFO, "Temporary index file {0} has been deleted", tempIndexFile.getName()); + } finally { + if (tempIndexFile.delete()) { + log.info("Temporary index file {} has been deleted", tempIndexFile.getName()); + } } } - } finally{ - indexAccessFile.close(); - indexAccessFile = null; - System.gc(); } - return indexFile; } @@ -372,12 +344,12 @@ private void checkFreeDiskSpace(List inputFiles) { usableSpace = f.getUsableSpace(); } } - LOGGER.log(Level.INFO, "Total expected store size is {0} Mb", + log.info("Total expected store size is {} Mb", new DecimalFormat("#,##0.0").format(totalSize / (1024 * 1024))); - LOGGER.log(Level.INFO, "Usable free space on the system is {0} Mb", + log.info("Usable free space on the system is {} Mb", new DecimalFormat("#,##0.0").format(usableSpace / (1024 * 1024))); if (totalSize / (double) usableSpace >= 0.66) { - throw new RuntimeException("Aborting because there isn' enough free disk space"); + throw new RuntimeException("Aborting because there isn't enough free disk space"); } } @@ -389,39 +361,32 @@ private void mergeFiles(List inputFiles, OutputStream outputStream) //Merge files for (File f : inputFiles) { if (f.exists()) { - FileInputStream fileInputStream = new FileInputStream(f); - BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream); - try { - LOGGER.log(Level.INFO, "Merging {0} size={1}", new Object[]{f.getName(), f.length()}); + try (FileInputStream fileInputStream = new FileInputStream(f); + BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { + log.info("Merging {} size={}", f.getName(), f.length()); byte[] buffer = new byte[8192]; int length; while ((length = bufferedInputStream.read(buffer)) > 0) { outputStream.write(buffer, 0, length); } - } finally { - bufferedInputStream.close(); - fileInputStream.close(); } } else { - LOGGER.log(Level.INFO, "Skip merging file {0} because it doesn't exist", f.getName()); + log.info("Skip merging file {} because it doesn't exist", f.getName()); } } - - LOGGER.log(Level.INFO, "Time to merge {0} s", ((System.nanoTime() - startTime) / 1000000000.0)); + log.debug("Time to merge {} s", ((System.nanoTime() - startTime) / 1000000000.0)); } //Cleanup files private void cleanup(List inputFiles) { for (File f : inputFiles) { - if (f.exists()) { - if (f.delete()) { - LOGGER.log(Level.INFO, "Deleted temporary file {0}", f.getName()); - } + if (f.exists() && f.delete()) { + log.info("Deleted temporary file {}", f.getName()); } } if (tempFolder.delete()) { - LOGGER.log(Level.INFO, "Deleted temporary folder at {0}", tempFolder.getAbsolutePath()); + log.info("Deleted temporary folder at {}", tempFolder.getAbsolutePath()); } } @@ -480,8 +445,8 @@ private DataOutputStream getIndexStream(int keyLength) private int getNumKeyCount() { int res = 0; - for (int i = 0; i < keyCounts.length; i++) { - if (keyCounts[i] != 0) { + for (final int count : keyCounts) { + if (count != 0) { res++; } } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java index 6ac350b..a82a717 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java @@ -14,16 +14,11 @@ package com.linkedin.paldb.impl; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.StoreReader; -import com.linkedin.paldb.api.StoreWriter; +import com.linkedin.paldb.api.*; import com.linkedin.paldb.utils.TempUtils; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.logging.Level; -import java.util.logging.Logger; +import org.slf4j.*; + +import java.io.*; /** @@ -31,7 +26,7 @@ */ public final class StoreImpl { - private final static Logger LOGGER = Logger.getLogger(StoreImpl.class.getName()); + private static final Logger log = LoggerFactory.getLogger(StoreImpl.class); private StoreImpl() { } @@ -40,7 +35,7 @@ public static StoreReader createReader(File file, Configuration config) { if (file == null || config == null) { throw new NullPointerException(); } - LOGGER.log(Level.INFO, "Initialize reader from file {0}", file.getName()); + log.info("Initialize reader from file {}", file.getName()); return new ReaderImpl(config, file); } @@ -48,13 +43,13 @@ public static StoreReader createReader(InputStream stream, Configuration config) if (stream == null || config == null) { throw new NullPointerException(); } - LOGGER.log(Level.INFO, "Initialize reader from stream, copying into temp folder"); + log.info("Initialize reader from stream, copying into temp folder"); try { File file = TempUtils.copyIntoTempFile("paldbtempreader", stream); - LOGGER.log(Level.INFO, "Copied stream into temp file {0}", file.getName()); + log.info("Copied stream into temp file {}", file.getName()); return new ReaderImpl(config, file); } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } } @@ -63,18 +58,18 @@ public static StoreWriter createWriter(File file, Configuration config) { throw new NullPointerException(); } try { - LOGGER.log(Level.INFO, "Initialize writer from file {0}", file.getName()); + log.info("Initialize writer from file {}", file.getName()); File parent = file.getParentFile(); if (parent != null && !parent.exists()) { if (parent.mkdirs()) { - LOGGER.log(Level.INFO, "Creating directories for path {0}", file.getName()); + log.info("Creating directories for path {}", file.getName()); } else { throw new RuntimeException(String.format("Couldn't create directory %s", parent)); } } return new WriterImpl(config, file); } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } } @@ -82,7 +77,7 @@ public static StoreWriter createWriter(OutputStream stream, Configuration config if (stream == null || config == null) { throw new NullPointerException(); } - LOGGER.info("Initialize writer from stream"); + log.info("Initialize writer from stream"); return new WriterImpl(config, stream); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java index 41421f0..70fa0b0 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java @@ -14,14 +14,10 @@ package com.linkedin.paldb.impl; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.StoreWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.logging.Level; -import java.util.logging.Logger; +import com.linkedin.paldb.api.*; +import org.slf4j.*; + +import java.io.*; /** @@ -30,7 +26,7 @@ public final class WriterImpl implements StoreWriter { // Logger - private final static Logger LOGGER = Logger.getLogger(WriterImpl.class.getName()); + private static final Logger log = LoggerFactory.getLogger(WriterImpl.class); // Configuration private final Configuration config; // Storage @@ -77,7 +73,7 @@ private WriterImpl(Configuration config, OutputStream stream, File file) { this.file = file; // Open storage - LOGGER.log(Level.INFO, "Opening writer storage"); + log.debug("Opening writer storage"); serialization = new StorageSerialization(config); storage = new StorageWriter(config, outputStream); opened = true; @@ -88,16 +84,16 @@ public void close() { checkOpen(); try { if (file != null) { - LOGGER.log(Level.INFO, "Closing writer storage, writing to file at " + file.getAbsolutePath()); + log.info("Closing writer storage, writing to file at {}", file.getAbsolutePath()); } else { - LOGGER.log(Level.INFO, "Closing writer storage, writing to stream"); + log.info("Closing writer storage, writing to stream"); } storage.close(); outputStream.close(); opened = false; } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } } @@ -116,7 +112,7 @@ public void put(Object key, Object value) { byte[] keyBytes = serialization.serializeKey(key); storage.put(keyBytes, serialization.serializeValue(value)); } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } } @@ -144,7 +140,7 @@ public void put(byte[] key, byte[] value) { try { storage.put(key, value); } catch (IOException ex) { - throw new RuntimeException(ex); + throw new UncheckedIOException(ex); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java b/paldb/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java index 83aae98..a6c6669 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java @@ -78,85 +78,72 @@ public int available() { } @Override - public void readFully(byte[] b) - throws IOException { + public void readFully(byte[] b) { readFully(b, 0, b.length); } @Override - public void readFully(byte[] b, int off, int len) - throws IOException { + public void readFully(byte[] b, int off, int len) { System.arraycopy(buf, pos, b, off, len); pos += len; } @Override - public int skipBytes(int n) - throws IOException { + public int skipBytes(int n) { pos += n; return n; } @Override - public boolean readBoolean() - throws IOException { + public boolean readBoolean() { return buf[pos++] == 1; } @Override - public byte readByte() - throws IOException { + public byte readByte() { return buf[pos++]; } @Override - public int readUnsignedByte() - throws IOException { + public int readUnsignedByte() { return buf[pos++] & 0xff; } @Override - public short readShort() - throws IOException { + public short readShort() { return (short) (((short) (buf[pos++] & 0xff) << 8) | ((short) (buf[pos++] & 0xff) << 0)); } @Override - public int readUnsignedShort() - throws IOException { + public int readUnsignedShort() { return (((int) (buf[pos++] & 0xff) << 8) | ((int) (buf[pos++] & 0xff) << 0)); } @Override - public char readChar() - throws IOException { + public char readChar() { return (char) readInt(); } @Override - public int readInt() - throws IOException { + public int readInt() { return (((buf[pos++] & 0xff) << 24) | ((buf[pos++] & 0xff) << 16) | ((buf[pos++] & 0xff) << 8) | ( (buf[pos++] & 0xff) << 0)); } @Override - public long readLong() - throws IOException { + public long readLong() { return (((long) (buf[pos++] & 0xff) << 56) | ((long) (buf[pos++] & 0xff) << 48) | ((long) (buf[pos++] & 0xff) << 40) | ((long) (buf[pos++] & 0xff) << 32) | ((long) (buf[pos++] & 0xff) << 24) | ((long) (buf[pos++] & 0xff) << 16) | ((long) (buf[pos++] & 0xff) << 8) | ((long) (buf[pos++] & 0xff) << 0)); } @Override - public float readFloat() - throws IOException { + public float readFloat() { return Float.intBitsToFloat(readInt()); } @Override - public double readDouble() - throws IOException { + public double readDouble() { return Double.longBitsToDouble(readLong()); } @@ -189,57 +176,49 @@ private void ensureAvail(int n) { } @Override - public void write(int b) - throws IOException { + public void write(int b) { ensureAvail(1); buf[pos++] = (byte) b; } @Override - public void write(byte[] b) - throws IOException { + public void write(byte[] b) { write(b, 0, b.length); } @Override - public void write(byte[] b, int off, int len) - throws IOException { + public void write(byte[] b, int off, int len) { ensureAvail(len); System.arraycopy(b, off, buf, pos, len); pos += len; } @Override - public void writeBoolean(boolean v) - throws IOException { + public void writeBoolean(boolean v) { ensureAvail(1); buf[pos++] = (byte) (v ? 1 : 0); } @Override - public void writeByte(int v) - throws IOException { + public void writeByte(int v) { ensureAvail(1); buf[pos++] = (byte) (v); } @Override - public void writeShort(int v) - throws IOException { + public void writeShort(int v) { ensureAvail(2); buf[pos++] = (byte) (0xff & (v >> 8)); buf[pos++] = (byte) (0xff & (v >> 0)); } @Override - public void writeChar(int v) - throws IOException { + public void writeChar(int v) { writeInt(v); } @Override - public void writeInt(int v) - throws IOException { + public void writeInt(int v) { ensureAvail(4); buf[pos++] = (byte) (0xff & (v >> 24)); buf[pos++] = (byte) (0xff & (v >> 16)); @@ -248,8 +227,7 @@ public void writeInt(int v) } @Override - public void writeLong(long v) - throws IOException { + public void writeLong(long v) { ensureAvail(8); buf[pos++] = (byte) (0xff & (v >> 56)); buf[pos++] = (byte) (0xff & (v >> 48)); @@ -262,15 +240,13 @@ public void writeLong(long v) } @Override - public void writeFloat(float v) - throws IOException { + public void writeFloat(float v) { ensureAvail(4); writeInt(Float.floatToIntBits(v)); } @Override - public void writeDouble(double v) - throws IOException { + public void writeDouble(double v) { ensureAvail(8); writeLong(Double.doubleToLongBits(v)); } @@ -299,59 +275,51 @@ public void writeUTF(String s) } @Override - public int read() - throws IOException { + public int read() { //is here just to implement ObjectInput return readUnsignedByte(); } @Override - public int read(byte[] b) - throws IOException { + public int read(byte[] b) { //is here just to implement ObjectInput readFully(b); return b.length; } @Override - public int read(byte[] b, int off, int len) - throws IOException { + public int read(byte[] b, int off, int len) { //is here just to implement ObjectInput readFully(b, off, len); return len; } @Override - public long skip(long n) - throws IOException { + public long skip(long n) { //is here just to implement ObjectInput pos += n; return n; } @Override - public void close() - throws IOException { + public void close() { //is here just to implement ObjectInput //do nothing } @Override - public void flush() - throws IOException { + public void flush() { //is here just to implement ObjectOutput //do nothing } @Override - public Object readObject() - throws ClassNotFoundException, IOException { + public Object readObject() { throw new UnsupportedOperationException("Not supported"); } @Override - public void writeObject(Object o) - throws IOException { + public void writeObject(Object o) { throw new UnsupportedOperationException("Not supported"); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/FormatVersion.java b/paldb/src/main/java/com/linkedin/paldb/utils/FormatVersion.java index ff251dd..64be1a4 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/FormatVersion.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/FormatVersion.java @@ -14,7 +14,7 @@ package com.linkedin.paldb.utils; -import java.io.IOException; +import java.io.*; import java.util.Arrays; @@ -42,13 +42,12 @@ public boolean is(FormatVersion fv) { * @return format version byte representation */ public byte[] getBytes() { - try { - DataInputOutput dio = new DataInputOutput(); + try (DataInputOutput dio = new DataInputOutput()) { dio.writeUTF(this.name()); byte[] res = dio.toByteArray(); return Arrays.copyOfRange(res, 1, res.length); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } } @@ -60,14 +59,13 @@ public byte[] getBytes() { */ public static FormatVersion fromBytes(byte[] bytes) { String version = null; - try { - byte[] withSize = new byte[bytes.length + 1]; - withSize[0] = (byte) bytes.length; - System.arraycopy(bytes, 0, withSize, 1, bytes.length); - DataInputOutput dio = new DataInputOutput(withSize); + byte[] withSize = new byte[bytes.length + 1]; + withSize[0] = (byte) bytes.length; + System.arraycopy(bytes, 0, withSize, 1, bytes.length); + try (DataInputOutput dio = new DataInputOutput(withSize)) { version = dio.readUTF(); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } try { return FormatVersion.valueOf(version); @@ -82,13 +80,12 @@ public static FormatVersion fromBytes(byte[] bytes) { * @return prefix byte representation */ public static byte[] getPrefixBytes() { - try { - DataInputOutput dio = new DataInputOutput(); + try (DataInputOutput dio = new DataInputOutput()) { dio.writeUTF("PALDB"); byte[] res = dio.toByteArray(); return Arrays.copyOfRange(res, 1, res.length); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/LongPacker.java b/paldb/src/main/java/com/linkedin/paldb/utils/LongPacker.java index dd2eebf..fd798c1 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/LongPacker.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/LongPacker.java @@ -42,11 +42,11 @@ private LongPacker() { * @return the number of bytes written * @throws IOException if an error occurs with the stream */ - static public int packLong(DataOutput os, long value) + public static int packLong(DataOutput os, long value) throws IOException { if (value < 0) { - throw new IllegalArgumentException("negative value: v=" + value); + throw new IllegalArgumentException("Negative value: v=" + value); } int i = 1; @@ -66,13 +66,11 @@ static public int packLong(DataOutput os, long value) * @param ba the byte array * @param value the long value * @return the number of bytes written - * @throws IOException if an error occurs with the stream */ - static public int packLong(byte[] ba, long value) - throws IOException { + public static int packLong(byte[] ba, long value) { if (value < 0) { - throw new IllegalArgumentException("negative value: v=" + value); + throw new IllegalArgumentException("Negative value: v=" + value); } int i = 1; @@ -92,7 +90,7 @@ static public int packLong(byte[] ba, long value) * @return the long value * @throws IOException if an error occurs with the stream */ - static public long unpackLong(DataInput is) + public static long unpackLong(DataInput is) throws IOException { long result = 0; @@ -112,7 +110,7 @@ static public long unpackLong(DataInput is) * @param ba byte array * @return the long value */ - static public long unpackLong(byte[] ba) { + public static long unpackLong(byte[] ba) { return unpackLong(ba, 0); } @@ -125,7 +123,7 @@ static public long unpackLong(byte[] ba) { * @param index index in ba * @return the long value */ - static public long unpackLong(byte[] ba, int index) { + public static long unpackLong(byte[] ba, int index) { long result = 0; for (int offset = 0; offset < 64; offset += 7) { long b = ba[index++]; @@ -146,7 +144,7 @@ static public long unpackLong(byte[] ba, int index) { * @return the number of bytes written * @throws IOException if an error occurs with the stream */ - static public int packInt(DataOutput os, int value) + public static int packInt(DataOutput os, int value) throws IOException { if (value < 0) { @@ -171,7 +169,7 @@ static public int packInt(DataOutput os, int value) * @return the long value * @throws IOException if an error occurs with the stream */ - static public int unpackInt(DataInput is) + public static int unpackInt(DataInput is) throws IOException { for (int offset = 0, result = 0; offset < 32; offset += 7) { int b = is.readUnsignedByte(); @@ -188,10 +186,8 @@ static public int unpackInt(DataInput is) * * @param bb The byte buffer * @return the long value - * @throws IOException if an error occurs with the stream */ - static public int unpackInt(ByteBuffer bb) - throws IOException { + public static int unpackInt(ByteBuffer bb) { for (int offset = 0, result = 0; offset < 32; offset += 7) { int b = bb.get() & 0xffff; result |= (b & 0x7F) << offset; diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java b/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java index 5ea361e..d6219c0 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java @@ -63,28 +63,17 @@ public static File createTempDir(String prefix) { */ public static File copyIntoTempFile(String fileName, InputStream inputStream) throws IOException { - BufferedInputStream bufferedStream = inputStream instanceof BufferedInputStream ? (BufferedInputStream) inputStream - : new BufferedInputStream(inputStream); - File destFile = null; - try { - destFile = File.createTempFile(fileName, null); - destFile.deleteOnExit(); - - FileOutputStream fileOutputStream = new FileOutputStream(destFile); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream); - try { - byte[] buffer = new byte[8192]; - int length; - while ((length = bufferedStream.read(buffer)) > 0) { - bufferedOutputStream.write(buffer, 0, length); - } - } finally { - bufferedOutputStream.close(); - fileOutputStream.close(); + File destFile = File.createTempFile(fileName, null); + destFile.deleteOnExit(); + try (BufferedInputStream bufferedStream = inputStream instanceof BufferedInputStream ? (BufferedInputStream) inputStream + : new BufferedInputStream(inputStream); + FileOutputStream fileOutputStream = new FileOutputStream(destFile); + BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream)) { + byte[] buffer = new byte[8192]; + int length; + while ((length = bufferedStream.read(buffer)) > 0) { + bufferedOutputStream.write(buffer, 0, length); } - } finally { - bufferedStream.close(); - inputStream.close(); } return destFile; } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java index 6ff368b..86eeae6 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -15,44 +15,49 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.*; -import com.linkedin.paldb.api.PalDB; -import com.linkedin.paldb.utils.DataInputOutput; -import com.linkedin.paldb.utils.FormatVersion; -import com.linkedin.paldb.utils.LongPacker; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; +import com.linkedin.paldb.utils.*; import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; +import org.testng.annotations.*; +import java.io.*; +import java.nio.file.*; +import java.util.*; public class TestStore { - private final File STORE_FOLDER = new File("data"); - private final File STORE_FILE = new File(STORE_FOLDER, "paldb.dat"); + private Path tempDir; + private File STORE_FILE = createTempFile(); - @BeforeClass - public void setUp() { - STORE_FILE.delete(); - STORE_FOLDER.delete(); - STORE_FOLDER.mkdir(); + @BeforeMethod + public void setUp() throws IOException { + tempDir = Files.createTempDirectory("tmp"); + STORE_FILE = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); + STORE_FILE.mkdir(); } - @AfterClass + @AfterMethod public void cleanUp() { - STORE_FILE.delete(); - STORE_FOLDER.delete(); + deleteDirectory(tempDir.toFile()); + } + + private boolean deleteDirectory(File directoryToBeDeleted) { + if (directoryToBeDeleted.isDirectory()) { + File[] allContents = directoryToBeDeleted.listFiles(); + if (allContents != null) { + for (File file : allContents) { + deleteDirectory(file); + } + } + } + return directoryToBeDeleted.delete(); + } + + private static File createTempFile() { + try { + return Files.createTempFile( "paldb", ".dat").toFile(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Test @@ -339,6 +344,7 @@ public void testDataOnTwoBuffers() int byteSize = serialization.serialize(values[0]).length + serialization.serialize(values[1]).length; //Write + System.out.println(STORE_FILE); writeStore(STORE_FILE, keys, values); //Read @@ -365,6 +371,7 @@ public void testDataSizeOnTwoBuffers() LongPacker.packInt(new DataInputOutput(), b1.length) + LongPacker.packInt(new DataInputOutput(), b2.length); //Write + System.out.println(STORE_FILE); writeStore(STORE_FILE, keys, values); //Read diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index 9d1920d..cdc4799 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -15,35 +15,75 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.*; -import com.linkedin.paldb.api.PalDB; +import org.testng.Assert; +import org.testng.annotations.*; import java.awt.*; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.File; -import java.io.IOException; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; +import java.io.*; +import java.nio.file.*; +import java.util.*; public class TestStoreReader { - private final File STORE_FOLDER = new File("data"); - private final File STORE_FILE = new File(STORE_FOLDER, "paldb.dat"); private StoreReader reader; + private Path tempDir; + private File STORE_FILE = createTempFile(); + + @BeforeMethod + public void setUp() throws IOException { + tempDir = Files.createTempDirectory("tmp"); + STORE_FILE = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); + STORE_FILE.mkdir(); + + Configuration configuration = new Configuration(); + configuration.registerSerializer(new PointSerializer()); + StoreWriter writer = PalDB.createWriter(STORE_FILE, configuration); + for (int i = 0; i < testValues.length; i++) { + writer.put(i, testValues[i]); + } + writer.close(); + + reader = PalDB.createReader(STORE_FILE, new Configuration()); + } + + @AfterMethod + public void cleanUp() { + try { + reader.close(); + } catch (Exception e) { + //nop + } + deleteDirectory(tempDir.toFile()); + } + + private boolean deleteDirectory(File directoryToBeDeleted) { + if (directoryToBeDeleted.isDirectory()) { + File[] allContents = directoryToBeDeleted.listFiles(); + if (allContents != null) { + for (File file : allContents) { + deleteDirectory(file); + } + } + } + return directoryToBeDeleted.delete(); + } + + private static File createTempFile() { + try { + return Files.createTempFile( "paldb", ".dat").toFile(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private final Object[] testValues = new Object[]{true, (byte) 1, 'a', 1.0, 1f, (short) 1, 1, 1l, "foo", new boolean[]{true}, new byte[]{1}, new char[]{'a'}, new double[]{1.0}, new float[]{1f}, new short[]{1}, new int[]{1}, new long[]{1l}, new String[]{"foo"}, new Object[]{"foo"}, new Point( 4, 56)}; - @BeforeMethod +/* @BeforeMethod public void setUp() { STORE_FILE.delete(); STORE_FOLDER.delete(); @@ -58,9 +98,9 @@ public void setUp() { writer.close(); reader = PalDB.createReader(STORE_FILE, new Configuration()); - } + }*/ - @AfterMethod +/* @AfterMethod public void cleanUp() { try { reader.close(); @@ -68,7 +108,7 @@ public void cleanUp() { } STORE_FILE.delete(); STORE_FOLDER.delete(); - } + }*/ @Test public void testFile() { diff --git a/paldb/src/test/resources/logback-test.xml b/paldb/src/test/resources/logback-test.xml new file mode 100644 index 0000000..cbb400f --- /dev/null +++ b/paldb/src/test/resources/logback-test.xml @@ -0,0 +1,15 @@ + + + + + + [%d{yyyy-MM-dd HH:mm:ss.SSS}] %-5level [%thread] %class{1}.%M:%L - %msg %n + + + + + + + + + \ No newline at end of file From 27de0453cb1b6b30c36585cb62f47e55ed97f350 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 20:50:38 +0200 Subject: [PATCH 02/20] Updated readme. Trying to setup codecov. --- .travis.yml | 4 ++-- README.md | 12 ++++++------ paldb/build.gradle | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e45e0d..3841833 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ matrix: - jdk: openjdk13 script: - - gradle build + - gradle build jacocoTestReport after_success: - - gradle jacocoTestReport coveralls + - bash <(curl -s https://codecov.io/bash) diff --git a/README.md b/README.md index 4792f15..091c12a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ PalDB ========== -[![Build Status](https://travis-ci.org/linkedin/PalDB.svg?branch=master)](https://travis-ci.org/linkedin/PalDB) -[![Coverage Status](https://coveralls.io/repos/linkedin/PalDB/badge.svg?branch=master&service=github)](https://coveralls.io/github/linkedin/PalDB?branch=master) +[![Build Status](https://travis-ci.org/soundvibe/PalDB.svg?branch=java11)](https://travis-ci.org/soundvibe/PalDB) +[![Coverage Status](https://codecov.io/github/soundvibe/PalDB/coverage.svg?branch=java11)](https://codecov.io/github/soundvibe/PalDB?branch=java11) PalDB is an embeddable write-once key-value store written in Java. @@ -75,14 +75,14 @@ Use it PalDB is available on Maven Central, hence just add the following dependency: ``` - com.linkedin.paldb + net.soundvibe.paldb paldb - 1.2.0 + 1.2.1 ``` Scala SBT ``` -libraryDependencies += "com.linkedin.paldb" % "paldb" % "1.2.0" +libraryDependencies += "net.soundvibe.paldb" % "paldb" % "1.2.1" ``` @@ -104,7 +104,7 @@ No, like a hashtable PalDB stores have no order. Build ----- -PalDB requires Java 6+ and gradle. The target Java version is 6. +PalDB requires Java 11+ and gradle. The target Java version is 11. ```bash gradle build diff --git a/paldb/build.gradle b/paldb/build.gradle index 80d28fa..1d73853 100644 --- a/paldb/build.gradle +++ b/paldb/build.gradle @@ -1,6 +1,5 @@ apply plugin: 'java' apply plugin: 'jacoco' -apply plugin: 'com.github.kt3k.coveralls' apply plugin: 'signing' apply plugin: 'maven' @@ -60,7 +59,7 @@ javadoc { jacocoTestReport { reports { - html.enabled = true + html.enabled = false xml.enabled = true } } From 099ff80635fac73ea51b8fb1aa44c3e48f48ca26 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 20:56:25 +0200 Subject: [PATCH 03/20] Adding gradle wrapper --- .gitignore | 2 - .travis.yml | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 55190 bytes gradle/wrapper/gradle-wrapper.properties | 6 + gradlew | 172 +++++++++++++++++++++++ gradlew.bat | 84 +++++++++++ 6 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat diff --git a/.gitignore b/.gitignore index c4914ae..a3c393f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,3 @@ out/ /ligradle .DS_Store /doc -gradle/ -gradlew* diff --git a/.travis.yml b/.travis.yml index 3841833..90a725b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ matrix: - jdk: openjdk13 script: - - gradle build jacocoTestReport + - ./gradlew build jacocoTestReport after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..87b738cbd051603d91cc39de6cb000dd98fe6b02 GIT binary patch literal 55190 zcmafaW0WS*vSoFbZQHhO+s0S6%`V%vZQJa!ZQHKus_B{g-pt%P_q|ywBQt-*Stldc z$+IJ3?^KWm27v+sf`9-50uuadKtMnL*BJ;1^6ynvR7H?hQcjE>7)art9Bu0Pcm@7C z@c%WG|JzYkP)<@zR9S^iR_sA`azaL$mTnGKnwDyMa;8yL_0^>Ba^)phg0L5rOPTbm7g*YIRLg-2^{qe^`rb!2KqS zk~5wEJtTdD?)3+}=eby3x6%i)sb+m??NHC^u=tcG8p$TzB<;FL(WrZGV&cDQb?O0GMe6PBV=V z?tTO*5_HTW$xea!nkc~Cnx#cL_rrUGWPRa6l+A{aiMY=<0@8y5OC#UcGeE#I>nWh}`#M#kIn-$A;q@u-p71b#hcSItS!IPw?>8 zvzb|?@Ahb22L(O4#2Sre&l9H(@TGT>#Py)D&eW-LNb!=S;I`ZQ{w;MaHW z#to!~TVLgho_Pm%zq@o{K3Xq?I|MVuVSl^QHnT~sHlrVxgsqD-+YD?Nz9@HA<;x2AQjxP)r6Femg+LJ-*)k%EZ}TTRw->5xOY z9#zKJqjZgC47@AFdk1$W+KhTQJKn7e>A&?@-YOy!v_(}GyV@9G#I?bsuto4JEp;5|N{orxi_?vTI4UF0HYcA( zKyGZ4<7Fk?&LZMQb6k10N%E*$gr#T&HsY4SPQ?yerqRz5c?5P$@6dlD6UQwZJ*Je9 z7n-@7!(OVdU-mg@5$D+R%gt82Lt%&n6Yr4=|q>XT%&^z_D*f*ug8N6w$`woqeS-+#RAOfSY&Rz z?1qYa5xi(7eTCrzCFJfCxc%j{J}6#)3^*VRKF;w+`|1n;Xaojr2DI{!<3CaP`#tXs z*`pBQ5k@JLKuCmovFDqh_`Q;+^@t_;SDm29 zCNSdWXbV?9;D4VcoV`FZ9Ggrr$i<&#Dx3W=8>bSQIU_%vf)#(M2Kd3=rN@^d=QAtC zI-iQ;;GMk|&A++W5#hK28W(YqN%?!yuW8(|Cf`@FOW5QbX|`97fxmV;uXvPCqxBD zJ9iI37iV)5TW1R+fV16y;6}2tt~|0J3U4E=wQh@sx{c_eu)t=4Yoz|%Vp<#)Qlh1V z0@C2ZtlT>5gdB6W)_bhXtcZS)`9A!uIOa`K04$5>3&8An+i9BD&GvZZ=7#^r=BN=k za+=Go;qr(M)B~KYAz|<^O3LJON}$Q6Yuqn8qu~+UkUKK~&iM%pB!BO49L+?AL7N7o z(OpM(C-EY753=G=WwJHE`h*lNLMNP^c^bBk@5MyP5{v7x>GNWH>QSgTe5 z!*GPkQ(lcbEs~)4ovCu!Zt&$${9$u(<4@9%@{U<-ksAqB?6F`bQ;o-mvjr)Jn7F&j$@`il1Mf+-HdBs<-`1FahTxmPMMI)@OtI&^mtijW6zGZ67O$UOv1Jj z;a3gmw~t|LjPkW3!EZ=)lLUhFzvO;Yvj9g`8hm%6u`;cuek_b-c$wS_0M4-N<@3l|88 z@V{Sd|M;4+H6guqMm4|v=C6B7mlpP(+It%0E;W`dxMOf9!jYwWj3*MRk`KpS_jx4c z=hrKBkFK;gq@;wUV2eqE3R$M+iUc+UD0iEl#-rECK+XmH9hLKrC={j@uF=f3UiceB zU5l$FF7#RKjx+6!JHMG5-!@zI-eG=a-!Bs^AFKqN_M26%cIIcSs61R$yuq@5a3c3& z4%zLs!g}+C5%`ja?F`?5-og0lv-;(^e<`r~p$x%&*89_Aye1N)9LNVk?9BwY$Y$$F^!JQAjBJvywXAesj7lTZ)rXuxv(FFNZVknJha99lN=^h`J2> zl5=~(tKwvHHvh|9-41@OV`c;Ws--PE%{7d2sLNbDp;A6_Ka6epzOSFdqb zBa0m3j~bT*q1lslHsHqaHIP%DF&-XMpCRL(v;MV#*>mB^&)a=HfLI7efblG z(@hzN`|n+oH9;qBklb=d^S0joHCsArnR1-h{*dIUThik>ot^!6YCNjg;J_i3h6Rl0ji)* zo(tQ~>xB!rUJ(nZjCA^%X;)H{@>uhR5|xBDA=d21p@iJ!cH?+%U|VSh2S4@gv`^)^ zNKD6YlVo$%b4W^}Rw>P1YJ|fTb$_(7C;hH+ z1XAMPb6*p^h8)e5nNPKfeAO}Ik+ZN_`NrADeeJOq4Ak;sD~ zTe77no{Ztdox56Xi4UE6S7wRVxJzWxKj;B%v7|FZ3cV9MdfFp7lWCi+W{}UqekdpH zdO#eoOuB3Fu!DU`ErfeoZWJbWtRXUeBzi zBTF-AI7yMC^ntG+8%mn(I6Dw}3xK8v#Ly{3w3_E?J4(Q5JBq~I>u3!CNp~Ekk&YH` z#383VO4O42NNtcGkr*K<+wYZ>@|sP?`AQcs5oqX@-EIqgK@Pmp5~p6O6qy4ml~N{D z{=jQ7k(9!CM3N3Vt|u@%ssTw~r~Z(}QvlROAkQQ?r8OQ3F0D$aGLh zny+uGnH5muJ<67Z=8uilKvGuANrg@s3Vu_lU2ajb?rIhuOd^E@l!Kl0hYIxOP1B~Q zggUmXbh$bKL~YQ#!4fos9UUVG#}HN$lIkM<1OkU@r>$7DYYe37cXYwfK@vrHwm;pg zbh(hEU|8{*d$q7LUm+x&`S@VbW*&p-sWrplWnRM|I{P;I;%U`WmYUCeJhYc|>5?&& zj}@n}w~Oo=l}iwvi7K6)osqa;M8>fRe}>^;bLBrgA;r^ZGgY@IC^ioRmnE&H4)UV5 zO{7egQ7sBAdoqGsso5q4R(4$4Tjm&&C|7Huz&5B0wXoJzZzNc5Bt)=SOI|H}+fbit z-PiF5(NHSy>4HPMrNc@SuEMDuKYMQ--G+qeUPqO_9mOsg%1EHpqoX^yNd~~kbo`cH zlV0iAkBFTn;rVb>EK^V6?T~t~3vm;csx+lUh_%ROFPy0(omy7+_wYjN!VRDtwDu^h4n|xpAMsLepm% zggvs;v8+isCW`>BckRz1MQ=l>K6k^DdT`~sDXTWQ<~+JtY;I~I>8XsAq3yXgxe>`O zZdF*{9@Z|YtS$QrVaB!8&`&^W->_O&-JXn1n&~}o3Z7FL1QE5R*W2W@=u|w~7%EeC1aRfGtJWxImfY-D3t!!nBkWM> zafu>^Lz-ONgT6ExjV4WhN!v~u{lt2-QBN&UxwnvdH|I%LS|J-D;o>@@sA62@&yew0 z)58~JSZP!(lX;da!3`d)D1+;K9!lyNlkF|n(UduR-%g>#{`pvrD^ClddhJyfL7C-(x+J+9&7EsC~^O`&}V%)Ut8^O_7YAXPDpzv8ir4 zl`d)(;imc6r16k_d^)PJZ+QPxxVJS5e^4wX9D=V2zH&wW0-p&OJe=}rX`*->XT=;_qI&)=WHkYnZx6bLoUh_)n-A}SF_ z9z7agNTM5W6}}ui=&Qs@pO5$zHsOWIbd_&%j^Ok5PJ3yUWQw*i4*iKO)_er2CDUME ztt+{Egod~W-fn^aLe)aBz)MOc_?i-stTj}~iFk7u^-gGSbU;Iem06SDP=AEw9SzuF zeZ|hKCG3MV(z_PJg0(JbqTRf4T{NUt%kz&}4S`)0I%}ZrG!jgW2GwP=WTtkWS?DOs znI9LY!dK+1_H0h+i-_~URb^M;4&AMrEO_UlDV8o?E>^3x%ZJyh$JuDMrtYL8|G3If zPf2_Qb_W+V?$#O; zydKFv*%O;Y@o_T_UAYuaqx1isMKZ^32JtgeceA$0Z@Ck0;lHbS%N5)zzAW9iz; z8tTKeK7&qw!8XVz-+pz>z-BeIzr*#r0nB^cntjQ9@Y-N0=e&ZK72vlzX>f3RT@i7@ z=z`m7jNk!9%^xD0ug%ptZnM>F;Qu$rlwo}vRGBIymPL)L|x}nan3uFUw(&N z24gdkcb7!Q56{0<+zu zEtc5WzG2xf%1<@vo$ZsuOK{v9gx^0`gw>@h>ZMLy*h+6ueoie{D#}}` zK2@6Xxq(uZaLFC%M!2}FX}ab%GQ8A0QJ?&!vaI8Gv=vMhd);6kGguDmtuOElru()) zuRk&Z{?Vp!G~F<1#s&6io1`poBqpRHyM^p;7!+L??_DzJ8s9mYFMQ0^%_3ft7g{PD zZd}8E4EV}D!>F?bzcX=2hHR_P`Xy6?FOK)mCj)Ym4s2hh z0OlOdQa@I;^-3bhB6mpw*X5=0kJv8?#XP~9){G-+0ST@1Roz1qi8PhIXp1D$XNqVG zMl>WxwT+K`SdO1RCt4FWTNy3!i?N>*-lbnn#OxFJrswgD7HjuKpWh*o@QvgF&j+CT z{55~ZsUeR1aB}lv#s_7~+9dCix!5(KR#c?K?e2B%P$fvrsZxy@GP#R#jwL{y#Ld$} z7sF>QT6m|}?V;msb?Nlohj7a5W_D$y+4O6eI;Zt$jVGymlzLKscqer9#+p2$0It&u zWY!dCeM6^B^Z;ddEmhi?8`scl=Lhi7W%2|pT6X6^%-=q90DS(hQ-%c+E*ywPvmoF(KqDoW4!*gmQIklm zk#!GLqv|cs(JRF3G?=AYY19{w@~`G3pa z@xR9S-Hquh*&5Yas*VI};(%9%PADn`kzm zeWMJVW=>>wap*9|R7n#!&&J>gq04>DTCMtj{P^d12|2wXTEKvSf?$AvnE!peqV7i4 zE>0G%CSn%WCW1yre?yi9*aFP{GvZ|R4JT}M%x_%Hztz2qw?&28l&qW<6?c6ym{f$d z5YCF+k#yEbjCN|AGi~-NcCG8MCF1!MXBFL{#7q z)HO+WW173?kuI}^Xat;Q^gb4Hi0RGyB}%|~j8>`6X4CPo+|okMbKy9PHkr58V4bX6<&ERU)QlF8%%huUz&f+dwTN|tk+C&&o@Q1RtG`}6&6;ncQuAcfHoxd5AgD7`s zXynq41Y`zRSiOY@*;&1%1z>oNcWTV|)sjLg1X8ijg1Y zbIGL0X*Sd}EXSQ2BXCKbJmlckY(@EWn~Ut2lYeuw1wg?hhj@K?XB@V_ZP`fyL~Yd3n3SyHU-RwMBr6t-QWE5TinN9VD4XVPU; zonIIR!&pGqrLQK)=#kj40Im%V@ij0&Dh0*s!lnTw+D`Dt-xmk-jmpJv$1-E-vfYL4 zqKr#}Gm}~GPE+&$PI@4ag@=M}NYi7Y&HW82Q`@Y=W&PE31D110@yy(1vddLt`P%N^ z>Yz195A%tnt~tvsSR2{m!~7HUc@x<&`lGX1nYeQUE(%sphTi>JsVqSw8xql*Ys@9B z>RIOH*rFi*C`ohwXjyeRBDt8p)-u{O+KWP;$4gg||%*u{$~yEj+Al zE(hAQRQ1k7MkCq9s4^N3ep*$h^L%2Vq?f?{+cicpS8lo)$Cb69b98au+m2J_e7nYwID0@`M9XIo1H~|eZFc8Hl!qly612ADCVpU zY8^*RTMX(CgehD{9v|^9vZ6Rab`VeZ2m*gOR)Mw~73QEBiktViBhR!_&3l$|be|d6 zupC`{g89Y|V3uxl2!6CM(RNpdtynaiJ~*DqSTq9Mh`ohZnb%^3G{k;6%n18$4nAqR zjPOrP#-^Y9;iw{J@XH9=g5J+yEVh|e=4UeY<^65`%gWtdQ=-aqSgtywM(1nKXh`R4 zzPP&7r)kv_uC7X9n=h=!Zrf<>X=B5f<9~Q>h#jYRD#CT7D~@6@RGNyO-#0iq0uHV1 zPJr2O4d_xLmg2^TmG7|dpfJ?GGa`0|YE+`2Rata9!?$j#e9KfGYuLL(*^z z!SxFA`$qm)q-YKh)WRJZ@S+-sD_1E$V?;(?^+F3tVcK6 z2fE=8hV*2mgiAbefU^uvcM?&+Y&E}vG=Iz!%jBF7iv){lyC`)*yyS~D8k+Mx|N3bm zI~L~Z$=W9&`x)JnO;8c>3LSDw!fzN#X3qi|0`sXY4?cz{*#xz!kvZ9bO=K3XbN z5KrgN=&(JbXH{Wsu9EdmQ-W`i!JWEmfI;yVTT^a-8Ch#D8xf2dtyi?7p z%#)W3n*a#ndFpd{qN|+9Jz++AJQO#-Y7Z6%*%oyEP5zs}d&kKIr`FVEY z;S}@d?UU=tCdw~EJ{b}=9x}S2iv!!8<$?d7VKDA8h{oeD#S-$DV)-vPdGY@x08n)@ zag?yLF_E#evvRTj4^CcrLvBL=fft&@HOhZ6Ng4`8ijt&h2y}fOTC~7GfJi4vpomA5 zOcOM)o_I9BKz}I`q)fu+Qnfy*W`|mY%LO>eF^a z;$)?T4F-(X#Q-m}!-k8L_rNPf`Mr<9IWu)f&dvt=EL+ESYmCvErd@8B9hd)afc(ZL94S z?rp#h&{7Ah5IJftK4VjATklo7@hm?8BX*~oBiz)jyc9FuRw!-V;Uo>p!CWpLaIQyt zAs5WN)1CCeux-qiGdmbIk8LR`gM+Qg=&Ve}w?zA6+sTL)abU=-cvU`3E?p5$Hpkxw znu0N659qR=IKnde*AEz_7z2pdi_Bh-sb3b=PdGO1Pdf_q2;+*Cx9YN7p_>rl``knY zRn%aVkcv1(W;`Mtp_DNOIECtgq%ufk-mu_<+Fu3Q17Tq4Rr(oeq)Yqk_CHA7LR@7@ zIZIDxxhS&=F2IQfusQ+Nsr%*zFK7S4g!U0y@3H^Yln|i;0a5+?RPG;ZSp6Tul>ezM z`40+516&719qT)mW|ArDSENle5hE2e8qY+zfeZoy12u&xoMgcP)4=&P-1Ib*-bAy` zlT?>w&B|ei-rCXO;sxo7*G;!)_p#%PAM-?m$JP(R%x1Hfas@KeaG%LO?R=lmkXc_MKZW}3f%KZ*rAN?HYvbu2L$ zRt_uv7~-IejlD1x;_AhwGXjB94Q=%+PbxuYzta*jw?S&%|qb=(JfJ?&6P=R7X zV%HP_!@-zO*zS}46g=J}#AMJ}rtWBr21e6hOn&tEmaM%hALH7nlm2@LP4rZ>2 zebe5aH@k!e?ij4Zwak#30|}>;`bquDQK*xmR=zc6vj0yuyC6+U=LusGnO3ZKFRpen z#pwzh!<+WBVp-!$MAc<0i~I%fW=8IO6K}bJ<-Scq>e+)951R~HKB?Mx2H}pxPHE@} zvqpq5j81_jtb_WneAvp<5kgdPKm|u2BdQx9%EzcCN&U{l+kbkhmV<1}yCTDv%&K^> zg;KCjwh*R1f_`6`si$h6`jyIKT7rTv5#k~x$mUyIw)_>Vr)D4fwIs@}{FSX|5GB1l z4vv;@oS@>Bu7~{KgUa_8eg#Lk6IDT2IY$41$*06{>>V;Bwa(-@N;ex4;D`(QK*b}{ z{#4$Hmt)FLqERgKz=3zXiV<{YX6V)lvYBr3V>N6ajeI~~hGR5Oe>W9r@sg)Na(a4- zxm%|1OKPN6^%JaD^^O~HbLSu=f`1px>RawOxLr+1b2^28U*2#h*W^=lSpSY4(@*^l z{!@9RSLG8Me&RJYLi|?$c!B0fP=4xAM4rerxX{xy{&i6=AqXueQAIBqO+pmuxy8Ib z4X^}r!NN3-upC6B#lt7&x0J;)nb9O~xjJMemm$_fHuP{DgtlU3xiW0UesTzS30L+U zQzDI3p&3dpONhd5I8-fGk^}@unluzu%nJ$9pzoO~Kk!>dLxw@M)M9?pNH1CQhvA`z zV;uacUtnBTdvT`M$1cm9`JrT3BMW!MNVBy%?@ZX%;(%(vqQAz<7I!hlDe|J3cn9=} zF7B;V4xE{Ss76s$W~%*$JviK?w8^vqCp#_G^jN0j>~Xq#Zru26e#l3H^{GCLEXI#n z?n~F-Lv#hU(bZS`EI9(xGV*jT=8R?CaK)t8oHc9XJ;UPY0Hz$XWt#QyLBaaz5+}xM zXk(!L_*PTt7gwWH*HLWC$h3Ho!SQ-(I||nn_iEC{WT3S{3V{8IN6tZ1C+DiFM{xlI zeMMk{o5;I6UvaC)@WKp9D+o?2Vd@4)Ue-nYci()hCCsKR`VD;hr9=vA!cgGL%3k^b(jADGyPi2TKr(JNh8mzlIR>n(F_hgiV(3@Ds(tjbNM7GoZ;T|3 zWzs8S`5PrA!9){jBJuX4y`f<4;>9*&NY=2Sq2Bp`M2(fox7ZhIDe!BaQUb@P(ub9D zlP8!p(AN&CwW!V&>H?yPFMJ)d5x#HKfwx;nS{Rr@oHqpktOg)%F+%1#tsPtq7zI$r zBo-Kflhq-=7_eW9B2OQv=@?|y0CKN77)N;z@tcg;heyW{wlpJ1t`Ap!O0`Xz{YHqO zI1${8Hag^r!kA<2_~bYtM=<1YzQ#GGP+q?3T7zYbIjN6Ee^V^b&9en$8FI*NIFg9G zPG$OXjT0Ku?%L7fat8Mqbl1`azf1ltmKTa(HH$Dqlav|rU{zP;Tbnk-XkGFQ6d+gi z-PXh?_kEJl+K98&OrmzgPIijB4!Pozbxd0H1;Usy!;V>Yn6&pu*zW8aYx`SC!$*ti zSn+G9p=~w6V(fZZHc>m|PPfjK6IN4(o=IFu?pC?+`UZAUTw!e`052{P=8vqT^(VeG z=psASIhCv28Y(;7;TuYAe>}BPk5Qg=8$?wZj9lj>h2kwEfF_CpK=+O6Rq9pLn4W)# zeXCKCpi~jsfqw7Taa0;!B5_C;B}e56W1s8@p*)SPzA;Fd$Slsn^=!_&!mRHV*Lmt| zBGIDPuR>CgS4%cQ4wKdEyO&Z>2aHmja;Pz+n|7(#l%^2ZLCix%>@_mbnyPEbyrHaz z>j^4SIv;ZXF-Ftzz>*t4wyq)ng8%0d;(Z_ExZ-cxwei=8{(br-`JYO(f23Wae_MqE z3@{Mlf^%M5G1SIN&en1*| zH~ANY1h3&WNsBy$G9{T=`kcxI#-X|>zLX2r*^-FUF+m0{k)n#GTG_mhG&fJfLj~K& zU~~6othMlvMm9<*SUD2?RD+R17|Z4mgR$L*R3;nBbo&Vm@39&3xIg;^aSxHS>}gwR zmzs?h8oPnNVgET&dx5^7APYx6Vv6eou07Zveyd+^V6_LzI$>ic+pxD_8s~ zC<}ucul>UH<@$KM zT4oI=62M%7qQO{}re-jTFqo9Z;rJKD5!X5$iwUsh*+kcHVhID08MB5cQD4TBWB(rI zuWc%CA}}v|iH=9gQ?D$1#Gu!y3o~p7416n54&Hif`U-cV?VrUMJyEqo_NC4#{puzU zzXEE@UppeeRlS9W*^N$zS`SBBi<@tT+<%3l@KhOy^%MWB9(A#*J~DQ;+MK*$rxo6f zcx3$3mcx{tly!q(p2DQrxcih|)0do_ZY77pyHGE#Q(0k*t!HUmmMcYFq%l$-o6%lS zDb49W-E?rQ#Hl``C3YTEdGZjFi3R<>t)+NAda(r~f1cT5jY}s7-2^&Kvo&2DLTPYP zhVVo-HLwo*vl83mtQ9)PR#VBg)FN}+*8c-p8j`LnNUU*Olm1O1Qqe62D#$CF#?HrM zy(zkX|1oF}Z=T#3XMLWDrm(|m+{1&BMxHY7X@hM_+cV$5-t!8HT(dJi6m9{ja53Yw z3f^`yb6Q;(e|#JQIz~B*=!-GbQ4nNL-NL z@^NWF_#w-Cox@h62;r^;Y`NX8cs?l^LU;5IWE~yvU8TqIHij!X8ydbLlT0gwmzS9} z@5BccG?vO;rvCs$mse1*ANi-cYE6Iauz$Fbn3#|ToAt5v7IlYnt6RMQEYLldva{~s zvr>1L##zmeoYgvIXJ#>bbuCVuEv2ZvZ8I~PQUN3wjP0UC)!U+wn|&`V*8?)` zMSCuvnuGec>QL+i1nCPGDAm@XSMIo?A9~C?g2&G8aNKjWd2pDX{qZ?04+2 zeyLw}iEd4vkCAWwa$ zbrHlEf3hfN7^1g~aW^XwldSmx1v~1z(s=1az4-wl} z`mM+G95*N*&1EP#u3}*KwNrPIgw8Kpp((rdEOO;bT1;6ea~>>sK+?!;{hpJ3rR<6UJb`O8P4@{XGgV%63_fs%cG8L zk9Fszbdo4tS$g0IWP1>t@0)E%-&9yj%Q!fiL2vcuL;90fPm}M==<>}Q)&sp@STFCY z^p!RzmN+uXGdtPJj1Y-khNyCb6Y$Vs>eZyW zPaOV=HY_T@FwAlleZCFYl@5X<<7%5DoO(7S%Lbl55?{2vIr_;SXBCbPZ(up;pC6Wx={AZL?shYOuFxLx1*>62;2rP}g`UT5+BHg(ju z&7n5QSvSyXbioB9CJTB#x;pexicV|9oaOpiJ9VK6EvKhl4^Vsa(p6cIi$*Zr0UxQ z;$MPOZnNae2Duuce~7|2MCfhNg*hZ9{+8H3?ts9C8#xGaM&sN;2lriYkn9W>&Gry! z3b(Xx1x*FhQkD-~V+s~KBfr4M_#0{`=Yrh90yj}Ph~)Nx;1Y^8<418tu!$1<3?T*~ z7Dl0P3Uok-7w0MPFQexNG1P5;y~E8zEvE49>$(f|XWtkW2Mj`udPn)pb%} zrA%wRFp*xvDgC767w!9`0vx1=q!)w!G+9(-w&p*a@WXg{?T&%;qaVcHo>7ca%KX$B z^7|KBPo<2;kM{2mRnF8vKm`9qGV%|I{y!pKm8B(q^2V;;x2r!1VJ^Zz8bWa)!-7a8 zSRf@dqEPlsj!7}oNvFFAA)75})vTJUwQ03hD$I*j6_5xbtd_JkE2`IJD_fQ;a$EkO z{fQ{~e%PKgPJsD&PyEvDmg+Qf&p*-qu!#;1k2r_(H72{^(Z)htgh@F?VIgK#_&eS- z$~(qInec>)XIkv@+{o6^DJLpAb>!d}l1DK^(l%#OdD9tKK6#|_R?-%0V!`<9Hj z3w3chDwG*SFte@>Iqwq`J4M&{aHXzyigT620+Vf$X?3RFfeTcvx_e+(&Q*z)t>c0e zpZH$1Z3X%{^_vylHVOWT6tno=l&$3 z9^eQ@TwU#%WMQaFvaYp_we%_2-9=o{+ck zF{cKJCOjpW&qKQquyp2BXCAP920dcrZ}T1@piukx_NY;%2W>@Wca%=Ch~x5Oj58Hv z;D-_ALOZBF(Mqbcqjd}P3iDbek#Dwzu`WRs`;hRIr*n0PV7vT+%Io(t}8KZ zpp?uc2eW!v28ipep0XNDPZt7H2HJ6oey|J3z!ng#1H~x_k%35P+Cp%mqXJ~cV0xdd z^4m5^K_dQ^Sg?$P`))ccV=O>C{Ds(C2WxX$LMC5vy=*44pP&)X5DOPYfqE${)hDg< z3hcG%U%HZ39=`#Ko4Uctg&@PQLf>?0^D|4J(_1*TFMOMB!Vv1_mnOq$BzXQdOGqgy zOp#LBZ!c>bPjY1NTXksZmbAl0A^Y&(%a3W-k>bE&>K?px5Cm%AT2E<&)Y?O*?d80d zgI5l~&Mve;iXm88Q+Fw7{+`PtN4G7~mJWR^z7XmYQ>uoiV!{tL)hp|= zS(M)813PM`d<501>{NqaPo6BZ^T{KBaqEVH(2^Vjeq zgeMeMpd*1tE@@);hGjuoVzF>Cj;5dNNwh40CnU+0DSKb~GEMb_# zT8Z&gz%SkHq6!;_6dQFYE`+b`v4NT7&@P>cA1Z1xmXy<2htaDhm@XXMp!g($ zw(7iFoH2}WR`UjqjaqOQ$ecNt@c|K1H1kyBArTTjLp%-M`4nzOhkfE#}dOpcd;b#suq8cPJ&bf5`6Tq>ND(l zib{VrPZ>{KuaIg}Y$W>A+nrvMg+l4)-@2jpAQ5h(Tii%Ni^-UPVg{<1KGU2EIUNGaXcEkOedJOusFT9X3%Pz$R+-+W+LlRaY-a$5r?4V zbPzgQl22IPG+N*iBRDH%l{Zh$fv9$RN1sU@Hp3m=M}{rX%y#;4(x1KR2yCO7Pzo>rw(67E{^{yUR`91nX^&MxY@FwmJJbyPAoWZ9Z zcBS$r)&ogYBn{DOtD~tIVJUiq|1foX^*F~O4hlLp-g;Y2wKLLM=?(r3GDqsPmUo*? zwKMEi*%f)C_@?(&&hk>;m07F$X7&i?DEK|jdRK=CaaNu-)pX>n3}@%byPKVkpLzBq z{+Py&!`MZ^4@-;iY`I4#6G@aWMv{^2VTH7|WF^u?3vsB|jU3LgdX$}=v7#EHRN(im zI(3q-eU$s~r=S#EWqa_2!G?b~ z<&brq1vvUTJH380=gcNntZw%7UT8tLAr-W49;9y^=>TDaTC|cKA<(gah#2M|l~j)w zY8goo28gj$n&zcNgqX1Qn6=<8?R0`FVO)g4&QtJAbW3G#D)uNeac-7cH5W#6i!%BH z=}9}-f+FrtEkkrQ?nkoMQ1o-9_b+&=&C2^h!&mWFga#MCrm85hW;)1pDt;-uvQG^D zntSB?XA*0%TIhtWDS!KcI}kp3LT>!(Nlc(lQN?k^bS8Q^GGMfo}^|%7s;#r+pybl@?KA++|FJ zr%se9(B|g*ERQU96az%@4gYrxRRxaM2*b}jNsG|0dQi;Rw{0WM0E>rko!{QYAJJKY z)|sX0N$!8d9E|kND~v|f>3YE|uiAnqbkMn)hu$if4kUkzKqoNoh8v|S>VY1EKmgO} zR$0UU2o)4i4yc1inx3}brso+sio{)gfbLaEgLahj8(_Z#4R-v) zglqwI%`dsY+589a8$Mu7#7_%kN*ekHupQ#48DIN^uhDxblDg3R1yXMr^NmkR z7J_NWCY~fhg}h!_aXJ#?wsZF$q`JH>JWQ9`jbZzOBpS`}-A$Vgkq7+|=lPx9H7QZG z8i8guMN+yc4*H*ANr$Q-3I{FQ-^;8ezWS2b8rERp9TMOLBxiG9J*g5=?h)mIm3#CGi4JSq1ohFrcrxx@`**K5%T}qbaCGldV!t zVeM)!U3vbf5FOy;(h08JnhSGxm)8Kqxr9PsMeWi=b8b|m_&^@#A3lL;bVKTBx+0v8 zLZeWAxJ~N27lsOT2b|qyp$(CqzqgW@tyy?CgwOe~^i;ZH zlL``i4r!>i#EGBNxV_P@KpYFQLz4Bdq{#zA&sc)*@7Mxsh9u%e6Ke`?5Yz1jkTdND zR8!u_yw_$weBOU}24(&^Bm|(dSJ(v(cBct}87a^X(v>nVLIr%%D8r|&)mi+iBc;B;x;rKq zd8*X`r?SZsTNCPQqoFOrUz8nZO?225Z#z(B!4mEp#ZJBzwd7jW1!`sg*?hPMJ$o`T zR?KrN6OZA1H{9pA;p0cSSu;@6->8aJm1rrO-yDJ7)lxuk#npUk7WNER1Wwnpy%u zF=t6iHzWU(L&=vVSSc^&D_eYP3TM?HN!Tgq$SYC;pSIPWW;zeNm7Pgub#yZ@7WPw#f#Kl)W4%B>)+8%gpfoH1qZ;kZ*RqfXYeGXJ_ zk>2otbp+1By`x^1V!>6k5v8NAK@T;89$`hE0{Pc@Q$KhG0jOoKk--Qx!vS~lAiypV zCIJ&6B@24`!TxhJ4_QS*S5;;Pk#!f(qIR7*(c3dN*POKtQe)QvR{O2@QsM%ujEAWEm) z+PM=G9hSR>gQ`Bv2(k}RAv2+$7qq(mU`fQ+&}*i%-RtSUAha>70?G!>?w%F(b4k!$ zvm;E!)2`I?etmSUFW7WflJ@8Nx`m_vE2HF#)_BiD#FaNT|IY@!uUbd4v$wTglIbIX zblRy5=wp)VQzsn0_;KdM%g<8@>#;E?vypTf=F?3f@SSdZ;XpX~J@l1;p#}_veWHp>@Iq_T z@^7|h;EivPYv1&u0~l9(a~>dV9Uw10QqB6Dzu1G~-l{*7IktljpK<_L8m0|7VV_!S zRiE{u97(%R-<8oYJ{molUd>vlGaE-C|^<`hppdDz<7OS13$#J zZ+)(*rZIDSt^Q$}CRk0?pqT5PN5TT`Ya{q(BUg#&nAsg6apPMhLTno!SRq1e60fl6GvpnwDD4N> z9B=RrufY8+g3_`@PRg+(+gs2(bd;5#{uTZk96CWz#{=&h9+!{_m60xJxC%r&gd_N! z>h5UzVX%_7@CUeAA1XFg_AF%(uS&^1WD*VPS^jcC!M2v@RHZML;e(H-=(4(3O&bX- zI6>usJOS+?W&^S&DL{l|>51ZvCXUKlH2XKJPXnHjs*oMkNM#ZDLx!oaM5(%^)5XaP zk6&+P16sA>vyFe9v`Cp5qnbE#r#ltR5E+O3!WnKn`56Grs2;sqr3r# zp@Zp<^q`5iq8OqOlJ`pIuyK@3zPz&iJ0Jcc`hDQ1bqos2;}O|$i#}e@ua*x5VCSx zJAp}+?Hz++tm9dh3Fvm_bO6mQo38al#>^O0g)Lh^&l82+&x)*<n7^Sw-AJo9tEzZDwyJ7L^i7|BGqHu+ea6(&7jKpBq>~V z8CJxurD)WZ{5D0?s|KMi=e7A^JVNM6sdwg@1Eg_+Bw=9j&=+KO1PG|y(mP1@5~x>d z=@c{EWU_jTSjiJl)d(>`qEJ;@iOBm}alq8;OK;p(1AdH$)I9qHNmxxUArdzBW0t+Qeyl)m3?D09770g z)hzXEOy>2_{?o%2B%k%z4d23!pZcoxyW1Ik{|m7Q1>fm4`wsRrl)~h z_=Z*zYL+EG@DV1{6@5@(Ndu!Q$l_6Qlfoz@79q)Kmsf~J7t1)tl#`MD<;1&CAA zH8;i+oBm89dTTDl{aH`cmTPTt@^K-%*sV+t4X9q0Z{A~vEEa!&rRRr=0Rbz4NFCJr zLg2u=0QK@w9XGE=6(-JgeP}G#WG|R&tfHRA3a9*zh5wNTBAD;@YYGx%#E4{C#Wlfo z%-JuW9=FA_T6mR2-Vugk1uGZvJbFvVVWT@QOWz$;?u6+CbyQsbK$>O1APk|xgnh_8 zc)s@Mw7#0^wP6qTtyNq2G#s?5j~REyoU6^lT7dpX{T-rhZWHD%dik*=EA7bIJgOVf_Ga!yC8V^tkTOEHe+JK@Fh|$kfNxO^= z#lpV^(ZQ-3!^_BhV>aXY~GC9{8%1lOJ}6vzXDvPhC>JrtXwFBC+!3a*Z-%#9}i z#<5&0LLIa{q!rEIFSFc9)>{-_2^qbOg5;_A9 ztQ))C6#hxSA{f9R3Eh^`_f${pBJNe~pIQ`tZVR^wyp}=gLK}e5_vG@w+-mp#Fu>e| z*?qBp5CQ5zu+Fi}xAs)YY1;bKG!htqR~)DB$ILN6GaChoiy%Bq@i+1ZnANC0U&D z_4k$=YP47ng+0NhuEt}6C;9-JDd8i5S>`Ml==9wHDQFOsAlmtrVwurYDw_)Ihfk35 zJDBbe!*LUpg%4n>BExWz>KIQ9vexUu^d!7rc_kg#Bf= z7TLz|l*y*3d2vi@c|pX*@ybf!+Xk|2*z$@F4K#MT8Dt4zM_EcFmNp31#7qT6(@GG? zdd;sSY9HHuDb=w&|K%sm`bYX#%UHKY%R`3aLMO?{T#EI@FNNFNO>p@?W*i0z(g2dt z{=9Ofh80Oxv&)i35AQN>TPMjR^UID-T7H5A?GI{MD_VeXZ%;uo41dVm=uT&ne2h0i zv*xI%9vPtdEK@~1&V%p1sFc2AA`9?H)gPnRdlO~URx!fiSV)j?Tf5=5F>hnO=$d$x zzaIfr*wiIc!U1K*$JO@)gP4%xp!<*DvJSv7p}(uTLUb=MSb@7_yO+IsCj^`PsxEl& zIxsi}s3L?t+p+3FXYqujGhGwTx^WXgJ1}a@Yq5mwP0PvGEr*qu7@R$9j>@-q1rz5T zriz;B^(ex?=3Th6h;7U`8u2sDlfS{0YyydK=*>-(NOm9>S_{U|eg(J~C7O zIe{|LK=Y`hXiF_%jOM8Haw3UtaE{hWdzo3BbD6ud7br4cODBtN(~Hl+odP0SSWPw;I&^m)yLw+nd#}3#z}?UIcX3=SssI}`QwY=% zAEXTODk|MqTx}2DVG<|~(CxgLyi*A{m>M@1h^wiC)4Hy>1K7@|Z&_VPJsaQoS8=ex zDL&+AZdQa>ylxhT_Q$q=60D5&%pi6+qlY3$3c(~rsITX?>b;({FhU!7HOOhSP7>bmTkC8KM%!LRGI^~y3Ug+gh!QM=+NZXznM)?L3G=4=IMvFgX3BAlyJ z`~jjA;2z+65D$j5xbv9=IWQ^&-K3Yh`vC(1Qz2h2`o$>Cej@XRGff!it$n{@WEJ^N z41qk%Wm=}mA*iwCqU_6}Id!SQd13aFER3unXaJJXIsSnxvG2(hSCP{i&QH$tL&TPx zDYJsuk+%laN&OvKb-FHK$R4dy%M7hSB*yj#-nJy?S9tVoxAuDei{s}@+pNT!vLOIC z8g`-QQW8FKp3cPsX%{)0B+x+OhZ1=L7F-jizt|{+f1Ga7%+!BXqjCjH&x|3%?UbN# zh?$I1^YokvG$qFz5ySK+Ja5=mkR&p{F}ev**rWdKMko+Gj^?Or=UH?SCg#0F(&a_y zXOh}dPv0D9l0RVedq1~jCNV=8?vZfU-Xi|nkeE->;ohG3U7z+^0+HV17~-_Mv#mV` zzvwUJJ15v5wwKPv-)i@dsEo@#WEO9zie7mdRAbgL2kjbW4&lk$vxkbq=w5mGKZK6@ zjXWctDkCRx58NJD_Q7e}HX`SiV)TZMJ}~zY6P1(LWo`;yDynY_5_L?N-P`>ALfmyl z8C$a~FDkcwtzK9m$tof>(`Vu3#6r#+v8RGy#1D2)F;vnsiL&P-c^PO)^B-4VeJteLlT@25sPa z%W~q5>YMjj!mhN})p$47VA^v$Jo6_s{!y?}`+h+VM_SN`!11`|;C;B};B&Z<@%FOG z_YQVN+zFF|q5zKab&e4GH|B;sBbKimHt;K@tCH+S{7Ry~88`si7}S)1E{21nldiu5 z_4>;XTJa~Yd$m4A9{Qbd)KUAm7XNbZ4xHbg3a8-+1uf*$1PegabbmCzgC~1WB2F(W zYj5XhVos!X!QHuZXCatkRsdEsSCc+D2?*S7a+(v%toqyxhjz|`zdrUvsxQS{J>?c& zvx*rHw^8b|v^7wq8KWVofj&VUitbm*a&RU_ln#ZFA^3AKEf<#T%8I!Lg3XEsdH(A5 zlgh&M_XEoal)i#0tcq8c%Gs6`xu;vvP2u)D9p!&XNt z!TdF_H~;`g@fNXkO-*t<9~;iEv?)Nee%hVe!aW`N%$cFJ(Dy9+Xk*odyFj72T!(b%Vo5zvCGZ%3tkt$@Wcx8BWEkefI1-~C_3y*LjlQ5%WEz9WD8i^ z2MV$BHD$gdPJV4IaV)G9CIFwiV=ca0cfXdTdK7oRf@lgyPx;_7*RRFk=?@EOb9Gcz zg~VZrzo*Snp&EE{$CWr)JZW)Gr;{B2ka6B!&?aknM-FENcl%45#y?oq9QY z3^1Y5yn&^D67Da4lI}ljDcphaEZw2;tlYuzq?uB4b9Mt6!KTW&ptxd^vF;NbX=00T z@nE1lIBGgjqs?ES#P{ZfRb6f!At51vk%<0X%d_~NL5b8UyfQMPDtfU@>ijA0NP3UU zh{lCf`Wu7cX!go`kUG`1K=7NN@SRGjUKuo<^;@GS!%iDXbJs`o6e`v3O8-+7vRkFm z)nEa$sD#-v)*Jb>&Me+YIW3PsR1)h=-Su)))>-`aRcFJG-8icomO4J@60 zw10l}BYxi{eL+Uu0xJYk-Vc~BcR49Qyyq!7)PR27D`cqGrik=?k1Of>gY7q@&d&Ds zt7&WixP`9~jjHO`Cog~RA4Q%uMg+$z^Gt&vn+d3&>Ux{_c zm|bc;k|GKbhZLr-%p_f%dq$eiZ;n^NxoS-Nu*^Nx5vm46)*)=-Bf<;X#?`YC4tLK; z?;u?shFbXeks+dJ?^o$l#tg*1NA?(1iFff@I&j^<74S!o;SWR^Xi);DM%8XiWpLi0 zQE2dL9^a36|L5qC5+&Pf0%>l&qQ&)OU4vjd)%I6{|H+pw<0(a``9w(gKD&+o$8hOC zNAiShtc}e~ob2`gyVZx59y<6Fpl*$J41VJ-H*e-yECWaDMmPQi-N8XI3 z%iI@ljc+d}_okL1CGWffeaejlxWFVDWu%e=>H)XeZ|4{HlbgC-Uvof4ISYQzZ0Um> z#Ov{k1c*VoN^f(gfiueuag)`TbjL$XVq$)aCUBL_M`5>0>6Ska^*Knk__pw{0I>jA zzh}Kzg{@PNi)fcAk7jMAdi-_RO%x#LQszDMS@_>iFoB+zJ0Q#CQJzFGa8;pHFdi`^ zxnTC`G$7Rctm3G8t8!SY`GwFi4gF|+dAk7rh^rA{NXzc%39+xSYM~($L(pJ(8Zjs* zYdN_R^%~LiGHm9|ElV4kVZGA*T$o@YY4qpJOxGHlUi*S*A(MrgQ{&xoZQo+#PuYRs zv3a$*qoe9gBqbN|y|eaH=w^LE{>kpL!;$wRahY(hhzRY;d33W)m*dfem@)>pR54Qy z ze;^F?mwdU?K+=fBabokSls^6_6At#1Sh7W*y?r6Ss*dmZP{n;VB^LDxM1QWh;@H0J z!4S*_5j_;+@-NpO1KfQd&;C7T`9ak;X8DTRz$hDNcjG}xAfg%gwZSb^zhE~O);NMO zn2$fl7Evn%=Lk!*xsM#(y$mjukN?A&mzEw3W5>_o+6oh62kq=4-`e3B^$rG=XG}Kd zK$blh(%!9;@d@3& zGFO60j1Vf54S}+XD?%*uk7wW$f`4U3F*p7@I4Jg7f`Il}2H<{j5h?$DDe%wG7jZQL zI{mj?t?Hu>$|2UrPr5&QyK2l3mas?zzOk0DV30HgOQ|~xLXDQ8M3o#;CNKO8RK+M; zsOi%)js-MU>9H4%Q)#K_me}8OQC1u;f4!LO%|5toa1|u5Q@#mYy8nE9IXmR}b#sZK z3sD395q}*TDJJA9Er7N`y=w*S&tA;mv-)Sx4(k$fJBxXva0_;$G6!9bGBw13c_Uws zXks4u(8JA@0O9g5f?#V~qR5*u5aIe2HQO^)RW9TTcJk28l`Syl>Q#ZveEE4Em+{?%iz6=V3b>rCm9F zPQQm@-(hfNdo2%n?B)u_&Qh7^^@U>0qMBngH8}H|v+Ejg*Dd(Y#|jgJ-A zQ_bQscil%eY}8oN7ZL+2r|qv+iJY?*l)&3W_55T3GU;?@Om*(M`u0DXAsQ7HSl56> z4P!*(%&wRCb?a4HH&n;lAmr4rS=kMZb74Akha2U~Ktni>>cD$6jpugjULq)D?ea%b zk;UW0pAI~TH59P+o}*c5Ei5L-9OE;OIBt>^(;xw`>cN2`({Rzg71qrNaE=cAH^$wP zNrK9Glp^3a%m+ilQj0SnGq`okjzmE7<3I{JLD6Jn^+oas=h*4>Wvy=KXqVBa;K&ri z4(SVmMXPG}0-UTwa2-MJ=MTfM3K)b~DzSVq8+v-a0&Dsv>4B65{dBhD;(d44CaHSM zb!0ne(*<^Q%|nuaL`Gb3D4AvyO8wyygm=1;9#u5x*k0$UOwx?QxR*6Od8>+ujfyo0 zJ}>2FgW_iv(dBK2OWC-Y=Tw!UwIeOAOUUC;h95&S1hn$G#if+d;*dWL#j#YWswrz_ zMlV=z+zjZJ%SlDhxf)vv@`%~$Afd)T+MS1>ZE7V$Rj#;J*<9Ld=PrK0?qrazRJWx) z(BTLF@Wk279nh|G%ZY7_lK7=&j;x`bMND=zgh_>>-o@6%8_#Bz!FnF*onB@_k|YCF z?vu!s6#h9bL3@tPn$1;#k5=7#s*L;FLK#=M89K^|$3LICYWIbd^qguQp02w5>8p-H z+@J&+pP_^iF4Xu>`D>DcCnl8BUwwOlq6`XkjHNpi@B?OOd`4{dL?kH%lt78(-L}eah8?36zw9d-dI6D{$s{f=M7)1 zRH1M*-82}DoFF^Mi$r}bTB5r6y9>8hjL54%KfyHxn$LkW=AZ(WkHWR;tIWWr@+;^^ zVomjAWT)$+rn%g`LHB6ZSO@M3KBA? z+W7ThSBgpk`jZHZUrp`F;*%6M5kLWy6AW#T{jFHTiKXP9ITrMlEdti7@&AT_a-BA!jc(Kt zWk>IdY-2Zbz?U1)tk#n_Lsl?W;0q`;z|t9*g-xE!(}#$fScX2VkjSiboKWE~afu5d z2B@9mvT=o2fB_>Mnie=TDJB+l`GMKCy%2+NcFsbpv<9jS@$X37K_-Y!cvF5NEY`#p z3sWEc<7$E*X*fp+MqsOyMXO=<2>o8)E(T?#4KVQgt=qa%5FfUG_LE`n)PihCz2=iNUt7im)s@;mOc9SR&{`4s9Q6)U31mn?}Y?$k3kU z#h??JEgH-HGt`~%)1ZBhT9~uRi8br&;a5Y3K_Bl1G)-y(ytx?ok9S*Tz#5Vb=P~xH z^5*t_R2It95=!XDE6X{MjLYn4Eszj9Y91T2SFz@eYlx9Z9*hWaS$^5r7=W5|>sY8}mS(>e9Ez2qI1~wtlA$yv2e-Hjn&K*P z2zWSrC~_8Wrxxf#%QAL&f8iH2%R)E~IrQLgWFg8>`Vnyo?E=uiALoRP&qT{V2{$79 z%9R?*kW-7b#|}*~P#cA@q=V|+RC9=I;aK7Pju$K-n`EoGV^-8Mk=-?@$?O37evGKn z3NEgpo_4{s>=FB}sqx21d3*=gKq-Zk)U+bM%Q_}0`XGkYh*+jRaP+aDnRv#Zz*n$pGp zEU9omuYVXH{AEx>=kk}h2iKt!yqX=EHN)LF}z1j zJx((`CesN1HxTFZ7yrvA2jTPmKYVij>45{ZH2YtsHuGzIRotIFj?(8T@ZWUv{_%AI zgMZlB03C&FtgJqv9%(acqt9N)`4jy4PtYgnhqev!r$GTIOvLF5aZ{tW5MN@9BDGu* zBJzwW3sEJ~Oy8is`l6Ly3an7RPtRr^1Iu(D!B!0O241Xua>Jee;Rc7tWvj!%#yX#m z&pU*?=rTVD7pF6va1D@u@b#V@bShFr3 zMyMbNCZwT)E-%L-{%$3?n}>EN>ai7b$zR_>=l59mW;tfKj^oG)>_TGCJ#HbLBsNy$ zqAqPagZ3uQ(Gsv_-VrZmG&hHaOD#RB#6J8&sL=^iMFB=gH5AIJ+w@sTf7xa&Cnl}@ zxrtzoNq>t?=(+8bS)s2p3>jW}tye0z2aY_Dh@(18-vdfvn;D?sv<>UgL{Ti08$1Q+ zZI3q}yMA^LK=d?YVg({|v?d1|R?5 zL0S3fw)BZazRNNX|7P4rh7!+3tCG~O8l+m?H} z(CB>8(9LtKYIu3ohJ-9ecgk+L&!FX~Wuim&;v$>M4 zUfvn<=Eok(63Ubc>mZrd8d7(>8bG>J?PtOHih_xRYFu1Hg{t;%+hXu2#x%a%qzcab zv$X!ccoj)exoOnaco_jbGw7KryOtuf(SaR-VJ0nAe(1*AA}#QV1lMhGtzD>RoUZ;WA?~!K{8%chYn?ttlz17UpDLlhTkGcVfHY6R<2r4E{mU zq-}D?+*2gAkQYAKrk*rB%4WFC-B!eZZLg4(tR#@kUQHIzEqV48$9=Q(~J_0 zy1%LSCbkoOhRO!J+Oh#;bGuXe;~(bIE*!J@i<%_IcB7wjhB5iF#jBn5+u~fEECN2* z!QFh!m<(>%49H12Y33+?$JxKV3xW{xSs=gxkxW-@Xds^|O1`AmorDKrE8N2-@ospk z=Au%h=f!`_X|G^A;XWL}-_L@D6A~*4Yf!5RTTm$!t8y&fp5_oqvBjW{FufS`!)5m% z2g(=9Ap6Y2y(9OYOWuUVGp-K=6kqQ)kM0P^TQT{X{V$*sN$wbFb-DaUuJF*!?EJPl zJev!UsOB^UHZ2KppYTELh+kqDw+5dPFv&&;;C~=u$Mt+Ywga!8YkL2~@g67}3wAQP zrx^RaXb1(c7vwU8a2se75X(cX^$M{FH4AHS7d2}heqqg4F0!1|Na>UtAdT%3JnS!B)&zelTEj$^b0>Oyfw=P-y-Wd^#dEFRUN*C{!`aJIHi<_YA2?piC%^ zj!p}+ZnBrM?ErAM+D97B*7L8U$K zo(IR-&LF(85p+fuct9~VTSdRjs`d-m|6G;&PoWvC&s8z`TotPSoksp;RsL4VL@CHf z_3|Tn%`ObgRhLmr60<;ya-5wbh&t z#ycN_)3P_KZN5CRyG%LRO4`Ot)3vY#dNX9!f!`_>1%4Q`81E*2BRg~A-VcN7pcX#j zrbl@7`V%n z6J53(m?KRzKb)v?iCuYWbH*l6M77dY4keS!%>}*8n!@ROE4!|7mQ+YS4dff1JJC(t z6Fnuf^=dajqHpH1=|pb(po9Fr8it^;2dEk|Ro=$fxqK$^Yix{G($0m-{RCFQJ~LqUnO7jJcjr zl*N*!6WU;wtF=dLCWzD6kW;y)LEo=4wSXQDIcq5WttgE#%@*m><@H;~Q&GniA-$in z`sjWFLgychS1kIJmPtd-w6%iKkj&dGhtB%0)pyy0M<4HZ@ZY0PWLAd7FCrj&i|NRh?>hZj*&FYnyu%Ur`JdiTu&+n z78d3n)Rl6q&NwVj_jcr#s5G^d?VtV8bkkYco5lV0LiT+t8}98LW>d)|v|V3++zLbHC(NC@X#Hx?21J0M*gP2V`Yd^DYvVIr{C zSc4V)hZKf|OMSm%FVqSRC!phWSyuUAu%0fredf#TDR$|hMZihJ__F!)Nkh6z)d=NC z3q4V*K3JTetxCPgB2_)rhOSWhuXzu+%&>}*ARxUaDeRy{$xK(AC0I=9%X7dmc6?lZNqe-iM(`?Xn3x2Ov>sej6YVQJ9Q42>?4lil?X zew-S>tm{=@QC-zLtg*nh5mQojYnvVzf3!4TpXPuobW_*xYJs;9AokrXcs!Ay z;HK>#;G$*TPN2M!WxdH>oDY6k4A6S>BM0Nimf#LfboKxJXVBC=RBuO&g-=+@O-#0m zh*aPG16zY^tzQLNAF7L(IpGPa+mDsCeAK3k=IL6^LcE8l0o&)k@?dz!79yxUquQIe($zm5DG z5RdXTv)AjHaOPv6z%99mPsa#8OD@9=URvHoJ1hYnV2bG*2XYBgB!-GEoP&8fLmWGg z9NG^xl5D&3L^io&3iYweV*qhc=m+r7C#Jppo$Ygg;jO2yaFU8+F*RmPL` zYxfGKla_--I}YUT353k}nF1zt2NO?+kofR8Efl$Bb^&llgq+HV_UYJUH7M5IoN0sT z4;wDA0gs55ZI|FmJ0}^Pc}{Ji-|#jdR$`!s)Di4^g3b_Qr<*Qu2rz}R6!B^;`Lj3sKWzjMYjexX)-;f5Y+HfkctE{PstO-BZan0zdXPQ=V8 zS8cBhnQyy4oN?J~oK0zl!#S|v6h-nx5to7WkdEk0HKBm;?kcNO*A+u=%f~l&aY*+J z>%^Dz`EQ6!+SEX$>?d(~|MNWU-}JTrk}&`IR|Ske(G^iMdk04)Cxd@}{1=P0U*%L5 zMFH_$R+HUGGv|ju2Z>5x(-aIbVJLcH1S+(E#MNe9g;VZX{5f%_|Kv7|UY-CM(>vf= z!4m?QS+AL+rUyfGJ;~uJGp4{WhOOc%2ybVP68@QTwI(8kDuYf?#^xv zBmOHCZU8O(x)=GVFn%tg@TVW1)qJJ_bU}4e7i>&V?r zh-03>d3DFj&@}6t1y3*yOzllYQ++BO-q!)zsk`D(z||)y&}o%sZ-tUF>0KsiYKFg6 zTONq)P+uL5Vm0w{D5Gms^>H1qa&Z##*X31=58*r%Z@Ko=IMXX{;aiMUp-!$As3{sq z0EEk02MOsgGm7$}E%H1ys2$yftNbB%1rdo@?6~0!a8Ym*1f;jIgfcYEF(I_^+;Xdr z2a>&oc^dF3pm(UNpazXgVzuF<2|zdPGjrNUKpdb$HOgNp*V56XqH`~$c~oSiqx;8_ zEz3fHoU*aJUbFJ&?W)sZB3qOSS;OIZ=n-*#q{?PCXi?Mq4aY@=XvlNQdA;yVC0Vy+ z{Zk6OO!lMYWd`T#bS8FV(`%flEA9El;~WjZKU1YmZpG#49`ku`oV{Bdtvzyz3{k&7 zlG>ik>eL1P93F zd&!aXluU_qV1~sBQf$F%sM4kTfGx5MxO0zJy<#5Z&qzNfull=k1_CZivd-WAuIQf> zBT3&WR|VD|=nKelnp3Q@A~^d_jN3@$x2$f@E~e<$dk$L@06Paw$);l*ewndzL~LuU zq`>vfKb*+=uw`}NsM}~oY}gW%XFwy&A>bi{7s>@(cu4NM;!%ieP$8r6&6jfoq756W z$Y<`J*d7nK4`6t`sZ;l%Oen|+pk|Ry2`p9lri5VD!Gq`U#Ms}pgX3ylAFr8(?1#&dxrtJgB>VqrlWZf61(r`&zMXsV~l{UGjI7R@*NiMJLUoK*kY&gY9kC@^}Fj* zd^l6_t}%Ku<0PY71%zQL`@}L}48M!@=r)Q^Ie5AWhv%#l+Rhu6fRpvv$28TH;N7Cl z%I^4ffBqx@Pxpq|rTJV)$CnxUPOIn`u278s9#ukn>PL25VMv2mff)-RXV&r`Dwid7}TEZxXX1q(h{R6v6X z&x{S_tW%f)BHc!jHNbnrDRjGB@cam{i#zZK*_*xlW@-R3VDmp)<$}S%t*@VmYX;1h zFWmpXt@1xJlc15Yjs2&e%)d`fimRfi?+fS^BoTcrsew%e@T^}wyVv6NGDyMGHSKIQ zC>qFr4GY?#S#pq!%IM_AOf`#}tPoMn7JP8dHXm(v3UTq!aOfEXNRtEJ^4ED@jx%le zvUoUs-d|2(zBsrN0wE(Pj^g5wx{1YPg9FL1)V1JupsVaXNzq4fX+R!oVX+q3tG?L= z>=s38J_!$eSzy0m?om6Wv|ZCbYVHDH*J1_Ndajoh&?L7h&(CVii&rmLu+FcI;1qd_ zHDb3Vk=(`WV?Uq;<0NccEh0s`mBXcEtmwt6oN99RQt7MNER3`{snV$qBTp={Hn!zz z1gkYi#^;P8s!tQl(Y>|lvz{5$uiXsitTD^1YgCp+1%IMIRLiSP`sJru0oY-p!FPbI)!6{XM%)(_Dolh1;$HlghB-&e><;zU&pc=ujpa-(+S&Jj zX1n4T#DJDuG7NP;F5TkoG#qjjZ8NdXxF0l58RK?XO7?faM5*Z17stidTP|a%_N z^e$D?@~q#Pf+708cLSWCK|toT1YSHfXVIs9Dnh5R(}(I;7KhKB7RD>f%;H2X?Z9eR z{lUMuO~ffT!^ew= z7u13>STI4tZpCQ?yb9;tSM-(EGb?iW$a1eBy4-PVejgMXFIV_Ha^XB|F}zK_gzdhM z!)($XfrFHPf&uyFQf$EpcAfk83}91Y`JFJOiQ;v5ca?)a!IxOi36tGkPk4S6EW~eq z>WiK`Vu3D1DaZ}515nl6>;3#xo{GQp1(=uTXl1~ z4gdWxr-8a$L*_G^UVd&bqW_nzMM&SlNW$8|$lAfo@zb+P>2q?=+T^qNwblP*RsN?N zdZE%^Zs;yAwero1qaoqMp~|KL=&npffh981>2om!fseU(CtJ=bW7c6l{U5(07*e0~ zJRbid6?&psp)ilmYYR3ZIg;t;6?*>hoZ3uq7dvyyq-yq$zH$yyImjfhpQb@WKENSP zl;KPCE+KXzU5!)mu12~;2trrLfs&nlEVOndh9&!SAOdeYd}ugwpE-9OF|yQs(w@C9 zoXVX`LP~V>%$<(%~tE*bsq(EFm zU5z{H@Fs^>nm%m%wZs*hRl=KD%4W3|(@j!nJr{Mmkl`e_uR9fZ-E{JY7#s6i()WXB0g-b`R{2r@K{2h3T+a>82>722+$RM*?W5;Bmo6$X3+Ieg9&^TU(*F$Q3 zT572!;vJeBr-)x?cP;^w1zoAM`nWYVz^<6N>SkgG3s4MrNtzQO|A?odKurb6DGZffo>DP_)S0$#gGQ_vw@a9JDXs2}hV&c>$ zUT0;1@cY5kozKOcbN6)n5v)l#>nLFL_x?2NQgurQH(KH@gGe>F|$&@ zq@2A!EXcIsDdzf@cWqElI5~t z4cL9gg7{%~4@`ANXnVAi=JvSsj95-7V& zME3o-%9~2?cvlH#twW~99=-$C=+b5^Yv}Zh4;Mg-!LS zw>gqc=}CzS9>v5C?#re>JsRY!w|Mtv#%O3%Ydn=S9cQarqkZwaM4z(gL~1&oJZ;t; zA5+g3O6itCsu93!G1J_J%Icku>b3O6qBW$1Ej_oUWc@MI)| zQ~eyS-EAAnVZp}CQnvG0N>Kc$h^1DRJkE7xZqJ0>p<>9*apXgBMI-v87E0+PeJ-K& z#(8>P_W^h_kBkI;&e_{~!M+TXt@z8Po*!L^8XBn{of)knd-xp{heZh~@EunB2W)gd zAVTw6ZZasTi>((qpBFh(r4)k zz&@Mc@ZcI-4d639AfcOgHOU+YtpZ)rC%Bc5gw5o~+E-i+bMm(A6!uE>=>1M;V!Wl4 z<#~muol$FsY_qQC{JDc8b=$l6Y_@_!$av^08`czSm!Xan{l$@GO-zPq1s>WF)G=wv zDD8j~Ht1pFj)*-b7h>W)@O&m&VyYci&}K|0_Z*w`L>1jnGfCf@6p}Ef*?wdficVe_ zmPRUZ(C+YJU+hIj@_#IiM7+$4kH#VS5tM!Ksz01siPc-WUe9Y3|pb4u2qnn zRavJiRpa zq?tr&YV?yKt<@-kAFl3s&Kq#jag$hN+Y%%kX_ytvpCsElgFoN3SsZLC>0f|m#&Jhu zp7c1dV$55$+k78FI2q!FT}r|}cIV;zp~#6X2&}22$t6cHx_95FL~T~1XW21VFuatb zpM@6w>c^SJ>Pq6{L&f9()uy)TAWf;6LyHH3BUiJ8A4}od)9sriz~e7}l7Vr0e%(=>KG1Jay zW0azuWC`(|B?<6;R)2}aU`r@mt_#W2VrO{LcX$Hg9f4H#XpOsAOX02x^w9+xnLVAt z^~hv2guE-DElBG+`+`>PwXn5kuP_ZiOO3QuwoEr)ky;o$n7hFoh}Aq0@Ar<8`H!n} zspCC^EB=6>$q*gf&M2wj@zzfBl(w_@0;h^*fC#PW9!-kT-dt*e7^)OIU{Uw%U4d#g zL&o>6`hKQUps|G4F_5AuFU4wI)(%9(av7-u40(IaI|%ir@~w9-rLs&efOR@oQy)}{ z&T#Qf`!|52W0d+>G!h~5A}7VJky`C3^fkJzt3|M&xW~x-8rSi-uz=qBsgODqbl(W#f{Ew#ui(K)(Hr&xqZs` zfrK^2)tF#|U=K|_U@|r=M_Hb;qj1GJG=O=d`~#AFAccecIaq3U`(Ds1*f*TIs=IGL zp_vlaRUtFNK8(k;JEu&|i_m39c(HblQkF8g#l|?hPaUzH2kAAF1>>Yykva0;U@&oRV8w?5yEK??A0SBgh?@Pd zJg{O~4xURt7!a;$rz9%IMHQeEZHR8KgFQixarg+MfmM_OeX#~#&?mx44qe!wt`~dd zqyt^~ML>V>2Do$huU<7}EF2wy9^kJJSm6HoAD*sRz%a|aJWz_n6?bz99h)jNMp}3k ztPVbos1$lC1nX_OK0~h>=F&v^IfgBF{#BIi&HTL}O7H-t4+wwa)kf3AE2-Dx@#mTA z!0f`>vz+d3AF$NH_-JqkuK1C+5>yns0G;r5ApsU|a-w9^j4c+FS{#+7- zH%skr+TJ~W_8CK_j$T1b;$ql_+;q6W|D^BNK*A+W5XQBbJy|)(IDA=L9d>t1`KX2b zOX(Ffv*m?e>! zS3lc>XC@IqPf1g-%^4XyGl*1v0NWnwZTW?z4Y6sncXkaA{?NYna3(n@(+n+#sYm}A zGQS;*Li$4R(Ff{obl3#6pUsA0fKuWurQo$mWXMNPV5K66V!XYOyc})^>889Hg3I<{V^Lj9($B4Zu$xRr=89-lDz9x`+I8q(vEAimx1K{sTbs|5x7S zZ+7o$;9&9>@3K;5-DVzGw=kp7ez%1*kxhGytdLS>Q)=xUWv3k_x(IsS8we39Tijvr z`GKk>gkZTHSht;5q%fh9z?vk%sWO}KR04G9^jleJ^@ovWrob7{1xy7V=;S~dDVt%S za$Q#Th%6g1(hiP>hDe}7lcuI94K-2~Q0R3A1nsb7Y*Z!DtQ(Ic<0;TDKvc6%1kBdJ z$hF!{uALB0pa?B^TC}#N5gZ|CKjy|BnT$7eaKj;f>Alqdb_FA3yjZ4CCvm)D&ibL) zZRi91HC!TIAUl<|`rK_6avGh`!)TKk=j|8*W|!vb9>HLv^E%t$`@r@piI(6V8pqDG zBON7~=cf1ZWF6jc{qkKm;oYBtUpIdau6s+<-o^5qNi-p%L%xAtn9OktFd{@EjVAT% z#?-MJ5}Q9QiK_jYYWs+;I4&!N^(mb!%4zx7qO6oCEDn=8oL6#*9XIJ&iJ30O`0vsFy|fEVkw}*jd&B6!IYi+~Y)qv6QlM&V9g0 zh)@^BVDB|P&#X{31>G*nAT}Mz-j~zd>L{v{9AxrxKFw8j;ccQ$NE0PZCc(7fEt1xd z`(oR2!gX6}R+Z77VkDz^{I)@%&HQT5q+1xlf*3R^U8q%;IT8-B53&}dNA7GW`Ki&= z$lrdH zDCu;j$GxW<&v_4Te7=AE2J0u1NM_7Hl9$u{z(8#%8vvrx2P#R7AwnY|?#LbWmROa; zOJzU_*^+n(+k;Jd{e~So9>OF>fPx$Hb$?~K1ul2xr>>o@**n^6IMu8+o3rDp(X$cC z`wQt9qIS>yjA$K~bg{M%kJ00A)U4L+#*@$8UlS#lN3YA{R{7{-zu#n1>0@(#^eb_% zY|q}2)jOEM8t~9p$X5fpT7BZQ1bND#^Uyaa{mNcFWL|MoYb@>y`d{VwmsF&haoJuS2W7azZU0{tu#Jj_-^QRc35tjW~ae&zhKk!wD}#xR1WHu z_7Fys#bp&R?VXy$WYa$~!dMxt2@*(>@xS}5f-@6eoT%rwH zv_6}M?+piNE;BqaKzm1kK@?fTy$4k5cqYdN8x-<(o6KelwvkTqC3VW5HEnr+WGQlF zs`lcYEm=HPpmM4;Ich7A3a5Mb3YyQs7(Tuz-k4O0*-YGvl+2&V(B&L1F8qfR0@vQM-rF<2h-l9T12eL}3LnNAVyY_z51xVr$%@VQ-lS~wf3mnHc zoM({3Z<3+PpTFCRn_Y6cbxu9v>_>eTN0>hHPl_NQQuaK^Mhrv zX{q#80ot;ptt3#js3>kD&uNs{G0mQp>jyc0GG?=9wb33hm z`y2jL=J)T1JD7eX3xa4h$bG}2ev=?7f>-JmCj6){Upo&$k{2WA=%f;KB;X5e;JF3IjQBa4e-Gp~xv- z|In&Rad7LjJVz*q*+splCj|{7=kvQLw0F@$vPuw4m^z=B^7=A4asK_`%lEf_oIJ-O z{L)zi4bd#&g0w{p1$#I&@bz3QXu%Y)j46HAJKWVfRRB*oXo4lIy7BcVl4hRs<%&iQ zr|)Z^LUJ>qn>{6y`JdabfNNFPX7#3`x|uw+z@h<`x{J4&NlDjnknMf(VW_nKWT!Jh zo1iWBqT6^BR-{T=4Ybe+?6zxP_;A5Uo{}Xel%*=|zRGm1)pR43K39SZ=%{MDCS2d$~}PE-xPw4ZK6)H;Zc&0D5p!vjCn0wCe&rVIhchR9ql!p2`g0b@JsC^J#n_r*4lZ~u0UHKwo(HaHUJDHf^gdJhTdTW z3i7Zp_`xyKC&AI^#~JMVZj^9WsW}UR#nc#o+ifY<4`M+?Y9NTBT~p`ONtAFf8(ltr*ER-Ig!yRs2xke#NN zkyFcaQKYv>L8mQdrL+#rjgVY>Z2_$bIUz(kaqL}cYENh-2S6BQK-a(VNDa_UewSW` zMgHi<3`f!eHsyL6*^e^W7#l?V|42CfAjsgyiJsA`yNfAMB*lAsJj^K3EcCzm1KT zDU2+A5~X%ax-JJ@&7>m`T;;}(-e%gcYQtj}?ic<*gkv)X2-QJI5I0tA2`*zZRX(;6 zJ0dYfMbQ+{9Rn3T@Iu4+imx3Y%bcf2{uT4j-msZ~eO)5Z_T7NC|Nr3)|NWjomhv=E zXaVin)MY)`1QtDyO7mUCjG{5+o1jD_anyKn73uflH*ASA8rm+S=gIfgJ);>Zx*hNG z!)8DDCNOrbR#9M7Ud_1kf6BP)x^p(|_VWCJ+(WGDbYmnMLWc?O4zz#eiP3{NfP1UV z(n3vc-axE&vko^f+4nkF=XK-mnHHQ7>w05$Q}iv(kJc4O3TEvuIDM<=U9@`~WdKN* zp4e4R1ncR_kghW}>aE$@OOc~*aH5OOwB5U*Z)%{LRlhtHuigxH8KuDwvq5{3Zg{Vr zrd@)KPwVKFP2{rXho(>MTZZfkr$*alm_lltPob4N4MmhEkv`J(9NZFzA>q0Ch;!Ut zi@jS_=0%HAlN+$-IZGPi_6$)ap>Z{XQGt&@ZaJ(es!Po5*3}>R4x66WZNsjE4BVgn z>}xm=V?F#tx#e+pimNPH?Md5hV7>0pAg$K!?mpt@pXg6UW9c?gvzlNe0 z3QtIWmw$0raJkjQcbv-7Ri&eX6Ks@@EZ&53N|g7HU<;V1pkc&$3D#8k!coJ=^{=vf z-pCP;vr2#A+i#6VA?!hs6A4P@mN62XYY$#W9;MwNia~89i`=1GoFESI+%Mbrmwg*0 zbBq4^bA^XT#1MAOum)L&ARDXJ6S#G>&*72f50M1r5JAnM1p7GFIv$Kf9eVR(u$KLt z9&hQ{t^i16zL1c(tRa~?qr?lbSN;1k;%;p*#gw_BwHJRjcYPTj6>y-rw*dFTnEs95 z`%-AoPL!P16{=#RI0 zUb6#`KR|v^?6uNnY`zglZ#Wd|{*rZ(x&Hk8N6ob6mpX~e^qu5kxvh$2TLJA$M=rx zc!#ot+sS+-!O<0KR6+Lx&~zgEhCsbFY{i_DQCihspM?e z-V}HemMAvFzXR#fV~a=Xf-;tJ1edd}Mry@^=9BxON;dYr8vDEK<<{ zW~rg(ZspxuC&aJo$GTM!9_sXu(EaQJNkV9AC(ob#uA=b4*!Uf}B*@TK=*dBvKKPAF z%14J$S)s-ws9~qKsf>DseEW(ssVQ9__YNg}r9GGx3AJiZR@w_QBlGP>yYh0lQCBtf zx+G;mP+cMAg&b^7J!`SiBwC81M_r0X9kAr2y$0(Lf1gZK#>i!cbww(hn$;fLIxRf? z!AtkSZc-h76KGSGz%48Oe`8ZBHkSXeVb!TJt_VC>$m<#}(Z}!(3h631ltKb3CDMw^fTRy%Ia!b&at`^g7Ew-%WLT9(#V0OP9CE?uj62s>`GI3NA z!`$U+i<`;IQyNBkou4|-7^9^ylac-Xu!M+V5p5l0Ve?J0wTSV+$gYtoc=+Ve*OJUJ z$+uIGALW?}+M!J9+M&#bT=Hz@{R2o>NtNGu1yS({pyteyb>*sg4N`KAD?`u3F#C1y z2K4FKOAPASGZTep54PqyCG(h3?kqQQAxDSW@>T2d!n;9C8NGS;3A8YMRcL>b=<<%M zMiWf$jY;`Ojq5S{kA!?28o)v$;)5bTL<4eM-_^h4)F#eeC2Dj*S`$jl^yn#NjJOYT zx%yC5Ww@eX*zsM)P(5#wRd=0+3~&3pdIH7CxF_2iZSw@>kCyd z%M}$1p((Bidw4XNtk&`BTkU{-PG)SXIZ)yQ!Iol6u8l*SQ1^%zC72FP zLvG>_Z0SReMvB%)1@+et0S{<3hV@^SY3V~5IY(KUtTR{*^xJ^2NN{sIMD9Mr9$~(C$GLNlSpzS=fsbw-DtHb_T|{s z9OR|sx!{?F``H!gVUltY7l~dx^a(2;OUV^)7 z%@hg`8+r&xIxmzZ;Q&v0X%9P)U0SE@r@(lKP%TO(>6I_iF{?PX(bez6v8Gp!W_nd5 z<8)`1jcT)ImNZp-9rr4_1MQ|!?#8sJQx{`~7)QZ75I=DPAFD9Mt{zqFrcrXCU9MG8 zEuGcy;nZ?J#M3!3DWW?Zqv~dnN6ijlIjPfJx(#S0cs;Z=jDjKY|$w2s4*Xa1Iz953sN2Lt!Vmk|%ZwOOqj`sA--5Hiaq8!C%LV zvWZ=bxeRV(&%BffMJ_F~~*FdcjhRVNUXu)MS(S#67rDe%Ler=GS+WysC1I2=Bmbh3s6wdS}o$0 zz%H08#SPFY9JPdL6blGD$D-AaYi;X!#zqib`(XX*i<*eh+2UEPzU4}V4RlC3{<>-~ zadGA8lSm>b7Z!q;D_f9DT4i)Q_}ByElGl*Cy~zX%IzHp)@g-itZB6xM70psn z;AY8II99e6P2drgtTG5>`^|7qg`9MTp%T~|1N3tBqV}2zgow3TFAH{XPor0%=HrkXnKyxyozHlJ6 zd3}OWkl?H$l#yZqOzZbMI+lDLoH48;s10!m1!K87g;t}^+A3f3e&w{EYhVPR0Km*- zh5-ku$Z|Ss{2?4pGm(Rz!0OQb^_*N`)rW{z)^Cw_`a(_L9j=&HEJl(!4rQy1IS)>- zeTIr>hOii`gc(fgYF(cs$R8l@q{mJzpoB5`5r>|sG zBpsY}RkY(g5`bj~D>(;F8v*DyjX(#nVLSs>)XneWI&%Wo>a0u#4A?N<1SK4D}&V1oN)76 z%S>a2n3n>G`YY1>0Hvn&AMtMuI_?`5?4y3w2Hnq4Qa2YH5 zxKdfM;k467djL31Y$0kd9FCPbU=pHBp@zaIi`Xkd80;%&66zvSqsq6%aY)jZacfvw ztkWE{ZV6V2WL9e}Dvz|!d96KqVkJU@5ryp#rReeWu>mSrOJxY^tWC9wd0)$+lZc%{ zY=c4#%OSyQJvQUuy^u}s8DN8|8T%TajOuaY^)R-&8s@r9D`(Ic4NmEu)fg1f!u`xUb;9t#rM z>}cY=648@d5(9A;J)d{a^*ORdVtJrZ77!g~^lZ9@)|-ojvW#>)Jhe8$7W3mhmQh@S zU=CSO+1gSsQ+Tv=x-BD}*py_Ox@;%#hPb&tqXqyUW9jV+fonnuCyVw=?HR>dAB~Fg z^vl*~y*4|)WUW*9RC%~O1gHW~*tJb^a-j;ae2LRNo|0S2`RX>MYqGKB^_ng7YRc@! zFxg1X!VsvXkNuv^3mI`F2=x6$(pZdw=jfYt1ja3FY7a41T07FPdCqFhU6%o|Yb6Z4 zpBGa=(ao3vvhUv#*S{li|EyujXQPUV;0sa5!0Ut)>tPWyC9e0_9(=v*z`TV5OUCcx zT=w=^8#5u~7<}8Mepqln4lDv*-~g^VoV{(+*4w(q{At6d^E-Usa2`JXty++Oh~on^ z;;WHkJsk2jvh#N|?(2PLl+g!M0#z_A;(#Uy=TzL&{Ei5G9#V{JbhKV$Qmkm%5tn!CMA? z@hM=b@2DZWTQ6>&F6WCq6;~~WALiS#@{|I+ucCmD6|tBf&e;$_)%JL8$oIQ%!|Xih1v4A$=7xNO zZVz$G8;G5)rxyD+M0$20L$4yukA_D+)xmK3DMTH3Q+$N&L%qB)XwYx&s1gkh=%qGCCPwnwhbT4p%*3R)I}S#w7HK3W^E%4w z2+7ctHPx3Q97MFYB48HfD!xKKb(U^K_4)Bz(5dvwyl*R?)k;uHEYVi|{^rvh)w7}t z`tnH{v9nlVHj2ign|1an_wz0vO)*`3RaJc#;(W-Q6!P&>+@#fptCgtUSn4!@b7tW0&pE2Qj@7}f#ugu4*C)8_}AMRuz^WG zc)XDcOPQjRaGptRD^57B83B-2NKRo!j6TBAJntJPHNQG;^Oz}zt5F^kId~miK3J@l ztc-IKp6qL!?u~q?qfGP0I~$5gvq#-0;R(oLU@sYayr*QH95fnrYA*E|n%&FP@Cz`a zSdJ~(c@O^>qaO`m9IQ8sd8!L<+)GPJDrL7{4{ko2gWOZel^3!($Gjt|B&$4dtfTmBmC>V`R&&6$wpgvdmns zxcmfS%9_ZoN>F~azvLFtA(9Q5HYT#A(byGkESnt{$Tu<73$W~reB4&KF^JBsoqJ6b zS?$D7DoUgzLO-?P`V?5_ub$nf1p0mF?I)StvPomT{uYjy!w&z$t~j&en=F~hw|O(1 zlV9$arQmKTc$L)Kupwz_zA~deT+-0WX6NzFPh&d+ly*3$%#?Ca9Z9lOJsGVoQ&1HNg+)tJ_sw)%oo*DK)iU~n zvL``LqTe=r=7SwZ@LB)9|3QB5`0(B9r(iR}0nUwJss-v=dXnwMRQFYSRK1blS#^g(3@z{`=8_CGDm!LESTWig zzm1{?AG&7`uYJ;PoFO$o8RWuYsV26V{>D-iYTnvq7igWx9@w$EC*FV^vpvDl@i9yp zPIqiX@hEZF4VqzI3Y)CHhR`xKN8poL&~ak|wgbE4zR%Dm(a@?bw%(7(!^>CM!^4@J z6Z)KhoQP;WBq_Z_&<@i2t2&xq>N>b;Np2rX?yK|-!14iE2T}E|jC+=wYe~`y38g3J z8QGZquvqBaG!vw&VtdXWX5*i5*% zJP~7h{?&E|<#l{klGPaun`IgAJ4;RlbRqgJz5rmHF>MtJHbfqyyZi53?Lhj=(Ku#& z__ubmZIxzSq3F90Xur!1)Vqe6b@!ueHA!93H~jdHmaS5Q^CULso}^poy)0Op6!{^9 zWyCyyIrdBP4fkliZ%*g+J-A!6VFSRF6Liu6G^^=W>cn81>4&7(c7(6vCGSAJ zQZ|S3mb|^Wf=yJ(h~rq`iiW~|n#$+KcblIR<@|lDtm!&NBzSG-1;7#YaU+-@=xIm4 zE}edTYd~e&_%+`dIqqgFntL-FxL3!m4yTNt<(^Vt9c6F(`?9`u>$oNxoKB29<}9FE zgf)VK!*F}nW?}l95%RRk8N4^Rf8)Xf;drT4<|lUDLPj^NPMrBPL;MX&0oGCsS za3}vWcF(IPx&W6{s%zwX{UxHX2&xLGfT{d9bWP!g;Lg#etpuno$}tHoG<4Kd*=kpU z;4%y(<^yj(UlG%l-7E9z_Kh2KoQ19qT3CR@Ghr>BAgr3Vniz3LmpC4g=g|A3968yD2KD$P7v$ zx9Q8`2&qH3&y-iv0#0+jur@}k`6C%7fKbCr|tHX2&O%r?rBpg`YNy~2m+ z*L7dP$RANzVUsG_Lb>=__``6vA*xpUecuGsL+AW?BeSwyoQfDlXe8R1*R1M{0#M?M zF+m19`3<`gM{+GpgW^=UmuK*yMh3}x)7P738wL8r@(Na6%ULPgbPVTa6gh5Q(SR0f znr6kdRpe^(LVM;6Rt(Z@Lsz3EX*ry6(WZ?w>#ZRelx)N%sE+MN>5G|Z8{%@b&D+Ov zPU{shc9}%;G7l;qbonIb_1m^Qc8ez}gTC-k02G8Rl?7={9zBz8uRX2{XJQ{vZhs67avlRn| zgRtWl0Lhjet&!YC47GIm%1gdq%T24_^@!W3pCywc89X4I5pnBCZDn(%!$lOGvS*`0!AoMtqxNPFgaMR zwoW$p;8l6v%a)vaNsesED3f}$%(>zICnoE|5JwP&+0XI}JxPccd+D^gx`g`=GsUc0 z9Uad|C+_@_0%JmcObGnS@3+J^0P!tg+fUZ_w#4rk#TlJYPXJiO>SBxzs9(J;XV9d{ zmTQE1(K8EYaz9p^XLbdWudyIPJlGPo0U*)fAh-jnbfm@SYD_2+?|DJ-^P+ojG{2{6 z>HJtedEjO@j_tqZ4;Zq1t5*5cWm~W?HGP!@_f6m#btM@46cEMhhK{(yI&jG)fwL1W z^n_?o@G8a-jYt!}$H*;{0#z8lANlo!9b@!c5K8<(#lPlpE!z86Yq#>WT&2} z;;G1$pD%iNoj#Z=&kij5&V1KHIhN-h<;{HC5wD)PvkF>CzlQOEx_0;-TJ*!#&{Wzt zKcvq^SZIdop}y~iouNqtU7K7+?eIz-v_rfNM>t#i+dD$s_`M;sjGubTdP)WI*uL@xPOLHt#~T<@Yz>xt50ZoTw;a(a}lNiDN-J${gOdE zx?8LOA|tv{Mb}=TTR=LcqMqbCJkKj+@;4Mu)Cu0{`~ohix6E$g&tff)aHeUAQQ%M? zIN4uSUTzC1iMEWL*W-in1y)C`E+R8j?4_?X4&2Zv5?QdkNMz(k} zw##^Ikx`#_s>i&CO_mu@vJJ*|3ePRDl5pq$9V^>D;g0R%l>lw;ttyM6Sy`NBF{)Lr zSk)V>mZr96+aHY%vTLLt%vO-+juw6^SO_ zYGJaGeWX6W(TOQx=5oTGXOFqMMU*uZyt>MR-Y`vxW#^&)H zk0!F8f*@v6NO@Z*@Qo)+hlX40EWcj~j9dGrLaq%1;DE_%#lffXCcJ;!ZyyyZTz74Q zb2WSly6sX{`gQeToQsi1-()5EJ1nJ*kXGD`xpXr~?F#V^sxE3qSOwRSaC9x9oa~jJ zTG9`E|q zC5Qs1xh}jzb5UPYF`3N9YuMnI7xsZ41P;?@c|%w zl=OxLr6sMGR+`LStLvh)g?fA5p|xbUD;yFAMQg&!PEDYxVYDfA>oTY;CFt`cg?Li1 z0b})!9Rvw&j#*&+D2))kXLL z0+j=?7?#~_}N-qdEIP>DQaZh#F(#e0WNLzwUAj@r694VJ8?Dr5_io2X49XYsG^ zREt0$HiNI~6VV!ycvao+0v7uT$_ilKCvsC+VDNg7yG1X+eNe^3D^S==F3ByiW0T^F zH6EsH^}Uj^VPIE&m)xlmOScYR(w750>hclqH~~dM2+;%GDXT`u4zG!p((*`Hwx41M z4KB+`hfT(YA%W)Ve(n+Gu9kuXWKzxg{1ff^xNQw>w%L-)RySTk9kAS92(X0Shg^Q? zx1YXg_TLC^?h6!4mBqZ9pKhXByu|u~gF%`%`vdoaGBN3^j4l!4x?Bw4Jd)Z4^di}! zXlG1;hFvc>H?bmmu1E7Vx=%vahd!P1#ZGJOJYNbaek^$DHt`EOE|Hlij+hX>ocQFSLVu|wz`|KVl@Oa;m2k6b*mNK2Vo{~l9>Qa3@B7G7#k?)aLx;w6U ze8bBq%vF?5v>#TspEoaII!N}sRT~>bh-VWJ7Q*1qsz%|G)CFmnttbq$Ogb{~YK_=! z{{0vhlW@g!$>|}$&4E3@k`KPElW6x#tSX&dfle>o!irek$NAbDzdd2pVeNzk4&qgJ zXvNF0$R96~g0x+R1igR=Xu&X_Hc5;!Ze&C)eUTB$9wW&?$&o8Yxhm5s(S`;?{> z*F?9Gr0|!OiKA>Rq-ae=_okB6&yMR?!JDer{@iQgIn=cGxs-u^!8Q$+N&pfg2WM&Z zulHu=Uh~U>fS{=Nm0x>ACvG*4R`Dx^kJ65&Vvfj`rSCV$5>c04N26Rt2S?*kh3JKq z9(3}5T?*x*AP(X2Ukftym0XOvg~r6Ms$2x&R&#}Sz23aMGU&7sU-cFvE3Eq`NBJe84VoftWF#v7PDAp`@V zRFCS24_k~;@~R*L)eCx@Q9EYmM)Sn}HLbVMyxx%{XnMBDc-YZ<(DXDBYUt8$u5Zh} zBK~=M9cG$?_m_M61YG+#|9Vef7LfbH>(C21&aC)x$^Lg}fa#SF){RX|?-xZjSOrn# z2ZAwUF)$VB<&S;R3FhNSQOV~8w%A`V9dWyLiy zgt7G=Z4t|zU3!dh5|s(@XyS|waBr$>@=^Dspmem8)@L`Ns{xl%rGdX!R(BiC5C7Vo zXetb$oC_iXS}2x_Hy}T(hUUNbO47Q@+^4Q`h>(R-;OxCyW#eoOeC51jzxnM1yxBrp zz6}z`(=cngs6X05e79o_B7@3K|Qpe3n38Py_~ zpi?^rj!`pq!7PHGliC$`-8A^Ib?2qgJJCW+(&TfOnFGJ+@-<<~`7BR0f4oSINBq&R z2CM`0%WLg_Duw^1SPwj-{?BUl2Y=M4e+7yL1{C&&f&zjF06#xf>VdLozgNye(BNgSD`=fFbBy0HIosLl@JwCQl^s;eTnc( z3!r8G=K>zb`|bLLI0N|eFJk%s)B>oJ^M@AQzqR;HUjLsOqW<0v>1ksT_#24*U@R3HJu*A^#1o#P3%3_jq>icD@<`tqU6ICEgZrME(xX#?i^Z z%Id$_uyQGlFD-CcaiRtRdGn|K`Lq5L-rx7`vYYGH7I=eLfHRozPiUtSe~Tt;IN2^gCXmf2#D~g2@9bhzK}3nphhG%d?V7+Zq{I2?Gt*!NSn_r~dd$ zqkUOg{U=MI?Ehx@`(X%rQB?LP=CjJ*V!rec{#0W2WshH$X#9zep!K)tzZoge*LYd5 z@g?-j5_mtMp>_WW`p*UNUZTFN{_+#m*bJzt{hvAdkF{W40{#L3w6gzPztnsA_4?&0 z(+>pv!zB16rR-(nm(^c>Z(its{ny677vT8sF564^mlZvJ!h65}OW%Hn|2OXbOQM%b z{6C54Z2v;^hyMQ;UH+HwFD2!F!VlQ}6Z{L0_9g5~CH0@Mqz?ZC`^QkhOU#$Lx<4`B zyZsa9uPF!rZDo8ZVfzzR#raQ>5|)k~_Ef*wDqG^76o)j!C4 zykvT*o$!-MBko@?{b~*Zf2*YMlImrK`cEp|#D7f%Twm<|C|dWD \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..6d57edc --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega From f4e4e6e0f1799d226b67ee12fb06c16e4f1ce2d2 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 20:59:10 +0200 Subject: [PATCH 04/20] set gradle exec permissions --- gradlew | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 gradlew diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 From 6ea9ecd0cb28a7e7b81ae0abb3adb712d00c45cc Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 21:06:44 +0200 Subject: [PATCH 05/20] Upgraded to gradle 6.0 --- gradle/wrapper/gradle-wrapper.properties | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index fde43ea..6ce793f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Wed Nov 13 18:13:51 EET 2019 -distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.0-bin.zip zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists From 7e6fac2e2f324e5e46b771d85650dffb6850580a Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 21:51:07 +0200 Subject: [PATCH 06/20] StoreReader and StoreWriter now both implement AutoCloseable. Optimization when deserializing empty byte arrays. --- .../com/linkedin/paldb/api/StoreReader.java | 95 ++++++++++--------- .../com/linkedin/paldb/api/StoreWriter.java | 13 +-- .../linkedin/paldb/impl/StorageReader.java | 11 +-- .../paldb/impl/StorageSerialization.java | 5 + .../linkedin/paldb/impl/StorageWriter.java | 2 +- .../linkedin/paldb/impl/TestStoreReader.java | 36 +------ 6 files changed, 69 insertions(+), 93 deletions(-) diff --git a/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java b/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java index 89052eb..5decf79 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java @@ -25,35 +25,36 @@ * get() method to fetch. Call the * close() to liberate resources when done. */ -public interface StoreReader { +public interface StoreReader extends AutoCloseable { /** * Closes the store reader and free resources. *

* A closed reader can't be reopened. */ - public void close(); + @Override + void close(); /** * Returns the reader's configuration. * * @return the store configuration */ - public Configuration getConfiguration(); + Configuration getConfiguration(); /** * Returns the store file. * * @return file */ - public File getFile(); + File getFile(); /** * Returns the number of keys in the store. * * @return key count */ - public long size(); + long size(); /** * Gets the value for key or null if not found. @@ -62,7 +63,7 @@ public interface StoreReader { * @param return type * @return value or null if not found */ - public K get(Object key); + K get(Object key); /** * Gets the value for key or defaultValue if not found. @@ -72,7 +73,7 @@ public interface StoreReader { * @param return type * @return value of defaultValue if not found */ - public K get(Object key, K defaultValue); + K get(Object key, K defaultValue); /** * Gets the int value for key. @@ -81,7 +82,7 @@ public interface StoreReader { * @return int value * @throws NotFoundException if not found */ - public int getInt(Object key) + int getInt(Object key) throws NotFoundException; /** @@ -91,7 +92,7 @@ public int getInt(Object key) * @param defaultValue default value * @return int value or defaultValue if not found */ - public int getInt(Object key, int defaultValue); + int getInt(Object key, int defaultValue); /** * Gets the long value for key. @@ -100,7 +101,7 @@ public int getInt(Object key) * @return long value * @throws NotFoundException if not found */ - public long getLong(Object key) + long getLong(Object key) throws NotFoundException; /** @@ -110,7 +111,7 @@ public long getLong(Object key) * @param defaultValue default value * @return long value or defaultValue if not found */ - public long getLong(Object key, long defaultValue); + long getLong(Object key, long defaultValue); /** * Gets the boolean value for key. @@ -119,7 +120,7 @@ public long getLong(Object key) * @return boolean value * @throws NotFoundException if not found */ - public boolean getBoolean(Object key) + boolean getBoolean(Object key) throws NotFoundException; /** @@ -129,7 +130,7 @@ public boolean getBoolean(Object key) * @param defaultValue default value * @return boolean value or defaultValue if not found */ - public boolean getBoolean(Object key, boolean defaultValue); + boolean getBoolean(Object key, boolean defaultValue); /** * Gets the float value for key. @@ -138,7 +139,7 @@ public boolean getBoolean(Object key) * @return float value * @throws NotFoundException if not found */ - public float getFloat(Object key) + float getFloat(Object key) throws NotFoundException; /** @@ -148,7 +149,7 @@ public float getFloat(Object key) * @param defaultValue default value * @return float value or defaultValue if not found */ - public float getFloat(Object key, float defaultValue); + float getFloat(Object key, float defaultValue); /** * Gets the double value for key. @@ -157,7 +158,7 @@ public float getFloat(Object key) * @return double value * @throws NotFoundException if not found */ - public double getDouble(Object key) + double getDouble(Object key) throws NotFoundException; /** @@ -167,7 +168,7 @@ public double getDouble(Object key) * @param defaultValue default value * @return double value or defaultValue if not found */ - public double getDouble(Object key, double defaultValue); + double getDouble(Object key, double defaultValue); /** * Gets the short value for key. @@ -176,7 +177,7 @@ public double getDouble(Object key) * @return short value * @throws NotFoundException if not found */ - public short getShort(Object key) + short getShort(Object key) throws NotFoundException; /** @@ -186,7 +187,7 @@ public short getShort(Object key) * @param defaultValue default value * @return short value or defaultValue if not found */ - public short getShort(Object key, short defaultValue); + short getShort(Object key, short defaultValue); /** * Gets the byte value for key. @@ -195,7 +196,7 @@ public short getShort(Object key) * @return byte value * @throws NotFoundException if not found */ - public byte getByte(Object key) + byte getByte(Object key) throws NotFoundException; /** @@ -205,7 +206,7 @@ public byte getByte(Object key) * @param defaultValue default value * @return byte value or defaultValue if not found */ - public byte getByte(Object key, byte defaultValue); + byte getByte(Object key, byte defaultValue); /** * Gets the string value for key or null if not found. @@ -213,7 +214,7 @@ public byte getByte(Object key) * @param key key to fetch * @return string value */ - public String getString(Object key); + String getString(Object key); /** * Gets the string value for key or defaultValue if not found. @@ -222,7 +223,7 @@ public byte getByte(Object key) * @param defaultValue default value * @return string value or defaultValue if not found */ - public String getString(Object key, String defaultValue); + String getString(Object key, String defaultValue); /** * Gets the char value for key. @@ -231,7 +232,7 @@ public byte getByte(Object key) * @return char value * @throws NotFoundException if not found */ - public char getChar(Object key) + char getChar(Object key) throws NotFoundException; /** @@ -241,7 +242,7 @@ public char getChar(Object key) * @param defaultValue default value * @return char value or defaultValue if not found */ - public char getChar(Object key, char defaultValue); + char getChar(Object key, char defaultValue); /** * Gets the object array value for key or null if not found. @@ -250,7 +251,7 @@ public char getChar(Object key) * @param return type * @return object array value or null if not found */ - public K[] getArray(Object key); + K[] getArray(Object key); /** * Gets the object array value for key or defaultValue if not found. @@ -260,7 +261,7 @@ public char getChar(Object key) * @param return type * @return object array value or defaultValue if not found */ - public K[] getArray(Object key, K[] defaultValue); + K[] getArray(Object key, K[] defaultValue); /** * Gets the int array value for key. @@ -269,7 +270,7 @@ public char getChar(Object key) * @return int array value * @throws NotFoundException if not found */ - public int[] getIntArray(Object key) + int[] getIntArray(Object key) throws NotFoundException; /** @@ -279,7 +280,7 @@ public int[] getIntArray(Object key) * @param defaultValue default value * @return int array value or defaultValue if not found */ - public int[] getIntArray(Object key, int[] defaultValue); + int[] getIntArray(Object key, int[] defaultValue); /** * Gets the long array value for key. @@ -288,7 +289,7 @@ public int[] getIntArray(Object key) * @return long array value * @throws NotFoundException if not found */ - public long[] getLongArray(Object key) + long[] getLongArray(Object key) throws NotFoundException; /** @@ -298,7 +299,7 @@ public long[] getLongArray(Object key) * @param defaultValue default value * @return long array value or defaultValue if not found */ - public long[] getLongArray(Object key, long[] defaultValue); + long[] getLongArray(Object key, long[] defaultValue); /** * Gets the boolean array value for key. @@ -307,7 +308,7 @@ public long[] getLongArray(Object key) * @return boolean array value * @throws NotFoundException if not found */ - public boolean[] getBooleanArray(Object key) + boolean[] getBooleanArray(Object key) throws NotFoundException; /** @@ -317,7 +318,7 @@ public boolean[] getBooleanArray(Object key) * @param defaultValue default value * @return boolean array value or defaultValue if not found */ - public boolean[] getBooleanArray(Object key, boolean[] defaultValue); + boolean[] getBooleanArray(Object key, boolean[] defaultValue); /** * Gets the float array value for key. @@ -326,7 +327,7 @@ public boolean[] getBooleanArray(Object key) * @return float array value * @throws NotFoundException if not found */ - public float[] getFloatArray(Object key) + float[] getFloatArray(Object key) throws NotFoundException; /** @@ -336,7 +337,7 @@ public float[] getFloatArray(Object key) * @param defaultValue default value * @return float array value or defaultValue if not found */ - public float[] getFloatArray(Object key, float[] defaultValue); + float[] getFloatArray(Object key, float[] defaultValue); /** * Gets the double array value for key. @@ -345,7 +346,7 @@ public float[] getFloatArray(Object key) * @return double array value * @throws NotFoundException if not found */ - public double[] getDoubleArray(Object key) + double[] getDoubleArray(Object key) throws NotFoundException; /** @@ -355,7 +356,7 @@ public double[] getDoubleArray(Object key) * @param defaultValue default value * @return double array value or defaultValue if not found */ - public double[] getDoubleArray(Object key, double[] defaultValue); + double[] getDoubleArray(Object key, double[] defaultValue); /** * Gets the short array value for key. @@ -364,7 +365,7 @@ public double[] getDoubleArray(Object key) * @return short array value * @throws NotFoundException if not found */ - public short[] getShortArray(Object key) + short[] getShortArray(Object key) throws NotFoundException; /** @@ -374,7 +375,7 @@ public short[] getShortArray(Object key) * @param defaultValue default value * @return short array value or defaultValue if not found */ - public short[] getShortArray(Object key, short[] defaultValue); + short[] getShortArray(Object key, short[] defaultValue); /** * Gets the byte array value for key. @@ -383,7 +384,7 @@ public short[] getShortArray(Object key) * @return byte array value * @throws NotFoundException if not found */ - public byte[] getByteArray(Object key) + byte[] getByteArray(Object key) throws NotFoundException; /** @@ -393,7 +394,7 @@ public byte[] getByteArray(Object key) * @param defaultValue default value * @return byte array value or defaultValue if not found */ - public byte[] getByteArray(Object key, byte[] defaultValue); + byte[] getByteArray(Object key, byte[] defaultValue); /** * Gets the char array value for key. @@ -402,7 +403,7 @@ public byte[] getByteArray(Object key) * @return char array value * @throws NotFoundException if not found */ - public char[] getCharArray(Object key) + char[] getCharArray(Object key) throws NotFoundException; /** @@ -412,7 +413,7 @@ public char[] getCharArray(Object key) * @param defaultValue default value * @return char array value or defaultValue if not found */ - public char[] getCharArray(Object key, char[] defaultValue); + char[] getCharArray(Object key, char[] defaultValue); /** * Gets the string array value for key or null if not found. @@ -421,7 +422,7 @@ public char[] getCharArray(Object key) * @return string array value or null if not found * @throws NotFoundException if not found */ - public String[] getStringArray(Object key) + String[] getStringArray(Object key) throws NotFoundException; /** @@ -431,7 +432,7 @@ public String[] getStringArray(Object key) * @param defaultValue default value * @return string array value or defaultValue if not found */ - public String[] getStringArray(Object key, String[] defaultValue); + String[] getStringArray(Object key, String[] defaultValue); /** * Gets the store iterable. @@ -442,7 +443,7 @@ public String[] getStringArray(Object key) * @param value type * @return iterable over store */ - public Iterable> iterable(); + Iterable> iterable(); /** * Gets the store keys iterable. @@ -450,5 +451,5 @@ public String[] getStringArray(Object key) * @param key type * @return iterable over keys */ - public Iterable keys(); + Iterable keys(); } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java b/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java index c8d4b5d..2a013f2 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java @@ -24,13 +24,14 @@ *

* Note that duplicates aren't allowed. */ -public interface StoreWriter { +public interface StoreWriter extends AutoCloseable { /** * Close the store writer and append the data to the final destination. A * closed writer can't be reopened. */ - public void close(); + @Override + void close(); /** * Return the writer configuration. Configuration values should always be @@ -39,7 +40,7 @@ public interface StoreWriter { * * @return the store configuration */ - public Configuration getConfiguration(); + Configuration getConfiguration(); /** * Put key-value to the store. @@ -49,7 +50,7 @@ public interface StoreWriter { * @throws NullPointerException if key or value is * null */ - public void put(Object key, Object value); + void put(Object key, Object value); /** * Put multiple key-values to the store. @@ -57,7 +58,7 @@ public interface StoreWriter { * @param keys a collection of keys * @param values a collection of values */ - public void putAll(Object[] keys, Object[] values); + void putAll(Object[] keys, Object[] values); /** * Put serialized key-value entry to the store.

Use only this method if @@ -68,5 +69,5 @@ public interface StoreWriter { * @throws NullPointerException if key or value is * null */ - public void put(byte[] key, byte[] value); + void put(byte[] key, byte[] value); } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index 491773e..575a684 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -97,9 +97,9 @@ public class StorageReader implements Iterable> { //Open file and read metadata long createdAt = 0; FormatVersion formatVersion = null; - FileInputStream inputStream = new FileInputStream(path); - DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream)); - try { + + try (FileInputStream inputStream = new FileInputStream(path); + DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream))) { int ignoredBytes = -2; //Byte mark @@ -166,11 +166,8 @@ public class StorageReader implements Iterable> { //Read index and data offset indexOffset = dataInputStream.readInt() + ignoredBytes; dataOffset = dataInputStream.readLong() + ignoredBytes; - } finally { - //Close metadata - dataInputStream.close(); - inputStream.close(); } + //Close metadata //Create Mapped file in read-only mode mappedFile = new RandomAccessFile(path, "r"); diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java index d73e3b2..6a1824c 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java @@ -1211,9 +1211,14 @@ private static String[] deserializeStringArray(DataInput is) return ret; } + private static final byte[] EMPTY_BYTES = new byte[0]; + private static byte[] deserializeByteArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); + if (size == 0) { + return EMPTY_BYTES; + } byte[] b = new byte[size]; is.readFully(b); return b; diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index 645138e..7c9dea7 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -213,7 +213,7 @@ private void writeMetadata(DataOutputStream dataOutputStream) dataOutputStream.writeInt(maxKeyLength); // For each keyLength - long datasLength = 0l; + long datasLength = 0L; for (int i = 0; i < keyCounts.length; i++) { if (keyCounts[i] > 0) { // Write the key length diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index cdc4799..25af513 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -39,12 +39,11 @@ public void setUp() throws IOException { Configuration configuration = new Configuration(); configuration.registerSerializer(new PointSerializer()); - StoreWriter writer = PalDB.createWriter(STORE_FILE, configuration); - for (int i = 0; i < testValues.length; i++) { - writer.put(i, testValues[i]); + try (StoreWriter writer = PalDB.createWriter(STORE_FILE, configuration)) { + for (int i = 0; i < testValues.length; i++) { + writer.put(i, testValues[i]); + } } - writer.close(); - reader = PalDB.createReader(STORE_FILE, new Configuration()); } @@ -83,33 +82,6 @@ private static File createTempFile() { new Object[]{true, (byte) 1, 'a', 1.0, 1f, (short) 1, 1, 1l, "foo", new boolean[]{true}, new byte[]{1}, new char[]{'a'}, new double[]{1.0}, new float[]{1f}, new short[]{1}, new int[]{1}, new long[]{1l}, new String[]{"foo"}, new Object[]{"foo"}, new Point( 4, 56)}; -/* @BeforeMethod - public void setUp() { - STORE_FILE.delete(); - STORE_FOLDER.delete(); - STORE_FOLDER.mkdir(); - - Configuration configuration = new Configuration(); - configuration.registerSerializer(new PointSerializer()); - StoreWriter writer = PalDB.createWriter(STORE_FILE, configuration); - for (int i = 0; i < testValues.length; i++) { - writer.put(i, testValues[i]); - } - writer.close(); - - reader = PalDB.createReader(STORE_FILE, new Configuration()); - }*/ - -/* @AfterMethod - public void cleanUp() { - try { - reader.close(); - } catch (Exception e) { - } - STORE_FILE.delete(); - STORE_FOLDER.delete(); - }*/ - @Test public void testFile() { Assert.assertEquals(reader.getFile(), STORE_FILE); From fac4bc811e545d2f646e2bb3c1b1e8cf318306c8 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 21:55:41 +0200 Subject: [PATCH 07/20] Updated codecov badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 091c12a..ddf192c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ PalDB ========== [![Build Status](https://travis-ci.org/soundvibe/PalDB.svg?branch=java11)](https://travis-ci.org/soundvibe/PalDB) -[![Coverage Status](https://codecov.io/github/soundvibe/PalDB/coverage.svg?branch=java11)](https://codecov.io/github/soundvibe/PalDB?branch=java11) +[![codecov](https://codecov.io/gh/soundvibe/PalDB/branch/java11/graph/badge.svg)](https://codecov.io/gh/soundvibe/PalDB) PalDB is an embeddable write-once key-value store written in Java. From 2ef2b69e37b106c7b267c2e7dc03e99d53381f66 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 22:30:09 +0200 Subject: [PATCH 08/20] Updated readme. StoreReader now implements Iterable. --- README.md | 27 +++++++++---------- .../com/linkedin/paldb/api/StoreReader.java | 14 ++++++++-- .../com/linkedin/paldb/impl/ReaderImpl.java | 7 ++++- .../linkedin/paldb/impl/TestStoreReader.java | 26 +++++++++--------- 4 files changed, 45 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index ddf192c..a622952 100644 --- a/README.md +++ b/README.md @@ -42,29 +42,28 @@ API documentation can be found [here](http://linkedin.github.com/PalDB/doc/javad How to write a store ```java -StoreWriter writer = PalDB.createWriter(new File("store.paldb")); -writer.put("foo", "bar"); -writer.put(1213, new int[] {1, 2, 3}); -writer.close(); +try (var writer = PalDB.createWriter(new File("store.paldb"))) { + writer.put("foo", "bar"); + writer.put(1213, new int[] {1, 2, 3}); +} ``` How to read a store ```java -StoreReader reader = PalDB.createReader(new File("store.paldb")); -String val1 = reader.get("foo"); -int[] val2 = reader.get(1213); -reader.close(); +try (var reader = PalDB.createReader(new File("store.paldb"))) { + String val1 = reader.get("foo"); + int[] val2 = reader.get(1213); +} ``` How to iterate on a store ```java -StoreReader reader = PalDB.createReader(new File("store.paldb")); -Iterable> iterable = reader.iterable(); -for (Map.Entry entry : iterable) { - String key = entry.getKey(); - String value = entry.getValue(); +try (var reader = PalDB.createReader(new File("store.paldb"))) { + for (var entry : reader) { + String key = (String) entry.getKey(); + String value = (String) entry.getValue(); + } } -reader.close(); ``` For Scala examples, see [here](https://gist.github.com/mbastian/9b9b49a4b96333da33ec) and [here](https://gist.github.com/mbastian/440a706f5e863bb65622). diff --git a/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java b/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java index 5decf79..05b800d 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java @@ -15,7 +15,7 @@ package com.linkedin.paldb.api; import java.io.File; -import java.util.Map; +import java.util.*; /** @@ -25,7 +25,7 @@ * get() method to fetch. Call the * close() to liberate resources when done. */ -public interface StoreReader extends AutoCloseable { +public interface StoreReader extends AutoCloseable, Iterable> { /** * Closes the store reader and free resources. @@ -445,6 +445,16 @@ String[] getStringArray(Object key) */ Iterable> iterable(); + /** + * Gets the store iterator. + *

+ * Note that entry objects are reused. + * + * @return iterable over store + */ + @Override + Iterator> iterator(); + /** * Gets the store keys iterable. * diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java index bd776ed..f2638c6 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java @@ -21,7 +21,7 @@ import org.slf4j.*; import java.io.*; -import java.util.Map; +import java.util.*; /** @@ -406,6 +406,11 @@ public Iterable> iterable() { return new ReaderIterable<>(storage, serialization); } + @Override + public Iterator> iterator() { + return iterable().iterator(); + } + @Override public Iterable keys() { checkOpen(); diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index 25af513..a53d3fd 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -211,16 +211,14 @@ public void testGetLongMissing() } @Test - public void testGetString() - throws Throwable { + public void testGetString() { Assert.assertEquals(reader.getString(8), "foo"); Assert.assertEquals(reader.getString(8, "bar"), "foo"); Assert.assertEquals(reader.getString(-1, "bar"), "bar"); } @Test - public void testGetStringMissing() - throws Throwable { + public void testGetStringMissing() { Assert.assertNull(reader.getString(-1)); } @@ -351,28 +349,24 @@ public void testGetStringArrayMissing() } @Test - public void testGetMissing() - throws Throwable { + public void testGetMissing() { Assert.assertNull(reader.get(-1)); } @Test - public void testGetArray() - throws Throwable { + public void testGetArray() { Assert.assertEquals(reader.getArray(18), new Object[]{"foo"}); Assert.assertEquals(reader.getArray(18, new Object[]{"bar"}), new Object[]{"foo"}); Assert.assertEquals(reader.getArray(-1, new Object[]{"bar"}), new Object[]{"bar"}); } @Test - public void testGetArrayMissing() - throws Throwable { + public void testGetArrayMissing() { Assert.assertNull(reader.getArray(-1)); } @Test - public void testGetPoint() - throws Throwable { + public void testGetPoint() { Assert.assertEquals(reader.get(19), new Point(4, 56)); } @@ -391,6 +385,14 @@ public void testIterator() { } } + @Test + public void testIterate() { + for (var entry: reader) { + Object val = testValues[(int) entry.getKey()]; + Assert.assertEquals(entry.getValue(), val); + } + } + @Test public void testKeyIterator() { Iterable iter = reader.keys(); From dedd8365f1c0e60d381cf27d536f820381092845 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Wed, 13 Nov 2019 22:46:42 +0200 Subject: [PATCH 09/20] Implemented PalDBConfigBuilder. --- README.md | 14 ++- .../java/com/linkedin/paldb/api/PalDB.java | 22 ++-- .../paldb/api/PalDBConfigBuilder.java | 114 ++++++++++++++++++ .../paldb/api/PalDBConfigBuilderTest.java | 31 +++++ .../linkedin/paldb/impl/TestStoreReader.java | 2 +- 5 files changed, 165 insertions(+), 18 deletions(-) create mode 100644 paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java create mode 100644 paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java diff --git a/README.md b/README.md index a622952..97ab1cb 100644 --- a/README.md +++ b/README.md @@ -141,10 +141,18 @@ Read parameters: + `cache.initial.capacity`, cache initial capacity (int) [default: 1000] + `cache.load.factor`, cache load factor (double) [default: 0.75] -Configuration values are passed at init time. Example: +Configuration values are passed at init time. Example using fluent builder: ```java -Configuration config = PalDB.newConfiguration(); -config.set(Configuration.CACHE_ENABLED, "true"); +var config = PalDBConfigBuilder.create() + .withMemoryMapSegmentSize(512 * 1024 * 1024) + .withMemoryMapDataEnabled(false) + .withIndexLoadFactor(0.75) + .withLRUCacheEnabled(true) + .withCacheSizeLimit(10_000) + .withCacheInitialCapacity(1000) + .withCacheLoadFactor(0.5) + .withEnableCompression(true) + .build(); StoreReader reader = PalDB.createReader(new File("store.paldb"), config); ``` diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java index 35feb30..e23a1c3 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java @@ -26,13 +26,7 @@ * This class is the entry point to obtain {@link com.linkedin.paldb.api.StoreReader} and * {@link com.linkedin.paldb.api.StoreWriter} interfaces. */ -public final class PalDB { - - /** - * This class is only static - */ - private PalDB() { - } +public interface PalDB { /** * Creates a store reader from the specified file with a default configuration. @@ -42,7 +36,7 @@ private PalDB() { * @param file a PalDB store file * @return a store reader */ - public static StoreReader createReader(File file) { + static StoreReader createReader(File file) { return StoreImpl.createReader(file, newConfiguration()); } @@ -55,7 +49,7 @@ public static StoreReader createReader(File file) { * @param config configuration * @return a store reader */ - public static StoreReader createReader(File file, Configuration config) { + static StoreReader createReader(File file, Configuration config) { return StoreImpl.createReader(file, config); } @@ -69,7 +63,7 @@ public static StoreReader createReader(File file, Configuration config) { * @param config configuration * @return a store reader */ - public static StoreReader createReader(InputStream stream, Configuration config) { + static StoreReader createReader(InputStream stream, Configuration config) { return StoreImpl.createReader(stream, config); } @@ -81,7 +75,7 @@ public static StoreReader createReader(InputStream stream, Configuration config) * @param file location of the output file * @return a store writer */ - public static StoreWriter createWriter(File file) { + static StoreWriter createWriter(File file) { return StoreImpl.createWriter(file, newConfiguration()); } @@ -94,7 +88,7 @@ public static StoreWriter createWriter(File file) { * @param config configuration * @return a store writer */ - public static StoreWriter createWriter(File file, Configuration config) { + static StoreWriter createWriter(File file, Configuration config) { return StoreImpl.createWriter(file, config); } @@ -108,7 +102,7 @@ public static StoreWriter createWriter(File file, Configuration config) { * @param config configuration * @return a store writer */ - public static StoreWriter createWriter(OutputStream stream, Configuration config) { + static StoreWriter createWriter(OutputStream stream, Configuration config) { return StoreImpl.createWriter(stream, config); } @@ -117,7 +111,7 @@ public static StoreWriter createWriter(OutputStream stream, Configuration config * * @return new configuration */ - public static Configuration newConfiguration() { + static Configuration newConfiguration() { return new Configuration(); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java b/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java new file mode 100644 index 0000000..9b986de --- /dev/null +++ b/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java @@ -0,0 +1,114 @@ +package com.linkedin.paldb.api; + +public final class PalDBConfigBuilder { + + private final Configuration palDbConfiguration; + + private PalDBConfigBuilder() { + this.palDbConfiguration = new Configuration(); + } + + public static PalDBConfigBuilder create() { + return new PalDBConfigBuilder(); + } + + /** + * PalDB configuration property. + *

+ * mmap.segment.size - memory map segment size [default: 1GB] + * @param bytes size in bytes + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withMemoryMapSegmentSize(long bytes) { + palDbConfiguration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(bytes)); + return this; + } + + /** + * PalDB configuration property. + *

+ * mmap.data.enabled - enable memory mapping for data [default: true] + * @param enabled flag + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withMemoryMapDataEnabled(boolean enabled) { + palDbConfiguration.set(Configuration.MMAP_DATA_ENABLED, String.valueOf(enabled)); + return this; + } + + /** + * PalDB configuration property. + *

+ * load.factor - index load factor [default: 0.75] + * @param loadFactor load factor value + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withIndexLoadFactor(double loadFactor) { + palDbConfiguration.set(Configuration.LOAD_FACTOR, String.valueOf(loadFactor)); + return this; + } + + /** + * PalDB configuration property. + *

+ * cache.enabled - LRU cache enabled [default: false] + * @param enabled flag + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withLRUCacheEnabled(boolean enabled) { + palDbConfiguration.set(Configuration.CACHE_ENABLED, String.valueOf(enabled)); + return this; + } + + /** + * PalDB configuration property. + *

+ * cache.bytes - cache limit [default: Xmx - 100MB] + * @param bytes size in bytes + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withCacheSizeLimit(long bytes) { + palDbConfiguration.set(Configuration.CACHE_BYTES, String.valueOf(bytes)); + return this; + } + + /** + * PalDB configuration property. + *

+ * cache.initial.capacity - cache initial capacity [default: 1000] + * @param size number of initial capacity + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withCacheInitialCapacity(int size) { + palDbConfiguration.set(Configuration.CACHE_INITIAL_CAPACITY, String.valueOf(size)); + return this; + } + + /** + * PalDB configuration property. + *

+ * cache.load.factor - cache load factor [default: 0.75] + * @param loadFactor load factor value + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withCacheLoadFactor(double loadFactor) { + palDbConfiguration.set(Configuration.CACHE_LOAD_FACTOR, String.valueOf(loadFactor)); + return this; + } + + /** + * PalDB configuration property. + *

+ * compression.enabled - enable compression [default: false] + * @param enabled flag + * @return this {@code CachemeerConfigBuilder} instance (for chaining) + */ + public PalDBConfigBuilder withEnableCompression(boolean enabled) { + palDbConfiguration.set(Configuration.COMPRESSION_ENABLED, String.valueOf(enabled)); + return this; + } + + public Configuration build() { + return new Configuration(palDbConfiguration); + } +} diff --git a/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java b/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java new file mode 100644 index 0000000..e253150 --- /dev/null +++ b/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java @@ -0,0 +1,31 @@ +package com.linkedin.paldb.api; + +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class PalDBConfigBuilderTest { + + @Test + public void testAllPropertiesSet() { + var config = PalDBConfigBuilder.create() + .withMemoryMapSegmentSize(500) + .withMemoryMapDataEnabled(false) + .withIndexLoadFactor(0.5) + .withLRUCacheEnabled(true) + .withCacheSizeLimit(200) + .withCacheInitialCapacity(100) + .withCacheLoadFactor(0.1) + .withEnableCompression(true) + .build(); + + assertEquals(500, config.getInt(Configuration.MMAP_SEGMENT_SIZE)); + assertFalse(config.getBoolean(Configuration.MMAP_DATA_ENABLED)); + assertEquals(0.5, config.getDouble(Configuration.LOAD_FACTOR)); + assertTrue(config.getBoolean(Configuration.CACHE_ENABLED)); + assertEquals(200, config.getInt(Configuration.CACHE_BYTES)); + assertEquals(100, config.getInt(Configuration.CACHE_INITIAL_CAPACITY)); + assertEquals(0.1, config.getDouble(Configuration.CACHE_LOAD_FACTOR)); + assertTrue(config.getBoolean(Configuration.COMPRESSION_ENABLED)); + } +} \ No newline at end of file diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index a53d3fd..b2a03b2 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -44,7 +44,7 @@ public void setUp() throws IOException { writer.put(i, testValues[i]); } } - reader = PalDB.createReader(STORE_FILE, new Configuration()); + reader = PalDB.createReader(STORE_FILE, PalDBConfigBuilder.create().build()); } @AfterMethod From 2f4e08816ed6a694c6490f93a3464691a7c6d77f Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Thu, 14 Nov 2019 01:53:30 +0200 Subject: [PATCH 10/20] StoreReader is now generic (StoreReader). --- CHANGELOG.md | 8 + README.md | 16 +- build.gradle | 3 - gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 5 +- .../java/com/linkedin/paldb/api/PalDB.java | 6 +- .../com/linkedin/paldb/api/StoreReader.java | 376 +------------ .../com/linkedin/paldb/impl/ReaderImpl.java | 294 +---------- .../com/linkedin/paldb/impl/StoreImpl.java | 8 +- .../com/linkedin/paldb/impl/TestStore.java | 433 +++++++-------- .../linkedin/paldb/impl/TestStoreReader.java | 494 ++++++++++-------- .../linkedin/paldb/utils/TestTempUtils.java | 19 +- 12 files changed, 520 insertions(+), 1144 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5df71df..65e4a59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ Change Log ========== +Version 2.0.0 +-------------------------- +- Bumped minimal java version to 11 +- StoreReader and StoreWriter implement AutoCloseable +- New config builder +- Reader and writer use generics + + Version 1.2.0 *(June 26th 2016)* -------------------------- diff --git a/README.md b/README.md index 97ab1cb..6a51f0f 100644 --- a/README.md +++ b/README.md @@ -43,25 +43,23 @@ API documentation can be found [here](http://linkedin.github.com/PalDB/doc/javad How to write a store ```java try (var writer = PalDB.createWriter(new File("store.paldb"))) { - writer.put("foo", "bar"); - writer.put(1213, new int[] {1, 2, 3}); + writer.put(1213, "foo"); } ``` How to read a store ```java -try (var reader = PalDB.createReader(new File("store.paldb"))) { - String val1 = reader.get("foo"); - int[] val2 = reader.get(1213); +try (var reader = PalDB.createReader(new File("store.paldb"))) { + String val = reader.get(1213); } ``` How to iterate on a store ```java -try (var reader = PalDB.createReader(new File("store.paldb"))) { +try (var reader = PalDB.createReader(new File("store.paldb"))) { for (var entry : reader) { - String key = (String) entry.getKey(); - String value = (String) entry.getValue(); + Integer key = entry.getKey(); + String value = entry.getValue(); } } ``` @@ -153,7 +151,7 @@ var config = PalDBConfigBuilder.create() .withCacheLoadFactor(0.5) .withEnableCompression(true) .build(); -StoreReader reader = PalDB.createReader(new File("store.paldb"), config); +StoreReader reader = PalDB.createReader(new File("store.paldb"), config); ``` A few tips on how configuration can affect performance: diff --git a/build.gradle b/build.gradle index 5c5e282..5c59efa 100644 --- a/build.gradle +++ b/build.gradle @@ -2,9 +2,6 @@ buildscript { repositories { mavenCentral() } - dependencies { - classpath "org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.8.3" - } } allprojects { diff --git a/gradle.properties b/gradle.properties index a6f9c9b..9c15eaa 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,6 @@ org.gradle.configureondemand=true ide.recursive=true org.gradle.parallel=false -VERSION_NAME=1.2.1 +VERSION_NAME=2.0.0 GROUP=net.soundvibe.paldb ARCHIVE_NAME=paldb diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 6ce793f..f6bebb9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ +#Wed Nov 13 23:43:01 EET 2019 +distributionUrl=https\://services.gradle.org/distributions/gradle-6.0-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.0-bin.zip -zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java index e23a1c3..2504a46 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java @@ -36,7 +36,7 @@ public interface PalDB { * @param file a PalDB store file * @return a store reader */ - static StoreReader createReader(File file) { + static StoreReader createReader(File file) { return StoreImpl.createReader(file, newConfiguration()); } @@ -49,7 +49,7 @@ static StoreReader createReader(File file) { * @param config configuration * @return a store reader */ - static StoreReader createReader(File file, Configuration config) { + static StoreReader createReader(File file, Configuration config) { return StoreImpl.createReader(file, config); } @@ -63,7 +63,7 @@ static StoreReader createReader(File file, Configuration config) { * @param config configuration * @return a store reader */ - static StoreReader createReader(InputStream stream, Configuration config) { + static StoreReader createReader(InputStream stream, Configuration config) { return StoreImpl.createReader(stream, config); } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java b/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java index 05b800d..0e049a4 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java @@ -25,7 +25,7 @@ * get() method to fetch. Call the * close() to liberate resources when done. */ -public interface StoreReader extends AutoCloseable, Iterable> { +public interface StoreReader extends AutoCloseable, Iterable> { /** * Closes the store reader and free resources. @@ -60,390 +60,27 @@ public interface StoreReader extends AutoCloseable, Iterablekey or null if not found. * * @param key key to fetch - * @param return type * @return value or null if not found */ - K get(Object key); + V get(K key); /** * Gets the value for key or defaultValue if not found. * * @param key key to fetch * @param defaultValue default value - * @param return type * @return value of defaultValue if not found */ - K get(Object key, K defaultValue); - - /** - * Gets the int value for key. - * - * @param key key to fetch - * @return int value - * @throws NotFoundException if not found - */ - int getInt(Object key) - throws NotFoundException; - - /** - * Gets the int value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return int value or defaultValue if not found - */ - int getInt(Object key, int defaultValue); - - /** - * Gets the long value for key. - * - * @param key key to fetch - * @return long value - * @throws NotFoundException if not found - */ - long getLong(Object key) - throws NotFoundException; - - /** - * Gets the long value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return long value or defaultValue if not found - */ - long getLong(Object key, long defaultValue); - - /** - * Gets the boolean value for key. - * - * @param key key to fetch - * @return boolean value - * @throws NotFoundException if not found - */ - boolean getBoolean(Object key) - throws NotFoundException; - - /** - * Gets the boolean value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return boolean value or defaultValue if not found - */ - boolean getBoolean(Object key, boolean defaultValue); - - /** - * Gets the float value for key. - * - * @param key key to fetch - * @return float value - * @throws NotFoundException if not found - */ - float getFloat(Object key) - throws NotFoundException; - - /** - * Gets the float value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return float value or defaultValue if not found - */ - float getFloat(Object key, float defaultValue); - - /** - * Gets the double value for key. - * - * @param key key to fetch - * @return double value - * @throws NotFoundException if not found - */ - double getDouble(Object key) - throws NotFoundException; - - /** - * Gets the double value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return double value or defaultValue if not found - */ - double getDouble(Object key, double defaultValue); - - /** - * Gets the short value for key. - * - * @param key key to fetch - * @return short value - * @throws NotFoundException if not found - */ - short getShort(Object key) - throws NotFoundException; - - /** - * Gets the short value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return short value or defaultValue if not found - */ - short getShort(Object key, short defaultValue); - - /** - * Gets the byte value for key. - * - * @param key key to fetch - * @return byte value - * @throws NotFoundException if not found - */ - byte getByte(Object key) - throws NotFoundException; - - /** - * Gets the byte value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return byte value or defaultValue if not found - */ - byte getByte(Object key, byte defaultValue); - - /** - * Gets the string value for key or null if not found. - * - * @param key key to fetch - * @return string value - */ - String getString(Object key); - - /** - * Gets the string value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return string value or defaultValue if not found - */ - String getString(Object key, String defaultValue); - - /** - * Gets the char value for key. - * - * @param key key to fetch - * @return char value - * @throws NotFoundException if not found - */ - char getChar(Object key) - throws NotFoundException; - - /** - * Gets the char value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return char value or defaultValue if not found - */ - char getChar(Object key, char defaultValue); - - /** - * Gets the object array value for key or null if not found. - * - * @param key key to fetch - * @param return type - * @return object array value or null if not found - */ - K[] getArray(Object key); - - /** - * Gets the object array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @param return type - * @return object array value or defaultValue if not found - */ - K[] getArray(Object key, K[] defaultValue); - - /** - * Gets the int array value for key. - * - * @param key key to fetch - * @return int array value - * @throws NotFoundException if not found - */ - int[] getIntArray(Object key) - throws NotFoundException; - - /** - * Gets the int array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return int array value or defaultValue if not found - */ - int[] getIntArray(Object key, int[] defaultValue); - - /** - * Gets the long array value for key. - * - * @param key key to fetch - * @return long array value - * @throws NotFoundException if not found - */ - long[] getLongArray(Object key) - throws NotFoundException; - - /** - * Gets the long array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return long array value or defaultValue if not found - */ - long[] getLongArray(Object key, long[] defaultValue); - - /** - * Gets the boolean array value for key. - * - * @param key key to fetch - * @return boolean array value - * @throws NotFoundException if not found - */ - boolean[] getBooleanArray(Object key) - throws NotFoundException; - - /** - * Gets the boolean array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return boolean array value or defaultValue if not found - */ - boolean[] getBooleanArray(Object key, boolean[] defaultValue); - - /** - * Gets the float array value for key. - * - * @param key key to fetch - * @return float array value - * @throws NotFoundException if not found - */ - float[] getFloatArray(Object key) - throws NotFoundException; - - /** - * Gets the float array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return float array value or defaultValue if not found - */ - float[] getFloatArray(Object key, float[] defaultValue); - - /** - * Gets the double array value for key. - * - * @param key key to fetch - * @return double array value - * @throws NotFoundException if not found - */ - double[] getDoubleArray(Object key) - throws NotFoundException; - - /** - * Gets the double array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return double array value or defaultValue if not found - */ - double[] getDoubleArray(Object key, double[] defaultValue); - - /** - * Gets the short array value for key. - * - * @param key key to fetch - * @return short array value - * @throws NotFoundException if not found - */ - short[] getShortArray(Object key) - throws NotFoundException; - - /** - * Gets the short array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return short array value or defaultValue if not found - */ - short[] getShortArray(Object key, short[] defaultValue); - - /** - * Gets the byte array value for key. - * - * @param key key to fetch - * @return byte array value - * @throws NotFoundException if not found - */ - byte[] getByteArray(Object key) - throws NotFoundException; - - /** - * Gets the byte array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return byte array value or defaultValue if not found - */ - byte[] getByteArray(Object key, byte[] defaultValue); - - /** - * Gets the char array value for key. - * - * @param key key to fetch - * @return char array value - * @throws NotFoundException if not found - */ - char[] getCharArray(Object key) - throws NotFoundException; - - /** - * Gets the char array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return char array value or defaultValue if not found - */ - char[] getCharArray(Object key, char[] defaultValue); - - /** - * Gets the string array value for key or null if not found. - * - * @param key key to fetch - * @return string array value or null if not found - * @throws NotFoundException if not found - */ - String[] getStringArray(Object key) - throws NotFoundException; - - /** - * Gets the string array value for key or defaultValue if not found. - * - * @param key key to fetch - * @param defaultValue default value - * @return string array value or defaultValue if not found - */ - String[] getStringArray(Object key, String[] defaultValue); + V get(K key, V defaultValue); /** * Gets the store iterable. *

* Note that entry objects are reused. * - * @param key type - * @param value type * @return iterable over store */ - Iterable> iterable(); + Iterable> iterable(); /** * Gets the store iterator. @@ -453,13 +90,12 @@ String[] getStringArray(Object key) * @return iterable over store */ @Override - Iterator> iterator(); + Iterator> iterator(); /** * Gets the store keys iterable. * - * @param key type * @return iterable over keys */ - Iterable keys(); + Iterable keys(); } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java index f2638c6..2142e48 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java @@ -14,9 +14,7 @@ package com.linkedin.paldb.impl; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.NotFoundException; -import com.linkedin.paldb.api.StoreReader; +import com.linkedin.paldb.api.*; import com.linkedin.paldb.utils.DataInputOutput; import org.slf4j.*; @@ -27,7 +25,7 @@ /** * Store reader implementation. */ -public final class ReaderImpl implements StoreReader { +public final class ReaderImpl implements StoreReader { // Logger private static final Logger log = LoggerFactory.getLogger(ReaderImpl.class); @@ -99,17 +97,17 @@ public File getFile() { } @Override - public K get(Object key) { - return (K) get(key, null); + public V get(K key) { + return get(key, null); } @Override - public K get(Object key, K defaultValue) { + public V get(K key, V defaultValue) { checkOpen(); if (key == null) { throw new NullPointerException("The key can't be null"); } - K value = cache.get(key); + V value = cache.get(key); if (value == null) { try { byte[] valueBytes = storage.get(serialization.serializeKey(key)); @@ -117,11 +115,11 @@ public K get(Object key, K defaultValue) { Object v = serialization.deserialize(dataInputOutput.reset(valueBytes)); cache.put(key, v); - return (K) v; + return (V) v; } else { return defaultValue; } - } catch (Exception ex) { + } catch (IOException | ClassNotFoundException ex) { throw new RuntimeException(ex); } } else if (value == StorageCache.NULL_VALUE) { @@ -131,288 +129,18 @@ public K get(Object key, K defaultValue) { } @Override - public int getInt(Object key, int defaultValue) { - return get(key, defaultValue); - } - - @Override - public int getInt(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Integer) val).intValue(); - } - - @Override - public long getLong(Object key, long defaultValue) { - return get(key, defaultValue); - } - - @Override - public long getLong(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Long) val).longValue(); - } - - @Override - public boolean getBoolean(Object key, boolean defaultValue) { - return get(key, defaultValue); - } - - @Override - public boolean getBoolean(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Boolean) val).booleanValue(); - } - - @Override - public float getFloat(Object key, float defaultValue) { - return get(key, defaultValue); - } - - @Override - public float getFloat(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Float) val).floatValue(); - } - - @Override - public double getDouble(Object key, double defaultValue) { - return get(key, defaultValue); - } - - @Override - public double getDouble(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Double) val).intValue(); - } - - @Override - public short getShort(Object key, short defaultValue) { - return get(key, defaultValue); - } - - @Override - public short getShort(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Short) val).shortValue(); - } - - @Override - public byte getByte(Object key, byte defaultValue) { - return get(key, defaultValue); - } - - @Override - public byte getByte(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Byte) val).byteValue(); - } - - @Override - public String getString(Object key, String defaultValue) { - return get(key, defaultValue); - } - - @Override - public String getString(Object key) { - return (String) get(key, null); - } - - @Override - public char getChar(Object key, char defaultValue) { - return get(key, defaultValue); - } - - @Override - public char getChar(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return ((Character) val).charValue(); - } - - @Override - public K[] getArray(Object key) { - return (K[]) get(key, null); - } - - @Override - public K[] getArray(Object key, K[] defaultValue) { - return (K[]) get(key, defaultValue); - } - - @Override - public int[] getIntArray(Object key, int[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public int[] getIntArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (int[]) val; - } - - @Override - public long[] getLongArray(Object key, long[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public long[] getLongArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (long[]) val; - } - - @Override - public boolean[] getBooleanArray(Object key, boolean[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public boolean[] getBooleanArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (boolean[]) val; - } - - @Override - public float[] getFloatArray(Object key, float[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public float[] getFloatArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (float[]) val; - } - - @Override - public double[] getDoubleArray(Object key, double[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public double[] getDoubleArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (double[]) val; - } - - @Override - public short[] getShortArray(Object key, short[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public short[] getShortArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (short[]) val; - } - - @Override - public byte[] getByteArray(Object key, byte[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public byte[] getByteArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (byte[]) val; - } - - @Override - public String[] getStringArray(Object key, String[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public String[] getStringArray(Object key) { - return (String[]) get(key, null); - } - - @Override - public char[] getCharArray(Object key, char[] defaultValue) { - return get(key, defaultValue); - } - - @Override - public char[] getCharArray(Object key) - throws NotFoundException { - Object val = get(key); - if (val == null) { - throw new NotFoundException(key); - } - return (char[]) val; - } - - @Override - public Iterable> iterable() { + public Iterable> iterable() { checkOpen(); return new ReaderIterable<>(storage, serialization); } @Override - public Iterator> iterator() { + public Iterator> iterator() { return iterable().iterator(); } @Override - public Iterable keys() { + public Iterable keys() { checkOpen(); return new ReaderKeyIterable<>(storage, serialization); } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java index a82a717..944dc72 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java @@ -31,15 +31,15 @@ public final class StoreImpl { private StoreImpl() { } - public static StoreReader createReader(File file, Configuration config) { + public static StoreReader createReader(File file, Configuration config) { if (file == null || config == null) { throw new NullPointerException(); } log.info("Initialize reader from file {}", file.getName()); - return new ReaderImpl(config, file); + return new ReaderImpl<>(config, file); } - public static StoreReader createReader(InputStream stream, Configuration config) { + public static StoreReader createReader(InputStream stream, Configuration config) { if (stream == null || config == null) { throw new NullPointerException(); } @@ -47,7 +47,7 @@ public static StoreReader createReader(InputStream stream, Configuration config) try { File file = TempUtils.copyIntoTempFile("paldbtempreader", stream); log.info("Copied stream into temp file {}", file.getName()); - return new ReaderImpl(config, file); + return new ReaderImpl<>(config, file); } catch (IOException ex) { throw new UncheckedIOException(ex); } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java index 86eeae6..341823f 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -23,16 +23,17 @@ import java.nio.file.*; import java.util.*; +import static com.linkedin.paldb.utils.TestTempUtils.deleteDirectory; + public class TestStore { private Path tempDir; - private File STORE_FILE = createTempFile(); + private File storeFile; @BeforeMethod public void setUp() throws IOException { tempDir = Files.createTempDirectory("tmp"); - STORE_FILE = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); - STORE_FILE.mkdir(); + storeFile = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); } @AfterMethod @@ -40,39 +41,17 @@ public void cleanUp() { deleteDirectory(tempDir.toFile()); } - private boolean deleteDirectory(File directoryToBeDeleted) { - if (directoryToBeDeleted.isDirectory()) { - File[] allContents = directoryToBeDeleted.listFiles(); - if (allContents != null) { - for (File file : allContents) { - deleteDirectory(file); - } - } - } - return directoryToBeDeleted.delete(); - } - - private static File createTempFile() { - try { - return Files.createTempFile( "paldb", ".dat").toFile(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - @Test public void testEmpty() { - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); + StoreWriter writer = PalDB.createWriter(storeFile, new Configuration()); writer.close(); - Assert.assertTrue(STORE_FILE.exists()); - - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); + Assert.assertTrue(storeFile.exists()); - Assert.assertEquals(reader.size(), 0); - Assert.assertNull(reader.get(1, null)); - - reader.close(); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), 0); + Assert.assertNull(reader.get(1, null)); + } } @Test @@ -90,17 +69,15 @@ public void testEmptyStream() { @Test public void testEmptyDefaultConfig() { - StoreWriter writer = PalDB.createWriter(STORE_FILE); + StoreWriter writer = PalDB.createWriter(storeFile); writer.close(); - Assert.assertTrue(STORE_FILE.exists()); - - StoreReader reader = PalDB.createReader(STORE_FILE); - - Assert.assertEquals(reader.size(), 0); - Assert.assertNull(reader.get(1, null)); + Assert.assertTrue(storeFile.exists()); - reader.close(); + try (StoreReader reader = PalDB.createReader(storeFile)) { + Assert.assertEquals(reader.size(), 0); + Assert.assertNull(reader.get(1, null)); + } } @Test @@ -177,82 +154,77 @@ public void write(int i) @Test(expectedExceptions = IllegalArgumentException.class) public void testInvalidSegmentSize() { - StoreWriter writer = PalDB.createWriter(STORE_FILE); + StoreWriter writer = PalDB.createWriter(storeFile); writer.close(); Configuration config = new Configuration(); config.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(1 + (long) Integer.MAX_VALUE)); - PalDB.createReader(STORE_FILE, config); + PalDB.createReader(storeFile, config); } @Test - public void testByteMarkEmpty() - throws IOException { - FileOutputStream fos = new FileOutputStream(STORE_FILE); - fos.write(12345); - fos.write(FormatVersion.getPrefixBytes()[0]); - fos.write(3456); - StoreWriter writer = PalDB.createWriter(fos, new Configuration()); - writer.close(); - - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - - Assert.assertEquals(reader.size(), 0); - Assert.assertNull(reader.get(1, null)); + public void testByteMarkEmpty() throws IOException { + try (FileOutputStream fos = new FileOutputStream(storeFile)) { + fos.write(12345); + fos.write(FormatVersion.getPrefixBytes()[0]); + fos.write(3456); + StoreWriter writer = PalDB.createWriter(fos, new Configuration()); + writer.close(); + } - reader.close(); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), 0); + Assert.assertNull(reader.get(1, null)); + } } @Test public void testOneKey() { - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); - writer.put(1, "foo"); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + writer.put(1, "foo"); + } - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.size(), 1); - Assert.assertEquals(reader.get(1), "foo"); - reader.close(); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), 1); + Assert.assertEquals(reader.get(1), "foo"); + } } @Test - public void testPutSerializedKey() - throws IOException { + public void testPutSerializedKey() throws IOException { StorageSerialization storageSerialization = new StorageSerialization(new Configuration()); byte[] serializedKey = storageSerialization.serializeKey(1); byte[] serializedValue = storageSerialization.serializeValue("foo"); - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); - writer.put(serializedKey, serializedValue); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + writer.put(serializedKey, serializedValue); + } - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.size(), 1); - Assert.assertEquals(reader.get(1), "foo"); - reader.close(); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), 1); + Assert.assertEquals(reader.get(1), "foo"); + } } @Test - public void testByteMarkOneKey() - throws IOException { - FileOutputStream fos = new FileOutputStream(STORE_FILE); - fos.write(12345); - fos.write(FormatVersion.getPrefixBytes()[0]); - fos.write(3456); - StoreWriter writer = PalDB.createWriter(fos, new Configuration()); - writer.put(1, "foo"); - writer.close(); - - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); + public void testByteMarkOneKey() throws IOException { + try (FileOutputStream fos = new FileOutputStream(storeFile); + StoreWriter writer = PalDB.createWriter(fos, new Configuration());) { + fos.write(12345); + fos.write(FormatVersion.getPrefixBytes()[0]); + fos.write(3456); + + writer.put(1, "foo"); + } - Assert.assertEquals(reader.size(), 1); - Assert.assertEquals(reader.get(1), "foo"); - reader.close(); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), 1); + Assert.assertEquals(reader.get(1), "foo"); + } } @Test - public void testTwoFirstKeyLength() - throws NotFoundException { + public void testTwoFirstKeyLength() { Integer key1 = 1; Integer key2 = 245; @@ -261,22 +233,22 @@ public void testTwoFirstKeyLength() testKeyLength(key2, 2); //Write - writeStore(STORE_FILE, new Object[]{key1, key2}, new Object[]{1, 6}); + writeStore(storeFile, new Object[]{key1, key2}, new Object[]{1, 6}); //Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.getInt(key1), 1); - Assert.assertEquals(reader.getInt(key2), 6); - Assert.assertNull(reader.get(0, null)); - Assert.assertNull(reader.get(6, null)); - Assert.assertNull(reader.get(244, null)); - Assert.assertNull(reader.get(246, null)); - Assert.assertNull(reader.get(1245, null)); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.get(key1).intValue(), 1); + Assert.assertEquals(reader.get(key2).intValue(), 6); + Assert.assertNull(reader.get(0, null)); + Assert.assertNull(reader.get(6, null)); + Assert.assertNull(reader.get(244, null)); + Assert.assertNull(reader.get(246, null)); + Assert.assertNull(reader.get(1245, null)); + } } @Test - public void testKeyLengthGap() - throws NotFoundException { + public void testKeyLengthGap() { Integer key1 = 1; Integer key2 = 2450; @@ -285,24 +257,24 @@ public void testKeyLengthGap() testKeyLength(key2, 3); //Write - writeStore(STORE_FILE, new Object[]{key1, key2}, new Object[]{1, 6}); + writeStore(storeFile, new Object[]{key1, key2}, new Object[]{1, 6}); //Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.getInt(key1), 1); - Assert.assertEquals(reader.getInt(key2), 6); - Assert.assertNull(reader.get(0, null)); - Assert.assertNull(reader.get(6, null)); - Assert.assertNull(reader.get(244, null)); - Assert.assertNull(reader.get(267, null)); - Assert.assertNull(reader.get(2449, null)); - Assert.assertNull(reader.get(2451, null)); - Assert.assertNull(reader.get(2454441, null)); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.get(key1).intValue(), 1); + Assert.assertEquals(reader.get(key2).intValue(), 6); + Assert.assertNull(reader.get(0, null)); + Assert.assertNull(reader.get(6, null)); + Assert.assertNull(reader.get(244, null)); + Assert.assertNull(reader.get(267, null)); + Assert.assertNull(reader.get(2449, null)); + Assert.assertNull(reader.get(2451, null)); + Assert.assertNull(reader.get(2454441, null)); + } } @Test - public void testKeyLengthStartTwo() - throws NotFoundException { + public void testKeyLengthStartTwo() { Integer key1 = 245; Integer key2 = 2450; @@ -311,31 +283,31 @@ public void testKeyLengthStartTwo() testKeyLength(key2, 3); //Write - writeStore(STORE_FILE, new Object[]{key1, key2}, new Object[]{1, 6}); + writeStore(storeFile, new Object[]{key1, key2}, new Object[]{1, 6}); //Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.getInt(key1), 1); - Assert.assertEquals(reader.getInt(key2), 6); - Assert.assertNull(reader.get(6, null)); - Assert.assertNull(reader.get(244, null)); - Assert.assertNull(reader.get(267, null)); - Assert.assertNull(reader.get(2449, null)); - Assert.assertNull(reader.get(2451, null)); - Assert.assertNull(reader.get(2454441, null)); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.get(key1).intValue(), 1); + Assert.assertEquals(reader.get(key2).intValue(), 6); + Assert.assertNull(reader.get(6, null)); + Assert.assertNull(reader.get(244, null)); + Assert.assertNull(reader.get(267, null)); + Assert.assertNull(reader.get(2449, null)); + Assert.assertNull(reader.get(2451, null)); + Assert.assertNull(reader.get(2454441, null)); + } } @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*duplicate.*") public void testDuplicateKeys() { - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); - writer.put(0, "ABC"); - writer.put(0, "DGE"); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + writer.put(0, "ABC"); + writer.put(0, "DGE"); + } } @Test - public void testDataOnTwoBuffers() - throws IOException { + public void testDataOnTwoBuffers() throws IOException { Object[] keys = new Object[]{1, 2, 3}; Object[] values = new Object[]{GenerateTestData.generateStringData(100), GenerateTestData .generateStringData(10000), GenerateTestData.generateStringData(100)}; @@ -344,21 +316,20 @@ public void testDataOnTwoBuffers() int byteSize = serialization.serialize(values[0]).length + serialization.serialize(values[1]).length; //Write - System.out.println(STORE_FILE); - writeStore(STORE_FILE, keys, values); + writeStore(storeFile, keys, values); //Read Configuration configuration = new Configuration(); configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize - 100)); - StoreReader reader = PalDB.createReader(STORE_FILE, configuration); - for (int i = 0; i < keys.length; i++) { - Assert.assertEquals(reader.get(keys[i], null), values[i]); + try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { + for (int i = 0; i < keys.length; i++) { + Assert.assertEquals(reader.get((Integer) keys[i], null), values[i]); + } } } @Test - public void testDataSizeOnTwoBuffers() - throws IOException { + public void testDataSizeOnTwoBuffers() throws IOException { Object[] keys = new Object[]{1, 2, 3}; Object[] values = new Object[]{GenerateTestData.generateStringData(100), GenerateTestData .generateStringData(10000), GenerateTestData.generateStringData(100)}; @@ -371,15 +342,15 @@ public void testDataSizeOnTwoBuffers() LongPacker.packInt(new DataInputOutput(), b1.length) + LongPacker.packInt(new DataInputOutput(), b2.length); //Write - System.out.println(STORE_FILE); - writeStore(STORE_FILE, keys, values); + writeStore(storeFile, keys, values); //Read Configuration configuration = new Configuration(); configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize + sizeSize + 3)); - StoreReader reader = PalDB.createReader(STORE_FILE, configuration); - for (int i = 0; i < keys.length; i++) { - Assert.assertEquals(reader.get(keys[i], null), values[i]); + try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { + for (int i = 0; i < keys.length; i++) { + Assert.assertEquals(reader.get((Integer) keys[i], null), values[i]); + } } } @@ -444,23 +415,23 @@ public void testReadDisk() { Configuration configuration = new Configuration(); //Write - StoreWriter writer = PalDB.createWriter(STORE_FILE, configuration); Object[] values = GenerateTestData.generateStringData(keys.length, 1000); - writer.putAll(keys, values); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(storeFile, configuration)) { + writer.putAll(keys, values); + } //Read configuration.set(Configuration.MMAP_DATA_ENABLED, "false"); - StoreReader reader = PalDB.createReader(STORE_FILE, configuration); - Assert.assertEquals(reader.size(), keys.length); - - for (int i = 0; i < keys.length; i++) { - Object key = keys[i]; - Object val = reader.getString(key, null); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { + Assert.assertEquals(reader.size(), keys.length); + + for (int i = 0; i < keys.length; i++) { + Integer key = keys[i]; + Object val = reader.get(key, null); + Assert.assertNotNull(val); + Assert.assertEquals(val, values[i]); + } } - reader.close(); } @Test @@ -469,28 +440,28 @@ public void testIterate() { String[] values = GenerateTestData.generateStringData(keys.length, 12); //Write - writeStore(STORE_FILE, keys, values); + writeStore(storeFile, keys, values); //Sets - Set keysSet = new HashSet(Arrays.asList(keys)); - Set valuesSet = new HashSet(Arrays.asList(values)); + Set keysSet = new HashSet<>(Arrays.asList(keys)); + Set valuesSet = new HashSet<>(Arrays.asList(values)); //Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Iterator> itr = reader.iterable().iterator(); - for (int i = 0; i < keys.length; i++) { - Assert.assertTrue(itr.hasNext()); - Map.Entry entry = itr.next(); - Assert.assertNotNull(entry); - Assert.assertTrue(keysSet.remove(entry.getKey())); - Assert.assertTrue(valuesSet.remove(entry.getValue())); - - Object valSearch = reader.get(entry.getKey(), null); - Assert.assertNotNull(valSearch); - Assert.assertEquals(valSearch, entry.getValue()); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + var itr = reader.iterable().iterator(); + for (int i = 0; i < keys.length; i++) { + Assert.assertTrue(itr.hasNext()); + var entry = itr.next(); + Assert.assertNotNull(entry); + Assert.assertTrue(keysSet.remove(entry.getKey())); + Assert.assertTrue(valuesSet.remove(entry.getValue())); + + Object valSearch = reader.get(entry.getKey(), null); + Assert.assertNotNull(valSearch); + Assert.assertEquals(valSearch, entry.getValue()); + } + Assert.assertFalse(itr.hasNext()); } - Assert.assertFalse(itr.hasNext()); - reader.close(); Assert.assertTrue(keysSet.isEmpty()); Assert.assertTrue(valuesSet.isEmpty()); @@ -498,98 +469,90 @@ public void testIterate() { // UTILITY - private void testReadKeyToString(Object[] keys) { + private void testReadKeyToString(K[] keys) { // Write - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); - Object[] values = GenerateTestData.generateStringData(keys.length, 10); - writer.putAll(keys, values); - writer.close(); - - // Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.size(), keys.length); - - for (int i = 0; i < keys.length; i++) { - Object key = keys[i]; - Object val = reader.getString(key, null); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + String[] values = GenerateTestData.generateStringData(keys.length, 10); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + writer.putAll(keys, values); + } + // Read + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), keys.length); + + for (int i = 0; i < keys.length; i++) { + K key = keys[i]; + String val = reader.get(key, null); + Assert.assertNotNull(val); + Assert.assertEquals(val, values[i]); + } } - - reader.close(); } - private void testReadKeyToInt(Object[] keys) { + private void testReadKeyToInt(K[] keys) { // Write - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); Integer[] values = GenerateTestData.generateIntData(keys.length); - writer.putAll(keys, values); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + writer.putAll(keys, values); + } - // Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.size(), keys.length); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), keys.length); - for (int i = 0; i < keys.length; i++) { - Object key = keys[i]; - Object val = reader.getInt(key, 0); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + for (int i = 0; i < keys.length; i++) { + K key = keys[i]; + Object val = reader.get(key, 0); + Assert.assertNotNull(val); + Assert.assertEquals(val, values[i]); + } } - - reader.close(); } - private void testReadKeyToNull(Object[] keys) { + private void testReadKeyToNull(K[] keys) { //Write - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); - Object[] values = new Object[keys.length]; - writer.putAll(keys, values); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + Object[] values = new Object[keys.length]; + writer.putAll(keys, values); + } - //Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.size(), keys.length); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), keys.length); - for (int i = 0; i < keys.length; i++) { - Object key = keys[i]; - Object val = reader.get(key, 0); - Assert.assertNull(val); - } - for (int i = 0; i < keys.length; i++) { - Object key = keys[i]; - Object val = reader.get(key, 0); - Assert.assertNull(val); - } + for (K key : keys) { + Object val = reader.get(key, null); + Assert.assertNull(val); + } - reader.close(); + for (K key : keys) { + Object val = reader.get(key, null); + Assert.assertNull(val); + } + } } - private void testReadKeyToIntArray(Object[] keys) { + private void testReadKeyToIntArray(K[] keys) { //Write - StoreWriter writer = PalDB.createWriter(STORE_FILE, new Configuration()); int[][] values = GenerateTestData.generateIntArrayData(keys.length, 100); - writer.putAll(keys, values); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + writer.putAll(keys, values); + } //Read - StoreReader reader = PalDB.createReader(STORE_FILE, new Configuration()); - Assert.assertEquals(reader.size(), keys.length); - - for (int i = 0; i < keys.length; i++) { - Object key = keys[i]; - int[] val = reader.getIntArray(key, null); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { + Assert.assertEquals(reader.size(), keys.length); + + for (int i = 0; i < keys.length; i++) { + K key = keys[i]; + int[] val = reader.get(key, null); + Assert.assertNotNull(val); + Assert.assertEquals(val, values[i]); + } } - - reader.close(); } private void writeStore(File location, Object[] keys, Object[] values) { - StoreWriter writer = PalDB.createWriter(location, new Configuration()); - writer.putAll(keys, values); - writer.close(); + try (StoreWriter writer = PalDB.createWriter(location, new Configuration())) { + writer.putAll(keys, values); + } } private void testKeyLength(Object key, int expectedLength) { diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index b2a03b2..f3f3ab9 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -21,394 +21,430 @@ import java.awt.*; import java.io.*; import java.nio.file.*; +import java.util.List; import java.util.*; +import static com.linkedin.paldb.utils.TestTempUtils.*; +import static org.testng.Assert.*; public class TestStoreReader { - private StoreReader reader; - private Path tempDir; - private File STORE_FILE = createTempFile(); + private File storeFile; @BeforeMethod public void setUp() throws IOException { tempDir = Files.createTempDirectory("tmp"); - STORE_FILE = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); - STORE_FILE.mkdir(); - - Configuration configuration = new Configuration(); - configuration.registerSerializer(new PointSerializer()); - try (StoreWriter writer = PalDB.createWriter(STORE_FILE, configuration)) { - for (int i = 0; i < testValues.length; i++) { - writer.put(i, testValues[i]); - } - } - reader = PalDB.createReader(STORE_FILE, PalDBConfigBuilder.create().build()); + storeFile = Files.createTempFile(tempDir, "paldb", ".dat").toFile(); } @AfterMethod public void cleanUp() { - try { - reader.close(); - } catch (Exception e) { - //nop - } deleteDirectory(tempDir.toFile()); } - private boolean deleteDirectory(File directoryToBeDeleted) { - if (directoryToBeDeleted.isDirectory()) { - File[] allContents = directoryToBeDeleted.listFiles(); - if (allContents != null) { - for (File file : allContents) { - deleteDirectory(file); - } + @SafeVarargs + private StoreReader readerForMany(V... values) { + Configuration configuration = new Configuration(); + configuration.registerSerializer(new PointSerializer()); + try (StoreWriter writer = PalDB.createWriter(storeFile, configuration)) { + for (int i = 0; i < values.length; i++) { + writer.put(i, values[i]); } } - return directoryToBeDeleted.delete(); + return PalDB.createReader(storeFile, PalDBConfigBuilder.create().build()); } - private static File createTempFile() { - try { - return Files.createTempFile( "paldb", ".dat").toFile(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + private StoreReader readerFor(V value) { + return readerForMany(value); } - - private final Object[] testValues = - new Object[]{true, (byte) 1, 'a', 1.0, 1f, (short) 1, 1, 1l, "foo", new boolean[]{true}, new byte[]{1}, new char[]{'a'}, new double[]{1.0}, new float[]{1f}, new short[]{1}, new int[]{1}, new long[]{1l}, new String[]{"foo"}, new Object[]{"foo"}, new Point( - 4, 56)}; - @Test public void testFile() { - Assert.assertEquals(reader.getFile(), STORE_FILE); + try (var reader = readerFor(true)) { + assertEquals(reader.getFile(), storeFile); + } } @Test public void testSize() { - Assert.assertEquals(reader.size(), testValues.length); + try (var reader = readerFor(true)) { + assertEquals(reader.size(), 1); + } } @Test(expectedExceptions = IllegalStateException.class) public void testStoreClosed() { + var reader = readerFor(true); reader.close(); reader.get(0); } @Test - public void testGetBoolean() - throws Throwable { - Assert.assertTrue(reader.getBoolean(0)); - Assert.assertTrue(reader.getBoolean(0, false)); - Assert.assertFalse(reader.getBoolean(-1, false)); + public void testGetBoolean() { + try (var reader = readerFor(true)) { + Assert.assertTrue(reader.get(0)); + Assert.assertTrue(reader.get(0, false)); + Assert.assertFalse(reader.get(-1, false)); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetBooleanMissing() - throws Throwable { - reader.getBoolean(-1); + @Test + public void testGetBooleanMissing() { + try (var reader = readerFor(true)) { + assertNull(reader.get(-1)); + } } @Test - public void testGetByte() - throws Throwable { - Assert.assertEquals(reader.getByte(1), (byte) 1); - Assert.assertEquals(reader.getByte(1, (byte) 5), (byte) 1); - Assert.assertEquals(reader.getByte(-1, (byte) 5), (byte) 5); + public void testGetByte() { + try (var reader = readerFor((byte)1)) { + assertEquals(reader.get(0).byteValue(), (byte) 1); + assertEquals(reader.get(0, (byte) 5).byteValue(), (byte) 1); + assertEquals(reader.get(-1, (byte) 5).byteValue(), (byte) 5); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetByteMissing() - throws Throwable { - reader.getByte(-1); + @Test + public void testGetByteMissing() { + try (var reader = readerFor((byte)1)) { + assertNull(reader.get(-1)); + } } @Test - public void testGetChar() - throws Throwable { - Assert.assertEquals(reader.getChar(2), (char) 'a'); - Assert.assertEquals(reader.getChar(2, (char) 'b'), (char) 'a'); - Assert.assertEquals(reader.getChar(-1, (char) 'b'), (char) 'b'); + public void testGetChar() { + try (var reader = readerFor('a')) { + assertEquals(reader.get(0).charValue(), 'a'); + assertEquals(reader.get(0, 'b').charValue(), 'a'); + assertEquals(reader.get(-1, 'b').charValue(), 'b'); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetCharMissing() - throws Throwable { - reader.getChar(-1); + @Test + public void testGetCharMissing() { + try (var reader = readerFor('a')) { + assertNull(reader.get(-1)); + } } @Test - public void testGetDouble() - throws Throwable { - Assert.assertEquals(reader.getDouble(3), 1.0); - Assert.assertEquals(reader.getDouble(3, 2.0), 1.0); - Assert.assertEquals(reader.getDouble(-1, 2.0), 2.0); + public void testGetDouble() { + try (var reader = readerFor(1.0)) { + assertEquals(reader.get(0).doubleValue(), 1.0); + assertEquals(reader.get(0, 2.0).doubleValue(), 1.0); + assertEquals(reader.get(-1, 2.0).doubleValue(), 2.0); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetDoubleMissing() - throws Throwable { - reader.getDouble(-1); + @Test + public void testGetDoubleMissing() { + try (var reader = readerFor(1.0)) { + assertNull(reader.get(-1)); + } } @Test - public void testGetFloat() - throws Throwable { - Assert.assertEquals(reader.getFloat(4), 1f); - Assert.assertEquals(reader.getFloat(4, 2f), 1f); - Assert.assertEquals(reader.getFloat(-1, 2f), 2f); + public void testGetFloat() { + try (var reader = readerFor(1f)) { + assertEquals(reader.get(0).floatValue(), 1f); + assertEquals(reader.get(0, 2f).floatValue(), 1f); + assertEquals(reader.get(-1, 2f).floatValue(), 2f); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetFloatMissing() - throws Throwable { - reader.getFloat(-1); + @Test + public void testGetFloatMissing() { + try (var reader = readerFor(1.0)) { + assertNull(reader.get(-1)); + } } @Test - public void testGetShort() - throws Throwable { - Assert.assertEquals(reader.getShort(5), (short) 1); - Assert.assertEquals(reader.getShort(5, (short) 2), (short) 1); - Assert.assertEquals(reader.getShort(-1, (short) 2), (short) 2); + public void testGetShort() { + try (var reader = readerFor((short) 1)) { + assertEquals(reader.get(0).shortValue(), (short) 1); + assertEquals(reader.get(0, (short) 2).shortValue(), (short) 1); + assertEquals(reader.get(-1, (short) 2).shortValue(), (short) 2); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetShortMissing() - throws Throwable { - reader.getShort(-1); + @Test + public void testGetShortMissing() { + try (var reader = readerFor((short) 1)) { + assertNull(reader.get(-1)); + } } @Test - public void testGetInt() - throws Throwable { - Assert.assertEquals(reader.getInt(6), 1); - Assert.assertEquals(reader.getInt(6, 2), 1); - Assert.assertEquals(reader.getInt(-1, 2), 2); + public void testGetInt() { + try (var reader = readerFor(1)) { + assertEquals(reader.get(0).intValue(), 1); + assertEquals(reader.get(0, 2).intValue(), 1); + assertEquals(reader.get(-1, 2).intValue(), 2); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetIntMissing() - throws Throwable { - reader.getInt(-1); + @Test + public void testGetIntMissing() { + try (var reader = readerFor(1)) { + assertNull(reader.get(-1)); + } } @Test - public void testGetLong() - throws Throwable { - Assert.assertEquals(reader.getLong(7), 1l); - Assert.assertEquals(reader.getLong(7, 2l), 1l); - Assert.assertEquals(reader.getLong(-1, 2l), 2l); + public void testGetLong() { + try (var reader = readerFor(1L)) { + assertEquals(reader.get(0).longValue(), 1L); + assertEquals(reader.get(0, 2L).longValue(), 1L); + assertEquals(reader.get(-1, 2L).longValue(), 2L); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetLongMissing() - throws Throwable { - reader.getLong(-1); + @Test + public void testGetLongMissing() { + try (var reader = readerFor(1L)) { + assertNull(reader.get(-1)); + } } @Test public void testGetString() { - Assert.assertEquals(reader.getString(8), "foo"); - Assert.assertEquals(reader.getString(8, "bar"), "foo"); - Assert.assertEquals(reader.getString(-1, "bar"), "bar"); + try (var reader = readerFor("foo")) { + assertEquals(reader.get(0), "foo"); + assertEquals(reader.get(0, "bar"), "foo"); + assertEquals(reader.get(-1, "bar"), "bar"); + } } @Test public void testGetStringMissing() { - Assert.assertNull(reader.getString(-1)); + try (var reader = readerFor("foo")) { + assertNull(reader.get(-1)); + } } @Test - public void testGetBooleanArray() - throws Throwable { - Assert.assertEquals(reader.getBooleanArray(9), new boolean[]{true}); - Assert.assertEquals(reader.getBooleanArray(9, new boolean[]{false}), new boolean[]{true}); - Assert.assertEquals(reader.getBooleanArray(-1, new boolean[]{false}), new boolean[]{false}); + public void testGetBooleanArray() { + try (var reader = readerFor(new boolean[]{true})) { + assertEquals(reader.get(0), new boolean[]{true}); + assertEquals(reader.get(0, new boolean[]{false}), new boolean[]{true}); + assertEquals(reader.get(-1, new boolean[]{false}), new boolean[]{false}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetBooleanArrayMissing() - throws Throwable { - reader.getBooleanArray(-1); + @Test + public void testGetBooleanArrayMissing() { + try (var reader = readerFor(new boolean[]{true})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetByteArray() - throws Throwable { - Assert.assertEquals(reader.getByteArray(10), new byte[]{1}); - Assert.assertEquals(reader.getByteArray(10, new byte[]{2}), new byte[]{1}); - Assert.assertEquals(reader.getByteArray(-1, new byte[]{2}), new byte[]{2}); + public void testGetByteArray() { + try (var reader = readerFor(new byte[]{1})) { + assertEquals(reader.get(0), new byte[]{1}); + assertEquals(reader.get(0, new byte[]{2}), new byte[]{1}); + assertEquals(reader.get(-1, new byte[]{2}), new byte[]{2}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetByteArrayMissing() - throws Throwable { - reader.getByteArray(-1); + @Test + public void testGetByteArrayMissing() { + try (var reader = readerFor(new byte[]{1})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetCharArray() - throws Throwable { - Assert.assertEquals(reader.getCharArray(11), new char[]{'a'}); - Assert.assertEquals(reader.getCharArray(11, new char[]{'b'}), new char[]{'a'}); - Assert.assertEquals(reader.getCharArray(-1, new char[]{'b'}), new char[]{'b'}); + public void testGetCharArray() { + try (var reader = readerFor(new char[]{'a'})) { + assertEquals(reader.get(0), new char[]{'a'}); + assertEquals(reader.get(0, new char[]{'b'}), new char[]{'a'}); + assertEquals(reader.get(-1, new char[]{'b'}), new char[]{'b'}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetCharArrayMissing() - throws Throwable { - reader.getCharArray(-1); + @Test + public void testGetCharArrayMissing() { + try (var reader = readerFor(new char[]{'a'})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetDoubleArray() - throws Throwable { - Assert.assertEquals(reader.getDoubleArray(12), new double[]{1.0}); - Assert.assertEquals(reader.getDoubleArray(12, new double[]{2.0}), new double[]{1.0}); - Assert.assertEquals(reader.getDoubleArray(-1, new double[]{2.0}), new double[]{2.0}); + public void testGetDoubleArray() { + try (var reader = readerFor(new double[]{1.0})) { + assertEquals(reader.get(0), new double[]{1.0}); + assertEquals(reader.get(0, new double[]{2.0}), new double[]{1.0}); + assertEquals(reader.get(-1, new double[]{2.0}), new double[]{2.0}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetDoubleArrayMissing() - throws Throwable { - reader.getDoubleArray(-1); + @Test + public void testGetDoubleArrayMissing() { + try (var reader = readerFor(new double[]{1.0})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetFloatArray() - throws Throwable { - Assert.assertEquals(reader.getFloatArray(13), new float[]{1f}); - Assert.assertEquals(reader.getFloatArray(13, new float[]{2f}), new float[]{1f}); - Assert.assertEquals(reader.getFloatArray(-1, new float[]{2f}), new float[]{2f}); + public void testGetFloatArray() { + try (var reader = readerFor(new float[]{1f})) { + assertEquals(reader.get(0), new float[]{1f}); + assertEquals(reader.get(0, new float[]{2f}), new float[]{1f}); + assertEquals(reader.get(-1, new float[]{2f}), new float[]{2f}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetFloatArrayMissing() - throws Throwable { - reader.getFloatArray(-1); + @Test + public void testGetFloatArrayMissing() { + try (var reader = readerFor(new float[]{1f})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetShortArray() - throws Throwable { - Assert.assertEquals(reader.getShortArray(14), new short[]{1}); - Assert.assertEquals(reader.getShortArray(14, new short[]{2}), new short[]{1}); - Assert.assertEquals(reader.getShortArray(-1, new short[]{2}), new short[]{2}); + public void testGetShortArray() { + try (var reader = readerFor(new short[]{1})) { + assertEquals(reader.get(0), new short[]{1}); + assertEquals(reader.get(0, new short[]{2}), new short[]{1}); + assertEquals(reader.get(-1, new short[]{2}), new short[]{2}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetShortArrayMissing() - throws Throwable { - reader.getShortArray(-1); + @Test + public void testGetShortArrayMissing() { + try (var reader = readerFor(new short[]{1})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetIntArray() - throws Throwable { - Assert.assertEquals(reader.getIntArray(15), new int[]{1}); - Assert.assertEquals(reader.getIntArray(15, new int[]{2}), new int[]{1}); - Assert.assertEquals(reader.getIntArray(-1, new int[]{2}), new int[]{2}); + public void testGetIntArray() { + try (var reader = readerFor(new int[]{1})) { + assertEquals(reader.get(0), new int[]{1}); + assertEquals(reader.get(0, new int[]{2}), new int[]{1}); + assertEquals(reader.get(-1, new int[]{2}), new int[]{2}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetIntArrayMissing() - throws Throwable { - reader.getIntArray(-1); + @Test + public void testGetIntArrayMissing() { + try (var reader = readerFor(new int[]{1})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetLongArray() - throws Throwable { - Assert.assertEquals(reader.getLongArray(16), new long[]{1l}); - Assert.assertEquals(reader.getLongArray(16, new long[]{2l}), new long[]{1l}); - Assert.assertEquals(reader.getLongArray(-1, new long[]{2l}), new long[]{2l}); + public void testGetLongArray() { + try (var reader = readerFor(new long[]{1L})) { + assertEquals(reader.get(0), new long[]{1L}); + assertEquals(reader.get(0, new long[]{2L}), new long[]{1L}); + assertEquals(reader.get(-1, new long[]{2L}), new long[]{2L}); + } } - @Test(expectedExceptions = NotFoundException.class) - public void testGetLongArrayMissing() - throws Throwable { - reader.getLongArray(-1); + @Test + public void testGetLongArrayMissing() { + try (var reader = readerFor(new long[]{1L})) { + assertNull(reader.get(-1)); + } } @Test - public void testGetStringArray() - throws Throwable { - Assert.assertEquals(reader.getStringArray(17), new String[]{"foo"}); - Assert.assertEquals(reader.getStringArray(17, new String[]{"bar"}), new String[]{"foo"}); - Assert.assertEquals(reader.getStringArray(-1, new String[]{"bar"}), new String[]{"bar"}); + public void testGetStringArray() { + try (var reader = readerFor(new String[]{"foo"})) { + assertEquals(reader.get(0), new String[]{"foo"}); + assertEquals(reader.get(0, new String[]{"bar"}), new String[]{"foo"}); + assertEquals(reader.get(-1, new String[]{"bar"}), new String[]{"bar"}); + } } @Test - public void testGetStringArrayMissing() - throws Throwable { - Assert.assertNull(reader.getStringArray(-1)); + public void testGetStringArrayMissing() { + try (var reader = readerFor(new String[]{"foo"})) { + assertNull(reader.get(-1)); + } } @Test public void testGetMissing() { - Assert.assertNull(reader.get(-1)); + try (var reader = readerFor("foo")) { + assertNull(reader.get(-1)); + } } @Test public void testGetArray() { - Assert.assertEquals(reader.getArray(18), new Object[]{"foo"}); - Assert.assertEquals(reader.getArray(18, new Object[]{"bar"}), new Object[]{"foo"}); - Assert.assertEquals(reader.getArray(-1, new Object[]{"bar"}), new Object[]{"bar"}); + try (var reader = readerFor(new Object[]{"foo"})) { + assertEquals(reader.get(0), new Object[]{"foo"}); + assertEquals(reader.get(0, new Object[]{"bar"}), new Object[]{"foo"}); + assertEquals(reader.get(-1, new Object[]{"bar"}), new Object[]{"bar"}); + } } @Test public void testGetArrayMissing() { - Assert.assertNull(reader.getArray(-1)); + try (var reader = readerFor(new Object[]{"foo"})) { + assertNull(reader.get(-1)); + } } @Test public void testGetPoint() { - Assert.assertEquals(reader.get(19), new Point(4, 56)); + try (var reader = readerFor(new Point(4, 56))) { + assertEquals(reader.get(0), new Point(4, 56)); + } } @Test public void testIterator() { - Iterable> iter = reader.iterable(); - Assert.assertNotNull(iter); - Iterator> itr = iter.iterator(); - Assert.assertNotNull(itr); - - for (int i = 0; i < testValues.length; i++) { - Assert.assertTrue(itr.hasNext()); - Map.Entry v = itr.next(); - Object val = testValues[v.getKey()]; - Assert.assertEquals(v.getValue(), val); + var values = List.of("foo", "bar"); + try (var reader = readerForMany(values.get(0), values.get(1))) { + var iter = reader.iterable(); + Assert.assertNotNull(iter); + var itr = iter.iterator(); + Assert.assertNotNull(itr); + + for (int i = 0; i < values.size(); i++) { + Assert.assertTrue(itr.hasNext()); + var v = itr.next(); + assertEquals(v.getValue(), values.get(v.getKey())); + } } } @Test public void testIterate() { - for (var entry: reader) { - Object val = testValues[(int) entry.getKey()]; - Assert.assertEquals(entry.getValue(), val); + var values = List.of("foo", "bar"); + try (var reader = readerForMany(values.get(0), values.get(1))) { + for (var entry: reader) { + var val = values.get(entry.getKey()); + assertEquals(entry.getValue(), val); + } } } @Test public void testKeyIterator() { - Iterable iter = reader.keys(); - Assert.assertNotNull(iter); - Iterator itr = iter.iterator(); - Assert.assertNotNull(itr); - - Set actual = new HashSet(); - Set expected = new HashSet(); - for (int i = 0; i < testValues.length; i++) { - Assert.assertTrue(itr.hasNext()); - Integer k = itr.next(); - actual.add(k); - expected.add(i); - } - Assert.assertEquals(actual, expected); + var values = List.of("foo", "bar"); + try (var reader = readerForMany(values.get(0), values.get(1))) { + var iter = reader.keys(); + Assert.assertNotNull(iter); + var itr = iter.iterator(); + Assert.assertNotNull(itr); + + Set actual = new HashSet<>(); + Set expected = new HashSet<>(); + for (int i = 0; i < values.size(); i++) { + Assert.assertTrue(itr.hasNext()); + Integer k = itr.next(); + actual.add(k); + expected.add(i); + } + assertEquals(actual, expected); + } } // UTILITY diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java b/paldb/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java index 5e7fb64..7dc2909 100644 --- a/paldb/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java +++ b/paldb/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java @@ -14,17 +14,26 @@ package com.linkedin.paldb.utils; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; import org.testng.Assert; import org.testng.annotations.Test; +import java.io.*; + public class TestTempUtils { + public static boolean deleteDirectory(File directoryToBeDeleted) { + if (directoryToBeDeleted.isDirectory()) { + File[] allContents = directoryToBeDeleted.listFiles(); + if (allContents != null) { + for (File file : allContents) { + deleteDirectory(file); + } + } + } + return directoryToBeDeleted.delete(); + } + @Test public void testTempDir() { File file = TempUtils.createTempDir("foo"); From 6307b3b2a176f0a26b9f90e560e9006b1329b4c7 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Thu, 14 Nov 2019 15:52:21 +0200 Subject: [PATCH 11/20] Generic writers. --- README.md | 6 ++-- .../com/linkedin/paldb/api/Configuration.java | 30 ++++++++----------- .../linkedin/paldb/api/NotFoundException.java | 28 ----------------- .../java/com/linkedin/paldb/api/PalDB.java | 6 ++-- .../com/linkedin/paldb/api/Serializer.java | 6 ++-- .../com/linkedin/paldb/api/StoreWriter.java | 6 ++-- .../com/linkedin/paldb/impl/StoreImpl.java | 8 ++--- .../com/linkedin/paldb/impl/WriterImpl.java | 6 ++-- .../com/linkedin/paldb/impl/TestStore.java | 25 ++++++++-------- 9 files changed, 44 insertions(+), 77 deletions(-) delete mode 100644 paldb/src/main/java/com/linkedin/paldb/api/NotFoundException.java diff --git a/README.md b/README.md index 6a51f0f..4e52b5e 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ API documentation can be found [here](http://linkedin.github.com/PalDB/doc/javad How to write a store ```java -try (var writer = PalDB.createWriter(new File("store.paldb"))) { +try (var writer = PalDB.createWriter(new File("store.paldb"))) { writer.put(1213, "foo"); } ``` @@ -74,12 +74,12 @@ PalDB is available on Maven Central, hence just add the following dependency: net.soundvibe.paldb paldb - 1.2.1 + 2.0.0 ``` Scala SBT ``` -libraryDependencies += "net.soundvibe.paldb" % "paldb" % "1.2.1" +libraryDependencies += "net.soundvibe.paldb" % "paldb" % "2.0.0" ``` diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java index 1298da4..a9283d2 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java @@ -43,28 +43,28 @@ public class Configuration implements Serializable { // Buffer segment size - public final static String MMAP_SEGMENT_SIZE = "mmap.segment.size"; + public static final String MMAP_SEGMENT_SIZE = "mmap.segment.size"; // Enable memory mapping for data - public final static String MMAP_DATA_ENABLED = "mmap.data.enabled"; + public static final String MMAP_DATA_ENABLED = "mmap.data.enabled"; // Load factor - public final static String LOAD_FACTOR = "load.factor"; + public static final String LOAD_FACTOR = "load.factor"; // Cache enabled - public final static String CACHE_ENABLED = "cache.enabled"; + public static final String CACHE_ENABLED = "cache.enabled"; // Cache limit (in bytes) - public final static String CACHE_BYTES = "cache.bytes"; + public static final String CACHE_BYTES = "cache.bytes"; // Cache initial capacity - public final static String CACHE_INITIAL_CAPACITY = "cache.initial.capacity"; + public static final String CACHE_INITIAL_CAPACITY = "cache.initial.capacity"; // Cache load factor - public final static String CACHE_LOAD_FACTOR = "cache.load.factor"; + public static final String CACHE_LOAD_FACTOR = "cache.load.factor"; // Enable compression - public final static String COMPRESSION_ENABLED = "compression.enabled"; + public static final String COMPRESSION_ENABLED = "compression.enabled"; // Property map - protected final Map properties = new HashMap(); + private final Map properties = new HashMap<>(); // Read only - protected final boolean readOnly; + private final boolean readOnly; // Serializers - protected final Serializers serializers; + private final Serializers serializers; /** * Default constructor that initializes default values. @@ -427,16 +427,12 @@ public boolean equals(Object o) { if (!properties.equals(that.properties)) { return false; } - if (!serializers.equals(that.serializers)) { - return false; - } - - return true; + return serializers.equals(that.serializers); } @Override public int hashCode() { - int result = properties != null ? properties.hashCode() : 0; + int result = properties.hashCode(); result = 31 * result + (serializers != null ? serializers.hashCode() : 0); return result; } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/NotFoundException.java b/paldb/src/main/java/com/linkedin/paldb/api/NotFoundException.java deleted file mode 100644 index 9983c28..0000000 --- a/paldb/src/main/java/com/linkedin/paldb/api/NotFoundException.java +++ /dev/null @@ -1,28 +0,0 @@ -/* -* Copyright 2015 LinkedIn Corp. All rights reserved. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -*/ - -package com.linkedin.paldb.api; - -/** - * Exception returned when an key can't be found. - * - * @see StoreReader - */ -@SuppressWarnings("serial") -public class NotFoundException extends Exception { - - public NotFoundException(Object key) { - super("The key '" + key.toString() + "' couldn't be found"); - } -} diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java index 2504a46..256b888 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java @@ -75,7 +75,7 @@ static StoreReader createReader(InputStream stream, Configuration con * @param file location of the output file * @return a store writer */ - static StoreWriter createWriter(File file) { + static StoreWriter createWriter(File file) { return StoreImpl.createWriter(file, newConfiguration()); } @@ -88,7 +88,7 @@ static StoreWriter createWriter(File file) { * @param config configuration * @return a store writer */ - static StoreWriter createWriter(File file, Configuration config) { + static StoreWriter createWriter(File file, Configuration config) { return StoreImpl.createWriter(file, config); } @@ -102,7 +102,7 @@ static StoreWriter createWriter(File file, Configuration config) { * @param config configuration * @return a store writer */ - static StoreWriter createWriter(OutputStream stream, Configuration config) { + static StoreWriter createWriter(OutputStream stream, Configuration config) { return StoreImpl.createWriter(stream, config); } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java b/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java index 04b00a1..844554e 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java @@ -39,7 +39,7 @@ public interface Serializer extends Serializable { * @param input instance * @throws IOException if an io error occurs */ - public void write(DataOutput dataOutput, K input) + void write(DataOutput dataOutput, K input) throws IOException; /** @@ -49,7 +49,7 @@ public void write(DataOutput dataOutput, K input) * @return new instance of type K. * @throws IOException if an io error occurs */ - public K read(DataInput dataInput) + K read(DataInput dataInput) throws IOException; /** @@ -59,5 +59,5 @@ public K read(DataInput dataInput) * @param instance instance to get weight for * @return the number of bytes the object uses in memory */ - public int getWeight(K instance); + int getWeight(K instance); } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java b/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java index 2a013f2..31cc446 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java @@ -24,7 +24,7 @@ *

* Note that duplicates aren't allowed. */ -public interface StoreWriter extends AutoCloseable { +public interface StoreWriter extends AutoCloseable { /** * Close the store writer and append the data to the final destination. A @@ -50,7 +50,7 @@ public interface StoreWriter extends AutoCloseable { * @throws NullPointerException if key or value is * null */ - void put(Object key, Object value); + void put(K key, V value); /** * Put multiple key-values to the store. @@ -58,7 +58,7 @@ public interface StoreWriter extends AutoCloseable { * @param keys a collection of keys * @param values a collection of values */ - void putAll(Object[] keys, Object[] values); + void putAll(K[] keys, V[] values); /** * Put serialized key-value entry to the store.

Use only this method if diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java index 944dc72..e5bcc65 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java @@ -53,7 +53,7 @@ public static StoreReader createReader(InputStream stream, Configurat } } - public static StoreWriter createWriter(File file, Configuration config) { + public static StoreWriter createWriter(File file, Configuration config) { if (file == null || config == null) { throw new NullPointerException(); } @@ -67,17 +67,17 @@ public static StoreWriter createWriter(File file, Configuration config) { throw new RuntimeException(String.format("Couldn't create directory %s", parent)); } } - return new WriterImpl(config, file); + return new WriterImpl<>(config, file); } catch (IOException ex) { throw new UncheckedIOException(ex); } } - public static StoreWriter createWriter(OutputStream stream, Configuration config) { + public static StoreWriter createWriter(OutputStream stream, Configuration config) { if (stream == null || config == null) { throw new NullPointerException(); } log.info("Initialize writer from stream"); - return new WriterImpl(config, stream); + return new WriterImpl<>(config, stream); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java index 70fa0b0..d392533 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java @@ -23,7 +23,7 @@ /** * Store writer implementation. */ -public final class WriterImpl implements StoreWriter { +public final class WriterImpl implements StoreWriter { // Logger private static final Logger log = LoggerFactory.getLogger(WriterImpl.class); @@ -103,7 +103,7 @@ public Configuration getConfiguration() { } @Override - public void put(Object key, Object value) { + public void put(K key, V value) { checkOpen(); if (key == null) { throw new NullPointerException(); @@ -117,7 +117,7 @@ public void put(Object key, Object value) { } @Override - public void putAll(Object[] keys, Object[] values) { + public void putAll(K[] keys, V[] values) { checkOpen(); if (keys == null || values == null) { throw new NullPointerException(); diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java index 341823f..979e467 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -145,8 +145,7 @@ public void testWriterNullStream() { public void testWriterNullConfigForStream() { PalDB.createWriter(new OutputStream() { @Override - public void write(int i) - throws IOException { + public void write(int i) { } }, null); @@ -180,7 +179,7 @@ public void testByteMarkEmpty() throws IOException { @Test public void testOneKey() { - try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { writer.put(1, "foo"); } @@ -196,7 +195,7 @@ public void testPutSerializedKey() throws IOException { byte[] serializedKey = storageSerialization.serializeKey(1); byte[] serializedValue = storageSerialization.serializeValue("foo"); - try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { writer.put(serializedKey, serializedValue); } @@ -209,7 +208,7 @@ public void testPutSerializedKey() throws IOException { @Test public void testByteMarkOneKey() throws IOException { try (FileOutputStream fos = new FileOutputStream(storeFile); - StoreWriter writer = PalDB.createWriter(fos, new Configuration());) { + StoreWriter writer = PalDB.createWriter(fos, new Configuration());) { fos.write(12345); fos.write(FormatVersion.getPrefixBytes()[0]); fos.write(3456); @@ -300,7 +299,7 @@ public void testKeyLengthStartTwo() { @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*duplicate.*") public void testDuplicateKeys() { - try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { writer.put(0, "ABC"); writer.put(0, "DGE"); } @@ -415,8 +414,8 @@ public void testReadDisk() { Configuration configuration = new Configuration(); //Write - Object[] values = GenerateTestData.generateStringData(keys.length, 1000); - try (StoreWriter writer = PalDB.createWriter(storeFile, configuration)) { + String[] values = GenerateTestData.generateStringData(keys.length, 1000); + try (StoreWriter writer = PalDB.createWriter(storeFile, configuration)) { writer.putAll(keys, values); } @@ -472,7 +471,7 @@ public void testIterate() { private void testReadKeyToString(K[] keys) { // Write String[] values = GenerateTestData.generateStringData(keys.length, 10); - try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { writer.putAll(keys, values); } // Read @@ -491,7 +490,7 @@ private void testReadKeyToString(K[] keys) { private void testReadKeyToInt(K[] keys) { // Write Integer[] values = GenerateTestData.generateIntData(keys.length); - try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { writer.putAll(keys, values); } @@ -509,7 +508,7 @@ private void testReadKeyToInt(K[] keys) { private void testReadKeyToNull(K[] keys) { //Write - try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { Object[] values = new Object[keys.length]; writer.putAll(keys, values); } @@ -532,7 +531,7 @@ private void testReadKeyToNull(K[] keys) { private void testReadKeyToIntArray(K[] keys) { //Write int[][] values = GenerateTestData.generateIntArrayData(keys.length, 100); - try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) { writer.putAll(keys, values); } @@ -550,7 +549,7 @@ private void testReadKeyToIntArray(K[] keys) { } private void writeStore(File location, Object[] keys, Object[] values) { - try (StoreWriter writer = PalDB.createWriter(location, new Configuration())) { + try (StoreWriter writer = PalDB.createWriter(location, new Configuration())) { writer.putAll(keys, values); } } From 8c617a8aa5755e79ed10223b9cd5ae11f51e7163 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Sat, 16 Nov 2019 18:54:07 +0200 Subject: [PATCH 12/20] StorageCache is now generic --- .../com/linkedin/paldb/api/Configuration.java | 4 +- .../com/linkedin/paldb/impl/ReaderImpl.java | 6 +- .../com/linkedin/paldb/impl/Serializers.java | 2 +- .../com/linkedin/paldb/impl/StorageCache.java | 42 ++++----- .../linkedin/paldb/impl/TestStorageCache.java | 88 +++++++++---------- 5 files changed, 69 insertions(+), 73 deletions(-) diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java index a9283d2..baaf7a5 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java @@ -405,7 +405,7 @@ public void registerSerializer(Serializer serializer) { * @param cls object class * @return serializer or null if not found */ - public Serializer getSerializer(Class cls) { + public Serializer getSerializer(Class cls) { return serializers.getSerializer(cls); } @@ -432,7 +432,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = properties.hashCode(); + int result = properties != null ? properties.hashCode() : 0; result = 31 * result + (serializers != null ? serializers.hashCode() : 0); return result; } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java index 2142e48..b5cd825 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java @@ -38,7 +38,7 @@ public final class ReaderImpl implements StoreReader { // Serialization private final StorageSerialization serialization; // Cache - private final StorageCache cache; + private final StorageCache cache; // File private final File file; // Opened? @@ -113,9 +113,9 @@ public V get(K key, V defaultValue) { byte[] valueBytes = storage.get(serialization.serializeKey(key)); if (valueBytes != null) { - Object v = serialization.deserialize(dataInputOutput.reset(valueBytes)); + V v = (V) serialization.deserialize(dataInputOutput.reset(valueBytes)); cache.put(key, v); - return (V) v; + return v; } else { return defaultValue; } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java index 7b85ff3..08ca4ba 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java @@ -69,7 +69,7 @@ public synchronized void registerSerializer(Serializer serializer) { * @param cls object class * @return serializer instance or null if not found */ - public Serializer getSerializer(Class cls) { + public Serializer getSerializer(Class cls) { SerializerWrapper w = getSerializerWrapper(cls); if (w != null) { return w.serializer; diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java index 7c2272d..6c46c91 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java @@ -34,7 +34,7 @@ * The cache estimates the size of the objects it contains so it consumes no more than the configured * memory limit. */ -public class StorageCache { +public class StorageCache { // Static null object to recognizes null from missing values protected static final Object NULL_VALUE = new Object(); @@ -47,11 +47,11 @@ public class StorageCache { * @param configuration configuration * @return new cache */ - static StorageCache initCache(Configuration configuration) { + static StorageCache initCache(Configuration configuration) { if (configuration.getBoolean(Configuration.CACHE_ENABLED) && configuration.getLong(Configuration.CACHE_BYTES) > 0) { - return new StorageCache(configuration); + return new StorageCache<>(configuration); } else { - return new DisabledCache(); + return new DisabledCache<>(); } } @@ -61,7 +61,7 @@ static StorageCache initCache(Configuration configuration) { * 24 bytes theoretical overhead per entry but more like 45 in practice */ static final int OVERHEAD = 50; - private final LinkedHashMap cache; + private final LinkedHashMap cache; private final Configuration configuration; private long maxWeight; private long currentWeight; @@ -72,14 +72,14 @@ static StorageCache initCache(Configuration configuration) { * @param config configuration */ private StorageCache(Configuration config) { - cache = new LinkedHashMap(config.getInt(Configuration.CACHE_INITIAL_CAPACITY), + cache = new LinkedHashMap<>(config.getInt(Configuration.CACHE_INITIAL_CAPACITY), config.getFloat(Configuration.CACHE_LOAD_FACTOR), true) { @Override - protected boolean removeEldestEntry(Map.Entry eldest) { + protected boolean removeEldestEntry(Map.Entry eldest) { boolean res = currentWeight > maxWeight; if (res) { - Object key = eldest.getKey(); - Object value = eldest.getValue(); + K key = eldest.getKey(); + V value = eldest.getValue(); currentWeight -= getWeight(key) + getWeight(value) + OVERHEAD; } return res; @@ -106,11 +106,10 @@ private StorageCache() { * StorageCache.NULL_VALUE. * * @param key key to get value for - * @param return type * @return value, null or StorageCache.NULL_VALUE */ - public K get(Object key) { - return (K) cache.get(key); + public V get(K key) { + return cache.get(key); } /** @@ -119,7 +118,7 @@ public K get(Object key) { * @param key key to test presence for * @return true if found, false otherwise */ - public boolean contains(Object key) { + public boolean contains(K key) { return cache.containsKey(key); } @@ -129,10 +128,10 @@ public boolean contains(Object key) { * @param key key * @param value value */ - public void put(Object key, Object value) { + public void put(K key, V value) { int weight = getWeight(key) + getWeight(value) + OVERHEAD; currentWeight += weight; - if (cache.put(key, value == null ? NULL_VALUE : value) != null) { + if (cache.put(key, value == null ? (V) NULL_VALUE : value) != null) { currentWeight -= weight; } } @@ -143,7 +142,7 @@ public void put(Object key, Object value) { * @param value value to get weight for * @return weight */ - private int getWeight(Object value) { + private int getWeight(T value) { if (value == null) { return 0; } @@ -199,7 +198,7 @@ private int getWeight(Object value) { } else if (value instanceof String) { return ((String) value).length() * 2 + 40; } else { - Serializer serializer = configuration.getSerializer(value.getClass()); + Serializer serializer = configuration.getSerializer((Class) value.getClass()); if (serializer != null) { return serializer.getWeight(value); } @@ -237,24 +236,25 @@ public long getWeight() { /** * Special inner class that overrides all cache's features when the cache is disabled. */ - private static class DisabledCache extends StorageCache { + private static class DisabledCache extends StorageCache { DisabledCache() { log.info("Cache disabled"); } @Override - public Object get(Object key) { + public V get(K key) { return null; } @Override - public boolean contains(Object key) { + public boolean contains(K key) { return false; } @Override - public void put(Object key, Object value) { + public void put(K key, V value) { + //nop } @Override diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java index bda6e46..c8874c9 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java @@ -14,50 +14,46 @@ package com.linkedin.paldb.impl; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.PalDB; -import com.linkedin.paldb.api.Serializer; - +import com.linkedin.paldb.api.*; import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; +import org.testng.annotations.*; public class TestStorageCache { - private final static int ENTRY_SIZE = 16 * 2 + StorageCache.OVERHEAD; + private static final int ENTRY_SIZE = 16 * 2 + StorageCache.OVERHEAD; - private Configuration _configuration; + private Configuration configuration; @BeforeMethod public void setUp() { - _configuration = PalDB.newConfiguration(); - _configuration.set(Configuration.CACHE_ENABLED, "true"); + configuration = PalDB.newConfiguration(); + configuration.set(Configuration.CACHE_ENABLED, "true"); } @Test public void testContainsValid() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 0); Assert.assertTrue(cache.contains(0)); } @Test public void testContainsInValid() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); Assert.assertFalse(cache.contains(0)); } @Test public void testEmpty() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); Assert.assertNull(cache.get(0)); Assert.assertEquals(cache.size(), 0); } @Test public void testPutOneItem() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 0); Assert.assertNotNull(cache.get(0)); Assert.assertEquals(cache.size(), 1); @@ -66,7 +62,7 @@ public void testPutOneItem() { @Test public void testPutTwice() { Integer second = 1; - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 1); cache.put(0, second); Assert.assertSame(cache.get(0), second); @@ -74,7 +70,7 @@ public void testPutTwice() { @Test public void testPutZeroSize() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.setMaxWeight(0); cache.put(0, 1); Assert.assertEquals(cache.size(), 0); @@ -82,7 +78,7 @@ public void testPutZeroSize() { @Test public void testPutTwiceObjectSize() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.setMaxWeight(ENTRY_SIZE); cache.put(0, 0); cache.put(1, 1); @@ -93,7 +89,7 @@ public void testPutTwiceObjectSize() { @Test public void putSameCheckWeight() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 0); long weight = cache.getWeight(); cache.put(0, 0); @@ -103,7 +99,7 @@ public void putSameCheckWeight() { @Test public void testPutGet() { int objs = 100; - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.setMaxWeight(ENTRY_SIZE * objs); for (int i = 0; i < objs; i++) { cache.put(i, i); @@ -118,7 +114,7 @@ public void testPutGet() { public void testCheckOrder() { int objs = 100; int capacity = 50; - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.setMaxWeight(ENTRY_SIZE * capacity); for (int i = 0; i < objs; i++) { cache.put(i, i); @@ -135,7 +131,7 @@ public void testCheckOrder() { @Test public void testCheckAccessOrderGet() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.setMaxWeight(ENTRY_SIZE * 3); cache.put(0, 0); cache.put(1, 1); @@ -149,7 +145,7 @@ public void testCheckAccessOrderGet() { @Test public void testCheckAccessOrderPut() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.setMaxWeight(ENTRY_SIZE * 3); cache.put(0, 0); cache.put(1, 1); @@ -163,112 +159,112 @@ public void testCheckAccessOrderPut() { @Test public void testWeightKeyObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 0); Assert.assertEquals(cache.getWeight(), ENTRY_SIZE); } @Test public void testWeightKeyArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(new Object[]{0, 1}, 0); Assert.assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); } @Test public void testWeightValueIntArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new int[]{1, 2}); Assert.assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); } @Test public void testWeightValueLongArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new long[]{1, 2}); Assert.assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); } @Test public void testWeightValueDoubleArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new double[]{1.0, 2.0}); Assert.assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); } @Test public void testWeightValueFloatArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new float[]{1.0F, 2.0F}); Assert.assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); } @Test public void testWeightValueBooleanArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new boolean[]{true, false}); Assert.assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueByteArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new byte[]{1, 2}); Assert.assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueShortArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new short[]{1, 2}); Assert.assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); } @Test public void testWeightValueCharArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new char[]{'a', 'b'}); Assert.assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); } @Test public void testWeightValueStringArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new String[]{"one", "two"}); Assert.assertEquals(cache.getWeight(), 16 + 46 * 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueInt2DArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new int[][]{{1, 2}, {3, 4}}); Assert.assertEquals(cache.getWeight(), 16 + 8 * 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueLong2DArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new long[][]{{1, 2}, {3, 4}}); Assert.assertEquals(cache.getWeight(), 16 + 16 * 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueStringObject() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new String("one")); Assert.assertEquals(cache.getWeight(), 16 + 46 + StorageCache.OVERHEAD); } @Test public void testWeightValueObjectArrayObjects() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new Object[]{0, 1}); Assert.assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); } @Test public void testNullValue() { - StorageCache cache = StorageCache.initCache(_configuration); + StorageCache cache = StorageCache.initCache(configuration); cache.put(0, null); Assert.assertEquals(cache.size(), 1); Assert.assertEquals(cache.get(0), StorageCache.NULL_VALUE); @@ -276,9 +272,9 @@ public void testNullValue() { @Test public void testDisabled() { - Configuration configuration = new Configuration(); - configuration.set(Configuration.CACHE_ENABLED, "false"); - StorageCache cache = StorageCache.initCache(configuration); + Configuration cfg = new Configuration(); + cfg.set(Configuration.CACHE_ENABLED, "false"); + StorageCache cache = StorageCache.initCache(cfg); Assert.assertEquals(cache.size(), 0); Assert.assertNull(cache.get("foo")); Assert.assertFalse(cache.contains("foo")); @@ -286,12 +282,12 @@ public void testDisabled() { @Test public void testDisabledPut() { - Configuration configuration = new Configuration(); - configuration.set(Configuration.CACHE_ENABLED, "false"); - StorageCache cache = StorageCache.initCache(configuration); + Configuration cfg = new Configuration(); + cfg.set(Configuration.CACHE_ENABLED, "false"); + StorageCache cache = StorageCache.initCache(cfg); cache.put(0, "foo"); Assert.assertEquals(cache.size(), 0); - Assert.assertNull(cache.get("foo")); - Assert.assertFalse(cache.contains("foo")); + Assert.assertNull(cache.get(1)); + Assert.assertFalse(cache.contains(2)); } } From fe89100db26bff76d71e91e9a48d2ce77346c230 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Sat, 16 Nov 2019 23:45:33 +0200 Subject: [PATCH 13/20] get() is now thread-safe --- .../com/linkedin/paldb/impl/ReaderImpl.java | 28 +-- .../linkedin/paldb/impl/StorageReader.java | 199 ++++++++++-------- .../paldb/impl/StorageSerialization.java | 8 +- .../linkedin/paldb/impl/StorageWriter.java | 4 +- .../com/linkedin/paldb/utils/HashUtils.java | 2 +- .../linkedin/paldb/api/TestConfiguration.java | 75 ++++--- .../linkedin/paldb/impl/TestSerializers.java | 38 ++-- .../linkedin/paldb/impl/TestStorageCache.java | 95 ++++----- .../paldb/impl/TestStorageSerialization.java | 151 +++++++------ .../com/linkedin/paldb/impl/TestStore.java | 137 ++++++------ .../linkedin/paldb/impl/TestStoreReader.java | 44 +++- 11 files changed, 403 insertions(+), 378 deletions(-) diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java index b5cd825..d5bc55a 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java @@ -107,25 +107,19 @@ public V get(K key, V defaultValue) { if (key == null) { throw new NullPointerException("The key can't be null"); } - V value = cache.get(key); - if (value == null) { - try { - byte[] valueBytes = storage.get(serialization.serializeKey(key)); - if (valueBytes != null) { - - V v = (V) serialization.deserialize(dataInputOutput.reset(valueBytes)); - cache.put(key, v); - return v; - } else { - return defaultValue; - } - } catch (IOException | ClassNotFoundException ex) { - throw new RuntimeException(ex); + + try { + byte[] valueBytes = storage.get(serialization.serializeKey(key)); + if (valueBytes != null) { + return (V) serialization.deserialize(new DataInputOutput(valueBytes)); + } else { + return defaultValue; } - } else if (value == StorageCache.NULL_VALUE) { - return null; + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); } - return value; } @Override diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index 575a684..dc724dd 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -32,10 +32,6 @@ public class StorageReader implements Iterable> { // Logger private static final Logger log = LoggerFactory.getLogger(StorageReader.class); - // Configuration - private final Configuration config; - // File path - private File path; // Buffer segment size private final long segmentSize; // Number of keys in the index @@ -46,12 +42,6 @@ public class StorageReader implements Iterable> { private final int[] slotSizes; // Number of slots for each key length private final int[] slots; - // Number of different key length - private final int keyLengthCount; - // Max key length - private final int maxKeyLength; - // Offset of the index in the channel - private final int indexOffset; // Offset of the index for different key length private final int[] indexOffsets; // Offset of the data in the channel @@ -60,33 +50,26 @@ public class StorageReader implements Iterable> { private final long[] dataOffsets; // Data size private final long dataSize; - // Index and data buffers - private MappedByteBuffer indexBuffer; - private MappedByteBuffer[] dataBuffers; + private final int maxSlotSize; // FileChannel - private RandomAccessFile mappedFile; - private FileChannel channel; + private final RandomAccessFile mappedFile; + private final FileChannel channel; // Use MMap for data? private final boolean mMapData; // Buffers - private final DataInputOutput sizeBuffer = new DataInputOutput(new byte[5]); - private final byte[] slotBuffer; - - private final HashUtils hashUtils; + private final ThreadLocal context; StorageReader(Configuration configuration, File file) throws IOException { - path = file; - config = configuration; + // File path + // Configuration if (!file.exists()) { throw new FileNotFoundException("File " + file.getAbsolutePath() + " not found"); } log.info("Opening file {}", file.getName()); //Config - segmentSize = config.getLong(Configuration.MMAP_SEGMENT_SIZE); - - hashUtils = new HashUtils(); + segmentSize = configuration.getLong(Configuration.MMAP_SEGMENT_SIZE); // Check valid segmentSize if (segmentSize > Integer.MAX_VALUE) { @@ -98,7 +81,9 @@ public class StorageReader implements Iterable> { long createdAt = 0; FormatVersion formatVersion = null; - try (FileInputStream inputStream = new FileInputStream(path); + // Offset of the index in the channel + int indexOffset; + try (FileInputStream inputStream = new FileInputStream(file); DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream))) { int ignoredBytes = -2; @@ -131,8 +116,10 @@ public class StorageReader implements Iterable> { //Metadata counters keyCount = dataInputStream.readInt(); - keyLengthCount = dataInputStream.readInt(); - maxKeyLength = dataInputStream.readInt(); + // Number of different key length + final int keyLengthCount = dataInputStream.readInt(); + // Max key length + final int maxKeyLength = dataInputStream.readInt(); //Read offset counts and keys indexOffsets = new int[maxKeyLength + 1]; @@ -141,7 +128,7 @@ public class StorageReader implements Iterable> { slots = new int[maxKeyLength + 1]; slotSizes = new int[maxKeyLength + 1]; - int maxSlotSize = 0; + int mSlotSize = 0; for (int i = 0; i < keyLengthCount; i++) { int keyLength = dataInputStream.readInt(); @@ -151,14 +138,13 @@ public class StorageReader implements Iterable> { indexOffsets[keyLength] = dataInputStream.readInt(); dataOffsets[keyLength] = dataInputStream.readLong(); - maxSlotSize = Math.max(maxSlotSize, slotSizes[keyLength]); + mSlotSize = Math.max(mSlotSize, slotSizes[keyLength]); } - - slotBuffer = new byte[maxSlotSize]; + maxSlotSize = mSlotSize; //Read serializers try { - Serializers.deserialize(dataInputStream, config.getSerializers()); + Serializers.deserialize(dataInputStream, configuration.getSerializers()); } catch (Exception e) { throw new RuntimeException(); } @@ -170,35 +156,21 @@ public class StorageReader implements Iterable> { //Close metadata //Create Mapped file in read-only mode - mappedFile = new RandomAccessFile(path, "r"); + mappedFile = new RandomAccessFile(file, "r"); channel = mappedFile.getChannel(); - long fileSize = path.length(); - - //Create index buffer - indexBuffer = channel.map(FileChannel.MapMode.READ_ONLY, indexOffset, dataOffset - indexOffset); + long fileSize = file.length(); //Create data buffers dataSize = fileSize - dataOffset; //Check if data size fits in memory map limit - if (!config.getBoolean(Configuration.MMAP_DATA_ENABLED)) { - //Use classical disk read - mMapData = false; - dataBuffers = null; - } else { - //Use Mmap - mMapData = true; - - //Build data buffers - int bufArraySize = (int) (dataSize / segmentSize) + ((dataSize % segmentSize != 0) ? 1 : 0); - dataBuffers = new MappedByteBuffer[bufArraySize]; - int bufIdx = 0; - for (long offset = 0; offset < dataSize; offset += segmentSize) { - long remainingFileSize = dataSize - offset; - long thisSegmentSize = Math.min(segmentSize, remainingFileSize); - dataBuffers[bufIdx++] = channel.map(FileChannel.MapMode.READ_ONLY, dataOffset + offset, thisSegmentSize); - } - } + mMapData = configuration.getBoolean(Configuration.MMAP_DATA_ENABLED); + context = mMapData ? + ThreadLocal.withInitial(() -> new ThreadContext( + initIndexBuffer(channel, indexOffset), + initDataBuffers(channel), + new byte[maxSlotSize])) : + ThreadLocal.withInitial(() -> new ThreadContext(initIndexBuffer(channel, indexOffset), new byte[maxSlotSize])); //logging if (log.isDebugEnabled()) { @@ -214,39 +186,59 @@ public class StorageReader implements Iterable> { } statMsg.append(" Index size: ").append(integerFormat.format((dataOffset - indexOffset) / (1024.0 * 1024.0))).append(" Mb\n"); statMsg.append(" Data size: ").append(integerFormat.format((fileSize - dataOffset) / (1024.0 * 1024.0))).append(" Mb\n"); - if (mMapData) { - statMsg.append(" Number of memory mapped data buffers: ").append(dataBuffers.length); - } else { - statMsg.append(" Memory mapped data disabled, using disk"); - } log.debug(statMsg.toString()); } } + private MappedByteBuffer initIndexBuffer(FileChannel channel, int indexOffset) { + try { + return channel.map(FileChannel.MapMode.READ_ONLY, indexOffset, dataOffset - indexOffset); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private MappedByteBuffer[] initDataBuffers(FileChannel channel) { + int bufArraySize = (int) (dataSize / segmentSize) + ((dataSize % segmentSize != 0) ? 1 : 0); + MappedByteBuffer[] result = new MappedByteBuffer[bufArraySize]; + int bufIdx = 0; + for (long offset = 0; offset < dataSize; offset += segmentSize) { + long remainingFileSize = dataSize - offset; + long thisSegmentSize = Math.min(segmentSize, remainingFileSize); + try { + result[bufIdx++] = channel.map(FileChannel.MapMode.READ_ONLY, dataOffset + offset, thisSegmentSize); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return result; + } + //Get the value for the given key or null - public byte[] get(byte[] key) - throws IOException { + public byte[] get(byte[] key) throws IOException { int keyLength = key.length; if (keyLength >= slots.length || keyCounts[keyLength] == 0) { return null; } - long hash = (long) hashUtils.hash(key); int numSlots = slots[keyLength]; int slotSize = slotSizes[keyLength]; - int indexOffset = indexOffsets[keyLength]; - long dataOffset = dataOffsets[keyLength]; + int ixOffset = indexOffsets[keyLength]; + long dtOffset = dataOffsets[keyLength]; + + var ctx = context.get(); + long hash = ctx.hash(key); for (int probe = 0; probe < numSlots; probe++) { int slot = (int) ((hash + probe) % numSlots); - indexBuffer.position(indexOffset + slot * slotSize); - indexBuffer.get(slotBuffer, 0, slotSize); + ctx.indexBuffer.position(ixOffset + slot * slotSize); + ctx.indexBuffer.get(ctx.slotBuffer, 0, slotSize); - long offset = LongPacker.unpackLong(slotBuffer, keyLength); - if (offset == 0) { + long offset = LongPacker.unpackLong(ctx.slotBuffer, keyLength); + if (offset == 0L) { return null; } - if (isKey(slotBuffer, key)) { - return mMapData ? getMMapBytes(dataOffset + offset) : getDiskBytes(dataOffset + offset); + if (isKey(ctx.slotBuffer, key)) { + return mMapData ? getMMapBytes(dtOffset + offset, ctx) : getDiskBytes(dtOffset + offset); } } return null; @@ -261,15 +253,36 @@ private boolean isKey(byte[] slotBuffer, byte[] key) { return true; } + private static final class ThreadContext { + + private final HashUtils.Murmur3A hash = new HashUtils.Murmur3A(42); + private final MappedByteBuffer indexBuffer; + private final MappedByteBuffer[] dataBuffers; + private final DataInputOutput sizeBuffer = new DataInputOutput(new byte[5]); + private final byte[] slotBuffer; + + private ThreadContext(MappedByteBuffer indexBuffer, byte[] slotBuffer) { + this(indexBuffer, null, slotBuffer); + } + + private ThreadContext(MappedByteBuffer indexBuffer, MappedByteBuffer[] dataBuffers, byte[] slotBuffer) { + this.indexBuffer = indexBuffer; + this.dataBuffers = dataBuffers; + this.slotBuffer = slotBuffer; + } + + public int hash(byte[] bytes) { + hash.reset(); + hash.update(bytes); + return hash.getIntValue() & 0x7fffffff; + } + } + //Close the reader channel - public void close() - throws IOException { + public void close() throws IOException { channel.close(); mappedFile.close(); - indexBuffer = null; - dataBuffers = null; - mappedFile = null; - channel = null; + context.remove(); } public int getKeyCount() { @@ -277,10 +290,9 @@ public int getKeyCount() { } //Read the data at the given offset, the data can be spread over multiple data buffers - private byte[] getMMapBytes(long offset) - throws IOException { + private byte[] getMMapBytes(long offset, ThreadContext ctx) throws IOException { //Read the first 4 bytes to get the size of the data - ByteBuffer buf = getDataBuffer(offset); + ByteBuffer buf = getDataBuffer(offset, ctx); int maxLen = (int) Math.min(5, dataSize - offset); int size; @@ -295,17 +307,17 @@ private byte[] getMMapBytes(long offset) //The size of the data is spread over multiple buffers int len = maxLen; int off = 0; - sizeBuffer.reset(); + ctx.sizeBuffer.reset(); while (len > 0) { - buf = getDataBuffer(offset + off); + buf = getDataBuffer(offset + off, ctx); int count = Math.min(len, buf.remaining()); - buf.get(sizeBuffer.getBuf(), off, count); + buf.get(ctx.sizeBuffer.getBuf(), off, count); off += count; len -= count; } - size = LongPacker.unpackInt(sizeBuffer); - offset += sizeBuffer.getPos(); - buf = getDataBuffer(offset); + size = LongPacker.unpackInt(ctx.sizeBuffer); + offset += ctx.sizeBuffer.getPos(); + buf = getDataBuffer(offset, ctx); } //Create output bytes @@ -319,7 +331,7 @@ private byte[] getMMapBytes(long offset) int len = size; int off = 0; while (len > 0) { - buf = getDataBuffer(offset); + buf = getDataBuffer(offset, ctx); int count = Math.min(len, buf.remaining()); buf.get(res, off, count); offset += count; @@ -332,7 +344,7 @@ private byte[] getMMapBytes(long offset) } //Get data from disk - private byte[] getDiskBytes(long offset) + private synchronized byte[] getDiskBytes(long offset) throws IOException { mappedFile.seek(dataOffset + offset); @@ -351,8 +363,8 @@ private byte[] getDiskBytes(long offset) } //Return the data buffer for the given position - private ByteBuffer getDataBuffer(long index) { - ByteBuffer buf = dataBuffers[(int) (index / segmentSize)]; + private ByteBuffer getDataBuffer(long index, ThreadContext ctx) { + ByteBuffer buf = ctx.dataBuffers[(int) (index / segmentSize)]; buf.position((int) (index % segmentSize)); return buf; } @@ -412,11 +424,12 @@ public boolean hasNext() { @Override public FastEntry next() { try { - indexBuffer.position(currentIndexOffset); + var ctx = context.get(); + ctx.indexBuffer.position(currentIndexOffset); long offset = 0; while (offset == 0) { - indexBuffer.get(currentSlotBuffer); + ctx.indexBuffer.get(currentSlotBuffer); offset = LongPacker.unpackLong(currentSlotBuffer, currentKeyLength); currentIndexOffset += currentSlotBuffer.length; } @@ -426,7 +439,7 @@ public FastEntry next() { if (withValue) { long valueOffset = currentDataOffset + offset; - value = mMapData ? getMMapBytes(valueOffset) : getDiskBytes(valueOffset); + value = mMapData ? getMMapBytes(valueOffset, ctx) : getDiskBytes(valueOffset); } entry.set(key, value); diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java index 6a1824c..5bb6b57 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java @@ -54,13 +54,13 @@ public StorageSerialization(Configuration config) { * @return key as byte array * @throws IOException if an io error occurs */ - public byte[] serializeKey(Object key) - throws IOException { + public byte[] serializeKey(Object key) throws IOException { if (key == null) { throw new NullPointerException(); } - serializeObject(key, dataInputOutput.reset(), false); - return dataInputOutput.toByteArray(); + var dataIO = new DataInputOutput(); + serializeObject(key, dataIO, false); + return dataIO.toByteArray(); } /** diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index 7c9dea7..e482e8d 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -247,8 +247,8 @@ private void writeMetadata(DataOutputStream dataOutputStream) //Write serializers try { Serializers.serialize(dataOutputStream, config.getSerializers()); - } catch (Exception e) { - throw new RuntimeException(); + } catch (IOException e) { + throw new UncheckedIOException(e); } //Write the position of the index and the data diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java b/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java index bcc707b..8dfd282 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java @@ -42,7 +42,7 @@ public int hash(byte[] bytes) { * * Originally developed for greenrobot by Markus Junginger. */ - private static class Murmur3A implements Checksum { + public static class Murmur3A implements Checksum { private static final int C1 = 0xcc9e2d51; private static final int C2 = 0x1b873593; diff --git a/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java b/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java index 39f1441..891cbdb 100644 --- a/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java +++ b/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java @@ -14,16 +14,13 @@ package com.linkedin.paldb.api; +import org.testng.annotations.Test; + import java.awt.*; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; +import java.io.*; import java.util.Arrays; -import org.testng.Assert; -import org.testng.annotations.Test; + +import static org.testng.Assert.*; public class TestConfiguration { @@ -32,8 +29,8 @@ public class TestConfiguration { public void testConfiguration() { Configuration c = new Configuration(); c.set("foo", "bar"); - Assert.assertEquals(c.get("foo", null), "bar"); - Assert.assertEquals(c.get("bar", "foo"), "foo"); + assertEquals(c.get("foo", null), "bar"); + assertEquals(c.get("bar", "foo"), "foo"); } @Test @@ -42,10 +39,10 @@ public void testConfigurationCopy() { c.set("foo", "bar"); Configuration r = new Configuration(c); - Assert.assertEquals(r.get("foo", null), "bar"); + assertEquals(r.get("foo", null), "bar"); c.set("foo", ""); - Assert.assertEquals(r.get("foo", null), "bar"); + assertEquals(r.get("foo", null), "bar"); } @Test(expectedExceptions = UnsupportedOperationException.class) @@ -59,7 +56,7 @@ public void testConfigurationReadOnly() { @Test public void testEqualsEmpty() { - Assert.assertEquals(new Configuration(), new Configuration()); + assertEquals(new Configuration(), new Configuration()); } @Test @@ -73,8 +70,8 @@ public void testEquals() { Configuration c3 = new Configuration(); c3.set("foo", "notbar"); - Assert.assertEquals(c1, c2); - Assert.assertNotEquals(c1, c3); + assertEquals(c1, c2); + assertNotEquals(c1, c3); } @Test @@ -83,8 +80,8 @@ public void testGetBoolean() { c.set("foo", "true"); c.set("bar", "false"); - Assert.assertTrue(c.getBoolean("foo")); - Assert.assertFalse(c.getBoolean("bar")); + assertTrue(c.getBoolean("foo")); + assertFalse(c.getBoolean("bar")); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -97,8 +94,8 @@ public void testGetBooleanDefault() { Configuration c = new Configuration(); c.set("foo", "true"); - Assert.assertTrue(c.getBoolean("foo", false)); - Assert.assertTrue(c.getBoolean("bar", true)); + assertTrue(c.getBoolean("foo", false)); + assertTrue(c.getBoolean("bar", true)); } @Test @@ -106,7 +103,7 @@ public void testGetDouble() { Configuration c = new Configuration(); c.set("foo", "1.0"); - Assert.assertEquals(c.getDouble("foo"), 1.0); + assertEquals(c.getDouble("foo"), 1.0); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -119,8 +116,8 @@ public void testGetDoubleDefault() { Configuration c = new Configuration(); c.set("foo", "1.0"); - Assert.assertEquals(c.getDouble("foo", 2.0), 1.0); - Assert.assertEquals(c.getDouble("bar", 2.0), 2.0); + assertEquals(c.getDouble("foo", 2.0), 1.0); + assertEquals(c.getDouble("bar", 2.0), 2.0); } @Test @@ -128,7 +125,7 @@ public void testGetFloat() { Configuration c = new Configuration(); c.set("foo", "1.0"); - Assert.assertEquals(c.getFloat("foo"), 1f); + assertEquals(c.getFloat("foo"), 1f); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -141,8 +138,8 @@ public void testGetFloatDefault() { Configuration c = new Configuration(); c.set("foo", "1.0"); - Assert.assertEquals(c.getFloat("foo", 2f), 1f); - Assert.assertEquals(c.getFloat("bar", 2f), 2f); + assertEquals(c.getFloat("foo", 2f), 1f); + assertEquals(c.getFloat("bar", 2f), 2f); } @Test @@ -150,7 +147,7 @@ public void testGetInt() { Configuration c = new Configuration(); c.set("foo", "1"); - Assert.assertEquals(c.getInt("foo"), 1); + assertEquals(c.getInt("foo"), 1); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -163,8 +160,8 @@ public void testGetIntDefault() { Configuration c = new Configuration(); c.set("foo", "1"); - Assert.assertEquals(c.getInt("foo", 2), 1); - Assert.assertEquals(c.getInt("bar", 2), 2); + assertEquals(c.getInt("foo", 2), 1); + assertEquals(c.getInt("bar", 2), 2); } @Test @@ -172,7 +169,7 @@ public void testGetShort() { Configuration c = new Configuration(); c.set("foo", "1"); - Assert.assertEquals(c.getShort("foo"), (short) 1); + assertEquals(c.getShort("foo"), (short) 1); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -185,8 +182,8 @@ public void testGetShortDefault() { Configuration c = new Configuration(); c.set("foo", "1"); - Assert.assertEquals(c.getShort("foo", (short) 2), (short) 1); - Assert.assertEquals(c.getShort("bar", (short) 2), (short) 2); + assertEquals(c.getShort("foo", (short) 2), (short) 1); + assertEquals(c.getShort("bar", (short) 2), (short) 2); } @Test @@ -194,7 +191,7 @@ public void testGetLong() { Configuration c = new Configuration(); c.set("foo", "1"); - Assert.assertEquals(c.getLong("foo"), 1l); + assertEquals(c.getLong("foo"), 1l); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -207,8 +204,8 @@ public void testGetLongDefault() { Configuration c = new Configuration(); c.set("foo", "1"); - Assert.assertEquals(c.getLong("foo", 2l), 1l); - Assert.assertEquals(c.getLong("bar", 2l), 2l); + assertEquals(c.getLong("foo", 2l), 1l); + assertEquals(c.getLong("bar", 2l), 2l); } @Test @@ -217,7 +214,7 @@ public void testGetClass() Configuration c = new Configuration(); c.set("foo", Integer.class.getName()); - Assert.assertEquals(c.getClass("foo"), Integer.class); + assertEquals(c.getClass("foo"), Integer.class); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -231,7 +228,7 @@ public void testGetList() { Configuration c = new Configuration(); c.set("foo", "foo,bar"); - Assert.assertEquals(c.getList("foo"), Arrays.asList("foo", "bar")); + assertEquals(c.getList("foo"), Arrays.asList("foo", "bar")); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -244,8 +241,8 @@ public void testGetListDefault() { Configuration c = new Configuration(); c.set("foo", "foo,bar"); - Assert.assertEquals(c.getList("foo", Arrays.asList("that")), Arrays.asList("foo", "bar")); - Assert.assertEquals(c.getList("bar", Arrays.asList("that")), Arrays.asList("that")); + assertEquals(c.getList("foo", Arrays.asList("that")), Arrays.asList("foo", "bar")); + assertEquals(c.getList("bar", Arrays.asList("that")), Arrays.asList("that")); } @Test @@ -268,7 +265,7 @@ public void testSerialization() in.close(); bis.close(); - Assert.assertEquals(sc, c); + assertEquals(sc, c); } // UTILITY diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java index daab9b9..a102bd9 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java @@ -16,14 +16,12 @@ import com.linkedin.paldb.api.Serializer; import com.linkedin.paldb.utils.DataInputOutput; +import org.testng.annotations.*; import java.awt.*; -import java.io.DataInput; -import java.io.DataOutput; +import java.io.*; -import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; +import static org.testng.Assert.*; public class TestSerializers { @@ -39,8 +37,8 @@ public void setUp() { public void testRegister() { ColorSerializer i = new ColorSerializer(); _serializers.registerSerializer(i); - Assert.assertSame(_serializers.getSerializer(Color.class), i); - Assert.assertEquals(_serializers.getIndex(Color.class), 0); + assertSame(_serializers.getSerializer(Color.class), i); + assertEquals(_serializers.getIndex(Color.class), 0); } @Test @@ -49,7 +47,7 @@ public void testRegisterTwice() { ColorSerializer i2 = new ColorSerializer(); _serializers.registerSerializer(i1); _serializers.registerSerializer(i2); - Assert.assertSame(_serializers.getSerializer(Color.class), i1); + assertSame(_serializers.getSerializer(Color.class), i1); } @Test @@ -58,32 +56,32 @@ public void testRegisterTwo() { PointSerializer f = new PointSerializer(); _serializers.registerSerializer(i); _serializers.registerSerializer(f); - Assert.assertSame(_serializers.getSerializer(Color.class), i); - Assert.assertEquals(_serializers.getIndex(Color.class), 0); - Assert.assertSame(_serializers.getSerializer(Point.class), f); - Assert.assertEquals(_serializers.getIndex(Point.class), 1); + assertSame(_serializers.getSerializer(Color.class), i); + assertEquals(_serializers.getIndex(Color.class), 0); + assertSame(_serializers.getSerializer(Point.class), f); + assertEquals(_serializers.getIndex(Point.class), 1); } @Test public void testGetSerializer() { ColorSerializer i = new ColorSerializer(); _serializers.registerSerializer(i); - Assert.assertNull(_serializers.getSerializer(Point.class)); - Assert.assertNotNull(_serializers.getSerializer(Color.class)); + assertNull(_serializers.getSerializer(Point.class)); + assertNotNull(_serializers.getSerializer(Color.class)); } @Test public void testGetIndex() { ColorSerializer i = new ColorSerializer(); _serializers.registerSerializer(i); - Assert.assertEquals(_serializers.getIndex(Color.class), 0); + assertEquals(_serializers.getIndex(Color.class), 0); } @Test public void testGetByIndex() { ColorSerializer i = new ColorSerializer(); _serializers.registerSerializer(i); - Assert.assertSame(_serializers.getSerializer(0), i); + assertSame(_serializers.getSerializer(0), i); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -112,16 +110,16 @@ public void testSerialize() throws Throwable { dio = new DataInputOutput(bytes); _serializers.clear(); Serializers.deserialize(dio, _serializers); - Assert.assertNotNull(_serializers.getSerializer(Color.class)); - Assert.assertEquals(_serializers.getIndex(Color.class), 0); - Assert.assertNotNull(_serializers.getSerializer(0)); + assertNotNull(_serializers.getSerializer(Color.class)); + assertEquals(_serializers.getIndex(Color.class), 0); + assertNotNull(_serializers.getSerializer(0)); } @Test public void testInterfaceType() throws Throwable { SerializerWithInterface i = new SerializerWithInterface(); _serializers.registerSerializer(i); - Assert.assertSame(_serializers.getSerializer(AnInterface.class), i); + assertSame(_serializers.getSerializer(AnInterface.class), i); } // HELPER diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java index c8874c9..051792e 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java @@ -15,9 +15,10 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.*; -import org.testng.Assert; import org.testng.annotations.*; +import static org.testng.Assert.*; + public class TestStorageCache { @@ -35,28 +36,28 @@ public void setUp() { public void testContainsValid() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 0); - Assert.assertTrue(cache.contains(0)); + assertTrue(cache.contains(0)); } @Test public void testContainsInValid() { StorageCache cache = StorageCache.initCache(configuration); - Assert.assertFalse(cache.contains(0)); + assertFalse(cache.contains(0)); } @Test public void testEmpty() { StorageCache cache = StorageCache.initCache(configuration); - Assert.assertNull(cache.get(0)); - Assert.assertEquals(cache.size(), 0); + assertNull(cache.get(0)); + assertEquals(cache.size(), 0); } @Test public void testPutOneItem() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 0); - Assert.assertNotNull(cache.get(0)); - Assert.assertEquals(cache.size(), 1); + assertNotNull(cache.get(0)); + assertEquals(cache.size(), 1); } @Test @@ -65,7 +66,7 @@ public void testPutTwice() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 1); cache.put(0, second); - Assert.assertSame(cache.get(0), second); + assertSame(cache.get(0), second); } @Test @@ -73,7 +74,7 @@ public void testPutZeroSize() { StorageCache cache = StorageCache.initCache(configuration); cache.setMaxWeight(0); cache.put(0, 1); - Assert.assertEquals(cache.size(), 0); + assertEquals(cache.size(), 0); } @Test @@ -82,9 +83,9 @@ public void testPutTwiceObjectSize() { cache.setMaxWeight(ENTRY_SIZE); cache.put(0, 0); cache.put(1, 1); - Assert.assertEquals(cache.size(), 1); - Assert.assertNull(cache.get(0)); - Assert.assertNotNull(cache.get(1)); + assertEquals(cache.size(), 1); + assertNull(cache.get(0)); + assertNotNull(cache.get(1)); } @Test @@ -93,7 +94,7 @@ public void putSameCheckWeight() { cache.put(0, 0); long weight = cache.getWeight(); cache.put(0, 0); - Assert.assertEquals(cache.getWeight(), weight); + assertEquals(cache.getWeight(), weight); } @Test @@ -104,9 +105,9 @@ public void testPutGet() { for (int i = 0; i < objs; i++) { cache.put(i, i); } - Assert.assertEquals(cache.size(), 100); + assertEquals(cache.size(), 100); for (int i = 0; i < objs; i++) { - Assert.assertNotNull(cache.get(i)); + assertNotNull(cache.get(i)); } } @@ -119,12 +120,12 @@ public void testCheckOrder() { for (int i = 0; i < objs; i++) { cache.put(i, i); } - Assert.assertEquals(cache.size(), capacity); + assertEquals(cache.size(), capacity); for (int i = 0; i < objs; i++) { if (i < capacity) { - Assert.assertNull(cache.get(i)); + assertNull(cache.get(i)); } else { - Assert.assertNotNull(cache.get(i)); + assertNotNull(cache.get(i)); } } } @@ -137,10 +138,10 @@ public void testCheckAccessOrderGet() { cache.put(1, 1); cache.get(0); cache.put(2, 2); - Assert.assertEquals(cache.size(), 3); + assertEquals(cache.size(), 3); cache.put(3, 2); - Assert.assertNull(cache.get(1)); - Assert.assertNotNull(cache.get(0)); + assertNull(cache.get(1)); + assertNotNull(cache.get(0)); } @Test @@ -151,123 +152,123 @@ public void testCheckAccessOrderPut() { cache.put(1, 1); cache.put(0, 0); cache.put(2, 2); - Assert.assertEquals(cache.size(), 3); + assertEquals(cache.size(), 3); cache.put(3, 2); - Assert.assertNull(cache.get(1)); - Assert.assertNotNull(cache.get(0)); + assertNull(cache.get(1)); + assertNotNull(cache.get(0)); } @Test public void testWeightKeyObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, 0); - Assert.assertEquals(cache.getWeight(), ENTRY_SIZE); + assertEquals(cache.getWeight(), ENTRY_SIZE); } @Test public void testWeightKeyArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(new Object[]{0, 1}, 0); - Assert.assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); } @Test public void testWeightValueIntArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new int[]{1, 2}); - Assert.assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); } @Test public void testWeightValueLongArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new long[]{1, 2}); - Assert.assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); } @Test public void testWeightValueDoubleArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new double[]{1.0, 2.0}); - Assert.assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); } @Test public void testWeightValueFloatArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new float[]{1.0F, 2.0F}); - Assert.assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); } @Test public void testWeightValueBooleanArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new boolean[]{true, false}); - Assert.assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueByteArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new byte[]{1, 2}); - Assert.assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueShortArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new short[]{1, 2}); - Assert.assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); } @Test public void testWeightValueCharArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new char[]{'a', 'b'}); - Assert.assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); } @Test public void testWeightValueStringArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new String[]{"one", "two"}); - Assert.assertEquals(cache.getWeight(), 16 + 46 * 2 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 46 * 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueInt2DArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new int[][]{{1, 2}, {3, 4}}); - Assert.assertEquals(cache.getWeight(), 16 + 8 * 2 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 8 * 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueLong2DArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new long[][]{{1, 2}, {3, 4}}); - Assert.assertEquals(cache.getWeight(), 16 + 16 * 2 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 16 * 2 + StorageCache.OVERHEAD); } @Test public void testWeightValueStringObject() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new String("one")); - Assert.assertEquals(cache.getWeight(), 16 + 46 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 46 + StorageCache.OVERHEAD); } @Test public void testWeightValueObjectArrayObjects() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, new Object[]{0, 1}); - Assert.assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); + assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); } @Test public void testNullValue() { StorageCache cache = StorageCache.initCache(configuration); cache.put(0, null); - Assert.assertEquals(cache.size(), 1); - Assert.assertEquals(cache.get(0), StorageCache.NULL_VALUE); + assertEquals(cache.size(), 1); + assertEquals(cache.get(0), StorageCache.NULL_VALUE); } @Test @@ -275,9 +276,9 @@ public void testDisabled() { Configuration cfg = new Configuration(); cfg.set(Configuration.CACHE_ENABLED, "false"); StorageCache cache = StorageCache.initCache(cfg); - Assert.assertEquals(cache.size(), 0); - Assert.assertNull(cache.get("foo")); - Assert.assertFalse(cache.contains("foo")); + assertEquals(cache.size(), 0); + assertNull(cache.get("foo")); + assertFalse(cache.contains("foo")); } @Test @@ -286,8 +287,8 @@ public void testDisabledPut() { cfg.set(Configuration.CACHE_ENABLED, "false"); StorageCache cache = StorageCache.initCache(cfg); cache.put(0, "foo"); - Assert.assertEquals(cache.size(), 0); - Assert.assertNull(cache.get(1)); - Assert.assertFalse(cache.contains(2)); + assertEquals(cache.size(), 0); + assertNull(cache.get(1)); + assertFalse(cache.contains(2)); } } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java index d7dfe4b..b360adf 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java @@ -14,25 +14,16 @@ package com.linkedin.paldb.impl; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.Serializer; -import com.linkedin.paldb.api.UnsupportedTypeException; +import com.linkedin.paldb.api.*; +import org.testng.Assert; +import org.testng.annotations.*; import java.awt.*; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; +import java.io.*; +import java.math.*; import java.util.Arrays; -import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; +import static org.testng.Assert.*; public class TestStorageSerialization { @@ -48,18 +39,18 @@ public void setUp() { @Test public void testCompressionEnabled() { - Assert.assertFalse(serialization.isCompressionEnabled()); + assertFalse(serialization.isCompressionEnabled()); Configuration config = new Configuration(); config.set(Configuration.COMPRESSION_ENABLED, "true"); StorageSerialization s = new StorageSerialization(config); - Assert.assertTrue(s.isCompressionEnabled()); + assertTrue(s.isCompressionEnabled()); } @Test public void testSerializeKey() throws IOException, ClassNotFoundException { Integer l = 1; Object d = serialization.deserialize(serialization.serializeKey(l)); - Assert.assertEquals(d, l); + assertEquals(d, l); } @Test @@ -73,7 +64,7 @@ public void testSerializeKeyDataOutput() throws IOException, ClassNotFoundExcept ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInputStream dis = new DataInputStream(bis); - Assert.assertEquals(serialization.deserialize(dis), l); + assertEquals(serialization.deserialize(dis), l); } @Test @@ -87,7 +78,7 @@ public void testSerializeValueDataOutput() throws IOException, ClassNotFoundExce ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInputStream dis = new DataInputStream(bis); - Assert.assertEquals(serialization.deserialize(dis), l); + assertEquals(serialization.deserialize(dis), l); } @Test(expectedExceptions = NullPointerException.class) @@ -100,7 +91,7 @@ public void testTransformValue() throws ClassNotFoundException, IOException { Integer l = 1; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - Assert.assertEquals(deserialize, l); + assertEquals(deserialize, l); } @Test @@ -108,8 +99,8 @@ public void testTransformList() throws ClassNotFoundException, IOException { Integer[] l = new Integer[]{1, 2}; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - Assert.assertEquals(deserialize.getClass(), int[].class); - Assert.assertEquals(deserialize, new int[]{1, 2}); + assertEquals(deserialize.getClass(), int[].class); + assertEquals(deserialize, new int[]{1, 2}); } @Test @@ -117,8 +108,8 @@ public void testTransformListWithNull() throws ClassNotFoundException, IOException { Integer[] l = new Integer[]{1, null, 2}; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - Assert.assertEquals(deserialize.getClass(), int[].class); - Assert.assertEquals(deserialize, new int[]{1, 0, 2}); + assertEquals(deserialize.getClass(), int[].class); + assertEquals(deserialize, new int[]{1, 0, 2}); } @Test @@ -126,8 +117,8 @@ public void testTransformListOfList() throws ClassNotFoundException, IOException { Integer[][] l = new Integer[][]{{1}, {2}}; Object deserialize = serialization.deserialize(serialization.serializeValue(l)); - Assert.assertEquals(deserialize.getClass(), int[][].class); - Assert.assertEquals(deserialize, new int[][]{{1}, {2}}); + assertEquals(deserialize.getClass(), int[][].class); + assertEquals(deserialize, new int[][]{{1}, {2}}); } @Test @@ -154,7 +145,7 @@ public int getWeight(Point instance) { }); Point p = new Point(42, 9); byte[] buf = serialization.serialize(p); - Assert.assertEquals(serialization.deserialize(buf), p); + assertEquals(serialization.deserialize(buf), p); } @Test @@ -181,7 +172,7 @@ public int getWeight(Point[] instance) { }); Point[] p = new Point[]{new Point(42, 9)}; byte[] buf = serialization.serialize(p); - Assert.assertEquals(serialization.deserialize(buf), p); + assertEquals(serialization.deserialize(buf), p); } @Test @@ -205,7 +196,7 @@ public int getWeight(ImplementsA instance) { }); ImplementsA a = new ImplementsA(42); byte[] buf = serialization.serialize(a); - Assert.assertEquals(serialization.deserialize(buf), a); + assertEquals(serialization.deserialize(buf), a); } @Test @@ -229,7 +220,7 @@ public int getWeight(A instance) { }); ImplementsA a = new ImplementsA(42); byte[] buf = serialization.serialize(a); - Assert.assertEquals(serialization.deserialize(buf), a); + assertEquals(serialization.deserialize(buf), a); } @Test @@ -246,8 +237,8 @@ public void testByte() for (byte val : vals) { byte[] buf = serialization.serialize(val); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Byte.class); - Assert.assertEquals(l2, val); + assertTrue(l2.getClass() == Byte.class); + assertEquals(l2, val); } } @@ -267,8 +258,8 @@ public void testInt() for (int i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Integer.class); - Assert.assertEquals(l2, i); + assertTrue(l2.getClass() == Integer.class); + assertEquals(l2, i); } } @@ -281,8 +272,8 @@ public void testShort() for (short i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Short.class); - Assert.assertEquals(l2, i); + assertTrue(l2.getClass() == Short.class); + assertEquals(l2, i); } } @@ -293,8 +284,8 @@ public void testDouble() for (double i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Double.class); - Assert.assertEquals(l2, i); + assertTrue(l2.getClass() == Double.class); + assertEquals(l2, i); } } @@ -305,8 +296,8 @@ public void testFloat() for (float i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Float.class); - Assert.assertEquals(l2, i); + assertTrue(l2.getClass() == Float.class); + assertEquals(l2, i); } } @@ -317,8 +308,8 @@ public void testChar() for (char i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Character.class); - Assert.assertEquals(l2, i); + assertTrue(l2.getClass() == Character.class); + assertEquals(l2, i); } } @@ -334,8 +325,8 @@ public void testLong() for (long i : vals) { byte[] buf = serialization.serialize(i); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Long.class); - Assert.assertEquals(l2, i); + assertTrue(l2.getClass() == Long.class); + assertEquals(l2, i); } } @@ -344,13 +335,13 @@ public void testBoolean() throws IOException, ClassNotFoundException { byte[] buf = serialization.serialize(true); Object l2 = serialization.deserialize(buf); - Assert.assertTrue(l2.getClass() == Boolean.class); - Assert.assertEquals(l2, true); + assertTrue(l2.getClass() == Boolean.class); + assertEquals(l2, true); byte[] buf2 = serialization.serialize(false); Object l22 = serialization.deserialize(buf2); - Assert.assertTrue(l22.getClass() == Boolean.class); - Assert.assertEquals(l22, false); + assertTrue(l22.getClass() == Boolean.class); + assertEquals(l22, false); } @Test @@ -358,7 +349,7 @@ public void testString() throws IOException, ClassNotFoundException { byte[] buf = serialization.serialize("Abcd"); String l2 = (String) serialization.deserialize(buf); - Assert.assertEquals(l2, "Abcd"); + assertEquals(l2, "Abcd"); } @Test @@ -366,7 +357,7 @@ public void testEmptyString() throws IOException, ClassNotFoundException { byte[] buf = serialization.serialize(""); String l2 = (String) serialization.deserialize(buf); - Assert.assertEquals(l2, ""); + assertEquals(l2, ""); } @Test @@ -378,7 +369,7 @@ public void testBigString() } byte[] buf = serialization.serialize(bigString); String l2 = (String) serialization.deserialize(buf); - Assert.assertEquals(l2, bigString); + assertEquals(l2, bigString); } @Test @@ -386,7 +377,7 @@ public void testClass() throws IOException, ClassNotFoundException { byte[] buf = serialization.serialize(String.class); Class l2 = (Class) serialization.deserialize(buf); - Assert.assertEquals(l2, String.class); + assertEquals(l2, String.class); } @Test @@ -394,7 +385,7 @@ public void testClass2() throws IOException, ClassNotFoundException { byte[] buf = serialization.serialize(long[].class); Class l2 = (Class) serialization.deserialize(buf); - Assert.assertEquals(l2, long[].class); + assertEquals(l2, long[].class); } @Test @@ -403,7 +394,7 @@ public void testUnicodeString() String s = "Ciudad Bolíva"; byte[] buf = serialization.serialize(s); Object l2 = serialization.deserialize(buf); - Assert.assertEquals(l2, s); + assertEquals(l2, s); } @Test @@ -411,7 +402,7 @@ public void testStringArray() throws ClassNotFoundException, IOException { String[] l = new String[]{"foo", "bar", ""}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (String[]) deserialize)); + assertTrue(Arrays.equals(l, (String[]) deserialize)); } @Test @@ -419,7 +410,7 @@ public void testObjectArray() throws ClassNotFoundException, IOException { Object[] l = new Object[]{"foo", 2, Boolean.TRUE}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (Object[]) deserialize)); + assertTrue(Arrays.equals(l, (Object[]) deserialize)); } @Test @@ -427,7 +418,7 @@ public void testBooleanArray() throws ClassNotFoundException, IOException { boolean[] l = new boolean[]{true, false}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (boolean[]) deserialize)); + assertTrue(Arrays.equals(l, (boolean[]) deserialize)); } @Test @@ -435,7 +426,7 @@ public void testDoubleArray() throws ClassNotFoundException, IOException { double[] l = new double[]{Math.PI, 1D}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (double[]) deserialize)); + assertTrue(Arrays.equals(l, (double[]) deserialize)); } @Test @@ -443,7 +434,7 @@ public void testFloatArray() throws ClassNotFoundException, IOException { float[] l = new float[]{1F, 1.234235F}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (float[]) deserialize)); + assertTrue(Arrays.equals(l, (float[]) deserialize)); } @Test @@ -451,7 +442,7 @@ public void testByteArray() throws ClassNotFoundException, IOException { byte[] l = new byte[]{1, 34, -5}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (byte[]) deserialize)); + assertTrue(Arrays.equals(l, (byte[]) deserialize)); } @Test @@ -459,7 +450,7 @@ public void testShortArray() throws ClassNotFoundException, IOException { short[] l = new short[]{1, 345, -5000}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (short[]) deserialize)); + assertTrue(Arrays.equals(l, (short[]) deserialize)); } @Test @@ -467,7 +458,7 @@ public void testCharArray() throws ClassNotFoundException, IOException { char[] l = new char[]{'1', 'a', '&'}; Object deserialize = serialization.deserialize(serialization.serialize(l)); - Assert.assertTrue(Arrays.equals(l, (char[]) deserialize)); + assertTrue(Arrays.equals(l, (char[]) deserialize)); } @Test @@ -476,7 +467,7 @@ public void testIntArray() int[][] l = new int[][]{{3, 5}, {-1200, 29999}, {3, 100000}, {-43999, 100000}}; for (int[] a : l) { Object deserialize = serialization.deserialize(serialization.serialize(a)); - Assert.assertTrue(Arrays.equals(a, (int[]) deserialize)); + assertTrue(Arrays.equals(a, (int[]) deserialize)); } } @@ -486,7 +477,7 @@ public void testLongArray() long[][] l = new long[][]{{3l, 5l}, {-1200l, 29999l}, {3l, 100000l}, {-43999l, 100000l}, {-123l, 12345678901234l}}; for (long[] a : l) { Object deserialize = serialization.deserialize(serialization.serialize(a)); - Assert.assertTrue(Arrays.equals(a, (long[]) deserialize)); + assertTrue(Arrays.equals(a, (long[]) deserialize)); } } @@ -495,7 +486,7 @@ public void testDoubleCompressedArray() throws ClassNotFoundException, IOException { double[] l = generateDoubleArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - Assert.assertTrue(Arrays.equals(l, (double[]) deserialize)); + assertTrue(Arrays.equals(l, (double[]) deserialize)); } @Test @@ -503,7 +494,7 @@ public void testFloatCompressedArray() throws ClassNotFoundException, IOException { float[] l = generateFloatArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - Assert.assertTrue(Arrays.equals(l, (float[]) deserialize)); + assertTrue(Arrays.equals(l, (float[]) deserialize)); } @Test @@ -511,7 +502,7 @@ public void testByteCompressedArray() throws ClassNotFoundException, IOException { byte[] l = generateByteArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - Assert.assertTrue(Arrays.equals(l, (byte[]) deserialize)); + assertTrue(Arrays.equals(l, (byte[]) deserialize)); } @Test @@ -519,7 +510,7 @@ public void testCharCompressedArray() throws ClassNotFoundException, IOException { char[] l = generateCharArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - Assert.assertTrue(Arrays.equals(l, (char[]) deserialize)); + assertTrue(Arrays.equals(l, (char[]) deserialize)); } @Test @@ -527,7 +518,7 @@ public void testShortCompressedArray() throws ClassNotFoundException, IOException { short[] l = generateShortArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - Assert.assertTrue(Arrays.equals(l, (short[]) deserialize)); + assertTrue(Arrays.equals(l, (short[]) deserialize)); } @Test @@ -535,7 +526,7 @@ public void testIntCompressedArray() throws ClassNotFoundException, IOException { int[] l = generateIntArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - Assert.assertTrue(Arrays.equals(l, (int[]) deserialize)); + assertTrue(Arrays.equals(l, (int[]) deserialize)); } @Test @@ -543,25 +534,25 @@ public void testLongCompressedArray() throws ClassNotFoundException, IOException { long[] l = generateLongArray(500); Object deserialize = serialization.deserialize(serialization.serialize(l, true)); - Assert.assertTrue(Arrays.equals(l, (long[]) deserialize)); + assertTrue(Arrays.equals(l, (long[]) deserialize)); } @Test public void testBigDecimal() throws IOException, ClassNotFoundException { BigDecimal d = new BigDecimal("445656.7889889895165654423236"); - Assert.assertEquals(d, serialization.deserialize(serialization.serialize(d))); + assertEquals(d, serialization.deserialize(serialization.serialize(d))); d = new BigDecimal("-53534534534534445656.7889889895165654423236"); - Assert.assertEquals(d, serialization.deserialize(serialization.serialize(d))); + assertEquals(d, serialization.deserialize(serialization.serialize(d))); } @Test public void testBigInteger() throws IOException, ClassNotFoundException { BigInteger d = new BigInteger("4456567889889895165654423236"); - Assert.assertEquals(d, serialization.deserialize(serialization.serialize(d))); + assertEquals(d, serialization.deserialize(serialization.serialize(d))); d = new BigInteger("-535345345345344456567889889895165654423236"); - Assert.assertEquals(d, serialization.deserialize(serialization.serialize(d))); + assertEquals(d, serialization.deserialize(serialization.serialize(d))); } @Test @@ -571,8 +562,8 @@ public void testMultiDimensionalIntArray() d[0] = new int[]{1, 3}; d[1] = new int[]{-3, 1}; Object res = serialization.deserialize(serialization.serialize(d)); - Assert.assertEquals(res.getClass(), int[][].class); - Assert.assertEquals(d, res); + assertEquals(res.getClass(), int[][].class); + assertEquals(d, res); } @Test @@ -582,8 +573,8 @@ public void testMultiDimensionalLongArray() d[0] = new long[]{1, 3}; d[1] = new long[]{-3, 1}; Object res = serialization.deserialize(serialization.serialize(d)); - Assert.assertEquals(res.getClass(), long[][].class); - Assert.assertEquals(d, res); + assertEquals(res.getClass(), long[][].class); + assertEquals(d, res); } // UTILITY diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java index 979e467..87dabd0 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -24,6 +24,7 @@ import java.util.*; import static com.linkedin.paldb.utils.TestTempUtils.deleteDirectory; +import static org.testng.Assert.*; public class TestStore { @@ -46,11 +47,11 @@ public void testEmpty() { StoreWriter writer = PalDB.createWriter(storeFile, new Configuration()); writer.close(); - Assert.assertTrue(storeFile.exists()); + assertTrue(storeFile.exists()); try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), 0); - Assert.assertNull(reader.get(1, null)); + assertEquals(reader.size(), 0); + assertNull(reader.get(1, null)); } } @@ -60,7 +61,7 @@ public void testEmptyStream() { StoreWriter writer = PalDB.createWriter(bos, new Configuration()); writer.close(); - Assert.assertTrue(bos.toByteArray().length > 0); + assertTrue(bos.toByteArray().length > 0); ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); StoreReader reader = PalDB.createReader(bis, new Configuration()); @@ -72,17 +73,17 @@ public void testEmptyDefaultConfig() { StoreWriter writer = PalDB.createWriter(storeFile); writer.close(); - Assert.assertTrue(storeFile.exists()); + assertTrue(storeFile.exists()); try (StoreReader reader = PalDB.createReader(storeFile)) { - Assert.assertEquals(reader.size(), 0); - Assert.assertNull(reader.get(1, null)); + assertEquals(reader.size(), 0); + assertNull(reader.get(1, null)); } } @Test public void testNewConfiguration() { - Assert.assertNotNull(PalDB.newConfiguration()); + assertNotNull(PalDB.newConfiguration()); } @Test @@ -92,7 +93,7 @@ public void testNoFolder() { StoreWriter writer = PalDB.createWriter(file, new Configuration()); writer.close(); - Assert.assertTrue(file.exists()); + assertTrue(file.exists()); } @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*not found.*") @@ -172,8 +173,8 @@ public void testByteMarkEmpty() throws IOException { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), 0); - Assert.assertNull(reader.get(1, null)); + assertEquals(reader.size(), 0); + assertNull(reader.get(1, null)); } } @@ -184,8 +185,8 @@ public void testOneKey() { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), 1); - Assert.assertEquals(reader.get(1), "foo"); + assertEquals(reader.size(), 1); + assertEquals(reader.get(1), "foo"); } } @@ -200,8 +201,8 @@ public void testPutSerializedKey() throws IOException { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), 1); - Assert.assertEquals(reader.get(1), "foo"); + assertEquals(reader.size(), 1); + assertEquals(reader.get(1), "foo"); } } @@ -217,8 +218,8 @@ public void testByteMarkOneKey() throws IOException { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), 1); - Assert.assertEquals(reader.get(1), "foo"); + assertEquals(reader.size(), 1); + assertEquals(reader.get(1), "foo"); } } @@ -236,13 +237,13 @@ public void testTwoFirstKeyLength() { //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.get(key1).intValue(), 1); - Assert.assertEquals(reader.get(key2).intValue(), 6); - Assert.assertNull(reader.get(0, null)); - Assert.assertNull(reader.get(6, null)); - Assert.assertNull(reader.get(244, null)); - Assert.assertNull(reader.get(246, null)); - Assert.assertNull(reader.get(1245, null)); + assertEquals(reader.get(key1).intValue(), 1); + assertEquals(reader.get(key2).intValue(), 6); + assertNull(reader.get(0, null)); + assertNull(reader.get(6, null)); + assertNull(reader.get(244, null)); + assertNull(reader.get(246, null)); + assertNull(reader.get(1245, null)); } } @@ -260,15 +261,15 @@ public void testKeyLengthGap() { //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.get(key1).intValue(), 1); - Assert.assertEquals(reader.get(key2).intValue(), 6); - Assert.assertNull(reader.get(0, null)); - Assert.assertNull(reader.get(6, null)); - Assert.assertNull(reader.get(244, null)); - Assert.assertNull(reader.get(267, null)); - Assert.assertNull(reader.get(2449, null)); - Assert.assertNull(reader.get(2451, null)); - Assert.assertNull(reader.get(2454441, null)); + assertEquals(reader.get(key1).intValue(), 1); + assertEquals(reader.get(key2).intValue(), 6); + assertNull(reader.get(0, null)); + assertNull(reader.get(6, null)); + assertNull(reader.get(244, null)); + assertNull(reader.get(267, null)); + assertNull(reader.get(2449, null)); + assertNull(reader.get(2451, null)); + assertNull(reader.get(2454441, null)); } } @@ -286,14 +287,14 @@ public void testKeyLengthStartTwo() { //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.get(key1).intValue(), 1); - Assert.assertEquals(reader.get(key2).intValue(), 6); - Assert.assertNull(reader.get(6, null)); - Assert.assertNull(reader.get(244, null)); - Assert.assertNull(reader.get(267, null)); - Assert.assertNull(reader.get(2449, null)); - Assert.assertNull(reader.get(2451, null)); - Assert.assertNull(reader.get(2454441, null)); + assertEquals(reader.get(key1).intValue(), 1); + assertEquals(reader.get(key2).intValue(), 6); + assertNull(reader.get(6, null)); + assertNull(reader.get(244, null)); + assertNull(reader.get(267, null)); + assertNull(reader.get(2449, null)); + assertNull(reader.get(2451, null)); + assertNull(reader.get(2454441, null)); } } @@ -322,7 +323,7 @@ public void testDataOnTwoBuffers() throws IOException { configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize - 100)); try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { for (int i = 0; i < keys.length; i++) { - Assert.assertEquals(reader.get((Integer) keys[i], null), values[i]); + assertEquals(reader.get((Integer) keys[i], null), values[i]); } } } @@ -348,7 +349,7 @@ public void testDataSizeOnTwoBuffers() throws IOException { configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize + sizeSize + 3)); try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { for (int i = 0; i < keys.length; i++) { - Assert.assertEquals(reader.get((Integer) keys[i], null), values[i]); + assertEquals(reader.get((Integer) keys[i], null), values[i]); } } } @@ -422,13 +423,13 @@ public void testReadDisk() { //Read configuration.set(Configuration.MMAP_DATA_ENABLED, "false"); try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { - Assert.assertEquals(reader.size(), keys.length); + assertEquals(reader.size(), keys.length); for (int i = 0; i < keys.length; i++) { Integer key = keys[i]; - Object val = reader.get(key, null); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + String val = reader.get(key, null); + assertNotNull(val); + assertEquals(val, values[i]); } } } @@ -449,21 +450,21 @@ public void testIterate() { try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { var itr = reader.iterable().iterator(); for (int i = 0; i < keys.length; i++) { - Assert.assertTrue(itr.hasNext()); + assertTrue(itr.hasNext()); var entry = itr.next(); - Assert.assertNotNull(entry); - Assert.assertTrue(keysSet.remove(entry.getKey())); - Assert.assertTrue(valuesSet.remove(entry.getValue())); + assertNotNull(entry); + assertTrue(keysSet.remove(entry.getKey())); + assertTrue(valuesSet.remove(entry.getValue())); Object valSearch = reader.get(entry.getKey(), null); - Assert.assertNotNull(valSearch); - Assert.assertEquals(valSearch, entry.getValue()); + assertNotNull(valSearch); + assertEquals(valSearch, entry.getValue()); } Assert.assertFalse(itr.hasNext()); } - Assert.assertTrue(keysSet.isEmpty()); - Assert.assertTrue(valuesSet.isEmpty()); + assertTrue(keysSet.isEmpty()); + assertTrue(valuesSet.isEmpty()); } // UTILITY @@ -476,13 +477,13 @@ private void testReadKeyToString(K[] keys) { } // Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), keys.length); + assertEquals(reader.size(), keys.length); for (int i = 0; i < keys.length; i++) { K key = keys[i]; String val = reader.get(key, null); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + assertNotNull(val); + assertEquals(val, values[i]); } } } @@ -495,13 +496,13 @@ private void testReadKeyToInt(K[] keys) { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), keys.length); + assertEquals(reader.size(), keys.length); for (int i = 0; i < keys.length; i++) { K key = keys[i]; Object val = reader.get(key, 0); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + assertNotNull(val); + assertEquals(val, values[i]); } } } @@ -514,16 +515,16 @@ private void testReadKeyToNull(K[] keys) { } try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), keys.length); + assertEquals(reader.size(), keys.length); for (K key : keys) { Object val = reader.get(key, null); - Assert.assertNull(val); + assertNull(val); } for (K key : keys) { Object val = reader.get(key, null); - Assert.assertNull(val); + assertNull(val); } } } @@ -537,13 +538,13 @@ private void testReadKeyToIntArray(K[] keys) { //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { - Assert.assertEquals(reader.size(), keys.length); + assertEquals(reader.size(), keys.length); for (int i = 0; i < keys.length; i++) { K key = keys[i]; int[] val = reader.get(key, null); - Assert.assertNotNull(val); - Assert.assertEquals(val, values[i]); + assertNotNull(val); + assertEquals(val, values[i]); } } } @@ -562,6 +563,6 @@ private void testKeyLength(Object key, int expectedLength) { } catch (IOException e) { throw new RuntimeException(e); } - Assert.assertEquals(keyLength, expectedLength); + assertEquals(keyLength, expectedLength); } } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index f3f3ab9..93fcaba 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -23,8 +23,10 @@ import java.nio.file.*; import java.util.List; import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; -import static com.linkedin.paldb.utils.TestTempUtils.*; +import static com.linkedin.paldb.utils.TestTempUtils.deleteDirectory; import static org.testng.Assert.*; public class TestStoreReader { @@ -47,7 +49,7 @@ public void cleanUp() { private StoreReader readerForMany(V... values) { Configuration configuration = new Configuration(); configuration.registerSerializer(new PointSerializer()); - try (StoreWriter writer = PalDB.createWriter(storeFile, configuration)) { + try (StoreWriter writer = PalDB.createWriter(storeFile, configuration)) { for (int i = 0; i < values.length; i++) { writer.put(i, values[i]); } @@ -83,9 +85,9 @@ public void testStoreClosed() { @Test public void testGetBoolean() { try (var reader = readerFor(true)) { - Assert.assertTrue(reader.get(0)); - Assert.assertTrue(reader.get(0, false)); - Assert.assertFalse(reader.get(-1, false)); + assertTrue(reader.get(0)); + assertTrue(reader.get(0, false)); + assertFalse(reader.get(-1, false)); } } @@ -408,7 +410,7 @@ public void testIterator() { Assert.assertNotNull(itr); for (int i = 0; i < values.size(); i++) { - Assert.assertTrue(itr.hasNext()); + assertTrue(itr.hasNext()); var v = itr.next(); assertEquals(v.getValue(), values.get(v.getKey())); } @@ -438,7 +440,7 @@ public void testKeyIterator() { Set actual = new HashSet<>(); Set expected = new HashSet<>(); for (int i = 0; i < values.size(); i++) { - Assert.assertTrue(itr.hasNext()); + assertTrue(itr.hasNext()); Integer k = itr.next(); actual.add(k); expected.add(i); @@ -447,6 +449,34 @@ public void testKeyIterator() { } } + @Test + public void testMultiThreadRead() throws InterruptedException { + int threadCount = 50; + final CountDownLatch latch = new CountDownLatch(threadCount); + final AtomicBoolean success = new AtomicBoolean(true); + var values = List.of("foobar", "any", "any value"); + try (var reader = readerForMany(values.get(0), values.get(1))) { + for(int i = 0; i < threadCount; i++) { + new Thread(() -> { + try { + for(int c = 0; c < 100000; c++) { + if(!success.get())break; + assertEquals(reader.get(1), "any"); + assertEquals(reader.get(0), "foobar"); + } + } catch (Throwable error){ + error.printStackTrace(); + success.set(false); + } finally { + latch.countDown(); + } + }).start(); + } + latch.await(); + assertTrue(success.get()); + } + } + // UTILITY public static class PointSerializer implements Serializer { From 972500c89a1290578495fb5e697dc5ee75d5ce89 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Sun, 17 Nov 2019 19:38:39 +0200 Subject: [PATCH 14/20] Removed LRU cache. Introduced bloom filter. Simplified custom serializers. --- .../com/linkedin/paldb/api/Configuration.java | 42 +-- .../paldb/api/PalDBConfigBuilder.java | 46 +-- .../com/linkedin/paldb/api/Serializer.java | 15 +- .../com/linkedin/paldb/impl/ReaderImpl.java | 7 - .../com/linkedin/paldb/impl/Serializers.java | 267 ++-------------- .../com/linkedin/paldb/impl/StorageCache.java | 265 ---------------- .../linkedin/paldb/impl/StorageReader.java | 49 ++- .../paldb/impl/StorageSerialization.java | 199 +++++------- .../linkedin/paldb/impl/StorageWriter.java | 51 +-- .../com/linkedin/paldb/utils/BloomFilter.java | 108 +++++++ .../linkedin/paldb/TestReadThroughput.java | 95 +++--- .../paldb/api/PalDBConfigBuilderTest.java | 12 +- .../linkedin/paldb/api/TestConfiguration.java | 45 +-- .../linkedin/paldb/impl/GenerateTestData.java | 7 +- .../linkedin/paldb/impl/TestSerializers.java | 144 ++------- .../linkedin/paldb/impl/TestStorageCache.java | 294 ------------------ .../paldb/impl/TestStorageSerialization.java | 32 +- .../com/linkedin/paldb/impl/TestStore.java | 14 +- .../linkedin/paldb/impl/TestStoreReader.java | 13 +- 19 files changed, 420 insertions(+), 1285 deletions(-) delete mode 100644 paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java create mode 100644 paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java delete mode 100644 paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java index baaf7a5..90df23d 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java @@ -15,11 +15,8 @@ package com.linkedin.paldb.api; import com.linkedin.paldb.impl.Serializers; -import java.io.Serializable; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; + +import java.util.*; /** @@ -40,7 +37,7 @@ * -Dpaldb.mmap.data.enabled=false). All property names should be prefixed * with paldb. */ -public class Configuration implements Serializable { +public class Configuration { // Buffer segment size public static final String MMAP_SEGMENT_SIZE = "mmap.segment.size"; @@ -48,16 +45,12 @@ public class Configuration implements Serializable { public static final String MMAP_DATA_ENABLED = "mmap.data.enabled"; // Load factor public static final String LOAD_FACTOR = "load.factor"; - // Cache enabled - public static final String CACHE_ENABLED = "cache.enabled"; - // Cache limit (in bytes) - public static final String CACHE_BYTES = "cache.bytes"; - // Cache initial capacity - public static final String CACHE_INITIAL_CAPACITY = "cache.initial.capacity"; - // Cache load factor - public static final String CACHE_LOAD_FACTOR = "cache.load.factor"; // Enable compression public static final String COMPRESSION_ENABLED = "compression.enabled"; + //Enable bloom filter + public static final String BLOOM_FILTER_ENABLED = "bloom.filter.enabled"; + //Bloom filter error rate + public static final String BLOOM_FILTER_ERROR_FACTOR = "bloom.filter.error.factor"; // Property map private final Map properties = new HashMap<>(); @@ -76,14 +69,9 @@ public Configuration() { putWithSystemPropertyDefault(MMAP_SEGMENT_SIZE, "1073741824"); putWithSystemPropertyDefault(MMAP_DATA_ENABLED, "true"); putWithSystemPropertyDefault(LOAD_FACTOR, "0.75"); - putWithSystemPropertyDefault(CACHE_ENABLED, "false"); - putWithSystemPropertyDefault(CACHE_INITIAL_CAPACITY, "1000"); - putWithSystemPropertyDefault(CACHE_LOAD_FACTOR, "0.75"); putWithSystemPropertyDefault(COMPRESSION_ENABLED, "false"); - - //Default cache size: (Xmx - 100mo); - long cacheMemory = Math.max(0, Runtime.getRuntime().maxMemory() - (100 * 1024 * 1024)); - putWithSystemPropertyDefault(CACHE_BYTES, String.valueOf(cacheMemory)); + putWithSystemPropertyDefault(BLOOM_FILTER_ENABLED, "false"); + putWithSystemPropertyDefault(BLOOM_FILTER_ERROR_FACTOR, "0.01"); //Serializers serializers = new Serializers(); @@ -395,20 +383,10 @@ public Class getClass(String key) * * @param serializer serializer to register */ - public void registerSerializer(Serializer serializer) { + public void registerSerializer(Serializer serializer) { serializers.registerSerializer(serializer); } - /** - * Gets the serializer for the given class or null if not found. - * - * @param cls object class - * @return serializer or null if not found - */ - public Serializer getSerializer(Class cls) { - return serializers.getSerializer(cls); - } - public Serializers getSerializers() { return serializers; } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java b/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java index 9b986de..1727256 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java @@ -51,60 +51,36 @@ public PalDBConfigBuilder withIndexLoadFactor(double loadFactor) { /** * PalDB configuration property. *

- * cache.enabled - LRU cache enabled [default: false] + * compression.enabled - enable compression [default: false] * @param enabled flag * @return this {@code CachemeerConfigBuilder} instance (for chaining) */ - public PalDBConfigBuilder withLRUCacheEnabled(boolean enabled) { - palDbConfiguration.set(Configuration.CACHE_ENABLED, String.valueOf(enabled)); - return this; - } - - /** - * PalDB configuration property. - *

- * cache.bytes - cache limit [default: Xmx - 100MB] - * @param bytes size in bytes - * @return this {@code CachemeerConfigBuilder} instance (for chaining) - */ - public PalDBConfigBuilder withCacheSizeLimit(long bytes) { - palDbConfiguration.set(Configuration.CACHE_BYTES, String.valueOf(bytes)); - return this; - } - - /** - * PalDB configuration property. - *

- * cache.initial.capacity - cache initial capacity [default: 1000] - * @param size number of initial capacity - * @return this {@code CachemeerConfigBuilder} instance (for chaining) - */ - public PalDBConfigBuilder withCacheInitialCapacity(int size) { - palDbConfiguration.set(Configuration.CACHE_INITIAL_CAPACITY, String.valueOf(size)); + public PalDBConfigBuilder withEnableCompression(boolean enabled) { + palDbConfiguration.set(Configuration.COMPRESSION_ENABLED, String.valueOf(enabled)); return this; } /** * PalDB configuration property. *

- * cache.load.factor - cache load factor [default: 0.75] - * @param loadFactor load factor value + * compression.enabled - enable bloom filter [default: true] + * @param enabled flag * @return this {@code CachemeerConfigBuilder} instance (for chaining) */ - public PalDBConfigBuilder withCacheLoadFactor(double loadFactor) { - palDbConfiguration.set(Configuration.CACHE_LOAD_FACTOR, String.valueOf(loadFactor)); + public PalDBConfigBuilder withEnableBloomFilter(boolean enabled) { + palDbConfiguration.set(Configuration.BLOOM_FILTER_ENABLED, String.valueOf(enabled)); return this; } /** * PalDB configuration property. *

- * compression.enabled - enable compression [default: false] - * @param enabled flag + * compression.enabled - bloom filter error rate [default: 0.01] + * @param errorFactor value, e.g. 0.01 equals 1% error rate * @return this {@code CachemeerConfigBuilder} instance (for chaining) */ - public PalDBConfigBuilder withEnableCompression(boolean enabled) { - palDbConfiguration.set(Configuration.COMPRESSION_ENABLED, String.valueOf(enabled)); + public PalDBConfigBuilder withBloomFilterErrorFactor(double errorFactor) { + palDbConfiguration.set(Configuration.BLOOM_FILTER_ERROR_FACTOR, String.valueOf(errorFactor)); return this; } diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java b/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java index 844554e..064e7c5 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java @@ -39,8 +39,7 @@ public interface Serializer extends Serializable { * @param input instance * @throws IOException if an io error occurs */ - void write(DataOutput dataOutput, K input) - throws IOException; + void write(DataOutput dataOutput, K input) throws IOException; /** * Reads the data input and creates the instance. @@ -49,15 +48,7 @@ void write(DataOutput dataOutput, K input) * @return new instance of type K. * @throws IOException if an io error occurs */ - K read(DataInput dataInput) - throws IOException; + K read(DataInput dataInput) throws IOException; - /** - * Returns the estimate number of bytes used to hold instance in memory. - *

- * This information is used by the cache so it can manages its memory usage. - * @param instance instance to get weight for - * @return the number of bytes the object uses in memory - */ - int getWeight(K instance); + Class serializedClass(); } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java index d5bc55a..af7ab00 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java @@ -31,14 +31,10 @@ public final class ReaderImpl implements StoreReader { private static final Logger log = LoggerFactory.getLogger(ReaderImpl.class); // Configuration private final Configuration config; - // Buffer - private final DataInputOutput dataInputOutput = new DataInputOutput(); // Storage private final StorageReader storage; // Serialization private final StorageSerialization serialization; - // Cache - private final StorageCache cache; // File private final File file; // Opened? @@ -63,9 +59,6 @@ public final class ReaderImpl implements StoreReader { throw new UncheckedIOException(ex); } opened = true; - - // Cache - cache = StorageCache.initCache(config); } @Override diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java index 08ca4ba..8641ee0 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java @@ -17,30 +17,23 @@ import com.linkedin.paldb.api.Serializer; import org.slf4j.*; -import java.io.*; -import java.lang.reflect.*; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; /** * Manages the custom serializers. */ -public final class Serializers implements Serializable { +public final class Serializers { // Logger private static final Logger log = LoggerFactory.getLogger(Serializers.class); - private AtomicInteger counter; - private Map serializers; - private Serializer[] serializersArray; + private final Map> serializerMap; /** * Default constructor. */ public Serializers() { - counter = new AtomicInteger(); - serializers = new HashMap<>(); - serializersArray = new Serializer[0]; + serializerMap = new HashMap<>(); } /** @@ -48,19 +41,10 @@ public Serializers() { * * @param serializer serializer */ - public synchronized void registerSerializer(Serializer serializer) { - Class objClass = getSerializerType(serializer); - if (!serializers.containsKey(objClass)) { - int index = counter.getAndIncrement(); - serializers.put(objClass, new SerializerWrapper(index, serializer)); - if (serializersArray.length <= index) { - serializersArray = Arrays.copyOf(serializersArray, index + 1); - } - serializersArray[index] = serializer; - - log.info("Registered new serializer '{}' for '{}' at index {}", serializer.getClass().getName(), - objClass.getName(), index); - } + public synchronized void registerSerializer(Serializer serializer) { + var className = serializer.serializedClass().getName(); + serializerMap.putIfAbsent(className, serializer); + log.info("Registered new serializer '{}' for '{}'", serializer.getClass().getName(), className); } /** @@ -69,235 +53,24 @@ public synchronized void registerSerializer(Serializer serializer) { * @param cls object class * @return serializer instance or null if not found */ - public Serializer getSerializer(Class cls) { - SerializerWrapper w = getSerializerWrapper(cls); - if (w != null) { - return w.serializer; - } - return null; - } - - /** - * Serializes this instance into an output stream. - * - * @param out data output - * @throws IOException if an io error occurs - */ - private void writeObject(ObjectOutputStream out) - throws IOException { - serialize(out, this); - } - - /** - * Serializes this class into a data output. - * - * @param out data output - * @param serializers serializers instance - * @throws IOException if an io error occurs - */ - static void serialize(DataOutput out, Serializers serializers) - throws IOException { - StringBuilder msg = new StringBuilder(String.format("Serialize %d serializer classes:", serializers.serializers.values().size())); - int size = serializers.serializers.values().size(); - - out.writeInt(size); - if (size > 0) { - for (SerializerWrapper sw : serializers.serializers.values()) { - int index = sw.index; - String name = sw.serializer.getClass().getName(); - - out.writeInt(index); - out.writeUTF(name); - - msg.append(String.format("%n (%d) %s", index, name)); - } - log.info(msg.toString()); - } - } - - /** - * Deserializes this instance from an input stream. - * - * @param in data input - * @throws IOException if an io error occurs - * @throws ClassNotFoundException if a class error occurs - */ - private void readObject(ObjectInputStream in) - throws IOException, ClassNotFoundException { - // Init - counter = new AtomicInteger(); - serializers = new HashMap<>(); - serializersArray = new Serializer[0]; - - deserialize(in, this); - } - - /** - * Deserializes this class from a data input. - * - * @param in data input - * @param serializers serializers instance - * @throws IOException if an io error occurs - * @throws ClassNotFoundException if a class error occurs - */ - static void deserialize(DataInput in, Serializers serializers) - throws IOException, ClassNotFoundException { - int size = in.readInt(); - if (size > 0) { - StringBuilder msg = new StringBuilder(String.format("Deserialize %d serializer classes:", size)); - - if (serializers.serializersArray.length < size) { - serializers.serializersArray = Arrays.copyOf(serializers.serializersArray, size); - } - - int max = 0; - for (int i = 0; i < size; i++) { - int index = in.readInt(); - max = Math.max(max, index); - String serializerClassName = in.readUTF(); - try { - Class serializerClass = (Class) Class.forName(serializerClassName); - Serializer serializerInstance = serializerClass.newInstance(); - serializers.serializers - .put(getSerializerType(serializerInstance), new SerializerWrapper(index, serializerInstance)); - serializers.serializersArray[index] = serializerInstance; - - msg.append(String.format("%n (%d) %s", index, serializerClassName)); - } catch (Exception ex) { - log.warn("Can't find the serializer '{}'", serializerClassName, ex); - } - } - serializers.counter.set(max + 1); - - log.info(msg.toString()); - } - } - - /** - * Clear all serializers. - */ - void clear() { - log.info("Clear all serializers"); - - serializers.clear(); - serializersArray = new Serializer[0]; - counter.set(0); - } - - /** - * Returns the serializer index associated with cls. - * - * @param cls object class - * @return serializer index - */ - int getIndex(Class cls) { - return getSerializerWrapper(cls).index; - } - - /** - * Returns the serializer and its index associated with cls. - * - * @param cls object clas - * @return serializer wrapper object - */ - private SerializerWrapper getSerializerWrapper(Class cls) { - SerializerWrapper w = serializers.get(cls); - if (w != null) { - return w; - } else { - // Try with interfaces implemented - for (Class c : cls.getInterfaces()) { - w = serializers.get(c); - if (w != null) { - return w; - } - } - } - return null; - } - - /** - * Returns the serializer given the index. - * - * @param index serializer index - * @return serializer - */ - Serializer getSerializer(int index) { - if (index >= serializersArray.length) { - throw new IllegalArgumentException(String.format("The serializer can't be found at index %d", index)); - } - return serializersArray[index]; - } - - /** - * Inner wrapper class that keeps the index attached to a serializer. - */ - private static class SerializerWrapper implements Serializable { - private int index; - private Serializer serializer; - - /** - * Used by deserialization. - */ - public SerializerWrapper() { - } - - public SerializerWrapper(int index, Serializer serializer) { - this.index = index; - this.serializer = serializer; - } - } - - /** - * Returns the serializer's generic type. - * - * @param instance serializer instance - * @return the class the serializer can serialize - */ - private static Class getSerializerType(Object instance) { - Type type = instance.getClass().getGenericInterfaces()[0]; - if (type instanceof ParameterizedType) { - Class cls = null; - Type clsType = ((ParameterizedType) type).getActualTypeArguments()[0]; - - if (clsType instanceof GenericArrayType) { - // Workaround for Java 6 (JDK bug 7151486) - cls = Array.newInstance((Class) ((GenericArrayType) clsType).getGenericComponentType(), 0).getClass(); - } else { - cls = (Class) clsType; - } + public Serializer getSerializer(Class cls) { + return getSerializer(cls.getName()); +} - if (Object.class.equals(cls)) { - throw new RuntimeException("The serializer type can't be object"); - } - return cls; - } else { - throw new RuntimeException(String - .format("The serializer class %s is not generic or has an unknown type", instance.getClass().getName())); - } + public Serializer getSerializer(String className) { + return serializerMap.get(className); } @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Serializers that = (Serializers) o; - - if (serializers.size() != that.serializers.size()) { - return false; - } - for (Map.Entry entry : serializers.entrySet()) { - SerializerWrapper sw = that.serializers.get(entry.getKey()); - if (sw == null || !sw.serializer.getClass().equals(entry.getValue().serializer.getClass())) { - return false; - } - } + if (this == o) return true; + if (!(o instanceof Serializers)) return false; + final Serializers that = (Serializers) o; + return Objects.equals(serializerMap, that.serializerMap); + } - return true; + @Override + public int hashCode() { + return Objects.hash(serializerMap); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java deleted file mode 100644 index 6c46c91..0000000 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageCache.java +++ /dev/null @@ -1,265 +0,0 @@ -/* -* Copyright 2015 LinkedIn Corp. All rights reserved. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -*/ - -package com.linkedin.paldb.impl; - -import com.linkedin.paldb.api.*; -import org.slf4j.*; - -import java.text.DecimalFormat; -import java.util.*; - - -/** - * LRU cache configured on the desired size in memory. - *

- *
This cache can be configured with the following properties:
- *
cache.enabled - LRU cache enabled
- *
cache.bytes - cache limit (bytes)
- *
cache.initial.capacity - cache initial capacity
- *
cache.load.factor - cache load factor
- *
- *

- * The cache estimates the size of the objects it contains so it consumes no more than the configured - * memory limit. - */ -public class StorageCache { - // Static null object to recognizes null from missing values - protected static final Object NULL_VALUE = new Object(); - - //Logger - private static final Logger log = LoggerFactory.getLogger(StorageCache.class); - - /** - * Factory to create and initialize the cache. - * - * @param configuration configuration - * @return new cache - */ - static StorageCache initCache(Configuration configuration) { - if (configuration.getBoolean(Configuration.CACHE_ENABLED) && configuration.getLong(Configuration.CACHE_BYTES) > 0) { - return new StorageCache<>(configuration); - } else { - return new DisabledCache<>(); - } - } - - /* - * Memory usage - * 632 bytes for the LinkedHashMap - * 24 bytes theoretical overhead per entry but more like 45 in practice - */ - static final int OVERHEAD = 50; - private final LinkedHashMap cache; - private final Configuration configuration; - private long maxWeight; - private long currentWeight; - - /** - * Cache constructor. - * - * @param config configuration - */ - private StorageCache(Configuration config) { - cache = new LinkedHashMap<>(config.getInt(Configuration.CACHE_INITIAL_CAPACITY), - config.getFloat(Configuration.CACHE_LOAD_FACTOR), true) { - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - boolean res = currentWeight > maxWeight; - if (res) { - K key = eldest.getKey(); - V value = eldest.getValue(); - currentWeight -= getWeight(key) + getWeight(value) + OVERHEAD; - } - return res; - } - }; - maxWeight = config.getLong(Configuration.CACHE_BYTES); - log.info("Cache initialized with maximum {} Mb usage", - new DecimalFormat("#,##0.00").format(maxWeight / (1024.0 * 1024.0))); - configuration = config; - } - - /** - * Private constructor used by the DisabledCache inner class. - */ - private StorageCache() { - cache = null; - configuration = null; - } - - /** - * Gets the value in the cache for key or null if not found. - *

- * If the value associated with key exists but is null, returns - * StorageCache.NULL_VALUE. - * - * @param key key to get value for - * @return value, null or StorageCache.NULL_VALUE - */ - public V get(K key) { - return cache.get(key); - } - - /** - * Returns true if the cache contains key. - * - * @param key key to test presence for - * @return true if found, false otherwise - */ - public boolean contains(K key) { - return cache.containsKey(key); - } - - /** - * Puts the key/value pair into the cache. - * - * @param key key - * @param value value - */ - public void put(K key, V value) { - int weight = getWeight(key) + getWeight(value) + OVERHEAD; - currentWeight += weight; - if (cache.put(key, value == null ? (V) NULL_VALUE : value) != null) { - currentWeight -= weight; - } - } - - /** - * Gets the weight for value. - * - * @param value value to get weight for - * @return weight - */ - private int getWeight(T value) { - if (value == null) { - return 0; - } - if (value.getClass().isArray()) { - Class cc = value.getClass().getComponentType(); - if (cc.isPrimitive()) { - if (cc.equals(int.class)) { - return ((int[]) value).length * 4; - } else if (cc.equals(long.class)) { - return ((long[]) value).length * 8; - } else if (cc.equals(double.class)) { - return ((double[]) value).length * 8; - } else if (cc.equals(float.class)) { - return ((float[]) value).length * 4; - } else if (cc.equals(boolean.class)) { - return ((boolean[]) value).length * 1; - } else if (cc.equals(byte.class)) { - return ((byte[]) value).length * 1; - } else if (cc.equals(short.class)) { - return ((short[]) value).length * 2; - } else if (cc.equals(char.class)) { - return ((char[]) value).length * 2; - } - } else if (cc.equals(String.class)) { - String[] v = (String[]) value; - int res = 0; - for (int i = 0; i < v.length; i++) { - res += v[i].length() * 2 + 40; - } - return res; - } else if (cc.equals(int[].class)) { - int[][] v = (int[][]) value; - int res = 0; - for (int i = 0; i < v.length; i++) { - res += v[i].length * 4; - } - return res; - } else if (cc.equals(long[].class)) { - long[][] v = (long[][]) value; - int res = 0; - for (int i = 0; i < v.length; i++) { - res += v[i].length * 8; - } - return res; - } else { - Object[] v = (Object[]) value; - int res = 0; - for (int i = 0; i < v.length; i++) { - res += getWeight(v[i]); - } - return res; - } - } else if (value instanceof String) { - return ((String) value).length() * 2 + 40; - } else { - Serializer serializer = configuration.getSerializer((Class) value.getClass()); - if (serializer != null) { - return serializer.getWeight(value); - } - } - return 16; - } - - /** - * Sets the max weight in the cache. - * - * @param maxWeight max weight - */ - public void setMaxWeight(long maxWeight) { - this.maxWeight = maxWeight; - } - - /** - * Gets the cache size. - * - * @return cache size - */ - public int size() { - return cache.size(); - } - - /** - * Gets the cache current weight. - * - * @return weight - */ - public long getWeight() { - return currentWeight; - } - - /** - * Special inner class that overrides all cache's features when the cache is disabled. - */ - private static class DisabledCache extends StorageCache { - - DisabledCache() { - log.info("Cache disabled"); - } - - @Override - public V get(K key) { - return null; - } - - @Override - public boolean contains(K key) { - return false; - } - - @Override - public void put(K key, V value) { - //nop - } - - @Override - public int size() { - return 0; - } - } -} diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index dc724dd..816660d 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -59,8 +59,9 @@ public class StorageReader implements Iterable> { // Buffers private final ThreadLocal context; - StorageReader(Configuration configuration, File file) - throws IOException { + private final BloomFilter bloomFilter; + + StorageReader(Configuration configuration, File file) throws IOException { // File path // Configuration if (!file.exists()) { @@ -83,8 +84,11 @@ public class StorageReader implements Iterable> { // Offset of the index in the channel int indexOffset; - try (FileInputStream inputStream = new FileInputStream(file); - DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream))) { + int bloomFilterBitSize = 0; + int bloomFilterHashFunctions = 0; + + try (FileInputStream inputStream = new FileInputStream(file); + DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream))) { int ignoredBytes = -2; //Byte mark @@ -116,6 +120,24 @@ public class StorageReader implements Iterable> { //Metadata counters keyCount = dataInputStream.readInt(); + + //read bloom filter bit size + bloomFilterBitSize = dataInputStream.readInt(); + //read bloom filter long array size + int bloomFilterLength = dataInputStream.readInt(); + //read bloom filter hash functions + bloomFilterHashFunctions = dataInputStream.readInt(); + if (bloomFilterLength > 0) { + //read bloom filter long array + long[] bits = new long[bloomFilterLength]; + for (int i = 0; i < bloomFilterLength; i++) { + bits[i] = dataInputStream.readLong(); + } + bloomFilter = new BloomFilter(bloomFilterHashFunctions, bloomFilterBitSize, bits); + } else { + bloomFilter = null; + } + // Number of different key length final int keyLengthCount = dataInputStream.readInt(); // Max key length @@ -142,13 +164,6 @@ public class StorageReader implements Iterable> { } maxSlotSize = mSlotSize; - //Read serializers - try { - Serializers.deserialize(dataInputStream, configuration.getSerializers()); - } catch (Exception e) { - throw new RuntimeException(); - } - //Read index and data offset indexOffset = dataInputStream.readInt() + ignoredBytes; dataOffset = dataInputStream.readLong() + ignoredBytes; @@ -186,6 +201,8 @@ public class StorageReader implements Iterable> { } statMsg.append(" Index size: ").append(integerFormat.format((dataOffset - indexOffset) / (1024.0 * 1024.0))).append(" Mb\n"); statMsg.append(" Data size: ").append(integerFormat.format((fileSize - dataOffset) / (1024.0 * 1024.0))).append(" Mb\n"); + statMsg.append(" Bloom filter size: ").append(integerFormat.format((bloomFilterBitSize / 8.0) / (1024.0 * 1024.0))).append(" Mb\n"); + statMsg.append(" Bloom filter hashes: ").append(bloomFilterHashFunctions).append("\n"); log.debug(statMsg.toString()); } } @@ -220,13 +237,18 @@ public byte[] get(byte[] key) throws IOException { if (keyLength >= slots.length || keyCounts[keyLength] == 0) { return null; } + + var ctx = context.get(); + long hash = ctx.hash(key); + if (bloomFilter != null && !bloomFilter.mightContain(hash, ctx.random)) { + return null; + } + int numSlots = slots[keyLength]; int slotSize = slotSizes[keyLength]; int ixOffset = indexOffsets[keyLength]; long dtOffset = dataOffsets[keyLength]; - var ctx = context.get(); - long hash = ctx.hash(key); for (int probe = 0; probe < numSlots; probe++) { int slot = (int) ((hash + probe) % numSlots); @@ -256,6 +278,7 @@ private boolean isKey(byte[] slotBuffer, byte[] key) { private static final class ThreadContext { private final HashUtils.Murmur3A hash = new HashUtils.Murmur3A(42); + private final Random random = new Random(); private final MappedByteBuffer indexBuffer; private final MappedByteBuffer[] dataBuffers; private final DataInputOutput sizeBuffer = new DataInputOutput(new byte[5]); diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java index 5bb6b57..b7a7948 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java @@ -54,7 +54,7 @@ public StorageSerialization(Configuration config) { * @return key as byte array * @throws IOException if an io error occurs */ - public byte[] serializeKey(Object key) throws IOException { + public byte[] serializeKey(K key) throws IOException { if (key == null) { throw new NullPointerException(); } @@ -70,8 +70,7 @@ public byte[] serializeKey(Object key) throws IOException { * @param dataOutput data output * @throws IOException if an io error occurs */ - public void serializeKey(Object key, DataOutput dataOutput) - throws IOException { + public void serializeKey(Object key, DataOutput dataOutput) throws IOException { serializeObject(key, dataOutput, false); } @@ -82,8 +81,7 @@ public void serializeKey(Object key, DataOutput dataOutput) * @return value as byte array * @throws IOException if an io error occurs */ - public byte[] serializeValue(Object value) - throws IOException { + public byte[] serializeValue(Object value) throws IOException { serializeObject(value, dataInputOutput.reset(), compression); return dataInputOutput.toByteArray(); @@ -96,8 +94,7 @@ public byte[] serializeValue(Object value) * @param dataOutput data output * @throws IOException if an io error occurs */ - public void serializeValue(Object value, DataOutput dataOutput) - throws IOException { + public void serializeValue(Object value, DataOutput dataOutput) throws IOException { serializeObject(value, dataOutput, compression); } @@ -106,15 +103,13 @@ public void serializeValue(Object value, DataOutput dataOutput) * * @param obj object to serialize * @param useCompression use compression - * @return serialized object in bytes * @throws IOException if an io error occurs */ - private void serializeObject(Object obj, DataOutput dataOutput, boolean useCompression) - throws IOException { + private void serializeObject(Object obj, DataOutput dataOutput, boolean useCompression) throws IOException { //Cast to primitive arrays if necessary if (obj != null && obj.getClass().isArray()) { if (obj instanceof Integer[]) { - obj = (int[]) getPrimitiveArray((Object[]) obj); + obj = (int[]) getPrimitiveArray((Integer[]) obj); } else if (obj instanceof Boolean[]) { obj = (boolean[]) getPrimitiveArray((Object[]) obj); } else if (obj instanceof Byte[]) { @@ -190,7 +185,7 @@ private static Object getPrimitiveArray(Object[][] array) { return null; } - private static Object getPrimitiveArray(Object[] array) { + private static Object getPrimitiveArray(T[] array) { Class arrayClass = array.getClass().getComponentType(); if (!arrayClass.isPrimitive()) { Class primitiveClass = getPrimitiveType(arrayClass); @@ -323,13 +318,11 @@ private static Class getPrimitiveType(Class type) { final static int CUSTOM = 114; final static String EMPTY_STRING = ""; - byte[] serialize(Object obj) - throws IOException { + byte[] serialize(Object obj) throws IOException { return serialize(obj, false); } - byte[] serialize(Object obj, boolean compress) - throws IOException { + byte[] serialize(Object obj, boolean compress) throws IOException { DataInputOutput ba = new DataInputOutput(); serialize(ba, obj, compress); @@ -337,19 +330,17 @@ byte[] serialize(Object obj, boolean compress) return ba.toByteArray(); } - private void serialize(final DataOutput out, final Object obj) - throws IOException { + private void serialize(final DataOutput out, final Object obj) throws IOException { serialize(out, obj, false); } - private void serialize(final DataOutput out, final Object obj, boolean compress) - throws IOException { + private void serialize(final DataOutput out, final Object obj, boolean compress) throws IOException { final Class clazz = obj != null ? obj.getClass() : null; if (obj == null) { out.write(NULL); } else if (clazz == Boolean.class) { - if (((Boolean) obj).booleanValue()) { + if ((boolean) obj) { out.write(BOOLEAN_TRUE); } else { out.write(BOOLEAN_FALSE); @@ -400,10 +391,11 @@ private void serialize(final DataOutput out, final Object obj, boolean compress) serializeLongLongArray(out, (long[][]) obj, compress); } else { // Custom - Serializer serializer = serializers.getSerializer(obj.getClass()); + var serializer = serializers.getSerializer(obj.getClass()); if (serializer != null) { - int index = serializers.getIndex(obj.getClass()); - out.write(CUSTOM + index); + var className = serializer.serializedClass().getName(); + out.write(CUSTOM); + out.writeChars(className); serializer.write(out, obj); } else if (obj instanceof Object[]) { serializeObjectArray(out, (Object[]) obj); @@ -413,8 +405,7 @@ private void serialize(final DataOutput out, final Object obj, boolean compress) } } - private static void serializeInt(final DataOutput out, final int val) - throws IOException { + private static void serializeInt(final DataOutput out, final int val) throws IOException { if (val == -1) { out.write(INTEGER_MINUS_1); } else if (val == 0) { @@ -449,8 +440,7 @@ private static void serializeInt(final DataOutput out, final int val) } } - private static void serializeDouble(final DataOutput out, final double val) - throws IOException { + private static void serializeDouble(final DataOutput out, final double val) throws IOException { if (val == -1d) { out.write(DOUBLE_MINUS_1); } else if (val == 0d) { @@ -469,8 +459,7 @@ private static void serializeDouble(final DataOutput out, final double val) } } - private static void serializeFloat(final DataOutput out, final float val) - throws IOException { + private static void serializeFloat(final DataOutput out, final float val) throws IOException { if (val == -1f) { out.write(FLOAT_MINUS_1); } else if (val == 0f) { @@ -489,8 +478,7 @@ private static void serializeFloat(final DataOutput out, final float val) } } - private static void serializeShort(final DataOutput out, final short val) - throws IOException { + private static void serializeShort(final DataOutput out, final short val) throws IOException { if (val == -1) { out.write(SHORT_MINUS_1); } else if (val == 0) { @@ -506,8 +494,7 @@ private static void serializeShort(final DataOutput out, final short val) } } - private static void serializeByte(final DataOutput out, final byte val) - throws IOException { + private static void serializeByte(final DataOutput out, final byte val) throws IOException { if (val == -1) { out.write(BYTE_MINUS_1); } else if (val == 0) { @@ -520,8 +507,7 @@ private static void serializeByte(final DataOutput out, final byte val) } } - private static void serializeLong(final DataOutput out, final long val) - throws IOException { + private static void serializeLong(final DataOutput out, final long val) throws IOException { if (val == -1) { out.write(LONG_MINUS_1); } else if (val == 0) { @@ -556,14 +542,12 @@ private static void serializeLong(final DataOutput out, final long val) } } - private static void serializeChar(final DataOutput out, final char val) - throws IOException { + private static void serializeChar(final DataOutput out, final char val) throws IOException { out.write(CHAR); out.writeChar(val); } - private static void serializeString(final DataOutput out, final String val) - throws IOException { + private static void serializeString(final DataOutput out, final String val) throws IOException { if (val.length() == 0) { out.write(STRING_EMPTY); } else { @@ -577,28 +561,24 @@ private static void serializeString(final DataOutput out, final String val) } } - private static void serializeBigInteger(final DataOutput out, final BigInteger val) - throws IOException { + private static void serializeBigInteger(final DataOutput out, final BigInteger val) throws IOException { out.write(BIGINTEGER); byte[] buf = val.toByteArray(); serializeByteArray(out, buf, false); } - private static void serializeBigDecimal(final DataOutput out, final BigDecimal val) - throws IOException { + private static void serializeBigDecimal(final DataOutput out, final BigDecimal val) throws IOException { out.write(BIGDECIMAL); serializeByteArray(out, val.unscaledValue().toByteArray(), false); LongPacker.packInt(out, val.scale()); } - private static void serializeClass(final DataOutput out, final Class val) - throws IOException { + private static void serializeClass(final DataOutput out, final Class val) throws IOException { out.write(CLASS); serializeString(out, val.getName()); } - private static void serializeBooleanArray(final DataOutput out, final boolean[] val) - throws IOException { + private static void serializeBooleanArray(final DataOutput out, final boolean[] val) throws IOException { out.write(BOOLEAN_ARRAY); LongPacker.packInt(out, val.length); for (boolean s : val) { @@ -606,8 +586,7 @@ private static void serializeBooleanArray(final DataOutput out, final boolean[] } } - private static void serializeShortArray(final DataOutput out, final short[] val, boolean compress) - throws IOException { + private static void serializeShortArray(final DataOutput out, final short[] val, boolean compress) throws IOException { if (compress && val.length > 250) { out.write(SHORT_ARRAY_C); byte[] b = Snappy.compress(val); @@ -622,8 +601,7 @@ private static void serializeShortArray(final DataOutput out, final short[] val, } } - private static void serializeDoubleArray(final DataOutput out, final double[] val, boolean compress) - throws IOException { + private static void serializeDoubleArray(final DataOutput out, final double[] val, boolean compress) throws IOException { if (compress && val.length > 250) { out.write(DOUBLE_ARRAY_C); byte[] b = Snappy.compress(val); @@ -638,8 +616,7 @@ private static void serializeDoubleArray(final DataOutput out, final double[] va } } - private static void serializeFloatArray(final DataOutput out, final float[] val, boolean compress) - throws IOException { + private static void serializeFloatArray(final DataOutput out, final float[] val, boolean compress) throws IOException { if (compress && val.length > 250) { out.write(FLOAT_ARRAY_C); byte[] b = Snappy.compress(val); @@ -654,8 +631,7 @@ private static void serializeFloatArray(final DataOutput out, final float[] val, } } - private static void serializeCharArray(final DataOutput out, final char[] val, boolean compress) - throws IOException { + private static void serializeCharArray(final DataOutput out, final char[] val, boolean compress) throws IOException { if (compress && val.length > 250) { out.write(CHAR_ARRAY_C); byte[] b = Snappy.compress(val); @@ -670,8 +646,7 @@ private static void serializeCharArray(final DataOutput out, final char[] val, b } } - private static void serializeIntArray(final DataOutput out, final int[] val, boolean compress) - throws IOException { + private static void serializeIntArray(final DataOutput out, final int[] val, boolean compress) throws IOException { int max = Integer.MIN_VALUE; int min = Integer.MAX_VALUE; for (int i : val) { @@ -711,8 +686,7 @@ private static void serializeIntArray(final DataOutput out, final int[] val, boo } } - private static void serializeIntIntArray(final DataOutput out, final int[][] val, boolean compress) - throws IOException { + private static void serializeIntIntArray(final DataOutput out, final int[][] val, boolean compress) throws IOException { out.write(INT_INT_ARRAY); LongPacker.packInt(out, val.length); @@ -721,8 +695,7 @@ private static void serializeIntIntArray(final DataOutput out, final int[][] val } } - private static void serializeLongArray(final DataOutput out, final long[] val, boolean compress) - throws IOException { + private static void serializeLongArray(final DataOutput out, final long[] val, boolean compress) throws IOException { long max = Long.MIN_VALUE; long min = Long.MAX_VALUE; for (long i : val) { @@ -768,8 +741,7 @@ private static void serializeLongArray(final DataOutput out, final long[] val, b } } - private static void serializeLongLongArray(final DataOutput out, final long[][] val, boolean compress) - throws IOException { + private static void serializeLongLongArray(final DataOutput out, final long[][] val, boolean compress) throws IOException { out.write(LONG_LONG_ARRAY); LongPacker.packInt(out, val.length); @@ -778,8 +750,7 @@ private static void serializeLongLongArray(final DataOutput out, final long[][] } } - private static void serializeByteArray(final DataOutput out, final byte[] val, boolean compress) - throws IOException { + private static void serializeByteArray(final DataOutput out, final byte[] val, boolean compress) throws IOException { if (compress && val.length > 250) { out.write(BYTE_ARRAY_C); byte[] b = Snappy.compress(val); @@ -792,8 +763,7 @@ private static void serializeByteArray(final DataOutput out, final byte[] val, b } } - private static void serializeStringArray(final DataOutput out, final String[] val) - throws IOException { + private static void serializeStringArray(final DataOutput out, final String[] val) throws IOException { out.write(STRING_ARRAY); LongPacker.packInt(out, val.length); for (String s : val) { @@ -801,8 +771,7 @@ private static void serializeStringArray(final DataOutput out, final String[] va } } - private void serializeObjectArray(final DataOutput out, final Object[] val) - throws IOException { + private void serializeObjectArray(final DataOutput out, final Object[] val) throws IOException { out.write(ARRAY_OBJECT); LongPacker.packInt(out, val.length); for (Object o : val) { @@ -810,8 +779,7 @@ private void serializeObjectArray(final DataOutput out, final Object[] val) } } - public Object deserialize(byte[] buf) - throws ClassNotFoundException, IOException { + public Object deserialize(byte[] buf) throws ClassNotFoundException, IOException { DataInputOutput bs = new DataInputOutput(buf); Object ret = deserialize(bs); if (bs.available() != 0) { @@ -821,14 +789,17 @@ public Object deserialize(byte[] buf) return ret; } - public Object deserialize(DataInput is) - throws IOException, ClassNotFoundException { + public Object deserialize(DataInput is) throws IOException, ClassNotFoundException { Object ret = null; final int head = is.readUnsignedByte(); if (head >= CUSTOM) { - Serializer serializer = serializers.getSerializer(head - CUSTOM); + var className = is.readUTF(); + Serializer serializer = serializers.getSerializer(className); + if (serializer == null) { + throw new ClassNotFoundException("Serializer not registered: " + className); + } ret = serializer.read(is); } else { switch (head) { @@ -1101,16 +1072,13 @@ private static String deserializeString(DataInput buf) return new String(b); } - private static Class deserializeClass(DataInput is) - throws IOException, ClassNotFoundException { + private static Class deserializeClass(DataInput is) throws IOException, ClassNotFoundException { is.readByte(); String className = deserializeString(is); - Class cls = Class.forName(className); - return cls; + return Class.forName(className); } - private static short[] deserializeShortArray(DataInput is) - throws IOException { + private static short[] deserializeShortArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); short[] ret = new short[size]; for (int i = 0; i < size; i++) { @@ -1119,8 +1087,7 @@ private static short[] deserializeShortArray(DataInput is) return ret; } - private static short[] deserializeShortCompressedArray(DataInput is) - throws IOException { + private static short[] deserializeShortCompressedArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); byte[] b = new byte[size]; is.readFully(b); @@ -1137,16 +1104,14 @@ private static float[] deserializeFloatArray(DataInput is) return ret; } - private static float[] deserializeFloatCompressedArray(DataInput is) - throws IOException { + private static float[] deserializeFloatCompressedArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); byte[] b = new byte[size]; is.readFully(b); return Snappy.uncompressFloatArray(b); } - private static double[] deserializeDoubleArray(DataInput is) - throws IOException { + private static double[] deserializeDoubleArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); double[] ret = new double[size]; for (int i = 0; i < size; i++) { @@ -1155,16 +1120,14 @@ private static double[] deserializeDoubleArray(DataInput is) return ret; } - private static double[] deserializeDoubleCompressedArray(DataInput is) - throws IOException { + private static double[] deserializeDoubleCompressedArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); byte[] b = new byte[size]; is.readFully(b); return Snappy.uncompressDoubleArray(b); } - private static char[] deserializeCharArray(DataInput is) - throws IOException { + private static char[] deserializeCharArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); char[] ret = new char[size]; for (int i = 0; i < size; i++) { @@ -1173,16 +1136,14 @@ private static char[] deserializeCharArray(DataInput is) return ret; } - private static char[] deserializeCharCompressedArray(DataInput is) - throws IOException { + private static char[] deserializeCharCompressedArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); byte[] b = new byte[size]; is.readFully(b); return Snappy.uncompressCharArray(b); } - private static boolean[] deserializeBooleanArray(DataInput is) - throws IOException { + private static boolean[] deserializeBooleanArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); boolean[] ret = new boolean[size]; for (int i = 0; i < size; i++) { @@ -1191,8 +1152,7 @@ private static boolean[] deserializeBooleanArray(DataInput is) return ret; } - private static String[] deserializeStringArray(DataInput is) - throws IOException { + private static String[] deserializeStringArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); String[] ret = new String[size]; for (int i = 0; i < size; i++) { @@ -1213,8 +1173,7 @@ private static String[] deserializeStringArray(DataInput is) private static final byte[] EMPTY_BYTES = new byte[0]; - private static byte[] deserializeByteArray(DataInput is) - throws IOException { + private static byte[] deserializeByteArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); if (size == 0) { return EMPTY_BYTES; @@ -1224,8 +1183,7 @@ private static byte[] deserializeByteArray(DataInput is) return b; } - private static byte[] deserializeByteCompressedArray(DataInput is) - throws IOException { + private static byte[] deserializeByteCompressedArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); byte[] b = new byte[size]; is.readFully(b); @@ -1243,8 +1201,7 @@ private Object[] deserializeArrayObject(DataInput is) return s; } - private static long[] deserializeArrayLongL(DataInput is) - throws IOException { + private static long[] deserializeArrayLongL(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); long[] ret = new long[size]; for (int i = 0; i < size; i++) { @@ -1253,8 +1210,7 @@ private static long[] deserializeArrayLongL(DataInput is) return ret; } - private static long[] deserializeArrayLongI(DataInput is) - throws IOException { + private static long[] deserializeArrayLongI(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); long[] ret = new long[size]; for (int i = 0; i < size; i++) { @@ -1263,8 +1219,7 @@ private static long[] deserializeArrayLongI(DataInput is) return ret; } - private static long[] deserializeArrayLongS(DataInput is) - throws IOException { + private static long[] deserializeArrayLongS(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); long[] ret = new long[size]; for (int i = 0; i < size; i++) { @@ -1273,8 +1228,7 @@ private static long[] deserializeArrayLongS(DataInput is) return ret; } - private static long[] deserializeArrayLongB(DataInput is) - throws IOException { + private static long[] deserializeArrayLongB(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); long[] ret = new long[size]; for (int i = 0; i < size; i++) { @@ -1286,16 +1240,14 @@ private static long[] deserializeArrayLongB(DataInput is) return ret; } - private static long[] deserializeArrayLongCompressed(DataInput is) - throws IOException { + private static long[] deserializeArrayLongCompressed(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); byte[] b = new byte[size]; is.readFully(b); return Snappy.uncompressLongArray(b); } - private static long[][] deserializeLongLongArray(DataInput is) - throws IOException { + private static long[][] deserializeLongLongArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); long[][] res = new long[size][]; for (int i = 0; i < size; i++) { @@ -1326,8 +1278,7 @@ private static long[][] deserializeLongLongArray(DataInput is) return res; } - private static int[][] deserializeIntIntArray(DataInput is) - throws IOException { + private static int[][] deserializeIntIntArray(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); int[][] res = new int[size][]; for (int i = 0; i < size; i++) { @@ -1355,8 +1306,7 @@ private static int[][] deserializeIntIntArray(DataInput is) return res; } - private static int[] deserializeArrayIntI(DataInput is) - throws IOException { + private static int[] deserializeArrayIntI(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); int[] ret = new int[size]; for (int i = 0; i < size; i++) { @@ -1365,8 +1315,7 @@ private static int[] deserializeArrayIntI(DataInput is) return ret; } - private static int[] deserializeArrayIntS(DataInput is) - throws IOException { + private static int[] deserializeArrayIntS(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); int[] ret = new int[size]; for (int i = 0; i < size; i++) { @@ -1375,8 +1324,7 @@ private static int[] deserializeArrayIntS(DataInput is) return ret; } - private static int[] deserializeArrayIntB(DataInput is) - throws IOException { + private static int[] deserializeArrayIntB(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); int[] ret = new int[size]; for (int i = 0; i < size; i++) { @@ -1388,8 +1336,7 @@ private static int[] deserializeArrayIntB(DataInput is) return ret; } - private static int[] deserializeArrayIntPack(DataInput is) - throws IOException { + private static int[] deserializeArrayIntPack(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); if (size < 0) { throw new EOFException(); @@ -1402,16 +1349,14 @@ private static int[] deserializeArrayIntPack(DataInput is) return ret; } - private static int[] deserializeArrayIntCompressed(DataInput is) - throws IOException { + private static int[] deserializeArrayIntCompressed(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); byte[] b = new byte[size]; is.readFully(b); return Snappy.uncompressIntArray(b); } - private static long[] deserializeArrayLongPack(DataInput is) - throws IOException { + private static long[] deserializeArrayLongPack(DataInput is) throws IOException { int size = LongPacker.unpackInt(is); if (size < 0) { throw new EOFException(); diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index e482e8d..e0ffbca 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -150,27 +150,31 @@ public void close() log.info("Number of keys: {}", keyCount); log.info("Number of values: {}", valueCount); + var bloomFilter = config.getBoolean(Configuration.BLOOM_FILTER_ENABLED) ? + new BloomFilter(keyCount, config.getDouble(Configuration.BLOOM_FILTER_ERROR_FACTOR, 0.01)) : + null; + + // Prepare files to merge List filesToMerge = new ArrayList<>(); try { + // Build index file + for (int i = 0; i < indexFiles.length; i++) { + if (indexFiles[i] != null) { + filesToMerge.add(buildIndex(i, bloomFilter)); + } + } //Write metadata file File metadataFile = new File(tempFolder, "metadata.dat"); metadataFile.deleteOnExit(); try (FileOutputStream metadataOutputStream = new FileOutputStream(metadataFile); DataOutputStream metadataDataOutputStream = new DataOutputStream(metadataOutputStream)) { - writeMetadata(metadataDataOutputStream); + writeMetadata(metadataDataOutputStream, bloomFilter); } - filesToMerge.add(metadataFile); - - // Build index file - for (int i = 0; i < indexFiles.length; i++) { - if (indexFiles[i] != null) { - filesToMerge.add(buildIndex(i)); - } - } + filesToMerge.add(0, metadataFile); // Stats collisions log.info("Number of collisions: {}", collisions); @@ -191,8 +195,7 @@ public void close() } } - private void writeMetadata(DataOutputStream dataOutputStream) - throws IOException { + private void writeMetadata(DataOutputStream dataOutputStream, BloomFilter bloomFilter) throws IOException { //Write format version dataOutputStream.writeUTF(FormatVersion.getLatestVersion().name()); @@ -206,6 +209,19 @@ private void writeMetadata(DataOutputStream dataOutputStream) //Write size (number of keys) dataOutputStream.writeInt(keyCount); + //write bloom filter bit size + dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.bitSize() : 0); + //write bloom filter long array size + dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.bits().length : 0); + //write bloom filter hash functions + dataOutputStream.writeInt(bloomFilter != null ? bloomFilter.hashFunctions() : 0); + //write bloom filter bits + if (bloomFilter != null) { + for (final long bit : bloomFilter.bits()) { + dataOutputStream.writeLong(bit); + } + } + //Write the number of different key length dataOutputStream.writeInt(keyLengthCount); @@ -244,21 +260,13 @@ private void writeMetadata(DataOutputStream dataOutputStream) } } - //Write serializers - try { - Serializers.serialize(dataOutputStream, config.getSerializers()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - //Write the position of the index and the data int indexOffset = dataOutputStream.size() + (Integer.SIZE / Byte.SIZE) + (Long.SIZE / Byte.SIZE); dataOutputStream.writeInt(indexOffset); dataOutputStream.writeLong(indexOffset + indexesLength); } - private File buildIndex(int keyLength) - throws IOException { + private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOException { long count = keyCounts[keyLength]; int slots = (int) Math.round(count / loadFactor); int offsetLength = maxOffsetLengths[keyLength]; @@ -288,6 +296,9 @@ private File buildIndex(int keyLength) // Hash long hash = hashUtils.hash(keyBuffer); + if (bloomFilter != null) { + bloomFilter.add(hash); + } boolean collision = false; for (int probe = 0; probe < count; probe++) { diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java new file mode 100644 index 0000000..8f5b1f8 --- /dev/null +++ b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java @@ -0,0 +1,108 @@ +package com.linkedin.paldb.utils; + +import java.util.*; + +import static java.lang.Math.log; + +public class BloomFilter { + private static final int BITS_IN_LONG = 64; + private final long[] bits; + private final int hashFunctions; // Number of hash functions + private static final double LN2 = 0.6931471805599453; // ln(2) + private final Random random = new Random(); + private final int sizeInBits; + + public BloomFilter(int elements, int sizeInBits) { + this.sizeInBits = sizeInBits; + this.hashFunctions = Math.max(1, (int) Math.round(LN2 * sizeInBits / elements)); + this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / BITS_IN_LONG))]; + } + + public BloomFilter(int expectedElements, double errorRate) { + this.sizeInBits = Math.max(BITS_IN_LONG, (int) Math.ceil( (-1 * expectedElements * log(errorRate)) / (LN2 * LN2))); + this.hashFunctions = Math.max(1, (int) Math.round(((double) sizeInBits / expectedElements) * LN2)); + this.bits = new long[Math.max(1, (int) Math.ceil((double) sizeInBits / BITS_IN_LONG))]; + } + + public BloomFilter(int hashFunctions, int bitSize, long[] bits) { + this.sizeInBits = bitSize; + this.bits = bits; + this.hashFunctions = hashFunctions; + } + + public void add(long hash) { + random.setSeed(hash); + + for (int i = 0; i < hashFunctions; i++) { + int value = Math.abs(random.nextInt() % sizeInBits); + setBit(value); + } + } + + public long[] bits() { + return bits; + } + + private void setBit(int position) { + int flagIndex = position / BITS_IN_LONG; + int bitIndexInFlag = position % BITS_IN_LONG; + bits[flagIndex] |= (1L << bitIndexInFlag); + } + + private boolean getBit(int position) { + int flagIndex = position / BITS_IN_LONG; + int bitIndexInFlag = position % BITS_IN_LONG; + return ((bits[flagIndex] >> bitIndexInFlag) & 1L) == 1; + } + + public boolean mightContain(long hash, Random random) { + random.setSeed(hash); + + for (int i = 0; i < hashFunctions; i++) { + int value = Math.abs(random.nextInt() % sizeInBits); + if (!getBit(value)) return false; + } + return true; + } + + public void clear() { + for (int i = 0; i < bits.length; i++) { + bits[i] = 0L; + } + } + + public int bitSize() { + return sizeInBits; + } + + public int hashFunctions() { + return hashFunctions; + } + + @Override + public int hashCode() { + return Arrays.hashCode(bits) ^ hashFunctions; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof BloomFilter)) return false; + final var that = (BloomFilter) o; + return Arrays.equals(this.bits, that.bits) && this.hashFunctions == that.hashFunctions; + } + + public void merge(BloomFilter other) { + if (other.hashFunctions != this.hashFunctions || other.bits.length != this.bits.length) { + throw new IllegalArgumentException("Incompatible bloom filters"); + } + + for (int i = 0; i < bits.length; i++) { + var oldBit = getBit(i); + if (!oldBit && other.getBit(i)) { + setBit(i); + } + } + } +} + diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java index 9549f7e..b658b28 100644 --- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java +++ b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java @@ -14,21 +14,14 @@ package com.linkedin.paldb; -import com.linkedin.paldb.api.Configuration; -import com.linkedin.paldb.api.PalDB; -import com.linkedin.paldb.api.StoreReader; -import com.linkedin.paldb.api.StoreWriter; +import com.linkedin.paldb.api.*; import com.linkedin.paldb.impl.GenerateTestData; -import com.linkedin.paldb.utils.DirectoryUtils; -import com.linkedin.paldb.utils.NanoBench; -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; +import com.linkedin.paldb.utils.*; import org.apache.commons.lang.RandomStringUtils; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; +import org.testng.annotations.*; + +import java.io.File; +import java.util.*; public class TestReadThroughput { @@ -50,7 +43,7 @@ public void cleanUp() { @Test public void testReadThroughput() { - List measures = new ArrayList(); + List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { Measure m = measure(i, 0, 0, false); @@ -78,68 +71,68 @@ public void testReadThroughputWithCache() { // UTILITY private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, final boolean frequentReads) { - + // Write store + File storeFile = new File(TEST_FOLDER, "paldb" + keysCount + "-" + valueLength + ".store"); // Generate keys long seed = 4242; final Integer[] keys = GenerateTestData.generateRandomIntKeys(keysCount, Integer.MAX_VALUE, seed); - - // Write store - File storeFile = new File(TEST_FOLDER, "paldb" + keysCount + "-" + valueLength + ".store"); - StoreWriter writer = PalDB.createWriter(storeFile, new Configuration()); - for (Integer key : keys) { - if (valueLength == 0) { - writer.put(key.toString(), Boolean.TRUE); - } else { - writer.put(key.toString(), RandomStringUtils.randomAlphabetic(valueLength)); - } - } - writer.close(); - + Configuration config = PalDB.newConfiguration(); // Get reader long cacheSize = 0; - Configuration config = PalDB.newConfiguration(); if (cacheSizeRatio > 0) { cacheSize = (long) (storeFile.length() * cacheSizeRatio); - config.set(Configuration.CACHE_ENABLED, "true"); - config.set(Configuration.CACHE_BYTES, String.valueOf(cacheSize)); + config.set(Configuration.BLOOM_FILTER_ENABLED, "true"); } else { - config.set(Configuration.CACHE_ENABLED, "false"); + config.set(Configuration.BLOOM_FILTER_ENABLED, "false"); + } + + + try (StoreWriter writer = PalDB.createWriter(storeFile, config)) { + for (Integer key : keys) { + if (valueLength == 0) { + writer.put(key.toString(), Boolean.TRUE.toString()); + } else { + writer.put(key.toString(), RandomStringUtils.randomAlphabetic(valueLength)); + } + } } - final StoreReader reader = PalDB.createReader(storeFile, config); - // Measure - NanoBench nanoBench = NanoBench.create(); - nanoBench.cpuOnly().warmUps(5).measurements(20).measure("Measure %d reads for %d keys with cache", new Runnable() { - @Override - public void run() { + int[] counter = new int[]{0, 0}; + try (StoreReader reader = PalDB.createReader(storeFile, config)) { + // Measure + NanoBench nanoBench = NanoBench.create(); + nanoBench.cpuOnly().warmUps(5).measurements(20).measure("Measure %d reads for %d keys with cache", () -> { Random r = new Random(); int length = keys.length; for (int i = 0; i < READS; i++) { - int index; + counter[1]++; + /*int index; if (i % 2 == 0 && frequentReads) { index = r.nextInt(length / 10); } else { index = r.nextInt(length); } - Integer key = keys[index]; - reader.get(key.toString()); - } - } - }); + Integer key = keys[index];*/ - // Close - reader.close(); + int key = r.nextInt(Integer.MAX_VALUE); + var value = reader.get(Integer.toString(key)); + if (value != null) { + counter[0]++; + } + } + }); - // Return measure - double rps = READS * nanoBench.getTps(); - return new Measure(storeFile.length(), rps, valueLength, cacheSize, keys.length); + // Return measure + double rps = READS * nanoBench.getTps(); + return new Measure(storeFile.length(), rps, counter[0], counter[1], keys.length); + } } private void report(String title, List measures) { System.out.println(title + "\n\n"); - System.out.println("FILE LENGTH;KEYS;RPS"); + System.out.println("FILE LENGTH;KEYS;RPS;VALUE LENGTH;TOTAL READS"); for (Measure m : measures) { - System.out.println(m.fileSize + ";" + m.keys + ";" + m.rps); + System.out.println(m.fileSize + ";" + m.keys + ";" + m.rps + ";" + m.valueLength + ";" + m.cacheSize); } } diff --git a/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java b/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java index e253150..8ce29f1 100644 --- a/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java +++ b/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java @@ -12,20 +12,16 @@ public void testAllPropertiesSet() { .withMemoryMapSegmentSize(500) .withMemoryMapDataEnabled(false) .withIndexLoadFactor(0.5) - .withLRUCacheEnabled(true) - .withCacheSizeLimit(200) - .withCacheInitialCapacity(100) - .withCacheLoadFactor(0.1) .withEnableCompression(true) + .withEnableBloomFilter(true) + .withBloomFilterErrorFactor(0.01) .build(); assertEquals(500, config.getInt(Configuration.MMAP_SEGMENT_SIZE)); assertFalse(config.getBoolean(Configuration.MMAP_DATA_ENABLED)); assertEquals(0.5, config.getDouble(Configuration.LOAD_FACTOR)); - assertTrue(config.getBoolean(Configuration.CACHE_ENABLED)); - assertEquals(200, config.getInt(Configuration.CACHE_BYTES)); - assertEquals(100, config.getInt(Configuration.CACHE_INITIAL_CAPACITY)); - assertEquals(0.1, config.getDouble(Configuration.CACHE_LOAD_FACTOR)); assertTrue(config.getBoolean(Configuration.COMPRESSION_ENABLED)); + assertTrue(config.getBoolean(Configuration.BLOOM_FILTER_ENABLED)); + assertEquals(0.01, config.getDouble(Configuration.BLOOM_FILTER_ERROR_FACTOR)); } } \ No newline at end of file diff --git a/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java b/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java index 891cbdb..4cf8512 100644 --- a/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java +++ b/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java @@ -18,8 +18,9 @@ import java.awt.*; import java.io.*; -import java.util.Arrays; +import java.util.*; +import static java.util.Collections.singletonList; import static org.testng.Assert.*; @@ -191,7 +192,7 @@ public void testGetLong() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getLong("foo"), 1l); + assertEquals(c.getLong("foo"), 1L); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -204,8 +205,8 @@ public void testGetLongDefault() { Configuration c = new Configuration(); c.set("foo", "1"); - assertEquals(c.getLong("foo", 2l), 1l); - assertEquals(c.getLong("bar", 2l), 2l); + assertEquals(c.getLong("foo", 2L), 1L); + assertEquals(c.getLong("bar", 2L), 2L); } @Test @@ -241,31 +242,8 @@ public void testGetListDefault() { Configuration c = new Configuration(); c.set("foo", "foo,bar"); - assertEquals(c.getList("foo", Arrays.asList("that")), Arrays.asList("foo", "bar")); - assertEquals(c.getList("bar", Arrays.asList("that")), Arrays.asList("that")); - } - - @Test - public void testSerialization() - throws Throwable { - Configuration c = new Configuration(); - c.set("foo", "bar"); - c.registerSerializer(new PointSerializer()); - - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(bos); - out.writeObject(c); - out.close(); - bos.close(); - - byte[] bytes = bos.toByteArray(); - ByteArrayInputStream bis = new ByteArrayInputStream(bytes); - ObjectInputStream in = new ObjectInputStream(bis); - Configuration sc = (Configuration) in.readObject(); - in.close(); - bis.close(); - - assertEquals(sc, c); + assertEquals(c.getList("foo", singletonList("that")), Arrays.asList("foo", "bar")); + assertEquals(c.getList("bar", singletonList("that")), singletonList("that")); } // UTILITY @@ -278,13 +256,14 @@ public Point read(DataInput input) { } @Override - public void write(DataOutput output, Point input) { - + public Class serializedClass() { + return Point.class; } @Override - public int getWeight(Point instance) { - return 0; + public void write(DataOutput output, Point input) { + } + } } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java b/paldb/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java index a851c19..924238a 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java @@ -14,11 +14,10 @@ package com.linkedin.paldb.impl; -import java.util.HashSet; -import java.util.Random; -import java.util.Set; import org.apache.commons.lang.RandomStringUtils; +import java.util.*; + /** * Utility to generate test data. @@ -53,7 +52,7 @@ public static Integer[] generateRandomIntKeys(int count, long seed) { public static Integer[] generateRandomIntKeys(int count, int range, long seed) { Random random = new Random(seed); - Set set = new HashSet(count); + Set set = new HashSet<>(count); while (set.size() < count) { set.add(random.nextInt(range)); } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java index a102bd9..7609285 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java @@ -15,7 +15,6 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.Serializer; -import com.linkedin.paldb.utils.DataInputOutput; import org.testng.annotations.*; import java.awt.*; @@ -26,100 +25,58 @@ public class TestSerializers { - private Serializers _serializers; + private Serializers serializers; @BeforeMethod public void setUp() { - _serializers = new Serializers(); + serializers = new Serializers(); } @Test public void testRegister() { ColorSerializer i = new ColorSerializer(); - _serializers.registerSerializer(i); - assertSame(_serializers.getSerializer(Color.class), i); - assertEquals(_serializers.getIndex(Color.class), 0); + serializers.registerSerializer(i); + assertSame(serializers.getSerializer(Color.class), i); } @Test public void testRegisterTwice() { ColorSerializer i1 = new ColorSerializer(); ColorSerializer i2 = new ColorSerializer(); - _serializers.registerSerializer(i1); - _serializers.registerSerializer(i2); - assertSame(_serializers.getSerializer(Color.class), i1); + serializers.registerSerializer(i1); + serializers.registerSerializer(i2); + assertSame(serializers.getSerializer(Color.class), i1); } @Test public void testRegisterTwo() { ColorSerializer i = new ColorSerializer(); PointSerializer f = new PointSerializer(); - _serializers.registerSerializer(i); - _serializers.registerSerializer(f); - assertSame(_serializers.getSerializer(Color.class), i); - assertEquals(_serializers.getIndex(Color.class), 0); - assertSame(_serializers.getSerializer(Point.class), f); - assertEquals(_serializers.getIndex(Point.class), 1); + serializers.registerSerializer(i); + serializers.registerSerializer(f); + assertSame(serializers.getSerializer(Color.class), i); + assertSame(serializers.getSerializer(Point.class), f); } @Test public void testGetSerializer() { ColorSerializer i = new ColorSerializer(); - _serializers.registerSerializer(i); - assertNull(_serializers.getSerializer(Point.class)); - assertNotNull(_serializers.getSerializer(Color.class)); + serializers.registerSerializer(i); + assertNull(serializers.getSerializer(Point.class)); + assertNotNull(serializers.getSerializer(Color.class)); } @Test - public void testGetIndex() { - ColorSerializer i = new ColorSerializer(); - _serializers.registerSerializer(i); - assertEquals(_serializers.getIndex(Color.class), 0); - } - - @Test - public void testGetByIndex() { - ColorSerializer i = new ColorSerializer(); - _serializers.registerSerializer(i); - assertSame(_serializers.getSerializer(0), i); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testGetByIndexMissing() { - _serializers.getSerializer(0); - } - - @Test(expectedExceptions = RuntimeException.class) - public void testMissingType() { - MissingTypeSerializer i = new MissingTypeSerializer(); - _serializers.registerSerializer(i); - } - - @Test(expectedExceptions = RuntimeException.class) - public void testObjectType() { - ObjectTypeSerializer i = new ObjectTypeSerializer(); - _serializers.registerSerializer(i); - } - - @Test - public void testSerialize() throws Throwable { - _serializers.registerSerializer(new ColorSerializer()); - DataInputOutput dio = new DataInputOutput(); - Serializers.serialize(dio, _serializers); - byte[] bytes = dio.toByteArray(); - dio = new DataInputOutput(bytes); - _serializers.clear(); - Serializers.deserialize(dio, _serializers); - assertNotNull(_serializers.getSerializer(Color.class)); - assertEquals(_serializers.getIndex(Color.class), 0); - assertNotNull(_serializers.getSerializer(0)); + public void testSerialize() { + serializers.registerSerializer(new ColorSerializer()); + assertNotNull(serializers.getSerializer(Color.class)); } @Test - public void testInterfaceType() throws Throwable { + public void testInterfaceType() { SerializerWithInterface i = new SerializerWithInterface(); - _serializers.registerSerializer(i); - assertSame(_serializers.getSerializer(AnInterface.class), i); + serializers.registerSerializer(i); + assertSame(serializers.getSerializer(AnInterface.class), i); } // HELPER @@ -132,71 +89,37 @@ public Color read(DataInput input) { } @Override - public void write(DataOutput output, Color input) { - + public Class serializedClass() { + return Color.class; } @Override - public int getWeight(Color instance) { - return 0; - } - } - - public static class PointSerializer implements Serializer { - - @Override - public Point read(DataInput input) { - return null; - } - - @Override - public void write(DataOutput output, Point input) { + public void write(DataOutput output, Color input) { } - @Override - public int getWeight(Point instance) { - return 0; - } } - public static class MissingTypeSerializer implements Serializer { + public static class PointSerializer implements Serializer { @Override - public Object read(DataInput input) { + public Point read(DataInput input) { return null; } @Override - public void write(DataOutput output, Object input) { - - } - - @Override - public int getWeight(Object instance) { - return 0; + public Class serializedClass() { + return Point.class; } - } - - public static class ObjectTypeSerializer implements Serializer { @Override - public Object read(DataInput input) { - return null; - } - - @Override - public void write(DataOutput output, Object input) { + public void write(DataOutput output, Point input) { } - @Override - public int getWeight(Object instance) { - return 0; - } } - public static interface AnInterface { + public interface AnInterface { } @@ -212,13 +135,14 @@ public AnInterface read(DataInput input) { } @Override - public void write(DataOutput output, AnInterface input) { - + public Class serializedClass() { + return AnInterface.class; } @Override - public int getWeight(AnInterface instance) { - return 0; + public void write(DataOutput output, AnInterface input) { + } + } } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java deleted file mode 100644 index 051792e..0000000 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageCache.java +++ /dev/null @@ -1,294 +0,0 @@ -/* -* Copyright 2015 LinkedIn Corp. All rights reserved. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -*/ - -package com.linkedin.paldb.impl; - -import com.linkedin.paldb.api.*; -import org.testng.annotations.*; - -import static org.testng.Assert.*; - - -public class TestStorageCache { - - private static final int ENTRY_SIZE = 16 * 2 + StorageCache.OVERHEAD; - - private Configuration configuration; - - @BeforeMethod - public void setUp() { - configuration = PalDB.newConfiguration(); - configuration.set(Configuration.CACHE_ENABLED, "true"); - } - - @Test - public void testContainsValid() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, 0); - assertTrue(cache.contains(0)); - } - - @Test - public void testContainsInValid() { - StorageCache cache = StorageCache.initCache(configuration); - assertFalse(cache.contains(0)); - } - - @Test - public void testEmpty() { - StorageCache cache = StorageCache.initCache(configuration); - assertNull(cache.get(0)); - assertEquals(cache.size(), 0); - } - - @Test - public void testPutOneItem() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, 0); - assertNotNull(cache.get(0)); - assertEquals(cache.size(), 1); - } - - @Test - public void testPutTwice() { - Integer second = 1; - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, 1); - cache.put(0, second); - assertSame(cache.get(0), second); - } - - @Test - public void testPutZeroSize() { - StorageCache cache = StorageCache.initCache(configuration); - cache.setMaxWeight(0); - cache.put(0, 1); - assertEquals(cache.size(), 0); - } - - @Test - public void testPutTwiceObjectSize() { - StorageCache cache = StorageCache.initCache(configuration); - cache.setMaxWeight(ENTRY_SIZE); - cache.put(0, 0); - cache.put(1, 1); - assertEquals(cache.size(), 1); - assertNull(cache.get(0)); - assertNotNull(cache.get(1)); - } - - @Test - public void putSameCheckWeight() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, 0); - long weight = cache.getWeight(); - cache.put(0, 0); - assertEquals(cache.getWeight(), weight); - } - - @Test - public void testPutGet() { - int objs = 100; - StorageCache cache = StorageCache.initCache(configuration); - cache.setMaxWeight(ENTRY_SIZE * objs); - for (int i = 0; i < objs; i++) { - cache.put(i, i); - } - assertEquals(cache.size(), 100); - for (int i = 0; i < objs; i++) { - assertNotNull(cache.get(i)); - } - } - - @Test - public void testCheckOrder() { - int objs = 100; - int capacity = 50; - StorageCache cache = StorageCache.initCache(configuration); - cache.setMaxWeight(ENTRY_SIZE * capacity); - for (int i = 0; i < objs; i++) { - cache.put(i, i); - } - assertEquals(cache.size(), capacity); - for (int i = 0; i < objs; i++) { - if (i < capacity) { - assertNull(cache.get(i)); - } else { - assertNotNull(cache.get(i)); - } - } - } - - @Test - public void testCheckAccessOrderGet() { - StorageCache cache = StorageCache.initCache(configuration); - cache.setMaxWeight(ENTRY_SIZE * 3); - cache.put(0, 0); - cache.put(1, 1); - cache.get(0); - cache.put(2, 2); - assertEquals(cache.size(), 3); - cache.put(3, 2); - assertNull(cache.get(1)); - assertNotNull(cache.get(0)); - } - - @Test - public void testCheckAccessOrderPut() { - StorageCache cache = StorageCache.initCache(configuration); - cache.setMaxWeight(ENTRY_SIZE * 3); - cache.put(0, 0); - cache.put(1, 1); - cache.put(0, 0); - cache.put(2, 2); - assertEquals(cache.size(), 3); - cache.put(3, 2); - assertNull(cache.get(1)); - assertNotNull(cache.get(0)); - } - - @Test - public void testWeightKeyObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, 0); - assertEquals(cache.getWeight(), ENTRY_SIZE); - } - - @Test - public void testWeightKeyArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(new Object[]{0, 1}, 0); - assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueIntArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new int[]{1, 2}); - assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueLongArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new long[]{1, 2}); - assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueDoubleArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new double[]{1.0, 2.0}); - assertEquals(cache.getWeight(), 16 + 16 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueFloatArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new float[]{1.0F, 2.0F}); - assertEquals(cache.getWeight(), 16 + 8 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueBooleanArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new boolean[]{true, false}); - assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueByteArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new byte[]{1, 2}); - assertEquals(cache.getWeight(), 16 + 2 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueShortArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new short[]{1, 2}); - assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueCharArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new char[]{'a', 'b'}); - assertEquals(cache.getWeight(), 16 + 4 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueStringArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new String[]{"one", "two"}); - assertEquals(cache.getWeight(), 16 + 46 * 2 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueInt2DArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new int[][]{{1, 2}, {3, 4}}); - assertEquals(cache.getWeight(), 16 + 8 * 2 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueLong2DArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new long[][]{{1, 2}, {3, 4}}); - assertEquals(cache.getWeight(), 16 + 16 * 2 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueStringObject() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new String("one")); - assertEquals(cache.getWeight(), 16 + 46 + StorageCache.OVERHEAD); - } - - @Test - public void testWeightValueObjectArrayObjects() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, new Object[]{0, 1}); - assertEquals(cache.getWeight(), 16 + 32 + StorageCache.OVERHEAD); - } - - @Test - public void testNullValue() { - StorageCache cache = StorageCache.initCache(configuration); - cache.put(0, null); - assertEquals(cache.size(), 1); - assertEquals(cache.get(0), StorageCache.NULL_VALUE); - } - - @Test - public void testDisabled() { - Configuration cfg = new Configuration(); - cfg.set(Configuration.CACHE_ENABLED, "false"); - StorageCache cache = StorageCache.initCache(cfg); - assertEquals(cache.size(), 0); - assertNull(cache.get("foo")); - assertFalse(cache.contains("foo")); - } - - @Test - public void testDisabledPut() { - Configuration cfg = new Configuration(); - cfg.set(Configuration.CACHE_ENABLED, "false"); - StorageCache cache = StorageCache.initCache(cfg); - cache.put(0, "foo"); - assertEquals(cache.size(), 0); - assertNull(cache.get(1)); - assertFalse(cache.contains(2)); - } -} diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java index b360adf..4de549a 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java @@ -139,9 +139,10 @@ public Point read(DataInput input) } @Override - public int getWeight(Point instance) { - return 0; + public Class serializedClass() { + return Point.class; } + }); Point p = new Point(42, 9); byte[] buf = serialization.serialize(p); @@ -166,9 +167,10 @@ public Point[] read(DataInput input) } @Override - public int getWeight(Point[] instance) { - return 0; + public Class serializedClass() { + return Point[].class; } + }); Point[] p = new Point[]{new Point(42, 9)}; byte[] buf = serialization.serialize(p); @@ -185,21 +187,22 @@ public ImplementsA read(DataInput dataInput) throws IOException { } @Override - public void write(DataOutput dataOutput, ImplementsA input) throws IOException { - dataOutput.writeInt(input.getVal()); + public Class serializedClass() { + return ImplementsA.class; } @Override - public int getWeight(ImplementsA instance) { - return 0; + public void write(DataOutput dataOutput, ImplementsA input) throws IOException { + dataOutput.writeInt(input.getVal()); } + }); ImplementsA a = new ImplementsA(42); byte[] buf = serialization.serialize(a); assertEquals(serialization.deserialize(buf), a); } - @Test + @Test(expectedExceptions = UnsupportedTypeException.class) public void testInheritedSerializer() throws Throwable { configuration.registerSerializer(new Serializer() { @@ -209,14 +212,15 @@ public A read(DataInput dataInput) throws IOException { } @Override - public void write(DataOutput dataOutput, A input) throws IOException { - dataOutput.writeInt(input.getVal()); + public Class serializedClass() { + return A.class; } @Override - public int getWeight(A instance) { - return 0; + public void write(DataOutput dataOutput, A input) throws IOException { + dataOutput.writeInt(input.getVal()); } + }); ImplementsA a = new ImplementsA(42); byte[] buf = serialization.serialize(a); @@ -642,7 +646,7 @@ private static char[] generateCharArray(int size) { return array; } - private static interface A { + private interface A { int getVal(); } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java index 87dabd0..6e68887 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -233,7 +233,7 @@ public void testTwoFirstKeyLength() { testKeyLength(key2, 2); //Write - writeStore(storeFile, new Object[]{key1, key2}, new Object[]{1, 6}); + writeStore(storeFile, new Integer[]{key1, key2}, new Integer[]{1, 6}); //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { @@ -257,7 +257,7 @@ public void testKeyLengthGap() { testKeyLength(key2, 3); //Write - writeStore(storeFile, new Object[]{key1, key2}, new Object[]{1, 6}); + writeStore(storeFile, new Integer[]{key1, key2}, new Integer[]{1, 6}); //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { @@ -283,7 +283,7 @@ public void testKeyLengthStartTwo() { testKeyLength(key2, 3); //Write - writeStore(storeFile, new Object[]{key1, key2}, new Object[]{1, 6}); + writeStore(storeFile, new Integer[]{key1, key2}, new Integer[]{1, 6}); //Read try (StoreReader reader = PalDB.createReader(storeFile, new Configuration())) { @@ -330,8 +330,8 @@ public void testDataOnTwoBuffers() throws IOException { @Test public void testDataSizeOnTwoBuffers() throws IOException { - Object[] keys = new Object[]{1, 2, 3}; - Object[] values = new Object[]{GenerateTestData.generateStringData(100), GenerateTestData + Integer[] keys = new Integer[]{1, 2, 3}; + String[] values = new String[]{GenerateTestData.generateStringData(100), GenerateTestData .generateStringData(10000), GenerateTestData.generateStringData(100)}; StorageSerialization serialization = new StorageSerialization(new Configuration()); @@ -549,8 +549,8 @@ private void testReadKeyToIntArray(K[] keys) { } } - private void writeStore(File location, Object[] keys, Object[] values) { - try (StoreWriter writer = PalDB.createWriter(location, new Configuration())) { + private void writeStore(File location, K[] keys, V[] values) { + try (StoreWriter writer = PalDB.createWriter(location, new Configuration())) { writer.putAll(keys, values); } } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java index 93fcaba..5f36c89 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java @@ -47,14 +47,14 @@ public void cleanUp() { @SafeVarargs private StoreReader readerForMany(V... values) { - Configuration configuration = new Configuration(); + var configuration = new Configuration(); configuration.registerSerializer(new PointSerializer()); try (StoreWriter writer = PalDB.createWriter(storeFile, configuration)) { for (int i = 0; i < values.length; i++) { writer.put(i, values[i]); } } - return PalDB.createReader(storeFile, PalDBConfigBuilder.create().build()); + return PalDB.createReader(storeFile, configuration); } private StoreReader readerFor(V value) { @@ -487,6 +487,11 @@ public Point read(DataInput input) return new Point(input.readInt(), input.readInt()); } + @Override + public Class serializedClass() { + return Point.class; + } + @Override public void write(DataOutput output, Point input) throws IOException { @@ -494,9 +499,5 @@ public void write(DataOutput output, Point input) output.writeInt(input.y); } - @Override - public int getWeight(Point instance) { - return 8; - } } } From 2004ccffba597a0c1288ab04d688b4c34c691fa1 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Sun, 17 Nov 2019 23:51:26 +0200 Subject: [PATCH 15/20] Use thread-safe murmur3 hash. Bloom filter optimizations. --- README.md | 26 ++-- paldb/build.gradle | 2 + .../api/errors/DuplicateKeyException.java | 8 + .../paldb/api/errors/OutOfDiskSpace.java | 8 + .../UnsupportedTypeException.java | 5 +- .../linkedin/paldb/impl/StorageReader.java | 17 +-- .../paldb/impl/StorageSerialization.java | 1 + .../linkedin/paldb/impl/StorageWriter.java | 14 +- .../com/linkedin/paldb/utils/BloomFilter.java | 25 ++- .../com/linkedin/paldb/utils/HashUtils.java | 15 +- .../linkedin/paldb/TestReadThroughput.java | 49 +++--- .../paldb/impl/TestStorageSerialization.java | 1 + .../linkedin/paldb/utils/BloomFilterTest.java | 142 ++++++++++++++++++ .../linkedin/paldb/utils/TestHashUtils.java | 5 +- 14 files changed, 235 insertions(+), 83 deletions(-) create mode 100644 paldb/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java create mode 100644 paldb/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java rename paldb/src/main/java/com/linkedin/paldb/api/{ => errors}/UnsupportedTypeException.java (90%) create mode 100644 paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java diff --git a/README.md b/README.md index 4e52b5e..604f8f6 100644 --- a/README.md +++ b/README.md @@ -129,15 +129,13 @@ Write parameters: + `load.factor`, index load factor (double) [default: 0.75] + `compression.enabled`, enable compression (boolean) [default: false] ++ `bloom.filter.enabled`, enable bloom filter (boolean) [default: false] ++ `bloom.filter.error.factor`, bloom filter error rate (double) [default: 0.01] Read parameters: + `mmap.data.enabled`, enable memory mapping for data (boolean) [default: true] + `mmap.segment.size`, memory map segment size (bytes) [default: 1GB] -+ `cache.enabled`, LRU cache enabled (boolean) [default: false] -+ `cache.bytes`, cache limit (bytes) [default: Xmx - 100MB] -+ `cache.initial.capacity`, cache initial capacity (int) [default: 1000] -+ `cache.load.factor`, cache load factor (double) [default: 0.75] Configuration values are passed at init time. Example using fluent builder: ```java @@ -145,11 +143,8 @@ var config = PalDBConfigBuilder.create() .withMemoryMapSegmentSize(512 * 1024 * 1024) .withMemoryMapDataEnabled(false) .withIndexLoadFactor(0.75) - .withLRUCacheEnabled(true) - .withCacheSizeLimit(10_000) - .withCacheInitialCapacity(1000) - .withCacheLoadFactor(0.5) .withEnableCompression(true) + .withEnableBloomFilter(true) .build(); StoreReader reader = PalDB.createReader(new File("store.paldb"), config); ``` @@ -157,7 +152,7 @@ StoreReader reader = PalDB.createReader(new File("store.paldb"), A few tips on how configuration can affect performance: + Disabling memory mapping will significantly reduce performance as disk seeks will be performed instead. -+ Enabling the cache makes sense when the value size is large and there's a significant cost in deserialization. Otherwise, the cache adds an overhead. The cache is also useful when memory mapping is disabled. ++ Enabling the bloom filter makes sense when you expect to miss finding some values. It will greatly increase read performance in this case. + Compression can be enabled when the store size is a concern and the values are large (e.g. a sparse matrix). By default, PalDB already uses a compact serialization. Snappy is used for compression. Custom serializer @@ -180,20 +175,20 @@ public class PointSerializer implements Serializer { output.writeInt(point.x); output.writeInt(point.y); } - + @Override - public int getWeight(Point instance) { - return 8; + public Class serializedClass() { + return Point.class; } } ``` -The `write` method serializes the instance to the `DataOutput`. The `read` method deserializes from `DataInput` and creates new object instances. The `getWeight` method returns the estimated memory used by an instance in bytes. The latter is used by the cache to evaluate the amount of memory it's currently using. +The `write` method serializes the instance to the `DataOutput`. The `read` method deserializes from `DataInput` and creates new object instances. Serializer implementation should be registered using the `Configuration`: ```java -Configuration configuration = PalDB.newConfiguration(); +var configuration = PalDB.newConfiguration(); configuration.registerSerializer(new PointSerializer()); ``` @@ -210,7 +205,7 @@ Limitations ----------- + PalDB is optimal in replacing the usage of large in-memory data storage but still use memory (off-heap, yet much less) to do its job. Disabling memory mapping and relying on seeks is possible but is not what PalDB has been optimized for. + The size of the index is limited to 2GB. There's no limitation in the data size however. -+ PalDB is not thread-safe at the moment so synchronization should be done externally if multi-threaded. ++ PalDB reader is thread-safe and writer is not thread-safe at the moment so synchronization should be done externally if multi-threaded. Contributions ----------- @@ -220,4 +215,5 @@ Any helpful feedback is more than welcome. This includes feature requests, bug r Copyright & License ------------------- +PalDB © 2019 Linas Naginionis. Licensed under the terms of the Apache License, Version 2.0. PalDB © 2015 LinkedIn Corp. Licensed under the terms of the Apache License, Version 2.0. diff --git a/paldb/build.gradle b/paldb/build.gradle index 1d73853..25fc8ef 100644 --- a/paldb/build.gradle +++ b/paldb/build.gradle @@ -41,6 +41,8 @@ perfTest { dependencies { compile 'org.xerial.snappy:snappy-java:1.1.7.3' compile 'org.slf4j:slf4j-api:1.7.28' + compile 'commons-codec:commons-codec:1.13' + testCompile 'org.testng:testng:7.0.0' testCompile 'commons-lang:commons-lang:2.6' diff --git a/paldb/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java b/paldb/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java new file mode 100644 index 0000000..32a7466 --- /dev/null +++ b/paldb/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java @@ -0,0 +1,8 @@ +package com.linkedin.paldb.api.errors; + +public class DuplicateKeyException extends RuntimeException { + + public DuplicateKeyException(String message) { + super(message); + } +} diff --git a/paldb/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java b/paldb/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java new file mode 100644 index 0000000..2b9d15e --- /dev/null +++ b/paldb/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java @@ -0,0 +1,8 @@ +package com.linkedin.paldb.api.errors; + +public class OutOfDiskSpace extends RuntimeException { + + public OutOfDiskSpace(String message) { + super(message); + } +} diff --git a/paldb/src/main/java/com/linkedin/paldb/api/UnsupportedTypeException.java b/paldb/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java similarity index 90% rename from paldb/src/main/java/com/linkedin/paldb/api/UnsupportedTypeException.java rename to paldb/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java index 21f719d..282589d 100644 --- a/paldb/src/main/java/com/linkedin/paldb/api/UnsupportedTypeException.java +++ b/paldb/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java @@ -12,14 +12,15 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ -package com.linkedin.paldb.api; +package com.linkedin.paldb.api.errors; + +import com.linkedin.paldb.api.StoreReader; /** * Exception returned when a type isn't supported * * @see StoreReader */ -@SuppressWarnings("serial") public class UnsupportedTypeException extends RuntimeException { public UnsupportedTypeException(Object obj) { diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index 816660d..aa9d6bd 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -238,12 +238,12 @@ public byte[] get(byte[] key) throws IOException { return null; } - var ctx = context.get(); - long hash = ctx.hash(key); - if (bloomFilter != null && !bloomFilter.mightContain(hash, ctx.random)) { - return null; + long hash = HashUtils.hash(key); + if (bloomFilter != null && !bloomFilter.mightContain(hash)) { + return null; } + var ctx = context.get(); int numSlots = slots[keyLength]; int slotSize = slotSizes[keyLength]; int ixOffset = indexOffsets[keyLength]; @@ -276,9 +276,6 @@ private boolean isKey(byte[] slotBuffer, byte[] key) { } private static final class ThreadContext { - - private final HashUtils.Murmur3A hash = new HashUtils.Murmur3A(42); - private final Random random = new Random(); private final MappedByteBuffer indexBuffer; private final MappedByteBuffer[] dataBuffers; private final DataInputOutput sizeBuffer = new DataInputOutput(new byte[5]); @@ -293,12 +290,6 @@ private ThreadContext(MappedByteBuffer indexBuffer, MappedByteBuffer[] dataBuffe this.dataBuffers = dataBuffers; this.slotBuffer = slotBuffer; } - - public int hash(byte[] bytes) { - hash.reset(); - hash.update(bytes); - return hash.getIntValue() & 0x7fffffff; - } } //Close the reader channel diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java index b7a7948..c487c01 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java @@ -15,6 +15,7 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.*; +import com.linkedin.paldb.api.errors.UnsupportedTypeException; import com.linkedin.paldb.utils.*; import org.xerial.snappy.Snappy; diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index e0ffbca..b0d1ab7 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -15,6 +15,7 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.Configuration; +import com.linkedin.paldb.api.errors.*; import com.linkedin.paldb.utils.*; import org.slf4j.*; @@ -59,8 +60,6 @@ public class StorageWriter { // Number of collisions private int collisions; - private HashUtils hashUtils; - StorageWriter(Configuration configuration, OutputStream stream) { config = configuration; loadFactor = config.getDouble(Configuration.LOAD_FACTOR); @@ -82,8 +81,7 @@ public class StorageWriter { dataLengths = new long[0]; maxOffsetLengths = new int[0]; keyCounts = new int[0]; - hashUtils = new HashUtils(); - } + } public void put(byte[] key, byte[] value) throws IOException { @@ -295,7 +293,7 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti long offset = LongPacker.unpackLong(tempIndexStream); // Hash - long hash = hashUtils.hash(keyBuffer); + long hash = HashUtils.hash(keyBuffer); if (bloomFilter != null) { bloomFilter.add(hash); } @@ -318,8 +316,8 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti collision = true; // Check for duplicates if (Arrays.equals(keyBuffer, Arrays.copyOf(slotBuffer, keyLength))) { - throw new RuntimeException( - String.format("A duplicate key has been found for for key bytes %s", Arrays.toString(keyBuffer))); + throw new DuplicateKeyException( + String.format("A duplicate key has been found for key bytes %s", Arrays.toString(keyBuffer))); } } } @@ -360,7 +358,7 @@ private void checkFreeDiskSpace(List inputFiles) { log.info("Usable free space on the system is {} Mb", new DecimalFormat("#,##0.0").format(usableSpace / (1024 * 1024))); if (totalSize / (double) usableSpace >= 0.66) { - throw new RuntimeException("Aborting because there isn't enough free disk space"); + throw new OutOfDiskSpace("Aborting because there isn't enough free disk space"); } } diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java index 8f5b1f8..c3eac21 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java @@ -1,5 +1,7 @@ package com.linkedin.paldb.utils; +import org.apache.commons.codec.digest.MurmurHash3; + import java.util.*; import static java.lang.Math.log; @@ -9,7 +11,6 @@ public class BloomFilter { private final long[] bits; private final int hashFunctions; // Number of hash functions private static final double LN2 = 0.6931471805599453; // ln(2) - private final Random random = new Random(); private final int sizeInBits; public BloomFilter(int elements, int sizeInBits) { @@ -31,14 +32,20 @@ public BloomFilter(int hashFunctions, int bitSize, long[] bits) { } public void add(long hash) { - random.setSeed(hash); - for (int i = 0; i < hashFunctions; i++) { - int value = Math.abs(random.nextInt() % sizeInBits); + int value = Math.abs(MurmurHash3.hash32(hash, i) % sizeInBits); setBit(value); } } + public boolean mightContain(long hash) { + for (int i = 0; i < hashFunctions; i++) { + int value = Math.abs(MurmurHash3.hash32(hash, i) % sizeInBits); + if (!getBit(value)) return false; + } + return true; + } + public long[] bits() { return bits; } @@ -55,16 +62,6 @@ private boolean getBit(int position) { return ((bits[flagIndex] >> bitIndexInFlag) & 1L) == 1; } - public boolean mightContain(long hash, Random random) { - random.setSeed(hash); - - for (int i = 0; i < hashFunctions; i++) { - int value = Math.abs(random.nextInt() % sizeInBits); - if (!getBit(value)) return false; - } - return true; - } - public void clear() { for (int i = 0; i < bits.length; i++) { bits[i] = 0L; diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java b/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java index 8dfd282..18d3733 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java @@ -14,27 +14,26 @@ package com.linkedin.paldb.utils; +import org.apache.commons.codec.digest.MurmurHash3; + import java.util.zip.Checksum; /** * Hashing utility. */ -public class HashUtils { +public final class HashUtils { - // Hash implementation - private final Murmur3A hash = new Murmur3A(42); + private HashUtils() { } - /** + /** * Returns the positive hash for the given bytes. * * @param bytes bytes to hash * @return hash */ - public int hash(byte[] bytes) { - hash.reset(); - hash.update(bytes); - return hash.getIntValue() & 0x7fffffff; + public static long hash(byte[] bytes) { + return MurmurHash3.hash32(bytes) & 0x7fffffff; } /** diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java index b658b28..303a890 100644 --- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java +++ b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java @@ -57,10 +57,24 @@ public void testReadThroughput() { @Test public void testReadThroughputWithCache() { - List measures = new ArrayList(); + List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { - Measure m = measure(i, 0, 0.2, false); + Measure m = measure(i, 0, 0.05, false); + + measures.add(m); + } + + report("READ THROUGHPUT WITH CACHE (Set int -> boolean)", measures); + } + + @Test + public void testReadThroughputWithCacheRandomFinds() { + + List measures = new ArrayList<>(); + int max = 10000000; + for (int i = 100; i <= max; i *= 10) { + Measure m = measure(i, 0, 0.05, true); measures.add(m); } @@ -70,22 +84,20 @@ public void testReadThroughputWithCache() { // UTILITY - private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, final boolean frequentReads) { + private Measure measure(int keysCount, int valueLength, double errorRate, final boolean randomReads) { // Write store File storeFile = new File(TEST_FOLDER, "paldb" + keysCount + "-" + valueLength + ".store"); // Generate keys long seed = 4242; final Integer[] keys = GenerateTestData.generateRandomIntKeys(keysCount, Integer.MAX_VALUE, seed); - Configuration config = PalDB.newConfiguration(); - // Get reader - long cacheSize = 0; - if (cacheSizeRatio > 0) { - cacheSize = (long) (storeFile.length() * cacheSizeRatio); - config.set(Configuration.BLOOM_FILTER_ENABLED, "true"); - } else { - config.set(Configuration.BLOOM_FILTER_ENABLED, "false"); + + var configBuilder = PalDBConfigBuilder.create(); + if (errorRate > 0) { + configBuilder.withEnableBloomFilter(true) + .withBloomFilterErrorFactor(errorRate); } + var config = configBuilder.build(); try (StoreWriter writer = PalDB.createWriter(storeFile, config)) { for (Integer key : keys) { @@ -102,19 +114,16 @@ private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, f // Measure NanoBench nanoBench = NanoBench.create(); nanoBench.cpuOnly().warmUps(5).measurements(20).measure("Measure %d reads for %d keys with cache", () -> { - Random r = new Random(); + Random r = new Random(42); int length = keys.length; for (int i = 0; i < READS; i++) { counter[1]++; - /*int index; - if (i % 2 == 0 && frequentReads) { - index = r.nextInt(length / 10); + int key; + if (randomReads) { + key = r.nextInt(Integer.MAX_VALUE); } else { - index = r.nextInt(length); + key = keys[r.nextInt(length)]; } - Integer key = keys[index];*/ - - int key = r.nextInt(Integer.MAX_VALUE); var value = reader.get(Integer.toString(key)); if (value != null) { counter[0]++; @@ -130,7 +139,7 @@ private Measure measure(int keysCount, int valueLength, double cacheSizeRatio, f private void report(String title, List measures) { System.out.println(title + "\n\n"); - System.out.println("FILE LENGTH;KEYS;RPS;VALUE LENGTH;TOTAL READS"); + System.out.println("FILE LENGTH;KEYS;RPS;VALUES FOUND;TOTAL READS"); for (Measure m : measures) { System.out.println(m.fileSize + ";" + m.keys + ";" + m.rps + ";" + m.valueLength + ";" + m.cacheSize); } diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java index 4de549a..110b701 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java @@ -15,6 +15,7 @@ package com.linkedin.paldb.impl; import com.linkedin.paldb.api.*; +import com.linkedin.paldb.api.errors.UnsupportedTypeException; import org.testng.Assert; import org.testng.annotations.*; diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java b/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java new file mode 100644 index 0000000..7120610 --- /dev/null +++ b/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java @@ -0,0 +1,142 @@ +package com.linkedin.paldb.utils; + +import org.testng.annotations.*; + +import java.lang.management.*; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.stream.*; + +import static org.testng.Assert.*; + +public class BloomFilterTest { + + private int elements = 1_000_000; + private int bitsize = 10_000_000; + private BloomFilter filter; + private Random prng; + private ThreadMXBean bean; + + @BeforeMethod + void setUp() { + bean = ManagementFactory.getThreadMXBean(); + prng = new Random(); + prng.setSeed(0); + filter = new BloomFilter(elements, bitsize); + } + + @Test + void should_calculate_bit_size_correctly() { + var sut = new BloomFilter(10_000, 0.01); + + assertEquals(95851 , sut.bitSize()); + } + + @Test + void should_calculate_hash_functions_correctly() { + var sut = new BloomFilter(10_000, 0.01); + + assertEquals(7 , sut.hashFunctions()); + } + + @Test + void should_provide_hashcode() { + var sut1 = new BloomFilter(10_000, 0.01); + sut1.add(HashUtils.hash("test".getBytes())); + var sut1_1 = new BloomFilter(10_000, 0.01); + sut1_1.add(HashUtils.hash("test".getBytes())); + + assertEquals(sut1.hashCode(), sut1_1.hashCode()); + } + + @Test + void should_test_multi_thread_get() { + var sut1 = new BloomFilter(10_000, 0.01); + sut1.add(HashUtils.hash("test".getBytes())); + + var futures = IntStream.range(0, 1000) + .mapToObj(i -> CompletableFuture.runAsync(() -> assertTrue(sut1.mightContain(HashUtils.hash("test".getBytes()))))) + .collect(Collectors.toList()); + + futures.forEach(CompletableFuture::join); + } + + @Test + void correctness() { + System.out.println("Testing correctness.\n"+ + "Creating a Set and filling it together with our filter..."); + filter.clear(); + Set inside = new HashSet<>((int)(elements / 0.75)); + while(inside.size() < elements) { + long v = prng.nextLong(); + inside.add(v); + filter.add(v); + assertTrue(filter.mightContain(v), "There should be no false negative: " + v); + } + + // testing + int found = 0, total = 0; + double rate = 0; + while (total < elements) { + long v = prng.nextLong(); + if (inside.contains(v)) continue; + total++; + found += filter.mightContain(v) ? 1 : 0; + + rate = (float) found / total; + if (total % 1000 == 0 || total == elements) { + System.out.format( + "\rElements incorrectly found to be inside: %8d/%-8d (%3.2f%%)", + found, total, 100*rate + ); + } + } + System.out.println("\n"); + + double ln2 = Math.log(2); + double expectedRate = Math.exp(-ln2*ln2 * bitsize / elements); + assertTrue(rate <= expectedRate * 1.10, "error rate p = e^(-ln2^2*m/n)"); + } + + @Test + void insertion() { + System.out.println("Testing insertion speed..."); + + filter.clear(); + long start = bean.getCurrentThreadCpuTime(); + for(int i=0; i 0); + Assert.assertTrue(HashUtils.hash(new byte[0]) > 0); } } From 25250f16348b2185cb6df40084c02f71e642abc0 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Mon, 18 Nov 2019 11:14:59 +0200 Subject: [PATCH 16/20] Bloom filter performance improvements. --- paldb/build.gradle | 3 +- .../linkedin/paldb/impl/StorageReader.java | 5 +- .../linkedin/paldb/impl/StorageWriter.java | 2 +- .../com/linkedin/paldb/utils/BloomFilter.java | 12 +- .../com/linkedin/paldb/utils/HashUtils.java | 200 ++++++++---------- .../linkedin/paldb/TestReadThroughput.java | 29 ++- .../linkedin/paldb/utils/BloomFilterTest.java | 49 +++-- .../linkedin/paldb/utils/TestHashUtils.java | 14 +- 8 files changed, 165 insertions(+), 149 deletions(-) diff --git a/paldb/build.gradle b/paldb/build.gradle index 25fc8ef..098c4e9 100644 --- a/paldb/build.gradle +++ b/paldb/build.gradle @@ -41,10 +41,9 @@ perfTest { dependencies { compile 'org.xerial.snappy:snappy-java:1.1.7.3' compile 'org.slf4j:slf4j-api:1.7.28' - compile 'commons-codec:commons-codec:1.13' - testCompile 'org.testng:testng:7.0.0' + testCompile 'commons-codec:commons-codec:1.13' testCompile 'commons-lang:commons-lang:2.6' testCompile 'ch.qos.logback:logback-classic:1.2.3' diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index aa9d6bd..b193531 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -238,8 +238,7 @@ public byte[] get(byte[] key) throws IOException { return null; } - long hash = HashUtils.hash(key); - if (bloomFilter != null && !bloomFilter.mightContain(hash)) { + if (bloomFilter != null && !bloomFilter.mightContain(key)) { return null; } @@ -248,7 +247,7 @@ public byte[] get(byte[] key) throws IOException { int slotSize = slotSizes[keyLength]; int ixOffset = indexOffsets[keyLength]; long dtOffset = dataOffsets[keyLength]; - + long hash = HashUtils.hash(key); for (int probe = 0; probe < numSlots; probe++) { int slot = (int) ((hash + probe) % numSlots); diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index b0d1ab7..32e564a 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -295,7 +295,7 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti // Hash long hash = HashUtils.hash(keyBuffer); if (bloomFilter != null) { - bloomFilter.add(hash); + bloomFilter.add(keyBuffer); } boolean collision = false; diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java index c3eac21..5d0787b 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java @@ -1,8 +1,6 @@ package com.linkedin.paldb.utils; -import org.apache.commons.codec.digest.MurmurHash3; - -import java.util.*; +import java.util.Arrays; import static java.lang.Math.log; @@ -31,16 +29,16 @@ public BloomFilter(int hashFunctions, int bitSize, long[] bits) { this.hashFunctions = hashFunctions; } - public void add(long hash) { + public void add(byte[] bytes) { for (int i = 0; i < hashFunctions; i++) { - int value = Math.abs(MurmurHash3.hash32(hash, i) % sizeInBits); + int value = Math.abs(HashUtils.hash(bytes, i) % sizeInBits); setBit(value); } } - public boolean mightContain(long hash) { + public boolean mightContain(byte[] bytes) { for (int i = 0; i < hashFunctions; i++) { - int value = Math.abs(MurmurHash3.hash32(hash, i) % sizeInBits); + int value = Math.abs(HashUtils.hash(bytes, i) % sizeInBits); if (!getBit(value)) return false; } return true; diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java b/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java index 18d3733..68ea57a 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java @@ -14,9 +14,8 @@ package com.linkedin.paldb.utils; -import org.apache.commons.codec.digest.MurmurHash3; - -import java.util.zip.Checksum; +import java.lang.invoke.*; +import java.nio.ByteOrder; /** @@ -26,133 +25,116 @@ public final class HashUtils { private HashUtils() { } - /** + /** * Returns the positive hash for the given bytes. * * @param bytes bytes to hash * @return hash */ public static long hash(byte[] bytes) { - return MurmurHash3.hash32(bytes) & 0x7fffffff; + return hash32(bytes) & 0x7fffffff; } - /** - * Hash implementation, inspired from java-common. - * - * Originally developed for greenrobot by Markus Junginger. - */ - public static class Murmur3A implements Checksum { - - private static final int C1 = 0xcc9e2d51; - private static final int C2 = 0x1b873593; + public static int hash(byte[] bytes, int seed) { + return hash32(bytes, bytes.length, seed); + } - private final int seed; + // Constants for 32 bit variant + private static final int C1_32 = 0xcc9e2d51; + private static final int C2_32 = 0x1b873593; + private static final int R1_32 = 15; + private static final int R2_32 = 13; + private static final int M_32 = 5; + private static final int N_32 = 0xe6546b64; - private int h1; - private int length; + public static final int DEFAULT_SEED = 104729; - private int partialK1; - private int partialK1Pos; + //** MurMur3 ** + /** + * Generates 32 bit hash from byte array with the default seed. + * + * @param data - input byte array + * @return 32 bit hash + */ + public static int hash32(final byte[] data) { + return hash32(data, 0, data.length, DEFAULT_SEED); + } - public Murmur3A(int seed) { - this.seed = seed; - h1 = seed; - } + /** + * Generates 32 bit hash from byte array with the given length and seed. + * + * @param data - input byte array + * @param length - length of array + * @param seed - seed. (default 0) + * @return 32 bit hash + */ + public static int hash32(final byte[] data, final int length, final int seed) { + return hash32(data, 0, length, seed); + } - @Override - public void update(int b) { - switch (partialK1Pos) { - case 0: - partialK1 = 0xff & b; - partialK1Pos = 1; - break; - case 1: - partialK1 |= (0xff & b) << 8; - partialK1Pos = 2; - break; - case 2: - partialK1 |= (0xff & b) << 16; - partialK1Pos = 3; - break; - case 3: - partialK1 |= (0xff & b) << 24; - applyK1(partialK1); - partialK1Pos = 0; - break; - } - length++; - } + private static final VarHandle INT_HANDLE = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); - @Override - public void update(byte[] b, int off, int len) { - while (partialK1Pos != 0 && len > 0) { - update(b[off]); - off++; - len--; - } - - int remainder = len & 3; - int stop = off + len - remainder; - for (int i = off; i < stop; i += 4) { - int k1 = getIntLE(b, i); - applyK1(k1); - } - length += stop - off; - - for (int i = 0; i < remainder; i++) { - update(b[stop + i]); - } - } + private static int getIntLE(byte[] array, int offset) { + return (int)INT_HANDLE.get(array, offset); + } - public void update(byte[] b) { - update(b, 0, b.length); + /** + * Generates 32 bit hash from byte array with the given length, offset and seed. + * + * @param data - input byte array + * @param offset - offset of data + * @param length - length of array + * @param seed - seed. (default 0) + * @return 32 bit hash + */ + public static int hash32(final byte[] data, final int offset, final int length, final int seed) { + int hash = seed; + final int nblocks = length >> 2; + + // body + for (int i = 0; i < nblocks; i++) { + final int i_4 = i << 2; + final int k = getIntLE(data, offset + i_4); + hash = mix32(k, hash); } - private void applyK1(int k1) { - k1 *= C1; - k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15); - k1 *= C2; - - h1 ^= k1; - h1 = (h1 << 13) | (h1 >>> 19); // ROTL32(h1,13); - h1 = h1 * 5 + 0xe6546b64; + // tail + final int idx = nblocks << 2; + int k1 = 0; + switch (length - idx) { + case 3: + k1 ^= data[offset + idx + 2] << 16; + case 2: + k1 ^= data[offset + idx + 1] << 8; + case 1: + k1 ^= data[offset + idx]; + + // mix functions + k1 *= C1_32; + k1 = Integer.rotateLeft(k1, R1_32); + k1 *= C2_32; + hash ^= k1; } - @Override - public long getValue() { - return 0xFFFFFFFFL & getIntValue(); - } + return fmix32(length, hash); + } - public int getIntValue() { - int finished = h1; - if (partialK1Pos > 0) { - int k1 = partialK1 * C1; - k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15); - k1 *= C2; - finished ^= k1; - } - finished ^= length; - - // fmix - finished ^= finished >>> 16; - finished *= 0x85ebca6b; - finished ^= finished >>> 13; - finished *= 0xc2b2ae35; - finished ^= finished >>> 16; - - return finished; - } + private static int mix32(int k, int hash) { + k *= C1_32; + k = Integer.rotateLeft(k, R1_32); + k *= C2_32; + hash ^= k; + return Integer.rotateLeft(hash, R2_32) * M_32 + N_32; + } - @Override - public void reset() { - h1 = seed; - length = 0; - partialK1Pos = 0; - } + private static int fmix32(final int length, int hash) { + hash ^= length; + hash ^= (hash >>> 16); + hash *= 0x85ebca6b; + hash ^= (hash >>> 13); + hash *= 0xc2b2ae35; + hash ^= (hash >>> 16); - private int getIntLE(byte[] bytes, int index) { - return (bytes[index] & 0xff) | ((bytes[index + 1] & 0xff) << 8) | - ((bytes[index + 2] & 0xff) << 16) | (bytes[index + 3] << 24); - } + return hash; } } diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java index 303a890..badfea0 100644 --- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java +++ b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java @@ -20,24 +20,33 @@ import org.apache.commons.lang.RandomStringUtils; import org.testng.annotations.*; -import java.io.File; +import java.io.*; +import java.nio.file.*; import java.util.*; public class TestReadThroughput { - private File TEST_FOLDER = new File("testreadthroughput"); - private final int READS = 500000; + private File testFolder = createTempDir(); + private static final int READS = 500000; @BeforeMethod public void setUp() { - DirectoryUtils.deleteDirectory(TEST_FOLDER); - TEST_FOLDER.mkdir(); + DirectoryUtils.deleteDirectory(testFolder); + testFolder.mkdir(); } @AfterMethod public void cleanUp() { - DirectoryUtils.deleteDirectory(TEST_FOLDER); + DirectoryUtils.deleteDirectory(testFolder); + } + + private static File createTempDir() { + try { + return Files.createTempDirectory("testreadthroughput").toFile(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Test @@ -74,7 +83,7 @@ public void testReadThroughputWithCacheRandomFinds() { List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { - Measure m = measure(i, 0, 0.05, true); + Measure m = measure(i, 0, 0.01, true); measures.add(m); } @@ -86,7 +95,7 @@ public void testReadThroughputWithCacheRandomFinds() { private Measure measure(int keysCount, int valueLength, double errorRate, final boolean randomReads) { // Write store - File storeFile = new File(TEST_FOLDER, "paldb" + keysCount + "-" + valueLength + ".store"); + File storeFile = new File(testFolder, "paldb" + keysCount + "-" + valueLength + ".store"); // Generate keys long seed = 4242; final Integer[] keys = GenerateTestData.generateRandomIntKeys(keysCount, Integer.MAX_VALUE, seed); @@ -139,9 +148,9 @@ private Measure measure(int keysCount, int valueLength, double errorRate, final private void report(String title, List measures) { System.out.println(title + "\n\n"); - System.out.println("FILE LENGTH;KEYS;RPS;VALUES FOUND;TOTAL READS"); + System.out.println("FILE LENGTH;\tKEYS;\tRPS;\tVALUES FOUND;\tTOTAL READS"); for (Measure m : measures) { - System.out.println(m.fileSize + ";" + m.keys + ";" + m.rps + ";" + m.valueLength + ";" + m.cacheSize); + System.out.println(m.fileSize + ";\t" + m.keys + ";\t" + m.rps + ";\t" + m.valueLength + ";\t" + m.cacheSize); } } diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java b/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java index 7120610..d3bfe80 100644 --- a/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java +++ b/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java @@ -42,9 +42,9 @@ void should_calculate_hash_functions_correctly() { @Test void should_provide_hashcode() { var sut1 = new BloomFilter(10_000, 0.01); - sut1.add(HashUtils.hash("test".getBytes())); + sut1.add("test".getBytes()); var sut1_1 = new BloomFilter(10_000, 0.01); - sut1_1.add(HashUtils.hash("test".getBytes())); + sut1_1.add("test".getBytes()); assertEquals(sut1.hashCode(), sut1_1.hashCode()); } @@ -52,10 +52,10 @@ void should_provide_hashcode() { @Test void should_test_multi_thread_get() { var sut1 = new BloomFilter(10_000, 0.01); - sut1.add(HashUtils.hash("test".getBytes())); + sut1.add("test".getBytes()); var futures = IntStream.range(0, 1000) - .mapToObj(i -> CompletableFuture.runAsync(() -> assertTrue(sut1.mightContain(HashUtils.hash("test".getBytes()))))) + .mapToObj(i -> CompletableFuture.runAsync(() -> assertTrue(sut1.mightContain("test".getBytes())))) .collect(Collectors.toList()); futures.forEach(CompletableFuture::join); @@ -67,21 +67,27 @@ void correctness() { "Creating a Set and filling it together with our filter..."); filter.clear(); Set inside = new HashSet<>((int)(elements / 0.75)); + var bytes = new byte[10]; while(inside.size() < elements) { - long v = prng.nextLong(); + var v = Math.abs(prng.nextLong()); + var pos = LongPacker.packLong(bytes, v); + var valueBytes = Arrays.copyOf(bytes, pos); + inside.add(v); - filter.add(v); - assertTrue(filter.mightContain(v), "There should be no false negative: " + v); + filter.add(valueBytes); + assertTrue(filter.mightContain(valueBytes), "There should be no false negative: " + v); } // testing int found = 0, total = 0; double rate = 0; while (total < elements) { - long v = prng.nextLong(); + var v = Math.abs(prng.nextLong()); if (inside.contains(v)) continue; + var pos = LongPacker.packLong(bytes, v); + var valueBytes = Arrays.copyOf(bytes, pos); total++; - found += filter.mightContain(v) ? 1 : 0; + found += filter.mightContain(valueBytes) ? 1 : 0; rate = (float) found / total; if (total % 1000 == 0 || total == elements) { @@ -101,10 +107,15 @@ void correctness() { @Test void insertion() { System.out.println("Testing insertion speed..."); - + var bytes = new byte[10]; filter.clear(); long start = bean.getCurrentThreadCpuTime(); - for(int i=0; i 0); + assertTrue(HashUtils.hash(new byte[0]) > 0); + } + + @Test + public void testSameHash() { + var bytes = "foo".getBytes(); + assertEquals(HashUtils.hash(bytes, 42), MurmurHash3.hash32(bytes, bytes.length, 42)); } } From 4fde9282ca161293322cc137bdfbe4ea6a74f027 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Mon, 18 Nov 2019 23:25:45 +0200 Subject: [PATCH 17/20] More efficient thread safety for reader. Requires java 13 from now on. --- README.md | 2 +- paldb/build.gradle | 6 +- .../linkedin/paldb/impl/StorageReader.java | 128 ++++++++---------- .../linkedin/paldb/TestReadThroughput.java | 95 +++++++++---- .../com/linkedin/paldb/impl/TestStore.java | 4 +- 5 files changed, 135 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index 604f8f6..b9d3ae9 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ No, like a hashtable PalDB stores have no order. Build ----- -PalDB requires Java 11+ and gradle. The target Java version is 11. +PalDB requires Java 13+ and gradle. The target Java version is 13. ```bash gradle build diff --git a/paldb/build.gradle b/paldb/build.gradle index 098c4e9..cd6996f 100644 --- a/paldb/build.gradle +++ b/paldb/build.gradle @@ -3,8 +3,8 @@ apply plugin: 'jacoco' apply plugin: 'signing' apply plugin: 'maven' -sourceCompatibility = JavaVersion.VERSION_11 -targetCompatibility = JavaVersion.VERSION_11 +sourceCompatibility = JavaVersion.VERSION_13 +targetCompatibility = JavaVersion.VERSION_13 group = GROUP version = VERSION_NAME @@ -35,7 +35,7 @@ task perfTest(type: Test) { perfTest { useTestNG() - maxHeapSize = "2g" + maxHeapSize = "4g" } dependencies { diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index b193531..5687d34 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -21,6 +21,7 @@ import java.io.*; import java.nio.*; import java.nio.channels.FileChannel; +import java.nio.charset.MalformedInputException; import java.text.*; import java.util.*; @@ -57,9 +58,9 @@ public class StorageReader implements Iterable> { // Use MMap for data? private final boolean mMapData; // Buffers - private final ThreadLocal context; - private final BloomFilter bloomFilter; + private final MappedByteBuffer indexBuffer; + private final MappedByteBuffer[] dataBuffers; StorageReader(Configuration configuration, File file) throws IOException { // File path @@ -180,12 +181,8 @@ public class StorageReader implements Iterable> { //Check if data size fits in memory map limit mMapData = configuration.getBoolean(Configuration.MMAP_DATA_ENABLED); - context = mMapData ? - ThreadLocal.withInitial(() -> new ThreadContext( - initIndexBuffer(channel, indexOffset), - initDataBuffers(channel), - new byte[maxSlotSize])) : - ThreadLocal.withInitial(() -> new ThreadContext(initIndexBuffer(channel, indexOffset), new byte[maxSlotSize])); + indexBuffer = initIndexBuffer(channel, indexOffset); + dataBuffers = initDataBuffers(channel); //logging if (log.isDebugEnabled()) { @@ -242,24 +239,22 @@ public byte[] get(byte[] key) throws IOException { return null; } - var ctx = context.get(); int numSlots = slots[keyLength]; int slotSize = slotSizes[keyLength]; int ixOffset = indexOffsets[keyLength]; long dtOffset = dataOffsets[keyLength]; long hash = HashUtils.hash(key); - + var slotBuffer = new byte[slotSize]; for (int probe = 0; probe < numSlots; probe++) { int slot = (int) ((hash + probe) % numSlots); - ctx.indexBuffer.position(ixOffset + slot * slotSize); - ctx.indexBuffer.get(ctx.slotBuffer, 0, slotSize); - - long offset = LongPacker.unpackLong(ctx.slotBuffer, keyLength); + indexBuffer.get(ixOffset + slot * slotSize, slotBuffer, 0, slotSize); + long offset = LongPacker.unpackLong(slotBuffer, keyLength); if (offset == 0L) { + log.info("Got offset 0, returning..."); return null; } - if (isKey(ctx.slotBuffer, key)) { - return mMapData ? getMMapBytes(dtOffset + offset, ctx) : getDiskBytes(dtOffset + offset); + if (isKey(slotBuffer, key)) { + return mMapData ? getMMapBytes(dtOffset + offset) : getDiskBytes(dtOffset + offset); } } return null; @@ -274,79 +269,82 @@ private boolean isKey(byte[] slotBuffer, byte[] key) { return true; } - private static final class ThreadContext { - private final MappedByteBuffer indexBuffer; - private final MappedByteBuffer[] dataBuffers; - private final DataInputOutput sizeBuffer = new DataInputOutput(new byte[5]); - private final byte[] slotBuffer; - - private ThreadContext(MappedByteBuffer indexBuffer, byte[] slotBuffer) { - this(indexBuffer, null, slotBuffer); - } - - private ThreadContext(MappedByteBuffer indexBuffer, MappedByteBuffer[] dataBuffers, byte[] slotBuffer) { - this.indexBuffer = indexBuffer; - this.dataBuffers = dataBuffers; - this.slotBuffer = slotBuffer; - } - } - //Close the reader channel public void close() throws IOException { channel.close(); mappedFile.close(); - context.remove(); } public int getKeyCount() { return keyCount; } + private int remaining(ByteBuffer buffer, int pos) { + return buffer.limit() - pos; + } + //Read the data at the given offset, the data can be spread over multiple data buffers - private byte[] getMMapBytes(long offset, ThreadContext ctx) throws IOException { - //Read the first 4 bytes to get the size of the data - ByteBuffer buf = getDataBuffer(offset, ctx); - int maxLen = (int) Math.min(5, dataSize - offset); + private byte[] getMMapBytes(long offset) throws IOException { + var buf = dataBuffers[(int) (offset / segmentSize)]; + var pos = (int) (offset % segmentSize); - int size; - if (buf.remaining() >= maxLen) { + int maxLen = (int) Math.min(5, dataSize - offset); + //Read the first 4 bytes to get the size of the data + int size = -1; + if (remaining(buf, pos) >= maxLen) { //Continuous read - int pos = buf.position(); - size = LongPacker.unpackInt(buf); + int oldPos = pos; + + //unpack int + for (int i = 0, result = 0; i < 32; i += 7) { + int b = buf.get(pos++) & 0xffff; + result |= (b & 0x7F) << i; + if ((b & 0x80) == 0) { + size = result; + break; + } + } + if (size == -1) throw new MalformedInputException(Integer.BYTES); // Used in case of data is spread over multiple buffers - offset += buf.position() - pos; + offset += pos - oldPos; } else { //The size of the data is spread over multiple buffers int len = maxLen; int off = 0; - ctx.sizeBuffer.reset(); - while (len > 0) { - buf = getDataBuffer(offset + off, ctx); - int count = Math.min(len, buf.remaining()); - buf.get(ctx.sizeBuffer.getBuf(), off, count); - off += count; - len -= count; + + try (var sizeBuffer = new DataInputOutput(new byte[5])) { + while (len > 0) { + buf = dataBuffers[(int) ( (offset + off) / segmentSize)]; + pos = (int) ( (offset + off) % segmentSize); + int count = Math.min(len, remaining(buf, pos)); + buf.get(pos, sizeBuffer.getBuf(), off, count); + off += count; + len -= count; + } + size = LongPacker.unpackInt(sizeBuffer); + offset += sizeBuffer.getPos(); + + buf = dataBuffers[(int) (offset / segmentSize)]; + pos = (int) (offset % segmentSize); } - size = LongPacker.unpackInt(ctx.sizeBuffer); - offset += ctx.sizeBuffer.getPos(); - buf = getDataBuffer(offset, ctx); } //Create output bytes byte[] res = new byte[size]; //Check if the data is one buffer - if (buf.remaining() >= size) { + if (remaining(buf, pos) >= size) { //Continuous read - buf.get(res, 0, size); + buf.get(pos, res, 0, size); } else { int len = size; int off = 0; while (len > 0) { - buf = getDataBuffer(offset, ctx); - int count = Math.min(len, buf.remaining()); - buf.get(res, off, count); + buf = dataBuffers[(int) (offset / segmentSize)]; + pos = (int) (offset % segmentSize); + int count = Math.min(len, remaining(buf, pos)); + buf.get(pos, res, off, count); offset += count; off += count; len -= count; @@ -375,13 +373,6 @@ private synchronized byte[] getDiskBytes(long offset) return res; } - //Return the data buffer for the given position - private ByteBuffer getDataBuffer(long index, ThreadContext ctx) { - ByteBuffer buf = ctx.dataBuffers[(int) (index / segmentSize)]; - buf.position((int) (index % segmentSize)); - return buf; - } - private String formatCreatedAt(long createdAt) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd G 'at' HH:mm:ss z"); Calendar cl = Calendar.getInstance(); @@ -437,12 +428,9 @@ public boolean hasNext() { @Override public FastEntry next() { try { - var ctx = context.get(); - ctx.indexBuffer.position(currentIndexOffset); - long offset = 0; while (offset == 0) { - ctx.indexBuffer.get(currentSlotBuffer); + indexBuffer.get(currentIndexOffset, currentSlotBuffer); offset = LongPacker.unpackLong(currentSlotBuffer, currentKeyLength); currentIndexOffset += currentSlotBuffer.length; } @@ -452,7 +440,7 @@ public FastEntry next() { if (withValue) { long valueOffset = currentDataOffset + offset; - value = mMapData ? getMMapBytes(valueOffset, ctx) : getDiskBytes(valueOffset); + value = mMapData ? getMMapBytes(valueOffset) : getDiskBytes(valueOffset); } entry.set(key, value); diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java index badfea0..025a44e 100644 --- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java +++ b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java @@ -21,8 +21,11 @@ import org.testng.annotations.*; import java.io.*; -import java.nio.file.*; +import java.nio.file.Files; import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; public class TestReadThroughput { @@ -55,7 +58,7 @@ public void testReadThroughput() { List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { - Measure m = measure(i, 0, 0, false); + Measure m = measure(i, 0, 0, false, 1); measures.add(m); } @@ -63,18 +66,32 @@ public void testReadThroughput() { report("READ THROUGHPUT (Set int -> boolean)", measures); } + @Test + public void testReadThroughputMultiThread() { + + List measures = new ArrayList<>(); + int max = 10000000; + for (int i = 100; i <= max; i *= 10) { + Measure m = measure(i, 0, 0, false, 4); + + measures.add(m); + } + + report("READ THROUGHPUT MULTI THREAD (Set int -> boolean)", measures); + } + @Test public void testReadThroughputWithCache() { List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { - Measure m = measure(i, 0, 0.05, false); + Measure m = measure(i, 0, 0.05, false, 1); measures.add(m); } - report("READ THROUGHPUT WITH CACHE (Set int -> boolean)", measures); + report("READ THROUGHPUT WITH BLOOM FILTER (Set int -> boolean)", measures); } @Test @@ -83,17 +100,31 @@ public void testReadThroughputWithCacheRandomFinds() { List measures = new ArrayList<>(); int max = 10000000; for (int i = 100; i <= max; i *= 10) { - Measure m = measure(i, 0, 0.01, true); + Measure m = measure(i, 0, 0.01, true, 1); measures.add(m); } - report("READ THROUGHPUT WITH CACHE (Set int -> boolean)", measures); + report("READ THROUGHPUT WITH BLOOM FILTER RANDOM FINDS (Set int -> boolean)", measures); + } + + @Test + public void testReadThroughputWithCacheRandomFindsMultipleThreads() { + + List measures = new ArrayList<>(); + int max = 10000000; + for (int i = 100; i <= max; i *= 10) { + Measure m = measure(i, 0, 0.01, true, 4); + + measures.add(m); + } + + report("READ THROUGHPUT WITH BLOOM FILTER RANDOM FINDS MULTITHREADED (Set int -> boolean)", measures); } // UTILITY - private Measure measure(int keysCount, int valueLength, double errorRate, final boolean randomReads) { + private Measure measure(int keysCount, int valueLength, double errorRate, boolean randomReads, int noOfThreads) { // Write store File storeFile = new File(testFolder, "paldb" + keysCount + "-" + valueLength + ".store"); // Generate keys @@ -118,36 +149,52 @@ private Measure measure(int keysCount, int valueLength, double errorRate, final } } - int[] counter = new int[]{0, 0}; + var totalCount = new AtomicInteger(0); + var findCount = new AtomicInteger(0); try (StoreReader reader = PalDB.createReader(storeFile, config)) { // Measure NanoBench nanoBench = NanoBench.create(); nanoBench.cpuOnly().warmUps(5).measurements(20).measure("Measure %d reads for %d keys with cache", () -> { - Random r = new Random(42); - int length = keys.length; - for (int i = 0; i < READS; i++) { - counter[1]++; - int key; - if (randomReads) { - key = r.nextInt(Integer.MAX_VALUE); - } else { - key = keys[r.nextInt(length)]; - } - var value = reader.get(Integer.toString(key)); - if (value != null) { - counter[0]++; + if (noOfThreads < 2) { + doWork(randomReads, keys, totalCount, findCount, reader); + } else { + var forkJoinPool = new ForkJoinPool(noOfThreads); + try { + forkJoinPool.submit(() -> IntStream.range(0, noOfThreads).parallel() + .forEach(i -> doWork(randomReads, keys, totalCount, findCount, reader)) + ).join(); + } finally { + forkJoinPool.shutdown(); } } }); // Return measure - double rps = READS * nanoBench.getTps(); - return new Measure(storeFile.length(), rps, counter[0], counter[1], keys.length); + double rps = READS * noOfThreads * nanoBench.getTps(); + return new Measure(storeFile.length(), rps, findCount.get(), totalCount.get(), keys.length); + } + } + + private void doWork(boolean randomReads, Integer[] keys, AtomicInteger totalCount, AtomicInteger findCount, StoreReader reader) { + Random r = new Random(42); + int length = keys.length; + for (int j = 0; j < READS; j++) { + totalCount.incrementAndGet(); + int key; + if (randomReads) { + key = r.nextInt(Integer.MAX_VALUE); + } else { + key = keys[r.nextInt(length)]; + } + var value = reader.get(Integer.toString(key)); + if (value != null) { + findCount.incrementAndGet(); + } } } private void report(String title, List measures) { - System.out.println(title + "\n\n"); + System.out.println(title); System.out.println("FILE LENGTH;\tKEYS;\tRPS;\tVALUES FOUND;\tTOTAL READS"); for (Measure m : measures) { System.out.println(m.fileSize + ";\t" + m.keys + ";\t" + m.rps + ";\t" + m.valueLength + ";\t" + m.cacheSize); diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java index 6e68887..0b839c6 100644 --- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java +++ b/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java @@ -209,7 +209,7 @@ public void testPutSerializedKey() throws IOException { @Test public void testByteMarkOneKey() throws IOException { try (FileOutputStream fos = new FileOutputStream(storeFile); - StoreWriter writer = PalDB.createWriter(fos, new Configuration());) { + StoreWriter writer = PalDB.createWriter(fos, new Configuration())) { fos.write(12345); fos.write(FormatVersion.getPrefixBytes()[0]); fos.write(3456); @@ -349,7 +349,7 @@ public void testDataSizeOnTwoBuffers() throws IOException { configuration.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(byteSize + sizeSize + 3)); try (StoreReader reader = PalDB.createReader(storeFile, configuration)) { for (int i = 0; i < keys.length; i++) { - assertEquals(reader.get((Integer) keys[i], null), values[i]); + assertEquals(reader.get(keys[i], null), values[i]); } } } From 300920386ab8b92848ad1742fc624ae21896ec21 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Tue, 19 Nov 2019 15:41:17 +0200 Subject: [PATCH 18/20] Small cleanup. --- .travis.yml | 2 -- README.md | 2 +- .../linkedin/paldb/impl/StorageReader.java | 24 +++++-------------- .../linkedin/paldb/impl/StorageWriter.java | 2 +- .../com/linkedin/paldb/utils/BloomFilter.java | 21 +++------------- .../utils/{HashUtils.java => Murmur3.java} | 4 ++-- .../com/linkedin/paldb/utils/TempUtils.java | 7 +----- .../linkedin/paldb/TestReadThroughput.java | 3 ++- .../{TestHashUtils.java => TestMurmur3.java} | 8 +++---- 9 files changed, 20 insertions(+), 53 deletions(-) rename paldb/src/main/java/com/linkedin/paldb/utils/{HashUtils.java => Murmur3.java} (98%) rename paldb/src/test/java/com/linkedin/paldb/utils/{TestHashUtils.java => TestMurmur3.java} (76%) diff --git a/.travis.yml b/.travis.yml index 90a725b..8228446 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ dist: trusty matrix: include: - - jdk: openjdk11 - - jdk: openjdk12 - jdk: openjdk13 script: diff --git a/README.md b/README.md index b9d3ae9..30fad21 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ Limitations ----------- + PalDB is optimal in replacing the usage of large in-memory data storage but still use memory (off-heap, yet much less) to do its job. Disabling memory mapping and relying on seeks is possible but is not what PalDB has been optimized for. + The size of the index is limited to 2GB. There's no limitation in the data size however. -+ PalDB reader is thread-safe and writer is not thread-safe at the moment so synchronization should be done externally if multi-threaded. ++ PalDB reader is thread-safe but writer is not thread-safe at the moment so synchronization should be done externally if multi-threaded. Contributions ----------- diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java index 5687d34..7c287d8 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java @@ -23,6 +23,7 @@ import java.nio.channels.FileChannel; import java.nio.charset.MalformedInputException; import java.text.*; +import java.time.Instant; import java.util.*; @@ -51,7 +52,6 @@ public class StorageReader implements Iterable> { private final long[] dataOffsets; // Data size private final long dataSize; - private final int maxSlotSize; // FileChannel private final RandomAccessFile mappedFile; private final FileChannel channel; @@ -163,8 +163,6 @@ public class StorageReader implements Iterable> { mSlotSize = Math.max(mSlotSize, slotSizes[keyLength]); } - maxSlotSize = mSlotSize; - //Read index and data offset indexOffset = dataInputStream.readInt() + ignoredBytes; dataOffset = dataInputStream.readLong() + ignoredBytes; @@ -243,14 +241,13 @@ public byte[] get(byte[] key) throws IOException { int slotSize = slotSizes[keyLength]; int ixOffset = indexOffsets[keyLength]; long dtOffset = dataOffsets[keyLength]; - long hash = HashUtils.hash(key); + long hash = Murmur3.hash(key); var slotBuffer = new byte[slotSize]; for (int probe = 0; probe < numSlots; probe++) { int slot = (int) ((hash + probe) % numSlots); indexBuffer.get(ixOffset + slot * slotSize, slotBuffer, 0, slotSize); long offset = LongPacker.unpackLong(slotBuffer, keyLength); if (offset == 0L) { - log.info("Got offset 0, returning..."); return null; } if (isKey(slotBuffer, key)) { @@ -261,12 +258,7 @@ public byte[] get(byte[] key) throws IOException { } private boolean isKey(byte[] slotBuffer, byte[] key) { - for (int i = 0; i < key.length; i++) { - if (slotBuffer[i] != key[i]) { - return false; - } - } - return true; + return Arrays.compare(slotBuffer, 0, key.length, key, 0, key.length) == 0; } //Close the reader channel @@ -279,7 +271,7 @@ public int getKeyCount() { return keyCount; } - private int remaining(ByteBuffer buffer, int pos) { + private static int remaining(ByteBuffer buffer, int pos) { return buffer.limit() - pos; } @@ -355,8 +347,7 @@ private byte[] getMMapBytes(long offset) throws IOException { } //Get data from disk - private synchronized byte[] getDiskBytes(long offset) - throws IOException { + private synchronized byte[] getDiskBytes(long offset) throws IOException { mappedFile.seek(dataOffset + offset); //Get size of data @@ -374,10 +365,7 @@ private synchronized byte[] getDiskBytes(long offset) } private String formatCreatedAt(long createdAt) { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd G 'at' HH:mm:ss z"); - Calendar cl = Calendar.getInstance(); - cl.setTimeInMillis(createdAt); - return sdf.format(cl.getTime()); + return Instant.ofEpochMilli(createdAt).toString(); } @Override diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java index 32e564a..4e0e622 100644 --- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java +++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java @@ -293,7 +293,7 @@ private File buildIndex(int keyLength, BloomFilter bloomFilter) throws IOExcepti long offset = LongPacker.unpackLong(tempIndexStream); // Hash - long hash = HashUtils.hash(keyBuffer); + long hash = Murmur3.hash(keyBuffer); if (bloomFilter != null) { bloomFilter.add(keyBuffer); } diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java index 5d0787b..78c12d9 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java @@ -31,14 +31,14 @@ public BloomFilter(int hashFunctions, int bitSize, long[] bits) { public void add(byte[] bytes) { for (int i = 0; i < hashFunctions; i++) { - int value = Math.abs(HashUtils.hash(bytes, i) % sizeInBits); + int value = Math.abs(Murmur3.hash(bytes, i) % sizeInBits); setBit(value); } } public boolean mightContain(byte[] bytes) { for (int i = 0; i < hashFunctions; i++) { - int value = Math.abs(HashUtils.hash(bytes, i) % sizeInBits); + int value = Math.abs(Murmur3.hash(bytes, i) % sizeInBits); if (!getBit(value)) return false; } return true; @@ -61,9 +61,7 @@ private boolean getBit(int position) { } public void clear() { - for (int i = 0; i < bits.length; i++) { - bits[i] = 0L; - } + Arrays.fill(bits, 0L); } public int bitSize() { @@ -86,18 +84,5 @@ public boolean equals(Object o) { final var that = (BloomFilter) o; return Arrays.equals(this.bits, that.bits) && this.hashFunctions == that.hashFunctions; } - - public void merge(BloomFilter other) { - if (other.hashFunctions != this.hashFunctions || other.bits.length != this.bits.length) { - throw new IllegalArgumentException("Incompatible bloom filters"); - } - - for (int i = 0; i < bits.length; i++) { - var oldBit = getBit(i); - if (!oldBit && other.getBit(i)) { - setBit(i); - } - } - } } diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java b/paldb/src/main/java/com/linkedin/paldb/utils/Murmur3.java similarity index 98% rename from paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java rename to paldb/src/main/java/com/linkedin/paldb/utils/Murmur3.java index 68ea57a..4b08fdf 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/HashUtils.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/Murmur3.java @@ -21,9 +21,9 @@ /** * Hashing utility. */ -public final class HashUtils { +public final class Murmur3 { - private HashUtils() { } + private Murmur3() { } /** * Returns the positive hash for the given bytes. diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java b/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java index d6219c0..b1ddaee 100644 --- a/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java +++ b/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java @@ -14,12 +14,7 @@ package com.linkedin.paldb.utils; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; /** diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java index 025a44e..daacceb 100644 --- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java +++ b/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java @@ -137,7 +137,8 @@ private Measure measure(int keysCount, int valueLength, double errorRate, boolea .withBloomFilterErrorFactor(errorRate); } - var config = configBuilder.build(); + var config = configBuilder + .build(); try (StoreWriter writer = PalDB.createWriter(storeFile, config)) { for (Integer key : keys) { diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/TestHashUtils.java b/paldb/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java similarity index 76% rename from paldb/src/test/java/com/linkedin/paldb/utils/TestHashUtils.java rename to paldb/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java index 73467fa..c9d7588 100644 --- a/paldb/src/test/java/com/linkedin/paldb/utils/TestHashUtils.java +++ b/paldb/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java @@ -20,21 +20,21 @@ import static org.testng.Assert.*; -public class TestHashUtils { +public class TestMurmur3 { @Test public void testHashEquals() { - assertEquals(HashUtils.hash("foo".getBytes()), HashUtils.hash("foo".getBytes())); + assertEquals(Murmur3.hash("foo".getBytes()), Murmur3.hash("foo".getBytes())); } @Test public void testEmpty() { - assertTrue(HashUtils.hash(new byte[0]) > 0); + assertTrue(Murmur3.hash(new byte[0]) > 0); } @Test public void testSameHash() { var bytes = "foo".getBytes(); - assertEquals(HashUtils.hash(bytes, 42), MurmurHash3.hash32(bytes, bytes.length, 42)); + assertEquals(Murmur3.hash(bytes, 42), MurmurHash3.hash32(bytes, bytes.length, 42)); } } From e6192f087026a110ec59467c2d668bd5b97d1593 Mon Sep 17 00:00:00 2001 From: Linas Naginionis Date: Tue, 19 Nov 2019 16:41:20 +0200 Subject: [PATCH 19/20] fixed javadoc --- gradle.properties | 3 +++ paldb/build.gradle | 1 + paldb/overview.html | 9 +++------ .../java/com/linkedin/paldb/api/Configuration.java | 1 + .../src/main/java/com/linkedin/paldb/api/PalDB.java | 12 ++++++++++++ .../java/com/linkedin/paldb/impl/Serializers.java | 1 + .../linkedin/paldb/impl/StorageSerialization.java | 1 + 7 files changed, 22 insertions(+), 6 deletions(-) diff --git a/gradle.properties b/gradle.properties index 9c15eaa..b395ddf 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,6 +2,9 @@ org.gradle.daemon=true org.gradle.configureondemand=true ide.recursive=true org.gradle.parallel=false +#release=true +signing.gnupg.executable=gpg +signing.gnupg.useLegacyGpg=true VERSION_NAME=2.0.0 GROUP=net.soundvibe.paldb diff --git a/paldb/build.gradle b/paldb/build.gradle index cd6996f..731e691 100644 --- a/paldb/build.gradle +++ b/paldb/build.gradle @@ -68,6 +68,7 @@ jacocoTestReport { if (project.hasProperty('release')) { signing { + useGpgCmd() sign configurations.archives } diff --git a/paldb/overview.html b/paldb/overview.html index 057ef2d..97e43be 100644 --- a/paldb/overview.html +++ b/paldb/overview.html @@ -2,7 +2,7 @@

PalDB is an embeddable persistent write-once key-value store.

-

Code samples

+

Code samples

How to write a store: 
@@ -23,20 +23,17 @@ 
Code samples

 How to iterate on a store:
 
 StoreReader reader = PalDB.createReader(new File("store.paldb"));
-Iterable> iterable = reader.iterable();
-for (Map.Entry entry : iterable) {
+Iterable<Map.Entry<String, String>> iterable = reader.iterable();
+for (Map.Entry<String, String> entry : iterable) {
   String key = entry.getKey();
   String value = entry.getValue();
 }
 reader.close();
 

 
-

 
API Changes

-

 
    
   
  • (April 2015) Initial API
    • 
 

-

 
 
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java
index 90df23d..86ff5cd 100644
--- a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java
+++ b/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java
@@ -382,6 +382,7 @@ public  Class getClass(String key)
    * The class for with the serializer is being registered is directly extracted from the class definition.
    *
    * @param serializer serializer to register
+   * @param  serializer type
    */
   public  void registerSerializer(Serializer serializer) {
     serializers.registerSerializer(serializer);
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java
index 256b888..2f40672 100644
--- a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java
+++ b/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java
@@ -34,6 +34,8 @@ public interface PalDB {
    * The file must exists.
    *
    * @param file a PalDB store file
+   * @param  key type
+   * @param  value type
    * @return a store reader
    */
   static  StoreReader createReader(File file) {
@@ -47,6 +49,8 @@ static  StoreReader createReader(File file) {
    *
    * @param file a PalDB store file
    * @param config configuration
+   * @param  key type
+   * @param  value type
    * @return a store reader
    */
   static  StoreReader createReader(File file, Configuration config) {
@@ -61,6 +65,8 @@ static  StoreReader createReader(File file, Configuration config) {
    *
    * @param stream an input stream on a PalDB store file
    * @param config configuration
+   * @param  key type
+   * @param  value type
    * @return a store reader
    */
   static  StoreReader createReader(InputStream stream, Configuration config) {
@@ -73,6 +79,8 @@ static  StoreReader createReader(InputStream stream, Configuration con
    * The parent folder is created if missing.
    *
    * @param file location of the output file
+   * @param  key type
+   * @param  value type
    * @return a store writer
    */
   static  StoreWriter createWriter(File file) {
@@ -86,6 +94,8 @@ static  StoreWriter createWriter(File file) {
    *
    * @param file location of the output file
    * @param config configuration
+   * @param  key type
+   * @param  value type
    * @return a store writer
    */
   static  StoreWriter createWriter(File file, Configuration config) {
@@ -100,6 +110,8 @@ static  StoreWriter createWriter(File file, Configuration config) {
    *
    * @param stream output stream
    * @param config configuration
+   * @param  key type
+   * @param  value type
    * @return a store writer
    */
   static  StoreWriter createWriter(OutputStream stream, Configuration config) {
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java
index 8641ee0..b068be8 100644
--- a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java
+++ b/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java
@@ -40,6 +40,7 @@ public Serializers() {
    * Registers the serializer.
    *
    * @param serializer serializer
+   * @param  serialized class type
    */
   public synchronized  void registerSerializer(Serializer serializer) {
     var className = serializer.serializedClass().getName();
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java
index c487c01..a369569 100644
--- a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java
+++ b/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java
@@ -52,6 +52,7 @@ public StorageSerialization(Configuration config) {
    * Serializes the key object and returns it as a byte array.
    *
    * @param key key to serialize
+   * @param  key type
    * @return key as byte array
    * @throws IOException if an io error occurs
    */

From ae29b64c722c582febbde11d4028a718e82a4a83 Mon Sep 17 00:00:00 2001
From: Linas Naginionis 
Date: Wed, 20 Nov 2019 15:00:20 +0200
Subject: [PATCH 20/20] Moved to maven.

---
 .gitignore                                    |   2 +
 .travis.yml                                   |   3 -
 CHANGELOG.md                                  |   4 +-
 README.md                                     |  16 +-
 build.gradle                                  |  16 --
 gradle.properties                             |  11 -
 gradle/wrapper/gradle-wrapper.jar             | Bin 55190 -> 0 bytes
 gradle/wrapper/gradle-wrapper.properties      |   6 -
 gradlew                                       | 172 ------------
 gradlew.bat                                   |  84 ------
 paldb/overview.html => overview.html          |   0
 paldb/build.gradle                            | 130 ---------
 pom.xml                                       | 249 ++++++++++++++++++
 settings.gradle                               |   1 -
 .../com/linkedin/paldb/api/Configuration.java |   0
 .../java/com/linkedin/paldb/api/PalDB.java    |   0
 .../paldb/api/PalDBConfigBuilder.java         |   0
 .../com/linkedin/paldb/api/Serializer.java    |   0
 .../com/linkedin/paldb/api/StoreReader.java   |   0
 .../com/linkedin/paldb/api/StoreWriter.java   |   0
 .../api/errors/DuplicateKeyException.java     |   0
 .../paldb/api/errors/OutOfDiskSpace.java      |   0
 .../api/errors/UnsupportedTypeException.java  |   0
 .../java/com/linkedin/paldb/api/package.html  |   0
 .../com/linkedin/paldb/impl/ReaderImpl.java   |   0
 .../linkedin/paldb/impl/ReaderIterable.java   |   0
 .../paldb/impl/ReaderKeyIterable.java         |   0
 .../com/linkedin/paldb/impl/Serializers.java  |   0
 .../linkedin/paldb/impl/StorageReader.java    |   0
 .../paldb/impl/StorageSerialization.java      |   0
 .../linkedin/paldb/impl/StorageWriter.java    |   0
 .../com/linkedin/paldb/impl/StoreImpl.java    |   0
 .../com/linkedin/paldb/impl/WriterImpl.java   |   0
 .../java/com/linkedin/paldb/impl/package.html |   0
 .../com/linkedin/paldb/utils/BloomFilter.java |   0
 .../linkedin/paldb/utils/DataInputOutput.java |   0
 .../linkedin/paldb/utils/FormatVersion.java   |   0
 .../com/linkedin/paldb/utils/LongPacker.java  |   0
 .../com/linkedin/paldb/utils/Murmur3.java     |   0
 .../com/linkedin/paldb/utils/TempUtils.java   |   0
 .../com/linkedin/paldb/utils/package.html     |   0
 .../paldb/api/PalDBConfigBuilderTest.java     |   4 +-
 .../linkedin/paldb/api/TestConfiguration.java |  54 ++--
 .../linkedin/paldb/impl/GenerateTestData.java |   2 +-
 .../linkedin/paldb/impl/TestSerializers.java  |   6 +-
 .../paldb/impl/TestStorageSerialization.java  |  48 ++--
 .../com/linkedin/paldb/impl/TestStore.java    |  79 +++---
 .../linkedin/paldb/impl/TestStoreReader.java  |  81 +++---
 .../performance}/TestMemoryUsageHashMap.java  |  12 +-
 .../performance}/TestReadThroughput.java      |  15 +-
 .../TestReadThroughputLevelDB.java            |  29 +-
 .../performance}/TestReadThrouputHashMap.java |  12 +-
 .../performance}/TestReadThrouputRocksDB.java |  33 +--
 .../paldb/performance}/TestStoreSize.java     |  21 +-
 .../performance}/utils/DirectoryUtils.java    |   2 +-
 .../paldb/performance}/utils/NanoBench.java   |   2 +-
 .../linkedin/paldb/utils/BloomFilterTest.java |   8 +-
 .../paldb/utils/TestFormatVersion.java        |  12 +-
 .../linkedin/paldb/utils/TestLongPacker.java  |  45 ++--
 .../com/linkedin/paldb/utils/TestMurmur3.java |   5 +-
 .../linkedin/paldb/utils/TestTempUtils.java   |  21 +-
 .../test/resources/logback-test.xml           |   0
 62 files changed, 498 insertions(+), 687 deletions(-)
 delete mode 100644 build.gradle
 delete mode 100644 gradle.properties
 delete mode 100644 gradle/wrapper/gradle-wrapper.jar
 delete mode 100644 gradle/wrapper/gradle-wrapper.properties
 delete mode 100755 gradlew
 delete mode 100644 gradlew.bat
 rename paldb/overview.html => overview.html (100%)
 delete mode 100644 paldb/build.gradle
 create mode 100644 pom.xml
 delete mode 100644 settings.gradle
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/Configuration.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/PalDB.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/Serializer.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/StoreReader.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/StoreWriter.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/api/package.html (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/ReaderImpl.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/ReaderIterable.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/Serializers.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/StorageReader.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/StorageSerialization.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/StorageWriter.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/StoreImpl.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/WriterImpl.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/impl/package.html (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/utils/BloomFilter.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/utils/DataInputOutput.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/utils/FormatVersion.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/utils/LongPacker.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/utils/Murmur3.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/utils/TempUtils.java (100%)
 rename {paldb/src => src}/main/java/com/linkedin/paldb/utils/package.html (100%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java (91%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/api/TestConfiguration.java (82%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/impl/GenerateTestData.java (97%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/impl/TestSerializers.java (97%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java (95%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/impl/TestStore.java (89%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/impl/TestStoreReader.java (82%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/TestMemoryUsageHashMap.java (88%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/TestReadThroughput.java (96%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/TestReadThroughputLevelDB.java (87%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/TestReadThrouputHashMap.java (89%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/TestReadThrouputRocksDB.java (88%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/TestStoreSize.java (81%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/utils/DirectoryUtils.java (94%)
 rename {paldb/src/perfTest/java/com/linkedin/paldb => src/test/java/com/linkedin/paldb/performance}/utils/NanoBench.java (99%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/utils/BloomFilterTest.java (98%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/utils/TestFormatVersion.java (62%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/utils/TestLongPacker.java (64%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/utils/TestMurmur3.java (92%)
 rename {paldb/src => src}/test/java/com/linkedin/paldb/utils/TestTempUtils.java (78%)
 rename {paldb/src => src}/test/resources/logback-test.xml (100%)

diff --git a/.gitignore b/.gitignore
index a3c393f..9aad68c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,5 @@ out/
 /ligradle
 .DS_Store
 /doc
+/target
+target/
diff --git a/.travis.yml b/.travis.yml
index 8228446..b215a48 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,8 +7,5 @@ matrix:
   include:
     - jdk: openjdk13
 
-script:
-  - ./gradlew build jacocoTestReport
-
 after_success:
   - bash <(curl -s https://codecov.io/bash)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 65e4a59..8ab630a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,10 +3,12 @@ Change Log
 
 Version 2.0.0
 --------------------------
-- Bumped minimal java version to 11
+- Bumped minimal java version to java 13
 - StoreReader and StoreWriter implement AutoCloseable
 - New config builder
 - Reader and writer use generics
+- Reader is now fully thread-safe
+- Bloom filters could be used for better performance
 
 
 Version 1.2.0 *(June 26th 2016)*
diff --git a/README.md b/README.md
index 30fad21..3865462 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 PalDB
 ==========
 
-[![Build Status](https://travis-ci.org/soundvibe/PalDB.svg?branch=java11)](https://travis-ci.org/soundvibe/PalDB)
+[![Build Status](https://travis-ci.org/soundvibe/PalDB.svg)](https://travis-ci.org/soundvibe/PalDB)
 [![codecov](https://codecov.io/gh/soundvibe/PalDB/branch/java11/graph/badge.svg)](https://codecov.io/gh/soundvibe/PalDB)
 
 PalDB is an embeddable write-once key-value store written in Java.
@@ -72,14 +72,14 @@ Use it
 PalDB is available on Maven Central, hence just add the following dependency:
 ```
 
-    net.soundvibe.paldb
+    net.soundvibe
     paldb
     2.0.0
 
 ```
 Scala SBT
 ```
-libraryDependencies += "net.soundvibe.paldb" % "paldb" % "2.0.0"
+libraryDependencies += "net.soundvibe" % "paldb" % "2.0.0"
 ```
 
 
@@ -101,26 +101,26 @@ No, like a hashtable PalDB stores have no order.
 Build
 -----
 
-PalDB requires Java 13+ and gradle. The target Java version is 13.
+PalDB requires Java 13+ and maven. The target Java version is 13.
 
 ```bash
-gradle build
+mvn build
 ```
 
 Performance tests are run separately from the build
 ```bash
-gradle perfTest
+mvn clean test -Dtag=performance
 ```
 
 Test
 ----
 
-We use the TestNG framework for our unit tests. You can run them via the `gradle clean test` command.
+We use the JUnit framework for our unit tests. You can run them via the `mvn clean test` command.
 
 Coverage
 --------
 
-Coverage is run using JaCoCo. You can run a report via `gradle jacocoTestReport`. The report will be generated in `paldb/build/reports/jacoco/test/html/`.
+Coverage is run using JaCoCo. You can run a report via `mvn jacoco:report`. The report will be generated in `paldb/build/reports/jacoco/test/html/`.
 
 Advanced configuration
 ----------------------
diff --git a/build.gradle b/build.gradle
deleted file mode 100644
index 5c59efa..0000000
--- a/build.gradle
+++ /dev/null
@@ -1,16 +0,0 @@
-buildscript {
-  repositories {
-    mavenCentral()
-  }
-}
-
-allprojects {
-  repositories {
-    mavenCentral()
-  }
-}
-
-allprojects {
-  apply plugin: 'eclipse'
-  apply plugin: 'idea'
-}
diff --git a/gradle.properties b/gradle.properties
deleted file mode 100644
index b395ddf..0000000
--- a/gradle.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-org.gradle.daemon=true
-org.gradle.configureondemand=true
-ide.recursive=true
-org.gradle.parallel=false
-#release=true
-signing.gnupg.executable=gpg
-signing.gnupg.useLegacyGpg=true
-
-VERSION_NAME=2.0.0
-GROUP=net.soundvibe.paldb
-ARCHIVE_NAME=paldb
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
deleted file mode 100644
index 87b738cbd051603d91cc39de6cb000dd98fe6b02..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 55190
zcmafaW0WS*vSoFbZQHhO+s0S6%`V%vZQJa!ZQHKus_B{g-pt%P_q|ywBQt-*Stldc
z$+IJ3?^KWm27v+sf`9-50uuadKtMnL*BJ;1^6ynvR7H?hQcjE>7)art9Bu0Pcm@7C
z@c%WG|JzYkP)<@zR9S^iR_sA`azaL$mTnGKnwDyMa;8yL_0^>Ba^)phg0L5rOPTbm7g*YIRLg-2^{qe^`rb!2KqS
zk~5wEJtTdD?)3+}=eby3x6%i)sb+m??NHC^u=tcG8p$TzB<;FL(WrZGV&cDQb?O0GMe6PBV=V
z?tTO*5_HTW$xea!nkc~Cnx#cL_rrUGWPRa6l+A{aiMY=<0@8y5OC#UcGeE#I>nWh}`#M#kIn-$A;q@u-p71b#hcSItS!IPw?>8
zvzb|?@Ahb22L(O4#2Sre&l9H(@TGT>#Py)D&eW-LNb!=S;I`ZQ{w;MaHW
z#to!~TVLgho_Pm%zq@o{K3Xq?I|MVuVSl^QHnT~sHlrVxgsqD-+YD?Nz9@HA<;x2AQjxP)r6Femg+LJ-*)k%EZ}TTRw->5xOY
z9#zKJqjZgC47@AFdk1$W+KhTQJKn7e>A&?@-YOy!v_(}GyV@9G#I?bsuto4JEp;5|N{orxi_?vTI4UF0HYcA(
zKyGZ4<7Fk?&LZMQb6k10N%E*$gr#T&HsY4SPQ?yerqRz5c?5P$@6dlD6UQwZJ*Je9
z7n-@7!(OVdU-mg@5$D+R%gt82Lt%&n6Yr4=|q>XT%&^z_D*f*ug8N6w$`woqeS-+#RAOfSY&Rz
z?1qYa5xi(7eTCrzCFJfCxc%j{J}6#)3^*VRKF;w+`|1n;Xaojr2DI{!<3CaP`#tXs
z*`pBQ5k@JLKuCmovFDqh_`Q;+^@t_;SDm29
zCNSdWXbV?9;D4VcoV`FZ9Ggrr$i<&#Dx3W=8>bSQIU_%vf)#(M2Kd3=rN@^d=QAtC
zI-iQ;;GMk|&A++W5#hK28W(YqN%?!yuW8(|Cf`@FOW5QbX|`97fxmV;uXvPCqxBD
zJ9iI37iV)5TW1R+fV16y;6}2tt~|0J3U4E=wQh@sx{c_eu)t=4Yoz|%Vp<#)Qlh1V
z0@C2ZtlT>5gdB6W)_bhXtcZS)`9A!uIOa`K04$5>3&8An+i9BD&GvZZ=7#^r=BN=k
za+=Go;qr(M)B~KYAz|<^O3LJON}$Q6Yuqn8qu~+UkUKK~&iM%pB!BO49L+?AL7N7o
z(OpM(C-EY753=G=WwJHE`h*lNLMNP^c^bBk@5MyP5{v7x>GNWH>QSgTe5
z!*GPkQ(lcbEs~)4ovCu!Zt&$${9$u(<4@9%@{U<-ksAqB?6F`bQ;o-mvjr)Jn7F&j$@`il1Mf+-HdBs<-`1FahTxmPMMI)@OtI&^mtijW6zGZ67O$UOv1Jj
z;a3gmw~t|LjPkW3!EZ=)lLUhFzvO;Yvj9g`8hm%6u`;cuek_b-c$wS_0M4-N<@3l|88
z@V{Sd|M;4+H6guqMm4|v=C6B7mlpP(+It%0E;W`dxMOf9!jYwWj3*MRk`KpS_jx4c
z=hrKBkFK;gq@;wUV2eqE3R$M+iUc+UD0iEl#-rECK+XmH9hLKrC={j@uF=f3UiceB
zU5l$FF7#RKjx+6!JHMG5-!@zI-eG=a-!Bs^AFKqN_M26%cIIcSs61R$yuq@5a3c3&
z4%zLs!g}+C5%`ja?F`?5-og0lv-;(^e<`r~p$x%&*89_Aye1N)9LNVk?9BwY$Y$$F^!JQAjBJvywXAesj7lTZ)rXuxv(FFNZVknJha99lN=^h`J2>
zl5=~(tKwvHHvh|9-41@OV`c;Ws--PE%{7d2sLNbDp;A6_Ka6epzOSFdqb
zBa0m3j~bT*q1lslHsHqaHIP%DF&-XMpCRL(v;MV#*>mB^&)a=HfLI7efblG
z(@hzN`|n+oH9;qBklb=d^S0joHCsArnR1-h{*dIUThik>ot^!6YCNjg;J_i3h6Rl0ji)*
zo(tQ~>xB!rUJ(nZjCA^%X;)H{@>uhR5|xBDA=d21p@iJ!cH?+%U|VSh2S4@gv`^)^
zNKD6YlVo$%b4W^}Rw>P1YJ|fTb$_(7C;hH+
z1XAMPb6*p^h8)e5nNPKfeAO}Ik+ZN_`NrADeeJOq4Ak;sD~
zTe77no{Ztdox56Xi4UE6S7wRVxJzWxKj;B%v7|FZ3cV9MdfFp7lWCi+W{}UqekdpH
zdO#eoOuB3Fu!DU`ErfeoZWJbWtRXUeBzi
zBTF-AI7yMC^ntG+8%mn(I6Dw}3xK8v#Ly{3w3_E?J4(Q5JBq~I>u3!CNp~Ekk&YH`
z#383VO4O42NNtcGkr*K<+wYZ>@|sP?`AQcs5oqX@-EIqgK@Pmp5~p6O6qy4ml~N{D
z{=jQ7k(9!CM3N3Vt|u@%ssTw~r~Z(}QvlROAkQQ?r8OQ3F0D$aGLh
zny+uGnH5muJ<67Z=8uilKvGuANrg@s3Vu_lU2ajb?rIhuOd^E@l!Kl0hYIxOP1B~Q
zggUmXbh$bKL~YQ#!4fos9UUVG#}HN$lIkM<1OkU@r>$7DYYe37cXYwfK@vrHwm;pg
zbh(hEU|8{*d$q7LUm+x&`S@VbW*&p-sWrplWnRM|I{P;I;%U`WmYUCeJhYc|>5?&&
zj}@n}w~Oo=l}iwvi7K6)osqa;M8>fRe}>^;bLBrgA;r^ZGgY@IC^ioRmnE&H4)UV5
zO{7egQ7sBAdoqGsso5q4R(4$4Tjm&&C|7Huz&5B0wXoJzZzNc5Bt)=SOI|H}+fbit
z-PiF5(NHSy>4HPMrNc@SuEMDuKYMQ--G+qeUPqO_9mOsg%1EHpqoX^yNd~~kbo`cH
zlV0iAkBFTn;rVb>EK^V6?T~t~3vm;csx+lUh_%ROFPy0(omy7+_wYjN!VRDtwDu^h4n|xpAMsLepm%
zggvs;v8+isCW`>BckRz1MQ=l>K6k^DdT`~sDXTWQ<~+JtY;I~I>8XsAq3yXgxe>`O
zZdF*{9@Z|YtS$QrVaB!8&`&^W->_O&-JXn1n&~}o3Z7FL1QE5R*W2W@=u|w~7%EeC1aRfGtJWxImfY-D3t!!nBkWM>
zafu>^Lz-ONgT6ExjV4WhN!v~u{lt2-QBN&UxwnvdH|I%LS|J-D;o>@@sA62@&yew0
z)58~JSZP!(lX;da!3`d)D1+;K9!lyNlkF|n(UduR-%g>#{`pvrD^ClddhJyfL7C-(x+J+9&7EsC~^O`&}V%)Ut8^O_7YAXPDpzv8ir4
zl`d)(;imc6r16k_d^)PJZ+QPxxVJS5e^4wX9D=V2zH&wW0-p&OJe=}rX`*->XT=;_qI&)=WHkYnZx6bLoUh_)n-A}SF_
z9z7agNTM5W6}}ui=&Qs@pO5$zHsOWIbd_&%j^Ok5PJ3yUWQw*i4*iKO)_er2CDUME
ztt+{Egod~W-fn^aLe)aBz)MOc_?i-stTj}~iFk7u^-gGSbU;Iem06SDP=AEw9SzuF
zeZ|hKCG3MV(z_PJg0(JbqTRf4T{NUt%kz&}4S`)0I%}ZrG!jgW2GwP=WTtkWS?DOs
znI9LY!dK+1_H0h+i-_~URb^M;4&AMrEO_UlDV8o?E>^3x%ZJyh$JuDMrtYL8|G3If
zPf2_Qb_W+V?$#O;
zydKFv*%O;Y@o_T_UAYuaqx1isMKZ^32JtgeceA$0Z@Ck0;lHbS%N5)zzAW9iz;
z8tTKeK7&qw!8XVz-+pz>z-BeIzr*#r0nB^cntjQ9@Y-N0=e&ZK72vlzX>f3RT@i7@
z=z`m7jNk!9%^xD0ug%ptZnM>F;Qu$rlwo}vRGBIymPL)L|x}nan3uFUw(&N
z24gdkcb7!Q56{0<+zu
zEtc5WzG2xf%1<@vo$ZsuOK{v9gx^0`gw>@h>ZMLy*h+6ueoie{D#}}`
zK2@6Xxq(uZaLFC%M!2}FX}ab%GQ8A0QJ?&!vaI8Gv=vMhd);6kGguDmtuOElru())
zuRk&Z{?Vp!G~F<1#s&6io1`poBqpRHyM^p;7!+L??_DzJ8s9mYFMQ0^%_3ft7g{PD
zZd}8E4EV}D!>F?bzcX=2hHR_P`Xy6?FOK)mCj)Ym4s2hh
z0OlOdQa@I;^-3bhB6mpw*X5=0kJv8?#XP~9){G-+0ST@1Roz1qi8PhIXp1D$XNqVG
zMl>WxwT+K`SdO1RCt4FWTNy3!i?N>*-lbnn#OxFJrswgD7HjuKpWh*o@QvgF&j+CT
z{55~ZsUeR1aB}lv#s_7~+9dCix!5(KR#c?K?e2B%P$fvrsZxy@GP#R#jwL{y#Ld$}
z7sF>QT6m|}?V;msb?Nlohj7a5W_D$y+4O6eI;Zt$jVGymlzLKscqer9#+p2$0It&u
zWY!dCeM6^B^Z;ddEmhi?8`scl=Lhi7W%2|pT6X6^%-=q90DS(hQ-%c+E*ywPvmoF(KqDoW4!*gmQIklm
zk#!GLqv|cs(JRF3G?=AYY19{w@~`G3pa
z@xR9S-Hquh*&5Yas*VI};(%9%PADn`kzm
zeWMJVW=>>wap*9|R7n#!&&J>gq04>DTCMtj{P^d12|2wXTEKvSf?$AvnE!peqV7i4
zE>0G%CSn%WCW1yre?yi9*aFP{GvZ|R4JT}M%x_%Hztz2qw?&28l&qW<6?c6ym{f$d
z5YCF+k#yEbjCN|AGi~-NcCG8MCF1!MXBFL{#7q
z)HO+WW173?kuI}^Xat;Q^gb4Hi0RGyB}%|~j8>`6X4CPo+|okMbKy9PHkr58V4bX6<&ERU)QlF8%%huUz&f+dwTN|tk+C&&o@Q1RtG`}6&6;ncQuAcfHoxd5AgD7`s
zXynq41Y`zRSiOY@*;&1%1z>oNcWTV|)sjLg1X8ijg1Y
zbIGL0X*Sd}EXSQ2BXCKbJmlckY(@EWn~Ut2lYeuw1wg?hhj@K?XB@V_ZP`fyL~Yd3n3SyHU-RwMBr6t-QWE5TinN9VD4XVPU;
zonIIR!&pGqrLQK)=#kj40Im%V@ij0&Dh0*s!lnTw+D`Dt-xmk-jmpJv$1-E-vfYL4
zqKr#}Gm}~GPE+&$PI@4ag@=M}NYi7Y&HW82Q`@Y=W&PE31D110@yy(1vddLt`P%N^
z>Yz195A%tnt~tvsSR2{m!~7HUc@x<&`lGX1nYeQUE(%sphTi>JsVqSw8xql*Ys@9B
z>RIOH*rFi*C`ohwXjyeRBDt8p)-u{O+KWP;$4gg||%*u{$~yEj+Al
zE(hAQRQ1k7MkCq9s4^N3ep*$h^L%2Vq?f?{+cicpS8lo)$Cb69b98au+m2J_e7nYwID0@`M9XIo1H~|eZFc8Hl!qly612ADCVpU
zY8^*RTMX(CgehD{9v|^9vZ6Rab`VeZ2m*gOR)Mw~73QEBiktViBhR!_&3l$|be|d6
zupC`{g89Y|V3uxl2!6CM(RNpdtynaiJ~*DqSTq9Mh`ohZnb%^3G{k;6%n18$4nAqR
zjPOrP#-^Y9;iw{J@XH9=g5J+yEVh|e=4UeY<^65`%gWtdQ=-aqSgtywM(1nKXh`R4
zzPP&7r)kv_uC7X9n=h=!Zrf<>X=B5f<9~Q>h#jYRD#CT7D~@6@RGNyO-#0iq0uHV1
zPJr2O4d_xLmg2^TmG7|dpfJ?GGa`0|YE+`2Rata9!?$j#e9KfGYuLL(*^z
z!SxFA`$qm)q-YKh)WRJZ@S+-sD_1E$V?;(?^+F3tVcK6
z2fE=8hV*2mgiAbefU^uvcM?&+Y&E}vG=Iz!%jBF7iv){lyC`)*yyS~D8k+Mx|N3bm
zI~L~Z$=W9&`x)JnO;8c>3LSDw!fzN#X3qi|0`sXY4?cz{*#xz!kvZ9bO=K3XbN
z5KrgN=&(JbXH{Wsu9EdmQ-W`i!JWEmfI;yVTT^a-8Ch#D8xf2dtyi?7p
z%#)W3n*a#ndFpd{qN|+9Jz++AJQO#-Y7Z6%*%oyEP5zs}d&kKIr`FVEY
z;S}@d?UU=tCdw~EJ{b}=9x}S2iv!!8<$?d7VKDA8h{oeD#S-$DV)-vPdGY@x08n)@
zag?yLF_E#evvRTj4^CcrLvBL=fft&@HOhZ6Ng4`8ijt&h2y}fOTC~7GfJi4vpomA5
zOcOM)o_I9BKz}I`q)fu+Qnfy*W`|mY%LO>eF^a
z;$)?T4F-(X#Q-m}!-k8L_rNPf`Mr<9IWu)f&dvt=EL+ESYmCvErd@8B9hd)afc(ZL94S
z?rp#h&{7Ah5IJftK4VjATklo7@hm?8BX*~oBiz)jyc9FuRw!-V;Uo>p!CWpLaIQyt
zAs5WN)1CCeux-qiGdmbIk8LR`gM+Qg=&Ve}w?zA6+sTL)abU=-cvU`3E?p5$Hpkxw
znu0N659qR=IKnde*AEz_7z2pdi_Bh-sb3b=PdGO1Pdf_q2;+*Cx9YN7p_>rl``knY
zRn%aVkcv1(W;`Mtp_DNOIECtgq%ufk-mu_<+Fu3Q17Tq4Rr(oeq)Yqk_CHA7LR@7@
zIZIDxxhS&=F2IQfusQ+Nsr%*zFK7S4g!U0y@3H^Yln|i;0a5+?RPG;ZSp6Tul>ezM
z`40+516&719qT)mW|ArDSENle5hE2e8qY+zfeZoy12u&xoMgcP)4=&P-1Ib*-bAy`
zlT?>w&B|ei-rCXO;sxo7*G;!)_p#%PAM-?m$JP(R%x1Hfas@KeaG%LO?R=lmkXc_MKZW}3f%KZ*rAN?HYvbu2L$
zRt_uv7~-IejlD1x;_AhwGXjB94Q=%+PbxuYzta*jw?S&%|qb=(JfJ?&6P=R7X
zV%HP_!@-zO*zS}46g=J}#AMJ}rtWBr21e6hOn&tEmaM%hALH7nlm2@LP4rZ>2
zebe5aH@k!e?ij4Zwak#30|}>;`bquDQK*xmR=zc6vj0yuyC6+U=LusGnO3ZKFRpen
z#pwzh!<+WBVp-!$MAc<0i~I%fW=8IO6K}bJ<-Scq>e+)951R~HKB?Mx2H}pxPHE@}
zvqpq5j81_jtb_WneAvp<5kgdPKm|u2BdQx9%EzcCN&U{l+kbkhmV<1}yCTDv%&K^>
zg;KCjwh*R1f_`6`si$h6`jyIKT7rTv5#k~x$mUyIw)_>Vr)D4fwIs@}{FSX|5GB1l
z4vv;@oS@>Bu7~{KgUa_8eg#Lk6IDT2IY$41$*06{>>V;Bwa(-@N;ex4;D`(QK*b}{
z{#4$Hmt)FLqERgKz=3zXiV<{YX6V)lvYBr3V>N6ajeI~~hGR5Oe>W9r@sg)Na(a4-
zxm%|1OKPN6^%JaD^^O~HbLSu=f`1px>RawOxLr+1b2^28U*2#h*W^=lSpSY4(@*^l
z{!@9RSLG8Me&RJYLi|?$c!B0fP=4xAM4rerxX{xy{&i6=AqXueQAIBqO+pmuxy8Ib
z4X^}r!NN3-upC6B#lt7&x0J;)nb9O~xjJMemm$_fHuP{DgtlU3xiW0UesTzS30L+U
zQzDI3p&3dpONhd5I8-fGk^}@unluzu%nJ$9pzoO~Kk!>dLxw@M)M9?pNH1CQhvA`z
zV;uacUtnBTdvT`M$1cm9`JrT3BMW!MNVBy%?@ZX%;(%(vqQAz<7I!hlDe|J3cn9=}
zF7B;V4xE{Ss76s$W~%*$JviK?w8^vqCp#_G^jN0j>~Xq#Zru26e#l3H^{GCLEXI#n
z?n~F-Lv#hU(bZS`EI9(xGV*jT=8R?CaK)t8oHc9XJ;UPY0Hz$XWt#QyLBaaz5+}xM
zXk(!L_*PTt7gwWH*HLWC$h3Ho!SQ-(I||nn_iEC{WT3S{3V{8IN6tZ1C+DiFM{xlI
zeMMk{o5;I6UvaC)@WKp9D+o?2Vd@4)Ue-nYci()hCCsKR`VD;hr9=vA!cgGL%3k^b(jADGyPi2TKr(JNh8mzlIR>n(F_hgiV(3@Ds(tjbNM7GoZ;T|3
zWzs8S`5PrA!9){jBJuX4y`f<4;>9*&NY=2Sq2Bp`M2(fox7ZhIDe!BaQUb@P(ub9D
zlP8!p(AN&CwW!V&>H?yPFMJ)d5x#HKfwx;nS{Rr@oHqpktOg)%F+%1#tsPtq7zI$r
zBo-Kflhq-=7_eW9B2OQv=@?|y0CKN77)N;z@tcg;heyW{wlpJ1t`Ap!O0`Xz{YHqO
zI1${8Hag^r!kA<2_~bYtM=<1YzQ#GGP+q?3T7zYbIjN6Ee^V^b&9en$8FI*NIFg9G
zPG$OXjT0Ku?%L7fat8Mqbl1`azf1ltmKTa(HH$Dqlav|rU{zP;Tbnk-XkGFQ6d+gi
z-PXh?_kEJl+K98&OrmzgPIijB4!Pozbxd0H1;Usy!;V>Yn6&pu*zW8aYx`SC!$*ti
zSn+G9p=~w6V(fZZHc>m|PPfjK6IN4(o=IFu?pC?+`UZAUTw!e`052{P=8vqT^(VeG
z=psASIhCv28Y(;7;TuYAe>}BPk5Qg=8$?wZj9lj>h2kwEfF_CpK=+O6Rq9pLn4W)#
zeXCKCpi~jsfqw7Taa0;!B5_C;B}e56W1s8@p*)SPzA;Fd$Slsn^=!_&!mRHV*Lmt|
zBGIDPuR>CgS4%cQ4wKdEyO&Z>2aHmja;Pz+n|7(#l%^2ZLCix%>@_mbnyPEbyrHaz
z>j^4SIv;ZXF-Ftzz>*t4wyq)ng8%0d;(Z_ExZ-cxwei=8{(br-`JYO(f23Wae_MqE
z3@{Mlf^%M5G1SIN&en1*|
zH~ANY1h3&WNsBy$G9{T=`kcxI#-X|>zLX2r*^-FUF+m0{k)n#GTG_mhG&fJfLj~K&
zU~~6othMlvMm9<*SUD2?RD+R17|Z4mgR$L*R3;nBbo&Vm@39&3xIg;^aSxHS>}gwR
zmzs?h8oPnNVgET&dx5^7APYx6Vv6eou07Zveyd+^V6_LzI$>ic+pxD_8s~
zC<}ucul>UH<@$KM
zT4oI=62M%7qQO{}re-jTFqo9Z;rJKD5!X5$iwUsh*+kcHVhID08MB5cQD4TBWB(rI
zuWc%CA}}v|iH=9gQ?D$1#Gu!y3o~p7416n54&Hif`U-cV?VrUMJyEqo_NC4#{puzU
zzXEE@UppeeRlS9W*^N$zS`SBBi<@tT+<%3l@KhOy^%MWB9(A#*J~DQ;+MK*$rxo6f
zcx3$3mcx{tly!q(p2DQrxcih|)0do_ZY77pyHGE#Q(0k*t!HUmmMcYFq%l$-o6%lS
zDb49W-E?rQ#Hl``C3YTEdGZjFi3R<>t)+NAda(r~f1cT5jY}s7-2^&Kvo&2DLTPYP
zhVVo-HLwo*vl83mtQ9)PR#VBg)FN}+*8c-p8j`LnNUU*Olm1O1Qqe62D#$CF#?HrM
zy(zkX|1oF}Z=T#3XMLWDrm(|m+{1&BMxHY7X@hM_+cV$5-t!8HT(dJi6m9{ja53Yw
z3f^`yb6Q;(e|#JQIz~B*=!-GbQ4nNL-NL
z@^NWF_#w-Cox@h62;r^;Y`NX8cs?l^LU;5IWE~yvU8TqIHij!X8ydbLlT0gwmzS9}
z@5BccG?vO;rvCs$mse1*ANi-cYE6Iauz$Fbn3#|ToAt5v7IlYnt6RMQEYLldva{~s
zvr>1L##zmeoYgvIXJ#>bbuCVuEv2ZvZ8I~PQUN3wjP0UC)!U+wn|&`V*8?)`
zMSCuvnuGec>QL+i1nCPGDAm@XSMIo?A9~C?g2&G8aNKjWd2pDX{qZ?04+2
zeyLw}iEd4vkCAWwa$
zbrHlEf3hfN7^1g~aW^XwldSmx1v~1z(s=1az4-wl}
z`mM+G95*N*&1EP#u3}*KwNrPIgw8Kpp((rdEOO;bT1;6ea~>>sK+?!;{hpJ3rR<6UJb`O8P4@{XGgV%63_fs%cG8L
zk9Fszbdo4tS$g0IWP1>t@0)E%-&9yj%Q!fiL2vcuL;90fPm}M==<>}Q)&sp@STFCY
z^p!RzmN+uXGdtPJj1Y-khNyCb6Y$Vs>eZyW
zPaOV=HY_T@FwAlleZCFYl@5X<<7%5DoO(7S%Lbl55?{2vIr_;SXBCbPZ(up;pC6Wx={AZL?shYOuFxLx1*>62;2rP}g`UT5+BHg(ju
z&7n5QSvSyXbioB9CJTB#x;pexicV|9oaOpiJ9VK6EvKhl4^Vsa(p6cIi$*Zr0UxQ
z;$MPOZnNae2Duuce~7|2MCfhNg*hZ9{+8H3?ts9C8#xGaM&sN;2lriYkn9W>&Gry!
z3b(Xx1x*FhQkD-~V+s~KBfr4M_#0{`=Yrh90yj}Ph~)Nx;1Y^8<418tu!$1<3?T*~
z7Dl0P3Uok-7w0MPFQexNG1P5;y~E8zEvE49>$(f|XWtkW2Mj`udPn)pb%}
zrA%wRFp*xvDgC767w!9`0vx1=q!)w!G+9(-w&p*a@WXg{?T&%;qaVcHo>7ca%KX$B
z^7|KBPo<2;kM{2mRnF8vKm`9qGV%|I{y!pKm8B(q^2V;;x2r!1VJ^Zz8bWa)!-7a8
zSRf@dqEPlsj!7}oNvFFAA)75})vTJUwQ03hD$I*j6_5xbtd_JkE2`IJD_fQ;a$EkO
z{fQ{~e%PKgPJsD&PyEvDmg+Qf&p*-qu!#;1k2r_(H72{^(Z)htgh@F?VIgK#_&eS-
z$~(qInec>)XIkv@+{o6^DJLpAb>!d}l1DK^(l%#OdD9tKK6#|_R?-%0V!`<9Hj
z3w3chDwG*SFte@>Iqwq`J4M&{aHXzyigT620+Vf$X?3RFfeTcvx_e+(&Q*z)t>c0e
zpZH$1Z3X%{^_vylHVOWT6tno=l&$3
z9^eQ@TwU#%WMQaFvaYp_we%_2-9=o{+ck
zF{cKJCOjpW&qKQquyp2BXCAP920dcrZ}T1@piukx_NY;%2W>@Wca%=Ch~x5Oj58Hv
z;D-_ALOZBF(Mqbcqjd}P3iDbek#Dwzu`WRs`;hRIr*n0PV7vT+%Io(t}8KZ
zpp?uc2eW!v28ipep0XNDPZt7H2HJ6oey|J3z!ng#1H~x_k%35P+Cp%mqXJ~cV0xdd
z^4m5^K_dQ^Sg?$P`))ccV=O>C{Ds(C2WxX$LMC5vy=*44pP&)X5DOPYfqE${)hDg<
z3hcG%U%HZ39=`#Ko4Uctg&@PQLf>?0^D|4J(_1*TFMOMB!Vv1_mnOq$BzXQdOGqgy
zOp#LBZ!c>bPjY1NTXksZmbAl0A^Y&(%a3W-k>bE&>K?px5Cm%AT2E<&)Y?O*?d80d
zgI5l~&Mve;iXm88Q+Fw7{+`PtN4G7~mJWR^z7XmYQ>uoiV!{tL)hp|=
zS(M)813PM`d<501>{NqaPo6BZ^T{KBaqEVH(2^Vjeq
zgeMeMpd*1tE@@);hGjuoVzF>Cj;5dNNwh40CnU+0DSKb~GEMb_#
zT8Z&gz%SkHq6!;_6dQFYE`+b`v4NT7&@P>cA1Z1xmXy<2htaDhm@XXMp!g($
zw(7iFoH2}WR`UjqjaqOQ$ecNt@c|K1H1kyBArTTjLp%-M`4nzOhkfE#}dOpcd;b#suq8cPJ&bf5`6Tq>ND(l
zib{VrPZ>{KuaIg}Y$W>A+nrvMg+l4)-@2jpAQ5h(Tii%Ni^-UPVg{<1KGU2EIUNGaXcEkOedJOusFT9X3%Pz$R+-+W+LlRaY-a$5r?4V
zbPzgQl22IPG+N*iBRDH%l{Zh$fv9$RN1sU@Hp3m=M}{rX%y#;4(x1KR2yCO7Pzo>rw(67E{^{yUR`91nX^&MxY@FwmJJbyPAoWZ9Z
zcBS$r)&ogYBn{DOtD~tIVJUiq|1foX^*F~O4hlLp-g;Y2wKLLM=?(r3GDqsPmUo*?
zwKMEi*%f)C_@?(&&hk>;m07F$X7&i?DEK|jdRK=CaaNu-)pX>n3}@%byPKVkpLzBq
z{+Py&!`MZ^4@-;iY`I4#6G@aWMv{^2VTH7|WF^u?3vsB|jU3LgdX$}=v7#EHRN(im
zI(3q-eU$s~r=S#EWqa_2!G?b~
z<&brq1vvUTJH380=gcNntZw%7UT8tLAr-W49;9y^=>TDaTC|cKA<(gah#2M|l~j)w
zY8goo28gj$n&zcNgqX1Qn6=<8?R0`FVO)g4&QtJAbW3G#D)uNeac-7cH5W#6i!%BH
z=}9}-f+FrtEkkrQ?nkoMQ1o-9_b+&=&C2^h!&mWFga#MCrm85hW;)1pDt;-uvQG^D
zntSB?XA*0%TIhtWDS!KcI}kp3LT>!(Nlc(lQN?k^bS8Q^GGMfo}^|%7s;#r+pybl@?KA++|FJ
zr%se9(B|g*ERQU96az%@4gYrxRRxaM2*b}jNsG|0dQi;Rw{0WM0E>rko!{QYAJJKY
z)|sX0N$!8d9E|kND~v|f>3YE|uiAnqbkMn)hu$if4kUkzKqoNoh8v|S>VY1EKmgO}
zR$0UU2o)4i4yc1inx3}brso+sio{)gfbLaEgLahj8(_Z#4R-v)
zglqwI%`dsY+589a8$Mu7#7_%kN*ekHupQ#48DIN^uhDxblDg3R1yXMr^NmkR
z7J_NWCY~fhg}h!_aXJ#?wsZF$q`JH>JWQ9`jbZzOBpS`}-A$Vgkq7+|=lPx9H7QZG
z8i8guMN+yc4*H*ANr$Q-3I{FQ-^;8ezWS2b8rERp9TMOLBxiG9J*g5=?h)mIm3#CGi4JSq1ohFrcrxx@`**K5%T}qbaCGldV!t
zVeM)!U3vbf5FOy;(h08JnhSGxm)8Kqxr9PsMeWi=b8b|m_&^@#A3lL;bVKTBx+0v8
zLZeWAxJ~N27lsOT2b|qyp$(CqzqgW@tyy?CgwOe~^i;ZH
zlL``i4r!>i#EGBNxV_P@KpYFQLz4Bdq{#zA&sc)*@7Mxsh9u%e6Ke`?5Yz1jkTdND
zR8!u_yw_$weBOU}24(&^Bm|(dSJ(v(cBct}87a^X(v>nVLIr%%D8r|&)mi+iBc;B;x;rKq
zd8*X`r?SZsTNCPQqoFOrUz8nZO?225Z#z(B!4mEp#ZJBzwd7jW1!`sg*?hPMJ$o`T
zR?KrN6OZA1H{9pA;p0cSSu;@6->8aJm1rrO-yDJ7)lxuk#npUk7WNER1Wwnpy%u
zF=t6iHzWU(L&=vVSSc^&D_eYP3TM?HN!Tgq$SYC;pSIPWW;zeNm7Pgub#yZ@7WPw#f#Kl)W4%B>)+8%gpfoH1qZ;kZ*RqfXYeGXJ_
zk>2otbp+1By`x^1V!>6k5v8NAK@T;89$`hE0{Pc@Q$KhG0jOoKk--Qx!vS~lAiypV
zCIJ&6B@24`!TxhJ4_QS*S5;;Pk#!f(qIR7*(c3dN*POKtQe)QvR{O2@QsM%ujEAWEm)
z+PM=G9hSR>gQ`Bv2(k}RAv2+$7qq(mU`fQ+&}*i%-RtSUAha>70?G!>?w%F(b4k!$
zvm;E!)2`I?etmSUFW7WflJ@8Nx`m_vE2HF#)_BiD#FaNT|IY@!uUbd4v$wTglIbIX
zblRy5=wp)VQzsn0_;KdM%g<8@>#;E?vypTf=F?3f@SSdZ;XpX~J@l1;p#}_veWHp>@Iq_T
z@^7|h;EivPYv1&u0~l9(a~>dV9Uw10QqB6Dzu1G~-l{*7IktljpK<_L8m0|7VV_!S
zRiE{u97(%R-<8oYJ{molUd>vlGaE-C|^<`hppdDz<7OS13$#J
zZ+)(*rZIDSt^Q$}CRk0?pqT5PN5TT`Ya{q(BUg#&nAsg6apPMhLTno!SRq1e60fl6GvpnwDD4N>
z9B=RrufY8+g3_`@PRg+(+gs2(bd;5#{uTZk96CWz#{=&h9+!{_m60xJxC%r&gd_N!
z>h5UzVX%_7@CUeAA1XFg_AF%(uS&^1WD*VPS^jcC!M2v@RHZML;e(H-=(4(3O&bX-
zI6>usJOS+?W&^S&DL{l|>51ZvCXUKlH2XKJPXnHjs*oMkNM#ZDLx!oaM5(%^)5XaP
zk6&+P16sA>vyFe9v`Cp5qnbE#r#ltR5E+O3!WnKn`56Grs2;sqr3r#
zp@Zp<^q`5iq8OqOlJ`pIuyK@3zPz&iJ0Jcc`hDQ1bqos2;}O|$i#}e@ua*x5VCSx
zJAp}+?Hz++tm9dh3Fvm_bO6mQo38al#>^O0g)Lh^&l82+&x)*<n7^Sw-AJo9tEzZDwyJ7L^i7|BGqHu+ea6(&7jKpBq>~V
z8CJxurD)WZ{5D0?s|KMi=e7A^JVNM6sdwg@1Eg_+Bw=9j&=+KO1PG|y(mP1@5~x>d
z=@c{EWU_jTSjiJl)d(>`qEJ;@iOBm}alq8;OK;p(1AdH$)I9qHNmxxUArdzBW0t+Qeyl)m3?D09770g
z)hzXEOy>2_{?o%2B%k%z4d23!pZcoxyW1Ik{|m7Q1>fm4`wsRrl)~h
z_=Z*zYL+EG@DV1{6@5@(Ndu!Q$l_6Qlfoz@79q)Kmsf~J7t1)tl#`MD<;1&CAA
zH8;i+oBm89dTTDl{aH`cmTPTt@^K-%*sV+t4X9q0Z{A~vEEa!&rRRr=0Rbz4NFCJr
zLg2u=0QK@w9XGE=6(-JgeP}G#WG|R&tfHRA3a9*zh5wNTBAD;@YYGx%#E4{C#Wlfo
z%-JuW9=FA_T6mR2-Vugk1uGZvJbFvVVWT@QOWz$;?u6+CbyQsbK$>O1APk|xgnh_8
zc)s@Mw7#0^wP6qTtyNq2G#s?5j~REyoU6^lT7dpX{T-rhZWHD%dik*=EA7bIJgOVf_Ga!yC8V^tkTOEHe+JK@Fh|$kfNxO^=
z#lpV^(ZQ-3!^_BhV>aXY~GC9{8%1lOJ}6vzXDvPhC>JrtXwFBC+!3a*Z-%#9}i
z#<5&0LLIa{q!rEIFSFc9)>{-_2^qbOg5;_A9
ztQ))C6#hxSA{f9R3Eh^`_f${pBJNe~pIQ`tZVR^wyp}=gLK}e5_vG@w+-mp#Fu>e|
z*?qBp5CQ5zu+Fi}xAs)YY1;bKG!htqR~)DB$ILN6GaChoiy%Bq@i+1ZnANC0U&D
z_4k$=YP47ng+0NhuEt}6C;9-JDd8i5S>`Ml==9wHDQFOsAlmtrVwurYDw_)Ihfk35
zJDBbe!*LUpg%4n>BExWz>KIQ9vexUu^d!7rc_kg#Bf=
z7TLz|l*y*3d2vi@c|pX*@ybf!+Xk|2*z$@F4K#MT8Dt4zM_EcFmNp31#7qT6(@GG?
zdd;sSY9HHuDb=w&|K%sm`bYX#%UHKY%R`3aLMO?{T#EI@FNNFNO>p@?W*i0z(g2dt
z{=9Ofh80Oxv&)i35AQN>TPMjR^UID-T7H5A?GI{MD_VeXZ%;uo41dVm=uT&ne2h0i
zv*xI%9vPtdEK@~1&V%p1sFc2AA`9?H)gPnRdlO~URx!fiSV)j?Tf5=5F>hnO=$d$x
zzaIfr*wiIc!U1K*$JO@)gP4%xp!<*DvJSv7p}(uTLUb=MSb@7_yO+IsCj^`PsxEl&
zIxsi}s3L?t+p+3FXYqujGhGwTx^WXgJ1}a@Yq5mwP0PvGEr*qu7@R$9j>@-q1rz5T
zriz;B^(ex?=3Th6h;7U`8u2sDlfS{0YyydK=*>-(NOm9>S_{U|eg(J~C7O
zIe{|LK=Y`hXiF_%jOM8Haw3UtaE{hWdzo3BbD6ud7br4cODBtN(~Hl+odP0SSWPw;I&^m)yLw+nd#}3#z}?UIcX3=SssI}`QwY=%
zAEXTODk|MqTx}2DVG<|~(CxgLyi*A{m>M@1h^wiC)4Hy>1K7@|Z&_VPJsaQoS8=ex
zDL&+AZdQa>ylxhT_Q$q=60D5&%pi6+qlY3$3c(~rsITX?>b;({FhU!7HOOhSP7>bmTkC8KM%!LRGI^~y3Ug+gh!QM=+NZXznM)?L3G=4=IMvFgX3BAlyJ
z`~jjA;2z+65D$j5xbv9=IWQ^&-K3Yh`vC(1Qz2h2`o$>Cej@XRGff!it$n{@WEJ^N
z41qk%Wm=}mA*iwCqU_6}Id!SQd13aFER3unXaJJXIsSnxvG2(hSCP{i&QH$tL&TPx
zDYJsuk+%laN&OvKb-FHK$R4dy%M7hSB*yj#-nJy?S9tVoxAuDei{s}@+pNT!vLOIC
z8g`-QQW8FKp3cPsX%{)0B+x+OhZ1=L7F-jizt|{+f1Ga7%+!BXqjCjH&x|3%?UbN#
zh?$I1^YokvG$qFz5ySK+Ja5=mkR&p{F}ev**rWdKMko+Gj^?Or=UH?SCg#0F(&a_y
zXOh}dPv0D9l0RVedq1~jCNV=8?vZfU-Xi|nkeE->;ohG3U7z+^0+HV17~-_Mv#mV`
zzvwUJJ15v5wwKPv-)i@dsEo@#WEO9zie7mdRAbgL2kjbW4&lk$vxkbq=w5mGKZK6@
zjXWctDkCRx58NJD_Q7e}HX`SiV)TZMJ}~zY6P1(LWo`;yDynY_5_L?N-P`>ALfmyl
z8C$a~FDkcwtzK9m$tof>(`Vu3#6r#+v8RGy#1D2)F;vnsiL&P-c^PO)^B-4VeJteLlT@25sPa
z%W~q5>YMjj!mhN})p$47VA^v$Jo6_s{!y?}`+h+VM_SN`!11`|;C;B};B&Z<@%FOG
z_YQVN+zFF|q5zKab&e4GH|B;sBbKimHt;K@tCH+S{7Ry~88`si7}S)1E{21nldiu5
z_4>;XTJa~Yd$m4A9{Qbd)KUAm7XNbZ4xHbg3a8-+1uf*$1PegabbmCzgC~1WB2F(W
zYj5XhVos!X!QHuZXCatkRsdEsSCc+D2?*S7a+(v%toqyxhjz|`zdrUvsxQS{J>?c&
zvx*rHw^8b|v^7wq8KWVofj&VUitbm*a&RU_ln#ZFA^3AKEf<#T%8I!Lg3XEsdH(A5
zlgh&M_XEoal)i#0tcq8c%Gs6`xu;vvP2u)D9p!&XNt
z!TdF_H~;`g@fNXkO-*t<9~;iEv?)Nee%hVe!aW`N%$cFJ(Dy9+Xk*odyFj72T!(b%Vo5zvCGZ%3tkt$@Wcx8BWEkefI1-~C_3y*LjlQ5%WEz9WD8i^
z2MV$BHD$gdPJV4IaV)G9CIFwiV=ca0cfXdTdK7oRf@lgyPx;_7*RRFk=?@EOb9Gcz
zg~VZrzo*Snp&EE{$CWr)JZW)Gr;{B2ka6B!&?aknM-FENcl%45#y?oq9QY
z3^1Y5yn&^D67Da4lI}ljDcphaEZw2;tlYuzq?uB4b9Mt6!KTW&ptxd^vF;NbX=00T
z@nE1lIBGgjqs?ES#P{ZfRb6f!At51vk%<0X%d_~NL5b8UyfQMPDtfU@>ijA0NP3UU
zh{lCf`Wu7cX!go`kUG`1K=7NN@SRGjUKuo<^;@GS!%iDXbJs`o6e`v3O8-+7vRkFm
z)nEa$sD#-v)*Jb>&Me+YIW3PsR1)h=-Su)))>-`aRcFJG-8icomO4J@60
zw10l}BYxi{eL+Uu0xJYk-Vc~BcR49Qyyq!7)PR27D`cqGrik=?k1Of>gY7q@&d&Ds
zt7&WixP`9~jjHO`Cog~RA4Q%uMg+$z^Gt&vn+d3&>Ux{_c
zm|bc;k|GKbhZLr-%p_f%dq$eiZ;n^NxoS-Nu*^Nx5vm46)*)=-Bf<;X#?`YC4tLK;
z?;u?shFbXeks+dJ?^o$l#tg*1NA?(1iFff@I&j^<74S!o;SWR^Xi);DM%8XiWpLi0
zQE2dL9^a36|L5qC5+&Pf0%>l&qQ&)OU4vjd)%I6{|H+pw<0(a``9w(gKD&+o$8hOC
zNAiShtc}e~ob2`gyVZx59y<6Fpl*$J41VJ-H*e-yECWaDMmPQi-N8XI3
z%iI@ljc+d}_okL1CGWffeaejlxWFVDWu%e=>H)XeZ|4{HlbgC-Uvof4ISYQzZ0Um>
z#Ov{k1c*VoN^f(gfiueuag)`TbjL$XVq$)aCUBL_M`5>0>6Ska^*Knk__pw{0I>jA
zzh}Kzg{@PNi)fcAk7jMAdi-_RO%x#LQszDMS@_>iFoB+zJ0Q#CQJzFGa8;pHFdi`^
zxnTC`G$7Rctm3G8t8!SY`GwFi4gF|+dAk7rh^rA{NXzc%39+xSYM~($L(pJ(8Zjs*
zYdN_R^%~LiGHm9|ElV4kVZGA*T$o@YY4qpJOxGHlUi*S*A(MrgQ{&xoZQo+#PuYRs
zv3a$*qoe9gBqbN|y|eaH=w^LE{>kpL!;$wRahY(hhzRY;d33W)m*dfem@)>pR54Qy
z
ze;^F?mwdU?K+=fBabokSls^6_6At#1Sh7W*y?r6Ss*dmZP{n;VB^LDxM1QWh;@H0J
z!4S*_5j_;+@-NpO1KfQd&;C7T`9ak;X8DTRz$hDNcjG}xAfg%gwZSb^zhE~O);NMO
zn2$fl7Evn%=Lk!*xsM#(y$mjukN?A&mzEw3W5>_o+6oh62kq=4-`e3B^$rG=XG}Kd
zK$blh(%!9;@d@3&
zGFO60j1Vf54S}+XD?%*uk7wW$f`4U3F*p7@I4Jg7f`Il}2H<{j5h?$DDe%wG7jZQL
zI{mj?t?Hu>$|2UrPr5&QyK2l3mas?zzOk0DV30HgOQ|~xLXDQ8M3o#;CNKO8RK+M;
zsOi%)js-MU>9H4%Q)#K_me}8OQC1u;f4!LO%|5toa1|u5Q@#mYy8nE9IXmR}b#sZK
z3sD395q}*TDJJA9Er7N`y=w*S&tA;mv-)Sx4(k$fJBxXva0_;$G6!9bGBw13c_Uws
zXks4u(8JA@0O9g5f?#V~qR5*u5aIe2HQO^)RW9TTcJk28l`Syl>Q#ZveEE4Em+{?%iz6=V3b>rCm9F
zPQQm@-(hfNdo2%n?B)u_&Qh7^^@U>0qMBngH8}H|v+Ejg*Dd(Y#|jgJ-A
zQ_bQscil%eY}8oN7ZL+2r|qv+iJY?*l)&3W_55T3GU;?@Om*(M`u0DXAsQ7HSl56>
z4P!*(%&wRCb?a4HH&n;lAmr4rS=kMZb74Akha2U~Ktni>>cD$6jpugjULq)D?ea%b
zk;UW0pAI~TH59P+o}*c5Ei5L-9OE;OIBt>^(;xw`>cN2`({Rzg71qrNaE=cAH^$wP
zNrK9Glp^3a%m+ilQj0SnGq`okjzmE7<3I{JLD6Jn^+oas=h*4>Wvy=KXqVBa;K&ri
z4(SVmMXPG}0-UTwa2-MJ=MTfM3K)b~DzSVq8+v-a0&Dsv>4B65{dBhD;(d44CaHSM
zb!0ne(*<^Q%|nuaL`Gb3D4AvyO8wyygm=1;9#u5x*k0$UOwx?QxR*6Od8>+ujfyo0
zJ}>2FgW_iv(dBK2OWC-Y=Tw!UwIeOAOUUC;h95&S1hn$G#if+d;*dWL#j#YWswrz_
zMlV=z+zjZJ%SlDhxf)vv@`%~$Afd)T+MS1>ZE7V$Rj#;J*<9Ld=PrK0?qrazRJWx)
z(BTLF@Wk279nh|G%ZY7_lK7=&j;x`bMND=zgh_>>-o@6%8_#Bz!FnF*onB@_k|YCF
z?vu!s6#h9bL3@tPn$1;#k5=7#s*L;FLK#=M89K^|$3LICYWIbd^qguQp02w5>8p-H
z+@J&+pP_^iF4Xu>`D>DcCnl8BUwwOlq6`XkjHNpi@B?OOd`4{dL?kH%lt78(-L}eah8?36zw9d-dI6D{$s{f=M7)1
zRH1M*-82}DoFF^Mi$r}bTB5r6y9>8hjL54%KfyHxn$LkW=AZ(WkHWR;tIWWr@+;^^
zVomjAWT)$+rn%g`LHB6ZSO@M3KBA?
z+W7ThSBgpk`jZHZUrp`F;*%6M5kLWy6AW#T{jFHTiKXP9ITrMlEdti7@&AT_a-BA!jc(Kt
zWk>IdY-2Zbz?U1)tk#n_Lsl?W;0q`;z|t9*g-xE!(}#$fScX2VkjSiboKWE~afu5d
z2B@9mvT=o2fB_>Mnie=TDJB+l`GMKCy%2+NcFsbpv<9jS@$X37K_-Y!cvF5NEY`#p
z3sWEc<7$E*X*fp+MqsOyMXO=<2>o8)E(T?#4KVQgt=qa%5FfUG_LE`n)PihCz2=iNUt7im)s@;mOc9SR&{`4s9Q6)U31mn?}Y?$k3kU
z#h??JEgH-HGt`~%)1ZBhT9~uRi8br&;a5Y3K_Bl1G)-y(ytx?ok9S*Tz#5Vb=P~xH
z^5*t_R2It95=!XDE6X{MjLYn4Eszj9Y91T2SFz@eYlx9Z9*hWaS$^5r7=W5|>sY8}mS(>e9Ez2qI1~wtlA$yv2e-Hjn&K*P
z2zWSrC~_8Wrxxf#%QAL&f8iH2%R)E~IrQLgWFg8>`Vnyo?E=uiALoRP&qT{V2{$79
z%9R?*kW-7b#|}*~P#cA@q=V|+RC9=I;aK7Pju$K-n`EoGV^-8Mk=-?@$?O37evGKn
z3NEgpo_4{s>=FB}sqx21d3*=gKq-Zk)U+bM%Q_}0`XGkYh*+jRaP+aDnRv#Zz*n$pGp
zEU9omuYVXH{AEx>=kk}h2iKt!yqX=EHN)LF}z1j
zJx((`CesN1HxTFZ7yrvA2jTPmKYVij>45{ZH2YtsHuGzIRotIFj?(8T@ZWUv{_%AI
zgMZlB03C&FtgJqv9%(acqt9N)`4jy4PtYgnhqev!r$GTIOvLF5aZ{tW5MN@9BDGu*
zBJzwW3sEJ~Oy8is`l6Ly3an7RPtRr^1Iu(D!B!0O241Xua>Jee;Rc7tWvj!%#yX#m
z&pU*?=rTVD7pF6va1D@u@b#V@bShFr3
zMyMbNCZwT)E-%L-{%$3?n}>EN>ai7b$zR_>=l59mW;tfKj^oG)>_TGCJ#HbLBsNy$
zqAqPagZ3uQ(Gsv_-VrZmG&hHaOD#RB#6J8&sL=^iMFB=gH5AIJ+w@sTf7xa&Cnl}@
zxrtzoNq>t?=(+8bS)s2p3>jW}tye0z2aY_Dh@(18-vdfvn;D?sv<>UgL{Ti08$1Q+
zZI3q}yMA^LK=d?YVg({|v?d1|R?5
zL0S3fw)BZazRNNX|7P4rh7!+3tCG~O8l+m?H}
z(CB>8(9LtKYIu3ohJ-9ecgk+L&!FX~Wuim&;v$>M4
zUfvn<=Eok(63Ubc>mZrd8d7(>8bG>J?PtOHih_xRYFu1Hg{t;%+hXu2#x%a%qzcab
zv$X!ccoj)exoOnaco_jbGw7KryOtuf(SaR-VJ0nAe(1*AA}#QV1lMhGtzD>RoUZ;WA?~!K{8%chYn?ttlz17UpDLlhTkGcVfHY6R<2r4E{mU
zq-}D?+*2gAkQYAKrk*rB%4WFC-B!eZZLg4(tR#@kUQHIzEqV48$9=Q(~J_0
zy1%LSCbkoOhRO!J+Oh#;bGuXe;~(bIE*!J@i<%_IcB7wjhB5iF#jBn5+u~fEECN2*
z!QFh!m<(>%49H12Y33+?$JxKV3xW{xSs=gxkxW-@Xds^|O1`AmorDKrE8N2-@ospk
z=Au%h=f!`_X|G^A;XWL}-_L@D6A~*4Yf!5RTTm$!t8y&fp5_oqvBjW{FufS`!)5m%
z2g(=9Ap6Y2y(9OYOWuUVGp-K=6kqQ)kM0P^TQT{X{V$*sN$wbFb-DaUuJF*!?EJPl
zJev!UsOB^UHZ2KppYTELh+kqDw+5dPFv&&;;C~=u$Mt+Ywga!8YkL2~@g67}3wAQP
zrx^RaXb1(c7vwU8a2se75X(cX^$M{FH4AHS7d2}heqqg4F0!1|Na>UtAdT%3JnS!B)&zelTEj$^b0>Oyfw=P-y-Wd^#dEFRUN*C{!`aJIHi<_YA2?piC%^
zj!p}+ZnBrM?ErAM+D97B*7L8U$K
zo(IR-&LF(85p+fuct9~VTSdRjs`d-m|6G;&PoWvC&s8z`TotPSoksp;RsL4VL@CHf
z_3|Tn%`ObgRhLmr60<;ya-5wbh&t
z#ycN_)3P_KZN5CRyG%LRO4`Ot)3vY#dNX9!f!`_>1%4Q`81E*2BRg~A-VcN7pcX#j
zrbl@7`V%n
z6J53(m?KRzKb)v?iCuYWbH*l6M77dY4keS!%>}*8n!@ROE4!|7mQ+YS4dff1JJC(t
z6Fnuf^=dajqHpH1=|pb(po9Fr8it^;2dEk|Ro=$fxqK$^Yix{G($0m-{RCFQJ~LqUnO7jJcjr
zl*N*!6WU;wtF=dLCWzD6kW;y)LEo=4wSXQDIcq5WttgE#%@*m><@H;~Q&GniA-$in
z`sjWFLgychS1kIJmPtd-w6%iKkj&dGhtB%0)pyy0M<4HZ@ZY0PWLAd7FCrj&i|NRh?>hZj*&FYnyu%Ur`JdiTu&+n
z78d3n)Rl6q&NwVj_jcr#s5G^d?VtV8bkkYco5lV0LiT+t8}98LW>d)|v|V3++zLbHC(NC@X#Hx?21J0M*gP2V`Yd^DYvVIr{C
zSc4V)hZKf|OMSm%FVqSRC!phWSyuUAu%0fredf#TDR$|hMZihJ__F!)Nkh6z)d=NC
z3q4V*K3JTetxCPgB2_)rhOSWhuXzu+%&>}*ARxUaDeRy{$xK(AC0I=9%X7dmc6?lZNqe-iM(`?Xn3x2Ov>sej6YVQJ9Q42>?4lil?X
zew-S>tm{=@QC-zLtg*nh5mQojYnvVzf3!4TpXPuobW_*xYJs;9AokrXcs!Ay
z;HK>#;G$*TPN2M!WxdH>oDY6k4A6S>BM0Nimf#LfboKxJXVBC=RBuO&g-=+@O-#0m
zh*aPG16zY^tzQLNAF7L(IpGPa+mDsCeAK3k=IL6^LcE8l0o&)k@?dz!79yxUquQIe($zm5DG
z5RdXTv)AjHaOPv6z%99mPsa#8OD@9=URvHoJ1hYnV2bG*2XYBgB!-GEoP&8fLmWGg
z9NG^xl5D&3L^io&3iYweV*qhc=m+r7C#Jppo$Ygg;jO2yaFU8+F*RmPL`
zYxfGKla_--I}YUT353k}nF1zt2NO?+kofR8Efl$Bb^&llgq+HV_UYJUH7M5IoN0sT
z4;wDA0gs55ZI|FmJ0}^Pc}{Ji-|#jdR$`!s)Di4^g3b_Qr<*Qu2rz}R6!B^;`Lj3sKWzjMYjexX)-;f5Y+HfkctE{PstO-BZan0zdXPQ=V8
zS8cBhnQyy4oN?J~oK0zl!#S|v6h-nx5to7WkdEk0HKBm;?kcNO*A+u=%f~l&aY*+J
z>%^Dz`EQ6!+SEX$>?d(~|MNWU-}JTrk}&`IR|Ske(G^iMdk04)Cxd@}{1=P0U*%L5
zMFH_$R+HUGGv|ju2Z>5x(-aIbVJLcH1S+(E#MNe9g;VZX{5f%_|Kv7|UY-CM(>vf=
z!4m?QS+AL+rUyfGJ;~uJGp4{WhOOc%2ybVP68@QTwI(8kDuYf?#^xv
zBmOHCZU8O(x)=GVFn%tg@TVW1)qJJ_bU}4e7i>&V?r
zh-03>d3DFj&@}6t1y3*yOzllYQ++BO-q!)zsk`D(z||)y&}o%sZ-tUF>0KsiYKFg6
zTONq)P+uL5Vm0w{D5Gms^>H1qa&Z##*X31=58*r%Z@Ko=IMXX{;aiMUp-!$As3{sq
z0EEk02MOsgGm7$}E%H1ys2$yftNbB%1rdo@?6~0!a8Ym*1f;jIgfcYEF(I_^+;Xdr
z2a>&oc^dF3pm(UNpazXgVzuF<2|zdPGjrNUKpdb$HOgNp*V56XqH`~$c~oSiqx;8_
zEz3fHoU*aJUbFJ&?W)sZB3qOSS;OIZ=n-*#q{?PCXi?Mq4aY@=XvlNQdA;yVC0Vy+
z{Zk6OO!lMYWd`T#bS8FV(`%flEA9El;~WjZKU1YmZpG#49`ku`oV{Bdtvzyz3{k&7
zlG>ik>eL1P93F
zd&!aXluU_qV1~sBQf$F%sM4kTfGx5MxO0zJy<#5Z&qzNfull=k1_CZivd-WAuIQf>
zBT3&WR|VD|=nKelnp3Q@A~^d_jN3@$x2$f@E~e<$dk$L@06Paw$);l*ewndzL~LuU
zq`>vfKb*+=uw`}NsM}~oY}gW%XFwy&A>bi{7s>@(cu4NM;!%ieP$8r6&6jfoq756W
z$Y<`J*d7nK4`6t`sZ;l%Oen|+pk|Ry2`p9lri5VD!Gq`U#Ms}pgX3ylAFr8(?1#&dxrtJgB>VqrlWZf61(r`&zMXsV~l{UGjI7R@*NiMJLUoK*kY&gY9kC@^}Fj*
zd^l6_t}%Ku<0PY71%zQL`@}L}48M!@=r)Q^Ie5AWhv%#l+Rhu6fRpvv$28TH;N7Cl
z%I^4ffBqx@Pxpq|rTJV)$CnxUPOIn`u278s9#ukn>PL25VMv2mff)-RXV&r`Dwid7}TEZxXX1q(h{R6v6X
z&x{S_tW%f)BHc!jHNbnrDRjGB@cam{i#zZK*_*xlW@-R3VDmp)<$}S%t*@VmYX;1h
zFWmpXt@1xJlc15Yjs2&e%)d`fimRfi?+fS^BoTcrsew%e@T^}wyVv6NGDyMGHSKIQ
zC>qFr4GY?#S#pq!%IM_AOf`#}tPoMn7JP8dHXm(v3UTq!aOfEXNRtEJ^4ED@jx%le
zvUoUs-d|2(zBsrN0wE(Pj^g5wx{1YPg9FL1)V1JupsVaXNzq4fX+R!oVX+q3tG?L=
z>=s38J_!$eSzy0m?om6Wv|ZCbYVHDH*J1_Ndajoh&?L7h&(CVii&rmLu+FcI;1qd_
zHDb3Vk=(`WV?Uq;<0NccEh0s`mBXcEtmwt6oN99RQt7MNER3`{snV$qBTp={Hn!zz
z1gkYi#^;P8s!tQl(Y>|lvz{5$uiXsitTD^1YgCp+1%IMIRLiSP`sJru0oY-p!FPbI)!6{XM%)(_Dolh1;$HlghB-&e><;zU&pc=ujpa-(+S&Jj
zX1n4T#DJDuG7NP;F5TkoG#qjjZ8NdXxF0l58RK?XO7?faM5*Z17stidTP|a%_N
z^e$D?@~q#Pf+708cLSWCK|toT1YSHfXVIs9Dnh5R(}(I;7KhKB7RD>f%;H2X?Z9eR
z{lUMuO~ffT!^ew=
z7u13>STI4tZpCQ?yb9;tSM-(EGb?iW$a1eBy4-PVejgMXFIV_Ha^XB|F}zK_gzdhM
z!)($XfrFHPf&uyFQf$EpcAfk83}91Y`JFJOiQ;v5ca?)a!IxOi36tGkPk4S6EW~eq
z>WiK`Vu3D1DaZ}515nl6>;3#xo{GQp1(=uTXl1~
z4gdWxr-8a$L*_G^UVd&bqW_nzMM&SlNW$8|$lAfo@zb+P>2q?=+T^qNwblP*RsN?N
zdZE%^Zs;yAwero1qaoqMp~|KL=&npffh981>2om!fseU(CtJ=bW7c6l{U5(07*e0~
zJRbid6?&psp)ilmYYR3ZIg;t;6?*>hoZ3uq7dvyyq-yq$zH$yyImjfhpQb@WKENSP
zl;KPCE+KXzU5!)mu12~;2trrLfs&nlEVOndh9&!SAOdeYd}ugwpE-9OF|yQs(w@C9
zoXVX`LP~V>%$<(%~tE*bsq(EFm
zU5z{H@Fs^>nm%m%wZs*hRl=KD%4W3|(@j!nJr{Mmkl`e_uR9fZ-E{JY7#s6i()WXB0g-b`R{2r@K{2h3T+a>82>722+$RM*?W5;Bmo6$X3+Ieg9&^TU(*F$Q3
zT572!;vJeBr-)x?cP;^w1zoAM`nWYVz^<6N>SkgG3s4MrNtzQO|A?odKurb6DGZffo>DP_)S0$#gGQ_vw@a9JDXs2}hV&c>$
zUT0;1@cY5kozKOcbN6)n5v)l#>nLFL_x?2NQgurQH(KH@gGe>F|$&@
zq@2A!EXcIsDdzf@cWqElI5~t
z4cL9gg7{%~4@`ANXnVAi=JvSsj95-7V&
zME3o-%9~2?cvlH#twW~99=-$C=+b5^Yv}Zh4;Mg-!LS
zw>gqc=}CzS9>v5C?#re>JsRY!w|Mtv#%O3%Ydn=S9cQarqkZwaM4z(gL~1&oJZ;t;
zA5+g3O6itCsu93!G1J_J%Icku>b3O6qBW$1Ej_oUWc@MI)|
zQ~eyS-EAAnVZp}CQnvG0N>Kc$h^1DRJkE7xZqJ0>p<>9*apXgBMI-v87E0+PeJ-K&
z#(8>P_W^h_kBkI;&e_{~!M+TXt@z8Po*!L^8XBn{of)knd-xp{heZh~@EunB2W)gd
zAVTw6ZZasTi>((qpBFh(r4)k
zz&@Mc@ZcI-4d639AfcOgHOU+YtpZ)rC%Bc5gw5o~+E-i+bMm(A6!uE>=>1M;V!Wl4
z<#~muol$FsY_qQC{JDc8b=$l6Y_@_!$av^08`czSm!Xan{l$@GO-zPq1s>WF)G=wv
zDD8j~Ht1pFj)*-b7h>W)@O&m&VyYci&}K|0_Z*w`L>1jnGfCf@6p}Ef*?wdficVe_
zmPRUZ(C+YJU+hIj@_#IiM7+$4kH#VS5tM!Ksz01siPc-WUe9Y3|pb4u2qnn
zRavJiRpa
zq?tr&YV?yKt<@-kAFl3s&Kq#jag$hN+Y%%kX_ytvpCsElgFoN3SsZLC>0f|m#&Jhu
zp7c1dV$55$+k78FI2q!FT}r|}cIV;zp~#6X2&}22$t6cHx_95FL~T~1XW21VFuatb
zpM@6w>c^SJ>Pq6{L&f9()uy)TAWf;6LyHH3BUiJ8A4}od)9sriz~e7}l7Vr0e%(=>KG1Jay
zW0azuWC`(|B?<6;R)2}aU`r@mt_#W2VrO{LcX$Hg9f4H#XpOsAOX02x^w9+xnLVAt
z^~hv2guE-DElBG+`+`>PwXn5kuP_ZiOO3QuwoEr)ky;o$n7hFoh}Aq0@Ar<8`H!n}
zspCC^EB=6>$q*gf&M2wj@zzfBl(w_@0;h^*fC#PW9!-kT-dt*e7^)OIU{Uw%U4d#g
zL&o>6`hKQUps|G4F_5AuFU4wI)(%9(av7-u40(IaI|%ir@~w9-rLs&efOR@oQy)}{
z&T#Qf`!|52W0d+>G!h~5A}7VJky`C3^fkJzt3|M&xW~x-8rSi-uz=qBsgODqbl(W#f{Ew#ui(K)(Hr&xqZs`
zfrK^2)tF#|U=K|_U@|r=M_Hb;qj1GJG=O=d`~#AFAccecIaq3U`(Ds1*f*TIs=IGL
zp_vlaRUtFNK8(k;JEu&|i_m39c(HblQkF8g#l|?hPaUzH2kAAF1>>Yykva0;U@&oRV8w?5yEK??A0SBgh?@Pd
zJg{O~4xURt7!a;$rz9%IMHQeEZHR8KgFQixarg+MfmM_OeX#~#&?mx44qe!wt`~dd
zqyt^~ML>V>2Do$huU<7}EF2wy9^kJJSm6HoAD*sRz%a|aJWz_n6?bz99h)jNMp}3k
ztPVbos1$lC1nX_OK0~h>=F&v^IfgBF{#BIi&HTL}O7H-t4+wwa)kf3AE2-Dx@#mTA
z!0f`>vz+d3AF$NH_-JqkuK1C+5>yns0G;r5ApsU|a-w9^j4c+FS{#+7-
zH%skr+TJ~W_8CK_j$T1b;$ql_+;q6W|D^BNK*A+W5XQBbJy|)(IDA=L9d>t1`KX2b
zOX(Ffv*m?e>!
zS3lc>XC@IqPf1g-%^4XyGl*1v0NWnwZTW?z4Y6sncXkaA{?NYna3(n@(+n+#sYm}A
zGQS;*Li$4R(Ff{obl3#6pUsA0fKuWurQo$mWXMNPV5K66V!XYOyc})^>889Hg3I<{V^Lj9($B4Zu$xRr=89-lDz9x`+I8q(vEAimx1K{sTbs|5x7S
zZ+7o$;9&9>@3K;5-DVzGw=kp7ez%1*kxhGytdLS>Q)=xUWv3k_x(IsS8we39Tijvr
z`GKk>gkZTHSht;5q%fh9z?vk%sWO}KR04G9^jleJ^@ovWrob7{1xy7V=;S~dDVt%S
za$Q#Th%6g1(hiP>hDe}7lcuI94K-2~Q0R3A1nsb7Y*Z!DtQ(Ic<0;TDKvc6%1kBdJ
z$hF!{uALB0pa?B^TC}#N5gZ|CKjy|BnT$7eaKj;f>Alqdb_FA3yjZ4CCvm)D&ibL)
zZRi91HC!TIAUl<|`rK_6avGh`!)TKk=j|8*W|!vb9>HLv^E%t$`@r@piI(6V8pqDG
zBON7~=cf1ZWF6jc{qkKm;oYBtUpIdau6s+<-o^5qNi-p%L%xAtn9OktFd{@EjVAT%
z#?-MJ5}Q9QiK_jYYWs+;I4&!N^(mb!%4zx7qO6oCEDn=8oL6#*9XIJ&iJ30O`0vsFy|fEVkw}*jd&B6!IYi+~Y)qv6QlM&V9g0
zh)@^BVDB|P&#X{31>G*nAT}Mz-j~zd>L{v{9AxrxKFw8j;ccQ$NE0PZCc(7fEt1xd
z`(oR2!gX6}R+Z77VkDz^{I)@%&HQT5q+1xlf*3R^U8q%;IT8-B53&}dNA7GW`Ki&=
z$lrdH
zDCu;j$GxW<&v_4Te7=AE2J0u1NM_7Hl9$u{z(8#%8vvrx2P#R7AwnY|?#LbWmROa;
zOJzU_*^+n(+k;Jd{e~So9>OF>fPx$Hb$?~K1ul2xr>>o@**n^6IMu8+o3rDp(X$cC
z`wQt9qIS>yjA$K~bg{M%kJ00A)U4L+#*@$8UlS#lN3YA{R{7{-zu#n1>0@(#^eb_%
zY|q}2)jOEM8t~9p$X5fpT7BZQ1bND#^Uyaa{mNcFWL|MoYb@>y`d{VwmsF&haoJuS2W7azZU0{tu#Jj_-^QRc35tjW~ae&zhKk!wD}#xR1WHu
z_7Fys#bp&R?VXy$WYa$~!dMxt2@*(>@xS}5f-@6eoT%rwH
zv_6}M?+piNE;BqaKzm1kK@?fTy$4k5cqYdN8x-<(o6KelwvkTqC3VW5HEnr+WGQlF
zs`lcYEm=HPpmM4;Ich7A3a5Mb3YyQs7(Tuz-k4O0*-YGvl+2&V(B&L1F8qfR0@vQM-rF<2h-l9T12eL}3LnNAVyY_z51xVr$%@VQ-lS~wf3mnHc
zoM({3Z<3+PpTFCRn_Y6cbxu9v>_>eTN0>hHPl_NQQuaK^Mhrv
zX{q#80ot;ptt3#js3>kD&uNs{G0mQp>jyc0GG?=9wb33hm
z`y2jL=J)T1JD7eX3xa4h$bG}2ev=?7f>-JmCj6){Upo&$k{2WA=%f;KB;X5e;JF3IjQBa4e-Gp~xv-
z|In&Rad7LjJVz*q*+splCj|{7=kvQLw0F@$vPuw4m^z=B^7=A4asK_`%lEf_oIJ-O
z{L)zi4bd#&g0w{p1$#I&@bz3QXu%Y)j46HAJKWVfRRB*oXo4lIy7BcVl4hRs<%&iQ
zr|)Z^LUJ>qn>{6y`JdabfNNFPX7#3`x|uw+z@h<`x{J4&NlDjnknMf(VW_nKWT!Jh
zo1iWBqT6^BR-{T=4Ybe+?6zxP_;A5Uo{}Xel%*=|zRGm1)pR43K39SZ=%{MDCS2d$~}PE-xPw4ZK6)H;Zc&0D5p!vjCn0wCe&rVIhchR9ql!p2`g0b@JsC^J#n_r*4lZ~u0UHKwo(HaHUJDHf^gdJhTdTW
z3i7Zp_`xyKC&AI^#~JMVZj^9WsW}UR#nc#o+ifY<4`M+?Y9NTBT~p`ONtAFf8(ltr*ER-Ig!yRs2xke#NN
zkyFcaQKYv>L8mQdrL+#rjgVY>Z2_$bIUz(kaqL}cYENh-2S6BQK-a(VNDa_UewSW`
zMgHi<3`f!eHsyL6*^e^W7#l?V|42CfAjsgyiJsA`yNfAMB*lAsJj^K3EcCzm1KT
zDU2+A5~X%ax-JJ@&7>m`T;;}(-e%gcYQtj}?ic<*gkv)X2-QJI5I0tA2`*zZRX(;6
zJ0dYfMbQ+{9Rn3T@Iu4+imx3Y%bcf2{uT4j-msZ~eO)5Z_T7NC|Nr3)|NWjomhv=E
zXaVin)MY)`1QtDyO7mUCjG{5+o1jD_anyKn73uflH*ASA8rm+S=gIfgJ);>Zx*hNG
z!)8DDCNOrbR#9M7Ud_1kf6BP)x^p(|_VWCJ+(WGDbYmnMLWc?O4zz#eiP3{NfP1UV
z(n3vc-axE&vko^f+4nkF=XK-mnHHQ7>w05$Q}iv(kJc4O3TEvuIDM<=U9@`~WdKN*
zp4e4R1ncR_kghW}>aE$@OOc~*aH5OOwB5U*Z)%{LRlhtHuigxH8KuDwvq5{3Zg{Vr
zrd@)KPwVKFP2{rXho(>MTZZfkr$*alm_lltPob4N4MmhEkv`J(9NZFzA>q0Ch;!Ut
zi@jS_=0%HAlN+$-IZGPi_6$)ap>Z{XQGt&@ZaJ(es!Po5*3}>R4x66WZNsjE4BVgn
z>}xm=V?F#tx#e+pimNPH?Md5hV7>0pAg$K!?mpt@pXg6UW9c?gvzlNe0
z3QtIWmw$0raJkjQcbv-7Ri&eX6Ks@@EZ&53N|g7HU<;V1pkc&$3D#8k!coJ=^{=vf
z-pCP;vr2#A+i#6VA?!hs6A4P@mN62XYY$#W9;MwNia~89i`=1GoFESI+%Mbrmwg*0
zbBq4^bA^XT#1MAOum)L&ARDXJ6S#G>&*72f50M1r5JAnM1p7GFIv$Kf9eVR(u$KLt
z9&hQ{t^i16zL1c(tRa~?qr?lbSN;1k;%;p*#gw_BwHJRjcYPTj6>y-rw*dFTnEs95
z`%-AoPL!P16{=#RI0
zUb6#`KR|v^?6uNnY`zglZ#Wd|{*rZ(x&Hk8N6ob6mpX~e^qu5kxvh$2TLJA$M=rx
zc!#ot+sS+-!O<0KR6+Lx&~zgEhCsbFY{i_DQCihspM?e
z-V}HemMAvFzXR#fV~a=Xf-;tJ1edd}Mry@^=9BxON;dYr8vDEK<<{
zW~rg(ZspxuC&aJo$GTM!9_sXu(EaQJNkV9AC(ob#uA=b4*!Uf}B*@TK=*dBvKKPAF
z%14J$S)s-ws9~qKsf>DseEW(ssVQ9__YNg}r9GGx3AJiZR@w_QBlGP>yYh0lQCBtf
zx+G;mP+cMAg&b^7J!`SiBwC81M_r0X9kAr2y$0(Lf1gZK#>i!cbww(hn$;fLIxRf?
z!AtkSZc-h76KGSGz%48Oe`8ZBHkSXeVb!TJt_VC>$m<#}(Z}!(3h631ltKb3CDMw^fTRy%Ia!b&at`^g7Ew-%WLT9(#V0OP9CE?uj62s>`GI3NA
z!`$U+i<`;IQyNBkou4|-7^9^ylac-Xu!M+V5p5l0Ve?J0wTSV+$gYtoc=+Ve*OJUJ
z$+uIGALW?}+M!J9+M&#bT=Hz@{R2o>NtNGu1yS({pyteyb>*sg4N`KAD?`u3F#C1y
z2K4FKOAPASGZTep54PqyCG(h3?kqQQAxDSW@>T2d!n;9C8NGS;3A8YMRcL>b=<<%M
zMiWf$jY;`Ojq5S{kA!?28o)v$;)5bTL<4eM-_^h4)F#eeC2Dj*S`$jl^yn#NjJOYT
zx%yC5Ww@eX*zsM)P(5#wRd=0+3~&3pdIH7CxF_2iZSw@>kCyd
z%M}$1p((Bidw4XNtk&`BTkU{-PG)SXIZ)yQ!Iol6u8l*SQ1^%zC72FP
zLvG>_Z0SReMvB%)1@+et0S{<3hV@^SY3V~5IY(KUtTR{*^xJ^2NN{sIMD9Mr9$~(C$GLNlSpzS=fsbw-DtHb_T|{s
z9OR|sx!{?F``H!gVUltY7l~dx^a(2;OUV^)7
z%@hg`8+r&xIxmzZ;Q&v0X%9P)U0SE@r@(lKP%TO(>6I_iF{?PX(bez6v8Gp!W_nd5
z<8)`1jcT)ImNZp-9rr4_1MQ|!?#8sJQx{`~7)QZ75I=DPAFD9Mt{zqFrcrXCU9MG8
zEuGcy;nZ?J#M3!3DWW?Zqv~dnN6ijlIjPfJx(#S0cs;Z=jDjKY|$w2s4*Xa1Iz953sN2Lt!Vmk|%ZwOOqj`sA--5Hiaq8!C%LV
zvWZ=bxeRV(&%BffMJ_F~~*FdcjhRVNUXu)MS(S#67rDe%Ler=GS+WysC1I2=Bmbh3s6wdS}o$0
zz%H08#SPFY9JPdL6blGD$D-AaYi;X!#zqib`(XX*i<*eh+2UEPzU4}V4RlC3{<>-~
zadGA8lSm>b7Z!q;D_f9DT4i)Q_}ByElGl*Cy~zX%IzHp)@g-itZB6xM70psn
z;AY8II99e6P2drgtTG5>`^|7qg`9MTp%T~|1N3tBqV}2zgow3TFAH{XPor0%=HrkXnKyxyozHlJ6
zd3}OWkl?H$l#yZqOzZbMI+lDLoH48;s10!m1!K87g;t}^+A3f3e&w{EYhVPR0Km*-
zh5-ku$Z|Ss{2?4pGm(Rz!0OQb^_*N`)rW{z)^Cw_`a(_L9j=&HEJl(!4rQy1IS)>-
zeTIr>hOii`gc(fgYF(cs$R8l@q{mJzpoB5`5r>|sG
zBpsY}RkY(g5`bj~D>(;F8v*DyjX(#nVLSs>)XneWI&%Wo>a0u#4A?N<1SK4D}&V1oN)76
z%S>a2n3n>G`YY1>0Hvn&AMtMuI_?`5?4y3w2Hnq4Qa2YH5
zxKdfM;k467djL31Y$0kd9FCPbU=pHBp@zaIi`Xkd80;%&66zvSqsq6%aY)jZacfvw
ztkWE{ZV6V2WL9e}Dvz|!d96KqVkJU@5ryp#rReeWu>mSrOJxY^tWC9wd0)$+lZc%{
zY=c4#%OSyQJvQUuy^u}s8DN8|8T%TajOuaY^)R-&8s@r9D`(Ic4NmEu)fg1f!u`xUb;9t#rM
z>}cY=648@d5(9A;J)d{a^*ORdVtJrZ77!g~^lZ9@)|-ojvW#>)Jhe8$7W3mhmQh@S
zU=CSO+1gSsQ+Tv=x-BD}*py_Ox@;%#hPb&tqXqyUW9jV+fonnuCyVw=?HR>dAB~Fg
z^vl*~y*4|)WUW*9RC%~O1gHW~*tJb^a-j;ae2LRNo|0S2`RX>MYqGKB^_ng7YRc@!
zFxg1X!VsvXkNuv^3mI`F2=x6$(pZdw=jfYt1ja3FY7a41T07FPdCqFhU6%o|Yb6Z4
zpBGa=(ao3vvhUv#*S{li|EyujXQPUV;0sa5!0Ut)>tPWyC9e0_9(=v*z`TV5OUCcx
zT=w=^8#5u~7<}8Mepqln4lDv*-~g^VoV{(+*4w(q{At6d^E-Usa2`JXty++Oh~on^
z;;WHkJsk2jvh#N|?(2PLl+g!M0#z_A;(#Uy=TzL&{Ei5G9#V{JbhKV$Qmkm%5tn!CMA?
z@hM=b@2DZWTQ6>&F6WCq6;~~WALiS#@{|I+ucCmD6|tBf&e;$_)%JL8$oIQ%!|Xih1v4A$=7xNO
zZVz$G8;G5)rxyD+M0$20L$4yukA_D+)xmK3DMTH3Q+$N&L%qB)XwYx&s1gkh=%qGCCPwnwhbT4p%*3R)I}S#w7HK3W^E%4w
z2+7ctHPx3Q97MFYB48HfD!xKKb(U^K_4)Bz(5dvwyl*R?)k;uHEYVi|{^rvh)w7}t
z`tnH{v9nlVHj2ign|1an_wz0vO)*`3RaJc#;(W-Q6!P&>+@#fptCgtUSn4!@b7tW0&pE2Qj@7}f#ugu4*C)8_}AMRuz^WG
zc)XDcOPQjRaGptRD^57B83B-2NKRo!j6TBAJntJPHNQG;^Oz}zt5F^kId~miK3J@l
ztc-IKp6qL!?u~q?qfGP0I~$5gvq#-0;R(oLU@sYayr*QH95fnrYA*E|n%&FP@Cz`a
zSdJ~(c@O^>qaO`m9IQ8sd8!L<+)GPJDrL7{4{ko2gWOZel^3!($Gjt|B&$4dtfTmBmC>V`R&&6$wpgvdmns
zxcmfS%9_ZoN>F~azvLFtA(9Q5HYT#A(byGkESnt{$Tu<73$W~reB4&KF^JBsoqJ6b
zS?$D7DoUgzLO-?P`V?5_ub$nf1p0mF?I)StvPomT{uYjy!w&z$t~j&en=F~hw|O(1
zlV9$arQmKTc$L)Kupwz_zA~deT+-0WX6NzFPh&d+ly*3$%#?Ca9Z9lOJsGVoQ&1HNg+)tJ_sw)%oo*DK)iU~n
zvL``LqTe=r=7SwZ@LB)9|3QB5`0(B9r(iR}0nUwJss-v=dXnwMRQFYSRK1blS#^g(3@z{`=8_CGDm!LESTWig
zzm1{?AG&7`uYJ;PoFO$o8RWuYsV26V{>D-iYTnvq7igWx9@w$EC*FV^vpvDl@i9yp
zPIqiX@hEZF4VqzI3Y)CHhR`xKN8poL&~ak|wgbE4zR%Dm(a@?bw%(7(!^>CM!^4@J
z6Z)KhoQP;WBq_Z_&<@i2t2&xq>N>b;Np2rX?yK|-!14iE2T}E|jC+=wYe~`y38g3J
z8QGZquvqBaG!vw&VtdXWX5*i5*%
zJP~7h{?&E|<#l{klGPaun`IgAJ4;RlbRqgJz5rmHF>MtJHbfqyyZi53?Lhj=(Ku#&
z__ubmZIxzSq3F90Xur!1)Vqe6b@!ueHA!93H~jdHmaS5Q^CULso}^poy)0Op6!{^9
zWyCyyIrdBP4fkliZ%*g+J-A!6VFSRF6Liu6G^^=W>cn81>4&7(c7(6vCGSAJ
zQZ|S3mb|^Wf=yJ(h~rq`iiW~|n#$+KcblIR<@|lDtm!&NBzSG-1;7#YaU+-@=xIm4
zE}edTYd~e&_%+`dIqqgFntL-FxL3!m4yTNt<(^Vt9c6F(`?9`u>$oNxoKB29<}9FE
zgf)VK!*F}nW?}l95%RRk8N4^Rf8)Xf;drT4<|lUDLPj^NPMrBPL;MX&0oGCsS
za3}vWcF(IPx&W6{s%zwX{UxHX2&xLGfT{d9bWP!g;Lg#etpuno$}tHoG<4Kd*=kpU
z;4%y(<^yj(UlG%l-7E9z_Kh2KoQ19qT3CR@Ghr>BAgr3Vniz3LmpC4g=g|A3968yD2KD$P7v$
zx9Q8`2&qH3&y-iv0#0+jur@}k`6C%7fKbCr|tHX2&O%r?rBpg`YNy~2m+
z*L7dP$RANzVUsG_Lb>=__``6vA*xpUecuGsL+AW?BeSwyoQfDlXe8R1*R1M{0#M?M
zF+m19`3<`gM{+GpgW^=UmuK*yMh3}x)7P738wL8r@(Na6%ULPgbPVTa6gh5Q(SR0f
znr6kdRpe^(LVM;6Rt(Z@Lsz3EX*ry6(WZ?w>#ZRelx)N%sE+MN>5G|Z8{%@b&D+Ov
zPU{shc9}%;G7l;qbonIb_1m^Qc8ez}gTC-k02G8Rl?7={9zBz8uRX2{XJQ{vZhs67avlRn|
zgRtWl0Lhjet&!YC47GIm%1gdq%T24_^@!W3pCywc89X4I5pnBCZDn(%!$lOGvS*`0!AoMtqxNPFgaMR
zwoW$p;8l6v%a)vaNsesED3f}$%(>zICnoE|5JwP&+0XI}JxPccd+D^gx`g`=GsUc0
z9Uad|C+_@_0%JmcObGnS@3+J^0P!tg+fUZ_w#4rk#TlJYPXJiO>SBxzs9(J;XV9d{
zmTQE1(K8EYaz9p^XLbdWudyIPJlGPo0U*)fAh-jnbfm@SYD_2+?|DJ-^P+ojG{2{6
z>HJtedEjO@j_tqZ4;Zq1t5*5cWm~W?HGP!@_f6m#btM@46cEMhhK{(yI&jG)fwL1W
z^n_?o@G8a-jYt!}$H*;{0#z8lANlo!9b@!c5K8<(#lPlpE!z86Yq#>WT&2}
z;;G1$pD%iNoj#Z=&kij5&V1KHIhN-h<;{HC5wD)PvkF>CzlQOEx_0;-TJ*!#&{Wzt
zKcvq^SZIdop}y~iouNqtU7K7+?eIz-v_rfNM>t#i+dD$s_`M;sjGubTdP)WI*uL@xPOLHt#~T<@Yz>xt50ZoTw;a(a}lNiDN-J${gOdE
zx?8LOA|tv{Mb}=TTR=LcqMqbCJkKj+@;4Mu)Cu0{`~ohix6E$g&tff)aHeUAQQ%M?
zIN4uSUTzC1iMEWL*W-in1y)C`E+R8j?4_?X4&2Zv5?QdkNMz(k}
zw##^Ikx`#_s>i&CO_mu@vJJ*|3ePRDl5pq$9V^>D;g0R%l>lw;ttyM6Sy`NBF{)Lr
zSk)V>mZr96+aHY%vTLLt%vO-+juw6^SO_
zYGJaGeWX6W(TOQx=5oTGXOFqMMU*uZyt>MR-Y`vxW#^&)H
zk0!F8f*@v6NO@Z*@Qo)+hlX40EWcj~j9dGrLaq%1;DE_%#lffXCcJ;!ZyyyZTz74Q
zb2WSly6sX{`gQeToQsi1-()5EJ1nJ*kXGD`xpXr~?F#V^sxE3qSOwRSaC9x9oa~jJ
zTG9`E|q
zC5Qs1xh}jzb5UPYF`3N9YuMnI7xsZ41P;?@c|%w
zl=OxLr6sMGR+`LStLvh)g?fA5p|xbUD;yFAMQg&!PEDYxVYDfA>oTY;CFt`cg?Li1
z0b})!9Rvw&j#*&+D2))kXLL
z0+j=?7?#~_}N-qdEIP>DQaZh#F(#e0WNLzwUAj@r694VJ8?Dr5_io2X49XYsG^
zREt0$HiNI~6VV!ycvao+0v7uT$_ilKCvsC+VDNg7yG1X+eNe^3D^S==F3ByiW0T^F
zH6EsH^}Uj^VPIE&m)xlmOScYR(w750>hclqH~~dM2+;%GDXT`u4zG!p((*`Hwx41M
z4KB+`hfT(YA%W)Ve(n+Gu9kuXWKzxg{1ff^xNQw>w%L-)RySTk9kAS92(X0Shg^Q?
zx1YXg_TLC^?h6!4mBqZ9pKhXByu|u~gF%`%`vdoaGBN3^j4l!4x?Bw4Jd)Z4^di}!
zXlG1;hFvc>H?bmmu1E7Vx=%vahd!P1#ZGJOJYNbaek^$DHt`EOE|Hlij+hX>ocQFSLVu|wz`|KVl@Oa;m2k6b*mNK2Vo{~l9>Qa3@B7G7#k?)aLx;w6U
ze8bBq%vF?5v>#TspEoaII!N}sRT~>bh-VWJ7Q*1qsz%|G)CFmnttbq$Ogb{~YK_=!
z{{0vhlW@g!$>|}$&4E3@k`KPElW6x#tSX&dfle>o!irek$NAbDzdd2pVeNzk4&qgJ
zXvNF0$R96~g0x+R1igR=Xu&X_Hc5;!Ze&C)eUTB$9wW&?$&o8Yxhm5s(S`;?{>
z*F?9Gr0|!OiKA>Rq-ae=_okB6&yMR?!JDer{@iQgIn=cGxs-u^!8Q$+N&pfg2WM&Z
zulHu=Uh~U>fS{=Nm0x>ACvG*4R`Dx^kJ65&Vvfj`rSCV$5>c04N26Rt2S?*kh3JKq
z9(3}5T?*x*AP(X2Ukftym0XOvg~r6Ms$2x&R&#}Sz23aMGU&7sU-cFvE3Eq`NBJe84VoftWF#v7PDAp`@V
zRFCS24_k~;@~R*L)eCx@Q9EYmM)Sn}HLbVMyxx%{XnMBDc-YZ<(DXDBYUt8$u5Zh}
zBK~=M9cG$?_m_M61YG+#|9Vef7LfbH>(C21&aC)x$^Lg}fa#SF){RX|?-xZjSOrn#
z2ZAwUF)$VB<&S;R3FhNSQOV~8w%A`V9dWyLiy
zgt7G=Z4t|zU3!dh5|s(@XyS|waBr$>@=^Dspmem8)@L`Ns{xl%rGdX!R(BiC5C7Vo
zXetb$oC_iXS}2x_Hy}T(hUUNbO47Q@+^4Q`h>(R-;OxCyW#eoOeC51jzxnM1yxBrp
zz6}z`(=cngs6X05e79o_B7@3K|Qpe3n38Py_~
zpi?^rj!`pq!7PHGliC$`-8A^Ib?2qgJJCW+(&TfOnFGJ+@-<<~`7BR0f4oSINBq&R
z2CM`0%WLg_Duw^1SPwj-{?BUl2Y=M4e+7yL1{C&&f&zjF06#xf>VdLozgNye(BNgSD`=fFbBy0HIosLl@JwCQl^s;eTnc(
z3!r8G=K>zb`|bLLI0N|eFJk%s)B>oJ^M@AQzqR;HUjLsOqW<0v>1ksT_#24*U@R3HJu*A^#1o#P3%3_jq>icD@<`tqU6ICEgZrME(xX#?i^Z
z%Id$_uyQGlFD-CcaiRtRdGn|K`Lq5L-rx7`vYYGH7I=eLfHRozPiUtSe~Tt;IN2^gCXmf2#D~g2@9bhzK}3nphhG%d?V7+Zq{I2?Gt*!NSn_r~dd$
zqkUOg{U=MI?Ehx@`(X%rQB?LP=CjJ*V!rec{#0W2WshH$X#9zep!K)tzZoge*LYd5
z@g?-j5_mtMp>_WW`p*UNUZTFN{_+#m*bJzt{hvAdkF{W40{#L3w6gzPztnsA_4?&0
z(+>pv!zB16rR-(nm(^c>Z(its{ny677vT8sF564^mlZvJ!h65}OW%Hn|2OXbOQM%b
z{6C54Z2v;^hyMQ;UH+HwFD2!F!VlQ}6Z{L0_9g5~CH0@Mqz?ZC`^QkhOU#$Lx<4`B
zyZsa9uPF!rZDo8ZVfzzR#raQ>5|)k~_Ef*wDqG^76o)j!C4
zykvT*o$!-MBko@?{b~*Zf2*YMlImrK`cEp|#D7f%Twm<|C|dWD \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m"'
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn () {
-    echo "$*"
-}
-
-die () {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=$((i+1))
-    done
-    case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Escape application args
-save () {
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
-    echo " "
-}
-APP_ARGS=$(save "$@")
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
-
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
-exec "$JAVACMD" "$@"
diff --git a/gradlew.bat b/gradlew.bat
deleted file mode 100644
index 6d57edc..0000000
--- a/gradlew.bat
+++ /dev/null
@@ -1,84 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m"
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
diff --git a/paldb/overview.html b/overview.html
similarity index 100%
rename from paldb/overview.html
rename to overview.html
diff --git a/paldb/build.gradle b/paldb/build.gradle
deleted file mode 100644
index 731e691..0000000
--- a/paldb/build.gradle
+++ /dev/null
@@ -1,130 +0,0 @@
-apply plugin: 'java'
-apply plugin: 'jacoco'
-apply plugin: 'signing'
-apply plugin: 'maven'
-
-sourceCompatibility = JavaVersion.VERSION_13
-targetCompatibility = JavaVersion.VERSION_13
-group = GROUP
-version = VERSION_NAME
-
-test {
-  useTestNG()
-}
-
-sourceSets {
-
-  perfTest {
-    java.srcDir file('src/perfTest/java')
-    resources.srcDir file('src/perfTest/resources')
-    compileClasspath += sourceSets.main.output + test.output
-    runtimeClasspath += sourceSets.main.output + test.output
-  }
-
-}
-
-jar {
-  from sourceSets.main.allSource
-}
-
-task perfTest(type: Test) {
-  testClassesDirs = sourceSets.perfTest.output.classesDirs
-  classpath = sourceSets.perfTest.runtimeClasspath
-}
-
-perfTest {
-  useTestNG()
-
-  maxHeapSize = "4g"
-}
-
-dependencies {
-  compile 'org.xerial.snappy:snappy-java:1.1.7.3'
-  compile 'org.slf4j:slf4j-api:1.7.28'
-
-  testCompile 'org.testng:testng:7.0.0'
-  testCompile 'commons-codec:commons-codec:1.13'
-  testCompile 'commons-lang:commons-lang:2.6'
-  testCompile 'ch.qos.logback:logback-classic:1.2.3'
-
-  perfTestCompile configurations.testCompile
-
-  // Only used for benchmark
-  perfTestCompile 'org.fusesource.leveldbjni:leveldbjni-all:1.8'
-  perfTestCompile 'org.rocksdb:rocksdbjni:4.0'
-}
-
-javadoc {
-  options.overview = "overview.html"
-}
-
-jacocoTestReport {
-  reports {
-    html.enabled = false
-    xml.enabled = true
-  }
-}
-
-if (project.hasProperty('release')) {
-
-  signing {
-    useGpgCmd()
-    sign configurations.archives
-  }
-
-  task javadocJar(type: Jar) {
-    classifier = 'javadoc'
-    from javadoc
-  }
-
-  task sourcesJar(type: Jar) {
-    classifier = 'sources'
-    from sourceSets.main.allSource
-  }
-
-  artifacts {
-    archives javadocJar, sourcesJar
-  }
-
-  uploadArchives {
-    repositories {
-      mavenDeployer {
-        // POM signature
-        beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }
-        // Target repository
-        repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") {
-          authentication(userName: ossrhUser, password: ossrhPassword)
-        }
-        pom.project {
-          group 'net.soundvibe.paldb'
-          name 'paldb'
-          description 'Embeddable persistent write-once key-value store'
-          packaging 'jar'
-          url 'https://github.com/soundvibe/PalDB'
-
-          scm {
-            connection 'scm:git@github.com:soundvibe/PalDB.git'
-            developerConnection 'scm:git@github.com:soundvibe/PalDB.git'
-            url 'https://github.com/soundvibe/PalDB'
-          }
-
-          licenses {
-            license {
-              name 'The Apache Software License, Version 2.0'
-              url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
-              distribution 'repo'
-            }
-          }
-
-          developers {
-            developer {
-              id 'lnaginionis'
-              name 'Linas Naginionis'
-              email 'lnaginionis@gmail.com'
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..1caba2a
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,249 @@
+
+
+    4.0.0
+
+    net.soundvibe
+    paldb
+    2.0.0
+    jar
+    paldb
+    Embeddable persistent write-once key-value store
+    https://github.com/soundvibe/paldb
+    
+        
+            The Apache Software License, Version 2.0
+            http://www.apache.org/licenses/LICENSE-2.0.txt
+            repo
+        
+    
+
+    
+        scm:git:https://github.com/soundvibe/paldb.git
+        scm:git:https://github.com/soundvibe/paldb.git
+        https://github.com/soundvibe/paldb.git
+        HEAD
+    
+    
+        
+            linasnaginionis
+            Linas Naginionis
+            lnaginionis@gmail.com
+        
+    
+
+    
+        13
+        UTF-8
+    
+
+    
+        
+            ossrh
+            https://oss.sonatype.org/content/repositories/snapshots
+        
+        
+            ossrh
+            https://oss.sonatype.org/service/local/staging/deploy/maven2/
+        
+    
+
+    
+
+        
+            org.xerial.snappy
+            snappy-java
+            1.1.7.3
+        
+
+        
+            org.slf4j
+            slf4j-api
+            1.7.28
+        
+
+        
+
+        
+            org.junit.jupiter
+            junit-jupiter
+            5.4.2
+            test
+        
+
+        
+            commons-codec
+            commons-codec
+            1.13
+            test
+        
+
+        
+            commons-lang
+            commons-lang
+            2.6
+            test
+        
+
+        
+            ch.qos.logback
+            logback-classic
+            1.2.3
+            test
+        
+
+        
+            org.fusesource.leveldbjni
+            leveldbjni-all
+            1.8
+            test
+        
+
+        
+            org.rocksdb
+            rocksdbjni
+            4.0
+            test
+        
+
+    
+
+    
+        
+            
+                maven-surefire-plugin
+                2.22.2
+                
+                    
+                        **/Test*.java
+                        **/*Test.java
+                        **/*Tests.java
+                        **/*TestCase.java
+                    
+                    
+                        **/performance/**
+                    
+                
+            
+
+            
+                org.apache.maven.plugins
+                maven-compiler-plugin
+                3.8.1
+                
+                    ${java.version}
+                    ${java.version}
+                
+            
+
+            
+                maven-source-plugin
+                3.1.0
+                
+                    
+                        attach-sources
+                        verify
+                        
+                            jar
+                        
+                    
+                
+            
+
+            
+                maven-release-plugin
+                2.5.3
+                
+                    deploy
+                    false
+                    true
+                
+            
+
+            
+                org.jacoco
+                jacoco-maven-plugin
+                0.8.4
+                
+                    
+                        
+                            prepare-agent
+                        
+                    
+                    
+                        report
+                        test
+                        
+                            report
+                        
+                    
+                
+            
+
+            
+                org.sonatype.plugins
+                nexus-staging-maven-plugin
+                1.6.8
+                true
+                
+                    ossrh
+                    https://oss.sonatype.org/
+                    true
+                
+            
+
+        
+    
+
+    
+        
+            release
+            
+                
+                    
+                    performRelease
+                    true
+                
+            
+            
+                
+                    
+                        org.apache.maven.plugins
+                        maven-javadoc-plugin
+                        3.0.1
+                        
+                            
+                                attach-javadocs
+                                
+                                    jar
+                                
+                                
+                                    false
+                                    -Xdoclint:none
+                                
+                            
+                        
+                    
+
+                    
+                        org.apache.maven.plugins
+                        maven-gpg-plugin
+                        1.6
+                        
+                            
+                                sign-artifacts
+                                verify
+                                
+                                    sign
+                                
+                            
+                        
+                    
+                
+            
+        
+
+    
+
+
diff --git a/settings.gradle b/settings.gradle
deleted file mode 100644
index 4191fd1..0000000
--- a/settings.gradle
+++ /dev/null
@@ -1 +0,0 @@
-include ':paldb'
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Configuration.java b/src/main/java/com/linkedin/paldb/api/Configuration.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/Configuration.java
rename to src/main/java/com/linkedin/paldb/api/Configuration.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDB.java b/src/main/java/com/linkedin/paldb/api/PalDB.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/PalDB.java
rename to src/main/java/com/linkedin/paldb/api/PalDB.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java b/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java
rename to src/main/java/com/linkedin/paldb/api/PalDBConfigBuilder.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/Serializer.java b/src/main/java/com/linkedin/paldb/api/Serializer.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/Serializer.java
rename to src/main/java/com/linkedin/paldb/api/Serializer.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java b/src/main/java/com/linkedin/paldb/api/StoreReader.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/StoreReader.java
rename to src/main/java/com/linkedin/paldb/api/StoreReader.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java b/src/main/java/com/linkedin/paldb/api/StoreWriter.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/StoreWriter.java
rename to src/main/java/com/linkedin/paldb/api/StoreWriter.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java b/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java
rename to src/main/java/com/linkedin/paldb/api/errors/DuplicateKeyException.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java b/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java
rename to src/main/java/com/linkedin/paldb/api/errors/OutOfDiskSpace.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java b/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java
rename to src/main/java/com/linkedin/paldb/api/errors/UnsupportedTypeException.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/api/package.html b/src/main/java/com/linkedin/paldb/api/package.html
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/api/package.html
rename to src/main/java/com/linkedin/paldb/api/package.html
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java b/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/ReaderImpl.java
rename to src/main/java/com/linkedin/paldb/impl/ReaderImpl.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderIterable.java b/src/main/java/com/linkedin/paldb/impl/ReaderIterable.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/ReaderIterable.java
rename to src/main/java/com/linkedin/paldb/impl/ReaderIterable.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java b/src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java
rename to src/main/java/com/linkedin/paldb/impl/ReaderKeyIterable.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java b/src/main/java/com/linkedin/paldb/impl/Serializers.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/Serializers.java
rename to src/main/java/com/linkedin/paldb/impl/Serializers.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java b/src/main/java/com/linkedin/paldb/impl/StorageReader.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/StorageReader.java
rename to src/main/java/com/linkedin/paldb/impl/StorageReader.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java b/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/StorageSerialization.java
rename to src/main/java/com/linkedin/paldb/impl/StorageSerialization.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java b/src/main/java/com/linkedin/paldb/impl/StorageWriter.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/StorageWriter.java
rename to src/main/java/com/linkedin/paldb/impl/StorageWriter.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java b/src/main/java/com/linkedin/paldb/impl/StoreImpl.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/StoreImpl.java
rename to src/main/java/com/linkedin/paldb/impl/StoreImpl.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java b/src/main/java/com/linkedin/paldb/impl/WriterImpl.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/WriterImpl.java
rename to src/main/java/com/linkedin/paldb/impl/WriterImpl.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/impl/package.html b/src/main/java/com/linkedin/paldb/impl/package.html
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/impl/package.html
rename to src/main/java/com/linkedin/paldb/impl/package.html
diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java b/src/main/java/com/linkedin/paldb/utils/BloomFilter.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/utils/BloomFilter.java
rename to src/main/java/com/linkedin/paldb/utils/BloomFilter.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java b/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/utils/DataInputOutput.java
rename to src/main/java/com/linkedin/paldb/utils/DataInputOutput.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/FormatVersion.java b/src/main/java/com/linkedin/paldb/utils/FormatVersion.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/utils/FormatVersion.java
rename to src/main/java/com/linkedin/paldb/utils/FormatVersion.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/LongPacker.java b/src/main/java/com/linkedin/paldb/utils/LongPacker.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/utils/LongPacker.java
rename to src/main/java/com/linkedin/paldb/utils/LongPacker.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/Murmur3.java b/src/main/java/com/linkedin/paldb/utils/Murmur3.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/utils/Murmur3.java
rename to src/main/java/com/linkedin/paldb/utils/Murmur3.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java b/src/main/java/com/linkedin/paldb/utils/TempUtils.java
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/utils/TempUtils.java
rename to src/main/java/com/linkedin/paldb/utils/TempUtils.java
diff --git a/paldb/src/main/java/com/linkedin/paldb/utils/package.html b/src/main/java/com/linkedin/paldb/utils/package.html
similarity index 100%
rename from paldb/src/main/java/com/linkedin/paldb/utils/package.html
rename to src/main/java/com/linkedin/paldb/utils/package.html
diff --git a/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java b/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java
similarity index 91%
rename from paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java
rename to src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java
index 8ce29f1..5cdf09c 100644
--- a/paldb/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java
+++ b/src/test/java/com/linkedin/paldb/api/PalDBConfigBuilderTest.java
@@ -1,8 +1,8 @@
 package com.linkedin.paldb.api;
 
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.Test;
 
-import static org.testng.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class PalDBConfigBuilderTest {
 
diff --git a/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java b/src/test/java/com/linkedin/paldb/api/TestConfiguration.java
similarity index 82%
rename from paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java
rename to src/test/java/com/linkedin/paldb/api/TestConfiguration.java
index 4cf8512..0988470 100644
--- a/paldb/src/test/java/com/linkedin/paldb/api/TestConfiguration.java
+++ b/src/test/java/com/linkedin/paldb/api/TestConfiguration.java
@@ -14,14 +14,15 @@
 
 package com.linkedin.paldb.api;
 
-import org.testng.annotations.Test;
+
+import org.junit.jupiter.api.Test;
 
 import java.awt.*;
 import java.io.*;
 import java.util.*;
 
 import static java.util.Collections.singletonList;
-import static org.testng.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 
 public class TestConfiguration {
@@ -46,13 +47,15 @@ public void testConfigurationCopy() {
     assertEquals(r.get("foo", null), "bar");
   }
 
-  @Test(expectedExceptions = UnsupportedOperationException.class)
+  @Test
   public void testConfigurationReadOnly() {
     Configuration c = new Configuration();
     c.set("foo", "bar");
 
-    Configuration r = new Configuration(c);
-    r.set("foo", "bar");
+    assertThrows(UnsupportedOperationException.class, () -> {
+      Configuration r = new Configuration(c);
+      r.set("foo", "bar");
+    });
   }
 
   @Test
@@ -85,9 +88,9 @@ public void testGetBoolean() {
     assertFalse(c.getBoolean("bar"));
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testGetBooleanMissing() {
-    new Configuration().getBoolean("foo");
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getBoolean("foo"));
   }
 
   @Test
@@ -107,9 +110,9 @@ public void testGetDouble() {
     assertEquals(c.getDouble("foo"), 1.0);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testGetDoubleMissing() {
-    new Configuration().getDouble("foo");
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getDouble("foo"));
   }
 
   @Test
@@ -129,9 +132,9 @@ public void testGetFloat() {
     assertEquals(c.getFloat("foo"), 1f);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testGetFloatMissing() {
-    new Configuration().getFloat("foo");
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getFloat("foo"));
   }
 
   @Test
@@ -151,9 +154,9 @@ public void testGetInt() {
     assertEquals(c.getInt("foo"), 1);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testGetIntMissing() {
-    new Configuration().getInt("foo");
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getInt("foo"));
   }
 
   @Test
@@ -173,9 +176,9 @@ public void testGetShort() {
     assertEquals(c.getShort("foo"), (short) 1);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testGetShortMissing() {
-    new Configuration().getShort("foo");
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getShort("foo"));
   }
 
   @Test
@@ -187,6 +190,11 @@ public void testGetShortDefault() {
     assertEquals(c.getShort("bar", (short) 2), (short) 2);
   }
 
+  @Test
+  public void testGetLongMissing() {
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getLong("foo"));
+  }
+
   @Test
   public void testGetLong() {
     Configuration c = new Configuration();
@@ -195,11 +203,6 @@ public void testGetLong() {
     assertEquals(c.getLong("foo"), 1L);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
-  public void testGetLongMissing() {
-    new Configuration().getLong("foo");
-  }
-
   @Test
   public void testGetLongDefault() {
     Configuration c = new Configuration();
@@ -218,10 +221,9 @@ public void testGetClass()
     assertEquals(c.getClass("foo"), Integer.class);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
-  public void testGetClassMissing()
-      throws ClassNotFoundException {
-    new Configuration().getClass("foo");
+  @Test
+  public void testGetClassMissing() {
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getClass("foo"));
   }
 
   @Test
@@ -232,9 +234,9 @@ public void testGetList() {
     assertEquals(c.getList("foo"), Arrays.asList("foo", "bar"));
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testGetListMissing() {
-    new Configuration().getList("foo");
+    assertThrows(IllegalArgumentException.class, () -> new Configuration().getList("foo"));
   }
 
   @Test
diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java b/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java
similarity index 97%
rename from paldb/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java
rename to src/test/java/com/linkedin/paldb/impl/GenerateTestData.java
index 924238a..2f1a79c 100644
--- a/paldb/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java
+++ b/src/test/java/com/linkedin/paldb/impl/GenerateTestData.java
@@ -98,7 +98,7 @@ public static Object[] generateCompoundKeys(int count) {
     Object[] res = new Object[count];
     Random random = new Random(345);
     for (int i = 0; i < count; i++) {
-      Object[] k = new Object[]{new Byte((byte) random.nextInt(10)), new Integer(i)};
+      Object[] k = new Object[]{Byte.valueOf((byte) random.nextInt(10)), Integer.valueOf(i)};
       res[i] = k;
     }
     return res;
diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java b/src/test/java/com/linkedin/paldb/impl/TestSerializers.java
similarity index 97%
rename from paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java
rename to src/test/java/com/linkedin/paldb/impl/TestSerializers.java
index 7609285..185d1f8 100644
--- a/paldb/src/test/java/com/linkedin/paldb/impl/TestSerializers.java
+++ b/src/test/java/com/linkedin/paldb/impl/TestSerializers.java
@@ -15,19 +15,19 @@
 package com.linkedin.paldb.impl;
 
 import com.linkedin.paldb.api.Serializer;
-import org.testng.annotations.*;
+import org.junit.jupiter.api.*;
 
 import java.awt.*;
 import java.io.*;
 
-import static org.testng.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 
 public class TestSerializers {
 
   private Serializers serializers;
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() {
     serializers = new Serializers();
   }
diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java b/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java
similarity index 95%
rename from paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java
rename to src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java
index 110b701..505ee11 100644
--- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java
+++ b/src/test/java/com/linkedin/paldb/impl/TestStorageSerialization.java
@@ -16,23 +16,21 @@
 
 import com.linkedin.paldb.api.*;
 import com.linkedin.paldb.api.errors.UnsupportedTypeException;
-import org.testng.Assert;
-import org.testng.annotations.*;
+import org.junit.jupiter.api.*;
 
 import java.awt.*;
 import java.io.*;
 import java.math.*;
 import java.util.Arrays;
 
-import static org.testng.Assert.*;
-
+import static org.junit.jupiter.api.Assertions.*;
 
 public class TestStorageSerialization {
 
   private Configuration configuration;
   private StorageSerialization serialization;
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() {
     configuration = new Configuration();
     serialization = new StorageSerialization(configuration);
@@ -82,9 +80,9 @@ public void testSerializeValueDataOutput() throws IOException, ClassNotFoundExce
     assertEquals(serialization.deserialize(dis), l);
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
-  public void testSerializeKeyNull() throws IOException, ClassNotFoundException {
-    serialization.serializeKey(null);
+  @Test
+  public void testSerializeKeyNull() {
+    assertThrows(NullPointerException.class, () -> serialization.serializeKey(null));
   }
 
   @Test
@@ -101,7 +99,7 @@ public void testTransformList()
     Integer[] l = new Integer[]{1, 2};
     Object deserialize = serialization.deserialize(serialization.serializeValue(l));
     assertEquals(deserialize.getClass(), int[].class);
-    assertEquals(deserialize, new int[]{1, 2});
+    assertArrayEquals((int[]) deserialize, new int[]{1, 2});
   }
 
   @Test
@@ -110,7 +108,7 @@ public void testTransformListWithNull()
     Integer[] l = new Integer[]{1, null, 2};
     Object deserialize = serialization.deserialize(serialization.serializeValue(l));
     assertEquals(deserialize.getClass(), int[].class);
-    assertEquals(deserialize, new int[]{1, 0, 2});
+    assertArrayEquals((int[])deserialize, new int[]{1, 0, 2});
   }
 
   @Test
@@ -119,7 +117,7 @@ public void testTransformListOfList()
     Integer[][] l = new Integer[][]{{1}, {2}};
     Object deserialize = serialization.deserialize(serialization.serializeValue(l));
     assertEquals(deserialize.getClass(), int[][].class);
-    assertEquals(deserialize, new int[][]{{1}, {2}});
+    assertArrayEquals((int[][])deserialize, new int[][]{{1}, {2}});
   }
 
   @Test
@@ -175,7 +173,7 @@ public Class serializedClass() {
     });
     Point[] p = new Point[]{new Point(42, 9)};
     byte[] buf = serialization.serialize(p);
-    assertEquals(serialization.deserialize(buf), p);
+    assertArrayEquals((Point[])serialization.deserialize(buf), p);
   }
 
   @Test
@@ -203,7 +201,7 @@ public void write(DataOutput dataOutput, ImplementsA input) throws IOException {
     assertEquals(serialization.deserialize(buf), a);
   }
 
-  @Test(expectedExceptions = UnsupportedTypeException.class)
+  @Test
   public void testInheritedSerializer() throws Throwable {
     configuration.registerSerializer(new Serializer() {
 
@@ -223,16 +221,17 @@ public void write(DataOutput dataOutput, A input) throws IOException {
       }
 
     });
-    ImplementsA a = new ImplementsA(42);
-    byte[] buf = serialization.serialize(a);
-    assertEquals(serialization.deserialize(buf), a);
+    assertThrows(UnsupportedTypeException.class, () -> {
+      ImplementsA a = new ImplementsA(42);
+      byte[] buf = serialization.serialize(a);
+      assertEquals(serialization.deserialize(buf), a);
+    });
   }
 
   @Test
-  public void testNull()
-      throws Throwable {
+  public void testNull() throws Throwable {
     byte[] buf = serialization.serialize(null);
-    Assert.assertNull(serialization.deserialize(buf));
+    assertNull(serialization.deserialize(buf));
   }
 
   @Test
@@ -247,10 +246,9 @@ public void testByte()
     }
   }
 
-  @Test(expectedExceptions = UnsupportedTypeException.class)
-  public void testNotSupported()
-      throws Throwable {
-    serialization.serialize(new Color(0, 0, 0));
+  @Test
+  public void testNotSupported() {
+    assertThrows(UnsupportedTypeException.class, () -> serialization.serialize(new Color(0, 0, 0)));
   }
 
   @Test
@@ -568,7 +566,7 @@ public void testMultiDimensionalIntArray()
     d[1] = new int[]{-3, 1};
     Object res = serialization.deserialize(serialization.serialize(d));
     assertEquals(res.getClass(), int[][].class);
-    assertEquals(d, res);
+    assertArrayEquals(d, (int[][])res);
   }
 
   @Test
@@ -579,7 +577,7 @@ public void testMultiDimensionalLongArray()
     d[1] = new long[]{-3, 1};
     Object res = serialization.deserialize(serialization.serialize(d));
     assertEquals(res.getClass(), long[][].class);
-    assertEquals(d, res);
+    assertArrayEquals(d, (long[][])res);
   }
 
   // UTILITY
diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java b/src/test/java/com/linkedin/paldb/impl/TestStore.java
similarity index 89%
rename from paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java
rename to src/test/java/com/linkedin/paldb/impl/TestStore.java
index 0b839c6..8e900bd 100644
--- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStore.java
+++ b/src/test/java/com/linkedin/paldb/impl/TestStore.java
@@ -15,29 +15,29 @@
 package com.linkedin.paldb.impl;
 
 import com.linkedin.paldb.api.*;
+import com.linkedin.paldb.api.errors.DuplicateKeyException;
 import com.linkedin.paldb.utils.*;
-import org.testng.Assert;
-import org.testng.annotations.*;
+import org.junit.jupiter.api.*;
 
 import java.io.*;
 import java.nio.file.*;
 import java.util.*;
 
 import static com.linkedin.paldb.utils.TestTempUtils.deleteDirectory;
-import static org.testng.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class TestStore {
 
   private Path tempDir;
   private File storeFile;
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() throws IOException {
     tempDir = Files.createTempDirectory("tmp");
     storeFile = Files.createTempFile(tempDir, "paldb", ".dat").toFile();
   }
 
-  @AfterMethod
+  @AfterEach
   public void cleanUp() {
     deleteDirectory(tempDir.toFile());
   }
@@ -96,70 +96,71 @@ public void testNoFolder() {
     assertTrue(file.exists());
   }
 
-  @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*not found.*")
+  @Test
   public void testReaderFileNotFound() {
-    PalDB.createReader(new File("notfound"), PalDB.newConfiguration());
+    assertThrows(RuntimeException.class, () -> PalDB.createReader(new File("notfound"), PalDB.newConfiguration()));
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testReaderNullFile() {
-    PalDB.createReader((File) null, PalDB.newConfiguration());
+    assertThrows(NullPointerException.class, () -> PalDB.createReader((File) null, PalDB.newConfiguration()));
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testReaderNullConfig() {
-    PalDB.createReader(new File("notfound"), null);
+    assertThrows(NullPointerException .class, () -> PalDB.createReader(new File("notfound"), null));
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testReaderNullStream() {
-    PalDB.createReader((InputStream) null, PalDB.newConfiguration());
+    assertThrows(NullPointerException.class, () -> PalDB.createReader((InputStream) null, PalDB.newConfiguration()));
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testReaderNullConfigForStream() {
-    PalDB.createReader(new InputStream() {
-      @Override
-      public int read()
-              throws IOException {
-        return 0;
-      }
-    }, null);
+    assertThrows(NullPointerException.class, () -> {
+      PalDB.createReader(new InputStream() {
+        @Override
+        public int read() {
+          return 0;
+        }
+      }, null);
+    });
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testWriterNullFile() {
-    PalDB.createWriter((File) null, PalDB.newConfiguration());
+    assertThrows(NullPointerException.class, () -> PalDB.createWriter((File) null, PalDB.newConfiguration()));
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testWriterNullConfig() {
-    PalDB.createWriter(new File("notfound"), null);
+    assertThrows(NullPointerException.class, () -> PalDB.createWriter(new File("notfound"), null));
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testWriterNullStream() {
-    PalDB.createWriter((OutputStream) null, PalDB.newConfiguration());
+    assertThrows(NullPointerException.class, () -> PalDB.createWriter((OutputStream) null, PalDB.newConfiguration()));
   }
 
-  @Test(expectedExceptions = NullPointerException.class)
+  @Test
   public void testWriterNullConfigForStream() {
-    PalDB.createWriter(new OutputStream() {
+    assertThrows(NullPointerException.class, () -> PalDB.createWriter(new OutputStream() {
       @Override
       public void write(int i) {
 
       }
-    }, null);
+    }, null));
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testInvalidSegmentSize() {
     StoreWriter writer = PalDB.createWriter(storeFile);
     writer.close();
 
     Configuration config = new Configuration();
     config.set(Configuration.MMAP_SEGMENT_SIZE, String.valueOf(1 + (long) Integer.MAX_VALUE));
-    PalDB.createReader(storeFile, config);
+    assertThrows(IllegalArgumentException.class, () -> PalDB.createReader(storeFile, config));
   }
 
   @Test
@@ -298,12 +299,12 @@ public void testKeyLengthStartTwo() {
     }
   }
 
-  @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*duplicate.*")
+  @Test
   public void testDuplicateKeys() {
-    try (StoreWriter writer = PalDB.createWriter(storeFile, new Configuration())) {
-      writer.put(0, "ABC");
-      writer.put(0, "DGE");
-    }
+    StoreWriter writer = PalDB.createWriter(storeFile, new Configuration());
+    writer.put(0, "ABC");
+    writer.put(0, "DGE");
+    assertThrows(DuplicateKeyException.class, writer::close);
   }
 
   @Test
@@ -460,7 +461,7 @@ public void testIterate() {
         assertNotNull(valSearch);
         assertEquals(valSearch, entry.getValue());
       }
-      Assert.assertFalse(itr.hasNext());
+      assertFalse(itr.hasNext());
     }
 
     assertTrue(keysSet.isEmpty());
@@ -544,7 +545,7 @@ private  void testReadKeyToIntArray(K[] keys) {
         K key = keys[i];
         int[] val = reader.get(key, null);
         assertNotNull(val);
-        assertEquals(val, values[i]);
+        assertArrayEquals(val, values[i]);
       }
     }
   }
diff --git a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java b/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java
similarity index 82%
rename from paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java
rename to src/test/java/com/linkedin/paldb/impl/TestStoreReader.java
index 5f36c89..363a5c4 100644
--- a/paldb/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java
+++ b/src/test/java/com/linkedin/paldb/impl/TestStoreReader.java
@@ -15,8 +15,7 @@
 package com.linkedin.paldb.impl;
 
 import com.linkedin.paldb.api.*;
-import org.testng.Assert;
-import org.testng.annotations.*;
+import org.junit.jupiter.api.*;
 
 import java.awt.*;
 import java.io.*;
@@ -27,20 +26,20 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static com.linkedin.paldb.utils.TestTempUtils.deleteDirectory;
-import static org.testng.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class TestStoreReader {
 
   private Path tempDir;
   private File storeFile;
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() throws IOException {
     tempDir = Files.createTempDirectory("tmp");
     storeFile = Files.createTempFile(tempDir, "paldb", ".dat").toFile();
   }
 
-  @AfterMethod
+  @AfterEach
   public void cleanUp() {
     deleteDirectory(tempDir.toFile());
   }
@@ -75,11 +74,11 @@ public void testSize() {
     }
   }
 
-  @Test(expectedExceptions = IllegalStateException.class)
+  @Test
   public void testStoreClosed() {
     var reader = readerFor(true);
     reader.close();
-    reader.get(0);
+    assertThrows(IllegalStateException.class, () -> reader.get(0));
   }
 
   @Test
@@ -229,9 +228,9 @@ public void testGetStringMissing() {
   @Test
   public void testGetBooleanArray() {
     try (var reader = readerFor(new boolean[]{true})) {
-      assertEquals(reader.get(0), new boolean[]{true});
-      assertEquals(reader.get(0, new boolean[]{false}), new boolean[]{true});
-      assertEquals(reader.get(-1, new boolean[]{false}), new boolean[]{false});
+      assertArrayEquals(reader.get(0), new boolean[]{true});
+      assertArrayEquals(reader.get(0, new boolean[]{false}), new boolean[]{true});
+      assertArrayEquals(reader.get(-1, new boolean[]{false}), new boolean[]{false});
     }
   }
 
@@ -245,9 +244,9 @@ public void testGetBooleanArrayMissing() {
   @Test
   public void testGetByteArray() {
     try (var reader = readerFor(new byte[]{1})) {
-      assertEquals(reader.get(0), new byte[]{1});
-      assertEquals(reader.get(0, new byte[]{2}), new byte[]{1});
-      assertEquals(reader.get(-1, new byte[]{2}), new byte[]{2});
+      assertArrayEquals(reader.get(0), new byte[]{1});
+      assertArrayEquals(reader.get(0, new byte[]{2}), new byte[]{1});
+      assertArrayEquals(reader.get(-1, new byte[]{2}), new byte[]{2});
     }
   }
 
@@ -261,9 +260,9 @@ public void testGetByteArrayMissing() {
   @Test
   public void testGetCharArray() {
     try (var reader = readerFor(new char[]{'a'})) {
-      assertEquals(reader.get(0), new char[]{'a'});
-      assertEquals(reader.get(0, new char[]{'b'}), new char[]{'a'});
-      assertEquals(reader.get(-1, new char[]{'b'}), new char[]{'b'});
+      assertArrayEquals(reader.get(0), new char[]{'a'});
+      assertArrayEquals(reader.get(0, new char[]{'b'}), new char[]{'a'});
+      assertArrayEquals(reader.get(-1, new char[]{'b'}), new char[]{'b'});
     }
   }
 
@@ -277,9 +276,9 @@ public void testGetCharArrayMissing() {
   @Test
   public void testGetDoubleArray() {
     try (var reader = readerFor(new double[]{1.0})) {
-      assertEquals(reader.get(0), new double[]{1.0});
-      assertEquals(reader.get(0, new double[]{2.0}), new double[]{1.0});
-      assertEquals(reader.get(-1, new double[]{2.0}), new double[]{2.0});
+      assertArrayEquals(reader.get(0), new double[]{1.0});
+      assertArrayEquals(reader.get(0, new double[]{2.0}), new double[]{1.0});
+      assertArrayEquals(reader.get(-1, new double[]{2.0}), new double[]{2.0});
     }
   }
 
@@ -293,9 +292,9 @@ public void testGetDoubleArrayMissing() {
   @Test
   public void testGetFloatArray() {
     try (var reader = readerFor(new float[]{1f})) {
-      assertEquals(reader.get(0), new float[]{1f});
-      assertEquals(reader.get(0, new float[]{2f}), new float[]{1f});
-      assertEquals(reader.get(-1, new float[]{2f}), new float[]{2f});
+      assertArrayEquals(reader.get(0), new float[]{1f});
+      assertArrayEquals(reader.get(0, new float[]{2f}), new float[]{1f});
+      assertArrayEquals(reader.get(-1, new float[]{2f}), new float[]{2f});
     }
   }
 
@@ -309,9 +308,9 @@ public void testGetFloatArrayMissing() {
   @Test
   public void testGetShortArray() {
     try (var reader = readerFor(new short[]{1})) {
-      assertEquals(reader.get(0), new short[]{1});
-      assertEquals(reader.get(0, new short[]{2}), new short[]{1});
-      assertEquals(reader.get(-1, new short[]{2}), new short[]{2});
+      assertArrayEquals(reader.get(0), new short[]{1});
+      assertArrayEquals(reader.get(0, new short[]{2}), new short[]{1});
+      assertArrayEquals(reader.get(-1, new short[]{2}), new short[]{2});
     }
   }
 
@@ -325,9 +324,9 @@ public void testGetShortArrayMissing() {
   @Test
   public void testGetIntArray() {
     try (var reader = readerFor(new int[]{1})) {
-      assertEquals(reader.get(0), new int[]{1});
-      assertEquals(reader.get(0, new int[]{2}), new int[]{1});
-      assertEquals(reader.get(-1, new int[]{2}), new int[]{2});
+      assertArrayEquals(reader.get(0), new int[]{1});
+      assertArrayEquals(reader.get(0, new int[]{2}), new int[]{1});
+      assertArrayEquals(reader.get(-1, new int[]{2}), new int[]{2});
     }
   }
 
@@ -341,9 +340,9 @@ public void testGetIntArrayMissing() {
   @Test
   public void testGetLongArray() {
     try (var reader = readerFor(new long[]{1L})) {
-      assertEquals(reader.get(0), new long[]{1L});
-      assertEquals(reader.get(0, new long[]{2L}), new long[]{1L});
-      assertEquals(reader.get(-1, new long[]{2L}), new long[]{2L});
+      assertArrayEquals(reader.get(0), new long[]{1L});
+      assertArrayEquals(reader.get(0, new long[]{2L}), new long[]{1L});
+      assertArrayEquals(reader.get(-1, new long[]{2L}), new long[]{2L});
     }
   }
 
@@ -357,9 +356,9 @@ public void testGetLongArrayMissing() {
   @Test
   public void testGetStringArray() {
     try (var reader = readerFor(new String[]{"foo"})) {
-      assertEquals(reader.get(0), new String[]{"foo"});
-      assertEquals(reader.get(0, new String[]{"bar"}), new String[]{"foo"});
-      assertEquals(reader.get(-1, new String[]{"bar"}), new String[]{"bar"});
+      assertArrayEquals(reader.get(0), new String[]{"foo"});
+      assertArrayEquals(reader.get(0, new String[]{"bar"}), new String[]{"foo"});
+      assertArrayEquals(reader.get(-1, new String[]{"bar"}), new String[]{"bar"});
     }
   }
 
@@ -380,9 +379,9 @@ public void testGetMissing() {
   @Test
   public void testGetArray() {
     try (var reader = readerFor(new Object[]{"foo"})) {
-      assertEquals(reader.get(0), new Object[]{"foo"});
-      assertEquals(reader.get(0, new Object[]{"bar"}), new Object[]{"foo"});
-      assertEquals(reader.get(-1, new Object[]{"bar"}), new Object[]{"bar"});
+      assertArrayEquals(reader.get(0), new Object[]{"foo"});
+      assertArrayEquals(reader.get(0, new Object[]{"bar"}), new Object[]{"foo"});
+      assertArrayEquals(reader.get(-1, new Object[]{"bar"}), new Object[]{"bar"});
     }
   }
 
@@ -405,9 +404,9 @@ public void testIterator() {
     var values = List.of("foo", "bar");
     try (var reader = readerForMany(values.get(0), values.get(1))) {
       var iter = reader.iterable();
-      Assert.assertNotNull(iter);
+      assertNotNull(iter);
       var itr = iter.iterator();
-      Assert.assertNotNull(itr);
+      assertNotNull(itr);
 
       for (int i = 0; i < values.size(); i++) {
         assertTrue(itr.hasNext());
@@ -433,9 +432,9 @@ public void testKeyIterator() {
     var values = List.of("foo", "bar");
     try (var reader = readerForMany(values.get(0), values.get(1))) {
       var iter = reader.keys();
-      Assert.assertNotNull(iter);
+      assertNotNull(iter);
       var itr = iter.iterator();
-      Assert.assertNotNull(itr);
+      assertNotNull(itr);
 
       Set actual = new HashSet<>();
       Set expected = new HashSet<>();
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestMemoryUsageHashMap.java b/src/test/java/com/linkedin/paldb/performance/TestMemoryUsageHashMap.java
similarity index 88%
rename from paldb/src/perfTest/java/com/linkedin/paldb/TestMemoryUsageHashMap.java
rename to src/test/java/com/linkedin/paldb/performance/TestMemoryUsageHashMap.java
index 7b5750f..cc9ba3e 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/TestMemoryUsageHashMap.java
+++ b/src/test/java/com/linkedin/paldb/performance/TestMemoryUsageHashMap.java
@@ -12,15 +12,15 @@
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
 
-package com.linkedin.paldb;
+package com.linkedin.paldb.performance;
 
-import com.linkedin.paldb.utils.NanoBench;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-import org.testng.annotations.Test;
+import com.linkedin.paldb.performance.utils.NanoBench;
+import org.junit.jupiter.api.*;
 
+import java.util.*;
 
+@Disabled
+@Tag("performance")
 public class TestMemoryUsageHashMap {
 
   private Set ref;
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java b/src/test/java/com/linkedin/paldb/performance/TestReadThroughput.java
similarity index 96%
rename from paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java
rename to src/test/java/com/linkedin/paldb/performance/TestReadThroughput.java
index daacceb..8610fb8 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughput.java
+++ b/src/test/java/com/linkedin/paldb/performance/TestReadThroughput.java
@@ -12,34 +12,35 @@
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
 
-package com.linkedin.paldb;
+package com.linkedin.paldb.performance;
 
 import com.linkedin.paldb.api.*;
 import com.linkedin.paldb.impl.GenerateTestData;
-import com.linkedin.paldb.utils.*;
+import com.linkedin.paldb.performance.utils.*;
 import org.apache.commons.lang.RandomStringUtils;
-import org.testng.annotations.*;
+import org.junit.jupiter.api.*;
 
 import java.io.*;
 import java.nio.file.Files;
 import java.util.*;
-import java.util.concurrent.*;
+import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.IntStream;
 
-
+@Disabled
+@Tag("performance")
 public class TestReadThroughput {
 
   private File testFolder = createTempDir();
   private static final int READS = 500000;
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() {
     DirectoryUtils.deleteDirectory(testFolder);
     testFolder.mkdir();
   }
 
-  @AfterMethod
+  @AfterEach
   public void cleanUp() {
     DirectoryUtils.deleteDirectory(testFolder);
   }
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughputLevelDB.java b/src/test/java/com/linkedin/paldb/performance/TestReadThroughputLevelDB.java
similarity index 87%
rename from paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughputLevelDB.java
rename to src/test/java/com/linkedin/paldb/performance/TestReadThroughputLevelDB.java
index 22265ff..3c0dda8 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThroughputLevelDB.java
+++ b/src/test/java/com/linkedin/paldb/performance/TestReadThroughputLevelDB.java
@@ -12,40 +12,33 @@
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
 
-package com.linkedin.paldb;
+package com.linkedin.paldb.performance;
 
 import com.linkedin.paldb.impl.GenerateTestData;
-import com.linkedin.paldb.utils.DirectoryUtils;
-import com.linkedin.paldb.utils.NanoBench;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
+import com.linkedin.paldb.performance.utils.*;
 import org.apache.commons.lang.RandomStringUtils;
-import org.iq80.leveldb.CompressionType;
-import org.iq80.leveldb.DB;
-import org.iq80.leveldb.Options;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
+import org.iq80.leveldb.*;
+import org.junit.jupiter.api.*;
 
-import static org.fusesource.leveldbjni.JniDBFactory.bytes;
-import static org.fusesource.leveldbjni.JniDBFactory.factory;
+import java.io.*;
+import java.util.*;
 
+import static org.fusesource.leveldbjni.JniDBFactory.*;
 
+@Disabled
+@Tag("performance")
 public class TestReadThroughputLevelDB {
 
   private File TEST_FOLDER = new File("testreadthroughputleveldb");
   private final int READS = 500000;
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() {
     DirectoryUtils.deleteDirectory(TEST_FOLDER);
     TEST_FOLDER.mkdir();
   }
 
-  @AfterMethod
+  @AfterEach
   public void cleanUp() {
     DirectoryUtils.deleteDirectory(TEST_FOLDER);
   }
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThrouputHashMap.java b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputHashMap.java
similarity index 89%
rename from paldb/src/perfTest/java/com/linkedin/paldb/TestReadThrouputHashMap.java
rename to src/test/java/com/linkedin/paldb/performance/TestReadThrouputHashMap.java
index b9f0e2b..615b10b 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThrouputHashMap.java
+++ b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputHashMap.java
@@ -12,15 +12,15 @@
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
 
-package com.linkedin.paldb;
+package com.linkedin.paldb.performance;
 
-import com.linkedin.paldb.utils.NanoBench;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-import org.testng.annotations.Test;
+import com.linkedin.paldb.performance.utils.NanoBench;
+import org.junit.jupiter.api.*;
 
+import java.util.*;
 
+@Disabled
+@Tag("performance")
 public class TestReadThrouputHashMap {
 
   private final int READS = 500000;
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThrouputRocksDB.java b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputRocksDB.java
similarity index 88%
rename from paldb/src/perfTest/java/com/linkedin/paldb/TestReadThrouputRocksDB.java
rename to src/test/java/com/linkedin/paldb/performance/TestReadThrouputRocksDB.java
index fe9c2fc..0f01ca1 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/TestReadThrouputRocksDB.java
+++ b/src/test/java/com/linkedin/paldb/performance/TestReadThrouputRocksDB.java
@@ -12,47 +12,36 @@
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
 
-package com.linkedin.paldb;
+package com.linkedin.paldb.performance;
 
 import com.linkedin.paldb.impl.GenerateTestData;
-import com.linkedin.paldb.utils.DirectoryUtils;
-import com.linkedin.paldb.utils.NanoBench;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
+import com.linkedin.paldb.performance.utils.*;
 import org.apache.commons.lang.RandomStringUtils;
-import org.rocksdb.CompactionStyle;
-import org.rocksdb.CompressionType;
-import org.rocksdb.Options;
-import org.rocksdb.ReadOptions;
-import org.rocksdb.RocksDB;
-import org.rocksdb.RocksDBException;
-import org.rocksdb.BlockBasedTableConfig;
-import org.rocksdb.BloomFilter;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.*;
+import org.rocksdb.*;
 
+import java.io.File;
+import java.util.*;
 
+@Disabled
+@Tag("performance")
 public class TestReadThrouputRocksDB {
 
   private File TEST_FOLDER = new File("testreadthroughputrocksdb");
   private final int READS = 500000;
 
-  @BeforeClass
+  @BeforeEach
   public void loadLibrary() {
     RocksDB.loadLibrary();
   }
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() {
     DirectoryUtils.deleteDirectory(TEST_FOLDER);
     TEST_FOLDER.mkdir();
   }
 
-  @AfterMethod
+  @AfterEach
   public void cleanUp() {
     DirectoryUtils.deleteDirectory(TEST_FOLDER);
   }
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/TestStoreSize.java b/src/test/java/com/linkedin/paldb/performance/TestStoreSize.java
similarity index 81%
rename from paldb/src/perfTest/java/com/linkedin/paldb/TestStoreSize.java
rename to src/test/java/com/linkedin/paldb/performance/TestStoreSize.java
index 38d559f..d2023d3 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/TestStoreSize.java
+++ b/src/test/java/com/linkedin/paldb/performance/TestStoreSize.java
@@ -12,30 +12,27 @@
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
 
-package com.linkedin.paldb;
+package com.linkedin.paldb.performance;
 
-import com.linkedin.paldb.api.Configuration;
-import com.linkedin.paldb.api.PalDB;
-import com.linkedin.paldb.api.StoreWriter;
+import com.linkedin.paldb.api.*;
 import com.linkedin.paldb.impl.GenerateTestData;
-import com.linkedin.paldb.utils.DirectoryUtils;
-import java.io.File;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
+import com.linkedin.paldb.performance.utils.DirectoryUtils;
+import org.junit.jupiter.api.*;
 
+import java.io.File;
+@Disabled
+@Tag("performance")
 public class TestStoreSize {
 
   private File TEST_FOLDER = new File("teststoresize");
 
-  @BeforeMethod
+  @BeforeEach
   public void setUp() {
     DirectoryUtils.deleteDirectory(TEST_FOLDER);
     TEST_FOLDER.mkdir();
   }
 
-  @AfterMethod
+  @AfterEach
   public void cleanUp() {
     DirectoryUtils.deleteDirectory(TEST_FOLDER);
   }
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/utils/DirectoryUtils.java b/src/test/java/com/linkedin/paldb/performance/utils/DirectoryUtils.java
similarity index 94%
rename from paldb/src/perfTest/java/com/linkedin/paldb/utils/DirectoryUtils.java
rename to src/test/java/com/linkedin/paldb/performance/utils/DirectoryUtils.java
index 7617c52..0b68317 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/utils/DirectoryUtils.java
+++ b/src/test/java/com/linkedin/paldb/performance/utils/DirectoryUtils.java
@@ -1,4 +1,4 @@
-package com.linkedin.paldb.utils;
+package com.linkedin.paldb.performance.utils;
 
 import java.io.File;
 
diff --git a/paldb/src/perfTest/java/com/linkedin/paldb/utils/NanoBench.java b/src/test/java/com/linkedin/paldb/performance/utils/NanoBench.java
similarity index 99%
rename from paldb/src/perfTest/java/com/linkedin/paldb/utils/NanoBench.java
rename to src/test/java/com/linkedin/paldb/performance/utils/NanoBench.java
index 5b7eaba..43b5637 100644
--- a/paldb/src/perfTest/java/com/linkedin/paldb/utils/NanoBench.java
+++ b/src/test/java/com/linkedin/paldb/performance/utils/NanoBench.java
@@ -1,4 +1,4 @@
-package com.linkedin.paldb.utils;
+package com.linkedin.paldb.performance.utils;
 
 import java.lang.management.ManagementFactory;
 import java.text.DecimalFormat;
diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java
similarity index 98%
rename from paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java
rename to src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java
index d3bfe80..b35b771 100644
--- a/paldb/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java
+++ b/src/test/java/com/linkedin/paldb/utils/BloomFilterTest.java
@@ -1,13 +1,15 @@
 package com.linkedin.paldb.utils;
 
-import org.testng.annotations.*;
+
+import org.junit.jupiter.api.*;
 
 import java.lang.management.*;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.*;
 
-import static org.testng.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
+
 
 public class BloomFilterTest {
 
@@ -17,7 +19,7 @@ public class BloomFilterTest {
     private Random prng;
     private ThreadMXBean bean;
 
-    @BeforeMethod
+    @BeforeEach
     void setUp() {
         bean = ManagementFactory.getThreadMXBean();
         prng = new Random();
diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java b/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java
similarity index 62%
rename from paldb/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java
rename to src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java
index 50e1ce3..2742231 100644
--- a/paldb/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java
+++ b/src/test/java/com/linkedin/paldb/utils/TestFormatVersion.java
@@ -14,29 +14,29 @@
 
 package com.linkedin.paldb.utils;
 
-import org.testng.Assert;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.Test;
 
+import static org.junit.jupiter.api.Assertions.*;
 
 public class TestFormatVersion {
 
   @Test
   public void testIs() {
-    Assert.assertTrue(FormatVersion.PALDB_V1.is(FormatVersion.PALDB_V1));
+    assertTrue(FormatVersion.PALDB_V1.is(FormatVersion.PALDB_V1));
   }
 
   @Test
   public void testBytes() {
-    Assert.assertEquals(FormatVersion.fromBytes(FormatVersion.PALDB_V1.getBytes()), FormatVersion.PALDB_V1);
+    assertEquals(FormatVersion.fromBytes(FormatVersion.PALDB_V1.getBytes()), FormatVersion.PALDB_V1);
   }
 
   @Test
   public void testPrefixBytes() {
-    Assert.assertEquals(FormatVersion.getPrefixBytes(), "PALDB".getBytes());
+    assertArrayEquals(FormatVersion.getPrefixBytes(), "PALDB".getBytes());
   }
 
   @Test
   public void testGetLastVersion() {
-    Assert.assertEquals(FormatVersion.getLatestVersion(), FormatVersion.values()[FormatVersion.values().length - 1]);
+    assertEquals(FormatVersion.getLatestVersion(), FormatVersion.values()[FormatVersion.values().length - 1]);
   }
 }
diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java b/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java
similarity index 64%
rename from paldb/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java
rename to src/test/java/com/linkedin/paldb/utils/TestLongPacker.java
index bb527b8..34b42db 100644
--- a/paldb/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java
+++ b/src/test/java/com/linkedin/paldb/utils/TestLongPacker.java
@@ -14,10 +14,12 @@
 
 package com.linkedin.paldb.utils;
 
+import org.junit.jupiter.api.Test;
+
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import org.testng.Assert;
-import org.testng.annotations.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
 
 
 public class TestLongPacker {
@@ -27,7 +29,7 @@ public void testPackInt()
       throws IOException {
     DataInputOutput dio = new DataInputOutput();
     LongPacker.packInt(dio.reset(), 42);
-    Assert.assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), 42);
+    assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), 42);
   }
 
   @Test
@@ -35,7 +37,7 @@ public void testPackIntZero()
       throws IOException {
     DataInputOutput dio = new DataInputOutput();
     LongPacker.packInt(dio.reset(), 0);
-    Assert.assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), 0);
+    assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), 0);
   }
 
   @Test
@@ -43,14 +45,13 @@ public void testPackIntMax()
       throws IOException {
     DataInputOutput dio = new DataInputOutput();
     LongPacker.packInt(dio.reset(), Integer.MAX_VALUE);
-    Assert.assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), Integer.MAX_VALUE);
+    assertEquals(LongPacker.unpackInt(dio.reset(dio.toByteArray())), Integer.MAX_VALUE);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
-  public void testPackIntNeg()
-      throws IOException {
+  @Test
+  public void testPackIntNeg() {
     DataInputOutput dio = new DataInputOutput();
-    LongPacker.packInt(dio.reset(), -42);
+    assertThrows(IllegalArgumentException.class, () -> LongPacker.packInt(dio.reset(), -42));
   }
 
   @Test
@@ -58,7 +59,7 @@ public void testPackLong()
       throws IOException {
     DataInputOutput dio = new DataInputOutput();
     LongPacker.packLong(dio.reset(), 42l);
-    Assert.assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), 42);
+    assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), 42);
   }
 
   @Test
@@ -66,7 +67,7 @@ public void testPackLongZero()
       throws IOException {
     DataInputOutput dio = new DataInputOutput();
     LongPacker.packLong(dio.reset(), 0l);
-    Assert.assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), 0l);
+    assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), 0l);
   }
 
   @Test
@@ -74,7 +75,7 @@ public void testPackLongBytes()
       throws IOException {
     byte[] buf = new byte[15];
     LongPacker.packLong(buf, 42l);
-    Assert.assertEquals(LongPacker.unpackLong(buf), 42l);
+    assertEquals(LongPacker.unpackLong(buf), 42l);
   }
 
   @Test
@@ -82,7 +83,7 @@ public void testPackLongMax()
       throws IOException {
     DataInputOutput dio = new DataInputOutput();
     LongPacker.packLong(dio.reset(), Long.MAX_VALUE);
-    Assert.assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), Long.MAX_VALUE);
+    assertEquals(LongPacker.unpackLong(dio.reset(dio.toByteArray())), Long.MAX_VALUE);
   }
 
   @Test
@@ -90,28 +91,26 @@ public void testPackLongBytesMax()
       throws IOException {
     byte[] buf = new byte[15];
     LongPacker.packLong(buf, Long.MAX_VALUE);
-    Assert.assertEquals(LongPacker.unpackLong(buf), Long.MAX_VALUE);
+    assertEquals(LongPacker.unpackLong(buf), Long.MAX_VALUE);
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
+  @Test
   public void testPackLongNeg()
       throws IOException {
     DataInputOutput dio = new DataInputOutput();
-    LongPacker.packLong(dio.reset(), -42l);
+    assertThrows(IllegalArgumentException.class, () -> LongPacker.packLong(dio.reset(), -42L));
   }
 
-  @Test(expectedExceptions = IllegalArgumentException.class)
-  public void testPackLongBytesNeg()
-      throws IOException {
-    LongPacker.packLong(new byte[15], -42l);
+  @Test
+  public void testPackLongBytesNeg() {
+    assertThrows(IllegalArgumentException.class, () -> LongPacker.packLong(new byte[15], -42L));
   }
 
   @Test
-  public void test()
-      throws IOException {
+  public void test() throws IOException {
     DataInputOutput dio = new DataInputOutput();
     LongPacker.packInt(dio.reset(), 5);
     ByteBuffer bb = ByteBuffer.wrap(dio.getBuf());
-    Assert.assertEquals(LongPacker.unpackInt(bb), 5);
+    assertEquals(LongPacker.unpackInt(bb), 5);
   }
 }
diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java b/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java
similarity index 92%
rename from paldb/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java
rename to src/test/java/com/linkedin/paldb/utils/TestMurmur3.java
index c9d7588..25faa75 100644
--- a/paldb/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java
+++ b/src/test/java/com/linkedin/paldb/utils/TestMurmur3.java
@@ -15,10 +15,9 @@
 package com.linkedin.paldb.utils;
 
 import org.apache.commons.codec.digest.MurmurHash3;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.*;
+import org.junit.jupiter.api.Test;
 
+import static org.junit.jupiter.api.Assertions.*;
 
 public class TestMurmur3 {
 
diff --git a/paldb/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java b/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java
similarity index 78%
rename from paldb/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java
rename to src/test/java/com/linkedin/paldb/utils/TestTempUtils.java
index 7dc2909..53480cd 100644
--- a/paldb/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java
+++ b/src/test/java/com/linkedin/paldb/utils/TestTempUtils.java
@@ -14,11 +14,12 @@
 
 package com.linkedin.paldb.utils;
 
-import org.testng.Assert;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.Test;
 
 import java.io.*;
 
+import static org.junit.jupiter.api.Assertions.*;
+
 
 public class TestTempUtils {
 
@@ -37,9 +38,9 @@ public static boolean deleteDirectory(File directoryToBeDeleted) {
   @Test
   public void testTempDir() {
     File file = TempUtils.createTempDir("foo");
-    Assert.assertTrue(file.exists());
-    Assert.assertTrue(file.isDirectory());
-    Assert.assertTrue(file.getName().contains("foo"));
+    assertTrue(file.exists());
+    assertTrue(file.isDirectory());
+    assertTrue(file.getName().contains("foo"));
     file.delete();
   }
 
@@ -48,10 +49,10 @@ public void testCopyIntoTempFile()
       throws IOException {
     ByteArrayInputStream bis = new ByteArrayInputStream("foo".getBytes());
     File file = TempUtils.copyIntoTempFile("bar", bis);
-    Assert.assertTrue(file.exists());
-    Assert.assertTrue(file.isFile());
-    Assert.assertTrue(file.getName().contains("bar"));
-    Assert.assertEquals(bis.available(), 0);
+    assertTrue(file.exists());
+    assertTrue(file.isFile());
+    assertTrue(file.getName().contains("bar"));
+    assertEquals(bis.available(), 0);
 
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     FileInputStream fis = new FileInputStream(file);
@@ -62,7 +63,7 @@ public void testCopyIntoTempFile()
     }
     fis.close();
     bos.close();
-    Assert.assertEquals(bos.toByteArray(), "foo".getBytes());
+    assertArrayEquals(bos.toByteArray(), "foo".getBytes());
   }
 }
 
diff --git a/paldb/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml
similarity index 100%
rename from paldb/src/test/resources/logback-test.xml
rename to src/test/resources/logback-test.xml