feat: improve BinaryWriter API

- extend default buffer size - allow HiveCipher to asynchronously perform crypto - implement HiveAesThreadedCipher for Hive Flutter - fix missing file in .gitignore In theory, no public API should be made incompatible here as HIveCipher was simply converted from `T Function()` to `FutureOr<T> Function()`. The idea behind the asynchronous operation is a) multithreading using e.g. the `compute()` function in Flutter or b) platform-native implementations making use of hardware-accelerated cryptographic implementations. Signed-off-by: TheOneWithTheBraid <[email protected]>
isar · Jun 10, 2022 · aa0ef45 · aa0ef45
1 parent 59a3b16
commit aa0ef45
Show file tree

Hide file tree

Showing 35 changed files with 623 additions and 172 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -36,7 +36,7 @@ jobs:
           channel: ${{ matrix.flutter-channel }}
       - name: Override dependency version
         run: |
-          echo -e "\ndependency_overrides:\n  win32: 2.6.1" >> pubspec.yaml
+          echo -e "\ndependency_overrides:\n  win32: 2.6.1\n  hive:\n    path: ../hive" >> pubspec.yaml
         working-directory: hive_flutter
       - name: Install dependencies
         run: flutter pub get

diff --git a/hive/README.md b/hive/README.md
@@ -164,6 +164,35 @@ class SettingsPage extends StatelessWidget {
 
 Boxes are cached and therefore fast enough to be used directly in the `build()` method of Flutter widgets.
 
+### Native AES crypto implementation
+
+When using Flutter, Hive supports native encryption using [package:cryptography](https://pub.dev/packages/cryptography)
+and [package:cryptography_flutter](https://pub.dev/packages/cryptography_flutter).
+
+Native AES implementations tremendously speed up operations on encrypted Boxes.
+
+Please follow these steps:
+
+1. add dependency to pubspec.yaml
+
+```yaml
+dependencies:
+  cryptography_flutter: ^2.0.2
+```
+
+2. enable native implementations
+
+```dart
+import 'package:cryptography_flutter/cryptography_flutter.dart';
+
+void main() {
+  // Enable Flutter cryptography
+  FlutterCryptography.enable();
+
+  // ....
+}
+```
+
 ## Benchmark
 
 |                                         1000 read iterations                                         |                                    1000 write iterations                                    |

diff --git a/hive/lib/src/backend/js/native/storage_backend_js.dart b/hive/lib/src/backend/js/native/storage_backend_js.dart
@@ -41,7 +41,7 @@ class StorageBackendJs extends StorageBackend {
 
   /// Not part of public API
   @visibleForTesting
-  dynamic encodeValue(Frame frame) {
+  Future<dynamic> encodeValue(Frame frame) async {
     var value = frame.value;
     if (_cipher == null) {
       if (value == null) {
@@ -66,7 +66,7 @@ class StorageBackendJs extends StorageBackend {
     if (_cipher == null) {
       frameWriter.write(value);
     } else {
-      frameWriter.writeEncrypted(value, _cipher!);
+      await frameWriter.writeEncrypted(value, _cipher!);
     }
 
     var bytes = frameWriter.toBytes();
@@ -76,7 +76,7 @@ class StorageBackendJs extends StorageBackend {
 
   /// Not part of public API
   @visibleForTesting
-  dynamic decodeValue(dynamic value) {
+  Future<dynamic> decodeValue(dynamic value) async {
     if (value is ByteBuffer) {
       var bytes = Uint8List.view(value);
       if (_isEncoded(bytes)) {
@@ -131,9 +131,9 @@ class StorageBackendJs extends StorageBackend {
     if (hasProperty(store, 'getAll') && !cursor) {
       var completer = Completer<Iterable<dynamic>>();
       var request = store.getAll(null);
-      request.onSuccess.listen((_) {
-        var values = (request.result as List).map(decodeValue);
-        completer.complete(values);
+      request.onSuccess.listen((_) async {
+        var futures = (request.result as List).map(decodeValue);
+        completer.complete(await Future.wait(futures));
       });
       request.onError.listen((_) {
         completer.completeError(request.error!);
@@ -178,7 +178,7 @@ class StorageBackendJs extends StorageBackend {
       if (frame.deleted) {
         await store.delete(frame.key);
       } else {
-        await store.put(encodeValue(frame), frame.key);
+        await store.put(await encodeValue(frame), frame.key);
       }
     }
   }

diff --git a/hive/lib/src/backend/storage_backend_memory.dart b/hive/lib/src/backend/storage_backend_memory.dart
@@ -27,8 +27,8 @@ class StorageBackendMemory extends StorageBackend {
 
   @override
   Future<void> initialize(
-      TypeRegistry registry, Keystore? keystore, bool lazy) {
-    var recoveryOffset = _frameHelper.framesFromBytes(
+      TypeRegistry registry, Keystore? keystore, bool lazy) async {
+    var recoveryOffset = await _frameHelper.framesFromBytes(
       _bytes!, // Initialized at constructor and nulled after initialization
       keystore,
       registry,

diff --git a/hive/lib/src/backend/vm/storage_backend_vm.dart b/hive/lib/src/backend/vm/storage_backend_vm.dart
@@ -49,6 +49,9 @@ class StorageBackendVm extends StorageBackend {
 
   bool _compactionScheduled = false;
 
+  /// a cache for asynchronouse I/O operations
+  final Set<Future<void>> _ongoingTransactions = {};
+
   /// Not part of public API
   StorageBackendVm(
       this._file, this._lockFile, this._crashRecovery, this._cipher)
@@ -101,101 +104,129 @@ class StorageBackendVm extends StorageBackend {
   }
 
   @override
-  Future<dynamic> readValue(Frame frame) {
-    return _sync.syncRead(() async {
-      await readRaf.setPosition(frame.offset);
+  Future<dynamic> readValue(Frame frame) async {
+    Future<dynamic> operation() async {
+      await Future.wait(_ongoingTransactions);
+      await _sync.syncRead(() async {
+        await readRaf.setPosition(frame.offset);
 
-      var bytes = await readRaf.read(frame.length!);
+        var bytes = await readRaf.read(frame.length!);
 
-      var reader = BinaryReaderImpl(bytes, registry);
-      var readFrame = reader.readFrame(cipher: _cipher, lazy: false);
+        var reader = BinaryReaderImpl(bytes, registry);
+        var readFrame = await reader.readFrame(cipher: _cipher, lazy: false);
 
-      if (readFrame == null) {
-        throw HiveError(
-            'Could not read value from box. Maybe your box is corrupted.');
-      }
+        if (readFrame == null) {
+          throw HiveError(
+              'Could not read value from box. Maybe your box is corrupted.');
+        }
 
-      return readFrame.value;
-    });
+        return readFrame.value;
+      });
+    }
+
+    final operationFuture = operation.call();
+    _ongoingTransactions.add(operationFuture);
+    final result = await operationFuture;
+    _ongoingTransactions.remove(operationFuture);
+    return result;
   }
 
   @override
-  Future<void> writeFrames(List<Frame> frames) {
-    return _sync.syncWrite(() async {
-      var writer = BinaryWriterImpl(registry);
+  Future<void> writeFrames(List<Frame> frames) async {
+    var writer = BinaryWriterImpl(registry);
 
-      for (var frame in frames) {
-        frame.length = writer.writeFrame(frame, cipher: _cipher);
-      }
+    for (var frame in frames) {
+      frame.length = await writer.writeFrame(frame, cipher: _cipher);
+    }
+    Future<void> operation() async {
+      // adding to the end of the queue
+      await Future.wait(_ongoingTransactions);
+      await _sync.syncWrite(() async {
+        final bytes = writer.toBytes();
+
+        final cachedOffset = writeOffset;
+        try {
+          /// TODO(TheOneWithTheBraid): implement real transactions with cache
+          await writeRaf.writeFrom(bytes);
+        } catch (e) {
+          await writeRaf.setPosition(cachedOffset);
+          rethrow;
+        }
+      });
+    }
 
-      try {
-        await writeRaf.writeFrom(writer.toBytes());
-      } catch (e) {
-        await writeRaf.setPosition(writeOffset);
-        rethrow;
-      }
+    final future = operation();
+    _ongoingTransactions.add(future);
+    future.then((value) => _ongoingTransactions.remove(future));
 
-      for (var frame in frames) {
-        frame.offset = writeOffset;
-        writeOffset += frame.length!;
-      }
-    });
+    for (var frame in frames) {
+      frame.offset = writeOffset;
+      writeOffset += frame.length!;
+    }
   }
 
   @override
-  Future<void> compact(Iterable<Frame> frames) {
+  Future<void> compact(Iterable<Frame> frames) async {
     if (_compactionScheduled) return Future.value();
     _compactionScheduled = true;
 
-    return _sync.syncReadWrite(() async {
-      await readRaf.setPosition(0);
-      var reader = BufferedFileReader(readRaf);
-
-      var fileDirectory = path.substring(0, path.length - 5);
-      var compactFile = File('$fileDirectory.hivec');
-      var compactRaf = await compactFile.open(mode: FileMode.write);
-      var writer = BufferedFileWriter(compactRaf);
-
-      var sortedFrames = frames.toList();
-      sortedFrames.sort((a, b) => a.offset.compareTo(b.offset));
-      try {
-        for (var frame in sortedFrames) {
-          if (frame.offset == -1) continue; // Frame has not been written yet
-          if (frame.offset != reader.offset) {
-            var skip = frame.offset - reader.offset;
-            if (reader.remainingInBuffer < skip) {
-              if (await reader.loadBytes(skip) < skip) {
-                throw HiveError('Could not compact box: Unexpected EOF.');
+    Future<void> operation() async {
+      await Future.wait(_ongoingTransactions);
+      await _sync.syncReadWrite(() async {
+        await readRaf.setPosition(0);
+        var reader = BufferedFileReader(readRaf);
+
+        var fileDirectory = path.substring(0, path.length - 5);
+        var compactFile = File('$fileDirectory.hivec');
+        var compactRaf = await compactFile.open(mode: FileMode.write);
+        var writer = BufferedFileWriter(compactRaf);
+
+        var sortedFrames = frames.toList();
+        sortedFrames.sort((a, b) => a.offset.compareTo(b.offset));
+        try {
+          for (var frame in sortedFrames) {
+            if (frame.offset == -1) continue; // Frame has not been written yet
+            if (frame.offset != reader.offset) {
+              var skip = frame.offset - reader.offset;
+              if (reader.remainingInBuffer < skip) {
+                if (await reader.loadBytes(skip) < skip) {
+                  throw HiveError('Could not compact box: Unexpected EOF.');
+                }
               }
+              reader.skip(skip);
             }
-            reader.skip(skip);
-          }
 
-          if (reader.remainingInBuffer < frame.length!) {
-            if (await reader.loadBytes(frame.length!) < frame.length!) {
-              throw HiveError('Could not compact box: Unexpected EOF.');
+            if (reader.remainingInBuffer < frame.length!) {
+              if (await reader.loadBytes(frame.length!) < frame.length!) {
+                throw HiveError('Could not compact box: Unexpected EOF.');
+              }
             }
+            await writer.write(reader.viewBytes(frame.length!));
           }
-          await writer.write(reader.viewBytes(frame.length!));
+          await writer.flush();
+        } finally {
+          await compactRaf.close();
         }
-        await writer.flush();
-      } finally {
-        await compactRaf.close();
-      }
 
-      await readRaf.close();
-      await writeRaf.close();
-      await compactFile.rename(path);
-      await open();
+        await readRaf.close();
+        await writeRaf.close();
+        await compactFile.rename(path);
+        await open();
 
-      var offset = 0;
-      for (var frame in sortedFrames) {
-        if (frame.offset == -1) continue;
-        frame.offset = offset;
-        offset += frame.length!;
-      }
-      _compactionScheduled = false;
-    });
+        var offset = 0;
+        for (var frame in sortedFrames) {
+          if (frame.offset == -1) continue;
+          frame.offset = offset;
+          offset += frame.length!;
+        }
+        _compactionScheduled = false;
+      });
+    }
+
+    final future = operation();
+    _ongoingTransactions.add(future);
+    await future;
+    _ongoingTransactions.remove(future);
   }
 
   @override
@@ -216,7 +247,8 @@ class StorageBackendVm extends StorageBackend {
   }
 
   @override
-  Future<void> close() {
+  Future<void> close() async {
+    await Future.wait(_ongoingTransactions);
     return _sync.syncReadWrite(_closeInternal);
   }
 
@@ -229,9 +261,17 @@ class StorageBackendVm extends StorageBackend {
   }
 
   @override
-  Future<void> flush() {
-    return _sync.syncWrite(() async {
-      await writeRaf.flush();
-    });
+  Future<void> flush() async {
+    Future<void> operation() async {
+      await Future.wait(_ongoingTransactions);
+      await _sync.syncWrite(() async {
+        await writeRaf.flush();
+      });
+    }
+
+    final future = operation();
+    _ongoingTransactions.add(future);
+    await future;
+    _ongoingTransactions.remove(future);
   }
 }
diff --git a/hive/lib/src/binary/binary_reader_impl.dart b/hive/lib/src/binary/binary_reader_impl.dart
@@ -241,8 +241,8 @@ class BinaryReaderImpl extends BinaryReader {
   }
 
   /// Not part of public API
-  Frame? readFrame(
-      {HiveCipher? cipher, bool lazy = false, int frameOffset = 0}) {
+  Future<Frame?> readFrame(
+      {HiveCipher? cipher, bool lazy = false, int frameOffset = 0}) async {
     // frame length is stored on 4 bytes
     if (availableBytes < 4) return null;
 
@@ -275,7 +275,7 @@ class BinaryReaderImpl extends BinaryReader {
     } else if (cipher == null) {
       frame = Frame(key, read());
     } else {
-      frame = Frame(key, readEncrypted(cipher));
+      frame = Frame(key, await readEncrypted(cipher));
     }
 
     frame
@@ -332,10 +332,10 @@ class BinaryReaderImpl extends BinaryReader {
   /// Not part of public API
   @pragma('vm:prefer-inline')
   @pragma('dart2js:tryInline')
-  dynamic readEncrypted(HiveCipher cipher) {
+  Future<dynamic> readEncrypted(HiveCipher cipher) async {
     var inpLength = availableBytes;
     var out = Uint8List(inpLength);
-    var outLength = cipher.decrypt(_buffer, _offset, inpLength, out, 0);
+    var outLength = await cipher.decrypt(_buffer, _offset, inpLength, out, 0);
     _offset += inpLength;
 
     var valueReader = BinaryReaderImpl(out, _typeRegistry, outLength);