feat: improve BinaryWriter API

- extend default buffer size - allow HiveCipher to asynchronously perform crypto - implement HiveAesThreadedCipher for Hive Flutter - fix missing file in .gitignore In theory, no public API should be made incompatible here as HIveCipher was simply converted from `T Function()` to `FutureOr<T> Function()`. The idea behind the asynchronous operation is a) multithreading using e.g. the `compute()` function in Flutter or b) platform-native implementations making use of hardware-accelerated cryptographic implementations. Signed-off-by: TheOneWithTheBraid <[email protected]>
isar · Jun 3, 2022 · 74d6835 · 74d6835
1 parent 7faa095
commit 74d6835
Show file tree

Hide file tree

Showing 28 changed files with 768 additions and 107 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -36,7 +36,7 @@ jobs:
           channel: ${{ matrix.flutter-channel }}
       - name: Override dependency version
         run: |
-          echo -e "\ndependency_overrides:\n  win32: 2.6.1" >> pubspec.yaml
+          echo -e "\ndependency_overrides:\n  win32: 2.6.1\n  hive:\n    path: ../hive" >> pubspec.yaml
         working-directory: hive_flutter
       - name: Install dependencies
         run: flutter pub get

diff --git a/hive/README.md b/hive/README.md
@@ -164,6 +164,35 @@ class SettingsPage extends StatelessWidget {
 
 Boxes are cached and therefore fast enough to be used directly in the `build()` method of Flutter widgets.
 
+### Native AES crypto implementation
+
+When using Flutter, Hive supports native encryption using [package:cryptography](https://pub.dev/packages/cryptography)
+and [package:cryptography_flutter](https://pub.dev/packages/cryptography_flutter).
+
+Native AES implementations tremendously speed up operations on encrypted Boxes.
+
+Please follow these steps:
+
+1. add dependency to pubspec.yaml
+
+```yaml
+dependencies:
+  cryptography_flutter: ^2.0.2
+```
+
+2. enable native implementations
+
+```dart
+import 'package:cryptography_flutter/cryptography_flutter.dart';
+
+void main() {
+  // Enable Flutter cryptography
+  FlutterCryptography.enable();
+
+  // ....
+}
+```
+
 ## Benchmark
 
 |                                         1000 read iterations                                         |                                    1000 write iterations                                    |

diff --git a/hive/lib/src/backend/js/native/storage_backend_js.dart b/hive/lib/src/backend/js/native/storage_backend_js.dart
@@ -41,7 +41,7 @@ class StorageBackendJs extends StorageBackend {
 
   /// Not part of public API
   @visibleForTesting
-  dynamic encodeValue(Frame frame) {
+  Future<dynamic> encodeValue(Frame frame) async {
     var value = frame.value;
     if (_cipher == null) {
       if (value == null) {
@@ -66,17 +66,17 @@ class StorageBackendJs extends StorageBackend {
     if (_cipher == null) {
       frameWriter.write(value);
     } else {
-      frameWriter.writeEncrypted(value, _cipher!);
+      await frameWriter.writeEncrypted(value, _cipher!);
     }
 
-    var bytes = frameWriter.toBytes();
+    var bytes = await frameWriter.toBytes();
     var sublist = bytes.sublist(0, bytes.length);
     return sublist.buffer;
   }
 
   /// Not part of public API
   @visibleForTesting
-  dynamic decodeValue(dynamic value) {
+  Future<dynamic> decodeValue(dynamic value) async {
     if (value is ByteBuffer) {
       var bytes = Uint8List.view(value);
       if (_isEncoded(bytes)) {
@@ -131,9 +131,9 @@ class StorageBackendJs extends StorageBackend {
     if (hasProperty(store, 'getAll') && !cursor) {
       var completer = Completer<Iterable<dynamic>>();
       var request = store.getAll(null);
-      request.onSuccess.listen((_) {
-        var values = (request.result as List).map(decodeValue);
-        completer.complete(values);
+      request.onSuccess.listen((_) async {
+        var futures = (request.result as List).map(decodeValue);
+        completer.complete(await Future.wait(futures));
       });
       request.onError.listen((_) {
         completer.completeError(request.error!);
@@ -178,7 +178,7 @@ class StorageBackendJs extends StorageBackend {
       if (frame.deleted) {
         await store.delete(frame.key);
       } else {
-        await store.put(encodeValue(frame), frame.key);
+        await store.put(await encodeValue(frame), frame.key);
       }
     }
   }

diff --git a/hive/lib/src/backend/storage_backend_memory.dart b/hive/lib/src/backend/storage_backend_memory.dart
@@ -27,8 +27,8 @@ class StorageBackendMemory extends StorageBackend {
 
   @override
   Future<void> initialize(
-      TypeRegistry registry, Keystore? keystore, bool lazy) {
-    var recoveryOffset = _frameHelper.framesFromBytes(
+      TypeRegistry registry, Keystore? keystore, bool lazy) async {
+    var recoveryOffset = await _frameHelper.framesFromBytes(
       _bytes!, // Initialized at constructor and nulled after initialization
       keystore,
       registry,

diff --git a/hive/lib/src/backend/vm/storage_backend_vm.dart b/hive/lib/src/backend/vm/storage_backend_vm.dart
@@ -108,7 +108,7 @@ class StorageBackendVm extends StorageBackend {
       var bytes = await readRaf.read(frame.length!);
 
       var reader = BinaryReaderImpl(bytes, registry);
-      var readFrame = reader.readFrame(cipher: _cipher, lazy: false);
+      var readFrame = await reader.readFrame(cipher: _cipher, lazy: false);
 
       if (readFrame == null) {
         throw HiveError(
@@ -125,11 +125,15 @@ class StorageBackendVm extends StorageBackend {
       var writer = BinaryWriterImpl(registry);
 
       for (var frame in frames) {
-        frame.length = writer.writeFrame(frame, cipher: _cipher);
+        frame.length = await writer.writeFrame(frame, cipher: _cipher);
       }
 
+      print(writer.operations);
+
       try {
-        await writeRaf.writeFrom(writer.toBytes());
+        final bytes = await writer.toBytes();
+        print(bytes);
+        await writeRaf.writeFrom(bytes);
       } catch (e) {
         await writeRaf.setPosition(writeOffset);
         rethrow;

diff --git a/hive/lib/src/binary/binary_reader_impl.dart b/hive/lib/src/binary/binary_reader_impl.dart
@@ -241,8 +241,8 @@ class BinaryReaderImpl extends BinaryReader {
   }
 
   /// Not part of public API
-  Frame? readFrame(
-      {HiveCipher? cipher, bool lazy = false, int frameOffset = 0}) {
+  Future<Frame?> readFrame(
+      {HiveCipher? cipher, bool lazy = false, int frameOffset = 0}) async {
     // frame length is stored on 4 bytes
     if (availableBytes < 4) return null;
 
@@ -275,7 +275,7 @@ class BinaryReaderImpl extends BinaryReader {
     } else if (cipher == null) {
       frame = Frame(key, read());
     } else {
-      frame = Frame(key, readEncrypted(cipher));
+      frame = Frame(key, await readEncrypted(cipher));
     }
 
     frame
@@ -332,10 +332,10 @@ class BinaryReaderImpl extends BinaryReader {
   /// Not part of public API
   @pragma('vm:prefer-inline')
   @pragma('dart2js:tryInline')
-  dynamic readEncrypted(HiveCipher cipher) {
+  Future<dynamic> readEncrypted(HiveCipher cipher) async {
     var inpLength = availableBytes;
     var out = Uint8List(inpLength);
-    var outLength = cipher.decrypt(_buffer, _offset, inpLength, out, 0);
+    var outLength = await cipher.decrypt(_buffer, _offset, inpLength, out, 0);
     _offset += inpLength;
 
     var valueReader = BinaryReaderImpl(out, _typeRegistry, outLength);

diff --git a/hive/lib/src/binary/binary_writer_impl.dart b/hive/lib/src/binary/binary_writer_impl.dart
@@ -1,3 +1,4 @@
+import 'dart:async';
 import 'dart:convert';
 import 'dart:typed_data';
 
@@ -11,7 +12,7 @@ import 'package:meta/meta.dart';
 
 /// Not part of public API
 class BinaryWriterImpl extends BinaryWriter {
-  static const _initBufferSize = 256;
+  static const _initBufferSize = 4096;
 
   final TypeRegistryImpl _typeRegistry;
   Uint8List _buffer = Uint8List(_initBufferSize);
@@ -20,6 +21,11 @@ class BinaryWriterImpl extends BinaryWriter {
 
   int _offset = 0;
 
+  /// contains pending binary operations on the output
+  ///
+  /// This is used to archive asynchronous encryption in background
+  final List<Future<Uint8List>> _ongoingOperations = [];
+
   @pragma('vm:prefer-inline')
   @pragma('dart2js:tryInline')
   ByteData get _byteData {
@@ -268,7 +274,7 @@ class BinaryWriterImpl extends BinaryWriter {
   }
 
   /// Not part of public API
-  int writeFrame(Frame frame, {HiveCipher? cipher}) {
+  Future<int> writeFrame(Frame frame, {HiveCipher? cipher}) async {
     ArgumentError.checkNotNull(frame);
 
     var startOffset = _offset;
@@ -281,7 +287,7 @@ class BinaryWriterImpl extends BinaryWriter {
       if (cipher == null) {
         write(frame.value);
       } else {
-        writeEncrypted(frame.value, cipher);
+        await writeEncrypted(frame.value, cipher);
       }
     }
 
@@ -394,23 +400,65 @@ class BinaryWriterImpl extends BinaryWriter {
   /// Not part of public API
   @pragma('vm:prefer-inline')
   @pragma('dart2js:tryInline')
-  void writeEncrypted(dynamic value, HiveCipher cipher,
-      {bool writeTypeId = true}) {
+  Future<void> writeEncrypted(dynamic value, HiveCipher cipher,
+      {bool writeTypeId = true}) async {
     var valueWriter = BinaryWriterImpl(_typeRegistry)
       ..write(value, writeTypeId: writeTypeId);
     var inp = valueWriter._buffer;
     var inpLength = valueWriter._offset;
 
     _reserveBytes(cipher.maxEncryptedSize(inp));
 
-    var len = cipher.encrypt(inp, 0, inpLength, _buffer, _offset);
+    // the supposed output length
+    int len;
+
+    /// compatibility to deprecated implementations
+    // ignore: deprecated_member_use_from_same_package
+    if (cipher.encrypt != null) {
+      // ignore: deprecated_member_use_from_same_package
+      len = await cipher.encrypt!.call(inp, 0, inpLength, _buffer, _offset);
+    } else {
+      final parallel = await cipher.encryptParallel(
+          inp, 0, inpLength, () => _buffer, () => _offset);
+
+      /// in case parallel operation is supported by the cipher, store the
+      /// operation
+
+      void applyOperation(Uint8List result) {
+        final outEnd = _offset + parallel.outputLength;
+        if (outEnd > _buffer.length) {
+          print(outEnd);
+          print(_buffer.length);
+          _buffer.length = outEnd;
+        }
+        _buffer.setRange(_offset, outEnd, result);
+        print(_buffer);
+      }
+
+      final operation = parallel.operation;
+      if (operation is Future<Uint8List>) {
+        _ongoingOperations.add(operation);
+        operation.then((result) {
+          applyOperation(result);
+          _ongoingOperations.remove(operation);
+        });
+      } else {
+        applyOperation(operation);
+      }
 
+      len = parallel.outputLength;
+    }
     _offset += len;
   }
 
   /// Not part of public API
-  Uint8List toBytes() {
-    return Uint8List.view(_buffer.buffer, 0, _offset);
+  FutureOr<Uint8List> toBytes() {
+    print('${_ongoingOperations.length} operations ongoing...');
+    if (_ongoingOperations.isEmpty) {
+      return Uint8List.view(_buffer.buffer, 0, _offset);
+    } else {
+      return Future.wait(_ongoingOperations).then((value) => toBytes());
+    }
   }
 
   @pragma('vm:prefer-inline')
@@ -425,4 +473,7 @@ class BinaryWriterImpl extends BinaryWriter {
     x |= x >> 16;
     return x + 1;
   }
+
+  @visibleForTesting
+  get operations => _ongoingOperations;
 }
diff --git a/hive/lib/src/binary/frame_helper.dart b/hive/lib/src/binary/frame_helper.dart
@@ -7,18 +7,18 @@ import 'package:hive/src/box/keystore.dart';
 /// Not part of public API
 class FrameHelper {
   /// Not part of public API
-  int framesFromBytes(
+  Future<int> framesFromBytes(
     Uint8List bytes,
     Keystore? keystore,
     TypeRegistry registry,
     HiveCipher? cipher,
-  ) {
+  ) async {
     var reader = BinaryReaderImpl(bytes, registry);
 
     while (reader.availableBytes != 0) {
       var frameOffset = reader.usedBytes;
 
-      var frame = reader.readFrame(
+      var frame = await reader.readFrame(
         cipher: cipher,
         lazy: false,
         frameOffset: frameOffset,

diff --git a/hive/lib/src/box/box_base_impl.dart b/hive/lib/src/box/box_base_impl.dart
@@ -166,7 +166,16 @@ abstract class BoxBaseImpl<E> implements BoxBase<E> {
   Future<void> close() async {
     if (!_open) return;
 
-    _open = false;
+    try {
+      await flush();
+    } catch (e) {
+      print('Box already flushed');
+    } finally {
+      _open = false;
+    }
+
+    print('Closing');
+
     await keystore.close();
     hive.unregisterBox(name);
 

diff --git a/hive/lib/src/box_collection.dart b/hive/lib/src/box_collection.dart
@@ -206,7 +206,7 @@ class CollectionBox<V> {
     // other stuff while the flusing is still in progress. Fortunately, hive has
     // a proper read / write queue, meaning that if we do actually want to write
     // something again, it'll wait until the flush is completed.
-    box.flush();
+    await box.flush();
   }
 
   Future<void> _flushOrMark() async {

diff --git a/hive/lib/src/crypto/hive_aes_cipher.dart b/hive/lib/src/crypto/hive_aes_cipher.dart
@@ -1,7 +1,7 @@
 part of hive;
 
 /// Default encryption algorithm. Uses AES256 CBC with PKCS7 padding.
-class HiveAesCipher implements HiveCipher {
+class HiveAesCipher extends HiveCipher {
   static final _ivRandom = Random.secure();
 
   late final AesCbcPkcs7 _cipher;
@@ -24,7 +24,7 @@ class HiveAesCipher implements HiveCipher {
   int calculateKeyCrc() => _keyCrc;
 
   @override
-  int decrypt(
+  FutureOr<int> decrypt(
       Uint8List inp, int inpOff, int inpLength, Uint8List out, int outOff) {
     var iv = inp.view(inpOff, 16);
 
@@ -36,14 +36,21 @@ class HiveAesCipher implements HiveCipher {
   Uint8List generateIv() => _ivRandom.nextBytes(16);
 
   @override
-  int encrypt(
-      Uint8List inp, int inpOff, int inpLength, Uint8List out, int outOff) {
+  FutureOr<ParallelEncryption> encryptParallel(
+      Uint8List inp,
+      int inpOff,
+      int inpLength,
+      Uint8List Function() getOut,
+      int Function() getOutOff,
+      ) {
     var iv = generateIv();
-    out.setAll(outOff, iv);
 
-    var len = _cipher.encrypt(iv, inp, 0, inpLength, out, outOff + 16);
+    final out = Uint8List.fromList(getOut());
+    out.setAll(getOutOff(), iv);
+
+    var len = _cipher.encrypt(iv, inp, 0, inpLength, out, getOutOff() + 16);
 
-    return len + 16;
+    return ParallelEncryption.serial(out);
   }
 
   @override