Merge pull request #1739 from matthiasn/refactor/database_updates

refactor: database updates via stream

integration_test/matrix_service_test.dart

@@ -13,6 +13,7 @@ import 'package:lotti/database/database.dart';
 import 'package:lotti/database/logging_db.dart';
 import 'package:lotti/database/settings_db.dart';
 import 'package:lotti/get_it.dart';
+import 'package:lotti/services/db_notification.dart';
 import 'package:lotti/sync/matrix/matrix_service.dart';
 import 'package:lotti/sync/secure_storage.dart';
 import 'package:lotti/sync/vector_clock.dart';
@@ -39,6 +40,18 @@ void main() {
 
     // create separate databases for each simulated device & suppress warning
     drift.driftRuntimeOptions.dontWarnAboutMultipleDatabases = true;
+
+    final mockUpdateNotifications = MockUpdateNotifications();
+
+    when(() => mockUpdateNotifications.updateStream).thenAnswer(
+      (_) => Stream<DatabaseType>.fromIterable([]),
+    );
+
+    when(() => mockUpdateNotifications.notifyUpdate(DatabaseType.journal))
+        .thenAnswer((_) {});
+
+    getIt.registerSingleton<UpdateNotifications>(mockUpdateNotifications);
+
     final aliceDb = JournalDb(overriddenFilename: 'alice_db.sqlite');
     final bobDb = JournalDb(overriddenFilename: 'bob_db.sqlite');
 

lib/blocs/journal/journal_page_cubit.dart

@@ -12,6 +12,7 @@ import 'package:lotti/database/database.dart';
 import 'package:lotti/database/fts5_db.dart';
 import 'package:lotti/database/settings_db.dart';
 import 'package:lotti/get_it.dart';
+import 'package:lotti/services/db_notification.dart';
 import 'package:lotti/utils/platform.dart';
 import 'package:rxdart/rxdart.dart';
 import 'package:visibility_detector/visibility_detector.dart';
@@ -101,14 +102,12 @@ class JournalPageCubit extends Cubit<JournalPageState> {
             }
           });
     } else {
-      _db
-          .watchJournalCount()
+      _updateNotifications.updateStream
           .throttleTime(
-            const Duration(seconds: 2),
-            leading: false,
-            trailing: true,
-          )
-          .where(makeDuplicateFilter())
+        const Duration(seconds: 1),
+        leading: false,
+        trailing: true,
+      )
           .listen((_) {
         if (_isVisible) {
           refreshQuery();
@@ -121,6 +120,7 @@ class JournalPageCubit extends Cubit<JournalPageState> {
   static const selectedEntryTypesKey = 'SELECTED_ENTRY_TYPES';
 
   final JournalDb _db = getIt<JournalDb>();
+  final UpdateNotifications _updateNotifications = getIt<UpdateNotifications>();
   bool _isVisible = false;
   static const _pageSize = 50;
   Set<String> _selectedEntryTypes = entryTypes.toSet();
@@ -308,15 +308,18 @@ class JournalPageCubit extends Cubit<JournalPageState> {
               .first;
 
       final isLastPage = newItems.length < _pageSize;
+
       if (isLastPage) {
         state.pagingController.appendLastPage(newItems);
       } else {
         final nextPageKey = pageKey + newItems.length;
         state.pagingController.appendPage(newItems, nextPageKey);
       }
       final finished = DateTime.now();
-      final duration = finished.difference(start);
-      debugPrint('_fetchPage $showTasks duration $duration');
+      final duration = finished.difference(start).inMicroseconds / 1000;
+      debugPrint(
+        '_fetchPage ${showTasks ? 'TASK' : 'JOURNAL'} duration $duration ms',
+      );
     } catch (error) {
       state.pagingController.error = error;
     }

lib/database/common.dart

@@ -1,15 +1,14 @@
 import 'dart:async';
 import 'dart:io';
-import 'dart:isolate';
 
 import 'package:drift/drift.dart';
-import 'package:drift/isolate.dart';
 import 'package:drift/native.dart';
-import 'package:flutter/foundation.dart';
 import 'package:intl/intl.dart';
-import 'package:lotti/get_it.dart';
 import 'package:lotti/utils/file_utils.dart';
 import 'package:path/path.dart' as p;
+import 'package:path_provider/path_provider.dart';
+import 'package:sqlite3/sqlite3.dart';
+import 'package:sqlite3_flutter_libs/sqlite3_flutter_libs.dart';
 
 Future<File> getDatabaseFile(String dbFileName) async {
   final dbFolder = getDocumentsDirectory();
@@ -33,83 +32,15 @@ LazyDatabase openDbConnection(
       return NativeDatabase.memory();
     }
 
-    final file = await getDatabaseFile(fileName);
-    debugPrint('DB LazyDatabase ${file.path}');
+    final dbFolder = await getApplicationDocumentsDirectory();
+    final file = File(p.join(dbFolder.path, fileName));
 
-    return NativeDatabase(file);
-  });
-}
-
-Future<DriftIsolate> createDriftIsolate(
-  String dbFileName, {
-  bool inMemory = false,
-}) async {
-  // this method is called from the main isolate. Since we can't use
-  // getApplicationDocumentsDirectory on a background isolate, we calculate
-  // the database path in the foreground isolate and then inform the
-  // background isolate about the path.
-  final dir = getDocumentsDirectory();
-  final path = p.join(dir.path, dbFileName);
-  final receivePort = ReceivePort();
-
-  await Isolate.spawn(
-    inMemory ? _startBackgroundInMem : _startBackground,
-    _IsolateStartRequest(receivePort.sendPort, path),
-  );
-
-  // _startBackground will send the DriftIsolate to this ReceivePort
-  return await receivePort.first as DriftIsolate;
-}
-
-void _startBackground(_IsolateStartRequest request) {
-  // this is the entry point from the background isolate! Let's create
-  // the database from the path we received
-  final executor = NativeDatabase(File(request.targetPath));
-  // we're using DriftIsolate.inCurrent here as this method already runs on a
-  // background isolate. If we used DriftIsolate.spawn, a third isolate would be
-  // started which is not what we want!
-  final driftIsolate = DriftIsolate.inCurrent(
-    () => DatabaseConnection(executor),
-  );
-  // inform the starting isolate about this, so that it can call .connect()
-  request.sendDriftIsolate.send(driftIsolate);
-}
-
-void _startBackgroundInMem(_IsolateStartRequest request) {
-  final executor = NativeDatabase.memory();
-  final driftIsolate = DriftIsolate.inCurrent(
-    () => DatabaseConnection(executor),
-  );
-  request.sendDriftIsolate.send(driftIsolate);
-}
-
-// used to bundle the SendPort and the target path, since isolate entry point
-// functions can only take one parameter.
-class _IsolateStartRequest {
-  _IsolateStartRequest(
-    this.sendDriftIsolate,
-    this.targetPath,
-  );
-
-  final SendPort sendDriftIsolate;
-  final String targetPath;
-}
+    if (Platform.isAndroid) {
+      await applyWorkaroundToOpenSqlite3OnOldAndroidVersions();
+    }
 
-DatabaseConnection getDatabaseConnection(String dbFileName) {
-  return DatabaseConnection.delayed(
-    Future.sync(() async {
-      final isolate = await getIt<Future<DriftIsolate>>(
-        instanceName: dbFileName,
-      );
-      return isolate.connect();
-    }),
-  );
-}
+    sqlite3.tempDirectory = (await getTemporaryDirectory()).path;
 
-DatabaseConnection getDbConnFromIsolate(DriftIsolate isolate) {
-  return DatabaseConnection.delayed(
-    Future.sync(() async {
-      return isolate.connect();
-    }),
-  );
+    return NativeDatabase.createInBackground(file);
+  });
 }

lib/database/database.dart

@@ -13,6 +13,7 @@ import 'package:lotti/database/common.dart';
 import 'package:lotti/database/conversions.dart';
 import 'package:lotti/database/logging_db.dart';
 import 'package:lotti/get_it.dart';
+import 'package:lotti/services/db_notification.dart';
 import 'package:lotti/sync/vector_clock.dart';
 import 'package:lotti/utils/file_utils.dart';
 import 'package:lotti/widgets/journal/entry_tools.dart';
@@ -40,9 +41,8 @@ class JournalDb extends _$JournalDb {
           ),
         );
 
-  JournalDb.connect(super.connection) : super.connect();
-
   bool inMemoryDatabase = false;
+  final UpdateNotifications _updateNotifications = getIt<UpdateNotifications>();
 
   @override
   int get schemaVersion => 19;
@@ -723,10 +723,7 @@ class JournalDb extends _$JournalDb {
       dashboard: upsertDashboardDefinition,
       categoryDefinition: upsertCategoryDefinition,
     );
+    _updateNotifications.notifyUpdate(DatabaseType.journal);
     return linesAffected;
   }
 }
-
-JournalDb getJournalDb() {
-  return JournalDb.connect(getDatabaseConnection(journalDbFileName));
-}

lib/database/journal_db/config_flags.dart

@@ -14,7 +14,13 @@ Future<void> initConfigFlags(
       status: true,
     ),
   );
-
+  await db.insertFlagIfNotExists(
+    const ConfigFlag(
+      name: attemptEmbedding,
+      description: 'Create LLM embedding',
+      status: false,
+    ),
+  );
   await db.insertFlagIfNotExists(
     const ConfigFlag(
       name: allowInvalidCertFlag,

lib/database/logging_db.dart

@@ -135,7 +135,3 @@ class LoggingDb extends _$LoggingDb {
     return allLogEntries(limit).watch();
   }
 }
-
-LoggingDb getLoggingDb() {
-  return LoggingDb.connect(getDatabaseConnection(loggingDbFileName));
-}

lib/database/settings_db.dart

@@ -55,7 +55,3 @@ class SettingsDb extends _$SettingsDb {
     }
   }
 }
-
-SettingsDb getSettingsDb() {
-  return SettingsDb.connect(getDatabaseConnection(settingsDbFileName));
-}

lib/database/sync_db.dart

@@ -97,7 +97,3 @@ class SyncDatabase extends _$SyncDatabase {
   @override
   int get schemaVersion => 1;
 }
-
-SyncDatabase getSyncDatabase() {
-  return SyncDatabase.connect(getDatabaseConnection(syncDbFileName));
-}

lib/get_it.dart

@@ -1,8 +1,6 @@
 import 'dart:async';
 
-import 'package:drift/isolate.dart';
 import 'package:get_it/get_it.dart';
-import 'package:lotti/database/common.dart';
 import 'package:lotti/database/database.dart';
 import 'package:lotti/database/editor_db.dart';
 import 'package:lotti/database/fts5_db.dart';
@@ -14,6 +12,7 @@ import 'package:lotti/logic/ai/ai_logic.dart';
 import 'package:lotti/logic/health_import.dart';
 import 'package:lotti/logic/persistence_logic.dart';
 import 'package:lotti/services/asr_service.dart';
+import 'package:lotti/services/db_notification.dart';
 import 'package:lotti/services/editor_state_service.dart';
 import 'package:lotti/services/entities_cache_service.dart';
 import 'package:lotti/services/link_service.dart';
@@ -28,29 +27,15 @@ import 'package:lotti/sync/outbox/outbox_service.dart';
 final getIt = GetIt.instance;
 
 Future<void> registerSingletons() async {
-  await getIt.registerSingleton<Future<DriftIsolate>>(
-    createDriftIsolate(journalDbFileName),
-    instanceName: journalDbFileName,
-  );
-
-  await getIt.registerSingleton<Future<DriftIsolate>>(
-    createDriftIsolate(loggingDbFileName),
-    instanceName: loggingDbFileName,
-  );
-
-  await getIt.registerSingleton<Future<DriftIsolate>>(
-    createDriftIsolate(syncDbFileName),
-    instanceName: syncDbFileName,
-  );
-
   getIt
     ..registerSingleton<Fts5Db>(Fts5Db())
-    ..registerSingleton<LoggingDb>(getLoggingDb())
-    ..registerSingleton<JournalDb>(getJournalDb())
+    ..registerSingleton<LoggingDb>(LoggingDb())
+    ..registerSingleton<UpdateNotifications>(UpdateNotifications())
+    ..registerSingleton<JournalDb>(JournalDb())
     ..registerSingleton<EditorDb>(EditorDb())
     ..registerSingleton<TagsService>(TagsService())
     ..registerSingleton<EntitiesCacheService>(EntitiesCacheService())
-    ..registerSingleton<SyncDatabase>(getSyncDatabase())
+    ..registerSingleton<SyncDatabase>(SyncDatabase())
     ..registerSingleton<AsrService>(AsrService())
     ..registerSingleton<VectorClockService>(VectorClockService())
     ..registerSingleton<TimeService>(TimeService())

lib/logic/ai/ai_logic.dart

@@ -1,6 +1,9 @@
 import 'package:flutter/foundation.dart';
 import 'package:langchain/langchain.dart';
 import 'package:lotti/classes/journal_entities.dart';
+import 'package:lotti/database/database.dart';
+import 'package:lotti/get_it.dart';
+import 'package:lotti/utils/consts.dart';
 import 'package:lotti/utils/platform.dart';
 import 'package:ollama_dart/ollama_dart.dart';
 
@@ -13,6 +16,13 @@ class AiLogic {
     JournalEntity? journalEntity, {
     String? linkedFromId,
   }) async {
+    final shouldAttemptEmbedding = await getIt<JournalDb>().getConfigFlag(
+      attemptEmbedding,
+    );
+    if (!shouldAttemptEmbedding) {
+      return;
+    }
+
     final markdown = journalEntity?.entryText?.markdown;
     final headline = switch (journalEntity) {
       Task() => journalEntity.data.title,

lib/main.dart

@@ -1,13 +1,11 @@
 import 'dart:async';
 import 'dart:io';
 
-import 'package:drift/isolate.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_animate/flutter_animate.dart';
 import 'package:flutter_riverpod/flutter_riverpod.dart';
 import 'package:hotkey_manager/hotkey_manager.dart';
 import 'package:lotti/beamer/beamer_app.dart';
-import 'package:lotti/database/common.dart';
 import 'package:lotti/database/logging_db.dart';
 import 'package:lotti/database/settings_db.dart';
 import 'package:lotti/get_it.dart';
@@ -34,14 +32,8 @@ Future<void> main() async {
 
     getIt
       ..registerSingleton<SecureStorage>(SecureStorage())
-      ..registerSingleton<Directory>(docDir);
-
-    await getIt.registerSingleton<Future<DriftIsolate>>(
-      createDriftIsolate(settingsDbFileName),
-      instanceName: settingsDbFileName,
-    );
-    getIt
-      ..registerSingleton<SettingsDb>(getSettingsDb())
+      ..registerSingleton<Directory>(docDir)
+      ..registerSingleton<SettingsDb>(SettingsDb())
       ..registerSingleton<WindowService>(WindowService());
 
     await getIt<WindowService>().restore();

lib/pages/settings/flags_page.dart

@@ -22,6 +22,7 @@ class FlagsPage extends StatelessWidget {
 
         const displayedItems = {
           privateFlag,
+          attemptEmbedding,
           enableNotificationsFlag,
           autoTranscribeFlag,
           recordLocationFlag,

lib/services/db_notification.dart

@@ -0,0 +1,20 @@
+import 'dart:async';
+
+enum DatabaseType {
+  journal,
+  setting,
+  sync,
+  logging,
+}
+
+class UpdateNotifications {
+  UpdateNotifications();
+
+  final _updateStreamController = StreamController<DatabaseType>.broadcast();
+
+  Stream<DatabaseType> get updateStream => _updateStreamController.stream;
+
+  void notifyUpdate(DatabaseType databaseType) {
+    _updateStreamController.add(databaseType);
+  }
+}