feat: add Matrix Sync message exchange

.github/workflows/flutter-analyze.yml

@@ -22,4 +22,4 @@ jobs:
           flutter-version: ${{ steps.fvm-config-action.outputs.FLUTTER_VERSION }}
           channel: ${{ steps.fvm-config-action.outputs.FLUTTER_CHANNEL }}
       - name: Run Flutter Analyze
-        run: make clean_analyze
+        run: flutter analyze

.github/workflows/flutter-matrix-test.yml

@@ -7,7 +7,35 @@ on:
 
 jobs:
   test:
-    name: Flutter Matrix Test on Linux
+    name: Matrix Test on Linux
+    runs-on: ubuntu-latest
+    steps:
+      - name: Update apt-get
+        run: sudo apt-get update
+      - name: Install libraries
+        run: sudo apt-get install libgtk-3-dev cmake libolm3 cmake-doc ninja-build libsecret-1-dev libjsoncpp-dev libsecret-1-0 sqlite3 libsqlite3-dev keybinder-3.0 network-manager mpv libmpv-dev
+      - uses: actions/checkout@v3
+      - uses: kuhnroyal/flutter-fvm-config-action@v2
+        id: fvm-config-action
+      - uses: subosito/flutter-action@v2
+        with:
+          flutter-version: ${{ steps.fvm-config-action.outputs.FLUTTER_VERSION }}
+          channel: ${{ steps.fvm-config-action.outputs.FLUTTER_CHANNEL }}
+      - uses: hoverkraft-tech/[email protected]
+        with:
+          cwd: "integration_test/docker"
+      - name: Run Matrix integration test
+        uses: GabrielBB/[email protected]
+        env:
+          SLOW_NETWORK: false
+        with:
+          working-directory: ./integration_test
+          run: |
+            ./setup_toxiproxy_docker.sh
+            ./run_matrix_tests.sh
+
+  test_degraded_network:
+    name: Matrix Test on Linux with degraded network
     runs-on: ubuntu-latest
     steps:
       - name: Update apt-get

Makefile

@@ -10,7 +10,6 @@ endif
 IOS_ARCHIVE_PATH = ./build/ios/archive/Runner.xcarchive
 IOS_EXPORT_PATH = ./build/ios/export
 MACOS_ARCHIVE_PATH = ./build/macos/archive/Runner.xcarchive
-WHISPER_CPP_VERSION = 1.4.2
 MACOS_EXPORT_PATH = ./build/macos/export
 LOTTI_VERSION := $(shell yq '.version' pubspec.yaml |  tr -d '"')
 
@@ -71,10 +70,6 @@ coverage_report:
 .PHONY: coverage
 coverage: test coverage_report
 
-.PHONY: check-null-safety
-check-null-safety:
-	$(FLUTTER_CMD) pub outdated --mode=null-safety
-
 .PHONY: build_runner
 build_runner: deps
 	dart run build_runner build --delete-conflicting-outputs
@@ -87,10 +82,6 @@ watch: l10n
 activate_fluttium:
 	$(FLUTTER_CMD) pub global activate fluttium_cli
 
-.PHONY: fluttium
-fluttium: get_whisper_cpp
-	fluttium test test_flows/habit_flow.yaml -d macOS --reporter expanded
-
 .PHONY: fluttium_linux
 fluttium_linux:
 	fluttium test test_flows/habit_flow.yaml --reporter expanded
@@ -121,7 +112,7 @@ bundle:
 #######################################
 
 .PHONY: ios_build_ipa
-ios_build_ipa: get_whisper_cpp
+ios_build_ipa:
 	$(FLUTTER_CMD) build ipa
 
 .PHONY: ios_build
@@ -163,7 +154,7 @@ ios_upload:
 ios: ios_build ios_fastlane_build ios_fastlane_upload
 
 .PHONY: macos_build_flutter
-macos_build_flutter: get_whisper_cpp
+macos_build_flutter:
 	$(FLUTTER_CMD) build macos
 
 .PHONY: macos_build
@@ -191,12 +182,6 @@ macos_upload:
 macos_open: macos_build macos_archive
 	open $(MACOS_ARCHIVE_PATH)
 
-.PHONY: get_whisper_cpp
-get_whisper_cpp:
-	cd whisper.cpp/ && \
-	wget -nc https://github.com/ggerganov/whisper.cpp/archive/refs/tags/v$(WHISPER_CPP_VERSION).zip && \
-	unzip -n v$(WHISPER_CPP_VERSION).zip
-
 .PHONY: macos_fastlane_build
 macos_fastlane_build:
 	cd macos && fastlane do_build && cd ..
@@ -262,12 +247,3 @@ icons:
 
 .PHONY: clean_test
 clean_test: clean deps build_runner l10n test
-
-.PHONY: clean_build_analyze
-clean_build_analyze: clean deps l10n build_runner analyze
-
-.PHONY: clean_analyze
-clean_analyze: clean deps l10n analyze
-
-.PHONY: clean_integration_test
-clean_integration_test: clean deps build_runner integration_test

integration_test/matrix_service_test.dart

@@ -1,13 +1,15 @@
 import 'dart:async';
 import 'dart:io';
 
+import 'package:drift/drift.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:lotti/classes/config.dart';
 import 'package:lotti/classes/entry_text.dart';
 import 'package:lotti/classes/journal_entities.dart';
 import 'package:lotti/classes/sync_message.dart';
+import 'package:lotti/database/database.dart';
 import 'package:lotti/database/logging_db.dart';
 import 'package:lotti/get_it.dart';
 import 'package:lotti/sync/matrix/matrix_service.dart';
@@ -19,20 +21,25 @@ import 'package:path_provider/path_provider.dart';
 import 'package:uuid/uuid.dart';
 
 import '../test/mocks/mocks.dart';
+import '../test/utils/utils.dart';
 
 void main() {
   TestWidgetsFlutterBinding.ensureInitialized();
 
   // description and how to run in https://github.com/matthiasn/lotti/pull/1695
   group('MatrixService Tests', () {
-    final mockLoggingDb = MockLoggingDb();
     final secureStorageMock = MockSecureStorage();
     const testUserEnv1 = 'TEST_USER1';
     const testUserEnv2 = 'TEST_USER2';
     const testServerEnv = 'TEST_SERVER';
     const testPasswordEnv = 'TEST_PASSWORD';
     const testSlowNetworkEnv = 'SLOW_NETWORK';
 
+    // create separate databases for each simulated device & suppress warning
+    driftRuntimeOptions.dontWarnAboutMultipleDatabases = true;
+    final aliceDb = JournalDb(overriddenFilename: 'alice_db.sqlite');
+    final bobDb = JournalDb(overriddenFilename: 'bob_db.sqlite');
+
     const testSlowNetwork = bool.fromEnvironment(testSlowNetworkEnv);
 
     if (testSlowNetwork) {
@@ -83,8 +90,8 @@ void main() {
       debugPrint('Created temporary docDir ${docDir.path}');
 
       getIt
-        ..registerSingleton<LoggingDb>(mockLoggingDb)
         ..registerSingleton<Directory>(docDir)
+        ..registerSingleton<LoggingDb>(LoggingDb())
         ..registerSingleton<SecureStorage>(secureStorageMock);
     });
 
@@ -104,6 +111,7 @@ void main() {
           matrixConfig: config1,
           hiveDbName: 'AliceDevice',
           deviceDisplayName: 'AliceDevice',
+          overriddenJournalDb: aliceDb,
         );
 
         await aliceDevice.login();
@@ -117,12 +125,14 @@ void main() {
 
         final joinRes = await aliceDevice.joinRoom(roomId);
         debugPrint('AliceDevice - room joined: $joinRes');
+        await aliceDevice.listenToTimeline();
 
         debugPrint('\n--- BobDevice goes live');
         final bobDevice = MatrixService(
           matrixConfig: config2,
           hiveDbName: 'BobDevice',
           deviceDisplayName: 'BobDevice',
+          overriddenJournalDb: bobDb,
         );
 
         await bobDevice.login();
@@ -131,10 +141,8 @@ void main() {
 
         final joinRes2 = await bobDevice.joinRoom(roomId);
         debugPrint('BobDevice - room joined: $joinRes2');
-
-        await Future<void>.delayed(
-          const Duration(seconds: defaultDelay * delayFactor),
-        );
+        await bobDevice.listenToTimeline();
+        await waitSeconds(defaultDelay * delayFactor);
 
         await waitUntil(() => aliceDevice.getUnverified().length == 1);
         await waitUntil(() => bobDevice.getUnverified().length == 1);
@@ -152,16 +160,12 @@ void main() {
         final incomingKeyVerificationRunnerStream =
             bobDevice.incomingKeyVerificationRunnerStream;
 
-        await Future<void>.delayed(
-          const Duration(seconds: defaultDelay * delayFactor),
-        );
+        await waitSeconds(defaultDelay * delayFactor);
 
         debugPrint('\n--- AliceDevice verifies BobDevice');
         await aliceDevice.verifyDevice(unverifiedAlice.first);
 
-        await Future<void>.delayed(
-          const Duration(seconds: defaultDelay * delayFactor),
-        );
+        await waitSeconds(defaultDelay * delayFactor);
 
         var emojisFromBob = '';
         var emojisFromAlice = '';
@@ -226,45 +230,81 @@ void main() {
         expect(aliceDevice.getUnverified(), isEmpty);
         expect(bobDevice.getUnverified(), isEmpty);
 
-        await Future<void>.delayed(
-          const Duration(seconds: defaultDelay * delayFactor),
-        );
-
-        final testEntry1 = JournalEntry(
-          meta: Metadata(
-            id: '32ea936e-dfc6-43bd-8722-d816c35eb489',
-            createdAt: DateTime(2024, 4, 6, 13),
-            dateFrom: DateTime(2024, 4, 6, 13),
-            dateTo: DateTime(2024, 4, 6, 14),
-            updatedAt: DateTime(2024, 4, 6, 13),
-            starred: true,
-            vectorClock: const VectorClock({'a': 11}),
-          ),
-          entryText: EntryText(
-            plainText: const Uuid().v1(),
-          ),
+        await waitSeconds(defaultDelay * delayFactor);
+
+        Future<void> sendTestMessage(
+          int index, {
+          required MatrixService device,
+          required String deviceName,
+        }) async {
+          final id = const Uuid().v1();
+          final now = DateTime.now();
+
+          await device.sendMatrixMsg(
+            SyncMessage.journalEntity(
+              journalEntity: JournalEntry(
+                meta: Metadata(
+                  id: id,
+                  createdAt: now,
+                  dateFrom: now,
+                  dateTo: now,
+                  updatedAt: now,
+                  starred: true,
+                  vectorClock: VectorClock({deviceName: index}),
+                ),
+                entryText: EntryText(
+                  plainText: 'Test $deviceName #$index - $now',
+                ),
+              ),
+              status: SyncEntryStatus.initial,
+            ),
+            myRoomId: roomId,
+          );
+        }
+
+        const n = testSlowNetwork ? 10 : 100;
+
+        debugPrint('\n--- AliceDevice sends $n message');
+        for (var i = 0; i < n; i++) {
+          await sendTestMessage(
+            i,
+            device: aliceDevice,
+            deviceName: 'aliceDevice',
+          );
+        }
+
+        debugPrint('\n--- BobDevice sends $n message');
+        for (var i = 0; i < n; i++) {
+          await sendTestMessage(
+            i,
+            device: bobDevice,
+            deviceName: 'bobDevice',
+          );
+        }
+
+        await waitUntilAsync(
+          () async => await aliceDb.getJournalCount() == 2 * n,
         );
+        debugPrint('\n--- AliceDevice finished receiving messages');
+        final aliceEntriesCount = await aliceDb.getJournalCount();
+        expect(aliceEntriesCount, 2 * n);
+        debugPrint('AliceDevice persisted entries: $aliceEntriesCount');
 
-        await aliceDevice.sendMatrixMsg(
-          SyncMessage.journalEntity(
-            journalEntity: testEntry1,
-            status: SyncEntryStatus.initial,
-          ),
-          myRoomId: roomId,
+        await waitUntilAsync(
+          () async => await bobDb.getJournalCount() == 2 * n,
         );
-
-        await Future<void>.delayed(
-          const Duration(seconds: defaultDelay * delayFactor),
-        );
-
-        final bobsTimelineEvents = await bobDevice.getTimelineEvents();
-        expect(bobsTimelineEvents?.length, 8);
+        debugPrint('\n--- BobDevice finished receiving messages');
+        final bobEntriesCount = await bobDb.getJournalCount();
+        expect(bobEntriesCount, 2 * n);
+        debugPrint('BobDevice persisted entries: $bobEntriesCount');
 
         debugPrint('\n--- Logging out AliceDevice and BobDevice');
+
         await aliceDevice.logout();
+        await waitSeconds(defaultDelay * delayFactor);
         await bobDevice.logout();
       },
-      timeout: const Timeout(Duration(minutes: 10)),
+      timeout: const Timeout(Duration(minutes: 15)),
     );
   });
 }
@@ -278,11 +318,3 @@ String extractEmojiString(Iterable<KeyVerificationEmoji>? emojis) {
   }
   return buffer.toString();
 }
-
-Future<void> waitUntil(
-  bool Function() condition,
-) async {
-  while (!condition()) {
-    await Future<void>.delayed(const Duration(milliseconds: 100));
-  }
-}

integration_test/setup_toxiproxy_docker.sh

@@ -4,6 +4,6 @@ cd "$(dirname "$0")" || exit
 
 cd docker || exit
 docker compose exec toxiproxy /toxiproxy-cli create -l toxiproxy:18008 -u dendrite:8008 dendrite-proxy
-docker compose exec toxiproxy /toxiproxy-cli toxic add -t latency -a latency=500 dendrite-proxy
-docker compose exec toxiproxy /toxiproxy-cli toxic add -t bandwidth -a rate=100 dendrite-proxy
+docker compose exec toxiproxy /toxiproxy-cli toxic add -t latency -a latency=200 dendrite-proxy
+docker compose exec toxiproxy /toxiproxy-cli toxic add -t bandwidth -a rate=300 dendrite-proxy
 cd ..

lib/database/conversions.dart

@@ -60,6 +60,7 @@ JournalDbEntity toDbEntity(JournalEntity entity) {
     task: task,
     taskStatus: taskStatus,
     dateTo: entity.meta.dateTo,
+    plainText: entity.entryText?.plainText,
     type: entity.map(
       journalEntry: (_) => 'JournalEntry',
       journalImage: (_) => 'JournalImage',

lib/database/database.dart

@@ -31,10 +31,12 @@ enum ConflictStatus {
   include: {'database.drift'},
 )
 class JournalDb extends _$JournalDb {
-  JournalDb({this.inMemoryDatabase = false})
-      : super(
+  JournalDb({
+    this.inMemoryDatabase = false,
+    String? overriddenFilename,
+  }) : super(
           openDbConnection(
-            journalDbFileName,
+            overriddenFilename ?? journalDbFileName,
             inMemoryDatabase: inMemoryDatabase,
           ),
         );