diff --git a/Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs b/Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs index 67ca3ef0..8b7769b6 100644 --- a/Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs +++ b/Wino.Core.Domain/Interfaces/IImapSynchronizerStrategy.cs @@ -17,16 +17,24 @@ public interface IImapSynchronizerStrategy /// Imap synchronizer that downloads messages. /// Cancellation token. /// List of new downloaded message ids that don't exist locally. - Task> HandleSynchronizationAsync(IImapClient client, MailItemFolder folder, IImapSynchronizer synchronizer, CancellationToken cancellationToken = default); + Task> HandleSynchronizationAsync(IImapClient client, + MailItemFolder folder, + IImapSynchronizer synchronizer, + CancellationToken cancellationToken = default); /// /// Downloads given set of messages from the folder. /// Folder is expected to be opened and synchronizer is connected. /// /// Synchronizer that performs the action. - /// Remote folder to download messages from. + /// Remote folder to download messages from. + /// Local folder to assign mails to. /// Set of message uniqueids. /// Cancellation token. - Task DownloadMessagesAsync(IImapSynchronizer synchronizer, IMailFolder folder, UniqueIdSet uniqueIdSet, CancellationToken cancellationToken = default); + Task DownloadMessagesAsync(IImapSynchronizer synchronizer, + IMailFolder remoteFolder, + MailItemFolder localFolder, + UniqueIdSet uniqueIdSet, + CancellationToken cancellationToken = default); } diff --git a/Wino.Core.Domain/Interfaces/IMailService.cs b/Wino.Core.Domain/Interfaces/IMailService.cs index 34331e16..544889ea 100644 --- a/Wino.Core.Domain/Interfaces/IMailService.cs +++ b/Wino.Core.Domain/Interfaces/IMailService.cs @@ -18,9 +18,13 @@ public interface IMailService /// Returns the single mail item with the given mail copy id. /// Caution: This method is not safe. Use other overrides. /// - /// - /// Task GetSingleMailItemAsync(string mailCopyId); + + /// + /// Returns the multiple mail item with the given mail copy ids. + /// Caution: This method is not safe. Use other overrides. + /// + Task> GetMailItemsAsync(IEnumerable mailCopyIds); Task> FetchMailsAsync(MailListInitializationOptions options, CancellationToken cancellationToken = default); /// @@ -88,6 +92,15 @@ public interface IMailService /// Native mail id of the message. Task IsMailExistsAsync(string mailCopyId); + /// + /// Checks whether the given mail copy ids exists in the database. + /// Safely used for Outlook to prevent downloading the same mail twice. + /// For Gmail, it should be avoided since one mail may belong to multiple folders. + /// + /// Native mail id of the messages. + /// List of Mail ids that already exists in the database. + Task> AreMailsExistsAsync(IEnumerable mailCopyIds); + /// /// Returns all mails for given folder id. /// diff --git a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs index 2b44cdbb..a18991a2 100644 --- a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs +++ b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs @@ -61,6 +61,7 @@ public interface IDefaultChangeProcessor Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken); Task GetMailCopyAsync(string mailCopyId); + Task> GetMailCopiesAsync(IEnumerable mailCopyIds); Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package); Task DeleteUserMailCacheAsync(Guid accountId); @@ -73,6 +74,7 @@ public interface IDefaultChangeProcessor /// Folder's local id. /// Whether mail exists in the folder or not. Task IsMailExistsInFolderAsync(string messageId, Guid folderId); + Task> AreMailsExistsAsync(IEnumerable mailCopyIds); } public interface IGmailChangeProcessor : IDefaultChangeProcessor @@ -139,9 +141,15 @@ public Task ChangeFlagStatusAsync(string mailCopyId, bool isFlagged) public Task IsMailExistsAsync(string messageId) => MailService.IsMailExistsAsync(messageId); + public Task> AreMailsExistsAsync(IEnumerable mailCopyIds) + => MailService.AreMailsExistsAsync(mailCopyIds); + public Task GetMailCopyAsync(string mailCopyId) => MailService.GetSingleMailItemAsync(mailCopyId); + public Task> GetMailCopiesAsync(IEnumerable mailCopyIds) + => MailService.GetMailItemsAsync(mailCopyIds); + public Task ChangeMailReadStatusAsync(string mailCopyId, bool isRead) => MailService.ChangeReadStatusAsync(mailCopyId, isRead); diff --git a/Wino.Core/Synchronizers/GmailSynchronizer.cs b/Wino.Core/Synchronizers/GmailSynchronizer.cs index 4fe88c02..823b11b0 100644 --- a/Wino.Core/Synchronizers/GmailSynchronizer.cs +++ b/Wino.Core/Synchronizers/GmailSynchronizer.cs @@ -1021,7 +1021,7 @@ public override async Task> OnlineSearchAsync(string queryText, L string pageToken = null; - var messagesToDownload = new List(); + List messagesToDownload = []; do { @@ -1030,7 +1030,7 @@ public override async Task> OnlineSearchAsync(string queryText, L // Ignore the folders if the query starts with these keywords. // User is trying to list everything. } - else if (folders?.Any() ?? false) + else if (folders?.Count > 0) { request.LabelIds = folders.Select(a => a.RemoteFolderId).ToList(); } @@ -1044,49 +1044,23 @@ public override async Task> OnlineSearchAsync(string queryText, L if (response.Messages == null) break; // Handle skipping manually - foreach (var message in response.Messages) - { - messagesToDownload.Add(message); - } + messagesToDownload.AddRange(response.Messages); pageToken = response.NextPageToken; } while (!string.IsNullOrEmpty(pageToken)); // Do not download messages that exists, but return them for listing. - var messageIds = messagesToDownload.Select(a => a.Id).ToList(); - - List downloadRequireMessageIds = new(); + var messageIds = messagesToDownload.Select(a => a.Id); - foreach (var messageId in messageIds) - { - var exists = await _gmailChangeProcessor.IsMailExistsAsync(messageId).ConfigureAwait(false); - - if (!exists) - { - downloadRequireMessageIds.Add(messageId); - } - } + var downloadRequireMessageIds = messageIds.Except(await _gmailChangeProcessor.AreMailsExistsAsync(messageIds)); // Download missing messages. await BatchDownloadMessagesAsync(downloadRequireMessageIds, cancellationToken); // Get results from database and return. - var searchResults = new List(); - - foreach (var messageId in messageIds) - { - var copy = await _gmailChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false); - - if (copy == null) continue; - - searchResults.Add(copy); - } - - return searchResults; - - // TODO: Return the search result ids. + return await _gmailChangeProcessor.GetMailCopiesAsync(messageIds); } public override async Task DownloadMissingMimeMessageAsync(IMailItem mailItem, diff --git a/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs b/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs index 59ef6163..a25e2341 100644 --- a/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs +++ b/Wino.Core/Synchronizers/ImapSync/ImapSynchronizationStrategyBase.cs @@ -84,19 +84,13 @@ protected async Task> HandleChangedUIdsAsync(IImapSynchronizer sync // Fetch the new mails in batch. var batchedMessageIds = newMessageIds.Batch(50).ToList(); - var downloadTasks = new List(); - // Create tasks for each batch. foreach (var group in batchedMessageIds) { downloadedMessageIds.AddRange(group.Select(a => MailkitClientExtensions.CreateUid(Folder.Id, a.Id))); - var task = DownloadMessagesAsync(synchronizer, remoteFolder, new UniqueIdSet(group), cancellationToken); - downloadTasks.Add(task); + await DownloadMessagesAsync(synchronizer, remoteFolder, Folder, new UniqueIdSet(group), cancellationToken).ConfigureAwait(false); } - // Wait for all batches to complete. - await Task.WhenAll(downloadTasks).ConfigureAwait(false); - return downloadedMessageIds; } @@ -167,6 +161,7 @@ protected async Task ManageUUIdBasedDeletedMessagesAsync(MailItemFolder localFol public async Task DownloadMessagesAsync(IImapSynchronizer synchronizer, IMailFolder folder, + MailItemFolder localFolder, UniqueIdSet uniqueIdSet, CancellationToken cancellationToken = default) { @@ -178,7 +173,7 @@ public async Task DownloadMessagesAsync(IImapSynchronizer synchronizer, var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage); - var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, Folder, cancellationToken).ConfigureAwait(false); + var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false); if (mailPackages != null) { @@ -187,7 +182,7 @@ public async Task DownloadMessagesAsync(IImapSynchronizer synchronizer, // Local draft is mapped. We don't need to create a new mail copy. if (package == null) continue; - await MailService.CreateMailAsync(Folder.MailAccountId, package).ConfigureAwait(false); + await MailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false); } } } diff --git a/Wino.Core/Synchronizers/ImapSynchronizer.cs b/Wino.Core/Synchronizers/ImapSynchronizer.cs index 788ff77a..87eddf1e 100644 --- a/Wino.Core/Synchronizers/ImapSynchronizer.cs +++ b/Wino.Core/Synchronizers/ImapSynchronizer.cs @@ -639,52 +639,48 @@ public override async Task> OnlineSearchAsync(string queryText, L { client = await _clientPool.GetClientAsync().ConfigureAwait(false); - var searchResults = new List(); - List searchResultFolderMailUids = new(); + List searchResults = []; + List searchResultFolderMailUids = []; foreach (var folder in folders) { - var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId).ConfigureAwait(false); + var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); // Look for subject and body. var query = SearchQuery.BodyContains(queryText).Or(SearchQuery.SubjectContains(queryText)); var searchResultsInFolder = await remoteFolder.SearchAsync(query, cancellationToken).ConfigureAwait(false); - var nonExisttingUniqueIds = new List(); + Dictionary searchResultsIdsInFolder = []; foreach (var searchResultId in searchResultsInFolder) { var folderMailUid = MailkitClientExtensions.CreateUid(folder.Id, searchResultId.Id); searchResultFolderMailUids.Add(folderMailUid); + searchResultsIdsInFolder.Add(folderMailUid, searchResultId); + } - bool exists = await _imapChangeProcessor.IsMailExistsAsync(folderMailUid); + // Populate no foundIds + var foundIds = await _imapChangeProcessor.AreMailsExistsAsync(searchResultsIdsInFolder.Select(a => a.Key)); + var notFoundIds = searchResultsIdsInFolder.Keys.Except(foundIds); - if (!exists) - { - nonExisttingUniqueIds.Add(searchResultId); - } + List nonExistingUniqueIds = []; + foreach (var nonExistingId in notFoundIds) + { + nonExistingUniqueIds.Add(searchResultsIdsInFolder[nonExistingId]); } - if (nonExisttingUniqueIds.Any()) + if (nonExistingUniqueIds.Count != 0) { var syncStrategy = _imapSynchronizationStrategyProvider.GetSynchronizationStrategy(client); - await syncStrategy.DownloadMessagesAsync(this, remoteFolder, new UniqueIdSet(nonExisttingUniqueIds, SortOrder.Ascending), cancellationToken).ConfigureAwait(false); - } - await remoteFolder.CloseAsync().ConfigureAwait(false); - } - - foreach (var messageId in searchResultFolderMailUids) - { - var copy = await _imapChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false); - - if (copy == null) continue; + await syncStrategy.DownloadMessagesAsync(this, remoteFolder, folder as MailItemFolder, new UniqueIdSet(nonExistingUniqueIds, SortOrder.Ascending), cancellationToken).ConfigureAwait(false); + } - searchResults.Add(copy); + await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - return searchResults; + return await _imapChangeProcessor.GetMailCopiesAsync(searchResultFolderMailUids); } catch (Exception ex) { @@ -700,8 +696,6 @@ public override async Task> OnlineSearchAsync(string queryText, L _clientPool.Release(client); } - - return new List(); } private async Task> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default) diff --git a/Wino.Core/Synchronizers/OutlookSynchronizer.cs b/Wino.Core/Synchronizers/OutlookSynchronizer.cs index 467bff20..fd9c7b5a 100644 --- a/Wino.Core/Synchronizers/OutlookSynchronizer.cs +++ b/Wino.Core/Synchronizers/OutlookSynchronizer.cs @@ -1054,18 +1054,7 @@ public override async Task> OnlineSearchAsync(string queryText, L } // Get results from database and return. - var searchResults = new List(); - - foreach (var messageId in existingMessageIds) - { - var copy = await _outlookChangeProcessor.GetMailCopyAsync(messageId).ConfigureAwait(false); - - if (copy == null) continue; - - searchResults.Add(copy); - } - - return searchResults; + return await _outlookChangeProcessor.GetMailCopiesAsync(existingMessageIds); } private async Task DownloadMimeMessageAsync(string messageId, CancellationToken cancellationToken = default) diff --git a/Wino.Services/MailService.cs b/Wino.Services/MailService.cs index c498fc27..10ae6887 100644 --- a/Wino.Services/MailService.cs +++ b/Wino.Services/MailService.cs @@ -143,7 +143,7 @@ public async Task> GetUnreadMailsByFolderIdAsync(Guid folderId) return unreadMails; } - private string BuildMailFetchQuery(MailListInitializationOptions options) + private static string BuildMailFetchQuery(MailListInitializationOptions options) { // If the search query is there, we should ignore some properties and trim it. //if (!string.IsNullOrEmpty(options.SearchQuery)) @@ -231,7 +231,7 @@ public async Task> FetchMailsAsync(MailListInitializationOptions // Avoid DBs calls as possible, storing info in a dictionary. foreach (var mail in mails) { - await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache).ConfigureAwait(false); + await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false); } // Remove items that has no assigned account or folder. @@ -244,12 +244,12 @@ public async Task> FetchMailsAsync(MailListInitializationOptions // Threading is disabled. Just return everything as it is. mails.Sort(options.SortingOptionType == SortingOptionType.ReceiveDate ? new DateComparer() : new NameComparer()); - return new List(mails); + return [.. mails]; } // Populate threaded items. - var threadedItems = new List(); + List threadedItems = []; // Each account items must be threaded separately. foreach (var group in mails.GroupBy(a => a.AssignedAccount.Id)) @@ -270,7 +270,7 @@ public async Task> FetchMailsAsync(MailListInitializationOptions foreach (var mail in accountThreadedItems) { cancellationToken.ThrowIfCancellationRequested(); - await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache).ConfigureAwait(false); + await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false); } if (accountThreadedItems != null) @@ -283,76 +283,65 @@ public async Task> FetchMailsAsync(MailListInitializationOptions cancellationToken.ThrowIfCancellationRequested(); return threadedItems; + } - // Recursive function to populate folder and account assignments for each mail item. - async Task LoadAssignedPropertiesWithCacheAsync(IMailItem mail, - Dictionary folderCache, - Dictionary accountCache) + /// + /// This method should used for operations with multiple mailItems. Don't use this for single mail items. + /// Called method should provide own instances for caches. + /// + private async Task LoadAssignedPropertiesWithCacheAsync(IMailItem mail, Dictionary folderCache, Dictionary accountCache, Dictionary contactCache) + { + if (mail is ThreadMailItem threadMailItem) { - if (mail is ThreadMailItem threadMailItem) + foreach (var childMail in threadMailItem.ThreadItems) { - foreach (var childMail in threadMailItem.ThreadItems) - { - await LoadAssignedPropertiesWithCacheAsync(childMail, folderCache, accountCache).ConfigureAwait(false); - } + await LoadAssignedPropertiesWithCacheAsync(childMail, folderCache, accountCache, contactCache).ConfigureAwait(false); } + } - if (mail is MailCopy mailCopy) + if (mail is MailCopy mailCopy) + { + var isFolderCached = folderCache.TryGetValue(mailCopy.FolderId, out MailItemFolder folderAssignment); + MailAccount accountAssignment = null; + if (!isFolderCached) { - MailAccount accountAssignment = null; + folderAssignment = await _folderService.GetFolderAsync(mailCopy.FolderId).ConfigureAwait(false); + folderCache.TryAdd(mailCopy.FolderId, folderAssignment); + } - var isFolderCached = folderCache.TryGetValue(mailCopy.FolderId, out MailItemFolder folderAssignment); - accountAssignment = null; - if (!isFolderCached) + if (folderAssignment != null) + { + var isAccountCached = accountCache.TryGetValue(folderAssignment.MailAccountId, out accountAssignment); + if (!isAccountCached) { - folderAssignment = await _folderService.GetFolderAsync(mailCopy.FolderId).ConfigureAwait(false); - if (!folderCache.ContainsKey(mailCopy.FolderId)) - { - folderCache.Add(mailCopy.FolderId, folderAssignment); - } - } + accountAssignment = await _accountService.GetAccountAsync(folderAssignment.MailAccountId).ConfigureAwait(false); - if (folderAssignment != null) - { - var isAccountCached = accountCache.TryGetValue(folderAssignment.MailAccountId, out accountAssignment); - if (!isAccountCached) - { - accountAssignment = await _accountService.GetAccountAsync(folderAssignment.MailAccountId).ConfigureAwait(false); - - if (!accountCache.ContainsKey(folderAssignment.MailAccountId)) - { - accountCache.Add(folderAssignment.MailAccountId, accountAssignment); - } - } + accountCache.TryAdd(folderAssignment.MailAccountId, accountAssignment); } + } - AccountContact contactAssignment = null; + AccountContact contactAssignment = null; - bool isContactCached = !string.IsNullOrEmpty(mailCopy.FromAddress) ? - contactCache.TryGetValue(mailCopy.FromAddress, out contactAssignment) : - false; + bool isContactCached = !string.IsNullOrEmpty(mailCopy.FromAddress) && + contactCache.TryGetValue(mailCopy.FromAddress, out contactAssignment); + + if (!isContactCached && accountAssignment != null) + { + contactAssignment = await GetSenderContactForAccountAsync(accountAssignment, mailCopy.FromAddress).ConfigureAwait(false); - if (!isContactCached && accountAssignment != null) + if (contactAssignment != null) { - contactAssignment = await GetSenderContactForAccountAsync(accountAssignment, mailCopy.FromAddress).ConfigureAwait(false); - - if (contactAssignment != null) - { - if (!contactCache.ContainsKey(mailCopy.FromAddress)) - { - contactCache.Add(mailCopy.FromAddress, contactAssignment); - } - } + contactCache.TryAdd(mailCopy.FromAddress, contactAssignment); } - - mailCopy.AssignedFolder = folderAssignment; - mailCopy.AssignedAccount = accountAssignment; - mailCopy.SenderContact = contactAssignment ?? CreateUnknownContact(mailCopy.FromName, mailCopy.FromAddress); } + + mailCopy.AssignedFolder = folderAssignment; + mailCopy.AssignedAccount = accountAssignment; + mailCopy.SenderContact = contactAssignment ?? CreateUnknownContact(mailCopy.FromName, mailCopy.FromAddress); } } - private AccountContact CreateUnknownContact(string fromName, string fromAddress) + private static AccountContact CreateUnknownContact(string fromName, string fromAddress) { if (string.IsNullOrEmpty(fromName) && string.IsNullOrEmpty(fromAddress)) { @@ -1071,4 +1060,38 @@ public async Task GetGmailArchiveComparisonResultA return new GmailArchiveComparisonResult(addedMails, removedMails); } + + public async Task> GetMailItemsAsync(IEnumerable mailCopyIds) + { + if (!mailCopyIds.Any()) return []; + + var query = new Query("MailCopy") + .WhereIn("MailCopy.Id", mailCopyIds) + .SelectRaw("MailCopy.*") + .GetRawQuery(); + + var mailCopies = await Connection.QueryAsync(query); + if (mailCopies?.Count == 0) return []; + + Dictionary folderCache = []; + Dictionary accountCache = []; + Dictionary contactCache = []; + + foreach (var mail in mailCopies) + { + await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false); + } + + return mailCopies; + } + + public async Task> AreMailsExistsAsync(IEnumerable mailCopyIds) + { + var query = new Query(nameof(MailCopy)) + .WhereIn("Id", mailCopyIds) + .Select("Id") + .GetRawQuery(); + + return await Connection.QueryScalarsAsync(query); + } }