From 6912f4518d785fdd34480fd620aecb8ef4341916 Mon Sep 17 00:00:00 2001 From: "Antti K. Koskela" Date: Sun, 22 Oct 2023 20:23:27 +0300 Subject: [PATCH 1/5] Add an optional Retry-loop around executing the search query --- documentation/Submit-PnPSearchQuery.md | 20 ++++- src/Commands/Search/SubmitSearchQuery.cs | 99 ++++++++++++++++-------- 2 files changed, 83 insertions(+), 36 deletions(-) diff --git a/documentation/Submit-PnPSearchQuery.md b/documentation/Submit-PnPSearchQuery.md index ab4427a2d..f2255644f 100644 --- a/documentation/Submit-PnPSearchQuery.md +++ b/documentation/Submit-PnPSearchQuery.md @@ -22,7 +22,7 @@ Submit-PnPSearchQuery [-Query] [-StartRow ] [-MaxResults [-RankingModelId ] [-ClientType ] [-CollapseSpecification ] [-HiddenConstraints ] [-TimeZoneId ] [-EnablePhonetic ] [-EnableStemming ] [-EnableQueryRules ] [-SourceId ] [-ProcessBestBets ] - [-ProcessPersonalFavorites ] [-RelevantResults] [-Connection ] + [-ProcessPersonalFavorites ] [-RelevantResults] [-Connection ] [-RetryCount ] ``` @@ -34,7 +34,7 @@ Submit-PnPSearchQuery [-Query] [-All] [-TrimDuplicates ] [-Pro [-CollapseSpecification ] [-HiddenConstraints ] [-TimeZoneId ] [-EnablePhonetic ] [-EnableStemming ] [-EnableQueryRules ] [-SourceId ] [-ProcessBestBets ] [-ProcessPersonalFavorites ] [-RelevantResults] - [-Connection ] + [-Connection ] [-RetryCount ] ``` ## DESCRIPTION @@ -346,6 +346,20 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -RetryCount +How many times to retry for a failed query. Default is 0 (no retries). Will wait 5 seconds between each retry. + +```yaml +Type: Int32 +Parameter Sets: (All) + +Required: False +Position: Named +Default value: 0 +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -SelectProperties The list of properties to return in the search results, separated by a comma. I.e. ComplianceTag,InformationProtectionLabelId. @@ -430,8 +444,6 @@ Accept pipeline input: False Accept wildcard characters: False ``` - - ## RELATED LINKS [Microsoft 365 Patterns and Practices](https://aka.ms/m365pnp) diff --git a/src/Commands/Search/SubmitSearchQuery.cs b/src/Commands/Search/SubmitSearchQuery.cs index bcbf002e5..3fdb70b0e 100644 --- a/src/Commands/Search/SubmitSearchQuery.cs +++ b/src/Commands/Search/SubmitSearchQuery.cs @@ -5,6 +5,8 @@ using Microsoft.SharePoint.Client.Search.Query; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Reflection; namespace PnP.PowerShell.Commands.Search { @@ -85,6 +87,9 @@ public class SubmitSearchQuery : PnPWebCmdlet [Parameter(Mandatory = false, ParameterSetName = ParameterAttribute.AllParameterSets)] public SwitchParameter RelevantResults; + [Parameter(Mandatory = false, ParameterSetName = ParameterAttribute.AllParameterSets)] + public int RetryCount = 0; + internal IEnumerable Run() { int startRow = StartRow; @@ -121,53 +126,83 @@ internal IEnumerable Run() keywordQuery.QueryText += " IndexDocId>" + lastDocId; } - var searchExec = new SearchExecutor(ClientContext); - var results = searchExec.ExecuteQuery(keywordQuery); - ClientContext.ExecuteQueryRetry(); - - if (results.Value != null) + // We'll always try at least once, even if RetryCount is 0 (default) + for (var iterator = 0; iterator <= RetryCount; iterator++) { - if (finalResults == null) + try { - finalResults = (PnPResultTableCollection)results.Value; - foreach (ResultTable resultTable in results.Value) + var searchExec = new SearchExecutor(ClientContext); + var results = searchExec.ExecuteQuery(keywordQuery); + ClientContext.ExecuteQueryRetry(); + + if (results.Value != null) { - if (resultTable.TableType == "RelevantResults") + if (finalResults == null) { - currentCount = resultTable.RowCount; - if (currentCount > 0) + finalResults = (PnPResultTableCollection)results.Value; + foreach (ResultTable resultTable in results.Value) { - lastDocId = resultTable.ResultRows.Last()["DocId"].ToString(); + if (resultTable.TableType == "RelevantResults") + { + currentCount = resultTable.RowCount; + if (currentCount > 0) + { + lastDocId = resultTable.ResultRows.Last()["DocId"].ToString(); + } + } } } - } - } - else - { - // we're in paging mode - foreach (ResultTable resultTable in results.Value) - { - PnPResultTable pnpResultTable = (PnPResultTable)resultTable; - var existingTable = finalResults.SingleOrDefault(t => t.TableType == resultTable.TableType); - if (existingTable != null) - { - existingTable.ResultRows.AddRange(pnpResultTable.ResultRows); - } else { - finalResults.Add(pnpResultTable); - } - if (pnpResultTable.TableType == "RelevantResults") - { - currentCount = resultTable.RowCount; - if (currentCount > 0) + // we're in paging mode + foreach (ResultTable resultTable in results.Value) { - lastDocId = resultTable.ResultRows.Last()["DocId"].ToString(); + PnPResultTable pnpResultTable = (PnPResultTable)resultTable; + var existingTable = finalResults.SingleOrDefault(t => t.TableType == resultTable.TableType); + if (existingTable != null) + { + existingTable.ResultRows.AddRange(pnpResultTable.ResultRows); + } + else + { + finalResults.Add(pnpResultTable); + } + if (pnpResultTable.TableType == "RelevantResults") + { + currentCount = resultTable.RowCount; + if (currentCount > 0) + { + lastDocId = resultTable.ResultRows.Last()["DocId"].ToString(); + } + } } } } + + // If we were successful (and didn't end in the catch block), we don't want to retry -> break out of retry loop + break; } + // If we're not retrying, or if we're on the last retry, don't catch the exception + catch (Exception ex) when (RetryCount > 0 && iterator < RetryCount) + { + // use reflection to find if the Exception has a property called "ServerErrorTypeName" and if so, check if its value is "Microsoft.Office.Server.Search.Query.InternalQueryErrorException" + var serverErrorTypeNameProperty = ex.GetType().GetProperty("ServerErrorTypeName", BindingFlags.Instance | BindingFlags.Public); + if (serverErrorTypeNameProperty != null) + { + var serverErrorTypeName = serverErrorTypeNameProperty.GetValue(ex); + if (serverErrorTypeName != null && serverErrorTypeName.ToString().Equals("Microsoft.Office.Server.Search.Query.InternalQueryErrorException", StringComparison.InvariantCultureIgnoreCase)) + { + // This was a "Microsoft.Office.Server.Search.Query.InternalQueryErrorException" which is safe to retry as-is (it's often transient) + // It's the one that says "Search has encountered a problem that prevents results from being returned. If the issue persists, please contact your administrator." + // Swallow the exception and retry (with incremental backoff) + Thread.Sleep(4000 * (iterator+1)); + continue; + } + } + // Rethrow the exception if it wasn't one warranting a retry + throw; + } } startRow += rowLimit; } while (currentCount == rowLimit && All.IsPresent); From 396b2c6e0464929282bfe0be38577fe1e41ab217 Mon Sep 17 00:00:00 2001 From: "Antti K. Koskela" Date: Sun, 22 Oct 2023 21:42:06 +0300 Subject: [PATCH 2/5] Remove the logic to look into the exception specifics with Reflection and just simply always retry --- src/Commands/Search/SubmitSearchQuery.cs | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/Commands/Search/SubmitSearchQuery.cs b/src/Commands/Search/SubmitSearchQuery.cs index 3fdb70b0e..588160c37 100644 --- a/src/Commands/Search/SubmitSearchQuery.cs +++ b/src/Commands/Search/SubmitSearchQuery.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading; -using System.Reflection; namespace PnP.PowerShell.Commands.Search { @@ -184,24 +183,11 @@ internal IEnumerable Run() } // If we're not retrying, or if we're on the last retry, don't catch the exception catch (Exception ex) when (RetryCount > 0 && iterator < RetryCount) - { - // use reflection to find if the Exception has a property called "ServerErrorTypeName" and if so, check if its value is "Microsoft.Office.Server.Search.Query.InternalQueryErrorException" - var serverErrorTypeNameProperty = ex.GetType().GetProperty("ServerErrorTypeName", BindingFlags.Instance | BindingFlags.Public); - if (serverErrorTypeNameProperty != null) - { - var serverErrorTypeName = serverErrorTypeNameProperty.GetValue(ex); - if (serverErrorTypeName != null && serverErrorTypeName.ToString().Equals("Microsoft.Office.Server.Search.Query.InternalQueryErrorException", StringComparison.InvariantCultureIgnoreCase)) - { - // This was a "Microsoft.Office.Server.Search.Query.InternalQueryErrorException" which is safe to retry as-is (it's often transient) - // It's the one that says "Search has encountered a problem that prevents results from being returned. If the issue persists, please contact your administrator." - // Swallow the exception and retry (with incremental backoff) - Thread.Sleep(4000 * (iterator+1)); + { + // Swallow the exception and retry (with incremental backoff) + Thread.Sleep(4000 * (iterator+1)); - continue; - } - } - // Rethrow the exception if it wasn't one warranting a retry - throw; + continue; } } startRow += rowLimit; From 34560fa1b2194da593005b4eb3072635dd8c546f Mon Sep 17 00:00:00 2001 From: "Antti K. Koskela" Date: Sun, 22 Oct 2023 22:27:12 +0300 Subject: [PATCH 3/5] Update wait time to match documentation. --- src/Commands/Search/SubmitSearchQuery.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Commands/Search/SubmitSearchQuery.cs b/src/Commands/Search/SubmitSearchQuery.cs index 588160c37..5c38bbe74 100644 --- a/src/Commands/Search/SubmitSearchQuery.cs +++ b/src/Commands/Search/SubmitSearchQuery.cs @@ -185,7 +185,7 @@ internal IEnumerable Run() catch (Exception ex) when (RetryCount > 0 && iterator < RetryCount) { // Swallow the exception and retry (with incremental backoff) - Thread.Sleep(4000 * (iterator+1)); + Thread.Sleep(5000 * (iterator+1)); continue; } From 4bfb880684289beb091c4d08bb91aa4690469332 Mon Sep 17 00:00:00 2001 From: Koen Zomers Date: Tue, 7 Nov 2023 16:25:18 +0100 Subject: [PATCH 4/5] Changing exceptions not to be swallowed but shown in verbose and the exception to be thrown if it still happens on the last attempt --- documentation/Submit-PnPSearchQuery.md | 21 +++++++++++++++++---- src/Commands/Search/SubmitSearchQuery.cs | 19 ++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/documentation/Submit-PnPSearchQuery.md b/documentation/Submit-PnPSearchQuery.md index b6e66e604..0601c3cad 100644 --- a/documentation/Submit-PnPSearchQuery.md +++ b/documentation/Submit-PnPSearchQuery.md @@ -22,7 +22,7 @@ Submit-PnPSearchQuery [-Query] [-StartRow ] [-MaxResults [-RankingModelId ] [-ClientType ] [-CollapseSpecification ] [-HiddenConstraints ] [-TimeZoneId ] [-EnablePhonetic ] [-EnableStemming ] [-EnableQueryRules ] [-SourceId ] [-ProcessBestBets ] - [-ProcessPersonalFavorites ] [-RelevantResults] [-Connection ] [-RetryCount ] + [-ProcessPersonalFavorites ] [-RelevantResults] [-Connection ] [-RetryCount ] [-Verbose] ``` @@ -34,7 +34,7 @@ Submit-PnPSearchQuery [-Query] [-All] [-TrimDuplicates ] [-Pro [-CollapseSpecification ] [-HiddenConstraints ] [-TimeZoneId ] [-EnablePhonetic ] [-EnableStemming ] [-EnableQueryRules ] [-SourceId ] [-ProcessBestBets ] [-ProcessPersonalFavorites ] [-RelevantResults] - [-Connection ] [-RetryCount ] + [-Connection ] [-RetryCount ] [-Verbose] ``` ## DESCRIPTION @@ -451,7 +451,20 @@ Accept pipeline input: False Accept wildcard characters: False ``` -## RELATED LINKS +### -Verbose +When provided, additional debug statements will be shown while executing the cmdlet. -[Microsoft 365 Patterns and Practices](https://aka.ms/m365pnp) +```yaml +Type: SwitchParameter +Parameter Sets: (All) + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +## RELATED LINKS +[Microsoft 365 Patterns and Practices](https://aka.ms/m365pnp) \ No newline at end of file diff --git a/src/Commands/Search/SubmitSearchQuery.cs b/src/Commands/Search/SubmitSearchQuery.cs index 5c38bbe74..48699f292 100644 --- a/src/Commands/Search/SubmitSearchQuery.cs +++ b/src/Commands/Search/SubmitSearchQuery.cs @@ -182,12 +182,21 @@ internal IEnumerable Run() break; } // If we're not retrying, or if we're on the last retry, don't catch the exception - catch (Exception ex) when (RetryCount > 0 && iterator < RetryCount) - { - // Swallow the exception and retry (with incremental backoff) - Thread.Sleep(5000 * (iterator+1)); + catch (Exception ex) + { + if (RetryCount > 0 && iterator < RetryCount) + { + var waitTime = 5 * (iterator + 1); + + WriteVerbose($"Search operation failed with exception {ex.Message}. Attempt {iterator + 1} out of {RetryCount}. Retrying in {waitTime} seconds."); - continue; + Thread.Sleep(TimeSpan.FromSeconds(waitTime)); + continue; + } + else + { + throw; + } } } startRow += rowLimit; From 6b136b7279488654501f3023ff76cc4185ca095d Mon Sep 17 00:00:00 2001 From: Koen Zomers Date: Tue, 7 Nov 2023 16:28:27 +0100 Subject: [PATCH 5/5] Added changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed1f3381..082fc65d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added GCC support for `Get-PnPAzureADUser` , `Add-PnPFlowOwner` , `Remove-PnPFlowOwner`, `Sync-PnPSharePointUserProfilesFromAzureActiveDirectory`, `New-PnPAzureADUserTemporaryAccessPass` and `Get-PnPAvailableSensitivityLabel` cmdlets. [#3484](https://github.com/pnp/powershell/pull/3484) - Added a devcontainer for easily building a minimal environment necessary to contribute to the project. [#3497](https://github.com/pnp/powershell/pull/3497) - Added `-RelativeUrl` parameter to `Connect-PnPOnline` cmdlet to allow specifying custom URLs for usage with `-WebLogin` method. [#3530](https://github.com/pnp/powershell/pull/3530) +- Added `-RetryCount` to `Submit-PnPSearchQuery` which allows for specifying the number of retries to perform when an exception occurs [#3528](https://github.com/pnp/powershell/pull/3528) ### Fixed @@ -84,6 +85,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Contributors +- Antti K. Koskela [koskila] - Kunj Balkrishna Sangani [kunj-sangani] - Antti K. Koskela [koskila] - Dave Paylor [paylord]