Fix bad option in publish test results

Tweak process runner tests. They seem to only fail on CI and are brittle. Added more debug information to the trace to help dx failures.
QutEcoacoustics · Apr 3, 2020 · 45e6232 · 45e6232
1 parent 2fd130e
commit 45e6232
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 14 deletions.
diff --git a/build/azure-pipelines-build.yml b/build/azure-pipelines-build.yml
@@ -144,7 +144,7 @@ jobs:
           - task: PublishTestResults@2
             inputs:
               testResultFormat: VSTest
-              testResultFiles: '**/*.trx'
+              testResultsFiles: '**/*.trx'
               searchFolder: $(Agent.TempDirectory)/Acoustics.Test_Results
               testRunTitle: "Acoustics.Test for ${{ platform.rid }} ${{ configuration }}"
               buildConfig: ${{ configuration }}
@@ -172,7 +172,7 @@ jobs:
           - task: PublishTestResults@2
             inputs:
               testResultFormat: VSTest
-              testResultFiles: '**/*.trx'
+              testResultsFiles: '**/*.trx'
               searchFolder: $(Agent.TempDirectory)/AED.Test_Results
               testRunTitle: ".Test for ${{ platform.rid }} ${{ configuration }}"
               buildConfig: ${{ configuration }}

diff --git a/tests/Acoustics.Test/Shared/ProcessRunnerTests.cs b/tests/Acoustics.Test/Shared/ProcessRunnerTests.cs
@@ -115,18 +115,20 @@ private bool RunFfprobeIndefinite(int index)
             var dest = PathHelper.GetTempFile(this.TestOutputDirectory, ".mp3");
             using (ProcessRunner runner = new ProcessRunner(AppConfigHelper.FfmpegExe))
             {
-                runner.WaitForExitMilliseconds = 1000;
+                runner.WaitForExitMilliseconds = 500;
                 runner.WaitForExit = true;
                 runner.MaxRetries = 1;
 
                 Assert.ThrowsException<ProcessRunner.ProcessMaximumRetriesException>(() =>
                 {
                     runner.Run($@"-i ""{path}"" -ar 8000 ""{dest}""", this.TestOutputDirectory.FullName);
+
+                    Assert.Fail($"Process running finished without timing out - this should not happen. Exit code: {runner.ExitCode}.\nStdout:\n{runner.StandardOutput}\nStdErr\n{runner.ErrorOutput}");
                 });
 
                 Assert.AreEqual(0, runner.StandardOutput.Length);
                 Assert.IsTrue(
-                    runner.ErrorOutput.Length > 1500,
+                    runner.ErrorOutput.Length > 1000,
                     $"Expected stderr to at least include ffmpeg header but it was only {runner.ErrorOutput.Length} chars. Index: {index}. StdErr:\n{runner.ErrorOutput}");
 
                 if (AppConfigHelper.IsWindows)

diff --git a/tests/Acoustics.Test/TestHelpers/RetryTestMethodAttribute.cs b/tests/Acoustics.Test/TestHelpers/RetryTestMethodAttribute.cs
@@ -32,7 +32,6 @@ public override TestResult[] Execute(ITestMethod testMethod)
 
             var result = new List<TestResult>();
 
-            var success = false;
             for (int count = 0; count < retryCount; count++)
             {
                 var testResults = base.Execute(testMethod);
@@ -41,20 +40,21 @@ public override TestResult[] Execute(ITestMethod testMethod)
                 var failed = testResults.FirstOrDefault((tr) => tr.Outcome == UnitTestOutcome.Failed);
                 if (failed != null)
                 {
-                    failed.TestContextMessages += $"Test iteration  {count + 1} of {retryCount} failed. Attempting to run again.";
-                    failed.Outcome = UnitTestOutcome.Inconclusive;
+                    var isLast = count + 1 == retryCount;
+                    var retryString = isLast ? " Attempting to run again." : " Exhausted retries, remaining failed.";
+                    failed.TestContextMessages += $"Test iteration {count + 1} of {retryCount} failed.{retryString}";
+
+                    if (!isLast)
+                    {
+                        failed.Outcome = UnitTestOutcome.Inconclusive;
+                    }
+
                     continue;
                 }
 
-                success = true;
                 break;
             }
 
-            if (!success)
-            {
-                result[^1].Outcome = UnitTestOutcome.Failed;
-            }
-
             return result.ToArray();
         }
     }

diff --git a/tests/Acoustics.Test/TestHelpers/TestSetup.cs b/tests/Acoustics.Test/TestHelpers/TestSetup.cs
@@ -21,7 +21,7 @@ public static void AssemblyInitialize(TestContext context)
                 enableMemoryLogger: true,
                 enableFileLogger: false,
                 colorConsole: false,
-                defaultLevel: Level.Verbose,
+                defaultLevel: Level.Info,
                 quietConsole: true);
         }
     }