Fixes for CI and tests

Further tweaks to runtime identifier tests. They're not always running the correct way. Temporarily enabled verbose logging to help with mac mp3 bugs Fixed string matching for error in ValidatesNonExistingExePaths Work done for #196
QutEcoacoustics · Mar 28, 2020 · d32268d · d32268d
1 parent ce8b199
commit d32268d
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 15 deletions.
diff --git a/tests/Acoustics.Test/AnalysisPrograms/MainEntryTests.cs b/tests/Acoustics.Test/AnalysisPrograms/MainEntryTests.cs
@@ -21,7 +21,7 @@ namespace Acoustics.Test.AnalysisPrograms
     public class MainEntryTests
     {
         [DoNotParallelize]
-        [RuntimeIdentifierSpecificDataTestMethod(RidType.Compiled)]
+        [RuntimeIdentifierSpecificDataTestMethod(RidType.CompiledIfSelfContained)]
         [TestCategory("UnsupportedAzurePipelinesPlatform")]
         [DataRow(WinX64)]
         [DataRow(OsxX64)]
@@ -43,7 +43,7 @@ public async Task DefaultCliWorks(string rid)
         }
 
         [DoNotParallelize]
-        [RuntimeIdentifierSpecificDataTestMethod(RidType.Compiled)]
+        [RuntimeIdentifierSpecificDataTestMethod(RidType.CompiledIfSelfContained)]
         [TestCategory("UnsupportedAzurePipelinesPlatform")]
         [DataRow(WinX64)]
         [DataRow(OsxX64)]
@@ -66,7 +66,7 @@ public async Task DefaultHelpWorks(string rid)
         }
 
         [DoNotParallelize]
-        [RuntimeIdentifierSpecificDataTestMethod(RidType.Compiled)]
+        [RuntimeIdentifierSpecificDataTestMethod(RidType.CompiledIfSelfContained)]
         [TestCategory("UnsupportedAzurePipelinesPlatform")]
         [DataRow(WinX64)]
         [DataRow(OsxX64)]
@@ -89,7 +89,7 @@ public async Task DefaultVersionWorks(string rid)
         }
 
         [DoNotParallelize]
-        [RuntimeIdentifierSpecificDataTestMethod(RidType.Compiled)]
+        [RuntimeIdentifierSpecificDataTestMethod(RidType.CompiledIfSelfContained)]
         [TestCategory("UnsupportedAzurePipelinesPlatform")]
         [DataRow(WinX64)]
         [DataRow(OsxX64)]

diff --git a/tests/Acoustics.Test/AudioAnalysisTools/Indices/IndexMatricesTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/Indices/IndexMatricesTests.cs
@@ -28,7 +28,7 @@ public static IEnumerable<object[]> ScaleCombinations
                 var combinations = from s in scales from d in dataSizes from k in keys select new object[] { s, d, k };
 
                 // these scales uses a lot of memory, our CI server can't handle it, so exclude them
-                if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("APPVEYOR")))
+                if (TestHelper.OnContinuousIntegrationServer())
                 {
                     combinations = combinations.Where(x => !((double)x[0] < 0.4 && (int)x[1] > 100_000));
                 }

diff --git a/tests/Acoustics.Test/TestHelpers/RuntimeIdentifierSpecificDataTestMethod.cs b/tests/Acoustics.Test/TestHelpers/RuntimeIdentifierSpecificDataTestMethod.cs
@@ -16,6 +16,7 @@ public enum RidType
     {
         Pseudo,
         Compiled,
+        CompiledIfSelfContained,
         Actual,
     }
 
@@ -52,16 +53,25 @@ public override TestResult[] Execute(ITestMethod testMethod)
             var actual = AppConfigHelper.RuntimeIdentifier;
             var compiled = BuildMetadata.CompiledRuntimeIdentifer;
 
+            var compiledEmpty = compiled.IsNullOrEmpty();
+            var selfContained = BuildMetadata.CompiledAsSelfContained;
+
             var actualRid = this.RuntimeIdentifierSource switch
             {
                 RidType.Actual => actual,
+                RidType.Compiled => compiled,
+
                 // compiled will be empty when the --runtime argument is not supplied
-                RidType.Compiled => string.IsNullOrEmpty(compiled) ? pseudo : compiled,
+                RidType.CompiledIfSelfContained when selfContained && !compiledEmpty => compiled,
+                RidType.CompiledIfSelfContained when !selfContained && !compiledEmpty => compiled,
+                RidType.CompiledIfSelfContained when selfContained && compiledEmpty => throw new InvalidOperationException(
+                    $"Compiled RID is empty! Must be non-empty for mode {nameof(RidType.CompiledIfSelfContained)}."),
+                RidType.CompiledIfSelfContained when !selfContained && compiledEmpty => pseudo,
                 RidType.Pseudo => pseudo,
                 _ => throw new InvalidOperationException($"RidType {this.RuntimeIdentifierSource} is not supported"),
             };
 
-            Trace.WriteLine($"RIDs: Pseudo={pseudo}, Actual={actual}, Compiled={compiled}. Using={this.RuntimeIdentifierSource} which is {actualRid}.");
+            string debugMessage = $"RIDs: Pseudo={pseudo}, Actual={actual}, Compiled={compiled}. Using={this.RuntimeIdentifierSource} which is {actualRid}.";
 
             if (rid != actualRid)
             {
@@ -71,6 +81,7 @@ public override TestResult[] Execute(ITestMethod testMethod)
                 {
                     new TestResult
                     {
+                        TestContextMessages = debugMessage,
                         Outcome = UnitTestOutcome.Inconclusive,
                         TestFailureException = new AssertInconclusiveException(message),
                     },

diff --git a/tests/Acoustics.Test/TestHelpers/TestHelper.cs b/tests/Acoustics.Test/TestHelpers/TestHelper.cs
@@ -411,6 +411,11 @@ public static bool CompareDates(DateTime dt1, DateTime dt2)
                                                                               || dt1.Millisecond == dt2.Millisecond - 2);
         }
 
+        public static bool OnContinuousIntegrationServer()
+        {
+            return !GetEnvironmentVariable("TF_BUILD").IsNullOrEmpty() || !GetEnvironmentVariable("APPVEYOR").IsNullOrEmpty();
+        }
+
         /// <summary>
         /// Datetimes may not be exactly equal.
         /// </summary>

diff --git a/tests/Acoustics.Test/TestHelpers/TestSetup.cs b/tests/Acoustics.Test/TestHelpers/TestSetup.cs
@@ -21,7 +21,7 @@ public static void AssemblyInitialize(TestContext context)
                 enableMemoryLogger: true,
                 enableFileLogger: false,
                 colorConsole: false,
-                defaultLevel: Level.Info,
+                defaultLevel: Level.Verbose,
                 quietConsole: true);
         }
     }

diff --git a/tests/Acoustics.Test/Tools/MasterAudioUtilityTests.cs b/tests/Acoustics.Test/Tools/MasterAudioUtilityTests.cs
@@ -324,16 +324,16 @@ public void ValidatesNonExistingExePaths()
             var randomFile = new FileInfo(@"X:\hello-my-dear\where-are-you\hey-its-adirectory\blah.exe");
 
             TestHelper.ExceptionMatches<FileNotFoundException>(
-                () => new FfmpegAudioUtility(randomFile, randomFile), "Could not find exe");
+                () => new FfmpegAudioUtility(randomFile, randomFile), "Could not find binary");
 
             //TestHelper.ExceptionMatches<FileNotFoundException>(
             //    () => new Mp3SpltAudioUtility(randomFile), "Could not find exe");
 
             TestHelper.ExceptionMatches<FileNotFoundException>(
-                () => new WavPackAudioUtility(randomFile), "Could not find exe");
+                () => new WavPackAudioUtility(randomFile), "Could not find binary");
 
             TestHelper.ExceptionMatches<FileNotFoundException>(
-                () => new SoxAudioUtility(randomFile), "Could not find exe");
+                () => new SoxAudioUtility(randomFile), "Could not find binary");
         }
 
         /// <summary>
@@ -343,15 +343,15 @@ public void ValidatesNonExistingExePaths()
         public void ValidatesNullExePaths()
         {
             TestHelper.ExceptionMatches<ArgumentNullException>(
-                () => new FfmpegAudioUtility(null, null), "Value cannot be null");
+                () => new FfmpegAudioUtility(null, null), "but was supplied with null");
 
             //TestHelper.ExceptionMatches<ArgumentNullException>(
-            //    () => new Mp3SpltAudioUtility(null), "Value cannot be null");
+            //    () => new Mp3SpltAudioUtility(null), "but was supplied with null");
 
             TestHelper.ExceptionMatches<ArgumentNullException>(
-                () => new WavPackAudioUtility(null), "Value cannot be null");
+                () => new WavPackAudioUtility(null), "but was supplied with null");
 
-            TestHelper.ExceptionMatches<ArgumentNullException>(() => new SoxAudioUtility(null), "Value cannot be null");
+            TestHelper.ExceptionMatches<ArgumentNullException>(() => new SoxAudioUtility(null), "but was supplied with null");
         }
 
         [TestMethod]