[xharness] Improve parsing of NUnitV2 xml reports to fix an issue wit…

…h failures in parameterized tests. (#9259) We weren't properly reporting all failures when there were multiple failures in parameterized tests, because we'd incorrectly skip too many xml nodes when we were looking for the next test failure.
dotnet · Aug 3, 2020 · 691e8ea · 691e8ea · xamarin-release-manager · Aug 3, 2020
1 parent 7e9a60d
commit 691e8ea
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 11 deletions.
diff --git a/...rosoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj b/...rosoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj
@@ -18,6 +18,7 @@
     <None Remove="Samples\TouchUnitSample.xml" />
     <None Remove="Samples\xUnitSample.xml" />
     <None Remove="Samples\NUnitV2SampleFailure.xml" />
+    <None Remove="Samples\TestCaseFailures.xml" />
   </ItemGroup>
 
   <ItemGroup>
@@ -52,6 +53,7 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </EmbeddedResource>
     <EmbeddedResource Include="Samples\NUnitV2SampleFailure.xml" />
+    <EmbeddedResource Include="Samples\TestCaseFailures.xml" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<test-results name="mtouch.dll" total="440" errors="4" failures="19" not-run="6" inconclusive="0" ignored="6" skipped="0" invalid="0" date="2020-07-31" time="03:17:46">
+  <environment nunit-version="3.6.0.0" clr-version="4.0.30319.42000" os-version="Unix 19.5.0.0" platform="Unix" cwd="..." machine-name="..." user="..." user-domain="..." />
+  <culture-info current-culture="en-US" current-uiculture="en-US" />
+  <test-suite type="Assembly" name="mtouch.dll" executed="True" result="Failure" success="False" time="3393.268" asserts="7560">
+    <properties>
+      <property name="_PID" value="72474" />
+      <property name="_APPDOMAIN" value="domain-" />
+    </properties>
+    <failure>
+      <message><![CDATA[One or more child tests had errors]]></message>
+      <stack-trace />
+    </failure>
+    <results>
+      <test-suite type="SetUpFixture" name="[default namespace]" executed="True" result="Failure" success="False" time="3393.255" asserts="7560">
+        <failure>
+          <message><![CDATA[One or more child tests had errors]]></message>
+          <stack-trace />
+        </failure>
+        <results>
+          <test-suite type="TestSuite" name="Xamarin" executed="True" result="Failure" success="False" time="3393.250" asserts="7560">
+            <failure>
+              <message><![CDATA[One or more child tests had errors]]></message>
+              <stack-trace />
+            </failure>
+            <results>
+              <test-suite type="TestFixture" name="MTouch" executed="True" result="Failure" success="False" time="2720.409" asserts="576">
+                <failure>
+                  <message><![CDATA[One or more child tests had errors]]></message>
+                  <stack-trace />
+                </failure>
+                <results>
+                  <test-suite type="ParameterizedTest" name="FastDev_LinkAll" executed="True" result="Failure" success="False" time="16.633" asserts="2">
+                    <failure>
+                      <message><![CDATA[One or more child tests had errors]]></message>
+                      <stack-trace />
+                    </failure>
+                    <results>
+                      <test-case name="Xamarin.MTouch.FastDev_LinkAll(iOS)" executed="True" result="Failure" success="False" time="8.162" asserts="1">
+                        <failure>
+                          <message>message</message>
+                          <stack-trace>stacktrace</stack-trace>
+                        </failure>
+                      </test-case>
+                    </results>
+                  </test-suite>
+                  <test-case name="Xamarin.MTouch.RebuildWhenReferenceSymbolsInCode" executed="True" result="Failure" success="False" time="0.443" asserts="0">
+                    <failure>
+                      <message>message</message>
+                      <stack-trace>stacktrace</stack-trace>
+                    </failure>
+                  </test-case>
+                </results>
+              </test-suite>
+            </results>
+          </test-suite>
+        </results>
+      </test-suite>
+    </results>
+  </test-suite>
+</test-results>
+
diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs
@@ -361,9 +361,31 @@ public void NUnitV2GenerateTestReport ()
 message</div>
 </li>
 <li>
-ErrorTest2: SingleLineErrorMessage</li>
-<li>
 NUnit.Tests.Assemblies.MockTestFixture.FailingTest: Intentional failure</li>
+<li>
+NUnit.Tests.Assemblies.MockTestFixture.TestWithException: System.ApplicationException : Intentional Exception</li>
+</ul>
+</div>
+";
+			Assert.AreEqual (expectedOutput, writer.ToString (), "Output");
+		}
+
+
+		[Test]
+		public void NUnitV2GenerateTestReportWithTestCaseFailures ()
+		{
+			using var writer = new StringWriter ();
+			using var stream = GetType ().Assembly.GetManifestResourceStream ("Microsoft.DotNet.XHarness.iOS.Shared.Tests.Samples.TestCaseFailures.xml");
+			using var reader = new StreamReader (stream);
+			resultParser.GenerateTestReport (writer, reader, XmlResultJargon.NUnitV2);
+			Console.WriteLine (writer.ToString ());
+			var expectedOutput =
+@"<div style='padding-left: 15px;'>
+<ul>
+<li>
+Xamarin.MTouch.FastDev_LinkAll(iOS): message</li>
+<li>
+Xamarin.MTouch.RebuildWhenReferenceSymbolsInCode: message</li>
 </ul>
 </div>
 ";

diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs
@@ -518,7 +518,14 @@ static void GenerateNUnitV2TestReport (TextWriter writer, XmlReader reader)
 			void write_failure ()
 			{
 				var name = reader ["name"];
-				var message = reader.ReadToDescendant ("message") ? reader.ReadElementContentAsString () : null;
+				string? message = null;
+				var depth = reader.Depth;
+				if (reader.ReadToDescendant ("message")) {
+					message = reader.ReadElementContentAsString ();
+					// ReadToParent
+					while (depth > reader.Depth && reader.Read ()) {
+					}
+				}
 				var message_block = message?.IndexOf ('\n') >= 0;
 				writer.WriteLine ("<li>");
 				writer.Write (name.AsHtml ());
@@ -544,16 +551,18 @@ void write_failure ()
 				if (result == "Error")
 					write_failure ();
 
-				if (!reader.ReadToDescendant ("test-case"))
-					continue;
+				var depth = reader.Depth;
 
-				do {
-					result = reader ["result"];
-					if (result != "Error" && result != "Failure")
-						continue;
+				while (reader.Read ()) {
+					if (reader.NodeType != XmlNodeType.Element || reader.Name != "test-case")
+							continue;
 
-					write_failure ();
-				} while (reader.ReadToNextSibling ("test-case"));
+					result = reader ["result"];
+					if (result == "Error" || result == "Failure")
+						write_failure ();
+					if (reader.Depth < depth)
+						break;
+				}
 			}
 
 			writer.WriteLine ("</ul>");