Add screen capture and improved logging to helix tests (#176)

Enable /screencaptureonerror for TAEF tests and upload the screenshots to helix storage.
Also upload te.wtl log file.

Report uploaded urls of both of these to the Azure DevOps test report so that someone investigating a test failure can link directly to them instead of having to go to through the Mission Control site.

Here is an example run showing this in action:
https://dev.azure.com/ms/microsoft-ui-xaml/_build/results?buildId=1332&view=ms.vss-test-web.test-result-details

Author: kmahone

build/Helix/ConvertWttLogToXUnit.cs

@@ -1,7 +1,8 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Xml.Linq;
 
 namespace HelixTestHelpers
@@ -16,14 +17,21 @@ public class TestPass
 
         public class TestResult
         {
+            public TestResult()
+            {
+                Screenshots = new List<string>();
+            }
+
             public string Name { get; set; }
             public bool Passed { get; set; }
             public bool CleanupPassed { get; set; }
             public TimeSpan ExecutionTime { get; set; }
             public string Details { get; set; }
+
+            public List<string> Screenshots { get; private set; }
         }
 
-        public static void ConvertWttLogToXUnitLog(string wttInputPath, string xunitOutputPath, string testNamePrefix)
+        public static void ConvertWttLogToXUnitLog(string wttInputPath, string xunitOutputPath, string testNamePrefix, string helixResultsContainerUri, string helixResultsContainerRsas)
         {
             var testPass = TestResultParser.ParseTestWttFile(wttInputPath, true);
             var results = testPass.TestResults;
@@ -80,7 +88,24 @@ public static void ConvertWttLogToXUnitLog(string wttInputPath, string xunitOutp
                     failure.SetAttributeValue("exception-type", "Exception");
 
                     var message = new XElement("message");
-                    message.Add(new XCData(result.Details));
+
+                    StringBuilder errorMessage = new StringBuilder();
+
+                    errorMessage.AppendLine("Log: " + GetUploadedFileUrl(wttInputPath, helixResultsContainerUri, helixResultsContainerRsas));
+                    
+                    if(result.Screenshots.Any())
+                    {
+                        errorMessage.AppendLine("Screenshots:");
+                        foreach(var screenshot in result.Screenshots)
+                        {
+                            errorMessage.AppendLine(GetUploadedFileUrl(screenshot, helixResultsContainerUri, helixResultsContainerRsas));
+                        }
+                    }
+
+                    errorMessage.AppendLine("Error Log: ");
+                    errorMessage.AppendLine(result.Details);
+
+                    message.Add(new XCData(errorMessage.ToString()));
                     failure.Add(message);
 
                     test.Add(failure);
@@ -272,6 +297,26 @@ public static TestPass ParseTestWttFile(string fileName, bool cleanupFailuresAre
                             currentResult.Details += "]";
                         }
                     }
+
+                    if (currentResult != null && element.Name == "Msg")
+                    {
+                        var dataElement = element.Element("Data");
+                        if (dataElement != null)
+                        {
+                            var supportingInfo = dataElement.Element("SupportingInfo");
+                            if (supportingInfo != null)
+                            {
+                                var screenshots = supportingInfo.Elements("Item")
+                                    .Where(item => GetAttributeValue(item, "Name") == "Screenshot")
+                                    .Select(item => GetAttributeValue(item, "Value"));
+
+                                foreach(var screenshot in screenshots)
+                                {
+                                    currentResult.Screenshots.Add(screenshot);
+                                }
+                            }
+                        }
+                    }
                 }
 
                 testPassStartTime = Int64.Parse(doc.Root.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value);
@@ -288,5 +333,23 @@ public static TestPass ParseTestWttFile(string fileName, bool cleanupFailuresAre
                 return testpass;
             }
         }
+
+        private static string GetAttributeValue(XElement element, string attributeName)
+        {
+            if(element.Attribute(attributeName) != null)
+            {
+                return element.Attribute(attributeName).Value;
+            }
+
+            return null;
+        }
+
+        private static string GetUploadedFileUrl(string filePath, string helixResultsContainerUri, string helixResultsContainerRsas)
+        {
+            var filename = Path.GetFileName(filePath);
+            return string.Format("{0}/{1}{2}", helixResultsContainerUri, filename, helixResultsContainerRsas);
+        }
     }
+
+
 }

build/Helix/ConvertWttLogToXUnit.ps1

@@ -9,6 +9,11 @@ Param(
     [string]$testNamePrefix
 )
 
+# Ideally these would be passed as parameters to the script. However ps makes it difficult to deal with string literals containing '&', so we just 
+# read the values directly from the environment variables
+$helixResultsContainerUri = $Env:HELIX_RESULTS_CONTAINER_URI
+$helixResultsContainerRsas = $Env:HELIX_RESULTS_CONTAINER_RSAS
+
 Add-Type -Language CSharp -ReferencedAssemblies System.Xml,System.Xml.Linq (Get-Content .\ConvertWttLogToXUnit.cs -Raw)
 
-[HelixTestHelpers.TestResultParser]::ConvertWttLogToXUnitLog($WttInputPath, $XUnitOutputPath, $testNamePrefix)
+[HelixTestHelpers.TestResultParser]::ConvertWttLogToXUnitLog($WttInputPath, $XUnitOutputPath, $testNamePrefix, $helixResultsContainerUri, $helixResultsContainerRsas)

build/Helix/runtests.cmd

@@ -1,6 +1,13 @@
-robocopy %HELIX_CORRELATION_PAYLOAD% . /s
+robocopy %HELIX_CORRELATION_PAYLOAD% . /s /NP
 
-te MUXControls.Test.dll MUXControlsTestApp.appx IXMPTestApp.appx /enablewttlogging /unicodeOutput:false /sessionTimeout:0:15 /testtimeout:0:10 %*
+te MUXControls.Test.dll MUXControlsTestApp.appx IXMPTestApp.appx /enablewttlogging /unicodeOutput:false /sessionTimeout:0:15 /testtimeout:0:10 /screenCaptureOnError %* 
+
+%HELIX_PYTHONPATH% %HELIX_SCRIPT_ROOT%\upload_result.py -result te.wtl -result_name te.wtl
+
+FOR %%I in (WexLogFileOutput\*.jpg) DO (
+    echo Uploading %%I to "%HELIX_RESULTS_CONTAINER_URI%/%%I%HELIX_RESULTS_CONTAINER_RSAS%"
+    %HELIX_PYTHONPATH% %HELIX_SCRIPT_ROOT%\upload_result.py -result %%I -result_name %%~nI%%~xI 
+)
 
 cd scripts
 powershell -ExecutionPolicy Bypass .\ConvertWttLogToXUnit.ps1 ..\te.wtl ..\testResults.xml %testnameprefix%