Blame acceptance tests (#2558)

* Upload on session end when HangDump started In rare cases where we have a crashing tests and short hang timeout, we get into situation where hang dumper started dumping, but then session ends because of the crash, and dumps are not uploaded because the session end is only uploading when there is crash dump enabled. After this change we look for any dump in any case and upload all the dumps that hang dumper was able to create. * Fix procdump condition * Move TestAssets.sln * Blame tests * Add acceptance tests for dump collecting * Fix build error * Install 3.1.0 runtime * Update to recent version of .NET5.0-rc1 that has the appropriate dumping code in runtime * Correct versions everywhere * Fix timeout
microsoft · Sep 16, 2020 · 1b1d0ca · 1b1d0ca
1 parent 50e6af6
commit 1b1d0ca
Show file tree

Hide file tree

Showing 24 changed files with 806 additions and 60 deletions.
diff --git a/global.json b/global.json
@@ -1,12 +1,12 @@
 {
   "sdk": {
-    "version": "5.0.100-rc.1.20380.12",
+    "version": "5.0.100-rc.1.20453.7",
     "rollForward": "minor",
     "allowPrerelease": false,
     "architecture": "x64"
   },
   "tools": {
-    "dotnet": "5.0.100-rc.1.20380.12"
+    "dotnet": "5.0.100-rc.1.20453.7"
   },
   "msbuild-sdks": {
     "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.20465.6",

diff --git a/scripts/build.ps1 b/scripts/build.ps1
@@ -82,7 +82,7 @@ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1
 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712
 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR
 $env:NUGET_EXE_Version = "3.4.3"
-$env:DOTNET_CLI_VERSION = "5.0.100-rc.1.20380.12"
+$env:DOTNET_CLI_VERSION = "5.0.100-rc.1.20453.7"
 # $env:DOTNET_RUNTIME_VERSION = "LATEST"
 $env:VSWHERE_VERSION = "2.0.2"
 $env:MSBUILD_VERSION = "15.0"
@@ -174,14 +174,17 @@ function Install-DotNetCli
     New-Item -ItemType directory -Path $dotnetInstallPath -Force | Out-Null
     & $dotnetInstallScript -Channel "master" -InstallDir $dotnetInstallPath -Version $env:DOTNET_CLI_VERSION
 
-    # Pull in additional shared frameworks.
-    # Get netcoreapp2.1 shared components.
-
     & $dotnetInstallScript -InstallDir "$dotnetInstallPath" -Runtime 'dotnet' -Version '2.1.0' -Channel '2.1.0' -Architecture x64 -NoPath
     $env:DOTNET_ROOT= $dotnetInstallPath
 
     & $dotnetInstallScript -InstallDir "${dotnetInstallPath}_x86" -Runtime 'dotnet' -Version '2.1.0' -Channel '2.1.0' -Architecture x86 -NoPath
     ${env:DOTNET_ROOT(x86)} = "${dotnetInstallPath}_x86"
+
+    & $dotnetInstallScript -InstallDir "$dotnetInstallPath" -Runtime 'dotnet' -Version '3.1.0' -Channel '3.1.0' -Architecture x64 -NoPath
+    $env:DOTNET_ROOT= $dotnetInstallPath
+
+    & $dotnetInstallScript -InstallDir "${dotnetInstallPath}_x86" -Runtime 'dotnet' -Version '3.1.0' -Channel '3.1.0' -Architecture x86 -NoPath
+    ${env:DOTNET_ROOT(x86)} = "${dotnetInstallPath}_x86"
 
     $env:DOTNET_MULTILEVEL_LOOKUP=0
 

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -111,7 +111,7 @@ VERSION=$(test -z $VERSION && grep TPVersionPrefix $TP_ROOT_DIR/scripts/build/Te
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 # Dotnet build doesnt support --packages yet. See https://github.com/dotnet/cli/issues/2712
 export NUGET_PACKAGES=$TP_PACKAGES_DIR
-DOTNET_CLI_VERSION="5.0.100-rc.1.20380.12"
+DOTNET_CLI_VERSION="5.0.100-rc.1.20453.7"
 #DOTNET_RUNTIME_VERSION="LATEST"
 
 #

diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs
@@ -4,7 +4,8 @@
 namespace Microsoft.TestPlatform.AcceptanceTests
 {
     using System;
-
+    using System.Collections.Generic;
+    using System.Linq;
     using Microsoft.TestPlatform.TestUtilities;
 
     public class AcceptanceTestBase : IntegrationTestBase
@@ -21,6 +22,7 @@ public class AcceptanceTestBase : IntegrationTestBase
         public const string DesktopTargetFramework = "net451";
         public const string Core21TargetFramework = "netcoreapp2.1";
         public const string Core31TargetFramework = "netcoreapp3.1";
+        public const string Core50TargetFramework = "net5.0";
 
         public const string DesktopFrameworkArgValue = ".NETFramework,Version=v4.5.1";
         public const string Net451FrameworkArgValue = ".NETFramework,Version=v4.5.1";
@@ -35,10 +37,23 @@ public class AcceptanceTestBase : IntegrationTestBase
 
         public const string Core21FrameworkArgValue = ".NETCoreApp,Version=v2.1";
         public const string Core31FrameworkArgValue = ".NETCoreApp,Version=v3.1";
+        public const string Core50FrameworkArgValue = ".NETCoreApp,Version=v5.0";
+
         public const string DesktopRunnerTargetRuntime = "win7-x64";
         public const string CoreRunnerTargetRuntime = "";
         public const string InIsolation = "/InIsolation";
 
+        public const string NETFX452_48 = "net452;net461;net472;net48";
+        public const string NETFX451_48 = "net452;net461;net472;net48";
+        public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0";
+        public const string NETFX452_NET50 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0";
+        public const string NETFX452_NET31 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1";
+
+        public static string And(string left, string right)
+        {
+            return string.Join(";", left, right);
+        }
+
         protected string FrameworkArgValue => DeriveFrameworkArgValue(this.testEnvironment);
 
         protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo)
@@ -56,6 +71,8 @@ protected static string DeriveFrameworkArgValue(IntegrationTestEnvironment testE
                     return Core21FrameworkArgValue;
                 case Core31TargetFramework:
                     return Core31FrameworkArgValue;
+                case Core50TargetFramework:
+                    return Core50FrameworkArgValue;
                 case Net451TargetFramework:
                     return Net451FrameworkArgValue;
                 case Net452TargetFramework:

diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs
@@ -5,14 +5,20 @@ namespace Microsoft.TestPlatform.AcceptanceTests
 {
     using Microsoft.VisualStudio.TestTools.UnitTesting;
     using System;
+    using System.Collections.Generic;
     using System.IO;
+    using System.Linq;
+    using System.Text;
     using System.Xml;
 
     [TestClass]
     public class BlameDataCollectorTests : AcceptanceTestBase
     {
         private readonly string resultsDir;
 
+        public const string NETCOREANDFX = "net452;net472;netcoreapp3.1";
+        public const string NET50 = "net5.0";
+
         public BlameDataCollectorTests()
         {
             this.resultsDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
@@ -61,6 +67,182 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo)
             this.VaildateOutput(true);
         }
 
+        [TestMethod]
+        [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")]
+        // should make no difference, keeping for easy debug 
+        // [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")]
+        public void HangDumpOnTimeout(RunnerInfo runnerInfo)
+        {
+            Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"));
+
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+            var assemblyPaths = this.GetAssetFullPath("timeout.dll");
+            var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
+            arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s""");
+            this.InvokeVsTest(arguments);
+
+            this.ValidateDump();
+        }
+
+        [TestMethod]
+        // net5.0 does not suppord dump on exit
+        [NetFrameworkRunner("net452;net472;netcoreapp3.1")]
+        // should make no difference, keeping for easy debug 
+        // [NetCoreRunner("net452;net472;netcoreapp3.1")]
+        public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo)
+        {
+            Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"));
+
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+            var assemblyPaths = this.GetAssetFullPath("timeout.dll");
+            var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
+            arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump""");
+            this.InvokeVsTest(arguments);
+
+            this.ValidateDump();
+        }
+
+        [TestMethod]
+        // net5.0 does not suppord dump on exit
+        [NetFrameworkRunner("net452;net472;netcoreapp3.1")]
+        // should make no difference, keeping for easy debug 
+        // [NetCoreRunner("net452;net472;netcoreapp3.1")]
+        public void CrashDumpOnExit(RunnerInfo runnerInfo)
+        {
+            Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"));
+
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+            var assemblyPaths = this.GetAssetFullPath("timeout.dll");
+            var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
+            arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true""");
+            this.InvokeVsTest(arguments);
+
+            this.ValidateDump();
+        }
+
+        [TestMethod]
+        [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")]
+        // should make no difference, keeping for easy debug 
+        // [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")]
+        public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo)
+        {
+            Environment.SetEnvironmentVariable("PROCDUMP_PATH", Path.Combine(this.testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"));
+
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+            var assemblyPaths = this.GetAssetFullPath("crash.dll");
+            var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
+            arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full""");
+            this.InvokeVsTest(arguments);
+
+            this.ValidateDump();
+        }
+
+        [TestMethod]
+        [NetFrameworkRunner(NET50)]
+        // should make no difference, keeping for easy debug 
+        // [NetCoreRunner(NET50)]
+        public void CrashDumpChildProcesses(RunnerInfo runnerInfo)
+        {
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+            var assemblyPaths = this.GetAssetFullPath("child-crash.dll");
+            var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
+            arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full""");
+            this.InvokeVsTest(arguments);
+
+            this.ValidateDump(2);
+        }
+
+        [TestMethod]
+        [NetFrameworkRunner("net452;net472;netcoreapp3.1;net5.0")]
+        // should make no difference, keeping for easy debug 
+        // [NetCoreRunner("net452;net472;netcoreapp3.1;net5.0")]
+        public void HangDumpChildProcesses(RunnerInfo runnerInfo)
+        {
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+            var assemblyPaths = this.GetAssetFullPath("child-hang.dll");
+            var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
+            arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s""");
+            this.InvokeVsTest(arguments);
+
+            this.ValidateDump(2);
+        }
+
+
+        private void ValidateDump(int expectedDumpCount = 1)
+        {
+            var attachments = this.StdOutWithWhiteSpace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)
+                .SkipWhile(l => !l.Contains("Attachments:")).Skip(1)
+                .Where(l => !string.IsNullOrWhiteSpace(l))
+                .ToList();
+
+            var output = string.Join(Environment.NewLine, attachments);
+            if (!attachments.Any(a => a.Contains("Sequence_")))
+            {
+                // sequence file is pretty flaky, and easily substituted by diag log
+                // throw new AssertFailedException("Expected Sequence file in Attachments, but there was none." 
+                //    + Environment.NewLine 
+                //    + output);
+            }
+
+            var dumps = attachments
+                .Where(a => a.EndsWith(".dmp"))
+                // On Windows we might collect conhost which tells us nothing 
+                // or WerFault in case we would start hanging during crash
+                // we don't want these to make cross-platform checks more difficult
+                // so we filter them out.
+                .Where(a => !a.Contains("WerFault") && !a.Contains("conhost"))
+                .Select(a => a.Trim()).ToList();
+
+            if (dumps.Count < expectedDumpCount)
+            {
+                throw new AssertFailedException($"Expected at least {expectedDumpCount} dump file in Attachments, but there were {dumps.Count}." 
+                    + Environment.NewLine 
+                    + string.Join(Environment.NewLine, dumps));
+            }
+
+            var nonExistingDumps = new List<string>();
+            var emptyDumps = new List<string>();
+            foreach (var dump in dumps)
+            {
+                if (!File.Exists(dump))
+                {
+                    nonExistingDumps.Add(dump);
+                }
+                else
+                {
+                    var file = new FileInfo(dump);
+                    if (file.Length == 0)
+                    {
+                        emptyDumps.Add(dump);
+                    }
+                }
+            }
+
+            // allow some child dumps to be missing, they manage to terminate early from time to time
+            if ((dumps.Count == 1 && nonExistingDumps.Any()) || (dumps.Count > 1 && nonExistingDumps.Count > 1)
+                || emptyDumps.Any())
+            {
+                var err = new StringBuilder();
+                err.AppendLine("Expected all dumps in the list of attachments to exist, and not be empty, but:");
+                if (nonExistingDumps.Any())
+                {
+                    err.AppendLine($"{nonExistingDumps.Count} don't exist:")
+                    .AppendLine(string.Join(Environment.NewLine, nonExistingDumps));
+                }
+
+                if (emptyDumps.Any())
+                {
+                    err.AppendLine($"{emptyDumps.Count} are empty:")
+                    .AppendLine(string.Join(Environment.NewLine, emptyDumps));
+                }
+
+                err.AppendLine("Reported attachments:")
+                .AppendLine(output);
+
+                throw new AssertFailedException(err.ToString());
+            }
+        }
+
         private void VaildateOutput(bool validateDumpFile = false)
         {
             bool isSequenceAttachmentReceived = false;
@@ -70,7 +252,7 @@ private void VaildateOutput(bool validateDumpFile = false)
             this.StdOutputContains("Sequence_");
             var resultFiles = Directory.GetFiles(this.resultsDir, "*", SearchOption.AllDirectories);
 
-            foreach(var file in resultFiles)
+            foreach (var file in resultFiles)
             {
                 if (file.Contains("Sequence_"))
                 {

diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs
@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Globalization;
+    using System.Reflection;
+    using TestUtilities;
+    using VisualStudio.TestTools.UnitTesting;
+
+    /// <summary>
+    /// Runs tests using the dotnet vstest.console.dll built against .NET Core 2.1.
+    /// Provide a list of target frameworks to run the tests from given as a ';' separated list, or using a constant containing that range such as 
+    /// AcceptanceTestBase.NETFX452_NET50 = "net452;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0" to determine which target framework of the project
+    /// to test. The target project must list those TFMs in the TargetFrameworks property in csproj.
+    /// </summary>
+    public class NetCoreRunner : Attribute, ITestDataSource
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="NetCoreTargetFrameworkDataSource"/> class.
+        /// </summary>
+        /// <param name="targetFrameworks">To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values.</param>
+        public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50)
+        {
+            foreach (var fmw in targetFrameworks.Split(';'))
+            {
+                this.dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw) });
+            }
+
+        }
+
+        private List<object[]> dataRows = new List<object[]>();
+
+        public IEnumerable<object[]> GetData(MethodInfo methodInfo)
+        {
+            return this.dataRows;
+        }
+
+        public string GetDisplayName(MethodInfo methodInfo, object[] data)
+        {
+            return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data));
+        }
+    }
+}
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs
@@ -63,18 +63,6 @@ public NetCoreTargetFrameworkDataSource(
             }
         }
 
-        /// <summary>
-        /// Initializes a new instance of the <see cref="NetCoreTargetFrameworkDataSource"/> class.
-        /// </summary>
-        /// <param name="targetFrameworks">To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values.</param>
-        public NetCoreTargetFrameworkDataSource(string[] targetFrameworks)
-        {
-            foreach (var fmw in targetFrameworks)
-            {
-                this.AddRunnerDataRow(IntegrationTestBase.CoreRunnerFramework, fmw);
-            }
-        }
-
         private void AddRunnerDataRow(string runnerFramework, string targetFramework)
         {
             var runnerInfo = new RunnerInfo(runnerFramework, targetFramework);