Use RunSettingsFilePath from when using dotnet test (#2272)

Uses the RunSettingsFilePath from project file when specified and falls
back to the one specified in console, or the default setting if none are
provided.

The path to the settings file can be specified in multiple ways,
for example as a path relative to the project file. This enables
`dotnet test` to be invoked against the project file directly.

```xml
<RunSettingsFilePath>$(ProjectDir)..\example.runsettings</RunSettingsFilePath>
```

Alternatively path can also be specified relatively to solution file, but that will
only work when running dotnet test against the solution. On the other hand the
setting will be the same in all projects.

```xml
<RunSettingsFilePath>$(SolutionDir)\example.runsettings</RunSettingsFilePath>
```

Fixes #2265

TestPlatform.sln

@@ -179,7 +179,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Exte
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests", "test\Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests\Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj", "{41248B96-6E15-4E5E-A78F-859897676814}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "coverlet.collector", "test\TestAssets\coverlet.collector\coverlet.collector.csproj", "{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.collector", "test\TestAssets\coverlet.collector\coverlet.collector.csproj", "{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProjectFileRunSettingsTestProject", "test\TestAssets\ProjectFileRunSettingsTestProject\ProjectFileRunSettingsTestProject.csproj", "{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -911,6 +913,18 @@ Global
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}.Release|x64.Build.0 = Release|Any CPU
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}.Release|x86.ActiveCfg = Release|Any CPU
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3}.Release|x86.Build.0 = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x64.Build.0 = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Debug|x86.Build.0 = Debug|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x64.ActiveCfg = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x64.Build.0 = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x86.ActiveCfg = Release|Any CPU
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -988,6 +1002,7 @@ Global
 		{236A71E3-01DA-4679-9DFF-16A8E079ACFF} = {5E7F18A8-F843-4C8A-AB02-4C7D9205C6CF}
 		{41248B96-6E15-4E5E-A78F-859897676814} = {020E15EA-731F-4667-95AF-226671E0C3AE}
 		{F1D8630D-97D5-4CD7-BC18-A5E1779FA6E3} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
+		{8E87F6E4-E884-4404-B2E5-CBFB28038AE5} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD}

scripts/build.ps1

@@ -213,6 +213,19 @@ function Invoke-Build
     Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}"
 }
 
+function Publish-PatchedDotnet { 
+    Write-Log "Publish-PatchedDotnet: Copy local dotnet installation to testArtifacts"
+    $dotnetPath = "$env:TP_TOOLS_DIR\dotnet\"
+
+    $dotnetTestArtifactsPath = "$env:TP_TESTARTIFACTS\dotnet\" 
+    $dotnetTestArtifactsSdkPath = "$env:TP_TESTARTIFACTS\dotnet\sdk\$env:DOTNET_CLI_VERSION\"
+    Copy-Item $dotnetPath $dotnetTestArtifactsPath -Force -Recurse
+
+    Write-Log "Publish-PatchedDotnet: Copy VSTest task artifacts to local dotnet installation to allow `dotnet test` to run with it"
+    $buildArtifactsPath = "$env:TP_ROOT_DIR\src\Microsoft.TestPlatform.Build\bin\$TPB_Configuration\$TPB_TargetFrameworkNS2_0\*"
+    Copy-Item $buildArtifactsPath $dotnetTestArtifactsSdkPath -Force
+}
+
 function Publish-Package
 {
     $timer = Start-Timer
@@ -901,9 +914,11 @@ Restore-Package
 Update-LocalizedResources
 Invoke-Build
 Publish-Package
+Publish-PatchedDotnet
 Publish-Tests
 Create-VsixPackage
 Create-NugetPackages
 Generate-Manifest
 Write-Log "Build complete. {$(Get-ElapsedTime($timer))}"
 if ($Script:ScriptFailed) { Exit 1 } else { Exit 0 }
+

src/Microsoft.TestPlatform.Build/Microsoft.TestPlatform.targets

@@ -31,7 +31,7 @@ Copyright (c) .NET Foundation. All rights reserved.
 
     <Microsoft.TestPlatform.Build.Tasks.VSTestTask
       TestFileFullPath="$(TargetPath)"
-      VSTestSetting="$(VSTestSetting)"
+      VSTestSetting="$([MSBuild]::ValueOrDefault($(VSTestSetting), '$(RunSettingsFilePath)'))"
       VSTestTestAdapterPath="$(VSTestTestAdapterPath)"
       VSTestFramework="$(TargetFrameworkMoniker)"
       VSTestPlatform="$(PlatformTarget)"

test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs

@@ -478,6 +478,37 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo
 
         #endregion
 
+        #region RunSettings defined in project file
+        /// <summary>
+        /// RunSettingsFilePath can be specified in .csproj and should be honored by `dotnet test`, this test 
+        /// checks that the settings were honored by translating an inconlusive test to failed "result", instead of the default "skipped".
+        /// This test depends on Microsoft.TestPlatform.Build\Microsoft.TestPlatform.targets being previously copied into the 
+        /// artifacts/testArtifacts/dotnet folder. This will allow the local copy of dotnet to pickup the VSTest msbuild task.
+        /// </summary>
+        /// <param name="runnerInfo"></param>
+        [TestMethod]
+        [NetFullTargetFrameworkDataSource]
+        [NetCoreTargetFrameworkDataSource]
+        public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo)
+        {
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var projectName = "ProjectFileRunSettingsTestProject.csproj";
+            var projectPath = this.GetProjectFullPath(projectName);
+            this.InvokeDotnetTest(projectPath);
+            this.ValidateSummaryStatus(0, 1, 0);
+
+            // make sure that we can revert the project settings back by providing a config from commandline 
+            // keeping this in the same test, because it is easier to see that we are reverting settings that 
+            // are honored by dotnet test, instead of just using the default, which would produce the same 
+            // result
+            var settingsPath = this.GetProjectAssetFullPath(projectName, "inconclusive.runsettings");
+            this.InvokeDotnetTest($"{projectPath} --settings {settingsPath}");
+            this.ValidateSummaryStatus(0, 0, 1);
+        }
+
+        #endregion
+
         private string GetRunsettingsFilePath(Dictionary<string, string> runConfigurationDictionary)
         {
             var runsettingsPath = Path.Combine(

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs

@@ -106,7 +106,18 @@ public static string PrepareArguments(string testAssembly, string testAdapterPat
         /// <param name="arguments">Arguments provided to <c>vstest.console</c>.exe</param>
         public void InvokeVsTest(string arguments)
         {
-            this.Execute(arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
+            this.ExecuteVsTestConsole(arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
+            this.FormatStandardOutCome();
+        }
+
+
+        /// <summary>
+        /// Invokes <c>vstest.console</c> with specified arguments.
+        /// </summary>
+        /// <param name="arguments">Arguments provided to <c>vstest.console</c>.exe</param>
+        public void InvokeDotnetTest(string arguments)
+        {
+            this.ExecutePatchedDotnet("test", arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
             this.FormatStandardOutCome();
         }
 
@@ -353,6 +364,17 @@ protected string GetAssetFullPath(string assetName, string targetFramework)
             return this.testEnvironment.GetTestAsset(assetName, targetFramework);
         }
 
+        protected string GetProjectFullPath(string projectName)
+        {
+            return this.testEnvironment.GetTestProject(projectName);
+        }
+
+        protected string GetProjectAssetFullPath(string projectName, string assetName)
+        {
+            var projectPath = this.testEnvironment.GetTestProject(projectName);
+            return Path.Combine(Path.GetDirectoryName(projectPath), assetName);
+        } 
+
         protected string GetTestAdapterPath(UnitTestFramework testFramework = UnitTestFramework.MSTest)
         {
             string adapterRelativePath = string.Empty;
@@ -478,7 +500,7 @@ private static string GetTestMethodName(string testFullName)
             return testMethodName;
         }
 
-        private void Execute(string args, out string stdOut, out string stdError, out int exitCode)
+        private void ExecuteVsTestConsole(string args, out string stdOut, out string stdError, out int exitCode)
         {
             if (this.IsNetCoreRunner())
             {
@@ -487,46 +509,88 @@ private void Execute(string args, out string stdOut, out string stdError, out in
 
             this.arguments = args;
 
-            using (Process vstestconsole = new Process())
+            this.ExecuteApplication(this.GetConsoleRunnerPath(), args, out stdOut, out stdError, out exitCode);
+        }
+
+        /// <summary>
+        /// Executes a local copy of dotnet that has VSTest task installed and possibly other modifications. Do not use this to 
+        /// do your builds or to run general tests, unless you want your changes to be reflected.
+        /// </summary>
+        /// <param name="command"></param>
+        /// <param name="args"></param>
+        /// <param name="stdOut"></param>
+        /// <param name="stdError"></param>
+        /// <param name="exitCode"></param>
+        private void ExecutePatchedDotnet(string command, string args, out string stdOut, out string stdError, out int exitCode)
+        {
+            var environmentVariables = new Dictionary<string, string> {
+                ["DOTNET_MULTILEVEL_LOOKUP"] = "0" 
+            };
+
+            var patchedDotnetPath = Path.Combine(this.testEnvironment.TestArtifactsDirectory, @"dotnet\dotnet.exe"); ;
+            this.ExecuteApplication(patchedDotnetPath, string.Join(" ", command, args), out stdOut, out stdError, out exitCode, environmentVariables);
+        }
+
+        private void ExecuteApplication(string path, string args, out string stdOut, out string stdError, out int exitCode, Dictionary<string, string> environmentVariables = null)
+        {
+            if (string.IsNullOrWhiteSpace(path))
+            {
+                throw new ArgumentException("Executable path must not be null or whitespace.", nameof(path));
+            }
+
+            var executableName = Path.GetFileName(path);
+
+            using (Process process = new Process())
             {
-                Console.WriteLine("IntegrationTestBase.Execute: Starting vstest.console.exe");
-                vstestconsole.StartInfo.FileName = this.GetConsoleRunnerPath();
-                vstestconsole.StartInfo.Arguments = args;
-                vstestconsole.StartInfo.UseShellExecute = false;
+                Console.WriteLine($"IntegrationTestBase.Execute: Starting {executableName}");
+                process.StartInfo.FileName = path;
+                process.StartInfo.Arguments = args;
+                process.StartInfo.UseShellExecute = false;
                 //vstestconsole.StartInfo.WorkingDirectory = testEnvironment.PublishDirectory;
-                vstestconsole.StartInfo.RedirectStandardError = true;
-                vstestconsole.StartInfo.RedirectStandardOutput = true;
-                vstestconsole.StartInfo.CreateNoWindow = true;
-                vstestconsole.StartInfo.StandardOutputEncoding = Encoding.UTF8;
-                vstestconsole.StartInfo.StandardErrorEncoding = Encoding.UTF8;
+                process.StartInfo.RedirectStandardError = true;
+                process.StartInfo.RedirectStandardOutput = true;
+                process.StartInfo.CreateNoWindow = true;
+                process.StartInfo.StandardOutputEncoding = Encoding.UTF8;
+                process.StartInfo.StandardErrorEncoding = Encoding.UTF8;
+                if (environmentVariables != null) {
+                    foreach (var variable in environmentVariables) {
+                        if (process.StartInfo.EnvironmentVariables.ContainsKey(variable.Key)) {
+                            process.StartInfo.EnvironmentVariables[variable.Key] = variable.Value;
+                        } 
+                        else
+                        {
+                            process.StartInfo.EnvironmentVariables.Add(variable.Key, variable.Value);
+                        }
+                    }
+                }
 
                 var stdoutBuffer = new StringBuilder();
                 var stderrBuffer = new StringBuilder();
-                vstestconsole.OutputDataReceived += (sender, eventArgs) =>
+                process.OutputDataReceived += (sender, eventArgs) =>
                 {
                     stdoutBuffer.Append(eventArgs.Data).Append(Environment.NewLine);
                 };
 
-                vstestconsole.ErrorDataReceived += (sender, eventArgs) => stderrBuffer.Append(eventArgs.Data).Append(Environment.NewLine);
+                process.ErrorDataReceived += (sender, eventArgs) => stderrBuffer.Append(eventArgs.Data).Append(Environment.NewLine);
 
-                Console.WriteLine("IntegrationTestBase.Execute: Path = {0}", vstestconsole.StartInfo.FileName);
-                Console.WriteLine("IntegrationTestBase.Execute: Arguments = {0}", vstestconsole.StartInfo.Arguments);
+                Console.WriteLine("IntegrationTestBase.Execute: Path = {0}", process.StartInfo.FileName);
+                Console.WriteLine("IntegrationTestBase.Execute: Arguments = {0}", process.StartInfo.Arguments);
 
                 Stopwatch stopwatch = new Stopwatch();
                 stopwatch.Start();
 
-                vstestconsole.Start();
-                vstestconsole.BeginOutputReadLine();
-                vstestconsole.BeginErrorReadLine();
-                if (!vstestconsole.WaitForExit(80 * 1000))
+                process.Start();
+                process.BeginOutputReadLine();
+                process.BeginErrorReadLine();
+                if (!process.WaitForExit(80 * 1000))
                 {
-                    Console.WriteLine("IntegrationTestBase.Execute: Timed out waiting for vstest.console.exe. Terminating the process.");
-                    vstestconsole.Kill();
+                    Console.WriteLine($"IntegrationTestBase.Execute: Timed out waiting for {executableName}. Terminating the process.");
+                    process.Kill();
                 }
                 else
                 {
                     // Ensure async buffers are flushed
-                    vstestconsole.WaitForExit();
+                    process.WaitForExit();
                 }
 
                 stopwatch.Stop();
@@ -535,11 +599,11 @@ private void Execute(string args, out string stdOut, out string stdError, out in
 
                 stdError = stderrBuffer.ToString();
                 stdOut = stdoutBuffer.ToString();
-                exitCode = vstestconsole.ExitCode;
+                exitCode = process.ExitCode;
 
                 Console.WriteLine("IntegrationTestBase.Execute: stdError = {0}", stdError);
                 Console.WriteLine("IntegrationTestBase.Execute: stdOut = {0}", stdOut);
-                Console.WriteLine("IntegrationTestBase.Execute: Stopped vstest.console.exe. Exit code = {0}", exitCode);
+                Console.WriteLine($"IntegrationTestBase.Execute: Stopped {executableName}. Exit code = {0}", exitCode);
             }
         }
 

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs

@@ -57,6 +57,7 @@ public IntegrationTestEnvironment()
             // Need to remove this assumption when we move to a CDP.
             this.PackageDirectory = Path.Combine(TestPlatformRootDirectory, @"packages");
             this.ToolsDirectory = Path.Combine(TestPlatformRootDirectory, @"tools");
+            this.TestArtifactsDirectory = Path.Combine(TestPlatformRootDirectory, "artifacts", "testArtifacts");
             this.RunnerFramework = "net451";
         }
 
@@ -193,6 +194,15 @@ public string ToolsDirectory
             private set;
         }
 
+        /// <summary>
+        /// Gets the test artifacts directory.
+        /// </summary>
+        public string TestArtifactsDirectory
+        {
+            get;
+            private set;
+        }
+
         /// <summary>
         /// Gets the application type.
         /// Supported values = <c>net451</c>, <c>netcoreapp1.0</c>.
@@ -293,5 +303,30 @@ private static Dictionary<string, string> GetDependencies(string testPlatformRoo
 
             return dependencyProps;
         }
+
+        /// <summary>
+        /// Gets the full path to a test asset.
+        /// </summary>
+        /// <param name="assetName">Name of the asset with extension. E.g. <c>SimpleUnitTest.csproj</c></param>
+        /// <returns>Full path to the test asset.</returns>
+        /// <remarks>
+        /// Test assets follow several conventions:
+        /// (a) They are built for supported frameworks. See <see cref="TargetFramework"/>.
+        /// (b) They are built for provided build configuration.
+        /// (c) Name of the test asset matches the parent directory name. E.g. <c>TestAssets\SimpleUnitTest\SimpleUnitTest.csproj</c> must
+        /// produce <c>TestAssets\SimpleUnitTest\SimpleUnitTest.csproj</c>
+        /// </remarks>
+        public string GetTestProject(string assetName)
+        {
+            var simpleAssetName = Path.GetFileNameWithoutExtension(assetName);
+            var assetPath = Path.Combine(
+                this.TestAssetsPath,
+                simpleAssetName,
+                assetName);
+
+            Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath);
+
+            return assetPath;
+        }
     }
 }

test/TestAssets/ProjectFileRunSettingsTestProject/ProjectFileRunSettingsTestProject.csproj

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <!-- Imports Common TestAssets props. -->
+  <Import Project="..\..\..\scripts\build\TestAssets.props" />
+  <Import Project="..\..\..\scripts\build\TestPlatform.Dependencies.props" />
+  <!-- Package dependency versions -->
+
+  <PropertyGroup>
+    <TargetFrameworks>netcoreapp2.1;net451</TargetFrameworks>
+    <PlatformTarget>x64</PlatformTarget>
+    <RunSettingsFilePath>fail.runsettings</RunSettingsFilePath>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="MSTest.TestFramework">
+      <Version>$(MSTestFrameworkVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="MSTest.TestAdapter">
+      <Version>$(MSTestAdapterVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk">
+      <Version>$(NETTestSdkPreviousVersion)</Version>
+    </PackageReference>
+  </ItemGroup>
+  <ItemGroup Condition=" '$(TargetFramework)' == 'net451' ">
+    <Reference Include="System.Runtime" />
+    <Reference Include="System.Windows.Forms" />
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+</Project>

test/TestAssets/ProjectFileRunSettingsTestProject/UnitTest1.cs

@@ -0,0 +1,20 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace ProjectFileRunSettingsTestProject
+{
+    [TestClass]
+    public class UnitTest1
+    {
+        [TestMethod]
+        public void TestMethod1()
+        {
+            // this project specifies runsettings in it's proj file
+            // that runsettings say that inconclusive should translate to 
+            // failed. 
+            // we can then easily figure out if the settings were applied 
+            // correctly if we set the test as failed, or did not apply if the 
+            // test is shown as skipped
+            Assert.Inconclusive();
+        }
+    }
+}