diff --git a/NUnitConsole.sln b/NUnitConsole.sln index 6071e231e..e49c3c84c 100644 --- a/NUnitConsole.sln +++ b/NUnitConsole.sln @@ -135,6 +135,16 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WpfTest", "src\TestData\wpf EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WpfApp", "src\TestData\WpfApp\WpfApp.csproj", "{93D182B7-2F63-4661-BB32-94F1716CF03F}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "OlderNUnit", "OlderNUnit", "{D31607D2-D689-488B-9F9F-92F47AC1D7F6}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.0", "src\TestData\NUnit3.0\NUnit3.0.csproj", "{2448CE80-59AA-41B7-83A0-768BE5520F7A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.0.1", "src\TestData\NUnit3.0.1\NUnit3.0.1.csproj", "{56416AD9-8E7B-4F72-B0E1-E75A024431F8}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.2", "src\TestData\NUnit3.2\NUnit3.2.csproj", "{8AFBB856-700A-4CCC-AE0E-9B622178E1E0}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3.10", "src\TestData\NUnit3.10\NUnit3.10.csproj", "{0555B97D-E918-455B-951C-74EFCDA8790A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -229,6 +239,22 @@ Global {93D182B7-2F63-4661-BB32-94F1716CF03F}.Debug|Any CPU.Build.0 = Debug|Any CPU {93D182B7-2F63-4661-BB32-94F1716CF03F}.Release|Any CPU.ActiveCfg = Release|Any CPU {93D182B7-2F63-4661-BB32-94F1716CF03F}.Release|Any CPU.Build.0 = Release|Any CPU + {2448CE80-59AA-41B7-83A0-768BE5520F7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2448CE80-59AA-41B7-83A0-768BE5520F7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2448CE80-59AA-41B7-83A0-768BE5520F7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2448CE80-59AA-41B7-83A0-768BE5520F7A}.Release|Any CPU.Build.0 = Release|Any CPU + {56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {56416AD9-8E7B-4F72-B0E1-E75A024431F8}.Release|Any CPU.Build.0 = Release|Any CPU + {8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8AFBB856-700A-4CCC-AE0E-9B622178E1E0}.Release|Any CPU.Build.0 = Release|Any CPU + {0555B97D-E918-455B-951C-74EFCDA8790A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0555B97D-E918-455B-951C-74EFCDA8790A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0555B97D-E918-455B-951C-74EFCDA8790A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0555B97D-E918-455B-951C-74EFCDA8790A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -266,6 +292,11 @@ Global {48DF1E40-93BA-436A-B460-5D1130316ADA} = {37D508B2-91E0-4B32-869B-DFF9E68EA213} {2F9D8932-2186-464F-BED6-7D7979C8FFA6} = {37D508B2-91E0-4B32-869B-DFF9E68EA213} {93D182B7-2F63-4661-BB32-94F1716CF03F} = {37D508B2-91E0-4B32-869B-DFF9E68EA213} + {D31607D2-D689-488B-9F9F-92F47AC1D7F6} = {37D508B2-91E0-4B32-869B-DFF9E68EA213} + {2448CE80-59AA-41B7-83A0-768BE5520F7A} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6} + {56416AD9-8E7B-4F72-B0E1-E75A024431F8} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6} + {8AFBB856-700A-4CCC-AE0E-9B622178E1E0} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6} + {0555B97D-E918-455B-951C-74EFCDA8790A} = {D31607D2-D689-488B-9F9F-92F47AC1D7F6} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D8E4FC26-5422-4C51-8BBC-D1AC0A578711} diff --git a/package-tests.cake b/package-tests.cake index 7c6ad2179..9bfdfe3b3 100644 --- a/package-tests.cake +++ b/package-tests.cake @@ -35,7 +35,7 @@ class MockAssemblyExpectedResult : ExpectedResult StandardRunnerTests.Add(new PackageTest( 1, "Net462Test", "Run mock-assembly.dll under .NET 4.6.2", - "testdata/net462/mock-assembly.dll", + "testdata/net462/mock-assembly.dll --trace:Debug", new MockAssemblyExpectedResult("net-4.6.2"))); AddToBothLists(new PackageTest( @@ -142,6 +142,46 @@ StandardRunnerTests.Add(new PackageTest( "testdata/net462/mock-assembly.dll testdata/net6.0/mock-assembly.dll", new MockAssemblyExpectedResult("net-4.6.2", "netcore-6.0"))); +////////////////////////////////////////////////////////////////////// +// TEST OLDER VERSIONS OF NUNIT SWITCHING API IF NEEDED +////////////////////////////////////////////////////////////////////// + +StandardRunnerTests.Add(new PackageTest( + 1, "NUnit30Test", + "Run a test under NUnit 3.0 using 2009 API", + "testdata/NUnit3.0/net462/NUnit3.0.dll", + new ExpectedResult("Passed") + { + Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.0.dll", "net462") } + })); + +StandardRunnerTests.Add(new PackageTest( + 1, "NUnit301Test", + "Run a test under NUnit 3.0.1 using 2009 API", + "testdata/NUnit3.0.1/net462/NUnit3.0.1.dll", + new ExpectedResult("Passed") + { + Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.0.1.dll", "net462") } + })); + +StandardRunnerTests.Add(new PackageTest( + 1, "NUnit32Test", + "Run a test under NUnit 3.2 using 20018 API", + "testdata/NUnit3.2/net462/NUnit3.2.dll", + new ExpectedResult("Passed") + { + Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.2.dll", "net462") } + })); + +StandardRunnerTests.Add(new PackageTest( + 1, "NUnit310Test", + "Run a test under NUnit 3.10 using 2018 API", + "testdata/NUnit3.10/net462/NUnit3.10.dll", + new ExpectedResult("Passed") + { + Assemblies = new[] { new ExpectedAssemblyResult("NUnit3.10.dll", "net462") } + })); + ////////////////////////////////////////////////////////////////////// // ASP.NETCORE TESTS ////////////////////////////////////////////////////////////////////// diff --git a/src/NUnitEngine/nunit.engine.api/Extensibility/IDriverFactory.cs b/src/NUnitEngine/nunit.engine.api/Extensibility/IDriverFactory.cs index fda047506..f5b0d050d 100644 --- a/src/NUnitEngine/nunit.engine.api/Extensibility/IDriverFactory.cs +++ b/src/NUnitEngine/nunit.engine.api/Extensibility/IDriverFactory.cs @@ -25,17 +25,19 @@ public interface IDriverFactory /// which the assembly is already known to reference. /// /// The domain in which the assembly will be loaded + /// The driver id. /// An AssemblyName referring to the test framework. /// - IFrameworkDriver GetDriver(AppDomain domain, AssemblyName reference); + IFrameworkDriver GetDriver(AppDomain domain, string id, AssemblyName reference); #else /// /// Gets a driver for a given test assembly and a framework /// which the assembly is already known to reference. /// + /// The driver id. /// An AssemblyName referring to the test framework. /// - IFrameworkDriver GetDriver(AssemblyName reference); + IFrameworkDriver GetDriver(string id, AssemblyName reference); #endif } } diff --git a/src/NUnitEngine/nunit.engine.api/Extensibility/IFrameworkDriver.cs b/src/NUnitEngine/nunit.engine.api/Extensibility/IFrameworkDriver.cs index 3f8f190bb..46ed1f841 100644 --- a/src/NUnitEngine/nunit.engine.api/Extensibility/IFrameworkDriver.cs +++ b/src/NUnitEngine/nunit.engine.api/Extensibility/IFrameworkDriver.cs @@ -18,7 +18,7 @@ public interface IFrameworkDriver /// Gets and sets the unique identifier for this driver, /// used to ensure that test ids are unique across drivers. /// - string ID { get; set; } + string ID { get; } /// /// Loads the tests in an assembly. diff --git a/src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnit3FrameworkDriverTests.cs b/src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnitFrameworkDriverTests.cs similarity index 77% rename from src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnit3FrameworkDriverTests.cs rename to src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnitFrameworkDriverTests.cs index 43d6b0b42..011c6b9c7 100644 --- a/src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnit3FrameworkDriverTests.cs +++ b/src/NUnitEngine/nunit.engine.core.tests/Drivers/NUnitFrameworkDriverTests.cs @@ -1,6 +1,5 @@ // Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt -#if NETFRAMEWORK using System; using System.Collections.Generic; using System.Reflection; @@ -13,22 +12,37 @@ namespace NUnit.Engine.Drivers { // Functional tests of the NUnitFrameworkDriver calling into the framework. - public class NUnit3FrameworkDriverTests +#if NETFRAMEWORK + [TestFixture("2009")] +#endif + [TestFixture("2018")] + public class NUnitFrameworkDriverTests { private const string MOCK_ASSEMBLY = "mock-assembly.dll"; - private const string LOAD_MESSAGE = "Method called without calling Load first"; + private const string LOAD_MESSAGE = "Method called without calling Load first. Possible error in runner."; private IDictionary _settings = new Dictionary(); - private NUnit3FrameworkDriver _driver; + private NUnitFrameworkDriver _driver; private string _mockAssemblyPath; + private string _whichApi; + public NUnitFrameworkDriverTests(string whichApi) + { + _whichApi = whichApi; + } + [SetUp] public void CreateDriver() { - var assemblyName = typeof(NUnit.Framework.TestAttribute).Assembly.GetName(); + var nunitRef = typeof(TestAttribute).Assembly.GetName(); _mockAssemblyPath = System.IO.Path.Combine(TestContext.CurrentContext.TestDirectory, MOCK_ASSEMBLY); - _driver = new NUnit3FrameworkDriver(AppDomain.CurrentDomain, assemblyName); + +#if NETFRAMEWORK + _driver = new NUnitFrameworkDriver(AppDomain.CurrentDomain, _whichApi, "99", nunitRef); +#else + _driver = new NUnitFrameworkDriver("99", nunitRef); +#endif } [Test] @@ -110,13 +124,41 @@ public void RunTestsAction_WithoutLoad_ThrowsInvalidOperationException() } [Test] - public void RunTestsAction_WithInvalidFilterElement_ThrowsNUnitEngineException() + public void RunTestsAction_WithInvalidFilterElement_ThrowsException() { _driver.Load(_mockAssemblyPath, _settings); var invalidFilter = "foo"; var ex = Assert.Catch(() => _driver.Run(new NullListener(), invalidFilter)); - Assert.That(ex, Is.TypeOf()); + + if (_whichApi == "2018") + { + Assert.That(ex, Is.TypeOf()); + Assert.That(ex.InnerException, Is.TypeOf()); + } + else + Assert.That(ex, Is.TypeOf()); + } + +#if NETFRAMEWORK + // Nested Class tests Api Selection in the driver + public class ApiSelectionTests() + { + [TestCase("4.2.2", "2018")] + [TestCase("3.14.0", "2018")] + [TestCase("3.2.0", "2018")] + [TestCase("3.0.1", "2009")] + [TestCase("3.0.0", "2009")] + public void CorrectApiIsSelected(string nunitVersion, string apiVersion) + { + var driver = new NUnitFrameworkDriver(AppDomain.CurrentDomain, "99", new AssemblyName() + { + Name = "nunit.framework", + Version = new Version(nunitVersion) + }); + + Assert.That(driver.API, Is.EqualTo(apiVersion)); + } } private class CallbackEventHandler : System.Web.UI.ICallbackEventHandler @@ -133,6 +175,7 @@ public void RaiseCallbackEvent(string eventArgument) _result = eventArgument; } } +#endif public class NullListener : ITestEventListener { @@ -143,4 +186,3 @@ public void OnTestEvent(string testEvent) } } } -#endif \ No newline at end of file diff --git a/src/NUnitEngine/nunit.engine.core.tests/Drivers/NotRunnableFrameworkDriverTests.cs b/src/NUnitEngine/nunit.engine.core.tests/Drivers/NotRunnableFrameworkDriverTests.cs index 0119e74ed..4bd37a513 100644 --- a/src/NUnitEngine/nunit.engine.core.tests/Drivers/NotRunnableFrameworkDriverTests.cs +++ b/src/NUnitEngine/nunit.engine.core.tests/Drivers/NotRunnableFrameworkDriverTests.cs @@ -14,7 +14,6 @@ namespace NUnit.Engine.Drivers // Functional tests of the NUnitFrameworkDriver calling into the framework. public abstract class NotRunnableFrameworkDriverTests { - private const string DRIVER_ID = "99"; private const string EXPECTED_ID = "99-1"; protected string? _expectedRunState; @@ -95,7 +94,6 @@ public void Run(string filePath, string expectedType) private IFrameworkDriver GetDriver(string filePath) { IFrameworkDriver driver = CreateDriver(filePath); - driver.ID = DRIVER_ID; return driver; } @@ -126,7 +124,7 @@ public InvalidAssemblyFrameworkDriverTests() protected override IFrameworkDriver CreateDriver(string filePath) { - return new InvalidAssemblyFrameworkDriver(filePath, _expectedReason ?? "Not Specified"); + return new InvalidAssemblyFrameworkDriver(filePath, "99", _expectedReason ?? "Not Specified"); } } @@ -142,7 +140,7 @@ public SkippedAssemblyFrameworkDriverTests() protected override IFrameworkDriver CreateDriver(string filePath) { - return new SkippedAssemblyFrameworkDriver(filePath); + return new SkippedAssemblyFrameworkDriver(filePath, "99"); } } } diff --git a/src/NUnitEngine/nunit.engine.core.tests/DummyExtensions.cs b/src/NUnitEngine/nunit.engine.core.tests/DummyExtensions.cs index 5d549efad..6f0c3b2fd 100644 --- a/src/NUnitEngine/nunit.engine.core.tests/DummyExtensions.cs +++ b/src/NUnitEngine/nunit.engine.core.tests/DummyExtensions.cs @@ -12,10 +12,10 @@ namespace NUnit.Engine [Extension] public class DummyFrameworkDriverExtension : IDriverFactory { -#if !NETFRAMEWORK - public IFrameworkDriver GetDriver(AssemblyName reference) +#if NETFRAMEWORK + public IFrameworkDriver GetDriver(AppDomain domain, string id, AssemblyName reference) #else - public IFrameworkDriver GetDriver(AppDomain domain, AssemblyName reference) + public IFrameworkDriver GetDriver(string id, AssemblyName reference) #endif { throw new NotImplementedException(); diff --git a/src/NUnitEngine/nunit.engine.core.tests/Runners/TestAgentRunnerExceptionTests.cs b/src/NUnitEngine/nunit.engine.core.tests/Runners/TestAgentRunnerExceptionTests.cs index 098237a5b..7df91d65d 100644 --- a/src/NUnitEngine/nunit.engine.core.tests/Runners/TestAgentRunnerExceptionTests.cs +++ b/src/NUnitEngine/nunit.engine.core.tests/Runners/TestAgentRunnerExceptionTests.cs @@ -22,8 +22,9 @@ public void Initialize() var driverService = Substitute.For(); driverService.GetDriver( AppDomain.CurrentDomain, + new TestPackage(), + string.Empty, string.Empty, - string.Empty, false).ReturnsForAnyArgs(_driver); _runner = new FakeTestAgentRunner(new TestPackage("mock-assembly.dll").SubPackages[0]) diff --git a/src/NUnitEngine/nunit.engine.core.tests/Services/DriverServiceTests.cs b/src/NUnitEngine/nunit.engine.core.tests/Services/DriverServiceTests.cs index 602c06242..538263071 100644 --- a/src/NUnitEngine/nunit.engine.core.tests/Services/DriverServiceTests.cs +++ b/src/NUnitEngine/nunit.engine.core.tests/Services/DriverServiceTests.cs @@ -7,7 +7,7 @@ using NUnit.Engine.Drivers; using NUnit.Engine.Extensibility; -namespace NUnit.Engine.Services.Tests +namespace NUnit.Engine.Services { [TestFixture] public class DriverServiceTests @@ -23,36 +23,25 @@ public void CreateDriverFactory() [TestCaseSource(nameof(DriverSelectionTestCases))] public void CorrectDriverIsUsed(string fileName, bool skipNonTestAssemblies, Type expectedType) { - var driver = _driverService.GetDriver(AppDomain.CurrentDomain, Path.Combine(TestContext.CurrentContext.TestDirectory, fileName), null, skipNonTestAssemblies); + var assemblyPath = Path.Combine(TestContext.CurrentContext.TestDirectory, fileName); + var driver = _driverService.GetDriver(AppDomain.CurrentDomain, new TestPackage(assemblyPath), assemblyPath, null, skipNonTestAssemblies); Assert.That(driver, Is.InstanceOf(expectedType)); } static TestCaseData[] DriverSelectionTestCases = new[] { - // TODO: make commented tests work -#if NETFRAMEWORK - new TestCaseData("mock-assembly.dll", false, typeof(NUnit3FrameworkDriver)), - new TestCaseData("mock-assembly.dll", true, typeof(NUnit3FrameworkDriver)), - //new TestCaseData("notest-assembly.dll", false, typeof(NUnit3FrameworkDriver)), -#elif NET5_0_OR_GREATER - new TestCaseData("mock-assembly.dll", false, typeof(NUnitNetCore31Driver)), - new TestCaseData("mock-assembly.dll", true, typeof(NUnitNetCore31Driver)), - //new TestCaseData("notest-assembly.dll", false, typeof(NUnitNetCore31Driver)), -#else - new TestCaseData("mock-assembly.dll", false, typeof(NUnitNetCore31Driver)), - new TestCaseData("mock-assembly.dll", true, typeof(NUnitNetCore31Driver)), - //new TestCaseData("notest-assembly.dll", false, typeof(NUnitNetCore31Driver)), -#endif -// Invalid cases should work with all target runtimes + new TestCaseData("mock-assembly.dll", false, typeof(NUnitFrameworkDriver)), + new TestCaseData("mock-assembly.dll", true, typeof(NUnitFrameworkDriver)), + new TestCaseData("notest-assembly.dll", false, typeof(NUnitFrameworkDriver)).Ignore("Assembly not present"), + new TestCaseData("notest-assembly.dll", true, typeof(SkippedAssemblyFrameworkDriver)).Ignore("Assembly not present"), + + // Invalid cases should work with all target runtimes new TestCaseData("mock-assembly.pdb", false, typeof(InvalidAssemblyFrameworkDriver)), new TestCaseData("mock-assembly.pdb", true, typeof(InvalidAssemblyFrameworkDriver)), new TestCaseData("junk.dll", false, typeof(InvalidAssemblyFrameworkDriver)), new TestCaseData("junk.dll", true, typeof(InvalidAssemblyFrameworkDriver)), new TestCaseData("nunit.engine.core.dll", false, typeof(InvalidAssemblyFrameworkDriver)), new TestCaseData("nunit.engine.core.dll", true, typeof(SkippedAssemblyFrameworkDriver)) -//#if !NET5_0_OR_GREATER // Not yet working -// new TestCaseData"notest-assembly.dll", true, typeof(SkippedAssemblyFrameworkDriver)) -//#endif }; [Test] diff --git a/src/NUnitEngine/nunit.engine.core.tests/TestData.cs b/src/NUnitEngine/nunit.engine.core.tests/TestData.cs deleted file mode 100644 index df2896dac..000000000 --- a/src/NUnitEngine/nunit.engine.core.tests/TestData.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt - -using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; - -namespace NUnit.Engine -{ - internal class TestData - { -#if NETCOREAPP3_1 - const string CURRENT_RUNTIME = "netcoreapp3.1"; -#elif NET6_0 - const string CURRENT_RUNTIME = "net6.0"; -#elif NET8_0 - const string CURRENT_RUNTIME = "net8.0"; -#else - const string CURRENT_RUNTIME = "net462"; -#endif - public static string MockAssemblyPath(string runtime) - => $"testdata/{runtime}/mock-assembly.dll"; - public static string NoTestAssemblyPath(string runtime) - => $"testdata/{runtime}/notest-assembly.dll"; - - [Test] - public void SelfTest() - { - VerifyFilePath(MockAssemblyPath(CURRENT_RUNTIME)); - } - - private void VerifyFilePath(string path) - { - Assert.That(File.Exists(path), $"File not found at {path}"); - } - } -} diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/DriverService.cs b/src/NUnitEngine/nunit.engine.core/Drivers/DriverService.cs index ccaec8db7..802a2e5ff 100644 --- a/src/NUnitEngine/nunit.engine.core/Drivers/DriverService.cs +++ b/src/NUnitEngine/nunit.engine.core/Drivers/DriverService.cs @@ -51,13 +51,13 @@ public DriverService() /// The value of any TargetFrameworkAttribute on the assembly, or null /// True if non-test assemblies should simply be skipped rather than reporting an error /// - public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string? targetFramework, bool skipNonTestAssemblies) + public IFrameworkDriver GetDriver(AppDomain domain, TestPackage package, string assemblyPath, string? targetFramework, bool skipNonTestAssemblies) { if (!File.Exists(assemblyPath)) - return new InvalidAssemblyFrameworkDriver(assemblyPath, "File not found: " + assemblyPath); + return new InvalidAssemblyFrameworkDriver(assemblyPath, package.ID, "File not found: " + assemblyPath); if (!PathUtils.IsAssemblyFileType(assemblyPath)) - return new InvalidAssemblyFrameworkDriver(assemblyPath, "File type is not supported"); + return new InvalidAssemblyFrameworkDriver(assemblyPath, package.ID, "File type is not supported"); if (targetFramework != null) { @@ -70,9 +70,9 @@ public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string? if (platform == "Silverlight" || platform == ".NETPortable" || platform == ".NETStandard" || platform == ".NETCompactFramework") if (skipNonTestAssemblies) - return new SkippedAssemblyFrameworkDriver(assemblyPath); + return new SkippedAssemblyFrameworkDriver(assemblyPath, package.ID); else - return new InvalidAssemblyFrameworkDriver(assemblyPath, platform + + return new InvalidAssemblyFrameworkDriver(assemblyPath, package.ID, platform + " test assemblies are not supported by this version of the engine"); } @@ -84,25 +84,22 @@ public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string? { foreach (var attr in assemblyDef.CustomAttributes) if (attr.AttributeType.FullName == "NUnit.Framework.NonTestAssemblyAttribute") - return new SkippedAssemblyFrameworkDriver(assemblyPath); + return new SkippedAssemblyFrameworkDriver(assemblyPath, package.ID); } - var references = new List(); - foreach (var cecilRef in assemblyDef.MainModule.AssemblyReferences) - references.Add(new AssemblyName(cecilRef.FullName)); - foreach (var factory in _factories) { log.Debug($"Trying {factory.GetType().Name}"); - foreach (var reference in references) + foreach (var cecilRef in assemblyDef.MainModule.AssemblyReferences) { - if (factory.IsSupportedTestFramework(reference)) + var assemblyName = new AssemblyName(cecilRef.FullName); + if (factory.IsSupportedTestFramework(assemblyName)) { #if NETFRAMEWORK - return factory.GetDriver(domain, reference); + return factory.GetDriver(domain, package.ID, assemblyName); #else - return factory.GetDriver(reference); + return factory.GetDriver(package.ID, assemblyName); #endif } } @@ -111,14 +108,14 @@ public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string? } catch (BadImageFormatException ex) { - return new InvalidAssemblyFrameworkDriver(assemblyPath, ex.Message); + return new InvalidAssemblyFrameworkDriver(assemblyPath, package.ID, ex.Message); } if (skipNonTestAssemblies) - return new SkippedAssemblyFrameworkDriver(assemblyPath); + return new SkippedAssemblyFrameworkDriver(assemblyPath, package.ID); else - return new InvalidAssemblyFrameworkDriver(assemblyPath, string.Format("No suitable tests found in '{0}'.\n" + - "Either assembly contains no tests or proper test driver has not been found.", assemblyPath)); + return new InvalidAssemblyFrameworkDriver(assemblyPath, package.ID, + $"No suitable tests found in '{assemblyPath}'.\r\nEither assembly contains no tests or proper test driver has not been found."); } } } diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/IDriverService.cs b/src/NUnitEngine/nunit.engine.core/Drivers/IDriverService.cs index fb3c26d3f..4dde75689 100644 --- a/src/NUnitEngine/nunit.engine.core/Drivers/IDriverService.cs +++ b/src/NUnitEngine/nunit.engine.core/Drivers/IDriverService.cs @@ -15,10 +15,11 @@ public interface IDriverService /// Get a driver suitable for loading and running tests in the specified assembly. /// /// The application domain in which to run the tests + /// The package for which the driver is to be used /// The path to the test assembly /// The value of any TargetFrameworkAttribute on the assembly, or null /// True if non-test assemblies should simply be skipped rather than reporting an error /// - IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string? targetFramework, bool skipNonTestAssemblies); + IFrameworkDriver GetDriver(AppDomain domain, TestPackage package, string assemblyPath, string? targetFramework, bool skipNonTestAssemblies); } } diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnit2DriverFactory.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnit2DriverFactory.cs index df4fb3688..f3dab35ce 100644 --- a/src/NUnitEngine/nunit.engine.core/Drivers/NUnit2DriverFactory.cs +++ b/src/NUnitEngine/nunit.engine.core/Drivers/NUnit2DriverFactory.cs @@ -43,7 +43,7 @@ public bool IsSupportedTestFramework(AssemblyName reference) /// The domain in which the assembly will be loaded /// The name of the test framework reference /// - public IFrameworkDriver GetDriver(AppDomain domain, AssemblyName reference) + public IFrameworkDriver GetDriver(AppDomain domain, string id, AssemblyName reference) { if (!IsSupportedTestFramework(reference)) throw new ArgumentException("Invalid framework", "reference"); diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnit3DriverFactory.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnit3DriverFactory.cs index 78bf43c6f..7205e8d0a 100644 --- a/src/NUnitEngine/nunit.engine.core/Drivers/NUnit3DriverFactory.cs +++ b/src/NUnitEngine/nunit.engine.core/Drivers/NUnit3DriverFactory.cs @@ -25,30 +25,31 @@ public bool IsSupportedTestFramework(AssemblyName reference) #if NETFRAMEWORK /// - /// Gets a driver for a given test assembly and a framework - /// which the assembly is already known to reference. + /// Gets a driver for a given test framework. /// /// The domain in which the assembly will be loaded /// An AssemblyName referring to the test framework. - /// - public IFrameworkDriver GetDriver(AppDomain domain, AssemblyName reference) + /// An IFrameworkDriver + public IFrameworkDriver GetDriver(AppDomain domain, string id, AssemblyName reference) { + Guard.ArgumentNotNullOrEmpty(id, nameof(id)); Guard.ArgumentValid(IsSupportedTestFramework(reference), "Invalid framework", "reference"); - return new NUnit3FrameworkDriver(domain, reference); + log.Info("Using NUnitFrameworkDriver"); + return new NUnitFrameworkDriver(domain, id, reference); } #else /// - /// Gets a driver for a given test assembly and a framework - /// which the assembly is already known to reference. + /// Gets a driver for a given test framework. /// /// An AssemblyName referring to the test framework. /// - public IFrameworkDriver GetDriver(AssemblyName reference) + public IFrameworkDriver GetDriver(string id, AssemblyName reference) { + Guard.ArgumentNotNullOrEmpty(id, nameof(id)); Guard.ArgumentValid(IsSupportedTestFramework(reference), "Invalid framework", "reference"); - log.Info("Using NUnitNetCore31Driver"); - return new NUnitNetCore31Driver(); + log.Info("Using NUnitFrameworkDriver"); + return new NUnitFrameworkDriver(id, reference); } #endif } diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnit3FrameworkDriver.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnit3FrameworkDriver.cs deleted file mode 100644 index 8084d79b0..000000000 --- a/src/NUnitEngine/nunit.engine.core/Drivers/NUnit3FrameworkDriver.cs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt - -#if NETFRAMEWORK -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Runtime.Serialization; -using NUnit.Common; -using NUnit.Engine.Internal; -using NUnit.Engine.Extensibility; -using System.Diagnostics.CodeAnalysis; - -namespace NUnit.Engine.Drivers -{ - /// - /// NUnitFrameworkDriver is used by the test-runner to load and run - /// tests using the NUnit framework assembly. - /// - public class NUnit3FrameworkDriver : IFrameworkDriver - { - private const string LOAD_MESSAGE = "Method called without calling Load first"; - - private static readonly string CONTROLLER_TYPE = "NUnit.Framework.Api.FrameworkController"; - private static readonly string LOAD_ACTION = CONTROLLER_TYPE + "+LoadTestsAction"; - private static readonly string EXPLORE_ACTION = CONTROLLER_TYPE + "+ExploreTestsAction"; - private static readonly string COUNT_ACTION = CONTROLLER_TYPE + "+CountTestsAction"; - private static readonly string RUN_ACTION = CONTROLLER_TYPE + "+RunTestsAction"; - private static readonly string STOP_RUN_ACTION = CONTROLLER_TYPE + "+StopRunAction"; - - static readonly ILogger log = InternalTrace.GetLogger("NUnitFrameworkDriver"); - - readonly AppDomain _testDomain; - readonly AssemblyName _reference; - string? _testAssemblyPath; - - object? _frameworkController; - - /// - /// Construct an NUnit3FrameworkDriver - /// - /// The application domain in which to create the FrameworkController - /// An AssemblyName referring to the test framework. - public NUnit3FrameworkDriver(AppDomain testDomain, AssemblyName reference) - { - _testDomain = testDomain; - _reference = reference; - } - - public string ID { get; set; } = string.Empty; - - /// - /// Loads the tests in an assembly. - /// - /// An Xml string representing the loaded test - public string Load(string testAssemblyPath, IDictionary settings) - { - Guard.ArgumentValid(File.Exists(testAssemblyPath), "Framework driver constructor called with a file name that doesn't exist.", "testAssemblyPath"); - - var idPrefix = string.IsNullOrEmpty(ID) ? "" : ID + "-"; - - // Normally, the runner should check for an invalid requested runtime, but we make sure here - var requestedRuntime = settings.ContainsKey(EnginePackageSettings.RequestedRuntimeFramework) - ? settings[EnginePackageSettings.RequestedRuntimeFramework] : null; - - _testAssemblyPath = testAssemblyPath; - - try - { - _frameworkController = CreateObject(CONTROLLER_TYPE, testAssemblyPath, idPrefix, settings); - } - catch (BadImageFormatException ex) when (requestedRuntime != null) - { - throw new NUnitEngineException($"Requested runtime {requestedRuntime} is not suitable for use with test assembly {testAssemblyPath}", ex); - } - catch (SerializationException ex) - { - throw new NUnitEngineException("The NUnit 3 driver cannot support this test assembly. Use a platform specific runner.", ex); - } - - CallbackHandler handler = new CallbackHandler(); - - var fileName = Path.GetFileName(_testAssemblyPath); - - log.Info("Loading {0} - see separate log file", fileName); - - CreateObject(LOAD_ACTION, _frameworkController, handler); - - log.Info("Loaded {0}", fileName); - - return handler.Result.ShouldNotBeNull(); - } - - public int CountTestCases(string filter) - { - CheckLoadWasCalled(); - - CallbackHandler handler = new CallbackHandler(); - - CreateObject(COUNT_ACTION, _frameworkController.ShouldNotBeNull(), filter, handler); - - return int.Parse(handler.Result.ShouldNotBeNull()); - } - - /// - /// Executes the tests in an assembly. - /// - /// An ITestEventHandler that receives progress notices - /// A filter that controls which tests are executed - /// An Xml string representing the result - public string Run(ITestEventListener? listener, string filter) - { - CheckLoadWasCalled(); - - var handler = new RunTestsCallbackHandler(listener); - - log.Info("Running {0} - see separate log file", Path.GetFileName(_testAssemblyPath.ShouldNotBeNull())); - CreateObject(RUN_ACTION, _frameworkController.ShouldNotBeNull(), filter, handler); - - return handler.Result.ShouldNotBeNull(); - } - - /// - /// Cancel the ongoing test run. If no test is running, the call is ignored. - /// - /// If true, cancel any ongoing test threads, otherwise wait for them to complete. - public void StopRun(bool force) - { - CreateObject(STOP_RUN_ACTION, _frameworkController.ShouldNotBeNull(), force, new CallbackHandler()); - } - - /// - /// Returns information about the tests in an assembly. - /// - /// A filter indicating which tests to include - /// An Xml string representing the tests - public string Explore(string filter) - { - CheckLoadWasCalled(); - - CallbackHandler handler = new CallbackHandler(); - - log.Info("Exploring {0} - see separate log file", Path.GetFileName(_testAssemblyPath.ShouldNotBeNull())); - CreateObject(EXPLORE_ACTION, _frameworkController.ShouldNotBeNull(), filter, handler); - - return handler.Result.ShouldNotBeNull(); - } - - private void CheckLoadWasCalled() - { - if (_frameworkController == null) - throw new InvalidOperationException(LOAD_MESSAGE); - } - - private object CreateObject(string typeName, params object?[]? args) - { - try - { - return _testDomain.CreateInstanceAndUnwrap( - _reference.FullName, typeName, false, 0, null, args, null, null )!; - } - catch (TargetInvocationException ex) - { - throw new NUnitEngineException("The NUnit 3 driver encountered an error while executing reflected code.", ex.InnerException); - } - } - } -} -#endif \ No newline at end of file diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi.cs new file mode 100644 index 000000000..8af86e081 --- /dev/null +++ b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi.cs @@ -0,0 +1,51 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +using System; +using System.Collections.Generic; + +namespace NUnit.Engine.Drivers +{ + public interface NUnitFrameworkApi + { + /// + /// Loads the tests in an assembly. + /// + /// An Xml string representing the loaded test + string Load(string testAssemblyPath, IDictionary settings); + + /// + /// Count the test cases that would be executed. + /// + /// An XML string representing the TestFilter to use in counting the tests + /// The number of test cases counted + int CountTestCases(string filter); + + /// + /// Executes the tests in an assembly synchronously. + /// + /// An ITestEventHandler that receives progress notices + /// A XML string representing the filter that controls which tests are executed + /// An Xml string representing the result + string Run(ITestEventListener? listener, string filter); + + /// + /// Executes the tests in an assembly asynchronously. + /// + /// A callback that receives XML progress notices + /// A filter that controls which tests are executed + void RunAsync(Action callback, string filter); + + /// + /// Returns information about the tests in an assembly. + /// + /// An XML string representing the filter that controls which tests are included + /// An Xml string representing the tests + string Explore(string filter); + + /// + /// Cancel the ongoing test run. If no test is running, the call is ignored. + /// + /// If true, cancel any ongoing test threads, otherwise wait for them to complete. + void StopRun(bool force); + } +} diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi2009.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi2009.cs new file mode 100644 index 000000000..acc022a58 --- /dev/null +++ b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi2009.cs @@ -0,0 +1,165 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +#if NETFRAMEWORK +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Runtime.Serialization; +using NUnit.Common; +using NUnit.Engine.Internal; + +namespace NUnit.Engine.Drivers +{ + /// + /// This is the original NUnit 3 API, which only works for .NET Framework. + /// As far as I can discover, it first appeared in pre-release 2.9.1, + /// on launchpad in 2009, hence the name. + /// + class NUnitFrameworkApi2009 : NUnitFrameworkApi + { + static readonly ILogger log = InternalTrace.GetLogger(nameof(NUnitFrameworkApi2009)); + + const string LOAD_MESSAGE = "Method called without calling Load first. Possible error in runner."; + const string INVALID_FRAMEWORK_MESSAGE = "Running tests against this version of the framework using this driver is not supported. Please update NUnit.Framework to the latest version."; + const string FAILED_TO_LOAD_ASSEMBLY = "Failed to load assembly "; + const string FAILED_TO_LOAD_NUNIT = "Failed to load the NUnit Framework in the test assembly"; + + const string CONTROLLER_TYPE = "NUnit.Framework.Api.FrameworkController"; + + private readonly string _driverId; + + private readonly AppDomain _testDomain; + private readonly AssemblyName _nunitRef; + + private string? _testAssemblyPath; + + private object? _frameworkController; + private Type? _frameworkControllerType; + + public NUnitFrameworkApi2009(AppDomain testDomain, string driverId, AssemblyName nunitRef) + { + Guard.ArgumentNotNull(testDomain, nameof(testDomain)); + Guard.ArgumentNotNull(driverId, nameof(driverId)); + Guard.ArgumentNotNull(nunitRef, nameof(nunitRef)); + + _testDomain = testDomain; + _driverId = driverId; + _nunitRef = nunitRef; + } + + public string Load(string testAssemblyPath, IDictionary settings) + { + Guard.ArgumentValid(File.Exists(testAssemblyPath), "Framework driver called with a file name that doesn't exist.", "testAssemblyPath"); + + log.Info($"Loading {testAssemblyPath} - see separate log file"); + + _testAssemblyPath = testAssemblyPath; + + // Normally, the caller should check for an invalid requested runtime, but we make sure here + var requestedRuntime = settings.ContainsKey(EnginePackageSettings.RequestedRuntimeFramework) + ? settings[EnginePackageSettings.RequestedRuntimeFramework] : null; + + var idPrefix = _driverId + "-"; + + try + { + _frameworkController = _testDomain.CreateInstanceAndUnwrap( + _nunitRef.FullName, + CONTROLLER_TYPE, + false, + 0, + null, + new object[] { _testAssemblyPath, idPrefix, settings }, + null, + null).ShouldNotBeNull(); + } + catch (BadImageFormatException ex) when (requestedRuntime != null) + { + throw new NUnitEngineException($"Requested runtime {requestedRuntime} is not suitable for use with test assembly {_testAssemblyPath}", ex); + } + catch (SerializationException ex) + { + throw new NUnitEngineException("The NUnit 3 driver cannot support this test assembly. Use a platform specific runner.", ex); + } + + _frameworkControllerType = _frameworkController.GetType(); + log.Debug($"Created FrameworkController {_frameworkControllerType.Name}"); + + return ExecuteAction(LOAD_ACTION); + } + + public int CountTestCases(string filter) + { + CheckLoadWasCalled(); + return int.Parse(ExecuteAction(COUNT_ACTION, filter)); + } + + public string Run(ITestEventListener? listener, string filter) + { + CheckLoadWasCalled(); + log.Info("Running {0} - see separate log file", Path.GetFileName(_testAssemblyPath.ShouldNotBeNull())); + return ExecuteAction(RUN_ACTION, listener, filter); + } + + public void RunAsync(Action callback, string filter) => throw new NotImplementedException(); + + public void StopRun(bool force) => ExecuteAction(STOP_RUN_ACTION, force); + + public string Explore(string filter) + { + CheckLoadWasCalled(); + log.Info("Exploring {0} - see separate log file", Path.GetFileName(_testAssemblyPath.ShouldNotBeNull())); + return ExecuteAction(EXPLORE_ACTION, filter); + } + + private void CheckLoadWasCalled() + { + if (_frameworkController == null) + throw new InvalidOperationException(LOAD_MESSAGE); + } + + // Actions with no extra arguments beyond controller and handler + const string LOAD_ACTION = CONTROLLER_TYPE + "+LoadTestsAction"; + private string ExecuteAction(string action) + { + CallbackHandler handler = new CallbackHandler(); + CreateObject(action, _frameworkController, handler); + return handler.Result.ShouldNotBeNull(); + } + + // Actions with one extra argument + const string EXPLORE_ACTION = CONTROLLER_TYPE + "+ExploreTestsAction"; + const string COUNT_ACTION = CONTROLLER_TYPE + "+CountTestsAction"; + const string STOP_RUN_ACTION = CONTROLLER_TYPE + "+StopRunAction"; + private string ExecuteAction(string action, object arg1) + { + CallbackHandler handler = new CallbackHandler(); + CreateObject(action, _frameworkController, arg1, handler); + return handler.Result.ShouldNotBeNull(); + } + + // Run action has two extra arguments and uses a special handler + const string RUN_ACTION = CONTROLLER_TYPE + "+RunTestsAction"; + private string ExecuteAction(string action, ITestEventListener? listener, string filter) + { + RunTestsCallbackHandler handler = new RunTestsCallbackHandler(listener); + CreateObject(action, _frameworkController, filter, handler); + return handler.Result.ShouldNotBeNull(); + } + + private object CreateObject(string typeName, params object?[]? args) + { + try + { + return _testDomain.CreateInstanceAndUnwrap( + _nunitRef.FullName, typeName, false, 0, null, args, null, null)!; + } + catch (TargetInvocationException ex) + { + throw new NUnitEngineException("The NUnit 3 driver encountered an error while executing reflected code.", ex.InnerException); + } + } + } +} +#endif diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi2018.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi2018.cs new file mode 100644 index 000000000..535388f19 --- /dev/null +++ b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkApi2018.cs @@ -0,0 +1,252 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Runtime.Serialization; +using NUnit.Common; +using NUnit.Engine.Internal; + +namespace NUnit.Engine.Drivers +{ + /// + /// This is the revised API, designed for use with .NET Core. It first + /// appears in our source code in 2018. This implementation is modified + /// to make it work under the .NET Framework as well as .NET Core. It + /// may be used for NUnit 3.10 or higher. + /// +#if NETFRAMEWORK + public class NUnitFrameworkApi2018 : MarshalByRefObject, NUnitFrameworkApi +#else + public class NUnitFrameworkApi2018 : NUnitFrameworkApi +#endif + { + static readonly ILogger log = InternalTrace.GetLogger(nameof(NUnitFrameworkApi2018)); + + const string LOAD_MESSAGE = "Method called without calling Load first. Possible error in runner."; + const string INVALID_FRAMEWORK_MESSAGE = "Running tests against this version of the framework using this driver is not supported. Please update NUnit.Framework to the latest version."; + const string FAILED_TO_LOAD_ASSEMBLY = "Failed to load assembly "; + const string FAILED_TO_LOAD_NUNIT = "Failed to load the NUnit Framework in the test assembly"; + + const string CONTROLLER_TYPE = "NUnit.Framework.Api.FrameworkController"; + + private readonly string _driverId; + + private readonly AssemblyName _nunitRef; + + private object? _frameworkController; + private Type? _frameworkControllerType; + +#if NETCOREAPP + private TestAssemblyLoadContext? _assemblyLoadContext; + private Assembly? _frameworkAssembly; +#endif + + private string? _testAssemblyPath; + + public NUnitFrameworkApi2018(string driverId, AssemblyName nunitRef) + { + Guard.ArgumentNotNull(driverId, nameof(driverId)); + Guard.ArgumentNotNull(nunitRef, nameof(nunitRef)); + + _driverId = driverId; + _nunitRef = nunitRef; + } + + public string Load(string testAssemblyPath, IDictionary settings) + { + Guard.ArgumentNotNull(testAssemblyPath, nameof(testAssemblyPath)); + Guard.ArgumentNotNull(settings, nameof(settings)); + Guard.ArgumentValid(File.Exists(testAssemblyPath), "Framework driver called with a file name that doesn't exist.", "testAssemblyPath"); + log.Info($"Loading {testAssemblyPath} - see separate log file"); + + _testAssemblyPath = Path.GetFullPath(testAssemblyPath); + var idPrefix = _driverId + "-"; + +#if NETFRAMEWORK + try + { + _frameworkController = AppDomain.CurrentDomain.CreateInstanceAndUnwrap( + _nunitRef.FullName, + CONTROLLER_TYPE, + false, + 0, + null, + new object[] { _testAssemblyPath, idPrefix, settings }, + null, + null).ShouldNotBeNull(); + } + catch (Exception ex) + { + string msg = $"Failed to load {_nunitRef.FullName}\r\n Codebase: {_nunitRef.CodeBase}"; + throw new Exception(msg, ex); + } + + _frameworkControllerType = _frameworkController?.GetType(); + log.Debug($"Created FrameworkController {_frameworkControllerType?.Name}"); + + var controllerAssembly = _frameworkControllerType?.Assembly?.GetName(); + log.Debug($"Controller assembly is {controllerAssembly}"); +#else + _assemblyLoadContext = new TestAssemblyLoadContext(testAssemblyPath); + + var testAssembly = LoadAssembly(testAssemblyPath); + _frameworkAssembly = LoadAssembly(_nunitRef); + + _frameworkController = CreateInstance(CONTROLLER_TYPE, testAssembly, idPrefix, settings); + if (_frameworkController == null) + { + log.Error(INVALID_FRAMEWORK_MESSAGE); + throw new NUnitEngineException(INVALID_FRAMEWORK_MESSAGE); + } +#endif + + _frameworkControllerType = _frameworkController?.GetType(); + log.Debug($"Created FrameworkController {_frameworkControllerType?.Name}"); + + log.Debug($"Loaded {testAssemblyPath}"); + return (string)ExecuteMethod(LOAD_METHOD); + } + + public int CountTestCases(string filter) + { + CheckLoadWasCalled(); + object? count = ExecuteMethod(COUNT_METHOD, filter); + return count != null ? (int)count : 0; + } + + public string Run(ITestEventListener? listener, string filter) + { + CheckLoadWasCalled(); + log.Info("Running {0} - see separate log file", Path.GetFileName(_testAssemblyPath.ShouldNotBeNull())); + Action? callback = null; + return (string)ExecuteMethod(RUN_METHOD, new[] { typeof(Action), typeof(string) }, callback, filter); + } + + public void RunAsync(Action callback, string filter) + { + CheckLoadWasCalled(); + log.Info("Running {0} - see separate log file", Path.GetFileName(_testAssemblyPath.ShouldNotBeNull())); + ExecuteMethod(RUN_ASYNC_METHOD, new[] { typeof(Action), typeof(string) }, callback, filter); + } + + public void StopRun(bool force) + { + ExecuteMethod(STOP_RUN_METHOD, force); + } + + public string Explore(string filter) + { + CheckLoadWasCalled(); + log.Info("Exploring {0} - see separate log file", Path.GetFileName(_testAssemblyPath.ShouldNotBeNull())); + return (string)ExecuteMethod(EXPLORE_METHOD, filter); + } + + private void CheckLoadWasCalled() + { + if (_frameworkController == null) + throw new InvalidOperationException(LOAD_MESSAGE); + } + +#if NETCOREAPP + private object CreateInstance(string typeName, params object?[]? args) + { + var type = _frameworkAssembly.ShouldNotBeNull().GetType(typeName, throwOnError: true)!; + return Activator.CreateInstance(type, args)!; + } + + private Assembly LoadAssembly(string assemblyPath) + { + Assembly assembly; + + try + { + assembly = _assemblyLoadContext?.LoadFromAssemblyPath(assemblyPath)!; + if (assembly == null) + throw new Exception("LoadFromAssemblyPath returned null"); + } + catch (Exception e) + { + var msg = string.Format(FAILED_TO_LOAD_ASSEMBLY + assemblyPath); + log.Error(msg); + throw new NUnitEngineException(msg, e); + } + + log.Debug($"Loaded {assemblyPath}"); + return assembly; + } + + private Assembly LoadAssembly(AssemblyName assemblyName) + { + Assembly assembly; + + try + { + assembly = _assemblyLoadContext?.LoadFromAssemblyName(assemblyName)!; + if (assembly == null) + throw new Exception("LoadFromAssemblyName returned null"); + } + catch (Exception e) + { + var msg = string.Format(FAILED_TO_LOAD_ASSEMBLY + assemblyName.FullName); + log.Error($"{FAILED_TO_LOAD_ASSEMBLY}\r\n{e}"); + throw new NUnitEngineException(FAILED_TO_LOAD_NUNIT, e); + } + + log.Debug($"Loaded {assemblyName.FullName}"); + return assembly; + } +#endif + + // API methods with no overloads + private static readonly string LOAD_METHOD = "LoadTests"; + private static readonly string EXPLORE_METHOD = "ExploreTests"; + private static readonly string COUNT_METHOD = "CountTests"; + private static readonly string STOP_RUN_METHOD = "StopRun"; + + // Execute methods with no overloads + private object ExecuteMethod(string methodName, params object?[] args) + { + log.Debug($"Calling API method {methodName} with args {string.Join("+", args)}"); + var method = _frameworkControllerType.ShouldNotBeNull().GetMethod(methodName, BindingFlags.Public | BindingFlags.Instance); + return ExecuteMethod(method, args); + } + + // API methods with overloads + private static readonly string RUN_METHOD = "RunTests"; + private static readonly string RUN_ASYNC_METHOD = "RunTests"; + + // Execute overloaded methods specifying argument types + private object ExecuteMethod(string methodName, Type[] ptypes, params object?[] args) + { + log.Debug($"Calling API method {methodName} with arg types {string.Join("+", ptypes)}"); + var method = _frameworkControllerType.ShouldNotBeNull().GetMethod(methodName, ptypes); + return ExecuteMethod(method, args); + } + + private object ExecuteMethod(MethodInfo? method, params object?[] args) + { + if (method == null) + throw new NUnitEngineException(INVALID_FRAMEWORK_MESSAGE); + + log.Debug($"Executing {method.DeclaringType}.{method.Name}"); + +#if NETFRAMEWORK + return method.Invoke(_frameworkController, args).ShouldNotBeNull(); +#else + //using (_assemblyLoadContext.ShouldNotBeNull().EnterContextualReflection()) + //{ + return method.Invoke(_frameworkController, args).ShouldNotBeNull(); + //} +#endif + } + +#if NETFRAMEWORK + public override object InitializeLifetimeService() + { + return null!; + } +#endif + } +} diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkDriver.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkDriver.cs new file mode 100644 index 000000000..b876ba368 --- /dev/null +++ b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitFrameworkDriver.cs @@ -0,0 +1,157 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +using System; +using System.Collections.Generic; +using System.Reflection; +using NUnit.Common; +using NUnit.Engine.Internal; +using NUnit.Engine.Extensibility; + +namespace NUnit.Engine.Drivers +{ + /// + /// NUnitFrameworkDriver is used by the test-runner to load and run + /// tests using the NUnit framework assembly, versions 3 and up. + /// + public class NUnitFrameworkDriver : IFrameworkDriver + { + static readonly Version MINIMUM_NUNIT_VERSION = new Version(3, 2, 0); + static readonly ILogger log = InternalTrace.GetLogger(nameof(NUnitFrameworkDriver)); + + readonly NUnitFrameworkApi _api; + +#if NETFRAMEWORK + /// + /// Construct an NUnitFrameworkDriver + /// + /// The application domain in which to create the FrameworkController + /// An AssemblyName referring to the test framework. + public NUnitFrameworkDriver(AppDomain testDomain, string id, AssemblyName nunitRef) + { + Guard.ArgumentNotNull(testDomain, nameof(testDomain)); + Guard.ArgumentNotNullOrEmpty(id, nameof(id)); + Guard.ArgumentNotNull(nunitRef, nameof(nunitRef)); + + ID = id; + + if (nunitRef.Version >= MINIMUM_NUNIT_VERSION) + { + API = "2018"; + _api = (NUnitFrameworkApi)testDomain.CreateInstanceFromAndUnwrap( + Assembly.GetExecutingAssembly().Location, + "NUnit.Engine.Drivers.NUnitFrameworkApi2018", + false, + 0, + null, + new object[] { ID, nunitRef }, + null, + null).ShouldNotBeNull(); + } + else + { + API = "2009"; + _api = new NUnitFrameworkApi2009(testDomain, ID, nunitRef); + } + } + + /// + /// Internal generic constructor used by our tests. + /// + /// The application domain in which to create the FrameworkController + /// An AssemblyName referring to the test framework. + internal NUnitFrameworkDriver(AppDomain testDomain, string api, string id, AssemblyName nunitRef) + { + Guard.ArgumentNotNull(testDomain, nameof(testDomain)); + Guard.ArgumentNotNull(api, nameof(api)); + Guard.ArgumentValid(api == "2009" || api == "2018", $"Invalid API specified: {api}", nameof(api)); + Guard.ArgumentNotNullOrEmpty(id, nameof(id)); + Guard.ArgumentNotNull(nunitRef, nameof(nunitRef)); + + ID = id; + API = api; + + _api = api == "2018" + ? (NUnitFrameworkApi)testDomain.CreateInstanceFromAndUnwrap( + Assembly.GetExecutingAssembly().Location, + typeof(NUnitFrameworkApi2018).FullName!, + false, + 0, + null, + new object[] { ID, nunitRef }, + null, + null).ShouldNotBeNull() + : new NUnitFrameworkApi2009(testDomain, ID, nunitRef); + } +#else + /// + /// Construct an NUnitFrameworkDriver + /// + /// An AssemblyName referring to the test framework. + public NUnitFrameworkDriver(string id, AssemblyName nunitRef) + { + Guard.ArgumentNotNullOrEmpty(id, nameof(id)); + Guard.ArgumentNotNull(nunitRef, nameof(nunitRef)); + + ID = id; + API = "2018"; + + _api = new NUnitFrameworkApi2018(ID, nunitRef); + } +#endif + + /// + /// String naming the API in use, for use by tests + /// + internal string API { get; } = string.Empty; + + /// + /// An id prefix that will be passed to the test framework and used as part of the + /// test ids created. + /// + public string ID { get; } + + /// + /// Loads the tests in an assembly. + /// + /// The path to the test assembly + /// The test settings + /// An XML string representing the loaded test + public string Load(string testAssemblyPath, IDictionary settings) + => _api.Load(testAssemblyPath, settings); + + /// + /// Counts the number of test cases for the loaded test assembly + /// + /// The XML test filter + /// The number of test cases + public int CountTestCases(string filter) => _api.CountTestCases(filter); + + /// + /// Executes the tests in an assembly. + /// + /// An ITestEventHandler that receives progress notices + /// A filter that controls which tests are executed + /// An Xml string representing the result + public string Run(ITestEventListener? listener, string filter) => _api.Run(null, filter); + + /// + /// Executes the tests in an assembly asynchronously. + /// + /// A callback that receives XML progress notices + /// A filter that controls which tests are executed + public void RunAsync(Action callback, string filter) => _api.RunAsync(callback, filter); + + /// + /// Cancel the ongoing test run. If no test is running, the call is ignored. + /// + /// If true, cancel any ongoing test threads, otherwise wait for them to complete. + public void StopRun(bool force) => _api.StopRun(force); + + /// + /// Returns information about the tests in an assembly. + /// + /// A filter indicating which tests to include + /// An Xml string representing the tests + public string Explore(string filter) => _api.Explore(filter); + } +} diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NUnitNetCore31Driver.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NUnitNetCore31Driver.cs deleted file mode 100644 index 7199f9a8f..000000000 --- a/src/NUnitEngine/nunit.engine.core/Drivers/NUnitNetCore31Driver.cs +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt - -#if NETCOREAPP3_1_OR_GREATER -using System; -using System.Linq; -using System.Collections.Generic; -using System.IO; -using NUnit.Engine.Internal; -using System.Reflection; -using NUnit.Engine.Extensibility; -using System.Diagnostics; -using NUnit.Common; - -namespace NUnit.Engine.Drivers -{ - /// - /// NUnitNetCore31Driver is used by the test-runner to load and run - /// tests using the NUnit framework assembly. It contains functionality to - /// correctly load assemblies from other directories, using APIs first available in - /// .NET Core 3.1. - /// - public class NUnitNetCore31Driver : IFrameworkDriver - { - const string LOAD_MESSAGE = "Method called without calling Load first"; - const string INVALID_FRAMEWORK_MESSAGE = "Running tests against this version of the framework using this driver is not supported. Please update NUnit.Framework to the latest version."; - const string FAILED_TO_LOAD_TEST_ASSEMBLY = "Failed to load the test assembly {0}"; - const string FAILED_TO_LOAD_NUNIT = "Failed to load the NUnit Framework in the test assembly"; - - static readonly string CONTROLLER_TYPE = "NUnit.Framework.Api.FrameworkController"; - static readonly string LOAD_METHOD = "LoadTests"; - static readonly string EXPLORE_METHOD = "ExploreTests"; - static readonly string COUNT_METHOD = "CountTests"; - static readonly string RUN_METHOD = "RunTests"; - static readonly string RUN_ASYNC_METHOD = "RunTests"; - static readonly string STOP_RUN_METHOD = "StopRun"; - - static ILogger log = InternalTrace.GetLogger(nameof(NUnitNetCore31Driver)); - - Assembly? _testAssembly; - Assembly? _frameworkAssembly; - object? _frameworkController; - Type? _frameworkControllerType; - TestAssemblyLoadContext? _assemblyLoadContext; - - /// - /// An id prefix that will be passed to the test framework and used as part of the - /// test ids created. - /// - public string ID { get; set; } = string.Empty; - - /// - /// Loads the tests in an assembly. - /// - /// The path to the test assembly - /// The test settings - /// An XML string representing the loaded test - public string Load(string assemblyPath, IDictionary settings) - { - log.Debug($"Loading {assemblyPath}"); - var idPrefix = string.IsNullOrEmpty(ID) ? "" : ID + "-"; - - assemblyPath = Path.GetFullPath(assemblyPath); //AssemblyLoadContext requires an absolute path - _assemblyLoadContext = new TestAssemblyLoadContext(assemblyPath); - - try - { - _testAssembly = _assemblyLoadContext.LoadFromAssemblyPath(assemblyPath); - } - catch (Exception e) - { - var msg = string.Format(FAILED_TO_LOAD_TEST_ASSEMBLY, assemblyPath); - log.Error(msg); - throw new NUnitEngineException(msg, e); - } - log.Debug($"Loaded {assemblyPath}"); - - var nunitRef = _testAssembly.GetReferencedAssemblies().FirstOrDefault(reference => string.Equals(reference.Name, "nunit.framework", StringComparison.OrdinalIgnoreCase)); - if (nunitRef == null) - { - log.Error(FAILED_TO_LOAD_NUNIT); - throw new NUnitEngineException(FAILED_TO_LOAD_NUNIT); - } - - try - { - _frameworkAssembly = _assemblyLoadContext.LoadFromAssemblyName(nunitRef); - } - catch (Exception e) - { - log.Error($"{FAILED_TO_LOAD_NUNIT}\r\n{e}"); - throw new NUnitEngineException(FAILED_TO_LOAD_NUNIT, e); - } - log.Debug("Loaded nunit.framework"); - - _frameworkController = CreateObject(CONTROLLER_TYPE, _testAssembly, idPrefix, settings); - if (_frameworkController == null) - { - log.Error(INVALID_FRAMEWORK_MESSAGE); - throw new NUnitEngineException(INVALID_FRAMEWORK_MESSAGE); - } - - _frameworkControllerType = _frameworkController.GetType(); - log.Debug($"Created FrameworkControler {_frameworkControllerType.Name}"); - - log.Info("Loading {0} - see separate log file", _testAssembly.FullName!); - return (string)ExecuteMethod(LOAD_METHOD); - } - - /// - /// Counts the number of test cases for the loaded test assembly - /// - /// The XML test filter - /// The number of test cases - public int CountTestCases(string filter) - { - CheckLoadWasCalled(); - object? count = ExecuteMethod(COUNT_METHOD, filter); - return count != null ? (int)count : 0; - } - - /// - /// Executes the tests in an assembly. - /// - /// An ITestEventHandler that receives progress notices - /// A filter that controls which tests are executed - /// An Xml string representing the result - public string Run(ITestEventListener? listener, string filter) - { - CheckLoadWasCalled(); - log.Info("Running {0} - see separate log file", _testAssembly.ShouldNotBeNull().FullName!); - Action? callback = listener != null ? listener.OnTestEvent : (Action?)null; - return (string)ExecuteMethod(RUN_METHOD, new[] { typeof(Action), typeof(string) }, callback, filter); - } - - /// - /// Executes the tests in an assembly asynchronously. - /// - /// A callback that receives XML progress notices - /// A filter that controls which tests are executed - public void RunAsync(Action callback, string filter) - { - CheckLoadWasCalled(); - log.Info("Running {0} - see separate log file", _testAssembly.ShouldNotBeNull().FullName!); - ExecuteMethod(RUN_ASYNC_METHOD, new[] { typeof(Action), typeof(string) }, callback, filter); - } - - /// - /// Cancel the ongoing test run. If no test is running, the call is ignored. - /// - /// If true, cancel any ongoing test threads, otherwise wait for them to complete. - public void StopRun(bool force) - { - ExecuteMethod(STOP_RUN_METHOD, force); - } - - /// - /// Returns information about the tests in an assembly. - /// - /// A filter indicating which tests to include - /// An Xml string representing the tests - public string Explore(string filter) - { - CheckLoadWasCalled(); - - log.Info("Exploring {0} - see separate log file", _testAssembly.ShouldNotBeNull().FullName!); - return (string)ExecuteMethod(EXPLORE_METHOD, filter); - } - - void CheckLoadWasCalled() - { - if (_frameworkController == null) - throw new InvalidOperationException(LOAD_MESSAGE); - } - - object CreateObject(string typeName, params object?[]? args) - { - var type = _frameworkAssembly.ShouldNotBeNull().GetType(typeName, throwOnError: true)!; - return Activator.CreateInstance(type, args)!; - } - - object ExecuteMethod(string methodName, params object?[] args) - { - var method = _frameworkControllerType.ShouldNotBeNull().GetMethod(methodName, BindingFlags.Public | BindingFlags.Instance); - return ExecuteMethod(method, args); - } - - object ExecuteMethod(string methodName, Type[] ptypes, params object?[] args) - { - var method = _frameworkControllerType.ShouldNotBeNull().GetMethod(methodName, ptypes); - return ExecuteMethod(method, args); - } - - object ExecuteMethod(MethodInfo? method, params object?[] args) - { - if (method == null) - { - throw new NUnitEngineException(INVALID_FRAMEWORK_MESSAGE); - } - - using (_assemblyLoadContext.ShouldNotBeNull().EnterContextualReflection()) - { - log.Debug($"Executing {method.DeclaringType}.{method.Name}"); - return method.Invoke(_frameworkController, args).ShouldNotBeNull(); - } - } - } -} -#endif \ No newline at end of file diff --git a/src/NUnitEngine/nunit.engine.core/Drivers/NotRunnableFrameworkDriver.cs b/src/NUnitEngine/nunit.engine.core/Drivers/NotRunnableFrameworkDriver.cs index 6967be78f..e7ceab661 100644 --- a/src/NUnitEngine/nunit.engine.core/Drivers/NotRunnableFrameworkDriver.cs +++ b/src/NUnitEngine/nunit.engine.core/Drivers/NotRunnableFrameworkDriver.cs @@ -36,16 +36,17 @@ public abstract class NotRunnableFrameworkDriver : IFrameworkDriver protected string _label; #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. - public NotRunnableFrameworkDriver(string assemblyPath, string message) + public NotRunnableFrameworkDriver(string assemblyPath, string id, string message) #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. { _name = Escape(Path.GetFileName(assemblyPath)); _fullname = Escape(Path.GetFullPath(assemblyPath)); _message = Escape(message); _type = new List { ".dll", ".exe" }.Contains(Path.GetExtension(assemblyPath)) ? "Assembly" : "Unknown"; + ID = id; } - public string ID { get; set; } + public string ID { get; } public string Load(string assemblyPath, IDictionary settings) @@ -90,21 +91,13 @@ private string GetLoadResult() _type, TestID, _name, _fullname, _runstate, _message); } - private string TestID - { - get - { - return string.IsNullOrEmpty(ID) - ? "1" - : ID + "-1"; - } - } + private string TestID => ID + "-1"; } public class InvalidAssemblyFrameworkDriver :NotRunnableFrameworkDriver { - public InvalidAssemblyFrameworkDriver(string assemblyPath, string message) - : base(assemblyPath, message) + public InvalidAssemblyFrameworkDriver(string assemblyPath, string id, string message) + : base(assemblyPath, id, message) { _runstate = "NotRunnable"; _result = "Failed"; @@ -114,8 +107,8 @@ public InvalidAssemblyFrameworkDriver(string assemblyPath, string message) public class SkippedAssemblyFrameworkDriver : NotRunnableFrameworkDriver { - public SkippedAssemblyFrameworkDriver(string assemblyPath) - : base(assemblyPath, "Skipping non-test assembly") + public SkippedAssemblyFrameworkDriver(string assemblyPath, string id) + : base(assemblyPath, id, "Skipping non-test assembly") { _runstate = "Runnable"; _result = "Skipped"; diff --git a/src/NUnitEngine/nunit.engine.core/Internal/ProvidedPathsAssemblyResolver.cs b/src/NUnitEngine/nunit.engine.core/Internal/ProvidedPathsAssemblyResolver.cs index 8b178e7bd..4a55fc192 100644 --- a/src/NUnitEngine/nunit.engine.core/Internal/ProvidedPathsAssemblyResolver.cs +++ b/src/NUnitEngine/nunit.engine.core/Internal/ProvidedPathsAssemblyResolver.cs @@ -12,6 +12,8 @@ public class ProvidedPathsAssemblyResolver { static readonly ILogger log = InternalTrace.GetLogger(typeof(ProvidedPathsAssemblyResolver)); + static readonly string THIS_ASSEMBLY_LOCATION = Assembly.GetExecutingAssembly().Location; + public ProvidedPathsAssemblyResolver() { _resolutionPaths = new List(); @@ -21,6 +23,8 @@ public void Install() { Debug.Assert(AppDomain.CurrentDomain.IsDefaultAppDomain()); AppDomain.CurrentDomain.AssemblyResolve += AssemblyResolve; + + AddPath(THIS_ASSEMBLY_LOCATION); } public void AddPath(string dirPath) diff --git a/src/NUnitEngine/nunit.engine.core/Runners/TestAgentRunner.cs b/src/NUnitEngine/nunit.engine.core/Runners/TestAgentRunner.cs index d32bd7766..d71e58f14 100644 --- a/src/NUnitEngine/nunit.engine.core/Runners/TestAgentRunner.cs +++ b/src/NUnitEngine/nunit.engine.core/Runners/TestAgentRunner.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Linq; using NUnit.Common; using NUnit.Engine.Drivers; using NUnit.Engine.Extensibility; @@ -17,10 +18,10 @@ namespace NUnit.Engine.Runners /// public abstract class TestAgentRunner : ITestEngineRunner { - private readonly List _drivers = new List(); - private readonly ProvidedPathsAssemblyResolver? _assemblyResolver; + private IFrameworkDriver? _driver; + protected AppDomain? TestDomain { get; set; } // Used to inject DriverService for testing @@ -47,10 +48,11 @@ public bool IsPackageLoaded public TestAgentRunner(TestPackage package) { Guard.ArgumentNotNull(package, nameof(package)); + //Guard.ArgumentValid(package.IsAssemblyPackage(), "TestAgentRunner requires a package with a single assembly", nameof(package)); var assemblyPackages = package.Select(p => !p.HasSubPackages()); Guard.ArgumentValid(assemblyPackages.Count == 1, "TestAgentRunner requires a package with a single assembly", nameof(package)); - TestPackage = assemblyPackages[0]; + TestPackage = package; // Bypass the resolver if not in the default AppDomain. This prevents trying to use the resolver within // NUnit's own automated tests (in a test AppDomain) which does not make sense anyway. @@ -71,27 +73,14 @@ public TestAgentRunner(TestPackage package) /// public TestEngineResult Explore(TestFilter filter) { - EnsurePackageIsLoaded(); - - var result = new TestEngineResult(); - - foreach (IFrameworkDriver driver in _drivers) + try { - string driverResult; - - try - { - driverResult = driver.Explore(filter.Text); - } - catch (Exception ex) when (!(ex is NUnitEngineException)) - { - throw new NUnitEngineException("An exception occurred in the driver while exploring tests.", ex); - } - - result.Add(driverResult); + return new TestEngineResult(GetLoadedDriver().Explore(filter.Text)); + } + catch (Exception ex) when (!(ex is NUnitEngineException)) + { + throw new NUnitEngineException("An exception occurred in the driver while exploring tests.", ex); } - - return result; } /// @@ -101,53 +90,34 @@ public TestEngineResult Explore(TestFilter filter) public virtual TestEngineResult Load() { Guard.OperationValid(TestDomain != null, "TestDomain is not set"); - AppDomain testDomain = TestDomain; var result = new TestEngineResult(); - // DirectRunner may be called with a single-assembly package, - // a set of assemblies as subpackages or even an arbitrary - // hierarchy of packages and subpackages with assemblies - // found in the terminal nodes. - var packagesToLoad = TestPackage.Select(p => !p.HasSubPackages()); + // The TestAgentRunner constructor guarantees that TestPackage has + // only a single assembly. + var assemblyPackage = TestPackage.Select(p => !p.HasSubPackages()).First(); if (DriverService == null) DriverService = new DriverService(); - _drivers.Clear(); + var testFile = assemblyPackage.FullName!; // We know it's an assembly - foreach (var subPackage in packagesToLoad) - { - var testFile = subPackage.FullName!; // We know it's an assembly - - string? targetFramework = subPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string?)null); - bool skipNonTestAssemblies = subPackage.GetSetting(EnginePackageSettings.SkipNonTestAssemblies, false); - - if (_assemblyResolver != null && !testDomain.IsDefaultAppDomain() - && subPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false)) - { - // It's OK to do this in the loop because the Add method - // checks to see if the path is already present. - _assemblyResolver.AddPathFromFile(testFile); - } - - IFrameworkDriver driver = DriverService.GetDriver(testDomain, testFile, targetFramework, skipNonTestAssemblies); + string? targetFramework = assemblyPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string?)null); + bool skipNonTestAssemblies = assemblyPackage.GetSetting(EnginePackageSettings.SkipNonTestAssemblies, false); - driver.ID = subPackage.ID; - result.Add(LoadDriver(driver, testFile, subPackage)); - _drivers.Add(driver); + if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain() + && assemblyPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false)) + { + // It's OK to do this in the loop because the Add method + // checks to see if the path is already present. + _assemblyResolver.AddPathFromFile(testFile); } - return result; - } - public virtual void Unload() { } - public TestEngineResult Reload() => Load(); + _driver = DriverService.GetDriver(TestDomain, assemblyPackage, testFile, targetFramework, skipNonTestAssemblies); - private static string LoadDriver(IFrameworkDriver driver, string testFile, TestPackage subPackage) - { try { - return driver.Load(testFile, subPackage.Settings); + return new TestEngineResult(_driver.Load(testFile, assemblyPackage.Settings)); } catch (Exception ex) when (ex is not NUnitEngineException) { @@ -155,6 +125,9 @@ private static string LoadDriver(IFrameworkDriver driver, string testFile, TestP } } + public virtual void Unload() { } + public TestEngineResult Reload() => Load(); + /// /// Count the test cases that would be run under /// the specified filter. @@ -163,23 +136,14 @@ private static string LoadDriver(IFrameworkDriver driver, string testFile, TestP /// The count of test cases public int CountTestCases(TestFilter filter) { - EnsurePackageIsLoaded(); - - int count = 0; - - foreach (IFrameworkDriver driver in _drivers) + try { - try - { - count += driver.CountTestCases(filter.Text); - } - catch (Exception ex) when (!(ex is NUnitEngineException)) - { - throw new NUnitEngineException("An exception occurred in the driver while counting test cases.", ex); - } + return GetLoadedDriver().CountTestCases(filter.Text); + } + catch (Exception ex) when (!(ex is NUnitEngineException)) + { + throw new NUnitEngineException("An exception occurred in the driver while counting test cases.", ex); } - - return count; } @@ -193,33 +157,15 @@ public int CountTestCases(TestFilter filter) /// public TestEngineResult Run(ITestEventListener? listener, TestFilter filter) { - EnsurePackageIsLoaded(); - - var result = new TestEngineResult(); - - foreach (IFrameworkDriver driver in _drivers) + try { - string driverResult; - - try - { - driverResult = driver.Run(listener, filter.Text); - } - catch (Exception ex) when (!(ex is NUnitEngineException)) - { - throw new NUnitEngineException("An exception occurred in the driver while running tests.", ex); - } - - result.Add(driverResult); + return new TestEngineResult(GetLoadedDriver().Run(listener, filter.Text)); } - - if (_assemblyResolver != null) + catch (Exception ex) when (!(ex is NUnitEngineException)) { - foreach (var package in TestPackage.Select(p => p.IsAssemblyPackage())) - _assemblyResolver.RemovePathFromFile(package.FullName!); // IsAssemblyPackage guarantees FullName is not null + throw new NUnitEngineException("An exception occurred in the driver while running tests.", ex); } - return result; } public AsyncTestEngineResult RunAsync(ITestEventListener? listener, TestFilter filter) @@ -245,25 +191,22 @@ public AsyncTestEngineResult RunAsync(ITestEventListener? listener, TestFilter f /// If true, cancel any ongoing test threads, otherwise wait for them to complete. public void StopRun(bool force) { - EnsurePackageIsLoaded(); - - foreach (IFrameworkDriver driver in _drivers) + try { - try - { - driver.StopRun(force); - } - catch (Exception ex) when (!(ex is NUnitEngineException)) - { - throw new NUnitEngineException("An exception occurred in the driver while stopping the run.", ex); - } + GetLoadedDriver().StopRun(force); + } + catch (Exception ex) when (!(ex is NUnitEngineException)) + { + throw new NUnitEngineException("An exception occurred in the driver while stopping the run.", ex); } } - private void EnsurePackageIsLoaded() + private IFrameworkDriver GetLoadedDriver() { if (!IsPackageLoaded) LoadResult = Load(); + + return _driver.ShouldNotBeNull(); } public void Dispose() diff --git a/src/NUnitEngine/nunit.engine.tests/Services/TestFilteringTests.cs b/src/NUnitEngine/nunit.engine.tests/Services/TestFilteringTests.cs index 71ff0a44b..667cf56fa 100644 --- a/src/NUnitEngine/nunit.engine.tests/Services/TestFilteringTests.cs +++ b/src/NUnitEngine/nunit.engine.tests/Services/TestFilteringTests.cs @@ -14,21 +14,19 @@ public class TestFilteringTests { private const string MOCK_ASSEMBLY = "mock-assembly.dll"; -#if NETCOREAPP3_1_OR_GREATER - private NUnitNetCore31Driver _driver; -#else - private NUnit3FrameworkDriver _driver; -#endif + private NUnitFrameworkDriver _driver; [SetUp] public void LoadAssembly() { var mockAssemblyPath = System.IO.Path.Combine(TestContext.CurrentContext.TestDirectory, MOCK_ASSEMBLY); -#if NETCOREAPP3_1_OR_GREATER - _driver = new NUnitNetCore31Driver(); + var nunitRef = typeof(TestAttribute).Assembly.GetName(); + var assemblyPath = typeof(TestAttribute).Assembly.Location; + string driverId = "99"; +#if NETFRAMEWORK + _driver = new NUnitFrameworkDriver(AppDomain.CurrentDomain, driverId, nunitRef); #else - var assemblyName = typeof(NUnit.Framework.TestAttribute).Assembly.GetName(); - _driver = new NUnit3FrameworkDriver(AppDomain.CurrentDomain, assemblyName); + _driver = new NUnitFrameworkDriver(driverId, nunitRef); #endif _driver.Load(mockAssemblyPath, new Dictionary()); } diff --git a/src/NUnitEngine/nunit.engine/Runners/ProcessRunner.cs b/src/NUnitEngine/nunit.engine/Runners/ProcessRunner.cs index f04d3b7cf..c708bc895 100644 --- a/src/NUnitEngine/nunit.engine/Runners/ProcessRunner.cs +++ b/src/NUnitEngine/nunit.engine/Runners/ProcessRunner.cs @@ -15,13 +15,8 @@ namespace NUnit.Engine.Runners /// public class ProcessRunner : TestEngineRunner { - // ProcessRunner is given a TestPackage containing a single assembly - // multiple assemblies, a project, multiple projects or a mix. It loads - // and runs all tests in a single remote agent process. - // - // If the input contains projects, which are not summarized at a lower - // level, the ProcessRunner should create an XML node for the entire - // project, aggregating the assembly results. + // ProcessRunner is given a TestPackage containing a single assembly. + // It loads and runs the test assembly in a single remote agent process. private static readonly Logger log = InternalTrace.GetLogger(typeof(ProcessRunner)); @@ -33,6 +28,9 @@ public class ProcessRunner : TestEngineRunner public ProcessRunner(IServiceLocator services, TestPackage package) : base(services, package) { _agency = Services.GetService(); + + var assemblyPackages = package.Select(p => !p.HasSubPackages()); + Guard.ArgumentValid(assemblyPackages.Count == 1, $"{GetType().Name} requires a package with a single assembly", nameof(package)); } /// diff --git a/src/NUnitEngine/nunit.engine/Runners/WorkItemTracker.cs b/src/NUnitEngine/nunit.engine/Runners/WorkItemTracker.cs index f18231a9b..88aa94b11 100644 --- a/src/NUnitEngine/nunit.engine/Runners/WorkItemTracker.cs +++ b/src/NUnitEngine/nunit.engine/Runners/WorkItemTracker.cs @@ -1,7 +1,9 @@ // Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt +using NUnit.Engine.Internal; using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Text; using System.Threading; @@ -63,6 +65,8 @@ public int CompareTo(InProgressItem? other) } } + private static readonly ILogger log = InternalTrace.GetLogger(nameof(InProgressItem)); + // items are keyed by id private readonly Dictionary _itemsInProcess = new Dictionary(); private readonly ManualResetEvent _allItemsComplete = new ManualResetEvent(false); diff --git a/src/TestData/FakeExtensions/FakeExtensions.cs b/src/TestData/FakeExtensions/FakeExtensions.cs index f0ac2be6e..41fb228bf 100644 --- a/src/TestData/FakeExtensions/FakeExtensions.cs +++ b/src/TestData/FakeExtensions/FakeExtensions.cs @@ -12,10 +12,10 @@ namespace NUnit.Engine.Tests [Extension] public class DummyFrameworkDriverExtension : IDriverFactory { -#if !NETFRAMEWORK - public IFrameworkDriver GetDriver(AssemblyName reference) +#if NETFRAMEWORK + public IFrameworkDriver GetDriver(AppDomain domain, string id, AssemblyName reference) #else - public IFrameworkDriver GetDriver(AppDomain domain, AssemblyName reference) + public IFrameworkDriver GetDriver(string id, AssemblyName reference) #endif { throw new NotImplementedException(); diff --git a/src/TestData/NUnit3.0.1/Class1.cs b/src/TestData/NUnit3.0.1/Class1.cs new file mode 100644 index 000000000..350273d1d --- /dev/null +++ b/src/TestData/NUnit3.0.1/Class1.cs @@ -0,0 +1,13 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +using System; +using NUnit.Framework; + +namespace NUnit.Engine.TestData +{ + public class NUnit_3_0_1_Test + { + [Test] + public void Test() { Assert.Pass("test passes"); } + } +} diff --git a/src/TestData/NUnit3.0.1/NUnit3.0.1.csproj b/src/TestData/NUnit3.0.1/NUnit3.0.1.csproj new file mode 100644 index 000000000..a88c94326 --- /dev/null +++ b/src/TestData/NUnit3.0.1/NUnit3.0.1.csproj @@ -0,0 +1,13 @@ + + + + NUnit.TestData + net462 + ../../../bin/$(Configuration)/testdata/nunit3.0.1 + + + + + + + diff --git a/src/TestData/NUnit3.0/NUnit3.0.csproj b/src/TestData/NUnit3.0/NUnit3.0.csproj new file mode 100644 index 000000000..ede18b3e3 --- /dev/null +++ b/src/TestData/NUnit3.0/NUnit3.0.csproj @@ -0,0 +1,13 @@ + + + + NUnit.TestData + net462 + ../../../bin/$(Configuration)/testdata/nunit3.0 + + + + + + + diff --git a/src/TestData/NUnit3.0/NUnit_3_0_Test.cs b/src/TestData/NUnit3.0/NUnit_3_0_Test.cs new file mode 100644 index 000000000..d2be48895 --- /dev/null +++ b/src/TestData/NUnit3.0/NUnit_3_0_Test.cs @@ -0,0 +1,13 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +using System; +using NUnit.Framework; + +namespace NUnit.Engine.TestData +{ + public class NUnit_3_0_Test + { + [Test] + public void Test() { Assert.Pass("test passes"); } + } +} diff --git a/src/TestData/NUnit3.10/NUnit3.10.csproj b/src/TestData/NUnit3.10/NUnit3.10.csproj new file mode 100644 index 000000000..ac6ddfb5f --- /dev/null +++ b/src/TestData/NUnit3.10/NUnit3.10.csproj @@ -0,0 +1,13 @@ + + + + NUnit.TestData + net462 + ../../../bin/$(Configuration)/testdata/nunit3.10 + + + + + + + diff --git a/src/TestData/NUnit3.10/NUnit_3_10_Test.cs b/src/TestData/NUnit3.10/NUnit_3_10_Test.cs new file mode 100644 index 000000000..ff2727b9d --- /dev/null +++ b/src/TestData/NUnit3.10/NUnit_3_10_Test.cs @@ -0,0 +1,13 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +using System; +using NUnit.Framework; + +namespace NUnit.Engine.TestData +{ + public class NUnit_3_10_Test + { + [Test] + public void Test() { Assert.Pass("test passes"); } + } +} diff --git a/src/TestData/NUnit3.2/NUnit3.2.csproj b/src/TestData/NUnit3.2/NUnit3.2.csproj new file mode 100644 index 000000000..d0a9e0376 --- /dev/null +++ b/src/TestData/NUnit3.2/NUnit3.2.csproj @@ -0,0 +1,13 @@ + + + + NUnit.TestData + net462 + ../../../bin/$(Configuration)/testdata/nunit3.2 + + + + + + + diff --git a/src/TestData/NUnit3.2/NUnit_3_2_Test.cs b/src/TestData/NUnit3.2/NUnit_3_2_Test.cs new file mode 100644 index 000000000..adb726a7b --- /dev/null +++ b/src/TestData/NUnit3.2/NUnit_3_2_Test.cs @@ -0,0 +1,13 @@ +// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt + +using System; +using NUnit.Framework; + +namespace NUnit.Engine.TestData +{ + public class NUnit_3_2_Test + { + [Test] + public void Test() { Assert.Pass("test passes"); } + } +}