From 28ab3bcae95d09e7014987b3a79722622089b388 Mon Sep 17 00:00:00 2001 From: abhishkk Date: Tue, 8 May 2018 12:47:34 +0530 Subject: [PATCH 1/6] Parent test result support for data driven tests --- src/Adapter/MSTest.CoreAdapter/Constants.cs | 9 ++ .../Execution/TestMethodRunner.cs | 47 +++++++++ .../Extensions/TestResultExtensions.cs | 3 + .../ObjectModel/UnitTestResult.cs | 19 ++++ .../Attributes/VSTestAttributes.cs | 15 +++ .../Execution/TestMethodRunnerTests.cs | 98 +++++++++++++++++-- 6 files changed, 181 insertions(+), 10 deletions(-) diff --git a/src/Adapter/MSTest.CoreAdapter/Constants.cs b/src/Adapter/MSTest.CoreAdapter/Constants.cs index 5f561f1519..2e91aa4f81 100644 --- a/src/Adapter/MSTest.CoreAdapter/Constants.cs +++ b/src/Adapter/MSTest.CoreAdapter/Constants.cs @@ -44,6 +44,12 @@ internal static class Constants internal static readonly TestProperty DoNotParallelizeProperty = TestProperty.Register("MSTestDiscoverer.DoNotParallelize", DoNotParallelizeLabel, typeof(bool), TestPropertyAttributes.Hidden, typeof(TestCase)); + internal static readonly TestProperty ExecutionIdProperty = TestProperty.Register("ExecutionId", ExecutionIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult)); + + internal static readonly TestProperty ParentExecIdProperty = TestProperty.Register("ParentExecId", ParentExecIdLabel, typeof(Guid), TestPropertyAttributes.Hidden, typeof(TestResult)); + + internal static readonly TestProperty InnerResultsCountProperty = TestProperty.Register("InnerResultsCount", InnerResultsCountLabel, typeof(int), TestPropertyAttributes.Hidden, typeof(TestResult)); + #endregion #region Private Constants @@ -59,6 +65,9 @@ internal static class Constants private const string PriorityLabel = "Priority"; private const string DeploymentItemsLabel = "DeploymentItems"; private const string DoNotParallelizeLabel = "DoNotParallelize"; + private const string ExecutionIdLabel = "ExecutionId"; + private const string ParentExecIdLabel = "ParentExecId"; + private const string InnerResultsCountLabel = "InnerResultsCount"; #endregion } diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 2ece924fcc..05e57dbe94 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -216,6 +216,15 @@ internal UnitTestResult[] RunTestMethod() List results = new List(); + // Parent result. Added in properties bag only when results are greater than 1. + var parentResultWatch = new Stopwatch(); + parentResultWatch.Start(); + var parentResult = new UTF.TestResult + { + Outcome = UTF.UnitTestOutcome.InProgress, + ExecutionId = Guid.NewGuid() + }; + if (this.testMethodInfo.TestMethodOptions.Executor != null) { UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes(false); @@ -373,7 +382,45 @@ internal UnitTestResult[] RunTestMethod() results.Add(new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Unknown, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) }); } + // Adds parent result info in results if required. + parentResultWatch.Stop(); + parentResult.Duration = parentResultWatch.Elapsed; + results = this.UpdateResultsWithParentInfo(results, parentResult); + return results.ToArray().ToUnitTestResults(); } + + /// + /// Updates given resutls with parent info if results are greater than 1. + /// Add parent results as first result in updated result. + /// + /// Results. + /// Parent results. + /// Updated results which contains parent result as first result. All other results contains parent result info. + private List UpdateResultsWithParentInfo(List results, UTF.TestResult parentResult) + { + // Add parent result only when results are greater than 1. + if (results.Count <= 1) + { + return results; + } + + // UpdatedResults contain parent result at first position and results contains parent info. + var updatedResults = new List(); + updatedResults.Add(parentResult); + + parentResult.ExecutionId = Guid.NewGuid(); + foreach (var result in results) + { + result.ExecutionId = Guid.NewGuid(); + result.ParentExecId = parentResult.ExecutionId; + parentResult.Outcome = UnitTestOutcomeExtensions.GetMoreImportantOutcome(parentResult.Outcome, result.Outcome); + parentResult.InnerResultsCount++; + + updatedResults.Add(result); + } + + return updatedResults; + } } } diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs index 2edc05d058..60f3794969 100644 --- a/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs +++ b/src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs @@ -47,6 +47,9 @@ public static UnitTestResult[] ToUnitTestResults(this UTF.TestResult[] testResul unitTestResult.DisplayName = testResults[i].DisplayName; unitTestResult.DatarowIndex = testResults[i].DatarowIndex; unitTestResult.ResultFiles = testResults[i].ResultFiles; + unitTestResult.ExecutionId = testResults[i].ExecutionId; + unitTestResult.ParentExecId = testResults[i].ParentExecId; + unitTestResult.InnerResultsCount = testResults[i].InnerResultsCount; unitTestResults[i] = unitTestResult; } diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs index b04e8d9404..c8aae7fd21 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs @@ -75,6 +75,21 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage) /// public string ErrorStackTrace { get; internal set; } + /// + /// Gets the execution id of the result + /// + public Guid ExecutionId { get; internal set; } + + /// + /// Gets the parent execution id of the result + /// + public Guid ParentExecId { get; internal set; } + + /// + /// Gets the inner results count of the result + /// + public int InnerResultsCount { get; internal set; } + /// /// Gets the duration of the result /// @@ -149,6 +164,10 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da EndTime = endTime }; + testResult.SetPropertyValue(Constants.ExecutionIdProperty, this.ExecutionId); + testResult.SetPropertyValue(Constants.ParentExecIdProperty, this.ParentExecId); + testResult.SetPropertyValue(Constants.InnerResultsCountProperty, this.InnerResultsCount); + if (!string.IsNullOrEmpty(this.StandardOut)) { TestResultMessage message = new TestResultMessage(TestResultMessage.StandardOutCategory, this.StandardOut); diff --git a/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs b/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs index 78977e1d80..474499fbbc 100644 --- a/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs +++ b/src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs @@ -411,6 +411,21 @@ public TestResult() /// public string TestContextMessages { get; set; } + /// + /// Gets or sets the execution id of the result. + /// + public Guid ExecutionId { get; set; } + + /// + /// Gets or sets the parent execution id of the result. + /// + public Guid ParentExecId { get; set; } + + /// + /// Gets or sets the inner results count of the result. + /// + public int InnerResultsCount { get; set; } + /// /// Gets or sets the duration of test execution. /// diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs index 3dbd30d23e..923c6b183c 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs @@ -429,9 +429,11 @@ public void RunTestMethodForMultipleResultsReturnMultipleResults() var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); var results = testMethodRunner.Execute(); - Assert.AreEqual(2, results.Length); - Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + Assert.AreEqual(3, results.Length); + + // results[0] is parent result. + Assert.AreEqual(AdapterTestOutcome.Passed, results[1].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[2].Outcome); } [TestMethodV1] @@ -543,9 +545,11 @@ public void RunTestMethodShouldSetDataRowIndexForDataDrivenTestsWhenDataIsProvid var results = testMethodRunner.RunTestMethod(); // check for datarowIndex - Assert.AreEqual(results[0].DatarowIndex, 0); - Assert.AreEqual(results[1].DatarowIndex, 1); - Assert.AreEqual(results[2].DatarowIndex, 2); + // 1st is parent result. + Assert.AreEqual(results[0].DatarowIndex, -1); + Assert.AreEqual(results[1].DatarowIndex, 0); + Assert.AreEqual(results[2].DatarowIndex, 1); + Assert.AreEqual(results[3].DatarowIndex, 2); } [TestMethodV1] @@ -570,9 +574,11 @@ public void RunTestMethodShoudlRunOnlyDataSourceTestsWhenBothDataSourceAndDataRo var results = testMethodRunner.RunTestMethod(); // check for datarowIndex as only DataSource Tests are Run - Assert.AreEqual(results[0].DatarowIndex, 0); - Assert.AreEqual(results[1].DatarowIndex, 1); - Assert.AreEqual(results[2].DatarowIndex, 2); + // 1st is parent result. + Assert.AreEqual(results[0].DatarowIndex, -1); + Assert.AreEqual(results[1].DatarowIndex, 0); + Assert.AreEqual(results[2].DatarowIndex, 1); + Assert.AreEqual(results[3].DatarowIndex, 2); } [TestMethodV1] @@ -640,8 +646,80 @@ public void RunTestMethodShouldSetResultFilesIfPresentForDataDrivenTests() this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); - CollectionAssert.Contains(results[0].ResultFiles.ToList(), "C:\\temp.txt"); CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); + CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); + } + + [TestMethodV1] + public void RunTestMethodShouldReturnParentResultForDataDrivenTests() + { + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); + + var attribs = new Attribute[] { dataSourceAttribute }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); + + var results = testMethodRunner.RunTestMethod(); + + // check for parent result + Assert.AreEqual(4, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldNotReturnParentResultForNonDataDrivenTests() + { + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); + + var attribs = new Attribute[] { dataSourceAttribute }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1 }); + + var results = testMethodRunner.RunTestMethod(); + + // Parent result should not exist. + Assert.AreEqual(1, results.Length); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldSetParentResultOutcomeProperly() + { + var testMethodAttributeMock = new Mock(); + testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[] + { + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }, + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } + }); + + var localTestMethodOptions = new TestMethodOptions + { + Timeout = 200, + Executor = testMethodAttributeMock.Object, + TestContext = this.testContextImplementation, + ExpectedException = null + }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + var results = testMethodRunner.Execute(); + Assert.AreEqual(3, results.Length); + + // Parent result should show proper aggregate outcome. + Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[1].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[2].Outcome); } #region Test data From 7829b4516fcb0b2277ba8839f2c9536c12570da9 Mon Sep 17 00:00:00 2001 From: abhishkk Date: Tue, 8 May 2018 15:36:50 +0530 Subject: [PATCH 2/6] smoke tests fix --- test/E2ETests/Automation.CLI/CLITestBase.cs | 10 ++++++++++ .../CustomTestExecutionExtensibilityTests.cs | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index f846b3e69a..059615ebc4 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -140,6 +140,16 @@ public void ValidateFailedTests(string source, params string[] failedTests) this.ValidateFailedTestsContain(source, failedTests); } + /// + /// Validates the count of failed tests. + /// + /// Expected failed tests count. + public void ValidateFailedTestsCount(int expectedFailedTestsCount) + { + // Make sure only expected number of tests failed and not more. + Assert.AreEqual(expectedFailedTestsCount, this.runEventsHandler.FailedTests.Count); + } + /// /// Validates if the test results have the specified set of skipped tests. /// diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs index 23f7cd81c9..eea8e4b3d1 100644 --- a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -38,7 +38,10 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() "CustomTestMethod2 (B)", "CustomTestMethod2 (B)", "CustomTestMethod2 (B)"); - this.ValidateFailedTests( + + // Parent results should fail and thus failed count should be 7. + this.ValidateFailedTestsCount(7); + this.ValidateFailedTestsContain( TestAssembly, "CustomTestMethod2 (A)", "CustomTestMethod2 (A)", From 311a492e8c9cf53879feb87ccfe642db9b5e7818 Mon Sep 17 00:00:00 2001 From: abhishkk Date: Tue, 8 May 2018 21:47:33 +0530 Subject: [PATCH 3/6] review comments --- .../Execution/TestMethodRunner.cs | 75 +++++++++++-------- .../Extensions/UnitTestOutcomeExtensions.cs | 4 +- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 05e57dbe94..457d26ae57 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -215,6 +215,7 @@ internal UnitTestResult[] RunTestMethod() Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null."); List results = new List(); + var isDataDriven = false; // Parent result. Added in properties bag only when results are greater than 1. var parentResultWatch = new Stopwatch(); @@ -230,6 +231,7 @@ internal UnitTestResult[] RunTestMethod() UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes(false); if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) { + isDataDriven = true; Stopwatch watch = new Stopwatch(); watch.Start(); @@ -305,6 +307,7 @@ internal UnitTestResult[] RunTestMethod() if (testDataSources != null && testDataSources.Length > 0) { + isDataDriven = true; foreach (var testDataSource in testDataSources) { foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) @@ -354,42 +357,52 @@ internal UnitTestResult[] RunTestMethod() this.testMethodInfo.TestMethodName); } - if (results != null && results.Count > 0) - { - // aggregate for data driven tests - UTF.UnitTestOutcome aggregateOutcome = UTF.UnitTestOutcome.Passed; + parentResultWatch.Stop(); + parentResult.Duration = parentResultWatch.Elapsed; - foreach (var result in results) - { - if (result.Outcome != UTF.UnitTestOutcome.Passed) - { - if (aggregateOutcome != UTF.UnitTestOutcome.Failed) - { - if (result.Outcome == UTF.UnitTestOutcome.Failed - || aggregateOutcome != UTF.UnitTestOutcome.Timeout) - { - aggregateOutcome = result.Outcome; - } - } - } - } + // Get aggregate outcome. + var aggregateOutcome = this.GetAggregateOutcome(results); + this.testContext.SetOutcome(aggregateOutcome); - this.testContext.SetOutcome(aggregateOutcome); - } - else + // Set a result in case no result is present. + if (!results.Any()) { - this.testContext.SetOutcome(UTF.UnitTestOutcome.Unknown); - results.Add(new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Unknown, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) }); + results.Add(new UTF.TestResult() { Outcome = aggregateOutcome, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) }); } - // Adds parent result info in results if required. - parentResultWatch.Stop(); - parentResult.Duration = parentResultWatch.Elapsed; - results = this.UpdateResultsWithParentInfo(results, parentResult); + // In case of data driven, set parent info in results. + if (isDataDriven) + { + parentResult.Outcome = aggregateOutcome; + results = this.UpdateResultsWithParentInfo(results, parentResult); + } return results.ToArray().ToUnitTestResults(); } + /// + /// Gets aggregate outcome. + /// + /// Results. + /// Aggregate outcome. + private UTF.UnitTestOutcome GetAggregateOutcome(List results) + { + // In case results are not present, set outcome as unknown. + if (!results.Any()) + { + return UTF.UnitTestOutcome.Unknown; + } + + // Get aggregate outcome. + var aggregateOutcome = results[0].Outcome; + foreach (var result in results) + { + UnitTestOutcomeExtensions.GetMoreImportantOutcome(aggregateOutcome, result.Outcome); + } + + return aggregateOutcome; + } + /// /// Updates given resutls with parent info if results are greater than 1. /// Add parent results as first result in updated result. @@ -399,22 +412,20 @@ internal UnitTestResult[] RunTestMethod() /// Updated results which contains parent result as first result. All other results contains parent result info. private List UpdateResultsWithParentInfo(List results, UTF.TestResult parentResult) { - // Add parent result only when results are greater than 1. - if (results.Count <= 1) + // Return results in case there are no results. + if (!results.Any()) { return results; } - // UpdatedResults contain parent result at first position and results contains parent info. + // UpdatedResults contain parent result at first position and remaining results has parent info updated. var updatedResults = new List(); updatedResults.Add(parentResult); - parentResult.ExecutionId = Guid.NewGuid(); foreach (var result in results) { result.ExecutionId = Guid.NewGuid(); result.ParentExecId = parentResult.ExecutionId; - parentResult.Outcome = UnitTestOutcomeExtensions.GetMoreImportantOutcome(parentResult.Outcome, result.Outcome); parentResult.InnerResultsCount++; updatedResults.Add(result); diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs index d123b9a661..2df6c87710 100644 --- a/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs +++ b/src/Adapter/MSTest.CoreAdapter/Extensions/UnitTestOutcomeExtensions.cs @@ -61,7 +61,9 @@ public static UnitTestOutcome ToUnitTestOutcome(this UTF.UnitTestOutcome framewo /// Outcome which has higher importance. internal static UTF.UnitTestOutcome GetMoreImportantOutcome(this UTF.UnitTestOutcome outcome1, UTF.UnitTestOutcome outcome2) { - return outcome1 < outcome2 ? outcome1 : outcome2; + var unitTestOutcome1 = outcome1.ToUnitTestOutcome(); + var unitTestOutcome2 = outcome2.ToUnitTestOutcome(); + return unitTestOutcome1 < unitTestOutcome2 ? outcome1 : outcome2; } } } From 1da1628c4948a87e4b401bb069f0402a90012638 Mon Sep 17 00:00:00 2001 From: abhishkk Date: Wed, 9 May 2018 15:37:04 +0530 Subject: [PATCH 4/6] review comments and UTs --- .../Execution/TestMethodRunner.cs | 2 +- .../Execution/TestMethodRunnerTests.cs | 170 ++++++++++++++++-- 2 files changed, 156 insertions(+), 16 deletions(-) diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 457d26ae57..4ac11553a2 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -397,7 +397,7 @@ private UTF.UnitTestOutcome GetAggregateOutcome(List results) var aggregateOutcome = results[0].Outcome; foreach (var result in results) { - UnitTestOutcomeExtensions.GetMoreImportantOutcome(aggregateOutcome, result.Outcome); + aggregateOutcome = UnitTestOutcomeExtensions.GetMoreImportantOutcome(aggregateOutcome, result.Outcome); } return aggregateOutcome; diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs index 923c6b183c..9419502f2d 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs @@ -429,11 +429,10 @@ public void RunTestMethodForMultipleResultsReturnMultipleResults() var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); var results = testMethodRunner.Execute(); - Assert.AreEqual(3, results.Length); + Assert.AreEqual(2, results.Length); - // results[0] is parent result. - Assert.AreEqual(AdapterTestOutcome.Passed, results[1].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[2].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); } [TestMethodV1] @@ -601,7 +600,10 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowDisplayNameIfProvided this.testablePlatformServiceProvider.MockReflectionOperations.Setup(ro => ro.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); - Assert.AreEqual(results[0].DisplayName, "DataRowTestDisplayName"); + + // 1st results should be parent result. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[1].DisplayName, "DataRowTestDisplayName"); } [TestMethodV1] @@ -623,7 +625,10 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowArgumentsIfNoDisplayN this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); - Assert.AreEqual(results[0].DisplayName, "DummyTestMethod (2,DummyString)"); + + // 1st results should be parent result. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[1].DisplayName, "DummyTestMethod (2,DummyString)"); } [TestMethodV1] @@ -651,7 +656,7 @@ public void RunTestMethodShouldSetResultFilesIfPresentForDataDrivenTests() } [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataDrivenTests() + public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTests() { var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); @@ -669,10 +674,12 @@ public void RunTestMethodShouldReturnParentResultForDataDrivenTests() // check for parent result Assert.AreEqual(4, results.Length); Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - } + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + } [TestMethodV1] - public void RunTestMethodShouldNotReturnParentResultForNonDataDrivenTests() + public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTestsContainingSingleTest() { var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); @@ -687,13 +694,74 @@ public void RunTestMethodShouldNotReturnParentResultForNonDataDrivenTests() var results = testMethodRunner.RunTestMethod(); - // Parent result should not exist. - Assert.AreEqual(1, results.Length); + // Parent result should exist. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTests() + { + UTF.TestResult testResult = new UTF.TestResult(); + testResult.ResultFiles = new List() { "C:\\temp.txt" }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); + + int dummyIntData1 = 1; + int dummyIntData2 = 2; + UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); + UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); + + var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + + var results = testMethodRunner.RunTestMethod(); + CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); + CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); + + // Parent result should exist. + Assert.AreEqual(3, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); + } + + [TestMethodV1] + public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTestsContainingSingleTest() + { + UTF.TestResult testResult = new UTF.TestResult(); + testResult.ResultFiles = new List() { "C:\\temp.txt" }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); + + int dummyIntData1 = 1; + UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); + + var attribs = new Attribute[] { dataRowAttribute1 }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + + var results = testMethodRunner.RunTestMethod(); + CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); + + // Parent result should exist. + Assert.AreEqual(2, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); } [TestMethodV1] - public void RunTestMethodShouldSetParentResultOutcomeProperly() + public void RunTestMethodShouldNotReturnParentResultForNonDataDrivenTests() { var testMethodAttributeMock = new Mock(); testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[] @@ -714,12 +782,84 @@ public void RunTestMethodShouldSetParentResultOutcomeProperly() var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); var results = testMethodRunner.Execute(); + Assert.AreEqual(2, results.Length); + + // Parent result should not exists as its not data driven test. + Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + } + + [TestMethodV1] + public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataSourceDataDrivenTests() + { + var testExecutedCount = 0; + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => + { + return (testExecutedCount++ == 0) ? + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; + }); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); + + var attribs = new Attribute[] { dataSourceAttribute }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); + + var results = testMethodRunner.RunTestMethod(); + + // check for parent result + Assert.AreEqual(4, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + + // Check for aggregate outcome. + Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[3].Outcome); + } + + [TestMethodV1] + public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataRowDataDrivenTests() + { + var testExecutedCount = 0; + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => + { + return (testExecutedCount++ == 0) ? + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; + }); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); + + int dummyIntData1 = 1; + int dummyIntData2 = 2; + UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); + UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); + + var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; + + // Setup mocks + this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); + + var results = testMethodRunner.RunTestMethod(); + + // Parent result should exist. Assert.AreEqual(3, results.Length); + Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); + Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); + Assert.AreEqual(Guid.Empty, results[0].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); + Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); - // Parent result should show proper aggregate outcome. + // Check for aggregate outcome. Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Passed, results[1].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[2].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); } #region Test data From fb92014fa68b14cd339f08b3636e3b1fc704727e Mon Sep 17 00:00:00 2001 From: abhishkk Date: Wed, 9 May 2018 15:48:25 +0530 Subject: [PATCH 5/6] UTs --- .../Extensions/TestResultExtensionsTests.cs | 24 +++++++++++++++++++ .../ObjectModel/UnitTestResultTests.cs | 21 ++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs index 518b7b2157..ad839a1d9b 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestResultExtensionsTests.cs @@ -171,6 +171,30 @@ public void ToUnitTestResultsForTestResultShouldSetDataRowIndex() Assert.AreEqual(1, convertedResults[0].DatarowIndex); } + [TestMethod] + public void ToUnitTestResultsForTestResultShouldSetParentInfo() + { + var executionId = Guid.NewGuid(); + var parentExecId = Guid.NewGuid(); + var innerResultsCount = 5; + + var results = new[] + { + new UTF.TestResult() + { + ExecutionId = executionId, + ParentExecId = parentExecId, + InnerResultsCount = innerResultsCount + } + }; + + var convertedResults = results.ToUnitTestResults(); + + Assert.AreEqual(executionId, convertedResults[0].ExecutionId); + Assert.AreEqual(parentExecId, convertedResults[0].ParentExecId); + Assert.AreEqual(innerResultsCount, convertedResults[0].InnerResultsCount); + } + [TestMethod] public void ToUnitTestResultsShouldHaveResultsFileProvidedToTestResult() { diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs index ce9b50329f..f6ce578efd 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs @@ -154,6 +154,27 @@ public void ToTestResultForUniTestResultWithNoResultFilesShouldReturnTestResultW Assert.AreEqual(testresult.Attachments.Count, 0); } + [TestMethod] + public void ToTestResultForUniTestResultWithParentInfoShouldReturnTestResultWithParentInfo() + { + var executionId = Guid.NewGuid(); + var parentExecId = Guid.NewGuid(); + var innerResultsCount = 5; + + UnitTestResult result = new UnitTestResult() + { + ExecutionId = executionId, + ParentExecId = parentExecId, + InnerResultsCount = innerResultsCount + }; + TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + Assert.AreEqual(executionId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ExecutionIdProperty)); + Assert.AreEqual(parentExecId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ParentExecIdProperty)); + Assert.AreEqual(innerResultsCount, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.InnerResultsCountProperty)); + } + [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomePassedShouldReturnTestOutcomePassed() { From 69252455425b7c7285a0c3e4b739ba59b4f38be3 Mon Sep 17 00:00:00 2001 From: abhishkk Date: Wed, 9 May 2018 16:16:51 +0530 Subject: [PATCH 6/6] review comments --- test/E2ETests/Automation.CLI/CLITestBase.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index 059615ebc4..2dc761445b 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -134,9 +134,7 @@ public void ValidatePassedTests(params string[] passedTests) /// public void ValidateFailedTests(string source, params string[] failedTests) { - // Make sure only expected number of tests failed and not more. - Assert.AreEqual(failedTests.Length, this.runEventsHandler.FailedTests.Count); - + this.ValidateFailedTestsCount(failedTests.Length); this.ValidateFailedTestsContain(source, failedTests); }