From 0444dc38d1a4edcd671dbc80bf676f865313eb17 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 23 Nov 2021 18:38:14 +0100 Subject: [PATCH 01/47] Complete attachment processors extension --- .../DesignMode/DesignModeClient.cs | 4 +- .../Execution/TestRunRequest.cs | 2 + .../DataCollection/AfterTestRunEndResult.cs | 11 +- .../DataCollection/DataCollectionManager.cs | 42 +- .../DataCollection/DataCollectorConfig.cs | 35 +- .../Interfaces/IDataCollectionManager.cs | 6 + .../DataCollectionExtensionManager.cs | 45 +- .../ExtensionFramework/TestPluginCache.cs | 8 +- .../TestPluginDiscoverer.cs | 13 +- .../Utilities/TestPluginInformation.cs | 9 + .../ITestRunAttachmentsProcessingManager.cs | 24 +- .../Interfaces/IDataCollectorCapabilities.cs | 10 + .../Logging/InternalTestLoggerEvents.cs | 17 +- .../DataCollectionRequestHandler.cs | 4 +- .../TestRequestSender.cs | 2 +- ...taCollectorAttachmentsProcessorsFactory.cs | 90 + .../TestRunAttachmentsProcessingManager.cs | 65 +- .../Client/InProcessProxyexecutionManager.cs | 2 +- .../Parallel/ParallelProxyExecutionManager.cs | 8 +- .../Parallel/ParallelRunDataAggregator.cs | 24 +- .../Parallel/ParallelRunEventsHandler.cs | 6 +- .../Client/ProxyExecutionManager.cs | 2 +- .../Client/ProxyOperationManager.cs | 7 +- .../Client/TestLoggerManager.cs | 2 +- .../DataCollection/DataCollectionResult.cs | 30 + .../DataCollectionTestRunEventsHandler.cs | 34 +- .../Interfaces/IProxyDataCollectionManager.cs | 4 +- .../ParallelDataCollectionEventsHandler.cs | 5 +- .../ProxyDataCollectionManager.cs | 8 +- .../EventHandlers/TestRequestHandler.cs | 2 +- .../Execution/BaseRunTests.cs | 2 + .../Execution/ExecutionManager.cs | 4 +- .../Friends.cs | 1 + .../TestEngine.cs | 14 +- .../Client/Events/TestRunCompleteEventArgs.cs | 10 +- .../TestRunAttachmentsProcessingPayload.cs | 12 + .../DataCollectorAttachmentProcessor.cs | 29 + .../IDataCollectorAttachmentProcessor.cs | 5 +- .../InvokedDataCollector.cs | 99 + .../PublicAPI/PublicAPI.Shipped.txt | 17 +- .../PublicAPI/net/PublicAPI.Shipped.txt | 5 +- .../Hosting/DotnetTestHostManager.cs | 3 +- .../CodeCoverageDataAttachmentsHandler.cs | 4 +- .../ITranslationLayerRequestSenderAsync.cs | 4 + .../Interfaces/IVsTestConsoleWrapperAsync.cs | 24 +- .../VsTestConsoleRequestSender.cs | 14 +- .../VsTestConsoleWrapper.cs | 14 + .../TestPlatformHelpers/TestRequestManager.cs | 6 +- .../DataCollectionTests.cs | 59 +- .../CodeCoverageTests.cs | 37 +- .../EventHandler/RunEventHandler.cs | 11 + .../Execution/TestRunRequestTests.cs | 18 +- .../TestPluginDiscovererTests.cs | 22 +- .../Logging/InternalTestLoggerEventsTests.cs | 4 +- .../DataCollectionRequestSenderTests.cs | 10 +- .../TestRequestSenderTests.cs | 6 +- ...lectorAttachmentsProcessorsFactoryTests.cs | 212 + ...estRunAttachmentsProcessingManagerTests.cs | 230 +- .../ParallelProxyExecutionManagerTests.cs | 13 +- .../ParallelRunDataAggregatorTests.cs | 111 +- .../Parallel/ParallelRunEventsHandlerTests.cs | 10 +- ...DataCollectionTestRunEventsHandlerTests.cs | 37 +- ...arallelDataCollectionEventsHandlerTests.cs | 15 +- .../ProxyDataCollectionManagerTests.cs | 13 +- .../TestLoggerManagerTests.cs | 8 +- .../BlameLoggerTests.cs | 6 +- .../HtmlLoggerTests.cs | 22 +- .../TrxLoggerTests.cs | 4 +- .../IntegrationTestBase.cs | 26 +- ...CodeCoverageDataAttachmentsHandlerTests.cs | 21 +- .../AttachmentProcessorDataCollector.csproj | 14 + .../SampleDataCollector.cs | 85 + test/TestAssets/TestAssets.sln | 14 + .../VsTestConsoleRequestSenderTests.cs | 138 +- .../VsTestConsoleWrapperAsyncTests.cs | 6 +- .../VsTestConsoleWrapperTests.cs | 6 +- .../CommunicationLayerIntegrationTests.cs | 8 +- .../DataCollectionManagerTests.cs | 49 +- .../TestRunResultAggregatorTests.cs | 8 +- .../Internal/ConsoleLoggerTests.cs | 40 +- .../RunSpecificTestsArgumentProcessorTests.cs | 4 +- .../RunTestsArgumentProcessorTests.cs | 2 +- .../TestRequestManagerTests.cs | 4150 +++++++++-------- 83 files changed, 3772 insertions(+), 2435 deletions(-) create mode 100644 src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs create mode 100644 src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs create mode 100644 src/Microsoft.TestPlatform.ObjectModel/DataCollector/Attributes/DataCollectorAttachmentProcessor.cs create mode 100644 src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs create mode 100644 test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs create mode 100644 test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj create mode 100644 test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs diff --git a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs index 31812dacfa..58d8e9d2a5 100644 --- a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs +++ b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs @@ -421,7 +421,7 @@ public void TestRunMessageHandler(object sender, TestRunMessageEventArgs e) SendTestMessage(e.Level, e.Message); break; - + default: throw new NotSupportedException($"Test message level '{e.Level}' is not supported."); } @@ -456,7 +456,7 @@ private void StartTestRun(TestRunRequestPayload testRunPayload, ITestRequestMana this.communicationManager.SendMessage(MessageType.TestMessage, testMessagePayload); var runCompletePayload = new TestRunCompletePayload() { - TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, true, ex, null, TimeSpan.MinValue), + TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, true, ex, null, null, TimeSpan.MinValue), LastRunTests = null }; diff --git a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs index 4d41f304b1..e3efa01cb8 100644 --- a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs @@ -431,6 +431,7 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs runCompleteArgs, Test runCompleteArgs.Error, // This is required as TMI adapter is sending attachments as List which cannot be type casted to Collection. runContextAttachments != null ? new Collection(runContextAttachments.ToList()) : null, + runCompleteArgs.InvokedDataCollectors, this.runRequestTimeTracker.Elapsed); // Ignore the time sent (runCompleteArgs.ElapsedTimeInRunningTests) @@ -587,6 +588,7 @@ private void HandleLoggerManagerTestRunComplete(TestRunCompletePayload testRunCo testRunCompletePayload.TestRunCompleteArgs.IsAborted, testRunCompletePayload.TestRunCompleteArgs.Error, testRunCompletePayload.TestRunCompleteArgs.AttachmentSets, + testRunCompletePayload.TestRunCompleteArgs.InvokedDataCollectors, this.runRequestTimeTracker.Elapsed); this.LoggerManager.HandleTestRunComplete(testRunCompleteArgs); } diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs index 1c65bb13d6..4c9438fbce 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs @@ -20,18 +20,27 @@ public class AfterTestRunEndResult /// /// The collection of attachment sets. /// + /// + /// The collection of the DataCollectors invoked during test session + /// /// /// The metrics. /// - public AfterTestRunEndResult(Collection attachmentSets, IDictionary metrics) + public AfterTestRunEndResult(Collection attachmentSets, + Collection invokedDataCollectors, + IDictionary metrics) { this.AttachmentSets = attachmentSets; + this.InvokedDataCollectors = invokedDataCollectors; this.Metrics = metrics; } [DataMember] public Collection AttachmentSets { get; private set; } + [DataMember] + public Collection InvokedDataCollectors { get; private set; } + [DataMember] public IDictionary Metrics { get; private set; } } diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs index eea1662899..a319e78517 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs @@ -10,6 +10,8 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector using System.Linq; using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; + using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Logging; using Microsoft.VisualStudio.TestPlatform.Common.Utilities; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -257,6 +259,21 @@ public Collection SessionEnded(bool isCancelled = false) return new Collection(result); } + /// + public Collection GetInvokedDataCollectors() + { + List invokedDataCollector = new List(); + foreach (DataCollectorInformation dataCollectorInformation in this.RunDataCollectors.Values) + { + invokedDataCollector.Add(new InvokedDataCollector(dataCollectorInformation.DataCollectorConfig.TypeUri, + dataCollectorInformation.DataCollectorConfig.DataCollectorType.AssemblyQualifiedName, + dataCollectorInformation.DataCollectorConfig.FilePath, + dataCollectorInformation.DataCollectorConfig.HasAttachmentsProcessor())); + } + + return new Collection(invokedDataCollector); + } + /// public void TestHostLaunched(int processId) { @@ -397,6 +414,29 @@ protected virtual bool TryGetUriFromFriendlyName(string friendlyName, out string return false; } + /// + /// Gets the DataCollectorConfig using uri. + /// + /// + /// The extension uri. + /// + /// + /// The . + /// + protected virtual DataCollectorConfig TryGetDataCollectorConfig(string extensionUri) + { + var extensionManager = this.dataCollectorExtensionManager; + foreach (var extension in extensionManager.TestExtensions) + { + if (string.Equals(extension.TestPluginInfo.IdentifierData, extensionUri, StringComparison.OrdinalIgnoreCase)) + { + return (DataCollectorConfig)extension.TestPluginInfo; + } + } + + return null; + } + protected virtual bool IsUriValid(string uri) { if (string.IsNullOrEmpty(uri)) @@ -470,7 +510,7 @@ private void LoadAndInitialize(DataCollectorSettings dataCollectorSettings, stri return; } - dataCollectorConfig = new DataCollectorConfig(dataCollector.GetType()); + dataCollectorConfig = this.TryGetDataCollectorConfig(dataCollectorUri); // Attempt to get the data collector information verifying that all of the required metadata for the collector is available. dataCollectorInfo = new DataCollectorInformation( diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs index de25804357..91b541b98d 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs @@ -32,6 +32,7 @@ public DataCollectorConfig(Type type) this.DataCollectorType = type; this.TypeUri = GetTypeUri(type); this.FriendlyName = GetFriendlyName(type); + this.AttachmentsProcessorType = GetAttachmentsProcessors(type); } /// @@ -63,10 +64,21 @@ public override ICollection Metadata { get { - return new object[] { this.TypeUri.ToString(), this.FriendlyName }; + return new object[] { this.TypeUri.ToString(), this.FriendlyName, this.FilePath, this.AttachmentsProcessorType != null }; } } + /// + /// Gets attachments processor + /// + public Type AttachmentsProcessorType { get; private set; } + + /// + /// Check if collector registers an attachement processor. + /// + /// True if collector registers an attachment processor. + public bool HasAttachmentsProcessor() => AttachmentsProcessorType != null; + /// /// Gets the Type Uri for the data collector. /// @@ -88,6 +100,27 @@ private static Uri GetTypeUri(Type dataCollectorType) return typeUri; } + /// + /// Gets the attachment processor for the data collector. + /// + /// The data collector to get the attachment processor for. + /// Type of the attachment processor. + private static Type GetAttachmentsProcessors(Type dataCollectorType) + { + Type attachmentsProcessor = null; + var attachmenstProcessors = GetAttributes(dataCollectorType, typeof(DataCollectorAttachmentProcessorAttribute)); + if (attachmenstProcessors != null && attachmenstProcessors.Length > 0) + { + var attachmenstProcessorsAttribute = (DataCollectorAttachmentProcessorAttribute)attachmenstProcessors[0]; + if (attachmenstProcessorsAttribute.Type != null) + { + attachmentsProcessor = attachmenstProcessorsAttribute.Type; + } + } + + return attachmentsProcessor; + } + /// /// Gets the friendly name for the data collector. /// diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionManager.cs b/src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionManager.cs index b43433c968..a18a852473 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionManager.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionManager.cs @@ -66,5 +66,11 @@ internal interface IDataCollectionManager : IDisposable /// Collection of session attachmentSet. /// Collection SessionEnded(bool isCancelled); + + /// + /// Return a collections of the invoked data collectors + /// + /// Collection of data collectors. + Collection GetInvokedDataCollectors(); } } \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs index 546cc5c54b..fa585077f3 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs @@ -51,7 +51,6 @@ protected DataCollectorExtensionManager( /// public static DataCollectorExtensionManager Create(IMessageLogger messageLogger) { - TestPluginManager.Instance.GetSpecificTestExtensions( TestPlatformConstants.DataCollectorEndsWithPattern, out var unfilteredTestExtensions, @@ -59,6 +58,28 @@ public static DataCollectorExtensionManager Create(IMessageLogger messageLogger) return new DataCollectorExtensionManager(unfilteredTestExtensions, filteredTestExtensions, messageLogger); } + + /// + /// Gets an instance of the DataCollectorExtensionManager. + /// + /// + /// File path that contains data collectors to load. + /// + /// + /// The message Logger. + /// + /// + /// The DataCollectorExtensionManager. + /// + public static DataCollectorExtensionManager Create(string extensionAssemblyFilePath, IMessageLogger messageLogger) + { + TestPluginManager.Instance.GetTestExtensions( + extensionAssemblyFilePath, + out var unfilteredTestExtensions, + out var filteredTestExtensions); + + return new DataCollectorExtensionManager(unfilteredTestExtensions, filteredTestExtensions, messageLogger); + } } /// @@ -75,10 +96,12 @@ public class DataCollectorMetadata : IDataCollectorCapabilities /// /// The friendly Name. /// - public DataCollectorMetadata(string extension, string friendlyName) + public DataCollectorMetadata(string extension, string friendlyName, string filePath, bool hasAttachmentProcessor) { this.ExtensionUri = extension; this.FriendlyName = friendlyName; + this.FilePath = filePath; + this.HasAttachmentProcessor = hasAttachmentProcessor; } /// @@ -98,5 +121,23 @@ public string FriendlyName get; private set; } + + /// + /// Check if the data collector has got attachment processor registered + /// + public bool HasAttachmentProcessor + { + get; + private set; + } + + /// + /// Gets the file path of assemblies that contains the data collector. + /// + public string FilePath + { + get; + private set; + } } } diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs index 5344b257ad..cc9a34b3de 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs @@ -346,7 +346,7 @@ internal Dictionary GetTestExtensions(), extensionAssembly); - if (extensions != null) + if (extensions != null && extensions.Count > 0) { return extensions; } @@ -578,6 +578,12 @@ private void LogExtensions() var loggers = this.TestExtensions.TestLoggers != null ? string.Join(",", this.TestExtensions.TestLoggers.Keys.ToArray()) : null; EqtTrace.Verbose("TestPluginCache: Loggers are '{0}'.", loggers); + + var testhosts = this.TestExtensions.TestHosts != null ? string.Join(",", this.TestExtensions.TestHosts.Keys.ToArray()) : null; + EqtTrace.Verbose("TestPluginCache: TestHosts are '{0}'.", testhosts); + + var dataCollectors = this.TestExtensions.DataCollectors != null ? string.Join(",", this.TestExtensions.DataCollectors.Keys.ToArray()) : null; + EqtTrace.Verbose("TestPluginCache: DataCollectors are '{0}'.", dataCollectors); } } diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index f08f565394..67bfa3c182 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -10,8 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework using System.Diagnostics; using System.Globalization; using System.Collections.Generic; - using System.Diagnostics.CodeAnalysis; - + using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; using Microsoft.VisualStudio.TestPlatform.Common.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -130,7 +129,7 @@ private void GetTestExtensionsFromFiles( assembly = Assembly.Load(new AssemblyName(assemblyName)); if (assembly != null) { - this.GetTestExtensionsFromAssembly(assembly, pluginInfos); + this.GetTestExtensionsFromAssembly(assembly, pluginInfos, file); } } catch (FileLoadException e) @@ -158,7 +157,7 @@ private void GetTestExtensionsFromFiles( /// /// Type of Extensions. /// - private void GetTestExtensionsFromAssembly(Assembly assembly, Dictionary pluginInfos) where TPluginInfo : TestPluginInformation + private void GetTestExtensionsFromAssembly(Assembly assembly, Dictionary pluginInfos, string filePath) where TPluginInfo : TestPluginInformation { Debug.Assert(assembly != null, "null assembly"); Debug.Assert(pluginInfos != null, "null pluginInfos"); @@ -192,7 +191,7 @@ private void GetTestExtensionsFromAssembly(Assembly ass { foreach (var type in types) { - GetTestExtensionFromType(type, extension, pluginInfos); + GetTestExtensionFromType(type, extension, pluginInfos, filePath); } } } @@ -215,13 +214,15 @@ private void GetTestExtensionsFromAssembly(Assembly ass private void GetTestExtensionFromType( Type type, Type extensionType, - Dictionary extensionCollection) + Dictionary extensionCollection, + string filePath) where TPluginInfo : TestPluginInformation { if (extensionType.GetTypeInfo().IsAssignableFrom(type.GetTypeInfo())) { var rawPluginInfo = Activator.CreateInstance(typeof(TPluginInfo), type); var pluginInfo = (TPluginInfo)rawPluginInfo; + pluginInfo.FilePath = filePath; if (pluginInfo == null || pluginInfo.IdentifierData == null) { diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestPluginInformation.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestPluginInformation.cs index e8aaa5a794..d78e28ed02 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestPluginInformation.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestPluginInformation.cs @@ -50,5 +50,14 @@ public string AssemblyQualifiedName get; private set; } + + /// + /// Gets the file path of the plugin + /// + public string FilePath + { + get; + internal set; + } } } diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs index d6b88d89c8..db8ff0584d 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs @@ -2,10 +2,13 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Threading; using System.Threading.Tasks; +using System.Xml; namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine { @@ -17,17 +20,34 @@ internal interface ITestRunAttachmentsProcessingManager /// /// Processes attachments and provides results through handler /// + /// RunSettings /// Collection of attachments + /// Collection of invoked data collectors /// EventHandler for handling test run attachments processing event /// Cancellation token - Task ProcessTestRunAttachmentsAsync(IRequestData requestData, IEnumerable attachments, ITestRunAttachmentsProcessingEventsHandler eventHandler, CancellationToken cancellationToken); + Task ProcessTestRunAttachmentsAsync(string runSettingsXml, IRequestData requestData, IEnumerable attachments, IEnumerable invokedDataCollector, ITestRunAttachmentsProcessingEventsHandler eventHandler, CancellationToken cancellationToken); /// /// Processes attachments /// + /// RunSettings /// Collection of attachments + /// Collection of invoked data collectors /// Cancellation token /// Collection of attachments. - Task> ProcessTestRunAttachmentsAsync(IRequestData requestData, IEnumerable attachments, CancellationToken cancellationToken); + Task> ProcessTestRunAttachmentsAsync(string runSettingsXml, IRequestData requestData, IEnumerable attachments, IEnumerable invokedDataCollector, CancellationToken cancellationToken); + } + + /// + /// Creates and return a list of available attachments processor + /// + internal interface IDataCollectorAttachmentsProcessorsFactory + { + /// + /// Creates and return a list of available attachments processor + /// + /// List of invoked data collectors + /// List of attachments processors + IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors); } } diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs b/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs index f8a4628c0c..b4c4a57c7f 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs @@ -12,5 +12,15 @@ public interface IDataCollectorCapabilities : ITestExtensionCapabilities /// Gets the friendly name corresponding to the data collector. /// string FriendlyName { get; } + + /// + /// Gets the file path of the assembly that contains the data collector. + /// + string FilePath { get; } + + /// + /// Check if the data collector has got attachment processor registered. + /// + bool HasAttachmentProcessor { get; } } } diff --git a/src/Microsoft.TestPlatform.Common/Logging/InternalTestLoggerEvents.cs b/src/Microsoft.TestPlatform.Common/Logging/InternalTestLoggerEvents.cs index adac44003e..d240c8744a 100644 --- a/src/Microsoft.TestPlatform.Common/Logging/InternalTestLoggerEvents.cs +++ b/src/Microsoft.TestPlatform.Common/Logging/InternalTestLoggerEvents.cs @@ -47,7 +47,7 @@ internal class InternalTestLoggerEvents : TestLoggerEvents, IDisposable private bool isBoundsOnLoggerEventQueueEnabled; private TestSessionMessageLogger testSessionMessageLogger; -#endregion + #endregion #region Constructor @@ -76,7 +76,7 @@ public InternalTestLoggerEvents(TestSessionMessageLogger testSessionMessageLogge this.testSessionMessageLogger.TestRunMessage += this.TestRunMessageHandler; } -#endregion + #endregion #region Events @@ -120,7 +120,7 @@ public InternalTestLoggerEvents(TestSessionMessageLogger testSessionMessageLogge /// public override event EventHandler DiscoveryComplete; -#endregion + #endregion #region IDisposable @@ -143,7 +143,7 @@ public void Dispose() this.loggerEventQueue.Dispose(); } -#endregion + #endregion #region Internal Methods @@ -307,12 +307,13 @@ internal void RaiseTestRunComplete(TestRunCompleteEventArgs args) /// Specifies whether the test run is aborted. /// Specifies the error that occurs during the test run. /// Run level attachment sets + /// Invoked data collectors /// Time elapsed in just running the tests. - internal void CompleteTestRun(ITestRunStatistics stats, bool isCanceled, bool isAborted, Exception error, Collection attachmentSet, TimeSpan elapsedTime) + internal void CompleteTestRun(ITestRunStatistics stats, bool isCanceled, bool isAborted, Exception error, Collection attachmentSet, Collection invokedDataCollectors, TimeSpan elapsedTime) { this.CheckDisposed(); - var args = new TestRunCompleteEventArgs(stats, isCanceled, isAborted, error, attachmentSet, elapsedTime); + var args = new TestRunCompleteEventArgs(stats, isCanceled, isAborted, error, attachmentSet, invokedDataCollectors, elapsedTime); // Sending 0 size as this event is not expected to contain any data. this.SafeInvokeAsync(() => this.TestRunComplete, args, 0, "InternalTestLoggerEvents.SendTestRunComplete"); @@ -321,7 +322,7 @@ internal void CompleteTestRun(ITestRunStatistics stats, bool isCanceled, bool is this.loggerEventQueue.Flush(); } -#endregion + #endregion #region Private Members @@ -462,6 +463,6 @@ private int GetSetting(string appSettingKey, int defaultValue) return value; } -#endregion + #endregion } } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index 7add8c6308..ef4ee8fd0a 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -5,6 +5,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollect { using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Globalization; using System.IO; using System.Net; @@ -389,7 +390,8 @@ private void HandleAfterTestRunEnd(Message message) } var attachmentsets = this.dataCollectionManager.SessionEnded(isCancelled); - var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, this.requestData.MetricsCollection.Metrics); + var invokedDataCollectors = this.dataCollectionManager.GetInvokedDataCollectors(); + var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, invokedDataCollectors, this.requestData.MetricsCollection.Metrics); // Dispose all datacollectors before sending attachments to vstest.console process. // As datacollector process exits itself on parent process(vstest.console) exits. diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index 16ec2d6c03..4e2cb19bf0 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -633,7 +633,7 @@ private void OnTestRunAbort(ITestRunEventsHandler testRunEventsHandler, Exceptio this.LogErrorMessage(string.Format(CommonResources.AbortedTestRun, reason)); // notify test run abort to vstest console wrapper. - var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, null, TimeSpan.Zero); var payload = new TestRunCompletePayload { TestRunCompleteArgs = completeArgs }; var rawMessage = this.dataSerializer.SerializePayload(MessageType.ExecutionComplete, payload); testRunEventsHandler.HandleRawMessage(rawMessage); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs new file mode 100644 index 0000000000..7852cda8d2 --- /dev/null +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -0,0 +1,90 @@ +using Microsoft.VisualStudio.TestPlatform.Common.DataCollector; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; +using Microsoft.VisualStudio.TestPlatform.Common.Logging; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; + +namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing +{ + public class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory + { + private static Uri CoverageUri = new Uri("datacollector://microsoft/CodeCoverage/2.0"); + private const string CoverageFriendlyName = "Code Coverage"; + + public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) + { + IDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); + bool addCodeCoverageAttachmentProcessors = true; + + if (invokedDataCollectors != null) + { + // We order files by filename descending so in case of the same collector from the same nuget but with different versions, we'll run the newer version. + // i.e. C:\Users\xxx\.nuget\packages\coverlet.collector + // /3.0.2 + // /3.0.3 + // /3.1.0 + foreach (var invokedDataCollector in invokedDataCollectors.OrderByDescending(d => d.FilePath)) + { + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Analyzing data collector attachment processor Uri: {invokedDataCollector.Uri} AssemblyQualifiedName: {invokedDataCollector.AssemblyQualifiedName} FilePath: {invokedDataCollector.FilePath} HasAttachmentProcessor: {invokedDataCollector.HasAttachmentProcessor}"); + + // We'll merge using only one AQN in case of more "same processors" in different assembly. + if (!invokedDataCollector.HasAttachmentProcessor || datacollectorsAttachmentsProcessors.ContainsKey(invokedDataCollector.AssemblyQualifiedName)) + { + continue; + } + + var dataCollectorExtensionManager = DataCollectorExtensionManager.Create(invokedDataCollector.FilePath, TestSessionMessageLogger.Instance); + var dataCollectorExtension = dataCollectorExtensionManager.TryGetTestExtension(invokedDataCollector.Uri); + if (dataCollectorExtension?.Metadata.HasAttachmentProcessor == true) + { + Type attachmentProcessorType = ((DataCollectorConfig)dataCollectorExtension.TestPluginInfo).AttachmentsProcessorType; + IDataCollectorAttachmentProcessor dataCollectorAttachmentProcessorInstance = null; + try + { + dataCollectorAttachmentProcessorInstance = TestPluginManager.CreateTestExtension(attachmentProcessorType); + } + catch (Exception ex) + { + EqtTrace.Error($"DataCollectorAttachmentsProcessorsFactory: Failed during the creation of data collector attachment processor '{attachmentProcessorType.AssemblyQualifiedName}'\n{ex}"); + } + + if (dataCollectorAttachmentProcessorInstance != null && !datacollectorsAttachmentsProcessors.ContainsKey(attachmentProcessorType.AssemblyQualifiedName)) + { + datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); + + if (invokedDataCollector.Uri.AbsoluteUri == CoverageUri.AbsoluteUri) + { + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Attachment data processor for data collector with friendly name '{CoverageFriendlyName}' found '{attachmentProcessorType.AssemblyQualifiedName}' inside '{invokedDataCollector.FilePath}'"); + addCodeCoverageAttachmentProcessors = false; + } + } + } + else + { + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: DataCollectorExtension not found for uri '{invokedDataCollector.Uri}'"); + } + } + } + + if (addCodeCoverageAttachmentProcessors) + { + datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); + } + + var finalDatacollectorsAttachmentsProcessors = new Dictionary(); + foreach (var attachementProcessor in datacollectorsAttachmentsProcessors) + { + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: valid data collector attachment processor found: '{attachementProcessor.Value.Item2.GetType().AssemblyQualifiedName}'"); + finalDatacollectorsAttachmentsProcessors.Add(attachementProcessor.Value.Item1, attachementProcessor.Value.Item2); + } + + return new ReadOnlyDictionary(finalDatacollectorsAttachmentsProcessors); + } + } +} diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs index 6eff75838a..3e8c23cb7e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Xml; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -15,42 +16,43 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing { /// /// Orchestrates test run attachments processing operations. /// - public class TestRunAttachmentsProcessingManager : ITestRunAttachmentsProcessingManager + internal class TestRunAttachmentsProcessingManager : ITestRunAttachmentsProcessingManager { private static string AttachmentsProcessingCompleted = "Completed"; private static string AttachmentsProcessingCanceled = "Canceled"; private static string AttachmentsProcessingFailed = "Failed"; private readonly ITestPlatformEventSource testPlatformEventSource; - private readonly IDataCollectorAttachmentProcessor[] dataCollectorAttachmentsProcessors; + private readonly IDataCollectorAttachmentsProcessorsFactory dataCollectorAttachmentsProcessorsFactory; /// /// Initializes a new instance of the class. /// - public TestRunAttachmentsProcessingManager(ITestPlatformEventSource testPlatformEventSource, params IDataCollectorAttachmentProcessor[] dataCollectorAttachmentsProcessors) + public TestRunAttachmentsProcessingManager(ITestPlatformEventSource testPlatformEventSource, IDataCollectorAttachmentsProcessorsFactory dataCollectorAttachmentsProcessorsFactory) { this.testPlatformEventSource = testPlatformEventSource ?? throw new ArgumentNullException(nameof(testPlatformEventSource)); - this.dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessors ?? throw new ArgumentNullException(nameof(dataCollectorAttachmentsProcessors)); + this.dataCollectorAttachmentsProcessorsFactory = dataCollectorAttachmentsProcessorsFactory ?? throw new ArgumentNullException(nameof(dataCollectorAttachmentsProcessorsFactory)); } /// - public async Task ProcessTestRunAttachmentsAsync(IRequestData requestData, IEnumerable attachments, ITestRunAttachmentsProcessingEventsHandler eventHandler, CancellationToken cancellationToken) + public async Task ProcessTestRunAttachmentsAsync(string runSettingsXml, IRequestData requestData, IEnumerable attachments, IEnumerable invokedDataCollector, ITestRunAttachmentsProcessingEventsHandler eventHandler, CancellationToken cancellationToken) { - await InternalProcessTestRunAttachmentsAsync(requestData, new Collection(attachments.ToList()), eventHandler, cancellationToken).ConfigureAwait(false); + await InternalProcessTestRunAttachmentsAsync(runSettingsXml, requestData, new Collection(attachments.ToList()), invokedDataCollector, eventHandler, cancellationToken).ConfigureAwait(false); } /// - public Task> ProcessTestRunAttachmentsAsync(IRequestData requestData, IEnumerable attachments, CancellationToken cancellationToken) + public Task> ProcessTestRunAttachmentsAsync(string runSettingsXml, IRequestData requestData, IEnumerable attachments, IEnumerable invokedDataCollector, CancellationToken cancellationToken) { - return InternalProcessTestRunAttachmentsAsync(requestData, new Collection(attachments.ToList()), null, cancellationToken); + return InternalProcessTestRunAttachmentsAsync(runSettingsXml, requestData, new Collection(attachments.ToList()), invokedDataCollector, null, cancellationToken); } - private async Task> InternalProcessTestRunAttachmentsAsync(IRequestData requestData, Collection attachments, ITestRunAttachmentsProcessingEventsHandler eventHandler, CancellationToken cancellationToken) + private async Task> InternalProcessTestRunAttachmentsAsync(string runSettingsXml, IRequestData requestData, Collection attachments, IEnumerable invokedDataCollector, ITestRunAttachmentsProcessingEventsHandler eventHandler, CancellationToken cancellationToken) { Stopwatch stopwatch = Stopwatch.StartNew(); @@ -58,13 +60,13 @@ private async Task> InternalProcessTestRunAttachmentsA { testPlatformEventSource.TestRunAttachmentsProcessingStart(attachments?.Count ?? 0); requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsSentForProcessing, attachments?.Count ?? 0); - - cancellationToken.ThrowIfCancellationRequested(); + + cancellationToken.ThrowIfCancellationRequested(); var taskCompletionSource = new TaskCompletionSource>(); using (cancellationToken.Register(() => taskCompletionSource.TrySetCanceled())) { - Task> task = Task.Run(async () => await ProcessAttachmentsAsync(new Collection(attachments.ToList()), eventHandler, cancellationToken)); + Task> task = Task.Run(async () => await ProcessAttachmentsAsync(runSettingsXml, new Collection(attachments.ToList()), invokedDataCollector, eventHandler, cancellationToken)); var completedTask = await Task.WhenAny(task, taskCompletionSource.Task).ConfigureAwait(false); @@ -96,18 +98,19 @@ private async Task> InternalProcessTestRunAttachmentsA } } - private async Task> ProcessAttachmentsAsync(Collection attachments, ITestRunAttachmentsProcessingEventsHandler eventsHandler, CancellationToken cancellationToken) + private async Task> ProcessAttachmentsAsync(string runSettingsXml, Collection attachments, IEnumerable invokedDataCollector, ITestRunAttachmentsProcessingEventsHandler eventsHandler, CancellationToken cancellationToken) { if (attachments == null || !attachments.Any()) return attachments; + var dataCollectionRunSettings = XmlRunSettingsUtilities.GetDataCollectionRunSettings(runSettingsXml); var logger = CreateMessageLogger(eventsHandler); - - for (int i = 0; i < dataCollectorAttachmentsProcessors.Length; i++) + IReadOnlyDictionary dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray()); + for (int i = 0; i < dataCollectorAttachmentsProcessors.Count; i++) { - var dataCollectorAttachmentsProcessor = dataCollectorAttachmentsProcessors[i]; + var dataCollectorAttachmentsProcessor = dataCollectorAttachmentsProcessors.ElementAt(i); int attachmentsHandlerIndex = i + 1; - ICollection attachmentProcessorUris = dataCollectorAttachmentsProcessor.GetExtensionUris()?.ToList(); + ICollection attachmentProcessorUris = dataCollectorAttachmentsProcessor.Value.GetExtensionUris()?.ToList(); if (attachmentProcessorUris != null && attachmentProcessorUris.Any()) { var attachmentsToBeProcessed = attachments.Where(dataCollectionAttachment => attachmentProcessorUris.Any(uri => uri.Equals(dataCollectionAttachment.Uri))).ToArray(); @@ -118,15 +121,31 @@ private async Task> ProcessAttachmentsAsync(Collection attachments.Remove(attachment); } - IProgress progressReporter = new Progress((int progress) => + IProgress progressReporter = new Progress((int progress) => eventsHandler?.HandleTestRunAttachmentsProcessingProgress( - new TestRunAttachmentsProcessingProgressEventArgs(attachmentsHandlerIndex, attachmentProcessorUris, progress, dataCollectorAttachmentsProcessors.Length))); + new TestRunAttachmentsProcessingProgressEventArgs(attachmentsHandlerIndex, attachmentProcessorUris, progress, dataCollectorAttachmentsProcessors.Count))); + + XmlElement configuration = null; + var collectorConfiguration = dataCollectionRunSettings?.DataCollectorSettingsList.SingleOrDefault(c => c.FriendlyName == dataCollectorAttachmentsProcessor.Key); + if (collectorConfiguration != null && collectorConfiguration.IsEnabled) + { + configuration = collectorConfiguration.Configuration; + } - ICollection processedAttachments = await dataCollectorAttachmentsProcessor.ProcessAttachmentSetsAsync(new Collection(attachmentsToBeProcessed), progressReporter, logger, cancellationToken).ConfigureAwait(false); + EqtTrace.Info($"TestRunAttachmentsProcessingManager: invocation of data collector attachment processor '{dataCollectorAttachmentsProcessor.Value.GetType().AssemblyQualifiedName}' with configuration '{(configuration == null ? "null" : configuration.OuterXml)}'"); + ICollection processedAttachments = await dataCollectorAttachmentsProcessor.Value.ProcessAttachmentSetsAsync( + configuration, + new Collection(attachmentsToBeProcessed), + progressReporter, + logger, + cancellationToken).ConfigureAwait(false); - foreach (var attachment in processedAttachments) + if (processedAttachments != null && processedAttachments.Any()) { - attachments.Add(attachment); + foreach (var attachment in processedAttachments) + { + attachments.Add(attachment); + } } } } @@ -136,7 +155,7 @@ private async Task> ProcessAttachmentsAsync(Collection } private Collection FinalizeOperation(IRequestData requestData, TestRunAttachmentsProcessingCompleteEventArgs completeArgs, Collection attachments, Stopwatch stopwatch, ITestRunAttachmentsProcessingEventsHandler eventHandler) - { + { testPlatformEventSource.TestRunAttachmentsProcessingStop(attachments.Count); requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsAfterProcessing, attachments.Count); requestData.MetricsCollection.Add(TelemetryDataConstants.AttachmentsProcessingState, completeArgs.Error != null ? AttachmentsProcessingFailed : completeArgs.IsCanceled ? AttachmentsProcessingCanceled : AttachmentsProcessingCompleted); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyexecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyexecutionManager.cs index 5371a5f53b..8647402846 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyexecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyexecutionManager.cs @@ -100,7 +100,7 @@ public int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler e eventHandler.HandleLogMessage(TestMessageLevel.Error, exception.ToString()); // Send a run complete to caller. - var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection(), TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection(), new Collection(), TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs index d5cc0cecb4..9264f2817a 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs @@ -243,7 +243,7 @@ private int StartTestRunPrivate(ITestRunEventsHandler runEventsHandler) this.runCompletedClients = 0; // One data aggregator per parallel run - this.currentRunDataAggregator = new ParallelRunDataAggregator(); + this.currentRunDataAggregator = new ParallelRunDataAggregator(this.actualTestRunCriteria.TestRunSettings); foreach (var concurrentManager in this.GetConcurrentManagerInstances()) { @@ -260,9 +260,7 @@ private ParallelRunEventsHandler GetEventsHandler(IProxyExecutionManager concurr if (concurrentManager is ProxyExecutionManagerWithDataCollection) { var concurrentManagerWithDataCollection = concurrentManager as ProxyExecutionManagerWithDataCollection; - - // TODO : use TestPluginCache to iterate over all IDataCollectorAttachments - var attachmentsProcessingManager = new TestRunAttachmentsProcessingManager(TestPlatformEventSource.Instance, new CodeCoverageDataAttachmentsHandler()); + var attachmentsProcessingManager = new TestRunAttachmentsProcessingManager(TestPlatformEventSource.Instance, new DataCollectorAttachmentsProcessorsFactory()); return new ParallelDataCollectionEventsHandler( this.requestData, @@ -345,7 +343,7 @@ private void StartTestRunOnConcurrentManager(IProxyExecutionManager proxyExecuti // Aborted is sent to allow the current execution manager replaced with another instance // Ensure that the test run aggregator in parallel run events handler doesn't add these statistics // (since the test run didn't even start) - var completeArgs = new TestRunCompleteEventArgs(null, false, true, null, new Collection(), TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, true, null, new Collection(), new Collection(), TimeSpan.Zero); handler.HandleTestRunComplete(completeArgs, null, null, null); }, TaskContinuationOptions.OnlyOnFaulted); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs index 78185e0c5c..2917a5fd06 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs @@ -20,6 +20,8 @@ internal class ParallelRunDataAggregator { #region PrivateFields + private readonly string runSettingsXml; + private List executorUris; private List testRunStatsList; @@ -30,11 +32,13 @@ internal class ParallelRunDataAggregator #endregion - public ParallelRunDataAggregator() + public ParallelRunDataAggregator(string runSettingsXml) { + this.runSettingsXml = runSettingsXml ?? throw new ArgumentNullException(nameof(runSettingsXml)); ElapsedTime = TimeSpan.Zero; RunContextAttachments = new Collection(); RunCompleteArgsAttachments = new List(); + InvokedDataCollectors = new Collection(); Exceptions = new List(); executorUris = new List(); testRunStatsList = new List(); @@ -53,6 +57,8 @@ public ParallelRunDataAggregator() public List RunCompleteArgsAttachments { get; } + public Collection InvokedDataCollectors { get; set; } + public List Exceptions { get; } public HashSet ExecutorUris => new HashSet(executorUris); @@ -61,6 +67,8 @@ public ParallelRunDataAggregator() public bool IsCanceled { get; private set; } + public string RunSettings => this.runSettingsXml; + #endregion #region Public Methods @@ -137,7 +145,8 @@ public void Aggregate( bool isAborted, bool isCanceled, ICollection runContextAttachments, - Collection runCompleteArgsAttachments) + Collection runCompleteArgsAttachments, + Collection invokedDataCollectors) { lock (dataUpdateSyncObject) { @@ -157,6 +166,17 @@ public void Aggregate( if (exception != null) Exceptions.Add(exception); if (executorUris != null) this.executorUris.AddRange(executorUris); if (testRunStats != null) testRunStatsList.Add(testRunStats); + + if (invokedDataCollectors != null && invokedDataCollectors.Count > 0) + { + foreach (var invokedDataCollector in invokedDataCollectors) + { + if (!this.InvokedDataCollectors.Contains(invokedDataCollector)) + { + this.InvokedDataCollectors.Add(invokedDataCollector); + } + } + } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs index dd7646d4cf..896f488b6e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs @@ -75,6 +75,7 @@ public virtual void HandleTestRunComplete( this.runDataAggregator.IsAborted, this.runDataAggregator.GetAggregatedException(), new Collection(this.runDataAggregator.RunCompleteArgsAttachments), + new Collection(this.runDataAggregator.InvokedDataCollectors), this.runDataAggregator.ElapsedTime); // Collect Final RunState @@ -112,7 +113,8 @@ protected bool HandleSingleTestRunComplete(TestRunCompleteEventArgs testRunCompl testRunCompleteArgs.IsAborted, testRunCompleteArgs.IsCanceled, runContextAttachments, - testRunCompleteArgs.AttachmentSets); + testRunCompleteArgs.AttachmentSets, + testRunCompleteArgs.InvokedDataCollectors); // Aggregate Run Data Metrics this.runDataAggregator.AggregateRunDataMetrics(testRunCompleteArgs.Metrics); @@ -153,7 +155,7 @@ public void HandleRawMessage(string rawMessage) var message = this.dataSerializer.DeserializeMessage(rawMessage); // Do not deserialize further - just send if not execution complete - if(!string.Equals(MessageType.ExecutionComplete, message.MessageType)) + if (!string.Equals(MessageType.ExecutionComplete, message.MessageType)) { this.actualRunEventsHandler.HandleRawMessage(rawMessage); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs index c5e90651a4..de275432e7 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs @@ -260,7 +260,7 @@ public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsH // message ensures another execution manager created to replace the current one. // This will help if the current execution manager is aborted due to irreparable // error and the test host is lost as well. - var completeArgs = new TestRunCompleteEventArgs(null, false, true, null, new Collection(), TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, true, null, new Collection(), new Collection(), TimeSpan.Zero); var testRunCompletePayload = new TestRunCompletePayload { TestRunCompleteArgs = completeArgs }; this.HandleRawMessage(this.dataSerializer.SerializePayload(MessageType.ExecutionComplete, testRunCompletePayload)); this.HandleTestRunComplete(completeArgs, null, null, null); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs index 4b2ecc07e9..4809ade961 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs @@ -308,7 +308,10 @@ public virtual void Close() // The upper bound for wait should be 100ms. var timeout = 100; EqtTrace.Verbose("ProxyOperationManager.Close: waiting for test host to exit for {0} ms", timeout); - this.testHostExited.Wait(timeout); + if (!this.testHostExited.Wait(timeout)) + { + EqtTrace.Warning("ProxyOperationManager: Timed out waiting for test host to exit. Will terminate process."); + } // Closing the communication channel. this.RequestSender.Close(); @@ -324,8 +327,6 @@ public virtual void Close() { this.initialized = false; - EqtTrace.Warning("ProxyOperationManager: Timed out waiting for test host to exit. Will terminate process."); - // Please clean up test host. this.TestHostManager.CleanTestHostAsync(CancellationToken.None).Wait(); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs index ca9372c181..25ba5e7e07 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs @@ -265,7 +265,7 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs e) try { this.loggerEvents.CompleteTestRun(e.TestRunStatistics, e.IsCanceled, e.IsAborted, e.Error, - e.AttachmentSets, e.ElapsedTimeInRunningTests); + e.AttachmentSets, e.InvokedDataCollectors, e.ElapsedTimeInRunningTests); } finally { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs new file mode 100644 index 0000000000..8c822286b2 --- /dev/null +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System.Collections.ObjectModel; + +namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection +{ + /// + /// Information returned after data collection. + /// + public class DataCollectionResult + { + public DataCollectionResult(Collection attachments, Collection invokedDataCollectors) + { + this.Attachments = attachments; + this.InvokedDataCollectors = invokedDataCollectors; + } + + /// + /// Get list of attachments + /// + public Collection Attachments { get; } + + /// + /// Get the list of the invoked data collectors. + /// + public Collection InvokedDataCollectors { get; } + } +} diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs index cf661ee5ed..12dbcf5e12 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs @@ -6,7 +6,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection using System; using System.Collections.Generic; using System.Collections.ObjectModel; - using System.Globalization; using System.Linq; using System.Threading; @@ -29,6 +28,7 @@ internal class DataCollectionTestRunEventsHandler : ITestRunEventsHandler2 private CancellationToken cancellationToken; private IDataSerializer dataSerializer; private Collection dataCollectionAttachmentSets; + private Collection invokedDataCollectors; /// /// Initializes a new instance of the class. @@ -91,21 +91,31 @@ public void HandleRawMessage(string rawMessage) if (string.Equals(MessageType.ExecutionComplete, message.MessageType)) { - this.dataCollectionAttachmentSets = this.proxyDataCollectionManager?.AfterTestRunEnd(this.cancellationToken.IsCancellationRequested, this); + var dataCollectionResult = this.proxyDataCollectionManager?.AfterTestRunEnd(this.cancellationToken.IsCancellationRequested, this); + this.dataCollectionAttachmentSets = dataCollectionResult?.Attachments; - if (this.dataCollectionAttachmentSets != null && this.dataCollectionAttachmentSets.Any()) - { - var testRunCompletePayload = + var testRunCompletePayload = this.dataSerializer.DeserializePayload(message); + if (this.dataCollectionAttachmentSets != null && this.dataCollectionAttachmentSets.Any()) + { GetCombinedAttachmentSets( testRunCompletePayload.TestRunCompleteArgs.AttachmentSets, this.dataCollectionAttachmentSets); + } - rawMessage = this.dataSerializer.SerializePayload( - MessageType.ExecutionComplete, - testRunCompletePayload); + this.invokedDataCollectors = dataCollectionResult?.InvokedDataCollectors; + if (this.invokedDataCollectors?.Count > 0) + { + foreach (var dataCollector in this.invokedDataCollectors) + { + testRunCompletePayload.TestRunCompleteArgs.InvokedDataCollectors.Add(dataCollector); + } } + + rawMessage = this.dataSerializer.SerializePayload( + MessageType.ExecutionComplete, + testRunCompletePayload); } this.testRunEventsHandler.HandleRawMessage(rawMessage); @@ -133,6 +143,14 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, runContextAttachments = GetCombinedAttachmentSets(this.dataCollectionAttachmentSets, runContextAttachments); } + if (this.invokedDataCollectors != null && this.invokedDataCollectors.Any()) + { + foreach (var dataCollector in this.invokedDataCollectors) + { + testRunCompleteArgs.InvokedDataCollectors.Add(dataCollector); + } + } + this.testRunEventsHandler.HandleTestRunComplete(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/Interfaces/IProxyDataCollectionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/Interfaces/IProxyDataCollectionManager.cs index 32e52ae2b8..5bc5e920af 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/Interfaces/IProxyDataCollectionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/Interfaces/IProxyDataCollectionManager.cs @@ -60,9 +60,9 @@ DataCollectionParameters BeforeTestRunStart( /// The run Events Handler. /// /// - /// The . + /// The . /// - Collection AfterTestRunEnd(bool isCanceled, ITestMessageEventHandler runEventsHandler); + DataCollectionResult AfterTestRunEnd(bool isCanceled, ITestMessageEventHandler runEventsHandler); /// /// Invoked after initialization of test host diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ParallelDataCollectionEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ParallelDataCollectionEventsHandler.cs index a50d5fe6ff..0d250772f9 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ParallelDataCollectionEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ParallelDataCollectionEventsHandler.cs @@ -56,13 +56,14 @@ public override void HandleTestRunComplete( if (parallelRunComplete) { - runDataAggregator.RunContextAttachments = attachmentsProcessingManager.ProcessTestRunAttachmentsAsync(requestData, runDataAggregator.RunContextAttachments, cancellationToken).Result ?? runDataAggregator.RunContextAttachments; + runDataAggregator.RunContextAttachments = attachmentsProcessingManager.ProcessTestRunAttachmentsAsync(runDataAggregator.RunSettings, requestData, runDataAggregator.RunContextAttachments, runDataAggregator.InvokedDataCollectors, cancellationToken).Result ?? runDataAggregator.RunContextAttachments; var completedArgs = new TestRunCompleteEventArgs(this.runDataAggregator.GetAggregatedRunStats(), this.runDataAggregator.IsCanceled, this.runDataAggregator.IsAborted, this.runDataAggregator.GetAggregatedException(), - runDataAggregator.RunContextAttachments, + this.runDataAggregator.RunContextAttachments, + this.runDataAggregator.InvokedDataCollectors, this.runDataAggregator.ElapsedTime); // Add Metrics from Test Host diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs index ffcb29f588..8757b1bc4a 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/ProxyDataCollectionManager.cs @@ -143,15 +143,15 @@ internal ProxyDataCollectionManager(IRequestData requestData, string settingsXml /// The run Events Handler. /// /// - /// The . + /// The . /// - public Collection AfterTestRunEnd(bool isCanceled, ITestMessageEventHandler runEventsHandler) + public DataCollectionResult AfterTestRunEnd(bool isCanceled, ITestMessageEventHandler runEventsHandler) { AfterTestRunEndResult afterTestRunEnd = null; this.InvokeDataCollectionServiceAction( () => { - EqtTrace.Info("ProxyDataCollectionManager.AfterTestRunEnd: Get attachment set for datacollector processId: {0} port: {1}", dataCollectionProcessId, dataCollectionPort); + EqtTrace.Info("ProxyDataCollectionManager.AfterTestRunEnd: Get attachment set and invoked data collectors processId: {0} port: {1}", dataCollectionProcessId, dataCollectionPort); afterTestRunEnd = this.dataCollectionRequestSender.SendAfterTestRunEndAndGetResult(runEventsHandler, isCanceled); }, runEventsHandler); @@ -164,7 +164,7 @@ public Collection AfterTestRunEnd(bool isCanceled, ITestMessageEv } } - return afterTestRunEnd?.AttachmentSets; + return new DataCollectionResult(afterTestRunEnd?.AttachmentSets, afterTestRunEnd?.InvokedDataCollectors); } /// diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs index bbfe0ee0ca..8b76a4cf05 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs @@ -178,7 +178,7 @@ public void SendExecutionComplete( curentArgs.IsCanceled, curentArgs.IsAborted, this.messageProcessingUnrecoverableError, - curentArgs.AttachmentSets, curentArgs.ElapsedTimeInRunningTests + curentArgs.AttachmentSets, curentArgs.InvokedDataCollectors, curentArgs.ElapsedTimeInRunningTests ); } var data = this.dataSerializer.SerializePayload( diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs index cd9ad61edf..8abee57b9e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs @@ -662,6 +662,8 @@ private void RaiseTestRunComplete( aborted, exception, attachments, + // Today we don't offer an extension to run collectors for test adapters. + new Collection(), elapsedTime); testRunCompleteEventArgs.Metrics = this.requestData.MetricsCollection.Metrics; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/ExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/ExecutionManager.cs index 558394bf18..fb814ae0c7 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/ExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/ExecutionManager.cs @@ -158,7 +158,7 @@ public void Cancel(ITestRunEventsHandler testRunEventsHandler) { if (this.activeTestRun == null) { - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, true, false, null, null, TimeSpan.Zero); + var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, true, false, null, null, null, TimeSpan.Zero); testRunEventsHandler.HandleTestRunComplete(testRunCompleteEventArgs, null, null, null); } else @@ -174,7 +174,7 @@ public void Abort(ITestRunEventsHandler testRunEventsHandler) { if (this.activeTestRun == null) { - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero); + var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero); testRunEventsHandler.HandleTestRunComplete(testRunCompleteEventArgs, null, null, null); } else diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs index 6caae78d89..b52f4adf02 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs @@ -10,5 +10,6 @@ [assembly: InternalsVisibleTo("datacollector.PlatformTests, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("testhost, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("testhost.x86, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.console, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index 804e769d77..9a4d5240cb 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -245,13 +245,17 @@ public IProxyExecutionManager GetExecutionManager( }; // parallelLevel = 1 for desktop should go via else route. - return (parallelLevel > 1 || !testHostManager.Shared) + var executionManager = (parallelLevel > 1 || !testHostManager.Shared) ? new ParallelProxyExecutionManager( requestData, proxyExecutionManagerCreator, parallelLevel, sharedHosts: testHostManager.Shared) : proxyExecutionManagerCreator(); + + EqtTrace.Verbose($"TestEngine.GetExecutionManager: Chosen execution manager '{executionManager.GetType().AssemblyQualifiedName}' ParallelLevel '{parallelLevel}' Shared host '{testHostManager.Shared}'"); + + return executionManager; } /// @@ -501,12 +505,12 @@ private bool ShouldRunInNoIsolation( private IRequestData GetRequestData(bool isTelemetryOptedIn) { return new RequestData - { - MetricsCollection = isTelemetryOptedIn + { + MetricsCollection = isTelemetryOptedIn ? (IMetricsCollection)new MetricsCollection() : new NoOpMetricsCollection(), - IsTelemetryOptedIn = isTelemetryOptedIn - }; + IsTelemetryOptedIn = isTelemetryOptedIn + }; } /// diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs index 1a2ff34ad6..6bc69a12ad 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs @@ -22,14 +22,16 @@ public class TestRunCompleteEventArgs : EventArgs /// Specifies whether the test run is aborted. /// Specifies the error encountered during the execution of the test run. /// Attachment sets associated with the run. + /// Invoked data collectors /// Time elapsed in just running tests - public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool isAborted, Exception error, Collection attachmentSets, TimeSpan elapsedTime) + public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool isAborted, Exception error, Collection attachmentSets, Collection invokedDataCollectors, TimeSpan elapsedTime) { this.TestRunStatistics = stats; this.IsCanceled = isCanceled; this.IsAborted = isAborted; this.Error = error; this.AttachmentSets = attachmentSets ?? new Collection(); // Ensuring attachmentSets are not null, so that new attachmentSets can be combined whenever required. + this.InvokedDataCollectors = invokedDataCollectors ?? new Collection(); // Ensuring that invoked data collectors are not null. this.ElapsedTimeInRunningTests = elapsedTime; } @@ -63,6 +65,12 @@ public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool [DataMember] public Collection AttachmentSets { get; private set; } + /// + /// Gets the invoked data collectors for the test session. + /// + [DataMember] + public Collection InvokedDataCollectors { get; private set; } + /// /// Gets the time elapsed in just running the tests. /// Value is set to TimeSpan.Zero in case of any error. diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Payloads/TestRunAttachmentsProcessingPayload.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Payloads/TestRunAttachmentsProcessingPayload.cs index 6abdb68082..8d160b4080 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Payloads/TestRunAttachmentsProcessingPayload.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Payloads/TestRunAttachmentsProcessingPayload.cs @@ -17,6 +17,18 @@ public class TestRunAttachmentsProcessingPayload [DataMember] public IEnumerable Attachments { get; set; } + /// + /// Collection of the invoked data collectors. + /// + [DataMember] + public IEnumerable InvokedDataCollectors { get; set; } + + /// + /// Gets or sets the settings used for the test run request. + /// + [DataMember] + public string RunSettings { get; set; } + /// /// Gets or sets whether Metrics should be collected or not. /// diff --git a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/Attributes/DataCollectorAttachmentProcessor.cs b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/Attributes/DataCollectorAttachmentProcessor.cs new file mode 100644 index 0000000000..08b32fd5d4 --- /dev/null +++ b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/Attributes/DataCollectorAttachmentProcessor.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection +{ + using System; + + /// + /// Registers an attachment processor for a data collector. + /// + public class DataCollectorAttachmentProcessorAttribute : Attribute + { + /// + /// Initializes a new instance of the class. + /// + /// + /// The type of the attachement data processor. + /// + public DataCollectorAttachmentProcessorAttribute(Type type) + { + this.Type = type; + } + + /// + /// Gets the data collector type uri. + /// + public Type Type { get; private set; } + } +} diff --git a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs index dfdc77c9fe..37565e3d89 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs @@ -7,7 +7,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; - + using System.Xml; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; /// @@ -40,11 +40,12 @@ public interface IDataCollectorAttachmentProcessor /// /// Reprocess attachments generated by independent test executions /// + /// Configuration of the attachment processor. Will be the same as the data collector that registers it. /// Attachments to be processed /// Progress reporter. Accepts integers from 0 to 100 /// Message logger /// Cancellation token /// Attachments after reprocessing - Task> ProcessAttachmentSetsAsync(ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); + Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs new file mode 100644 index 0000000000..3580b5279f --- /dev/null +++ b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel +{ + public sealed class InvokedDataCollector : IEquatable + { + /// + /// Initialize an InvokedDataCollector + /// + /// Data collector Uri + /// Data collector assembly qualified name + /// Data collector file path + /// True if data collector registers an attachment processor + public InvokedDataCollector(Uri uri, string assemblyQualifiedName, string filePath, bool hasAttachmentProcessor) + { + this.Uri = uri; + this.AssemblyQualifiedName = assemblyQualifiedName; + this.FilePath = filePath; + this.HasAttachmentProcessor = hasAttachmentProcessor; + } + + /// + /// DataCollector uri. + /// + [DataMember] + public Uri Uri { get; private set; } + + /// + /// AssemblyQualifiedName of data collector. + /// + [DataMember] + public string AssemblyQualifiedName { get; private set; } + + /// + /// Data collector file path. + /// + [DataMember] + public string FilePath { get; private set; } + + /// + /// True if the collector registers an attachments processor. + /// + [DataMember] + public bool HasAttachmentProcessor { get; private set; } + + /// + /// Compares InvokedDataCollector instances for equality. + /// + /// InvokedDataCollector instance + /// True if objects are equal + public bool Equals(InvokedDataCollector other) + { + if (other is null) + { + return false; + } + + return this.HasAttachmentProcessor == other.HasAttachmentProcessor && + this.Uri.AbsoluteUri == other.Uri.AbsoluteUri && + this.AssemblyQualifiedName == other.AssemblyQualifiedName && + this.FilePath == other.FilePath; + } + + /// + /// Compares InvokedDataCollector instances for equality. + /// + /// InvokedDataCollector instance + /// True if objects are equal + public override bool Equals(object obj) + => this.Equals(obj as InvokedDataCollector); + + /// + /// Returns the object hashcode + /// + /// Hashcode value + public override int GetHashCode() + { + unchecked + { + var hashCode = this.Uri.GetHashCode(); + hashCode = (hashCode * 397) ^ this.AssemblyQualifiedName.GetHashCode(); + hashCode = (hashCode * 397) ^ (this.FilePath != null ? this.FilePath.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ this.HasAttachmentProcessor.GetHashCode(); + return hashCode; + } + } + + /// + /// Return string representation for the current object + /// + /// String representation + public override string ToString() + => $"Uri: '{Uri}' AssemblyQualifiedName: '{AssemblyQualifiedName}' FilePath: '{FilePath}' HasAttachmentProcessor: '{HasAttachmentProcessor}'"; + } +} diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt index a6f4408765..5d8ad972d4 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt @@ -324,8 +324,12 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcess Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.Attachments.get -> System.Collections.Generic.IEnumerable Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.Attachments.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.InvokedDataCollectors.get -> System.Collections.Generic.IEnumerable +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.InvokedDataCollectors.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.CollectMetrics.get -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.CollectMetrics.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.RunSettings.get -> string +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.RunSettings.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingPayload.TestRunAttachmentsProcessingPayload() -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingProgressEventArgs Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunAttachmentsProcessingProgressEventArgs.AttachmentProcessorsCount.get -> long @@ -339,6 +343,8 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunChangedEventArgs.N Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunChangedEventArgs.TestRunChangedEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics stats, System.Collections.Generic.IEnumerable newTestResults, System.Collections.Generic.IEnumerable activeTests) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunChangedEventArgs.TestRunStatistics.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.TestRunCompleteEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics stats, bool isCanceled, bool isAborted, System.Exception error, System.Collections.ObjectModel.Collection attachmentSets, System.Collections.ObjectModel.Collection invokedDataCollectors, System.TimeSpan elapsedTime) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.InvokedDataCollectors.get -> System.Collections.ObjectModel.Collection Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.AttachmentSets.get -> System.Collections.ObjectModel.Collection Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.ElapsedTimeInRunningTests.get -> System.TimeSpan Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.Error.get -> System.Exception @@ -346,7 +352,6 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs. Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.IsCanceled.get -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.Metrics.get -> System.Collections.Generic.IDictionary Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.Metrics.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.TestRunCompleteEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics stats, bool isCanceled, bool isAborted, System.Exception error, System.Collections.ObjectModel.Collection attachmentSets, System.TimeSpan elapsedTime) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.TestRunStatistics.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria.AdapterSourceMap.get -> System.Collections.Generic.Dictionary> @@ -871,3 +876,13 @@ static readonly Microsoft.VisualStudio.TestPlatform.ObjectModel.UapConstants.Uap virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.Properties.get -> System.Collections.Generic.IEnumerable virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.ProtectedGetPropertyValue(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestProperty property, object defaultValue) -> object virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.ProtectedSetPropertyValue(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestProperty property, object value) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.InvokedDataCollector(System.Uri uri, string assemblyQualifiedName, string filePath, bool hasAttachmentProcessor) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.Equals(Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector other) -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.AssemblyQualifiedName.get -> string +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.FilePath.get -> string +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.HasAttachmentProcessor.get -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.Uri.get -> System.Uri +override Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.Equals(object obj) -> bool +override Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.GetHashCode() -> int +override Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.ToString() -> string \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt index 7ac21e41e1..7929196a23 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt @@ -68,7 +68,7 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectionSi Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectionSink.SendData(Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectionContext dataCollectionContext, string key, string value) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.GetExtensionUris() -> System.Collections.Generic.IEnumerable -Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Xml.XmlElement configurationElement, System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.SupportsIncrementalProcessing.get -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments.GetExtensionUri() -> System.Uri @@ -241,3 +241,6 @@ static Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities.XmlRunSettingsU static readonly Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.RequestId.Empty -> Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.RequestId virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectionLogger.LogException(Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectionContext context, System.Exception ex, Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectorMessageLevel level) -> void virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollector.Dispose(bool disposing) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectorAttachmentProcessorAttribute +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectorAttachmentProcessorAttribute.DataCollectorAttachmentProcessorAttribute(System.Type type) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectorAttachmentProcessorAttribute.Type.get -> System.Type \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs index 65a4c52eaf..e9cb3c90f9 100644 --- a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs +++ b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs @@ -258,7 +258,6 @@ public virtual TestProcessStartInfo GetTestHostProcessStartInfo( // Try find testhost.exe (or the architecture specific version). We ship those ngened executables for Windows because they have faster startup time. We ship them only for some platforms. // When user specified path to dotnet.exe don't try to find the exexutable, because we will always use the testhost.dll together with their dotnet.exe. // We use dotnet.exe on Windows/ARM. - // TODO: Check if we're on ARM64 win using env var PROCESSARCHITECTURE bool testHostExeFound = false; if (!useCustomDotnetHostpath && this.platformEnvironment.OperatingSystem.Equals(PlatformOperatingSystem.Windows) @@ -480,7 +479,7 @@ public virtual TestProcessStartInfo GetTestHostProcessStartInfo( if (dotnetRoot != null) { - EqtTrace.Verbose($"DotnetTestHostmanager.LaunchTestHostAsync: Found '{dotnetRootEnvName}' in env variables, value '{dotnetRoot}'"); + EqtTrace.Verbose($"DotnetTestHostmanager.LaunchTestHostAsync: Found '{dotnetRootEnvName}' in env variables, value '{dotnetRoot}', forwarding to '{dotnetRootEnvName.Replace(prefix, string.Empty)}'"); startInfo.EnvironmentVariables.Add(dotnetRootEnvName.Replace(prefix, string.Empty), dotnetRoot); } else diff --git a/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs b/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs index 371b71046f..e4a2fd4210 100644 --- a/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs +++ b/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs @@ -11,7 +11,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Utilities using System.Reflection; using System.Threading; using System.Threading.Tasks; - + using System.Xml; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -42,7 +42,7 @@ public IEnumerable GetExtensionUris() yield return CodeCoverageDataCollectorUri; } - public async Task> ProcessAttachmentSetsAsync(ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + public async Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) { if ((attachments?.Any()) != true) return new Collection(); diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/ITranslationLayerRequestSenderAsync.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/ITranslationLayerRequestSenderAsync.cs index aa07882a6b..d8acafe788 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/ITranslationLayerRequestSenderAsync.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/ITranslationLayerRequestSenderAsync.cs @@ -136,11 +136,15 @@ Task StopTestSessionAsync( /// /// /// Collection of attachments. + /// Collection of invoked data collectors. + /// RunSettings configuration /// Enables metrics collection. /// Events handler. /// Cancellation token. Task ProcessTestRunAttachmentsAsync( IEnumerable attachments, + IEnumerable invokedDataCollectors, + string runSettings, bool collectMetrics, ITestRunAttachmentsProcessingEventsHandler testRunAttachmentsProcessingCompleteEventsHandler, CancellationToken cancellationToken); diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/IVsTestConsoleWrapperAsync.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/IVsTestConsoleWrapperAsync.cs index 3474af900f..478b44fa25 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/IVsTestConsoleWrapperAsync.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/Interfaces/IVsTestConsoleWrapperAsync.cs @@ -334,7 +334,7 @@ Task RunTestsWithCustomTestHostAsync( /// /// Enables metrics collection (used for telemetry). /// Event handler to receive session complete event. - /// Cancellation token. + /// Cancellation token. Task ProcessTestRunAttachmentsAsync( IEnumerable attachments, string processingSettings, @@ -343,6 +343,28 @@ Task ProcessTestRunAttachmentsAsync( ITestRunAttachmentsProcessingEventsHandler eventsHandler, CancellationToken cancellationToken); + /// + /// Gets back all attachments to test platform for additional processing (for example merging). + /// + /// + /// Collection of attachments. + /// Collection of invoked data collectors. + /// XML processing settings. + /// + /// Indicates that all test executions are done and all data is provided. + /// + /// Enables metrics collection (used for telemetry). + /// Event handler to receive session complete event. + /// Cancellation token. + Task ProcessTestRunAttachmentsAsync( + IEnumerable attachments, + IEnumerable invokedDataCollectors, + string processingSettings, + bool isLastBatch, + bool collectMetrics, + ITestRunAttachmentsProcessingEventsHandler eventsHandler, + CancellationToken cancellationToken); + /// /// See . /// diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs index be26f63f1f..ef1853a361 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs @@ -870,12 +870,16 @@ public void EndSession() /// public Task ProcessTestRunAttachmentsAsync( IEnumerable attachments, + IEnumerable invokedDataCollectors, + string runSettings, bool collectMetrics, ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, CancellationToken cancellationToken) { return this.SendMessageAndListenAndReportAttachmentsProcessingResultAsync( attachments, + invokedDataCollectors, + runSettings, collectMetrics, testSessionEventsHandler, cancellationToken); @@ -1227,7 +1231,7 @@ private void SendMessageAndListenAndReportTestResults( TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun); var completeArgs = new TestRunCompleteEventArgs( - null, false, true, exception, null, TimeSpan.Zero); + null, false, true, exception, null, null, TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); // Earlier we were closing the connection with vstest.console in case of exceptions. @@ -1310,7 +1314,7 @@ private async Task SendMessageAndListenAndReportTestResultsAsync( TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun); var completeArgs = new TestRunCompleteEventArgs( - null, false, true, exception, null, TimeSpan.Zero); + null, false, true, exception, null, null, TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); // Earlier we were closing the connection with vstest.console in case of exceptions. @@ -1326,6 +1330,8 @@ private async Task SendMessageAndListenAndReportTestResultsAsync( private async Task SendMessageAndListenAndReportAttachmentsProcessingResultAsync( IEnumerable attachments, + IEnumerable invokedDataCollectors, + string runSettings, bool collectMetrics, ITestRunAttachmentsProcessingEventsHandler eventHandler, CancellationToken cancellationToken) @@ -1335,6 +1341,8 @@ private async Task SendMessageAndListenAndReportAttachmentsProcessingResultAsync var payload = new TestRunAttachmentsProcessingPayload { Attachments = attachments, + InvokedDataCollectors = invokedDataCollectors, + RunSettings = runSettings, CollectMetrics = collectMetrics }; @@ -1403,7 +1411,7 @@ private async Task SendMessageAndListenAndReportAttachmentsProcessingResultAsync EqtTrace.Error("Aborting Test Session End Operation: {0}", exception); eventHandler.HandleLogMessage( TestMessageLevel.Error, - TranslationLayerResources.AbortedTestRunAttachmentsProcessing); + TranslationLayerResources.AbortedTestRunAttachmentsProcessing); eventHandler.HandleTestRunAttachmentsProcessingComplete( new TestRunAttachmentsProcessingCompleteEventArgs(false, exception), null); diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs index dbfeaf780f..56f3a0b664 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs @@ -936,6 +936,7 @@ await this.requestSender.StartTestRunWithCustomHostAsync( /// public async Task ProcessTestRunAttachmentsAsync( IEnumerable attachments, + IEnumerable invokedDataCollectors, string processingSettings, bool isLastBatch, bool collectMetrics, @@ -947,10 +948,23 @@ public async Task ProcessTestRunAttachmentsAsync( await this.EnsureInitializedAsync().ConfigureAwait(false); await requestSender.ProcessTestRunAttachmentsAsync( attachments, + invokedDataCollectors, + processingSettings, collectMetrics, testSessionEventsHandler, cancellationToken).ConfigureAwait(false); } + + /// + public Task ProcessTestRunAttachmentsAsync( + IEnumerable attachments, + string processingSettings, + bool isLastBatch, + bool collectMetrics, + ITestRunAttachmentsProcessingEventsHandler eventsHandler, + CancellationToken cancellationToken) + => ProcessTestRunAttachmentsAsync(attachments, Enumerable.Empty(), processingSettings, isLastBatch, collectMetrics, eventsHandler, cancellationToken); + #endregion private void EnsureInitialized() diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index c450e968c9..698975f79d 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -93,9 +93,7 @@ public TestRequestManager() IsTelemetryOptedIn(), CommandLineOptions.Instance.IsDesignMode), new ProcessHelper(), - new TestRunAttachmentsProcessingManager( - TestPlatformEventSource.Instance, - new CodeCoverageDataAttachmentsHandler())) + new TestRunAttachmentsProcessingManager(TestPlatformEventSource.Instance, new DataCollectorAttachmentsProcessorsFactory())) { } @@ -400,8 +398,10 @@ public void ProcessTestRunAttachments( this.currentAttachmentsProcessingCancellationTokenSource = new CancellationTokenSource(); Task task = this.attachmentsProcessingManager.ProcessTestRunAttachmentsAsync( + attachmentsProcessingPayload.RunSettings, requestData, attachmentsProcessingPayload.Attachments, + attachmentsProcessingPayload.InvokedDataCollectors, attachmentsProcessingEventsHandler, this.currentAttachmentsProcessingCancellationTokenSource.Token); task.Wait(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 9a74582e5c..252e5a3f77 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -6,8 +6,9 @@ namespace Microsoft.TestPlatform.AcceptanceTests using System; using System.Collections.Generic; using System.IO; + using System.Linq; using System.Xml; - + using System.Xml.Linq; using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; @@ -103,6 +104,62 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn TryRemoveDirectory(resultsDir); } + [TestMethod] + [NetFullTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource] + public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + + var resultsDir = GetResultsDirectory(); + var assemblyPath = this.BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var secondAssemblyPath = this.BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + string runSettings = this.GetRunsettingsFilePath(resultsDir); + string diagFileName = Path.Combine(resultsDir, "diaglog.txt"); + var extensionsPath = Path.Combine( + this.testEnvironment.TestAssetsPath, + Path.GetFileNameWithoutExtension("AttachmentProcessorDataCollector"), + "bin", + IntegrationTestEnvironment.BuildConfiguration, + this.testEnvironment.RunnerFramework); + var arguments = PrepareArguments(new string[] { assemblyPath, secondAssemblyPath }, null, runSettings, this.FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: resultsDir); + arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{extensionsPath}"); + + XElement runSettingsXml = XElement.Load(runSettings); + + // Today we merge only in the case of ParallelProxyExecutionManager executor, that is chosen if: + // (parallelLevel > 1 || !testHostManager.Shared) -> "src\Microsoft.TestPlatform.CrossPlatEngine\TestEngine.cs" line ~248 + // So we'll merge always in case of DotnetTestHostManager(Shared = false) or in case of DefaultTestHostManager(DisableAppDomain = true) or if MaxCpuCount > 1 + // For NetFull test we need to have more than one test library and MaxCpuCount > 1 + runSettingsXml.Add(new XElement("RunConfiguration", new XElement("MaxCpuCount", 2))); + + // Set datacollector parameters + runSettingsXml.Element("DataCollectionRunSettings") + .Element("DataCollectors") + .Element("DataCollector") + .Add(new XElement("Configuration", new XElement("MergeFile", "MergedFile.txt"))); + runSettingsXml.Save(runSettings); + + this.InvokeVsTest(arguments); + this.ValidateSummaryStatus(2, 2, 2); + + string mergedFile = Directory.GetFiles(resultsDir, "MergedFile.txt", SearchOption.AllDirectories).Single(); + List fileContent = new List(); + using (StreamReader streamReader = new StreamReader(mergedFile)) + { + while (!streamReader.EndOfStream) + { + string line = streamReader.ReadLine(); + Assert.IsTrue(line.StartsWith("SessionEnded_Handler_")); + fileContent.Add(line); + } + } + + Assert.AreEqual(2, fileContent.Distinct().Count()); + + TryRemoveDirectory(resultsDir); + } + private static void CreateDataCollectionRunSettingsFile(string destinationRunsettingsPath, Dictionary dataCollectionAttributes) { var doc = new XmlDocument(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index 711e5a5cb0..f6bc9d8c9d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -109,9 +109,18 @@ public void TestRunWithCodeCoverageParallel(RunnerInfo runnerInfo) } [TestMethod] - [NetFullTargetFrameworkDataSource] + [NetFullTargetFrameworkDataSource()] [NetCoreTargetFrameworkDataSource] - public async Task TestRunWithCodeCoverageAndAttachmentsProcessing(RunnerInfo runnerInfo) + public async Task TestRunWithCodeCoverageAndAttachmentsProcessingWithInvokedDataCollectors(RunnerInfo runnerInfo) + => await TestRunWithCodeCoverageAndAttachmentsProcessingInternal(runnerInfo, true); + + [TestMethod] + [NetFullTargetFrameworkDataSource()] + [NetCoreTargetFrameworkDataSource] + public async Task TestRunWithCodeCoverageAndAttachmentsProcessingWithoutInvokedDataCollectors(RunnerInfo runnerInfo) + => await TestRunWithCodeCoverageAndAttachmentsProcessingInternal(runnerInfo, false); + + private async Task TestRunWithCodeCoverageAndAttachmentsProcessingInternal(RunnerInfo runnerInfo, bool withInvokedDataCollectors) { // arrange AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); @@ -122,9 +131,16 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessing(RunnerInfo run Assert.AreEqual(6, this.runEventHandler.TestResults.Count); Assert.AreEqual(2, this.runEventHandler.Attachments.Count); + Assert.AreEqual(2, this.runEventHandler.InvokedDataCollectors.Count); // act - await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, null, true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); + await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync( + runEventHandler.Attachments, + withInvokedDataCollectors ? runEventHandler.InvokedDataCollectors : null, + withInvokedDataCollectors ? this.GetCodeCoverageRunSettings(1) : null, + true, + true, + testRunAttachmentsProcessingEventHandler, CancellationToken.None); // Assert testRunAttachmentsProcessingEventHandler.EnsureSuccess(); @@ -171,9 +187,10 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingNoMetrics(Runne Assert.AreEqual(6, this.runEventHandler.TestResults.Count); Assert.AreEqual(2, this.runEventHandler.Attachments.Count); + Assert.AreEqual(2, this.runEventHandler.InvokedDataCollectors.Count); // act - await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, null, true, false, testRunAttachmentsProcessingEventHandler, CancellationToken.None); + await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, runEventHandler.InvokedDataCollectors, this.GetCodeCoverageRunSettings(1), true, false, testRunAttachmentsProcessingEventHandler, CancellationToken.None); // Assert testRunAttachmentsProcessingEventHandler.EnsureSuccess(); @@ -218,9 +235,10 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingModuleDuplicate Assert.AreEqual(9, this.runEventHandler.TestResults.Count); Assert.AreEqual(3, this.runEventHandler.Attachments.Count); + Assert.AreEqual(3, this.runEventHandler.InvokedDataCollectors.Count); // act - await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, null, true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); + await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, runEventHandler.InvokedDataCollectors, this.GetCodeCoverageRunSettings(1), true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); // Assert testRunAttachmentsProcessingEventHandler.EnsureSuccess(); @@ -267,9 +285,10 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingSameReportForma Assert.AreEqual(6, this.runEventHandler.TestResults.Count); Assert.AreEqual(2, this.runEventHandler.Attachments.Count); + Assert.AreEqual(2, this.runEventHandler.InvokedDataCollectors.Count); // act - await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, null, true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); + await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, runEventHandler.InvokedDataCollectors, this.GetCodeCoverageRunSettings(1), true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); // Assert testRunAttachmentsProcessingEventHandler.EnsureSuccess(); @@ -320,9 +339,10 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingDifferentReport Assert.AreEqual(12, this.runEventHandler.TestResults.Count); Assert.AreEqual(4, this.runEventHandler.Attachments.Count); + Assert.AreEqual(4, this.runEventHandler.InvokedDataCollectors.Count); // act - await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, null, true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); + await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, runEventHandler.InvokedDataCollectors, this.GetCodeCoverageRunSettings(1, outputFormat: "Coverage"), true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); // Assert testRunAttachmentsProcessingEventHandler.EnsureSuccess(); @@ -373,8 +393,9 @@ public async Task EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runn Assert.AreEqual(6, this.runEventHandler.TestResults.Count); Assert.AreEqual(2, this.runEventHandler.Attachments.Count); + Assert.AreEqual(2, this.runEventHandler.InvokedDataCollectors.Count); - await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, null, true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); + await this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(runEventHandler.Attachments, runEventHandler.InvokedDataCollectors, this.GetCodeCoverageRunSettings(1), true, true, testRunAttachmentsProcessingEventHandler, CancellationToken.None); // act this.vstestConsoleWrapper?.EndSession(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs index c4bd80e385..10c9f7c40c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs @@ -23,6 +23,11 @@ public class RunEventHandler : ITestRunEventsHandler2 /// public List Attachments { get; private set; } + /// + /// Gets the list of the invoked data collectors. + /// + public List InvokedDataCollectors { get; private set; } + /// /// Gets the metrics. /// @@ -45,6 +50,7 @@ public RunEventHandler() this.TestResults = new List(); this.Errors = new List(); this.Attachments = new List(); + this.InvokedDataCollectors = new List(); } public void EnsureSuccess() @@ -80,6 +86,11 @@ public void HandleTestRunComplete( this.Attachments.AddRange(testRunCompleteArgs.AttachmentSets); } + if (testRunCompleteArgs.InvokedDataCollectors != null) + { + this.InvokedDataCollectors.AddRange(testRunCompleteArgs.InvokedDataCollectors); + } + this.Metrics = testRunCompleteArgs.Metrics; } diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/Execution/TestRunRequestTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/Execution/TestRunRequestTests.cs index 55745d4e82..15a5c2a44c 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/Execution/TestRunRequestTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/Execution/TestRunRequestTests.cs @@ -310,7 +310,7 @@ public void HandleRawMessageShouldAddVSTestDataPointsIfTelemetryOptedIn() this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) .Returns(new TestRunCompletePayload() { - TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.MinValue) + TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.MinValue) }); this.testRunRequest.HandleRawMessage(string.Empty); @@ -327,7 +327,7 @@ public void HandleRawMessageShouldInvokeHandleTestRunCompleteOfLoggerManager() this.mockDataSerializer.Setup(x => x.DeserializeMessage(It.IsAny())) .Returns(new Message() { MessageType = MessageType.ExecutionComplete }); - var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, + var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, null, null, TimeSpan.FromSeconds(0)); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) .Returns(new TestRunCompletePayload() @@ -348,7 +348,7 @@ public void HandleRawMessageShouldNotInvokeHandleTestRunCompleteOfLoggerManagerW this.mockDataSerializer.Setup(x => x.DeserializeMessage(It.IsAny())) .Returns(new Message() { MessageType = MessageType.ExecutionComplete }); - var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, + var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, null, null, TimeSpan.FromSeconds(0)); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) .Returns(new TestRunCompletePayload() @@ -388,7 +388,7 @@ public void HandleRawMessageShouldInvokeShouldInvokeHandleTestRunStatsChangeOfLo .Returns(new Message() { MessageType = MessageType.ExecutionComplete }); var testRunChangedEventArgs = new TestRunChangedEventArgs(mockStats.Object, testResults, activeTestCases); - var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, + var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, null, null, TimeSpan.FromSeconds(0)); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) @@ -453,7 +453,7 @@ public void HandleTestRunCompleteShouldInvokeHandleTestRunStatsChangeOfLoggerMan "A") }; var testRunChangedEventArgs = new TestRunChangedEventArgs(mockStats.Object, testResults, activeTestCases); - var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, + var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, null, null, TimeSpan.FromSeconds(0)); testRunRequest.ExecuteAsync(); @@ -466,7 +466,7 @@ public void HandleTestRunCompleteShouldInvokeHandleTestRunStatsChangeOfLoggerMan [TestMethod] public void HandleTestRunCompleteShouldInvokeHandleTestRunCompleteOfLoggerManager() { - var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, + var testRunCompleteEvent = new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, null, null, TimeSpan.FromSeconds(0)); testRunRequest.ExecuteAsync(); @@ -499,6 +499,7 @@ public void HandleTestRunCompleteShouldCollectMetrics() false, null, null, + null, TimeSpan.FromSeconds(0)); testRunCompeleteEventsArgs.Metrics = dict; @@ -528,6 +529,7 @@ public void HandleTestRunCompleteShouldHandleListAttachments() false, null, null, + null, TimeSpan.FromSeconds(0)); // Act @@ -555,6 +557,7 @@ public void HandleTestRunCompleteShouldHandleCollectionAttachments() false, null, null, + null, TimeSpan.FromSeconds(0)); // Act @@ -580,6 +583,7 @@ public void HandleTestRunCompleteShouldHandleEmptyAttachments() false, null, null, + null, TimeSpan.FromSeconds(0)); // Act @@ -597,7 +601,7 @@ public void HandleTestRunCompleteShouldCloseExecutionManager() this.testRunRequest.OnRunCompletion += (s, e) => events.Add("complete"); this.testRunRequest.ExecuteAsync(); - this.testRunRequest.HandleTestRunComplete(new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, null, TimeSpan.FromSeconds(0)), null, null, null); + this.testRunRequest.HandleTestRunComplete(new TestRunCompleteEventArgs(new TestRunStatistics(1, null), false, false, null, null, null, TimeSpan.FromSeconds(0)), null, null, null); Assert.AreEqual(2, events.Count); Assert.AreEqual("close", events[0]); diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs index 55f20f3543..3f623486e7 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs @@ -7,6 +7,8 @@ namespace TestPlatform.Common.UnitTests.ExtensionFramework using System.Collections.Generic; using System.Linq; using System.Reflection; + using System.Threading; + using System.Threading.Tasks; using System.Xml; using Microsoft.VisualStudio.TestPlatform.Common.DataCollector; @@ -139,7 +141,7 @@ public void GetTestExtensionsInformationShouldNotAbortOnFaultyExtensions() var testExtensions = this.testPluginDiscoverer.GetTestExtensionsInformation(pathToExtensions); - Assert.That.DoesNotThrow(() =>this.testPluginDiscoverer.GetTestExtensionsInformation(pathToExtensions)); + Assert.That.DoesNotThrow(() => this.testPluginDiscoverer.GetTestExtensionsInformation(pathToExtensions)); } #region Implementations @@ -320,6 +322,7 @@ public class ADataCollectorInheritingFromAnotherDataCollector : InvalidDataColle [DataCollectorFriendlyName("Foo")] [DataCollectorTypeUri("datacollector://foo/bar")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] public class ValidDataCollector : DataCollector { public override void Initialize( @@ -331,6 +334,21 @@ public override void Initialize( { } } + + public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor + { + public bool SupportsIncrementalProcessing => throw new NotImplementedException(); + + public IEnumerable GetExtensionUris() + { + throw new NotImplementedException(); + } + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } #endregion internal class FaultyTestExecutorPluginInformation : TestExtensionPluginInformation @@ -339,7 +357,7 @@ internal class FaultyTestExecutorPluginInformation : TestExtensionPluginInformat /// Default constructor /// /// The Type. - public FaultyTestExecutorPluginInformation(Type type): base(type) + public FaultyTestExecutorPluginInformation(Type type) : base(type) { throw new Exception(); } diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Logging/InternalTestLoggerEventsTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/Logging/InternalTestLoggerEventsTests.cs index 0cd28995da..b1e368757b 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Logging/InternalTestLoggerEventsTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Logging/InternalTestLoggerEventsTests.cs @@ -134,7 +134,7 @@ public void CompleteTestRunShouldInvokeRegisteredEventHandler() loggerEvents.EnableEvents(); // Send the test run complete event. - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan()); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan()); var waitSuccess = waitHandle.WaitOne(500); Assert.IsTrue(waitSuccess, "Event must be raised within timeout."); @@ -206,7 +206,7 @@ public void CompleteTestRunShouldThrowExceptionIfAlreadyDisposed() Assert.ThrowsException(() => { - loggerEvents.CompleteTestRun(null, true, false, null, null, new TimeSpan()); + loggerEvents.CompleteTestRun(null, true, false, null, null, null, new TimeSpan()); }); } diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs index 1e1c9cca49..d3d578d53b 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs @@ -38,22 +38,28 @@ public void SendAfterTestRunEndAndGetResultShouldReturnAttachments() var displayName = "CustomDataCollector"; var attachment = new AttachmentSet(datacollectorUri, displayName); attachment.Attachments.Add(new UriDataAttachment(attachmentUri, "filename.txt")); - + var invokedDataCollector = new InvokedDataCollector(datacollectorUri, typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())).Returns( - new AfterTestRunEndResult(new Collection() { attachment }, new Dictionary())); + new AfterTestRunEndResult(new Collection() { attachment }, new Collection() { invokedDataCollector }, new Dictionary())); this.mockCommunicationManager.Setup(x => x.ReceiveMessage()).Returns(new Message() { MessageType = MessageType.AfterTestRunEndResult, Payload = null }); var result = this.requestSender.SendAfterTestRunEndAndGetResult(null, false); Assert.IsNotNull(result); Assert.IsNotNull(result.AttachmentSets); + Assert.IsNotNull(result.AttachmentSets); Assert.IsNotNull(result.Metrics); Assert.AreEqual(1, result.AttachmentSets.Count); + Assert.AreEqual(1, result.InvokedDataCollectors.Count); Assert.AreEqual(0, result.Metrics.Count); Assert.IsNotNull(result.AttachmentSets[0]); Assert.AreEqual(displayName, result.AttachmentSets[0].DisplayName); Assert.AreEqual(datacollectorUri, result.AttachmentSets[0].Uri); Assert.AreEqual(attachmentUri, result.AttachmentSets[0].Attachments[0].Uri); + Assert.IsNotNull(result.InvokedDataCollectors[0]); + Assert.AreEqual(datacollectorUri, result.InvokedDataCollectors[0].Uri); + Assert.AreEqual(invokedDataCollector.FilePath, result.InvokedDataCollectors[0].FilePath); + Assert.AreEqual(invokedDataCollector.AssemblyQualifiedName, result.InvokedDataCollectors[0].AssemblyQualifiedName); } [TestMethod] diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestRequestSenderTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestRequestSenderTests.cs index 5b40d4d776..5ddb461f90 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestRequestSenderTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/TestRequestSenderTests.cs @@ -582,7 +582,7 @@ public void StartTestRunShouldNotifyExecutionCompleteOnRunCompleteMessageReceive { var testRunCompletePayload = new TestRunCompletePayload { - TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.MaxValue), + TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.MaxValue), LastRunTests = new TestRunChangedEventArgs(null, null, null), RunAttachments = new List() }; @@ -606,7 +606,7 @@ public void StartTestRunShouldStopServerOnRunCompleteMessageReceived() { var testRunCompletePayload = new TestRunCompletePayload { - TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.MaxValue), + TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.MaxValue), LastRunTests = new TestRunChangedEventArgs(null, null, null), RunAttachments = new List() }; @@ -704,7 +704,7 @@ public void StartTestRunShouldNotNotifyExecutionCompleteIfClientDisconnectedAndO { var testRunCompletePayload = new TestRunCompletePayload { - TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.MaxValue), + TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.MaxValue), LastRunTests = new TestRunChangedEventArgs(null, null, null), RunAttachments = new List() }; diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs new file mode 100644 index 0000000000..a10e739de0 --- /dev/null +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.DataCollectorAttachmentsProcessorsFactoryTests +{ + using Microsoft.TestPlatform.TestUtilities; + using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + using Microsoft.VisualStudio.TestPlatform.Utilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; + + [TestClass] + public class DataCollectorAttachmentsProcessorsFactoryTests + { + private readonly DataCollectorAttachmentsProcessorsFactory dataCollectorAttachmentsProcessorsFactory = new DataCollectorAttachmentsProcessorsFactory(); + + [TestInitialize] + public void Init() + { + TestPluginCacheHelper.SetupMockExtensions(typeof(DataCollectorAttachmentsProcessorsFactoryTests)); + } + + [TestCleanup] + public void Cleanup() + { + TestPluginCacheHelper.ResetExtensionsCache(); + } + + [TestMethod] + public void Create_ShouldReturnListOfAttachmentProcessors() + { + // arrange + List invokedDataCollectors = new List(); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://Sample"), typeof(SampleDataCollector).AssemblyQualifiedName, typeof(SampleDataCollector).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData2"), typeof(SampleData2Collector).AssemblyQualifiedName, typeof(SampleData2Collector).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData3"), typeof(SampleData3Collector).AssemblyQualifiedName, typeof(SampleData3Collector).Assembly.Location, true)); + // act + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray()); + + // assert + Assert.AreEqual(3, dataCollectorAttachmentsProcessors.Count); + + Assert.IsTrue(dataCollectorAttachmentsProcessors.ContainsKey("Sample")); + Assert.IsTrue(dataCollectorAttachmentsProcessors.ContainsKey("SampleData3")); + Assert.IsTrue(dataCollectorAttachmentsProcessors.ContainsKey("Code Coverage")); + + Assert.AreEqual(typeof(DataCollectorAttachmentProcessor).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Sample"].GetType().AssemblyQualifiedName); + Assert.AreEqual(typeof(DataCollectorAttachmentProcessor2).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["SampleData3"].GetType().AssemblyQualifiedName); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Code Coverage"].GetType().AssemblyQualifiedName); + } + + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public void Create_EmptyOrNullInvokedDataCollector_ShouldReturnCodeCoverageDataAttachmentsHandler(bool empty) + { + // act + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(empty ? new InvokedDataCollector[0] : null); + + //assert + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Code Coverage"].GetType().AssemblyQualifiedName); + } + + [TestMethod] + public void Create_ShouldNotFailIfWrongDataCollectorAttachmentProcessor() + { + // arrange + List invokedDataCollectors = new List(); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData4"), typeof(SampleData4Collector).AssemblyQualifiedName, typeof(SampleData4Collector).Assembly.Location, true)); + + // act + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray()); + + // assert + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Code Coverage"].GetType().AssemblyQualifiedName); + } + + [TestMethod] + public void Create_ShouldNotAddTwoTimeCodeCoverageDataAttachmentsHandler() + { + // arrange + List invokedDataCollectors = new List(); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://microsoft/CodeCoverage/2.0"), typeof(SampleData5Collector).AssemblyQualifiedName, typeof(SampleData5Collector).Assembly.Location, true)); + + // act + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray()); + + // assert + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["SampleData5"].GetType().AssemblyQualifiedName); + } + } + + [DataCollectorFriendlyName("Sample")] + [DataCollectorTypeUri("datacollector://Sample")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] + public class SampleDataCollector : DataCollector + { + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } + } + + [DataCollectorFriendlyName("SampleData2")] + [DataCollectorTypeUri("datacollector://SampleData2")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] + public class SampleData2Collector : DataCollector + { + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } + } + + [DataCollectorFriendlyName("SampleData3")] + [DataCollectorTypeUri("datacollector://SampleData3")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor2))] + public class SampleData3Collector : DataCollector + { + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } + } + + [DataCollectorFriendlyName("SampleData4")] + [DataCollectorTypeUri("datacollector://SampleData4")] + [DataCollectorAttachmentProcessor(typeof(string))] + public class SampleData4Collector : DataCollector + { + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } + } + + [DataCollectorFriendlyName("SampleData5")] + [DataCollectorTypeUri("datacollector://microsoft/CodeCoverage/2.0")] + [DataCollectorAttachmentProcessor(typeof(CodeCoverageDataAttachmentsHandler))] + public class SampleData5Collector : DataCollector + { + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } + } + + public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor + { + public bool SupportsIncrementalProcessing => throw new NotImplementedException(); + + public IEnumerable GetExtensionUris() + { + throw new NotImplementedException(); + } + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } + + public class DataCollectorAttachmentProcessor2 : IDataCollectorAttachmentProcessor + { + public bool SupportsIncrementalProcessing => throw new NotImplementedException(); + + public IEnumerable GetExtensionUris() + { + throw new NotImplementedException(); + } + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } +} diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs index a51adcc7d6..4bc8a806bc 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs @@ -5,15 +5,18 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.TestRunAttachmentsPro { using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Linq; using System.Threading; using System.Threading.Tasks; + using System.Xml; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -30,6 +33,7 @@ public class TestRunAttachmentsProcessingManagerTests private readonly Mock mockEventSource; private readonly Mock mockAttachmentHandler1; private readonly Mock mockAttachmentHandler2; + private readonly Mock mockDataCollectorAttachmentsProcessorsFactory; private readonly Mock mockEventsHandler; private readonly TestRunAttachmentsProcessingManager manager; private readonly CancellationTokenSource cancellationTokenSource; @@ -44,11 +48,17 @@ public TestRunAttachmentsProcessingManagerTests() mockAttachmentHandler1 = new Mock(); mockAttachmentHandler2 = new Mock(); mockEventsHandler = new Mock(); + mockDataCollectorAttachmentsProcessorsFactory = new Mock(); mockAttachmentHandler1.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); mockAttachmentHandler2.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri2) }); + mockDataCollectorAttachmentsProcessorsFactory.Setup(p => p.Create(It.IsAny())) + .Returns(new ReadOnlyDictionary(new Dictionary() + { + { "friendlyNameA", mockAttachmentHandler1.Object } , { "friendlyNameB" ,mockAttachmentHandler2.Object } + })); - manager = new TestRunAttachmentsProcessingManager(mockEventSource.Object, mockAttachmentHandler1.Object, mockAttachmentHandler2.Object); + manager = new TestRunAttachmentsProcessingManager(mockEventSource.Object, mockDataCollectorAttachmentsProcessorsFactory.Object); cancellationTokenSource = new CancellationTokenSource(); } @@ -60,7 +70,7 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT List inputAttachments = new List(); // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); // assert VerifyCompleteEvent(false, false); @@ -71,8 +81,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT mockEventSource.Verify(s => s.TestRunAttachmentsProcessingStop(0)); mockAttachmentHandler1.Verify(h => h.GetExtensionUris(), Times.Never); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 0, outputCount: 0); } @@ -84,14 +94,14 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnNoAttachments_IfNoA List inputAttachments = new List(); // act - var result = await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); + var result = await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], cancellationTokenSource.Token); // assert Assert.AreEqual(0, result.Count); mockAttachmentHandler1.Verify(h => h.GetExtensionUris(), Times.Never); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 0, outputCount: 0); } @@ -106,7 +116,7 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturn1NotProcessedAttach }; // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); // assert VerifyCompleteEvent(false, false, inputAttachments[0]); @@ -114,8 +124,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturn1NotProcessedAttach mockEventsHandler.Verify(h => h.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1); } @@ -130,15 +140,15 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturn1NotProcessedAttach }; // act - var result = await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); + var result = await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], cancellationTokenSource.Token); // assert Assert.AreEqual(1, result.Count); Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1); } @@ -157,10 +167,10 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturn1ProcessedAttachmen new AttachmentSet(new Uri(uri1), "uri1_output") }; - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachments); + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachments); // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); // assert VerifyCompleteEvent(false, false, outputAttachments[0]); @@ -168,8 +178,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturn1ProcessedAttachmen mockEventsHandler.Verify(h => h.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1); } @@ -188,10 +198,10 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturn1ProcessedAttachmen new AttachmentSet(new Uri(uri1), "uri1_output") }; - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachments); + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachments); // act - var result = await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); + var result = await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], cancellationTokenSource.Token); // assert Assert.AreEqual(1, result.Count); @@ -200,8 +210,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturn1ProcessedAttachmen mockEventSource.Verify(s => s.TestRunAttachmentsProcessingStop(1)); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1); } @@ -215,10 +225,10 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT new AttachmentSet(new Uri(uri1), "uri1_input") }; - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Throws(new Exception("exception message")); + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Throws(new Exception("exception message")); // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); // assert VerifyCompleteEvent(false, true, inputAttachments[0]); @@ -226,8 +236,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "exception message"), Times.Once); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1, status: "Failed"); } @@ -241,18 +251,18 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachments_ new AttachmentSet(new Uri(uri1), "uri1_input") }; - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Throws(new Exception("exception message")); + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Throws(new Exception("exception message")); // act - var result = await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); + var result = await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], cancellationTokenSource.Token); // assert Assert.AreEqual(1, result.Count); Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1, status: "Failed"); } @@ -268,15 +278,15 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT }; // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); // assert VerifyCompleteEvent(true, false, inputAttachments[0]); mockEventsHandler.Verify(h => h.HandleTestRunAttachmentsProcessingProgress(It.IsAny()), Times.Never); mockAttachmentHandler1.Verify(h => h.GetExtensionUris(), Times.Never); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1, status: "Canceled"); } @@ -292,15 +302,15 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachments_ }; // act - var result = await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); + var result = await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], cancellationTokenSource.Token); // assert Assert.AreEqual(1, result.Count); Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUris(), Times.Never); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1, status: "Canceled"); } @@ -328,11 +338,11 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProcessedAttachment new AttachmentSet(new Uri(uri2), "uri2_output") }; - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler1); - mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler2); + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(null, It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler1); + mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(null, It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler2); // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); // assert VerifyCompleteEvent(false, false, inputAttachments[4], outputAttachmentsForHandler1.First(), outputAttachmentsForHandler2.First()); @@ -340,8 +350,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProcessedAttachment mockEventsHandler.Verify(h => h.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); VerifyMetrics(inputCount: 5, outputCount: 3); } @@ -369,11 +379,11 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProcessedAttachment new AttachmentSet(new Uri(uri2), "uri2_output") }; - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler1); - mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler2); + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler1); + mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).ReturnsAsync(outputAttachmentsForHandler2); // act - var result = await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); + var result = await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], cancellationTokenSource.Token); // assert Assert.AreEqual(3, result.Count); @@ -382,8 +392,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProcessedAttachment Assert.IsTrue(result.Contains(outputAttachmentsForHandler2[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); VerifyMetrics(inputCount: 5, outputCount: 3); } @@ -404,7 +414,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT var innerTaskCompletionSource = new TaskCompletionSource(); - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Returns((ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((string configElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => { try { @@ -433,7 +444,7 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT }); // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); Console.WriteLine("Attachments processing done"); await innerTaskCompletionSource.Task; @@ -447,8 +458,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Informational, "Attachments processing was cancelled.")); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1, status: "Canceled"); } @@ -469,7 +480,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachments_ var innerTaskCompletionSource = new TaskCompletionSource(); - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Returns((ICollection i1, IProgress p, IMessageLogger logger, CancellationToken cancellation) => + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress p, IMessageLogger logger, CancellationToken cancellation) => { try { @@ -497,7 +509,7 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachments_ }); // act - var result = await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); + var result = await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], cancellationTokenSource.Token); Console.WriteLine("Attachments processing done"); await innerTaskCompletionSource.Task; @@ -507,8 +519,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachments_ Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); VerifyMetrics(inputCount: 1, outputCount: 1, status: "Canceled"); } @@ -536,16 +548,17 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProperlySendProgres var innerTaskCompletionSource = new TaskCompletionSource(); int counter = 0; - mockEventsHandler.Setup(h => h.HandleTestRunAttachmentsProcessingProgress(It.IsAny())).Callback(() => - { + mockEventsHandler.Setup(h => h.HandleTestRunAttachmentsProcessingProgress(It.IsAny())).Callback(() => + { counter++; - if(counter == 6) + if (counter == 6) { innerTaskCompletionSource.TrySetResult(null); } }); - mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Returns((ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(null, It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => { progress.Report(25); progress.Report(50); @@ -555,7 +568,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProperlySendProgres return Task.FromResult(outputAttachments1); }); - mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())).Returns((ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(null, It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => { progress.Report(50); logger.SendMessage(TestMessageLevel.Informational, "info"); @@ -564,7 +578,7 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProperlySendProgres }); // act - await manager.ProcessTestRunAttachmentsAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, CancellationToken.None); + await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, CancellationToken.None); // assert await innerTaskCompletionSource.Task; @@ -578,8 +592,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProperlySendProgres mockEventsHandler.Verify(h => h.HandleTestRunAttachmentsProcessingProgress(It.Is(a => VerifyProgressArgsForTwoHandlers(a, 2, 100, uri2)))); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), CancellationToken.None)); - mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[1])), It.IsAny>(), It.IsAny(), CancellationToken.None)); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), CancellationToken.None)); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[1])), It.IsAny>(), It.IsAny(), CancellationToken.None)); mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Informational, "info")); mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "error")); @@ -587,6 +601,94 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnProperlySendProgres VerifyMetrics(inputCount: 2, outputCount: 2); } + [TestMethod] + public async Task ProcessTestRunAttachmentsAsync_ShouldNotFailIfRunsettingsIsNull() + { + // arrange + List inputAttachments = new List + { + new AttachmentSet(new Uri(uri1), "uri1_input"), + new AttachmentSet(new Uri(uri2), "uri2_input") + }; + ICollection outputAttachments = new List + { + new AttachmentSet(new Uri(uri2), "uri2_output") + }; + mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + { + // assert + Assert.IsNull(configurationElement); + return Task.FromResult(outputAttachments); + }); + + // act + await manager.ProcessTestRunAttachmentsAsync(null, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); + + // assert + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); + } + + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task ProcessTestRunAttachmentsAsync_ShouldFlowCorrectDataCollectorConfiguration(bool withConfig) + { + // arrange + List inputAttachments = new List + { + new AttachmentSet(new Uri(uri1), "uri1_input"), + new AttachmentSet(new Uri(uri2), "uri2_input") + }; + + List invokedDataCollectors = new List + { + new InvokedDataCollector(new Uri(uri1), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) + }; + + string runSettingsXml = +$@" + + + + + + Value + + + + + +"; + + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + { + // assert + if (withConfig) + { + Assert.IsNotNull(configurationElement); + Assert.AreEqual("Value", configurationElement.InnerXml); + } + else + { + Assert.IsNull(configurationElement); + } + + ICollection outputAttachments = new List + { + new AttachmentSet(new Uri(uri2), "uri2_output") + }; + return Task.FromResult(outputAttachments); + }); + + // act + await manager.ProcessTestRunAttachmentsAsync(runSettingsXml, mockRequestData.Object, inputAttachments, invokedDataCollectors, mockEventsHandler.Object, cancellationTokenSource.Token); + + // assert + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); + } + private void VerifyMetrics(int inputCount, int outputCount, string status = "Completed") { mockEventSource.Verify(s => s.TestRunAttachmentsProcessingStart(inputCount)); @@ -616,9 +718,9 @@ private bool VerifyProgressArgs(TestRunAttachmentsProcessingProgressEventArgs ar private bool VerifyProgressArgsForTwoHandlers(TestRunAttachmentsProcessingProgressEventArgs args, long handlerIndex, long progress, string uri) { - return progress == args.CurrentAttachmentProcessorProgress && - args.CurrentAttachmentProcessorIndex == handlerIndex && - args.CurrentAttachmentProcessorUris.First().AbsoluteUri == uri && + return progress == args.CurrentAttachmentProcessorProgress && + args.CurrentAttachmentProcessorIndex == handlerIndex && + args.CurrentAttachmentProcessorUris.First().AbsoluteUri == uri && args.AttachmentProcessorsCount == 2; } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs index 78b7376bcf..ca4700c66e 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyExecutionManagerTests.cs @@ -160,7 +160,7 @@ public void StartTestRunWithSourcesShouldNotSendCompleteUntilAllSourcesAreProces [TestMethod] public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCollectionEnabled() { - var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); this.mockTestHostManager = new Mock(); this.mockRequestSender = new Mock(); this.mockDataCollectionManager = new Mock(); @@ -175,12 +175,12 @@ public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCo [TestMethod] public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCollectionEnabledAndCreatorWithDataCollection() { - var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); this.mockTestHostManager = new Mock(); this.mockRequestSender = new Mock(); this.mockDataCollectionManager = new Mock(); var proxyDataCollectionManager = new ProxyExecutionManagerWithDataCollection(this.mockRequestData.Object, this.mockRequestSender.Object, this.mockTestHostManager.Object, this.mockDataCollectionManager.Object); - var managers = new List>(); + var managers = new List>(); this.proxyManagerFunc = () => { this.proxyManagerFuncCalled = true; @@ -195,13 +195,13 @@ public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfDataCo Assert.IsTrue(this.proxyManagerFuncCalled); var handler = parallelExecutionManager.GetHandlerForGivenManager(managers.Last().Object); - Assert.IsTrue(handler is ParallelDataCollectionEventsHandler); + Assert.IsTrue(handler is ParallelDataCollectionEventsHandler); } [TestMethod] public void HandlePartialRunCompleteShouldCreateNewProxyExecutionManagerIfIsAbortedIsTrue() { - var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, true, true, null, null, null, TimeSpan.Zero); this.mockTestHostManager = new Mock(); this.mockRequestSender = new Mock(); var parallelExecutionManager = this.SetupExecutionManager(this.proxyManagerFunc, 2, setupTestCases: true); @@ -362,7 +362,7 @@ public void StartTestRunShouldAggregateRunData() } var completeArgs = new TestRunCompleteEventArgs(new - TestRunStatistics(5, stats), isCanceled, isAborted, null, runAttachments, timespan); + TestRunStatistics(5, stats), isCanceled, isAborted, null, runAttachments, new Collection(), timespan); handler.HandleTestRunComplete(completeArgs, null, runAttachments, executorUris); }); } @@ -480,6 +480,7 @@ private static TestRunCompleteEventArgs CreateTestRunCompleteArgs(bool isCancele isAborted, null, null, + null, TimeSpan.FromMilliseconds(1)); } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs index 75be8e2a8c..ed917f2e4e 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs @@ -19,7 +19,7 @@ public class ParallelRunDataAggregatorTests [TestMethod] public void ParallelRunDataAggregatorConstructorShouldInitializeAggregatorVars() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); Assert.AreEqual(aggregator.ElapsedTime, TimeSpan.Zero, "Timespan must be initialized to zero."); @@ -41,12 +41,12 @@ public void ParallelRunDataAggregatorConstructorShouldInitializeAggregatorVars() [TestMethod] public void AggregateShouldAggregateRunCompleteAttachmentsCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var attachmentSet1 = new Collection(); attachmentSet1.Add(new AttachmentSet(new Uri("x://hello1"), "hello1")); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet1); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet1, null); Assert.AreEqual(1, aggregator.RunCompleteArgsAttachments.Count, "RunCompleteArgsAttachments List must have data."); @@ -54,7 +54,7 @@ public void AggregateShouldAggregateRunCompleteAttachmentsCorrectly() var attachmentSet2 = new Collection(); attachmentSet2.Add(new AttachmentSet(new Uri("x://hello2"), "hello2")); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet2); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet2, null); Assert.AreEqual(2, aggregator.RunCompleteArgsAttachments.Count, "RunCompleteArgsAttachments List must have aggregated data."); } @@ -62,51 +62,82 @@ public void AggregateShouldAggregateRunCompleteAttachmentsCorrectly() [TestMethod] public void AggregateShouldAggregateRunContextAttachmentsCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var attachmentSet1 = new Collection(); attachmentSet1.Add(new AttachmentSet(new Uri("x://hello1"), "hello1")); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet1, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet1, null, null); Assert.AreEqual(1, aggregator.RunContextAttachments.Count, "RunContextAttachments List must have data."); var attachmentSet2 = new Collection(); attachmentSet2.Add(new AttachmentSet(new Uri("x://hello2"), "hello2")); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet2, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet2, null, null); Assert.AreEqual(2, aggregator.RunContextAttachments.Count, "RunContextAttachments List must have aggregated data."); } + [TestMethod] + public void AggregateShouldAggregateInvokedCollectorsCorrectly() + { + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); + + var invokedDataCollectors = new Collection() + { + new InvokedDataCollector(new Uri("datacollector://sample"),typeof(string).AssemblyQualifiedName,typeof(string).Assembly.Location,false) + }; + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors); + Assert.AreEqual(1, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have data."); + + var invokedDataCollectors2 = new Collection() + { + new InvokedDataCollector(new Uri("datacollector://sample2"),typeof(int).AssemblyQualifiedName,typeof(int).Assembly.Location,false) + }; + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors2); + Assert.AreEqual(2, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have aggregated data."); + + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors2); + + Assert.AreEqual(2, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have aggregated data."); + Assert.AreEqual(invokedDataCollectors[0].AssemblyQualifiedName, aggregator.InvokedDataCollectors[0].AssemblyQualifiedName); + Assert.AreEqual(invokedDataCollectors[0].FilePath, aggregator.InvokedDataCollectors[0].FilePath); + Assert.AreEqual(invokedDataCollectors[0].Uri, aggregator.InvokedDataCollectors[0].Uri); + Assert.AreEqual(invokedDataCollectors2[0].AssemblyQualifiedName, aggregator.InvokedDataCollectors[1].AssemblyQualifiedName); + Assert.AreEqual(invokedDataCollectors2[0].FilePath, aggregator.InvokedDataCollectors[1].FilePath); + Assert.AreEqual(invokedDataCollectors2[0].Uri, aggregator.InvokedDataCollectors[1].Uri); + } + [TestMethod] public void AggregateShouldAggregateAbortedAndCanceledCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.IsFalse(aggregator.IsAborted, "Aborted must be false"); Assert.IsFalse(aggregator.IsCanceled, "Canceled must be false"); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: true, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must be true"); Assert.IsFalse(aggregator.IsCanceled, "Canceled must still be false"); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: true, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must continue be true"); Assert.IsTrue(aggregator.IsCanceled, "Canceled must be true"); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must continue be true"); @@ -117,26 +148,26 @@ public void AggregateShouldAggregateAbortedAndCanceledCorrectly() [TestMethod] public void AggregateShouldAggregateTimeSpanCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.AreEqual(TimeSpan.Zero, aggregator.ElapsedTime, "Timespan must be zero"); aggregator.Aggregate(null, null, null, TimeSpan.FromMilliseconds(100), isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.AreEqual(TimeSpan.FromMilliseconds(100), aggregator.ElapsedTime, "Timespan must be 100ms"); aggregator.Aggregate(null, null, null, TimeSpan.FromMilliseconds(200), isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.AreEqual(TimeSpan.FromMilliseconds(200), aggregator.ElapsedTime, "Timespan should be Max of all 200ms"); aggregator.Aggregate(null, null, null, TimeSpan.FromMilliseconds(150), isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.AreEqual(TimeSpan.FromMilliseconds(200), aggregator.ElapsedTime, "Timespan should be Max of all i.e. 200ms"); } @@ -144,16 +175,16 @@ public void AggregateShouldAggregateTimeSpanCorrectly() [TestMethod] public void AggregateShouldAggregateExceptionsCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); aggregator.Aggregate(null, null, exception: null, elapsedTime: TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); Assert.IsNull(aggregator.GetAggregatedException(), "Aggregated exception must be null"); var exception1 = new NotImplementedException(); aggregator.Aggregate(null, null, exception: exception1, elapsedTime: TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); var aggregatedException = aggregator.GetAggregatedException() as AggregateException; Assert.IsNotNull(aggregatedException, "Aggregated exception must NOT be null"); @@ -163,7 +194,7 @@ public void AggregateShouldAggregateExceptionsCorrectly() var exception2 = new NotSupportedException(); aggregator.Aggregate(null, null, exception: exception2, elapsedTime: TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null); aggregatedException = aggregator.GetAggregatedException() as AggregateException; Assert.IsNotNull(aggregatedException, "Aggregated exception must NOT be null"); @@ -175,20 +206,20 @@ public void AggregateShouldAggregateExceptionsCorrectly() [TestMethod] public void AggregateShouldAggregateExecutorUrisCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, null); Assert.AreEqual(0, aggregator.ExecutorUris.Count, "ExecutorUris List must not have data."); var uri1 = "x://hello1"; - aggregator.Aggregate(null, new List() { uri1 }, null, TimeSpan.Zero, false, false, null, null); + aggregator.Aggregate(null, new List() { uri1 }, null, TimeSpan.Zero, false, false, null, null, null); Assert.AreEqual(1, aggregator.ExecutorUris.Count, "ExecutorUris List must have data."); Assert.IsTrue(aggregator.ExecutorUris.Contains(uri1), "ExecutorUris List must have correct data."); var uri2 = "x://hello2"; - aggregator.Aggregate(null, new List() { uri2 }, null, TimeSpan.Zero, false, false, null, null); + aggregator.Aggregate(null, new List() { uri2 }, null, TimeSpan.Zero, false, false, null, null, null); Assert.AreEqual(2, aggregator.ExecutorUris.Count, "ExecutorUris List must have aggregated data."); Assert.IsTrue(aggregator.ExecutorUris.Contains(uri2), "ExecutorUris List must have correct data."); @@ -197,9 +228,9 @@ public void AggregateShouldAggregateExecutorUrisCorrectly() [TestMethod] public void AggregateShouldAggregateRunStatsCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, null); var runStats = aggregator.GetAggregatedRunStats(); Assert.AreEqual(0, runStats.ExecutedTests, "RunStats must not have data."); @@ -211,7 +242,7 @@ public void AggregateShouldAggregateRunStatsCorrectly() stats1.Add(TestOutcome.NotFound, 4); stats1.Add(TestOutcome.None, 2); - aggregator.Aggregate(new TestRunStatistics(12, stats1), null, null, TimeSpan.Zero, false, false, null, null); + aggregator.Aggregate(new TestRunStatistics(12, stats1), null, null, TimeSpan.Zero, false, false, null, null, null); runStats = aggregator.GetAggregatedRunStats(); Assert.AreEqual(12, runStats.ExecutedTests, "RunStats must have aggregated data."); @@ -229,7 +260,7 @@ public void AggregateShouldAggregateRunStatsCorrectly() stats2.Add(TestOutcome.NotFound, 1); stats2.Add(TestOutcome.None, 3); - aggregator.Aggregate(new TestRunStatistics(11, stats2), null, null, TimeSpan.Zero, false, false, null, null); + aggregator.Aggregate(new TestRunStatistics(11, stats2), null, null, TimeSpan.Zero, false, false, null, null, null); runStats = aggregator.GetAggregatedRunStats(); Assert.AreEqual(23, runStats.ExecutedTests, "RunStats must have aggregated data."); @@ -243,7 +274,7 @@ public void AggregateShouldAggregateRunStatsCorrectly() [TestMethod] public void AggregateRunDataMetricsShouldAggregateMetricsCorrectly() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); aggregator.AggregateRunDataMetrics(null); @@ -254,7 +285,7 @@ public void AggregateRunDataMetricsShouldAggregateMetricsCorrectly() [TestMethod] public void AggregateRunDataMetricsShouldAddTotalTestsRun() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); dict.Add(TelemetryDataConstants.TotalTestsRanByAdapter, 2); @@ -271,7 +302,7 @@ public void AggregateRunDataMetricsShouldAddTotalTestsRun() [TestMethod] public void AggregateRunDataMetricsShouldAddTimeTakenToRunTests() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); dict.Add(TelemetryDataConstants.TimeTakenToRunTestsByAnAdapter, .02091); @@ -288,7 +319,7 @@ public void AggregateRunDataMetricsShouldAddTimeTakenToRunTests() [TestMethod] public void AggregateRunDataMetricsShouldAddTimeTakenByAllAdapters() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); dict.Add(TelemetryDataConstants.TimeTakenByAllAdaptersInSec, .02091); @@ -305,7 +336,7 @@ public void AggregateRunDataMetricsShouldAddTimeTakenByAllAdapters() [TestMethod] public void AggregateRunDataMetricsShouldNotAggregateRunState() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); dict.Add(TelemetryDataConstants.RunState, "Completed"); @@ -321,7 +352,7 @@ public void AggregateRunDataMetricsShouldNotAggregateRunState() [TestMethod] public void GetAggregatedRunDataMetricsShouldReturnEmptyIfMetricAggregatorIsEmpty() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); @@ -334,7 +365,7 @@ public void GetAggregatedRunDataMetricsShouldReturnEmptyIfMetricAggregatorIsEmpt [TestMethod] public void GetAggregatedRunDataMetricsShouldReturnEmptyIfMetricsIsNull() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); @@ -347,7 +378,7 @@ public void GetAggregatedRunDataMetricsShouldReturnEmptyIfMetricsIsNull() [TestMethod] public void GetRunDataMetricsShouldAddTotalAdaptersUsedIfMetricsIsNotEmpty() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); dict.Add(TelemetryDataConstants.TotalTestsRanByAdapter, 2); @@ -364,7 +395,7 @@ public void GetRunDataMetricsShouldAddTotalAdaptersUsedIfMetricsIsNotEmpty() [TestMethod] public void GetRunDataMetricsShouldAddNumberOfAdapterDiscoveredIfMetricsIsEmpty() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); dict.Add(TelemetryDataConstants.TimeTakenToRunTestsByAnAdapter + "executor:MSTestV1", .02091); @@ -381,7 +412,7 @@ public void GetRunDataMetricsShouldAddNumberOfAdapterDiscoveredIfMetricsIsEmpty( [TestMethod] public void GetRunDataMetricsShouldNotAddTotalAdaptersUsedIfMetricsIsEmpty() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); aggregator.AggregateRunDataMetrics(dict); @@ -394,7 +425,7 @@ public void GetRunDataMetricsShouldNotAddTotalAdaptersUsedIfMetricsIsEmpty() [TestMethod] public void GetRunDataMetricsShouldNotAddNumberOfAdapterDiscoveredIfMetricsIsEmpty() { - var aggregator = new ParallelRunDataAggregator(); + var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); var dict = new Dictionary(); aggregator.AggregateRunDataMetrics(dict); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunEventsHandlerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunEventsHandlerTests.cs index 11e65f0c84..3e28c73fe0 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunEventsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunEventsHandlerTests.cs @@ -46,7 +46,7 @@ public void TestInit() this.parallelRunEventsHandler = new ParallelRunEventsHandler(this.mockRequestData.Object, this.mockProxyExecutionManager.Object, this.mockTestRunEventsHandler.Object, this.mockParallelProxyExecutionManager.Object, - new ParallelRunDataAggregator(), this.mockDataSerializer.Object); + new ParallelRunDataAggregator(Constants.EmptyRunSettings), this.mockDataSerializer.Object); } [TestMethod] @@ -116,7 +116,7 @@ public void LaunchProcessWithDebuggerAttachedShouldJustPassOnTheEventToRunEvents [TestMethod] public void HandleRunCompleteShouldNotCallLastChunkResultsIfNotPresent() { - var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.Zero); this.mockParallelProxyExecutionManager.Setup(mp => mp.HandlePartialRunComplete( this.mockProxyExecutionManager.Object, completeArgs, null, null, null)).Returns(false); @@ -137,7 +137,7 @@ public void HandleRunCompleteShouldCallLastChunkResultsIfPresent() { string payload = "RunStats"; var lastChunk = new TestRunChangedEventArgs(null, null, null); - var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.Zero); this.mockDataSerializer.Setup(mds => mds.SerializePayload(MessageType.TestRunStatsChange, lastChunk)) .Returns(payload); @@ -160,7 +160,7 @@ public void HandleRunCompleteShouldCallLastChunkResultsIfPresent() public void HandleRunCompleteShouldCallTestRunCompleteOnActualHandlerIfParallelMaangerReturnsCompleteAsTrue() { string payload = "ExecComplete"; - var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.Zero); this.mockParallelProxyExecutionManager.Setup(mp => mp.HandlePartialRunComplete( this.mockProxyExecutionManager.Object, completeArgs, null, null, null)).Returns(true); @@ -189,7 +189,7 @@ public void HandleRunCompleteShouldCollectMetrics() var mockMetricsCollector = new Mock(); this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(mockMetricsCollector.Object); - var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero); + var completeArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.Zero); this.mockParallelProxyExecutionManager.Setup(mp => mp.HandlePartialRunComplete( this.mockProxyExecutionManager.Object, completeArgs, null, null, null)).Returns(true); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs index 00e439873a..1744ba41cc 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs @@ -54,7 +54,7 @@ public void HandleRawMessageShouldSendMessageToBaseTestRunEventsHandler() [TestMethod] public void HandleRawMessageShouldGetDataCollectorAttachments() { - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new TimeSpan()); + var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new Collection(), new TimeSpan()); this.mockDataSerializer.Setup(x => x.DeserializeMessage(It.IsAny())).Returns(new Message() { MessageType = MessageType.ExecutionComplete }); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) @@ -69,7 +69,7 @@ public void HandleRawMessageShouldGetDataCollectorAttachments() [TestMethod] public void HandleRawMessageShouldInvokeAfterTestRunEndPassingFalseIfRequestNotCancelled() { - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new TimeSpan()); + var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new Collection(), new TimeSpan()); this.mockDataSerializer.Setup(x => x.DeserializeMessage(It.IsAny())).Returns(new Message() { MessageType = MessageType.ExecutionComplete }); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) @@ -88,7 +88,7 @@ public void HandleRawMessageShouldInvokeAfterTestRunEndPassingFalseIfRequestNotC [TestMethod] public void HandleRawMessageShouldInvokeAfterTestRunEndPassingTrueIfRequestCancelled() { - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new TimeSpan()); + var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new Collection(), new TimeSpan()); this.mockDataSerializer.Setup(x => x.DeserializeMessage(It.IsAny())).Returns(new Message() { MessageType = MessageType.ExecutionComplete }); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) @@ -105,6 +105,37 @@ public void HandleRawMessageShouldInvokeAfterTestRunEndPassingTrueIfRequestCance Times.Once); } + [TestMethod] + public void HandleRawMessageShouldInvokeAfterTestRunEndAndReturnInvokedDataCollectors() + { + var invokedDataCollectors = new Collection(); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, true)); + + var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new Collection(), new TimeSpan()); + this.mockDataSerializer.Setup(x => x.DeserializeMessage(It.IsAny())).Returns(new Message() { MessageType = MessageType.ExecutionComplete }); + this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())) + .Returns(new TestRunCompletePayload() { TestRunCompleteArgs = testRunCompleteEventArgs }); + this.proxyDataCollectionManager.Setup(p => p.AfterTestRunEnd(It.IsAny(), It.IsAny())) + .Returns(new DataCollectionResult(null, invokedDataCollectors)); + this.mockDataSerializer.Setup(r => r.SerializePayload(It.IsAny(), It.IsAny())).Callback((string message, object o) => + { + Assert.AreEqual(1, ((TestRunCompletePayload)o).TestRunCompleteArgs.InvokedDataCollectors.Count); + Assert.AreEqual(invokedDataCollectors[0], ((TestRunCompletePayload)o).TestRunCompleteArgs.InvokedDataCollectors[0]); + }); + + testRunEventHandler = new DataCollectionTestRunEventsHandler(this.baseTestRunEventsHandler.Object, this.proxyDataCollectionManager.Object, this.mockDataSerializer.Object, CancellationToken.None); + testRunEventHandler.HandleRawMessage(string.Empty); + + var testRunCompleteEventArgs2 = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new Collection(), new TimeSpan()); + testRunEventHandler.HandleTestRunComplete(testRunCompleteEventArgs2, null, null, null); + Assert.AreEqual(1, testRunCompleteEventArgs2.InvokedDataCollectors.Count); + Assert.AreEqual(invokedDataCollectors[0], testRunCompleteEventArgs2.InvokedDataCollectors[0]); + + this.proxyDataCollectionManager.Verify( + dcm => dcm.AfterTestRunEnd(false, It.IsAny()), + Times.Once); + } + #region Get Combined Attachments [TestMethod] public void GetCombinedAttachmentSetsShouldReturnCombinedAttachments() diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ParallelDataCollectionEventsHandlerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ParallelDataCollectionEventsHandlerTests.cs index 6823e8efc1..50e4e1b802 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ParallelDataCollectionEventsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ParallelDataCollectionEventsHandlerTests.cs @@ -15,6 +15,7 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.DataCollection using System.Collections.ObjectModel; using System.Threading; using System.Threading.Tasks; + using Constants = VisualStudio.TestPlatform.ObjectModel.Constants; [TestClass] public class ParallelDataCollectionEventsHandlerTests @@ -40,7 +41,7 @@ public ParallelDataCollectionEventsHandlerTests() mockTestRunAttachmentsProcessingManager = new Mock(); cancellationTokenSource = new CancellationTokenSource(); parallelDataCollectionEventsHandler = new ParallelDataCollectionEventsHandler(mockRequestData.Object, mockProxyExecutionManager.Object, mockTestRunEventsHandler.Object, - mockParallelProxyExecutionManager.Object, new ParallelRunDataAggregator(), mockTestRunAttachmentsProcessingManager.Object, cancellationTokenSource.Token); + mockParallelProxyExecutionManager.Object, new ParallelRunDataAggregator(Constants.EmptyRunSettings), mockTestRunAttachmentsProcessingManager.Object, cancellationTokenSource.Token); mockParallelProxyExecutionManager.Setup(m => m.HandlePartialRunComplete(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>())).Returns(true); } @@ -61,14 +62,14 @@ public void HandleTestRunComplete_ShouldCallProcessTestRunAttachmentsAsyncWithAt new AttachmentSet(new Uri(uri1), "uri1_input1") }; - mockTestRunAttachmentsProcessingManager.Setup(f => f.ProcessTestRunAttachmentsAsync(mockRequestData.Object, It.IsAny>(), It.IsAny())).Returns(Task.FromResult(outputAttachments)); + mockTestRunAttachmentsProcessingManager.Setup(f => f.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(Task.FromResult(outputAttachments)); // act - parallelDataCollectionEventsHandler.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromSeconds(1)), null, inputAttachments, null); + parallelDataCollectionEventsHandler.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromSeconds(1)), null, inputAttachments, null); // assert mockTestRunEventsHandler.Verify(h => h.HandleTestRunComplete(It.IsAny(), It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(outputAttachments[0])), It.IsAny>())); - mockTestRunAttachmentsProcessingManager.Verify(f => f.ProcessTestRunAttachmentsAsync(mockRequestData.Object, It.Is>(a => a.Count == 3), cancellationTokenSource.Token)); + mockTestRunAttachmentsProcessingManager.Verify(f => f.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, It.Is>(a => a.Count == 3), It.IsAny>(), cancellationTokenSource.Token)); } [TestMethod] @@ -82,14 +83,14 @@ public void HandleTestRunComplete_ShouldCallProcessTestRunAttachmentsAsyncWithAt new AttachmentSet(new Uri(uri3), "uri3_input1") }; - mockTestRunAttachmentsProcessingManager.Setup(f => f.ProcessTestRunAttachmentsAsync(mockRequestData.Object, It.IsAny>(), It.IsAny())).Returns(Task.FromResult((Collection)null)); + mockTestRunAttachmentsProcessingManager.Setup(f => f.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(Task.FromResult((Collection)null)); // act - parallelDataCollectionEventsHandler.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromSeconds(1)), null, inputAttachments, null); + parallelDataCollectionEventsHandler.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromSeconds(1)), null, inputAttachments, null); // assert mockTestRunEventsHandler.Verify(h => h.HandleTestRunComplete(It.IsAny(), It.IsAny(), It.Is>(c => c.Count == 3), It.IsAny>())); - mockTestRunAttachmentsProcessingManager.Verify(f => f.ProcessTestRunAttachmentsAsync(mockRequestData.Object, It.Is>(a => a.Count == 3), cancellationTokenSource.Token)); + mockTestRunAttachmentsProcessingManager.Verify(f => f.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, It.Is>(a => a.Count == 3), It.IsAny>(), cancellationTokenSource.Token)); } } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs index e69707fd9c..37c60a1612 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/ProxyDataCollectionManagerTests.cs @@ -225,26 +225,27 @@ public void SendBeforeTestRunStartAndGetResultShouldBeInvokedWithCorrectTestSour public void AfterTestRunEndShouldReturnAttachments(bool telemetryOptedIn) { var attachments = new Collection(); + var invokedDataCollectors = new Collection(); var dispName = "MockAttachments"; var uri = new Uri("Mock://Attachments"); var attachmentSet = new AttachmentSet(uri, dispName); attachments.Add(attachmentSet); this.mockRequestData.Setup(m => m.IsTelemetryOptedIn).Returns(telemetryOptedIn); - var metrics = new Dictionary() + var metrics = new Dictionary() { {"key", "value"} }; - this.mockDataCollectionRequestSender.Setup(x => x.SendAfterTestRunEndAndGetResult(It.IsAny(), It.IsAny())).Returns(new AfterTestRunEndResult(attachments, metrics)); + this.mockDataCollectionRequestSender.Setup(x => x.SendAfterTestRunEndAndGetResult(It.IsAny(), It.IsAny())).Returns(new AfterTestRunEndResult(attachments, invokedDataCollectors, metrics)); var result = this.proxyDataCollectionManager.AfterTestRunEnd(false, null); Assert.IsNotNull(result); - Assert.AreEqual(1, result.Count); - Assert.IsNotNull(result[0]); - Assert.AreEqual(dispName, result[0].DisplayName); - Assert.AreEqual(uri, result[0].Uri); + Assert.AreEqual(1, result.Attachments.Count); + Assert.IsNotNull(result.Attachments[0]); + Assert.AreEqual(dispName, result.Attachments[0].DisplayName); + Assert.AreEqual(uri, result.Attachments[0].Uri); if (telemetryOptedIn) { diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestLoggerManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestLoggerManagerTests.cs index 9221db7145..9acef39540 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestLoggerManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/TestLoggerManagerTests.cs @@ -165,7 +165,7 @@ public void HandleTestRunCompleteShouldInvokeTestRunCompleteHandlerOfLoggers() testLoggerManager.InitializeLoggerByUri(new Uri(loggerUri), new Dictionary()); testLoggerManager.EnableLogging(); - testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, new TimeSpan())); + testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, null, new TimeSpan())); waitHandle.WaitOne(); Assert.AreEqual(1, counter); @@ -182,7 +182,7 @@ public void HandleTestRunCompleteShouldNotInvokeTestRunCompleteHandlerOfLoggersI testLoggerManager.EnableLogging(); testLoggerManager.Dispose(); - testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, new TimeSpan())); + testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, null, new TimeSpan())); Assert.AreEqual(0, counter); } @@ -197,8 +197,8 @@ public void HandleTestRunCompleteShouldDisposeLoggerManager() testLoggerManager.InitializeLoggerByUri(new Uri(loggerUri), new Dictionary()); testLoggerManager.EnableLogging(); - testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, new TimeSpan())); - testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, new TimeSpan())); // count should not increase because of second call. + testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, null, new TimeSpan())); + testLoggerManager.HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, null, new TimeSpan())); // count should not increase because of second call. Assert.AreEqual(1, counter); } diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameLoggerTests.cs b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameLoggerTests.cs index 958763d7c3..2f53aee914 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameLoggerTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/BlameLoggerTests.cs @@ -83,7 +83,7 @@ public void TestRunCompleteHandlerShouldNotReadFileIfTestRunNotAborted() // Setup and Raise event this.mockBlameReaderWriter.Setup(x => x.ReadTestSequence(It.IsAny())); - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(1, 0, 0, 0)); // Verify Call this.mockBlameReaderWriter.Verify(x => x.ReadTestSequence(It.IsAny()), Times.Never); @@ -105,7 +105,7 @@ public void TestRunCompleteHandlerShouldReturnIfUriAttachmentIsNull() this.blameLogger.Initialize(loggerEvents, (string)null); // Setup and Raise event - loggerEvents.CompleteTestRun(null, false, true, null, new Collection(attachmentSetList), new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, true, null, new Collection(attachmentSetList), new Collection(), new TimeSpan(1, 0, 0, 0)); // Verify Call this.mockBlameReaderWriter.Verify(x => x.ReadTestSequence(It.IsAny()), Times.Never); @@ -144,7 +144,7 @@ private void InitializeAndVerify(int count) // Setup and Raise event this.mockBlameReaderWriter.Setup(x => x.ReadTestSequence(It.IsAny())).Returns(testCaseList); - loggerEvents.CompleteTestRun(null, false, true, null, new Collection(attachmentSetList), new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, true, null, new Collection(attachmentSetList), new Collection(), new TimeSpan(1, 0, 0, 0)); // Verify Call this.mockBlameReaderWriter.Verify(x => x.ReadTestSequence(It.Is(str => str.EndsWith(".xml"))), Times.Exactly(count)); diff --git a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/HtmlLoggerTests.cs b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/HtmlLoggerTests.cs index 7458ba6098..63976c48f2 100644 --- a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/HtmlLoggerTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/HtmlLoggerTests.cs @@ -389,7 +389,7 @@ public void TestCompleteHandlerShouldKeepTackOfSummary() { }).Returns(new Mock().Object); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); Assert.AreEqual(4, this.htmlLogger.TestRunDetails.Summary.TotalTests, "summary should keep track of total tests"); Assert.AreEqual(1, this.htmlLogger.TestRunDetails.Summary.FailedTests, "summary should keep track of failed tests"); @@ -412,7 +412,7 @@ public void TestCompleteHandlerShouldCreateCustumHtmlFileNamewithLogFileNameKey( this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); this.htmlLogger.Initialize(new Mock().Object, parameters); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); Assert.IsTrue(this.htmlLogger.HtmlFilePath.Contains("TestResult")); } @@ -430,7 +430,7 @@ public void TestCompleteHandlerShouldCreateCustumHtmlFileNameWithLogPrefix() this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); this.htmlLogger.Initialize(new Mock().Object, parameters); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); Assert.IsFalse(this.htmlLogger.HtmlFilePath.Contains("__")); } @@ -448,7 +448,7 @@ public void TestCompleteHandlerShouldCreateCustumHtmlFileNameWithLogPrefixIfTarg this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); this.htmlLogger.Initialize(new Mock().Object, parameters); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); Assert.IsTrue(this.htmlLogger.HtmlFilePath.Contains("sample_net451")); } @@ -469,7 +469,7 @@ public void TestCompleteHandlerShouldCreateCustumHtmlFileNameWithLogPrefixNull() }).Returns(new Mock().Object); this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); this.mockFileHelper.Verify(x => x.GetStream(It.IsAny(), FileMode.Create, FileAccess.ReadWrite), Times.Once); } @@ -487,7 +487,7 @@ public void TestCompleteHandlerShouldThrowExceptionWithLogPrefixIfTargetFramewor this.htmlLogger.Initialize(new Mock().Object, parameters); - Assert.ThrowsException(() => this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero))); + Assert.ThrowsException(() => this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero))); } [TestMethod] @@ -513,7 +513,7 @@ public void TestCompleteHandlerShouldCreateFileCorrectly() }).Returns(new Mock().Object); this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); this.mockFileHelper.Verify(x => x.GetStream(It.IsAny(), FileMode.Create, FileAccess.ReadWrite), Times.Once); } @@ -530,7 +530,7 @@ public void TestCompleteHandlerShouldDeleteFileCorrectly() }); this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); this.mockFileHelper.Verify(x => x.Delete(It.IsAny()), Times.Once); } @@ -547,7 +547,7 @@ public void TestCompleteHandlerShouldCallHtmlTransformerCorrectly() }).Returns(new Mock().Object); this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); this.mockHtmlTransformer.Verify(x => x.Transform(It.IsAny(), It.IsAny()), Times.Once); } @@ -563,7 +563,7 @@ public void TestCompleteHandlerShouldWriteToXmlSerializerCorrectly() }).Returns(new Mock().Object); this.htmlLogger.TestResultHandler(new object(), resultEventArg1.Object); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); this.mockXmlSerializer.Verify(x => x.WriteObject(It.IsAny(), It.IsAny()), Times.Once); Assert.IsTrue(htmlLogger.XmlFilePath.Contains(".xml")); @@ -577,7 +577,7 @@ public void TestCompleteHandlerShouldNotDivideByZeroWhenThereAre0TestResults() { }).Returns(new Mock().Object); - this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.htmlLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); Assert.AreEqual(0, this.htmlLogger.TestRunDetails.Summary.TotalTests); Assert.AreEqual(0, this.htmlLogger.TestRunDetails.Summary.PassPercentage); diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs index 4b9aacccbf..360eaaf83d 100644 --- a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs @@ -541,7 +541,7 @@ public void TestRunCompleteHandlerShouldReportFailedOutcomeIfTestRunIsAborted() TestRunMessageEventArgs trme = new TestRunMessageEventArgs(TestMessageLevel.Error, message); this.testableTrxLogger.TestMessageHandler(new object(), trme); - this.testableTrxLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero)); + this.testableTrxLogger.TestRunCompleteHandler(new object(), new TestRunCompleteEventArgs(null, false, true, null, null, null, TimeSpan.Zero)); Assert.AreEqual(this.testableTrxLogger.TestResultOutcome, TrxLoggerObjectModel.TestOutcome.Failed); } @@ -941,7 +941,7 @@ private static TestCase CreateTestCase(string testCaseName) private static TestRunCompleteEventArgs CreateTestRunCompleteEventArgs() { var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, - new Collection(), new TimeSpan(1, 0, 0, 0)); + new Collection(), new Collection(), new TimeSpan(1, 0, 0, 0)); return testRunCompleteEventArgs; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index b7181a2fe6..3921f84c96 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -68,16 +68,22 @@ public IntegrationTestBase() /// /// Prepare arguments for vstest.console.exe. /// - /// Name of the test assembly. + /// List of test assemblies. /// Path to test adapter. /// Text of run settings. /// /// /// Command line arguments string. - public static string PrepareArguments(string testAssembly, string testAdapterPath, string runSettings, + public static string PrepareArguments(string[] testAssemblies, string testAdapterPath, string runSettings, string framework, string inIsolation = "", string resultsDirectory = null) { - var arguments = testAssembly.AddDoubleQuote(); + var arguments = ""; + foreach (var path in testAssemblies) + { + arguments += path.AddDoubleQuote() + " "; + } + + arguments = arguments.Trim(); if (!string.IsNullOrWhiteSpace(testAdapterPath)) { @@ -113,6 +119,20 @@ public static string PrepareArguments(string testAssembly, string testAdapterPat return arguments; } + /// + /// Prepare arguments for vstest.console.exe. + /// + /// Name of the test assembly. + /// Path to test adapter. + /// Text of run settings. + /// + /// + /// Command line arguments string. + public static string PrepareArguments(string testAssembly, string testAdapterPath, string runSettings, + string framework, string inIsolation = "", string resultsDirectory = null) + => PrepareArguments(new string[] { testAssembly }, testAdapterPath, runSettings, framework, inIsolation, resultsDirectory); + + /// /// Invokes vstest.console with specified arguments. /// diff --git a/test/Microsoft.TestPlatform.Utilities.UnitTests/CodeCoverageDataAttachmentsHandlerTests.cs b/test/Microsoft.TestPlatform.Utilities.UnitTests/CodeCoverageDataAttachmentsHandlerTests.cs index e805dde084..9e49656cc9 100644 --- a/test/Microsoft.TestPlatform.Utilities.UnitTests/CodeCoverageDataAttachmentsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.Utilities.UnitTests/CodeCoverageDataAttachmentsHandlerTests.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; + using System.Xml; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -17,6 +18,7 @@ public class CodeCoverageDataAttachmentsHandlerTests { private readonly Mock> mockProgressReporter; + private readonly XmlElement configurationElement; private readonly CodeCoverageDataAttachmentsHandler coverageDataAttachmentsHandler; private readonly string _filePrefix; @@ -26,6 +28,9 @@ public class CodeCoverageDataAttachmentsHandlerTests public CodeCoverageDataAttachmentsHandlerTests() { + var doc = new XmlDocument(); + doc.LoadXml(""); + configurationElement = doc.DocumentElement; mockProgressReporter = new Mock>(); coverageDataAttachmentsHandler = new CodeCoverageDataAttachmentsHandler(); #if NETFRAMEWORK @@ -54,12 +59,12 @@ public async Task HandleDataCollectionAttachmentSetsShouldReturnEmptySetWhenNoAt { Collection attachment = new Collection(); ICollection resultAttachmentSets = await - coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(attachment, mockProgressReporter.Object, null, CancellationToken.None); + coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, attachment, mockProgressReporter.Object, null, CancellationToken.None); Assert.IsNotNull(resultAttachmentSets); Assert.IsTrue(resultAttachmentSets.Count == 0); - resultAttachmentSets = await coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(null, mockProgressReporter.Object, null, CancellationToken.None); + resultAttachmentSets = await coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, null, mockProgressReporter.Object, null, CancellationToken.None); Assert.IsNotNull(resultAttachmentSets); Assert.IsTrue(resultAttachmentSets.Count == 0); @@ -75,7 +80,7 @@ public async Task HandleDataCollectionAttachmentSetsShouldReturnInputIfOnly1Atta Collection attachment = new Collection { attachmentSet }; ICollection resultAttachmentSets = await - coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(attachment, mockProgressReporter.Object, null, CancellationToken.None); + coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, attachment, mockProgressReporter.Object, null, CancellationToken.None); Assert.IsNotNull(resultAttachmentSets); Assert.IsTrue(resultAttachmentSets.Count == 1); @@ -95,7 +100,7 @@ public async Task HandleDataCollectionAttachmentSetsShouldReturnInputIf2Differen Collection attachment = new Collection { attachmentSet }; ICollection resultAttachmentSets = await - coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(attachment, mockProgressReporter.Object, null, CancellationToken.None); + coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, attachment, mockProgressReporter.Object, null, CancellationToken.None); Assert.IsNotNull(resultAttachmentSets); Assert.IsTrue(resultAttachmentSets.Count == 1); @@ -116,7 +121,7 @@ public async Task HandleDataCollectionAttachmentSetsShouldReturnInputIf2SameForm Collection attachment = new Collection { attachmentSet }; ICollection resultAttachmentSets = await - coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(attachment, mockProgressReporter.Object, null, CancellationToken.None); + coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, attachment, mockProgressReporter.Object, null, CancellationToken.None); Assert.IsNotNull(resultAttachmentSets); Assert.IsTrue(resultAttachmentSets.Count == 1); @@ -133,7 +138,7 @@ public async Task HandleDataCollectionAttachmentSetsShouldReturnInputIfOnly1Logs Collection attachment = new Collection { attachmentSet }; ICollection resultAttachmentSets = await - coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(attachment, mockProgressReporter.Object, null, CancellationToken.None); + coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, attachment, mockProgressReporter.Object, null, CancellationToken.None); Assert.IsNotNull(resultAttachmentSets); Assert.IsTrue(resultAttachmentSets.Count == 1); @@ -154,7 +159,7 @@ public async Task HandleDataCollectionAttachmentSetsShouldReturnInputIfOnlySever Collection attachment = new Collection { attachmentSet, attachmentSet1 }; ICollection resultAttachmentSets = await - coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(attachment, mockProgressReporter.Object, null, CancellationToken.None); + coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, attachment, mockProgressReporter.Object, null, CancellationToken.None); Assert.IsNotNull(resultAttachmentSets); Assert.IsTrue(resultAttachmentSets.Count == 2); @@ -176,7 +181,7 @@ public async Task HandleDataCollectionAttachmentSetsShouldThrowIfCancellationReq attachmentSet }; - await Assert.ThrowsExceptionAsync(async () => await coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(attachment, mockProgressReporter.Object, null, cts.Token)); + await Assert.ThrowsExceptionAsync(async () => await coverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(configurationElement, attachment, mockProgressReporter.Object, null, cts.Token)); Assert.AreEqual(2, attachment.Count); diff --git a/test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj b/test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj new file mode 100644 index 0000000000..2410868208 --- /dev/null +++ b/test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj @@ -0,0 +1,14 @@ + + + netcoreapp2.1;net451 + netcoreapp3.1 + false + 15.0.0.0 + 1.6.0 + $(MSBuildProjectDirectory)\RunSettings.runsettings + + + + + + diff --git a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs new file mode 100644 index 0000000000..5a82c1999c --- /dev/null +++ b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AttachmentProcessorDataCollector +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.IO; + using System.Linq; + using System.Net.Mail; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + + [DataCollectorFriendlyName("SampleDataCollector")] + [DataCollectorTypeUri("my://sample/datacollector")] + [DataCollectorAttachmentProcessor(typeof(SampleDataCollectorAttachmentProcessor))] + public class SampleDataCollector : DataCollector + { + private DataCollectionSink dataCollectionSink; + private DataCollectionEnvironmentContext context; + private string tempDirectoryPath = Path.GetTempPath(); + + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + events.SessionEnd += this.SessionEnded_Handler; + this.dataCollectionSink = dataSink; + this.context = environmentContext; + } + + private void SessionEnded_Handler(object sender, SessionEndEventArgs e) + { + string tmpAttachment = Path.Combine(tempDirectoryPath, Guid.NewGuid().ToString("N"), "DataCollectorAttachmentProcessor_1.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(tmpAttachment)); + File.WriteAllText(tmpAttachment, $"SessionEnded_Handler_{Guid.NewGuid().ToString("N")}"); + this.dataCollectionSink.SendFileAsync(this.context.SessionDataCollectionContext, tmpAttachment, true); + } + } + + public class SampleDataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor + { + public bool SupportsIncrementalProcessing => true; + + public IEnumerable GetExtensionUris() + => new List() { new Uri("my://sample/datacollector") }; + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + string finalFileName = configurationElement.FirstChild.InnerText; + StringBuilder stringBuilder = new StringBuilder(); + string finalFolder = null; + foreach (var attachmentSet in attachments) + { + foreach (var attachment in attachmentSet.Attachments.OrderBy(f => f.Uri.AbsolutePath)) + { + if (finalFolder is null) + { + finalFolder = Path.GetDirectoryName(attachment.Uri.AbsolutePath); + } + + stringBuilder.AppendLine(File.ReadAllText(attachment.Uri.AbsolutePath)); + } + } + + File.WriteAllText(Path.Combine(finalFolder, finalFileName), stringBuilder.ToString()); + + List mergedAttachment = new List(); + var mergedAttachmentSet = new AttachmentSet(new Uri("my://sample/datacollector"), "SampleDataCollector"); + mergedAttachmentSet.Attachments.Add(UriDataAttachment.CreateFrom(Path.Combine(finalFolder, finalFileName), string.Empty)); + mergedAttachment.Add(mergedAttachmentSet); + + return Task.FromResult((ICollection)new Collection(mergedAttachment)); + } + } +} \ No newline at end of file diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index 8aa29c6ac7..883397e994 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -90,6 +90,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "problematic-child", "proble EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ParametrizedTestProject", "ParametrizedTestProject\ParametrizedTestProject.csproj", "{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataCollector", "AttachmentProcessorDataCollector\AttachmentProcessorDataCollector.csproj", "{16F51720-29D0-472A-93FA-2604D61991B7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -592,6 +594,18 @@ Global {CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|x64.Build.0 = Release|Any CPU {CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|x86.ActiveCfg = Release|Any CPU {CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|x86.Build.0 = Release|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Debug|x64.ActiveCfg = Debug|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Debug|x64.Build.0 = Debug|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Debug|x86.ActiveCfg = Debug|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Debug|x86.Build.0 = Debug|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Release|Any CPU.Build.0 = Release|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Release|x64.ActiveCfg = Release|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Release|x64.Build.0 = Release|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Release|x86.ActiveCfg = Release|Any CPU + {16F51720-29D0-472A-93FA-2604D61991B7}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs index c07b2698b8..22e38b309a 100644 --- a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs +++ b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs @@ -28,6 +28,7 @@ namespace Microsoft.TestPlatform.VsTestConsole.TranslationLayer.UnitTests using TestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult; using Payloads = Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; + using NuGet.Frameworks; [TestClass] public class VsTestConsoleRequestSenderTests @@ -687,7 +688,7 @@ public void DiscoverTestsShouldLogErrorWhenProcessExited() Assert.IsTrue(c.IsCancellationRequested); }).Returns(Task.FromResult((Message)null)); - mockHandler.Setup(mh => mh.HandleDiscoveryComplete( It.IsAny(), null)).Callback(() => manualEvent.Set()); + mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), null)).Callback(() => manualEvent.Set()); this.requestSender.DiscoverTests(sources, null, new TestPlatformOptions(), null, mockHandler.Object); @@ -730,7 +731,7 @@ public void StartTestRunShouldCompleteWithZeroTests() var mockHandler = new Mock(); - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var payload = new TestRunCompletePayload() @@ -759,7 +760,7 @@ public async Task StartTestRunAsyncShouldCompleteWithZeroTests() var mockHandler = new Mock(); - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var payload = new TestRunCompletePayload() @@ -792,7 +793,7 @@ public void StartTestRunShouldCompleteWithSingleTestAndMessage() var testResult = new VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); testResult.Outcome = TestOutcome.Passed; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, @@ -847,7 +848,7 @@ public async Task StartTestRunAsyncShouldCompleteWithSingleTestAndMessage() var testResult = new TestResult(testCase); testResult.Outcome = TestOutcome.Passed; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, @@ -947,7 +948,7 @@ public void StartTestRunWithCustomHostShouldComplete() var testResult = new TestResult(testCase); testResult.Outcome = TestOutcome.Passed; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1011,7 +1012,7 @@ public async Task StartTestRunAsyncWithCustomHostShouldComplete() var testResult = new TestResult(testCase); testResult.Outcome = TestOutcome.Passed; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1075,7 +1076,7 @@ public void StartTestRunWithCustomHostShouldNotAbortAndSendErrorToVstestConsoleI var testResult = new TestResult(testCase); testResult.Outcome = TestOutcome.Passed; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1124,7 +1125,7 @@ public async Task StartTestRunAsyncWithCustomHostShouldNotAbortAndSendErrorToVst var testResult = new TestResult(testCase); testResult.Outcome = TestOutcome.Passed; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1213,7 +1214,7 @@ public void StartTestRunWithSelectedTestsShouldCompleteWithZeroTests() this.InitializeCommunication(); var mockHandler = new Mock(); - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var payload = new TestRunCompletePayload() @@ -1240,7 +1241,7 @@ public async Task StartTestRunAsyncWithSelectedTestsShouldCompleteWithZeroTests( await this.InitializeCommunicationAsync(); var mockHandler = new Mock(); - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var payload = new TestRunCompletePayload() @@ -1274,7 +1275,7 @@ public void StartTestRunWithSelectedTestsShouldCompleteWithSingleTestAndMessage( var testCaseList = new List() { testCase }; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1328,7 +1329,7 @@ public async Task StartTestRunAsyncWithSelectedTestsShouldCompleteWithSingleTest var testCaseList = new List() { testCase }; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1385,7 +1386,7 @@ public void StartTestRunWithSelectedTestsHavingTraitsShouldReturnTestRunComplete var testCaseList = new List() { testCase }; TestRunChangedEventArgs receivedChangeEventArgs = null; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, new List { testResult }, null); var payload = new TestRunCompletePayload() @@ -1441,7 +1442,7 @@ public async Task StartTestRunAsyncWithSelectedTestsHavingTraitsShouldReturnTest var testCaseList = new List() { testCase }; TestRunChangedEventArgs receivedChangeEventArgs = null; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, new List { testResult }, null); var payload = new TestRunCompletePayload() @@ -1497,7 +1498,7 @@ public void StartTestRunWithSelectedTestsHavingTraitsShouldReturnTestRunStatsWit var testCaseList = new List() { testCase }; TestRunChangedEventArgs receivedChangeEventArgs = null; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs( @@ -1554,7 +1555,7 @@ public async Task StartTestRunAsyncWithSelectedTestsHavingTraitsShouldReturnTest var testCaseList = new List() { testCase }; TestRunChangedEventArgs receivedChangeEventArgs = null; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs( @@ -1608,7 +1609,7 @@ public void StartTestRunWithCustomHostWithSelectedTestsComplete() var testCaseList = new List() { testCase }; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1670,7 +1671,7 @@ public async Task StartTestRunWithCustomHostAsyncWithSelectedTestsShouldComplete var testCaseList = new List() { testCase }; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var testsChangedArgs = new TestRunChangedEventArgs(null, new List() { testResult }, null); @@ -1729,7 +1730,7 @@ public void StartTestRunWithCustomHostInParallelShouldCallCustomHostMultipleTime var p2 = new TestProcessStartInfo() { FileName = "Y" }; var message1 = CreateMessage(MessageType.CustomTestHostLaunch, p1); var message2 = CreateMessage(MessageType.CustomTestHostLaunch, p2); - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var completepayload = new TestRunCompletePayload() { ExecutorUris = null, @@ -1768,7 +1769,7 @@ public async Task StartTestRunWithCustomHostAsyncInParallelShouldCallCustomHostM var p2 = new TestProcessStartInfo() { FileName = "Y" }; var message1 = CreateMessage(MessageType.CustomTestHostLaunch, p1); var message2 = CreateMessage(MessageType.CustomTestHostLaunch, p2); - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var completepayload = new TestRunCompletePayload() { ExecutorUris = null, @@ -1838,7 +1839,7 @@ public void StartTestRunShouldLogErrorOnProcessExited() var mockHandler = new Mock(); var manualEvent = new ManualResetEvent(false); var sources = new List { "1.dll" }; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var payload = new TestRunCompletePayload() { @@ -1869,7 +1870,7 @@ public async Task StartTestRunAsyncShouldLogErrorOnProcessExited() { var mockHandler = new Mock(); var sources = new List { "1.dll" }; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var payload = new TestRunCompletePayload() { @@ -1904,9 +1905,9 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithZeroAttachment var mockHandler = new Mock(); var payload = new TestRunAttachmentsProcessingCompletePayload() - { + { AttachmentsProcessingCompleteEventArgs = new TestRunAttachmentsProcessingCompleteEventArgs(false, null), - Attachments = new AttachmentSet[0] + Attachments = new AttachmentSet[0] }; var attachmentsProcessingComplete = new Message() @@ -1915,8 +1916,19 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithZeroAttachment Payload = JToken.FromObject(payload) }; this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(attachmentsProcessingComplete)); + this.mockCommunicationManager.Setup(cm => cm.SendMessage(It.IsAny(), It.IsAny())).Callback((string _, object o) => + { + Assert.AreEqual(Constants.EmptyRunSettings, ((TestRunAttachmentsProcessingPayload)o).RunSettings); + Assert.AreEqual(1, ((TestRunAttachmentsProcessingPayload)o).InvokedDataCollectors.Count()); + }); - await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, true, mockHandler.Object, CancellationToken.None); + await this.requestSender.ProcessTestRunAttachmentsAsync( + new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + Constants.EmptyRunSettings, + true, + mockHandler.Object, + CancellationToken.None); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingStart, It.IsAny())); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingCancel), Times.Never); @@ -1931,19 +1943,31 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithOneAttachment( var mockHandler = new Mock(); - var payload = new TestRunAttachmentsProcessingCompletePayload() + var payload = new TestRunAttachmentsProcessingCompletePayload() { AttachmentsProcessingCompleteEventArgs = new TestRunAttachmentsProcessingCompleteEventArgs(true, new Exception("msg")), - Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") } + Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") } }; var attachmentsProcessingComplete = new Message() { MessageType = MessageType.TestRunAttachmentsProcessingComplete, Payload = JToken.FromObject(payload) }; + this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(attachmentsProcessingComplete)); + this.mockCommunicationManager.Setup(cm => cm.SendMessage(It.IsAny(), It.IsAny())).Callback((string _, object o) => + { + Assert.AreEqual(Constants.EmptyRunSettings, ((TestRunAttachmentsProcessingPayload)o).RunSettings); + Assert.AreEqual(1, ((TestRunAttachmentsProcessingPayload)o).InvokedDataCollectors.Count()); + }); - await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, true, mockHandler.Object, CancellationToken.None); + await this.requestSender.ProcessTestRunAttachmentsAsync( + new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + Constants.EmptyRunSettings, + true, + mockHandler.Object, + CancellationToken.None); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingStart, It.IsAny())); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingCancel), Times.Never); @@ -1974,10 +1998,21 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithOneAttachmentA var message = CreateMessage(MessageType.TestMessage, mpayload); this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(message)); + this.mockCommunicationManager.Setup(cm => cm.SendMessage(It.IsAny(), It.IsAny())).Callback((string _, object o) => + { + Assert.AreEqual(Constants.EmptyRunSettings, ((TestRunAttachmentsProcessingPayload)o).RunSettings); + Assert.AreEqual(1, ((TestRunAttachmentsProcessingPayload)o).InvokedDataCollectors.Count()); + }); mockHandler.Setup(mh => mh.HandleLogMessage(It.IsAny(), It.IsAny())).Callback( () => this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(attachmentsProcessingComplete))); - await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, false, mockHandler.Object, CancellationToken.None); + await this.requestSender.ProcessTestRunAttachmentsAsync( + new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + Constants.EmptyRunSettings, + false, + mockHandler.Object, + CancellationToken.None); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingStart, It.IsAny())); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingCancel), Times.Never); @@ -2015,11 +2050,22 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithOneAttachmentA Payload = JToken.FromObject(progressPayload) }; this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(attachmentsProcessingProgress)); + this.mockCommunicationManager.Setup(cm => cm.SendMessage(It.IsAny(), It.IsAny())).Callback((string _, object o) => + { + Assert.AreEqual(Constants.EmptyRunSettings, ((TestRunAttachmentsProcessingPayload)o).RunSettings); + Assert.AreEqual(1, ((TestRunAttachmentsProcessingPayload)o).InvokedDataCollectors.Count()); + }); mockHandler.Setup(mh => mh.HandleTestRunAttachmentsProcessingProgress(It.IsAny())).Callback( () => this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(attachmentsProcessingComplete))); - await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, false, mockHandler.Object, CancellationToken.None); + await this.requestSender.ProcessTestRunAttachmentsAsync( + new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + Constants.EmptyRunSettings, + false, + mockHandler.Object, + CancellationToken.None); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingStart, It.IsAny())); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingCancel), Times.Never); @@ -2051,13 +2097,24 @@ public async Task ProcessTestRunAttachmentsAsyncShouldSendCancelMessageIfCancell var message = CreateMessage(MessageType.TestMessage, mpayload); this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(message)); + this.mockCommunicationManager.Setup(cm => cm.SendMessage(It.IsAny(), It.IsAny())).Callback((string _, object o) => + { + Assert.AreEqual(Constants.EmptyRunSettings, ((TestRunAttachmentsProcessingPayload)o).RunSettings); + Assert.AreEqual(1, ((TestRunAttachmentsProcessingPayload)o).InvokedDataCollectors.Count()); + }); mockHandler.Setup(mh => mh.HandleLogMessage(It.IsAny(), It.IsAny())).Callback(() => { cts.Cancel(); this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(attachmentsProcessingComplete)); }); - await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, false, mockHandler.Object, cts.Token); + await this.requestSender.ProcessTestRunAttachmentsAsync( + new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + Constants.EmptyRunSettings, + false, + mockHandler.Object, + cts.Token); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingStart, It.IsAny())); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingCancel)); @@ -2086,8 +2143,19 @@ public async Task ProcessTestRunAttachmentsAsyncShouldSendCancelMessageIfCancell }; this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(attachmentsProcessingComplete)); + this.mockCommunicationManager.Setup(cm => cm.SendMessage(It.IsAny(), It.IsAny())).Callback((string _, object o) => + { + Assert.AreEqual(Constants.EmptyRunSettings, ((TestRunAttachmentsProcessingPayload)o).RunSettings); + Assert.AreEqual(1, ((TestRunAttachmentsProcessingPayload)o).InvokedDataCollectors.Count()); + }); - await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, true, mockHandler.Object, cts.Token); + await this.requestSender.ProcessTestRunAttachmentsAsync( + new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + Constants.EmptyRunSettings, + true, + mockHandler.Object, + cts.Token); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingStart, It.IsAny())); mockCommunicationManager.Verify(c => c.SendMessage(MessageType.TestRunAttachmentsProcessingCancel)); @@ -2101,7 +2169,7 @@ public async Task ProcessTestRunAttachmentsAsyncShouldAbortOnExceptionInSendMess var mockHandler = new Mock(); this.mockCommunicationManager.Setup(cm => cm.SendMessage(MessageType.TestRunAttachmentsProcessingStart, It.IsAny())).Throws(new IOException()); - await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, false, mockHandler.Object, CancellationToken.None); + await this.requestSender.ProcessTestRunAttachmentsAsync(new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, new List(), Constants.EmptyRunSettings, false, mockHandler.Object, CancellationToken.None); mockHandler.Verify(mh => mh.HandleTestRunAttachmentsProcessingComplete(It.Is(a => !a.IsCanceled && a.Error is IOException), null), Times.Once, "Attachments Processing Complete must be called"); mockHandler.Verify(mh => mh.HandleLogMessage(TestMessageLevel.Error, It.IsAny()), Times.Once, "TestMessage event must be called"); @@ -2519,7 +2587,7 @@ private void SetupMockCommunicationForRunRequest(Mock moc var testResult = new VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); testResult.Outcome = TestOutcome.Passed; - var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.FromMilliseconds(1)); + var dummyCompleteArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, TimeSpan.FromMilliseconds(1)); var dummyLastRunArgs = new TestRunChangedEventArgs(null, null, null); var payload = new TestRunCompletePayload() diff --git a/test/TranslationLayer.UnitTests/VsTestConsoleWrapperAsyncTests.cs b/test/TranslationLayer.UnitTests/VsTestConsoleWrapperAsyncTests.cs index a4a85627f9..e92b43081c 100644 --- a/test/TranslationLayer.UnitTests/VsTestConsoleWrapperAsyncTests.cs +++ b/test/TranslationLayer.UnitTests/VsTestConsoleWrapperAsyncTests.cs @@ -564,17 +564,19 @@ await this.consoleWrapper.RunTestsWithCustomTestHostAsync( public async Task ProcessTestRunAttachmentsAsyncShouldSucceed() { var attachments = new Collection(); + var invokedDataCollectors = new Collection(); var cancellationToken = new CancellationToken(); await this.consoleWrapper.ProcessTestRunAttachmentsAsync( attachments, - null, + invokedDataCollectors, + Constants.EmptyRunSettings, true, true, new Mock().Object, cancellationToken); - this.mockRequestSender.Verify(rs => rs.ProcessTestRunAttachmentsAsync(attachments, true, It.IsAny(), cancellationToken)); + this.mockRequestSender.Verify(rs => rs.ProcessTestRunAttachmentsAsync(attachments, invokedDataCollectors, Constants.EmptyRunSettings, true, It.IsAny(), cancellationToken)); } [TestMethod] diff --git a/test/TranslationLayer.UnitTests/VsTestConsoleWrapperTests.cs b/test/TranslationLayer.UnitTests/VsTestConsoleWrapperTests.cs index 2c4f86e188..d47cb5440b 100644 --- a/test/TranslationLayer.UnitTests/VsTestConsoleWrapperTests.cs +++ b/test/TranslationLayer.UnitTests/VsTestConsoleWrapperTests.cs @@ -558,17 +558,19 @@ public void RunTestsWithSelectedTestsAndACustomHostShouldSucceedWhenUsingSession public async Task ProcessTestRunAttachmentsAsyncShouldSucceed() { var attachments = new Collection(); + var invokedDataCollectors = new Collection(); var cancellationToken = new CancellationToken(); await this.consoleWrapper.ProcessTestRunAttachmentsAsync( attachments, - null, + invokedDataCollectors, + Constants.EmptyRunSettings, true, true, new Mock().Object, cancellationToken); - this.mockRequestSender.Verify(rs => rs.ProcessTestRunAttachmentsAsync(attachments, true, It.IsAny(), cancellationToken)); + this.mockRequestSender.Verify(rs => rs.ProcessTestRunAttachmentsAsync(attachments, invokedDataCollectors, Constants.EmptyRunSettings, true, It.IsAny(), cancellationToken)); } [TestMethod] diff --git a/test/datacollector.PlatformTests/CommunicationLayerIntegrationTests.cs b/test/datacollector.PlatformTests/CommunicationLayerIntegrationTests.cs index 974ab32c6f..4040508d97 100644 --- a/test/datacollector.PlatformTests/CommunicationLayerIntegrationTests.cs +++ b/test/datacollector.PlatformTests/CommunicationLayerIntegrationTests.cs @@ -70,11 +70,11 @@ public void AfterTestRunShouldSendGetAttachments() proxyDataCollectionManager.BeforeTestRunStart(true, true, this.mockTestMessageEventHandler.Object); - var attachments = proxyDataCollectionManager.AfterTestRunEnd(false, this.mockTestMessageEventHandler.Object); + var dataCollectionResult = proxyDataCollectionManager.AfterTestRunEnd(false, this.mockTestMessageEventHandler.Object); - Assert.AreEqual("CustomDataCollector", attachments[0].DisplayName); - Assert.AreEqual("my://custom/datacollector", attachments[0].Uri.ToString()); - Assert.IsTrue(attachments[0].Attachments[0].Uri.ToString().Contains("filename.txt")); + Assert.AreEqual("CustomDataCollector", dataCollectionResult.Attachments[0].DisplayName); + Assert.AreEqual("my://custom/datacollector", dataCollectionResult.Attachments[0].Uri.ToString()); + Assert.IsTrue(dataCollectionResult.Attachments[0].Attachments[0].Uri.ToString().Contains("filename.txt")); } } diff --git a/test/datacollector.UnitTests/DataCollectionManagerTests.cs b/test/datacollector.UnitTests/DataCollectionManagerTests.cs index 26828281eb..0b017379c1 100644 --- a/test/datacollector.UnitTests/DataCollectionManagerTests.cs +++ b/test/datacollector.UnitTests/DataCollectionManagerTests.cs @@ -7,6 +7,8 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.UnitTests using System.Collections.Generic; using System.Linq; using System.Reflection; + using System.Threading; + using System.Threading.Tasks; using System.Xml; using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces; @@ -78,6 +80,8 @@ public void InitializeDataCollectorsShouldLoadDataCollector() this.dataCollectionManager.InitializeDataCollectors(dataCollectorSettings); Assert.IsTrue(this.dataCollectionManager.RunDataCollectors.ContainsKey(this.mockDataCollector.Object.GetType())); + Assert.AreEqual(typeof(AttachmentProcessorDataCollector2), this.dataCollectionManager.RunDataCollectors[this.mockDataCollector.Object.GetType()].DataCollectorConfig.AttachmentsProcessorType); + Assert.IsTrue(this.dataCollectionManager.RunDataCollectors[this.mockDataCollector.Object.GetType()].DataCollectorConfig.Metadata.Contains(true)); this.mockDataCollector.Verify(x => x.Initialize(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); } @@ -244,8 +248,8 @@ public void InitializeDataCollectorsShouldReturnOtherThanCodeCoverageEnvironment this.dataCollectorSettings = string.Format(this.defaultRunSettings, string.Format(this.defaultDataCollectionSettings, "Code Coverage", "my://custom/ccdatacollector", this.mockDataCollector.Object.GetType().AssemblyQualifiedName, typeof(DataCollectionManagerTests).GetTypeInfo().Assembly.Location, string.Empty) + string.Format(this.defaultDataCollectionSettings, this.friendlyName, this.uri, this.mockDataCollector.Object.GetType().AssemblyQualifiedName, typeof(DataCollectionManagerTests).GetTypeInfo().Assembly.Location, string.Empty)); - - var result = this.dataCollectionManager.InitializeDataCollectors(this.dataCollectorSettings); + + var result = this.dataCollectionManager.InitializeDataCollectors(this.dataCollectorSettings); Assert.AreEqual(3, result.Count); Assert.AreEqual("clrie", result["cor_profiler"]); @@ -355,6 +359,16 @@ public void SessionEndedShouldReturnEmptyCollectionIfDataCollectionIsNotEnabled( Assert.AreEqual(0, result.Count); } + [TestMethod] + public void GetInvokedDataCollectorsShouldReturnDataCollector() + { + var dataCollectorSettingsWithNullFriendlyName = string.Format(this.defaultRunSettings, string.Format(this.defaultDataCollectionSettings, string.Empty, uri, this.mockDataCollector.Object.GetType().AssemblyQualifiedName, typeof(DataCollectionManagerTests).GetTypeInfo().Assembly.Location, string.Empty).Replace("friendlyName=\"\"", string.Empty)); + this.dataCollectionManager.InitializeDataCollectors(dataCollectorSettingsWithNullFriendlyName); + var invokedDataCollector = this.dataCollectionManager.GetInvokedDataCollectors(); + Assert.AreEqual(1, invokedDataCollector.Count); + Assert.IsTrue(invokedDataCollector[0].HasAttachmentProcessor); + } + [TestMethod] public void SessionEndedShouldReturnAttachments() { @@ -554,10 +568,26 @@ protected override DataCollector TryGetTestExtension(string extensionUri) return null; } + + protected override DataCollectorConfig TryGetDataCollectorConfig(string extensionUri) + { + if (extensionUri.Equals("my://custom/datacollector")) + { + return new DataCollectorConfig(dataCollector.GetType()); + } + + if (extensionUri.Equals("my://custom/ccdatacollector")) + { + return new DataCollectorConfig(ccDataCollector.GetType()); + } + + return null; + } } [DataCollectorFriendlyName("CustomDataCollector")] [DataCollectorTypeUri("my://custom/datacollector")] + [DataCollectorAttachmentProcessor(typeof(AttachmentProcessorDataCollector2))] public abstract class DataCollector2 : DataCollector { } @@ -567,4 +597,19 @@ public abstract class DataCollector2 : DataCollector public abstract class CodeCoverageDataCollector : DataCollector { } + + public class AttachmentProcessorDataCollector2 : IDataCollectorAttachmentProcessor + { + public bool SupportsIncrementalProcessing => throw new NotImplementedException(); + + public IEnumerable GetExtensionUris() + { + throw new NotImplementedException(); + } + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } } diff --git a/test/vstest.console.UnitTests/CommandLine/TestRunResultAggregatorTests.cs b/test/vstest.console.UnitTests/CommandLine/TestRunResultAggregatorTests.cs index ffd898b325..5795a32591 100644 --- a/test/vstest.console.UnitTests/CommandLine/TestRunResultAggregatorTests.cs +++ b/test/vstest.console.UnitTests/CommandLine/TestRunResultAggregatorTests.cs @@ -59,7 +59,7 @@ public void TestRunMessageHandlerForMessageLevelErrorSetsOutcomeToFailed() [TestMethod] public void TestRunCompletionHandlerForTestRunStatisticsNullSetsOutcomeToFailed() { - var messageArgs = new TestRunCompleteEventArgs(null, false, false, null, null, new TimeSpan()); + var messageArgs = new TestRunCompleteEventArgs(null, false, false, null, null, null, new TimeSpan()); mockTestRunRequest.Raise(tr => tr.OnRunCompletion += null, messageArgs); Assert.AreEqual(TestOutcome.Failed, resultAggregator.Outcome); } @@ -71,7 +71,7 @@ public void TestRunCompletionHandlerForTestRunStatsWithOneOrMoreFailingTestsSets testOutcomeDict.Add(TestOutcome.Failed, 1); var stats = new TestableTestRunStats(testOutcomeDict); - var messageArgs = new TestRunCompleteEventArgs(stats, false, false, null, null, new TimeSpan()); + var messageArgs = new TestRunCompleteEventArgs(stats, false, false, null, null, null, new TimeSpan()); this.mockTestRunRequest.Raise(tr => tr.OnRunCompletion += null, messageArgs); Assert.AreEqual(TestOutcome.Failed, resultAggregator.Outcome); } @@ -83,7 +83,7 @@ public void TestRunCompletionHandlerForCanceledRunShouldSetsOutcomeToFailed() testOutcomeDict.Add(TestOutcome.Passed, 1); var stats = new TestableTestRunStats(testOutcomeDict); - var messageArgs = new TestRunCompleteEventArgs(stats, true, false, null, null, new TimeSpan()); + var messageArgs = new TestRunCompleteEventArgs(stats, true, false, null, null, null, new TimeSpan()); this.mockTestRunRequest.Raise(tr => tr.OnRunCompletion += null, messageArgs); Assert.AreEqual(TestOutcome.Failed, resultAggregator.Outcome); } @@ -95,7 +95,7 @@ public void TestRunCompletionHandlerForAbortedRunShouldSetsOutcomeToFailed() testOutcomeDict.Add(TestOutcome.Passed, 1); var stats = new TestableTestRunStats(testOutcomeDict); - var messageArgs = new TestRunCompleteEventArgs(stats, false, true, null, null, new TimeSpan()); + var messageArgs = new TestRunCompleteEventArgs(stats, false, true, null, null, null, new TimeSpan()); this.mockTestRunRequest.Raise(tr => tr.OnRunCompletion += null, messageArgs); Assert.AreEqual(TestOutcome.Failed, resultAggregator.Outcome); } diff --git a/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs b/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs index 811877fb09..91f7e136a2 100644 --- a/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs +++ b/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs @@ -348,7 +348,7 @@ public void TestRunErrorMessageShowShouldTestRunFailed() // Act. Raise an event on mock object loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, message)); - loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), new Collection(), TimeSpan.FromSeconds(1))); loggerEvents.WaitForEventCompletion(); // Verify @@ -373,7 +373,7 @@ public void InQuietModeTestErrorMessageShouldShowTestRunFailed() // Act. Raise an event on mock object loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, message)); - loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), new Collection(), TimeSpan.FromSeconds(1))); loggerEvents.WaitForEventCompletion(); // Verify @@ -397,7 +397,7 @@ public void InQuietModeTestWarningMessageShouldNotShow() // Act. Raise an event on mock object loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Warning, message)); - loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), new Collection(), TimeSpan.FromSeconds(1))); loggerEvents.WaitForEventCompletion(); // Verify @@ -422,7 +422,7 @@ public void InNormalModeTestWarningAndErrorMessagesShouldShow() // Act. Raise an event on mock object loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Warning, message)); loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, errorMessage)); - loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), new Collection(), TimeSpan.FromSeconds(1))); loggerEvents.WaitForEventCompletion(); // Verify @@ -609,7 +609,7 @@ public void TestResultHandlerShouldShowFailedTestsAndPassedTestsForQuietVebosity loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); } - loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), new Collection(), TimeSpan.FromSeconds(1))); loggerEvents.WaitForEventCompletion(); this.mockOutput.Verify(o => o.Write(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary, @@ -661,7 +661,7 @@ public void TestResultHandlerShouldNotShowformattedFailedTestsAndPassedTestsForO loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); } - loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), new Collection(), TimeSpan.FromSeconds(1))); loggerEvents.WaitForEventCompletion(); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary, CommandLineResources.PassedTestIndicator, 2, 1, 0, 1, "1 m 2 s", "TestSourcePassed", "(net451)"), OutputLevel.Information), Times.Never); @@ -830,7 +830,7 @@ public void TestRunCompleteHandlerShouldWriteToConsoleIfTestsPass() { loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); } - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryTotalTests, 1), OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryPassedTests, 1), OutputLevel.Information), Times.Once()); @@ -853,7 +853,7 @@ public void TestRunCompleteHandlerShouldWriteToConsoleIfTestsFail() { loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); } - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryTotalTests, 1), OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryFailedTests, 1), OutputLevel.Information), Times.Once()); @@ -877,7 +877,7 @@ public void TestRunCompleteHandlerShouldWriteToConsoleIfTestsCanceled() { loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); } - loggerEvents.CompleteTestRun(null, true, false, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, true, false, null, null, null, new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryForCanceledOrAbortedRun), OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryFailedTests, 1), OutputLevel.Information), Times.Once()); @@ -895,7 +895,7 @@ public void TestRunCompleteHandlerShouldWriteToConsoleIfTestsCanceledWithoutRunn parameters.Add("verbosity", "normal"); this.consoleLogger.Initialize(loggerEvents, parameters); - loggerEvents.CompleteTestRun(null, true, false, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, true, false, null, null, null, new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.TestRunCanceled, OutputLevel.Error), Times.Once()); } @@ -913,7 +913,7 @@ public void TestRunCompleteHandlerShouldWriteToConsoleIfTestsAborted() { loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); } - loggerEvents.CompleteTestRun(null, false, true, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, true, null, null, null, new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryForCanceledOrAbortedRun), OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.TestRunAborted, OutputLevel.Error), Times.Once()); @@ -928,7 +928,7 @@ public void TestRunCompleteHandlerShouldWriteToConsoleIfTestsAbortedWithoutRunni parameters.Add("verbosity", "normal"); this.consoleLogger.Initialize(loggerEvents, parameters); - loggerEvents.CompleteTestRun(null, false, true, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, true, null, null, null, new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.TestRunAborted, OutputLevel.Error), Times.Once()); } @@ -1036,10 +1036,10 @@ public void PrintTimeHandlerShouldPrintElapsedTimeOnConsole() { loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); } - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(1, 0, 0, 0)); - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(0, 1, 0, 0)); - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(0, 0, 1, 0)); - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(0, 0, 0, 1)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(0, 1, 0, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(0, 0, 1, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(0, 0, 0, 1)); // Verify PrintTimeSpan with different formats this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.ExecutionTimeFormatString, 1, CommandLineResources.Days), OutputLevel.Information), Times.Once()); @@ -1149,7 +1149,7 @@ public void AttachmentInformationShouldBeWrittenToConsoleIfAttachmentsArePresent { attachmentSet }; - loggerEvents.CompleteTestRun(null, false, false, null, new Collection(attachmentSetList), new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, new Collection(attachmentSetList), new Collection(), new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.AttachmentOutputFormat, uriDataAttachment.Uri.LocalPath), OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.AttachmentOutputFormat, uriDataAttachment1.Uri.LocalPath), OutputLevel.Information), Times.Once()); @@ -1188,7 +1188,7 @@ public void ResultsInHeirarchichalOrderShouldReportCorrectCount() loggerEvents.RaiseTestResult(new TestResultEventArgs(result2)); loggerEvents.RaiseTestResult(new TestResultEventArgs(result3)); - loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(1, 0, 0, 0)); + loggerEvents.CompleteTestRun(null, false, false, null, null, null, new TimeSpan(1, 0, 0, 0)); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryFailedTests, 1), OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryPassedTests, 1), OutputLevel.Information), Times.Once()); @@ -1231,8 +1231,8 @@ private void Setup() var testresult1 = new ObjectModel.TestResult(testcase) { Outcome = TestOutcome.Failed, - Duration = duration1, - StartTime = DateTime.Now - duration1, + Duration = duration1, + StartTime = DateTime.Now - duration1, EndTime = DateTime.Now }; diff --git a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs index 3a79cd4301..b100a49197 100644 --- a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs @@ -527,7 +527,7 @@ public void ExecutorShouldDisplayWarningIfNoTestsAreExecuted() mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Returns(1).Raises(tr => tr.OnRunCompletion += null, - new TestRunCompleteEventArgs(mockTestRunStats.Object, false, false, null, null, new TimeSpan())); + new TestRunCompleteEventArgs(mockTestRunStats.Object, false, false, null, null, null, new TimeSpan())); mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); @@ -556,7 +556,7 @@ public void ExecutorShouldNotDisplayWarningIfTestsAreExecuted() mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); mockTestRunRequest.Setup(tr => tr.ExecuteAsync()).Returns(1).Raises(tr => tr.OnRunCompletion += null, - new TestRunCompleteEventArgs(testRunStats, false, false, null, null, new TimeSpan())); + new TestRunCompleteEventArgs(testRunStats, false, false, null, null, null, new TimeSpan())); mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockTestRunRequest.Object); mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(mockDiscoveryRequest.Object); diff --git a/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs index f838cd183f..4a58431e54 100644 --- a/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs @@ -249,7 +249,7 @@ private ArgumentProcessorResult RunRunArgumentProcessorExecuteWithMockSetup(ITes list.Add(new TestCase("Test2", new Uri("http://FooTestUri2"), "Source2")); var mockTestRunStats = new Mock(); - var args = new TestRunCompleteEventArgs(mockTestRunStats.Object, false, false, null, null, new TimeSpan()); + var args = new TestRunCompleteEventArgs(mockTestRunStats.Object, false, false, null, null, null, new TimeSpan()); mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Returns(testRunRequest); diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index d8440633b5..0ab9bad45b 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -3,107 +3,107 @@ namespace vstest.console.UnitTests.TestPlatformHelpers { - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - - using Microsoft.VisualStudio.TestPlatform.Client.RequestHelper; - using Microsoft.VisualStudio.TestPlatform.CommandLine; - using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; - using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; - using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; - using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; - using Microsoft.VisualStudio.TestPlatform.Common.Logging; - using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; - using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; - using Microsoft.VisualStudio.TestPlatform.ObjectModel; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - using System.Runtime.Versioning; - using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; - - using Moq; - - using vstest.console.UnitTests.TestDoubles; - using Microsoft.VisualStudio.TestPlatform.Utilities; - using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + + using Microsoft.VisualStudio.TestPlatform.Client.RequestHelper; + using Microsoft.VisualStudio.TestPlatform.CommandLine; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; + using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using Microsoft.VisualStudio.TestPlatform.Common.Logging; + using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; + using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.Runtime.Versioning; + using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; + + using Moq; + + using vstest.console.UnitTests.TestDoubles; + using Microsoft.VisualStudio.TestPlatform.Utilities; + using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; [TestClass] - public class TestRequestManagerTests - { - private DummyLoggerEvents mockLoggerEvents; - private CommandLineOptions commandLineOptions; - private Mock mockTestPlatform; - private Mock mockOutput; - private Mock mockDiscoveryRequest; - private Mock mockRunRequest; - private Mock mockAssemblyMetadataProvider; - private InferHelper inferHelper; - private ITestRequestManager testRequestManager; - private Mock mockTestPlatformEventSource; - private ProtocolConfig protocolConfig; - private Task mockMetricsPublisherTask; - private Mock mockMetricsPublisher; - private Mock mockProcessHelper; - private Mock mockAttachmentsProcessingManager; - - private const string DefaultRunsettings = @" + public class TestRequestManagerTests + { + private DummyLoggerEvents mockLoggerEvents; + private CommandLineOptions commandLineOptions; + private Mock mockTestPlatform; + private Mock mockOutput; + private Mock mockDiscoveryRequest; + private Mock mockRunRequest; + private Mock mockAssemblyMetadataProvider; + private InferHelper inferHelper; + private ITestRequestManager testRequestManager; + private Mock mockTestPlatformEventSource; + private ProtocolConfig protocolConfig; + private Task mockMetricsPublisherTask; + private Mock mockMetricsPublisher; + private Mock mockProcessHelper; + private Mock mockAttachmentsProcessingManager; + + private const string DefaultRunsettings = @" "; - public TestRequestManagerTests() - { - this.mockLoggerEvents = new DummyLoggerEvents(TestSessionMessageLogger.Instance); - this.commandLineOptions = new DummyCommandLineOptions(); - this.mockOutput = new Mock(); - this.mockTestPlatform = new Mock(); - this.mockDiscoveryRequest = new Mock(); - this.mockRunRequest = new Mock(); - this.mockTestPlatformEventSource = new Mock(); - this.protocolConfig = new ProtocolConfig(); - this.mockAssemblyMetadataProvider = new Mock(); - this.inferHelper = new InferHelper(this.mockAssemblyMetadataProvider.Object); - var testRunResultAggregator = new DummyTestRunResultAggregator(); - this.mockProcessHelper = new Mock(); - - this.mockMetricsPublisher = new Mock(); - this.mockMetricsPublisherTask = Task.FromResult(this.mockMetricsPublisher.Object); - this.mockAttachmentsProcessingManager = new Mock(); - this.testRequestManager = new TestRequestManager( - this.commandLineOptions, - this.mockTestPlatform.Object, - testRunResultAggregator, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - this.mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(this.mockDiscoveryRequest.Object); - this.mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(this.mockRunRequest.Object); - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.X86); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework40)); - this.mockProcessHelper.Setup(x => x.GetCurrentProcessId()).Returns(1234); - this.mockProcessHelper.Setup(x => x.GetProcessName(It.IsAny())).Returns("dotnet.exe"); - } - - [TestCleanup] - public void Cleanup() - { - CommandLineOptions.Instance.Reset(); + public TestRequestManagerTests() + { + this.mockLoggerEvents = new DummyLoggerEvents(TestSessionMessageLogger.Instance); + this.commandLineOptions = new DummyCommandLineOptions(); + this.mockOutput = new Mock(); + this.mockTestPlatform = new Mock(); + this.mockDiscoveryRequest = new Mock(); + this.mockRunRequest = new Mock(); + this.mockTestPlatformEventSource = new Mock(); + this.protocolConfig = new ProtocolConfig(); + this.mockAssemblyMetadataProvider = new Mock(); + this.inferHelper = new InferHelper(this.mockAssemblyMetadataProvider.Object); + var testRunResultAggregator = new DummyTestRunResultAggregator(); + this.mockProcessHelper = new Mock(); + + this.mockMetricsPublisher = new Mock(); + this.mockMetricsPublisherTask = Task.FromResult(this.mockMetricsPublisher.Object); + this.mockAttachmentsProcessingManager = new Mock(); + this.testRequestManager = new TestRequestManager( + this.commandLineOptions, + this.mockTestPlatform.Object, + testRunResultAggregator, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + this.mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(this.mockDiscoveryRequest.Object); + this.mockTestPlatform.Setup(tp => tp.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(this.mockRunRequest.Object); + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.X86); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework40)); + this.mockProcessHelper.Setup(x => x.GetCurrentProcessId()).Returns(1234); + this.mockProcessHelper.Setup(x => x.GetProcessName(It.IsAny())).Returns("dotnet.exe"); + } + + [TestCleanup] + public void Cleanup() + { + CommandLineOptions.Instance.Reset(); // Opt out the Telemetry Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "0"); @@ -121,159 +121,159 @@ public void TestRequestManagerShouldNotInitializeConsoleLoggerIfDesignModeIsSet( this.inferHelper, this.mockMetricsPublisherTask, this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - Assert.IsFalse(this.mockLoggerEvents.EventsSubscribed()); - } - - [TestMethod] - public void InitializeExtensionsShouldCallTestPlatformToClearAndUpdateExtensions() - { - var paths = new List() { "a", "b" }; - this.testRequestManager.InitializeExtensions(paths, false); - - this.mockTestPlatform.Verify(mt => mt.ClearExtensions(), Times.Once); - this.mockTestPlatform.Verify(mt => mt.UpdateExtensions(paths, false), Times.Once); - } - - [TestMethod] - public void ResetShouldResetCommandLineOptionsInstance() - { - var oldInstance = CommandLineOptions.Instance; - this.testRequestManager.ResetOptions(); - - var newInstance = CommandLineOptions.Instance; - - Assert.AreNotEqual(oldInstance, newInstance, "CommandLineOptions must be cleaned up"); - } - - [TestMethod] - public void DiscoverTestsShouldReadTheBatchSizeFromSettingsAndSetItForDiscoveryCriteria() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = - @" + this.mockAttachmentsProcessingManager.Object); + + Assert.IsFalse(this.mockLoggerEvents.EventsSubscribed()); + } + + [TestMethod] + public void InitializeExtensionsShouldCallTestPlatformToClearAndUpdateExtensions() + { + var paths = new List() { "a", "b" }; + this.testRequestManager.InitializeExtensions(paths, false); + + this.mockTestPlatform.Verify(mt => mt.ClearExtensions(), Times.Once); + this.mockTestPlatform.Verify(mt => mt.UpdateExtensions(paths, false), Times.Once); + } + + [TestMethod] + public void ResetShouldResetCommandLineOptionsInstance() + { + var oldInstance = CommandLineOptions.Instance; + this.testRequestManager.ResetOptions(); + + var newInstance = CommandLineOptions.Instance; + + Assert.AreNotEqual(oldInstance, newInstance, "CommandLineOptions must be cleaned up"); + } + + [TestMethod] + public void DiscoverTestsShouldReadTheBatchSizeFromSettingsAndSetItForDiscoveryCriteria() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = + @" 15 " - }; - - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); - Assert.AreEqual(15, actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent); - } - - [TestMethod] - public void DiscoverTestsShouldCallTestPlatformAndSucceed() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = DefaultRunsettings - }; - - var createDiscoveryRequestCalled = 0; - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - createDiscoveryRequestCalled++; - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - var mockDiscoveryRegistrar = new Mock(); - - string testCaseFilterValue = "TestFilter"; - CommandLineOptions.Instance.TestCaseFilterValue = testCaseFilterValue; - this.testRequestManager = new TestRequestManager(CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, this.protocolConfig); - - Assert.AreEqual(testCaseFilterValue, actualDiscoveryCriteria.TestCaseFilter, "TestCaseFilter must be set"); + }; + + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); + Assert.AreEqual(15, actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent); + } + + [TestMethod] + public void DiscoverTestsShouldCallTestPlatformAndSucceed() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = DefaultRunsettings + }; + + var createDiscoveryRequestCalled = 0; + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + createDiscoveryRequestCalled++; + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + var mockDiscoveryRegistrar = new Mock(); + + string testCaseFilterValue = "TestFilter"; + CommandLineOptions.Instance.TestCaseFilterValue = testCaseFilterValue; + this.testRequestManager = new TestRequestManager(CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, this.protocolConfig); + + Assert.AreEqual(testCaseFilterValue, actualDiscoveryCriteria.TestCaseFilter, "TestCaseFilter must be set"); Assert.AreEqual(1, createDiscoveryRequestCalled, "CreateDiscoveryRequest must be invoked only once."); Assert.AreEqual(2, actualDiscoveryCriteria.Sources.Count(), "All Sources must be used for discovery request"); Assert.AreEqual("a", actualDiscoveryCriteria.Sources.First(), "First Source in list is incorrect"); Assert.AreEqual("b", actualDiscoveryCriteria.Sources.ElementAt(1), "Second Source in list is incorrect"); - // Default frequency is set to 10, unless specified in runsettings. - Assert.AreEqual(10, actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent); - - mockDiscoveryRegistrar.Verify(md => md.RegisterDiscoveryEvents(It.IsAny()), Times.Once); - mockDiscoveryRegistrar.Verify(md => md.UnregisterDiscoveryEvents(It.IsAny()), Times.Once); - - mockDiscoveryRequest.Verify(md => md.DiscoverAsync(), Times.Once); - - mockTestPlatformEventSource.Verify(mt => mt.DiscoveryRequestStart(), Times.Once); - mockTestPlatformEventSource.Verify(mt => mt.DiscoveryRequestStop(), Times.Once); - } - - [TestMethod] - public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = DefaultRunsettings - }; - - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - - var mockDiscoveryRegistrar = new Mock(); - - string testCaseFilterValue = "TestFilter"; - CommandLineOptions.Instance.TestCaseFilterValue = testCaseFilterValue; - this.testRequestManager = new TestRequestManager(CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); - - // Verify. - Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); - } - - - [TestMethod] - public void DiscoverTestsShouldCollectMetrics() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll", "b.dll" }, - RunSettings = @" + // Default frequency is set to 10, unless specified in runsettings. + Assert.AreEqual(10, actualDiscoveryCriteria.FrequencyOfDiscoveredTestsEvent); + + mockDiscoveryRegistrar.Verify(md => md.RegisterDiscoveryEvents(It.IsAny()), Times.Once); + mockDiscoveryRegistrar.Verify(md => md.UnregisterDiscoveryEvents(It.IsAny()), Times.Once); + + mockDiscoveryRequest.Verify(md => md.DiscoverAsync(), Times.Once); + + mockTestPlatformEventSource.Verify(mt => mt.DiscoveryRequestStart(), Times.Once); + mockTestPlatformEventSource.Verify(mt => mt.DiscoveryRequestStop(), Times.Once); + } + + [TestMethod] + public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = DefaultRunsettings + }; + + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + + var mockDiscoveryRegistrar = new Mock(); + + string testCaseFilterValue = "TestFilter"; + CommandLineOptions.Instance.TestCaseFilterValue = testCaseFilterValue; + this.testRequestManager = new TestRequestManager(CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + + // Verify. + Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); + } + + + [TestMethod] + public void DiscoverTestsShouldCollectMetrics() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll", "b.dll" }, + RunSettings = @" 2 x86 @@ -284,706 +284,706 @@ public void DiscoverTestsShouldCollectMetrics() 169.254.193.190 " - }; + }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); // Verify. - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out var targetDevice)); - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.MaxCPUcount, out var maxcount)); - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetPlatform, out var targetPlatform)); - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.DisableAppDomain, out var disableAppDomain)); - Assert.AreEqual("Other", targetDevice); - Assert.AreEqual(2, maxcount); - Assert.AreEqual("X86", targetPlatform.ToString()); - Assert.AreEqual(true, disableAppDomain); - } - - [TestMethod] - public void DiscoverTestsShouldCollectTargetDeviceLocalMachineIfTargetDeviceStringisEmpty() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = @" + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out var targetDevice)); + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.MaxCPUcount, out var maxcount)); + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetPlatform, out var targetPlatform)); + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.DisableAppDomain, out var disableAppDomain)); + Assert.AreEqual("Other", targetDevice); + Assert.AreEqual(2, maxcount); + Assert.AreEqual("X86", targetPlatform.ToString()); + Assert.AreEqual(true, disableAppDomain); + } + + [TestMethod] + public void DiscoverTestsShouldCollectTargetDeviceLocalMachineIfTargetDeviceStringisEmpty() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = @" " - }; - - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); - - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); - - // Verify. - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out var targetDevice)); - Assert.AreEqual("Local Machine", targetDevice); - } - - [TestMethod] - public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsDevice() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = @" + }; + + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); + + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + + // Verify. + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out var targetDevice)); + Assert.AreEqual("Local Machine", targetDevice); + } + + [TestMethod] + public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsDevice() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = @" Device " - }; - - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); - - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); - - // Verify. - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out object targetDevice)); - Assert.AreEqual("Device", targetDevice); - } - - [TestMethod] - public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsEmulator() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = @" + }; + + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); + + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + + // Verify. + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out object targetDevice)); + Assert.AreEqual("Device", targetDevice); + } + + [TestMethod] + public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsEmulator() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = @" Emulator 8.1 U1 WVGA 4 inch 512MB " - }; - - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); - - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); - - // Verify. - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out var targetDevice)); - Assert.AreEqual("Emulator 8.1 U1 WVGA 4 inch 512MB", targetDevice); - } - - [TestMethod] - public void DiscoverTestsShouldCollectCommands() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = @" + }; + + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); + + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + + // Verify. + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out var targetDevice)); + Assert.AreEqual("Emulator 8.1 U1 WVGA 4 inch 512MB", targetDevice); + } + + [TestMethod] + public void DiscoverTestsShouldCollectCommands() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = @" Device " - }; - - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); - - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - CommandLineOptions.Instance.Parallel = true; - CommandLineOptions.Instance.EnableCodeCoverage = true; - CommandLineOptions.Instance.InIsolation = true; - CommandLineOptions.Instance.UseVsixExtensions = true; - CommandLineOptions.Instance.SettingsFile = @"c://temp/.runsettings"; - - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); - - // Verify - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); - - var commandLineArray = commandLineSwitches.ToString(); - - Assert.IsTrue(commandLineArray.Contains("/Parallel")); - Assert.IsTrue(commandLineArray.Contains("/EnableCodeCoverage")); - Assert.IsTrue(commandLineArray.Contains("/InIsolation")); - Assert.IsTrue(commandLineArray.Contains("/UseVsixExtensions")); - Assert.IsTrue(commandLineArray.Contains("/settings//.RunSettings")); - } - - [TestMethod] - public void DiscoverTestsShouldCollectTestSettings() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = @" + }; + + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); + + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + CommandLineOptions.Instance.Parallel = true; + CommandLineOptions.Instance.EnableCodeCoverage = true; + CommandLineOptions.Instance.InIsolation = true; + CommandLineOptions.Instance.UseVsixExtensions = true; + CommandLineOptions.Instance.SettingsFile = @"c://temp/.runsettings"; + + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + + // Verify + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); + + var commandLineArray = commandLineSwitches.ToString(); + + Assert.IsTrue(commandLineArray.Contains("/Parallel")); + Assert.IsTrue(commandLineArray.Contains("/EnableCodeCoverage")); + Assert.IsTrue(commandLineArray.Contains("/InIsolation")); + Assert.IsTrue(commandLineArray.Contains("/UseVsixExtensions")); + Assert.IsTrue(commandLineArray.Contains("/settings//.RunSettings")); + } + + [TestMethod] + public void DiscoverTestsShouldCollectTestSettings() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = @" Device " - }; + }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); - CommandLineOptions.Instance.SettingsFile = @"c://temp/.testsettings"; + CommandLineOptions.Instance.SettingsFile = @"c://temp/.testsettings"; - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); // Verify Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); var commandLineArray = commandLineSwitches.ToString(); - Assert.IsTrue(commandLineArray.Contains("/settings//.TestSettings")); - } + Assert.IsTrue(commandLineArray.Contains("/settings//.TestSettings")); + } - [TestMethod] - public void DiscoverTestsShouldCollectVsmdiFile() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + [TestMethod] + public void DiscoverTestsShouldCollectVsmdiFile() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = @" + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = @" Device " - }; + }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); - CommandLineOptions.Instance.SettingsFile = @"c://temp/.vsmdi"; + CommandLineOptions.Instance.SettingsFile = @"c://temp/.vsmdi"; - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); // Verify Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); var commandLineArray = commandLineSwitches.ToString(); - Assert.IsTrue(commandLineArray.Contains("/settings//.vsmdi")); - } + Assert.IsTrue(commandLineArray.Contains("/settings//.vsmdi")); + } - [TestMethod] - public void DiscoverTestsShouldCollectTestRunConfigFile() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + [TestMethod] + public void DiscoverTestsShouldCollectTestRunConfigFile() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = @" + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = @" Device " - }; - - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - var mockDiscoveryRegistrar = new Mock(); - - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - CommandLineOptions.Instance.SettingsFile = @"c://temp/.testrunConfig"; - - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); - - // Verify - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); - - var commandLineArray = commandLineSwitches.ToString(); - - Assert.IsTrue(commandLineArray.Contains("/settings//.testrunConfig")); - } - - [TestMethod] - public void DiscoverTestsShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignMode() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockDiscoveryRegistrar = new Mock(); + + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => { actualRequestData = requestData; }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + CommandLineOptions.Instance.SettingsFile = @"c://temp/.testrunConfig"; + + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + + // Verify + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); + + var commandLineArray = commandLineSwitches.ToString(); + + Assert.IsTrue(commandLineArray.Contains("/settings//.testrunConfig")); + } + + [TestMethod] + public void DiscoverTestsShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignMode() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - this.commandLineOptions.IsDesignMode = true; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); - - Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); - } - - [TestMethod] - public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignMode() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = - $@" + }; + this.commandLineOptions.IsDesignMode = true; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); + + Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); + } + + [TestMethod] + public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignMode() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = + $@" {Constants.DotNetFramework46} {Architecture.ARM} " - }; - this.commandLineOptions.IsDesignMode = true; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.X86); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework451)); - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); - - Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); - } - - [TestMethod] - public void DiscoverTestsShouldUpdateFrameworkAndPlatformInCommandLineScenariosIfNotSpecified() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + this.commandLineOptions.IsDesignMode = true; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.X86); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework451)); + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Never); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Never); + + Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); + } + + [TestMethod] + public void DiscoverTestsShouldUpdateFrameworkAndPlatformInCommandLineScenariosIfNotSpecified() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - this.commandLineOptions.IsDesignMode = false; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); - - Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); - } - - [TestMethod] - public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformInCommandLineScenariosIfSpecifiedButInferred() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + this.commandLineOptions.IsDesignMode = false; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, new Mock().Object, this.protocolConfig); + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); + + Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); + } + + [TestMethod] + public void DiscoverTestsShouldNotUpdateFrameworkAndPlatformInCommandLineScenariosIfSpecifiedButInferred() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - this.commandLineOptions.IsDesignMode = false; - this.commandLineOptions.TargetFrameworkVersion = Framework.DefaultFramework; - this.commandLineOptions.TargetArchitecture = Architecture.X86; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, - new Mock().Object, this.protocolConfig); - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - - Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); - } - - [TestMethod] - public void DiscoverTestsShouldPublishMetrics() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a", "b" } - }; - var mockProtocolConfig = new ProtocolConfig { Version = 2 }; - var mockDiscoveryRegistrar = new Mock(); - - // Act - this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); - - // Verify. - this.mockMetricsPublisher.Verify(mp => mp.PublishMetrics(TelemetryDataConstants.TestDiscoveryCompleteEvent, It.IsAny>()), Times.Once); - } - - [TestMethod] - public void CancelShouldNotThrowExceptionIfTestRunRequestHasBeenDisposed() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = DefaultRunsettings - }; - - var mockRunEventsRegistrar = new Mock(); - var mockCustomlauncher = new Mock(); - - this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); - this.testRequestManager.CancelTestRun(); - } - - [TestMethod] - public void AbortShouldNotThrowExceptionIfTestRunRequestHasBeenDisposed() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = DefaultRunsettings - }; - - var mockRunEventsRegistrar = new Mock(); - var mockCustomlauncher = new Mock(); - - this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); - this.testRequestManager.AbortTestRun(); - } - - [TestMethod] - public void RunTestsShouldReadTheBatchSizeFromSettingsAndSetItForTestRunCriteria() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = - @" + }; + this.commandLineOptions.IsDesignMode = false; + this.commandLineOptions.TargetFrameworkVersion = Framework.DefaultFramework; + this.commandLineOptions.TargetArchitecture = Architecture.X86; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, + new Mock().Object, this.protocolConfig); + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + + Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains(nameof(Architecture.ARM))); + } + + [TestMethod] + public void DiscoverTestsShouldPublishMetrics() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a", "b" } + }; + var mockProtocolConfig = new ProtocolConfig { Version = 2 }; + var mockDiscoveryRegistrar = new Mock(); + + // Act + this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); + + // Verify. + this.mockMetricsPublisher.Verify(mp => mp.PublishMetrics(TelemetryDataConstants.TestDiscoveryCompleteEvent, It.IsAny>()), Times.Once); + } + + [TestMethod] + public void CancelShouldNotThrowExceptionIfTestRunRequestHasBeenDisposed() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = DefaultRunsettings + }; + + var mockRunEventsRegistrar = new Mock(); + var mockCustomlauncher = new Mock(); + + this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); + this.testRequestManager.CancelTestRun(); + } + + [TestMethod] + public void AbortShouldNotThrowExceptionIfTestRunRequestHasBeenDisposed() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = DefaultRunsettings + }; + + var mockRunEventsRegistrar = new Mock(); + var mockCustomlauncher = new Mock(); + + this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); + this.testRequestManager.AbortTestRun(); + } + + [TestMethod] + public void RunTestsShouldReadTheBatchSizeFromSettingsAndSetItForTestRunCriteria() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = + @" 15 " - }; - - TestRunCriteria actualTestRunCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - Assert.AreEqual(15, actualTestRunCriteria.FrequencyOfRunStatsChangeEvent); - } - - [TestMethod] - public void RunTestsShouldNotThrowForFramework35() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + TestRunCriteria actualTestRunCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + Assert.AreEqual(15, actualTestRunCriteria.FrequencyOfRunStatsChangeEvent); + } + + [TestMethod] + public void RunTestsShouldNotThrowForFramework35() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" Framework35 " - }; - - TestRunCriteria actualTestRunCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockDiscoveryRequest.Object); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Returns(new FrameworkName(Constants.DotNetFramework35)); - - var mockRunEventsRegistrar = new Mock(); - var mockCustomlauncher = new Mock(); - - this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); - - mockRunEventsRegistrar.Verify(lw => lw.LogWarning("Framework35 is not supported. For projects targeting .Net Framework 3.5, test will run in CLR 4.0 \"compatibility mode\"."), Times.Once); - mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStart(), Times.Once); - mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStop(), Times.Once); - } - - [TestMethod] - public void RunTestsShouldPassSameProtocolConfigInRequestData() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = DefaultRunsettings - }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualRequestData = requestData; - }).Returns(mockDiscoveryRequest.Object); - - // Act. - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); - - // Verify. - Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); - } - - [TestMethod] - public void RunTestsShouldCollectCommands() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = DefaultRunsettings - }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualRequestData = requestData; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - CommandLineOptions.Instance.Parallel = true; - CommandLineOptions.Instance.EnableCodeCoverage = true; - CommandLineOptions.Instance.InIsolation = true; - CommandLineOptions.Instance.UseVsixExtensions = true; - CommandLineOptions.Instance.SettingsFile = @"c://temp/.runsettings"; - - // Act. - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); - - // Verify - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); - - var commandLineArray = commandLineSwitches.ToString(); - - Assert.IsTrue(commandLineArray.Contains("/Parallel")); - Assert.IsTrue(commandLineArray.Contains("/EnableCodeCoverage")); - Assert.IsTrue(commandLineArray.Contains("/InIsolation")); - Assert.IsTrue(commandLineArray.Contains("/UseVsixExtensions")); - Assert.IsTrue(commandLineArray.Contains("/settings//.RunSettings")); - } - - [TestMethod] - public void RunTestsShouldCollectTelemetryForLegacySettings() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = @" + }; + + TestRunCriteria actualTestRunCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockDiscoveryRequest.Object); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Returns(new FrameworkName(Constants.DotNetFramework35)); + + var mockRunEventsRegistrar = new Mock(); + var mockCustomlauncher = new Mock(); + + this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); + + mockRunEventsRegistrar.Verify(lw => lw.LogWarning("Framework35 is not supported. For projects targeting .Net Framework 3.5, test will run in CLR 4.0 \"compatibility mode\"."), Times.Once); + mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStart(), Times.Once); + mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStop(), Times.Once); + } + + [TestMethod] + public void RunTestsShouldPassSameProtocolConfigInRequestData() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = DefaultRunsettings + }; + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualRequestData = requestData; + }).Returns(mockDiscoveryRequest.Object); + + // Act. + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); + + // Verify. + Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); + } + + [TestMethod] + public void RunTestsShouldCollectCommands() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = DefaultRunsettings + }; + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualRequestData = requestData; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + CommandLineOptions.Instance.Parallel = true; + CommandLineOptions.Instance.EnableCodeCoverage = true; + CommandLineOptions.Instance.InIsolation = true; + CommandLineOptions.Instance.UseVsixExtensions = true; + CommandLineOptions.Instance.SettingsFile = @"c://temp/.runsettings"; + + // Act. + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); + + // Verify + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.CommandLineSwitches, out var commandLineSwitches)); + + var commandLineArray = commandLineSwitches.ToString(); + + Assert.IsTrue(commandLineArray.Contains("/Parallel")); + Assert.IsTrue(commandLineArray.Contains("/EnableCodeCoverage")); + Assert.IsTrue(commandLineArray.Contains("/InIsolation")); + Assert.IsTrue(commandLineArray.Contains("/UseVsixExtensions")); + Assert.IsTrue(commandLineArray.Contains("/settings//.RunSettings")); + } + + [TestMethod] + public void RunTestsShouldCollectTelemetryForLegacySettings() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = @" @@ -1000,15 +1000,15 @@ public void RunTestsShouldCollectTelemetryForLegacySettings() " - }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualRequestData = requestData; - }).Returns(mockDiscoveryRequest.Object); + }; + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualRequestData = requestData; + }).Returns(mockDiscoveryRequest.Object); this.testRequestManager = new TestRequestManager( CommandLineOptions.Instance, @@ -1018,76 +1018,76 @@ public void RunTestsShouldCollectTelemetryForLegacySettings() this.inferHelper, this.mockMetricsPublisherTask, this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); + this.mockAttachmentsProcessingManager.Object); // Act. this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); - // Verify - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue("VS.TestRun.LegacySettings.Elements", out var legacySettingsNodes)); - StringAssert.Equals("Deployment, Scripts, Execution, AssemblyResolution, Timeouts, Hosts", legacySettingsNodes); - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue("VS.TestRun.LegacySettings.DeploymentAttributes", out var deploymentAttributes)); - StringAssert.Equals("enabled, deploySatelliteAssemblies", deploymentAttributes); - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue("VS.TestRun.LegacySettings.ExecutionAttributes", out var executionAttributes)); - StringAssert.Equals("hostProcessPlatform, parallelTestCount", executionAttributes); - - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TestSettingsUsed, out var testSettingsUsed)); - Assert.IsFalse((bool)testSettingsUsed); - } - - [TestMethod] - public void RunTestsShouldCollectTelemetryForTestSettingsEmbeddedInsideRunSettings() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = @" + // Verify + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue("VS.TestRun.LegacySettings.Elements", out var legacySettingsNodes)); + StringAssert.Equals("Deployment, Scripts, Execution, AssemblyResolution, Timeouts, Hosts", legacySettingsNodes); + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue("VS.TestRun.LegacySettings.DeploymentAttributes", out var deploymentAttributes)); + StringAssert.Equals("enabled, deploySatelliteAssemblies", deploymentAttributes); + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue("VS.TestRun.LegacySettings.ExecutionAttributes", out var executionAttributes)); + StringAssert.Equals("hostProcessPlatform, parallelTestCount", executionAttributes); + + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TestSettingsUsed, out var testSettingsUsed)); + Assert.IsFalse((bool)testSettingsUsed); + } + + [TestMethod] + public void RunTestsShouldCollectTelemetryForTestSettingsEmbeddedInsideRunSettings() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = @" true ..\..\Foo.testsettings " - }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualRequestData = requestData; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - // Act. - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); - - // Verify - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TestSettingsUsed, out var testSettingsUsed)); - Assert.IsTrue((bool)testSettingsUsed); - } - - [TestMethod] - public void RunTestsShouldCollectMetrics() - { - // Opt in the Telemetry - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = @" + }; + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualRequestData = requestData; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + // Act. + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); + + // Verify + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TestSettingsUsed, out var testSettingsUsed)); + Assert.IsTrue((bool)testSettingsUsed); + } + + [TestMethod] + public void RunTestsShouldCollectMetrics() + { + // Opt in the Telemetry + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = @" 2 x86 @@ -1098,548 +1098,548 @@ public void RunTestsShouldCollectMetrics() 169.254.193.190 " - }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; - IRequestData actualRequestData = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualRequestData = requestData; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager = new TestRequestManager( - CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - // Act. - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); + }; + var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + IRequestData actualRequestData = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualRequestData = requestData; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager = new TestRequestManager( + CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + // Act. + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); // Verify Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetDevice, out var targetDevice)); Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.MaxCPUcount, out var maxcount)); - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetPlatform, out var targetPlatform)); - Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.DisableAppDomain, out var disableAppDomain)); - Assert.AreEqual("Other", targetDevice); - Assert.AreEqual(2, maxcount); - Assert.AreEqual("X86", targetPlatform.ToString()); - Assert.AreEqual(true, disableAppDomain); - } - - [TestMethod] - public void RunTestsWithSourcesShouldCallTestPlatformAndSucceed() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = DefaultRunsettings - }; - - var createRunRequestCalled = 0; - TestRunCriteria observedCriteria = null; - var mockRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - createRunRequestCalled++; - observedCriteria = runCriteria; - }).Returns(mockRunRequest.Object); - - var mockRunEventsRegistrar = new Mock(); - var mockCustomlauncher = new Mock(); - - string testCaseFilterValue = "TestFilter"; - payload.TestPlatformOptions = new TestPlatformOptions { TestCaseFilter = testCaseFilterValue }; - this.testRequestManager = new TestRequestManager(CommandLineOptions.Instance, - this.mockTestPlatform.Object, - TestRunResultAggregator.Instance, - this.mockTestPlatformEventSource.Object, - this.inferHelper, - this.mockMetricsPublisherTask, - this.mockProcessHelper.Object, - this.mockAttachmentsProcessingManager.Object); - - this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); - - Assert.AreEqual(testCaseFilterValue, observedCriteria.TestCaseFilter, "TestCaseFilter must be set"); + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.TargetPlatform, out var targetPlatform)); + Assert.IsTrue(actualRequestData.MetricsCollection.Metrics.TryGetValue(TelemetryDataConstants.DisableAppDomain, out var disableAppDomain)); + Assert.AreEqual("Other", targetDevice); + Assert.AreEqual(2, maxcount); + Assert.AreEqual("X86", targetPlatform.ToString()); + Assert.AreEqual(true, disableAppDomain); + } + + [TestMethod] + public void RunTestsWithSourcesShouldCallTestPlatformAndSucceed() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = DefaultRunsettings + }; + + var createRunRequestCalled = 0; + TestRunCriteria observedCriteria = null; + var mockRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + createRunRequestCalled++; + observedCriteria = runCriteria; + }).Returns(mockRunRequest.Object); + + var mockRunEventsRegistrar = new Mock(); + var mockCustomlauncher = new Mock(); + + string testCaseFilterValue = "TestFilter"; + payload.TestPlatformOptions = new TestPlatformOptions { TestCaseFilter = testCaseFilterValue }; + this.testRequestManager = new TestRequestManager(CommandLineOptions.Instance, + this.mockTestPlatform.Object, + TestRunResultAggregator.Instance, + this.mockTestPlatformEventSource.Object, + this.inferHelper, + this.mockMetricsPublisherTask, + this.mockProcessHelper.Object, + this.mockAttachmentsProcessingManager.Object); + + this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); + + Assert.AreEqual(testCaseFilterValue, observedCriteria.TestCaseFilter, "TestCaseFilter must be set"); Assert.AreEqual(1, createRunRequestCalled, "CreateRunRequest must be invoked only once."); Assert.AreEqual(2, observedCriteria.Sources.Count(), "All Sources must be used for discovery request"); Assert.AreEqual("a", observedCriteria.Sources.First(), "First Source in list is incorrect"); Assert.AreEqual("b", observedCriteria.Sources.ElementAt(1), "Second Source in list is incorrect"); - // Check for the default value for the frequency - Assert.AreEqual(10, observedCriteria.FrequencyOfRunStatsChangeEvent); - mockRunEventsRegistrar.Verify(md => md.RegisterTestRunEvents(It.IsAny()), Times.Once); - mockRunEventsRegistrar.Verify(md => md.UnregisterTestRunEvents(It.IsAny()), Times.Once); - - mockRunRequest.Verify(md => md.ExecuteAsync(), Times.Once); - - mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStart(), Times.Once); - mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStop(), Times.Once); - } - - [TestMethod] - public void RunTestsMultipleCallsShouldNotRunInParallel() - { - var payload1 = new TestRunRequestPayload() - { - Sources = new List() { "a" }, - RunSettings = DefaultRunsettings - }; - - var payload2 = new TestRunRequestPayload() - { - Sources = new List() { "b" }, - RunSettings = DefaultRunsettings - }; - - var mockRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(mockRunRequest.Object); - - var mockRunEventsRegistrar1 = new Mock(); - var mockRunEventsRegistrar2 = new Mock(); - - // Setup the second one to wait - var sw = new Stopwatch(); - sw.Start(); - - long run1Start = 0; - long run1Stop = 0; - long run2Start = 0; - long run2Stop = 0; - mockRunEventsRegistrar1.Setup(md => md.RegisterTestRunEvents(It.IsAny())).Callback(() => - { - Thread.Sleep(10); - run1Start = sw.ElapsedMilliseconds; - Thread.Sleep(1); - }); - mockRunEventsRegistrar1.Setup(md => md.UnregisterTestRunEvents(It.IsAny())).Callback(() => - { - Thread.Sleep(10); - run1Stop = sw.ElapsedMilliseconds; - Thread.Sleep(10); - }); - - mockRunEventsRegistrar2.Setup(md => md.RegisterTestRunEvents(It.IsAny())).Callback(() => - { - Thread.Sleep(10); - run2Start = sw.ElapsedMilliseconds; - Thread.Sleep(10); - }); - mockRunEventsRegistrar2.Setup(md => md.UnregisterTestRunEvents(It.IsAny())).Callback(() => - { - Thread.Sleep(10); - run2Stop = sw.ElapsedMilliseconds; - }); - - var mockCustomlauncher = new Mock(); - var task1 = Task.Run(() => - { - this.testRequestManager.RunTests(payload1, mockCustomlauncher.Object, mockRunEventsRegistrar1.Object, this.protocolConfig); - }); - var task2 = Task.Run(() => - { - this.testRequestManager.RunTests(payload2, mockCustomlauncher.Object, mockRunEventsRegistrar2.Object, this.protocolConfig); - }); - - Task.WaitAll(task1, task2); - - if (run1Start < run2Start) - { - Assert.IsTrue((run2Stop > run2Start) - && (run2Start > run1Stop) - && (run1Stop > run1Start)); - } - else - { - Assert.IsTrue((run1Stop > run1Start) - && (run1Start > run2Stop) - && (run2Stop > run2Start)); - } - } - - [TestMethod] - public void RunTestsShouldPublishMetrics() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = DefaultRunsettings - }; - - var mockRunEventsRegistrar = new Mock(); - var mockCustomlauncher = new Mock(); - - this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); - - this.mockMetricsPublisher.Verify(mp => mp.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, It.IsAny>()), Times.Once); - } - - // TODO: add tests in design mode and executor that they are handling all the exceptions properly including printing inner exception. - - [TestMethod] - public void RunTestsIfThrowsTestPlatformExceptionShouldThrowOut() - { - Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new TestPlatformException("HelloWorld"))); - } - - [TestMethod] - public void RunTestsIfThrowsSettingsExceptionShouldThrowOut() - { - Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new SettingsException("HelloWorld"))); - } - - [TestMethod] - public void RunTestsIfThrowsInvalidOperationExceptionShouldThrowOut() - { - Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new InvalidOperationException("HelloWorld"))); - } - - [TestMethod] - public void RunTestsIfThrowsExceptionShouldThrowOut() - { - Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new NotImplementedException("HelloWorld"))); - } - - [TestMethod] - public void DiscoverTestsIfThrowsTestPlatformExceptionShouldThrowOut() - { - Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new TestPlatformException("HelloWorld"))); - } - - [TestMethod] - public void DiscoverTestsIfThrowsSettingsExceptionShouldThrowOut() - { - Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new SettingsException("HelloWorld"))); - } - - [TestMethod] - public void DiscoverTestsIfThrowsInvalidOperationExceptionShouldThrowOut() - { - Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new InvalidOperationException("HelloWorld"))); - } - - [TestMethod] - public void DiscoverTestsIfThrowsExceptionShouldThrowOut() - { - Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new NotImplementedException("HelloWorld"))); - } - - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public void DiscoverTestsShouldUpdateDesignModeAndCollectSourceInformation(bool designModeValue) - { - var runsettings = ".NETFramework,Version=v4.5"; - var discoveryPayload = CreateDiscoveryPayload(runsettings); - this.commandLineOptions.IsDesignMode = designModeValue; - - this.testRequestManager.DiscoverTests(discoveryPayload, new Mock().Object, this.protocolConfig); - - var designmode = $"{designModeValue}"; - this.mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny())); - - var collectSourceInformation = $"{designModeValue}"; - this.mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny())); - } - - [TestMethod] - public void DiscoverTestsShouldNotUpdateDesignModeIfUserHasSetDesignModeInRunSettings() - { - var runsettings = "False.NETFramework,Version=v4.5"; - var discoveryPayload = CreateDiscoveryPayload(runsettings); - this.commandLineOptions.IsDesignMode = true; - - this.testRequestManager.DiscoverTests(discoveryPayload, new Mock().Object, this.protocolConfig); - - var designmode = "False"; - this.mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny())); - } - - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public void RunTestsShouldUpdateDesignModeIfRunnerIsInDesignMode(bool designModeValue) - { - var runsettings = - ".NETFramework,Version=v4.5"; - var payload = new TestRunRequestPayload - { - RunSettings = runsettings, - Sources = new List { "c:\\testproject.dll" } - }; - this.commandLineOptions.IsDesignMode = designModeValue; - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - var designmode = $"{designModeValue}"; - this.mockTestPlatform.Verify(tp => tp.CreateTestRunRequest(It.IsAny(), It.Is(rc => rc.TestRunSettings.Contains(designmode)), It.IsAny())); - } - - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public void DiscoverTestsShouldNotUpdateCollectSourceInformationIfUserHasSetItInRunSettings(bool val) - { - var runsettings = $"{val}"; - var discoveryPayload = CreateDiscoveryPayload(runsettings); - - this.testRequestManager.DiscoverTests(discoveryPayload, new Mock().Object, this.protocolConfig); - - var collectSourceInformation = $"{val}"; - this.mockTestPlatform.Verify( - tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny())); - } - - [TestMethod] - public void RunTestsShouldShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignMode() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + // Check for the default value for the frequency + Assert.AreEqual(10, observedCriteria.FrequencyOfRunStatsChangeEvent); + mockRunEventsRegistrar.Verify(md => md.RegisterTestRunEvents(It.IsAny()), Times.Once); + mockRunEventsRegistrar.Verify(md => md.UnregisterTestRunEvents(It.IsAny()), Times.Once); + + mockRunRequest.Verify(md => md.ExecuteAsync(), Times.Once); + + mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStart(), Times.Once); + mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStop(), Times.Once); + } + + [TestMethod] + public void RunTestsMultipleCallsShouldNotRunInParallel() + { + var payload1 = new TestRunRequestPayload() + { + Sources = new List() { "a" }, + RunSettings = DefaultRunsettings + }; + + var payload2 = new TestRunRequestPayload() + { + Sources = new List() { "b" }, + RunSettings = DefaultRunsettings + }; + + var mockRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(mockRunRequest.Object); + + var mockRunEventsRegistrar1 = new Mock(); + var mockRunEventsRegistrar2 = new Mock(); + + // Setup the second one to wait + var sw = new Stopwatch(); + sw.Start(); + + long run1Start = 0; + long run1Stop = 0; + long run2Start = 0; + long run2Stop = 0; + mockRunEventsRegistrar1.Setup(md => md.RegisterTestRunEvents(It.IsAny())).Callback(() => + { + Thread.Sleep(10); + run1Start = sw.ElapsedMilliseconds; + Thread.Sleep(1); + }); + mockRunEventsRegistrar1.Setup(md => md.UnregisterTestRunEvents(It.IsAny())).Callback(() => + { + Thread.Sleep(10); + run1Stop = sw.ElapsedMilliseconds; + Thread.Sleep(10); + }); + + mockRunEventsRegistrar2.Setup(md => md.RegisterTestRunEvents(It.IsAny())).Callback(() => + { + Thread.Sleep(10); + run2Start = sw.ElapsedMilliseconds; + Thread.Sleep(10); + }); + mockRunEventsRegistrar2.Setup(md => md.UnregisterTestRunEvents(It.IsAny())).Callback(() => + { + Thread.Sleep(10); + run2Stop = sw.ElapsedMilliseconds; + }); + + var mockCustomlauncher = new Mock(); + var task1 = Task.Run(() => + { + this.testRequestManager.RunTests(payload1, mockCustomlauncher.Object, mockRunEventsRegistrar1.Object, this.protocolConfig); + }); + var task2 = Task.Run(() => + { + this.testRequestManager.RunTests(payload2, mockCustomlauncher.Object, mockRunEventsRegistrar2.Object, this.protocolConfig); + }); + + Task.WaitAll(task1, task2); + + if (run1Start < run2Start) + { + Assert.IsTrue((run2Stop > run2Start) + && (run2Start > run1Stop) + && (run1Stop > run1Start)); + } + else + { + Assert.IsTrue((run1Stop > run1Start) + && (run1Start > run2Stop) + && (run2Stop > run2Start)); + } + } + + [TestMethod] + public void RunTestsShouldPublishMetrics() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = DefaultRunsettings + }; + + var mockRunEventsRegistrar = new Mock(); + var mockCustomlauncher = new Mock(); + + this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); + + this.mockMetricsPublisher.Verify(mp => mp.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, It.IsAny>()), Times.Once); + } + + // TODO: add tests in design mode and executor that they are handling all the exceptions properly including printing inner exception. + + [TestMethod] + public void RunTestsIfThrowsTestPlatformExceptionShouldThrowOut() + { + Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new TestPlatformException("HelloWorld"))); + } + + [TestMethod] + public void RunTestsIfThrowsSettingsExceptionShouldThrowOut() + { + Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new SettingsException("HelloWorld"))); + } + + [TestMethod] + public void RunTestsIfThrowsInvalidOperationExceptionShouldThrowOut() + { + Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new InvalidOperationException("HelloWorld"))); + } + + [TestMethod] + public void RunTestsIfThrowsExceptionShouldThrowOut() + { + Assert.ThrowsException(() => RunTestsIfThrowsExceptionShouldThrowOut(new NotImplementedException("HelloWorld"))); + } + + [TestMethod] + public void DiscoverTestsIfThrowsTestPlatformExceptionShouldThrowOut() + { + Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new TestPlatformException("HelloWorld"))); + } + + [TestMethod] + public void DiscoverTestsIfThrowsSettingsExceptionShouldThrowOut() + { + Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new SettingsException("HelloWorld"))); + } + + [TestMethod] + public void DiscoverTestsIfThrowsInvalidOperationExceptionShouldThrowOut() + { + Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new InvalidOperationException("HelloWorld"))); + } + + [TestMethod] + public void DiscoverTestsIfThrowsExceptionShouldThrowOut() + { + Assert.ThrowsException(() => DiscoverTestsIfThrowsExceptionShouldThrowOut(new NotImplementedException("HelloWorld"))); + } + + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public void DiscoverTestsShouldUpdateDesignModeAndCollectSourceInformation(bool designModeValue) + { + var runsettings = ".NETFramework,Version=v4.5"; + var discoveryPayload = CreateDiscoveryPayload(runsettings); + this.commandLineOptions.IsDesignMode = designModeValue; + + this.testRequestManager.DiscoverTests(discoveryPayload, new Mock().Object, this.protocolConfig); + + var designmode = $"{designModeValue}"; + this.mockTestPlatform.Verify( + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny())); + + var collectSourceInformation = $"{designModeValue}"; + this.mockTestPlatform.Verify( + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny())); + } + + [TestMethod] + public void DiscoverTestsShouldNotUpdateDesignModeIfUserHasSetDesignModeInRunSettings() + { + var runsettings = "False.NETFramework,Version=v4.5"; + var discoveryPayload = CreateDiscoveryPayload(runsettings); + this.commandLineOptions.IsDesignMode = true; + + this.testRequestManager.DiscoverTests(discoveryPayload, new Mock().Object, this.protocolConfig); + + var designmode = "False"; + this.mockTestPlatform.Verify( + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(designmode)), It.IsAny())); + } + + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public void RunTestsShouldUpdateDesignModeIfRunnerIsInDesignMode(bool designModeValue) + { + var runsettings = + ".NETFramework,Version=v4.5"; + var payload = new TestRunRequestPayload + { + RunSettings = runsettings, + Sources = new List { "c:\\testproject.dll" } + }; + this.commandLineOptions.IsDesignMode = designModeValue; + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + var designmode = $"{designModeValue}"; + this.mockTestPlatform.Verify(tp => tp.CreateTestRunRequest(It.IsAny(), It.Is(rc => rc.TestRunSettings.Contains(designmode)), It.IsAny())); + } + + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public void DiscoverTestsShouldNotUpdateCollectSourceInformationIfUserHasSetItInRunSettings(bool val) + { + var runsettings = $"{val}"; + var discoveryPayload = CreateDiscoveryPayload(runsettings); + + this.testRequestManager.DiscoverTests(discoveryPayload, new Mock().Object, this.protocolConfig); + + var collectSourceInformation = $"{val}"; + this.mockTestPlatform.Verify( + tp => tp.CreateDiscoveryRequest(It.IsAny(), It.Is(dc => dc.RunSettings.Contains(collectSourceInformation)), It.IsAny())); + } + + [TestMethod] + public void RunTestsShouldShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignMode() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - - this.commandLineOptions.IsDesignMode = true; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); - - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); - - } - - [TestMethod] - public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignModeButInferred() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - $@" + }; + + this.commandLineOptions.IsDesignMode = true; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); + + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); + + } + + [TestMethod] + public void RunTestsShouldNotUpdateFrameworkAndPlatformIfSpecifiedInDesignModeButInferred() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + $@" {Constants.DotNetFramework46} {Architecture.ARM} " - }; - - this.commandLineOptions.IsDesignMode = true; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.X86); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework451)); - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); - } - - [TestMethod] - [DataRow("x86")] - [DataRow("X86")] - [DataRow("ARM")] - [DataRow("aRm")] - public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignModeButInferred(string targetPlatform) - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - $@" + }; + + this.commandLineOptions.IsDesignMode = true; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.X86); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework451)); + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); + } + + [TestMethod] + [DataRow("x86")] + [DataRow("X86")] + [DataRow("ARM")] + [DataRow("aRm")] + public void RunTestsShouldNotUpdatePlatformIfSpecifiedInDesignModeButInferred(string targetPlatform) + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + $@" {targetPlatform} " - }; - - this.commandLineOptions.IsDesignMode = true; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.X86); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework451)); - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(targetPlatform)); - } - - [TestMethod] - public void RunTestsShouldUpdateFrameworkAndPlatformInCommandLineScenarios() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = true; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.X86); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework451)); + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(targetPlatform)); + } + + [TestMethod] + public void RunTestsShouldUpdateFrameworkAndPlatformInCommandLineScenarios() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - - this.commandLineOptions.IsDesignMode = false; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); - - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); - } - - [TestMethod] - public void RunTestsShouldNotpdateFrameworkAndPlatformInCommandLineScenariosIfSpecifiedButInferred() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = false; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); + + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); + } + + [TestMethod] + public void RunTestsShouldNotpdateFrameworkAndPlatformInCommandLineScenariosIfSpecifiedButInferred() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - - this.commandLineOptions.IsDesignMode = false; - this.commandLineOptions.TargetArchitecture = Architecture.X86; - this.commandLineOptions.TargetFrameworkVersion = Framework.DefaultFramework; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); - - Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); - } - - [TestMethod] - public void RunTestsWithTestCasesShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignMode() - { - var actualSources = new List() { "1.dll", "2.dll" }; - var payload = new TestRunRequestPayload() - { - TestCases = new List() { - new TestCase(){Source = actualSources[0]}, - new TestCase() { Source = actualSources[0]}, - new TestCase() { Source = actualSources[1] } - }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = false; + this.commandLineOptions.TargetArchitecture = Architecture.X86; + this.commandLineOptions.TargetFrameworkVersion = Framework.DefaultFramework; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny()), Times.Once); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny()), Times.Once); + + Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); + } + + [TestMethod] + public void RunTestsWithTestCasesShouldUpdateFrameworkAndPlatformIfNotSpecifiedInDesignMode() + { + var actualSources = new List() { "1.dll", "2.dll" }; + var payload = new TestRunRequestPayload() + { + TestCases = new List() { + new TestCase(){Source = actualSources[0]}, + new TestCase() { Source = actualSources[0]}, + new TestCase() { Source = actualSources[1] } + }, + RunSettings = + @" " - }; - - List archSources = new List(), fxSources = new List(); - - this.commandLineOptions.IsDesignMode = true; - this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())).Callback(source => archSources.Add(source)) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Callback(source => fxSources.Add(source)) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); - - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); - CollectionAssert.AreEqual(actualSources, archSources); - CollectionAssert.AreEqual(actualSources, fxSources); - } - - [TestMethod] - public void RunTestShouldThrowExceptionIfRunSettingWithDCHasTestSettingsInIt() - { - var settingXml = @" + }; + + List archSources = new List(), fxSources = new List(); + + this.commandLineOptions.IsDesignMode = true; + this.mockAssemblyMetadataProvider.Setup(a => a.GetArchitecture(It.IsAny())).Callback(source => archSources.Add(source)) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup(a => a.GetFrameWork(It.IsAny())).Callback(source => fxSources.Add(source)) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); + + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(actualTestRunCriteria.TestRunSettings.Contains(nameof(Architecture.ARM))); + CollectionAssert.AreEqual(actualSources, archSources); + CollectionAssert.AreEqual(actualSources, fxSources); + } + + [TestMethod] + public void RunTestShouldThrowExceptionIfRunSettingWithDCHasTestSettingsInIt() + { + var settingXml = @" C:\temp.testsettings true @@ -1654,32 +1654,32 @@ public void RunTestShouldThrowExceptionIfRunSettingWithDCHasTestSettingsInIt() "; - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = settingXml - }; - - this.commandLineOptions.EnableCodeCoverage = false; - bool exceptionThrown = false; - - try - { - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - } - catch (SettingsException ex) - { - exceptionThrown = true; - Assert.IsTrue(ex.Message.Contains(@"C:\temp.testsettings"), ex.Message); - } - - Assert.IsTrue(exceptionThrown, "Initialize should throw exception"); - } - - [TestMethod] - public void RunTestShouldThrowExceptionIfRunSettingWithDCHasTestSettingsAndEnableCodeCoverageTrue() - { - var settingXml = @" + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = settingXml + }; + + this.commandLineOptions.EnableCodeCoverage = false; + bool exceptionThrown = false; + + try + { + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + } + catch (SettingsException ex) + { + exceptionThrown = true; + Assert.IsTrue(ex.Message.Contains(@"C:\temp.testsettings"), ex.Message); + } + + Assert.IsTrue(exceptionThrown, "Initialize should throw exception"); + } + + [TestMethod] + public void RunTestShouldThrowExceptionIfRunSettingWithDCHasTestSettingsAndEnableCodeCoverageTrue() + { + var settingXml = @" C:\temp.testsettings true @@ -1694,35 +1694,35 @@ public void RunTestShouldThrowExceptionIfRunSettingWithDCHasTestSettingsAndEnabl "; - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = settingXml - }; - - this.commandLineOptions.EnableCodeCoverage = true; - bool exceptionThrown = false; - - try - { - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - } - catch (SettingsException ex) - { - exceptionThrown = true; - Assert.IsTrue(ex.Message.Contains(@"C:\temp.testsettings"), ex.Message); - } - - Assert.IsTrue(exceptionThrown, "Initialize should throw exception"); - } - - [TestMethod] - public void RunTestShouldNotThrowExceptionIfRunSettingHasCodeCoverageDCAndTestSettingsInItWithEnableCoverageTrue() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = @" + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = settingXml + }; + + this.commandLineOptions.EnableCodeCoverage = true; + bool exceptionThrown = false; + + try + { + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + } + catch (SettingsException ex) + { + exceptionThrown = true; + Assert.IsTrue(ex.Message.Contains(@"C:\temp.testsettings"), ex.Message); + } + + Assert.IsTrue(exceptionThrown, "Initialize should throw exception"); + } + + [TestMethod] + public void RunTestShouldNotThrowExceptionIfRunSettingHasCodeCoverageDCAndTestSettingsInItWithEnableCoverageTrue() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = @" C:\temp.testsettings true @@ -1734,52 +1734,52 @@ public void RunTestShouldNotThrowExceptionIfRunSettingHasCodeCoverageDCAndTestSe " - }; - - this.commandLineOptions.EnableCodeCoverage = true; - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - } - - [TestMethod] - public void RunTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.EnableCodeCoverage = true; + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + } + + [TestMethod] + public void RunTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - - this.commandLineOptions.IsDesignMode = false; - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; - Assert.AreEqual(1, loggerSettingsList.Count); - Assert.AreEqual("Console", loggerSettingsList[0].FriendlyName); - Assert.IsNotNull(loggerSettingsList[0].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[0].CodeBase); - } - - [TestMethod] - public void RunTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalseInRunSettings() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = false; + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; + Assert.AreEqual(1, loggerSettingsList.Count); + Assert.AreEqual("Console", loggerSettingsList[0].FriendlyName); + Assert.IsNotNull(loggerSettingsList[0].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[0].CodeBase); + } + + [TestMethod] + public void RunTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalseInRunSettings() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" False @@ -1794,35 +1794,35 @@ public void RunTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalseInRun " - }; - - this.commandLineOptions.IsDesignMode = true; - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; - Assert.AreEqual(2, loggerSettingsList.Count); - Assert.IsNotNull(loggerSettingsList[0].Configuration); - Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); - Assert.AreEqual("Console", loggerSettingsList[1].FriendlyName); - Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[1].CodeBase); - } - - [TestMethod] - public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalseInRunSettings() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = true; + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; + Assert.AreEqual(2, loggerSettingsList.Count); + Assert.IsNotNull(loggerSettingsList[0].Configuration); + Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); + Assert.AreEqual("Console", loggerSettingsList[1].FriendlyName); + Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[1].CodeBase); + } + + [TestMethod] + public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalseInRunSettings() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" False @@ -1837,131 +1837,131 @@ public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsIfDesignModeSetFalse " - }; - this.commandLineOptions.IsDesignMode = true; - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, - new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; - Assert.AreEqual(2, loggerSettingsList.Count); - Assert.IsNotNull(loggerSettingsList[0].Configuration); - Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); - Assert.AreEqual("Console", loggerSettingsList[1].FriendlyName); - Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[1].CodeBase); - } - - [TestMethod] - public void RunTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + this.commandLineOptions.IsDesignMode = true; + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, + new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; + Assert.AreEqual(2, loggerSettingsList.Count); + Assert.IsNotNull(loggerSettingsList[0].Configuration); + Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); + Assert.AreEqual("Console", loggerSettingsList[1].FriendlyName); + Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[1].CodeBase); + } + + [TestMethod] + public void RunTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" True " - }; - - this.commandLineOptions.IsDesignMode = false; - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains("LoggerRunSettings")); - } - - [TestMethod] - public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = false; + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + Assert.IsFalse(actualTestRunCriteria.TestRunSettings.Contains("LoggerRunSettings")); + } + + [TestMethod] + public void DiscoverTestsShouldAddConsoleLoggerInRunSettingsInNonDesignMode() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - this.commandLineOptions.IsDesignMode = false; - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, - new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; - Assert.AreEqual(1, loggerSettingsList.Count); - Assert.AreEqual("Console", loggerSettingsList[0].FriendlyName); - Assert.IsNotNull(loggerSettingsList[0].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[0].CodeBase); - } - - [TestMethod] - public void DiscoverTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + this.commandLineOptions.IsDesignMode = false; + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, + new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; + Assert.AreEqual(1, loggerSettingsList.Count); + Assert.AreEqual("Console", loggerSettingsList[0].FriendlyName); + Assert.IsNotNull(loggerSettingsList[0].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[0].CodeBase); + } + + [TestMethod] + public void DiscoverTestsShouldNotAddConsoleLoggerInRunSettingsInDesignMode() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" True " - }; - this.commandLineOptions.IsDesignMode = false; - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, - new Mock().Object, this.protocolConfig); - - Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains("LoggerRunSettings")); - } - - [TestMethod] - public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInNonDesignMode() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + this.commandLineOptions.IsDesignMode = false; + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, + new Mock().Object, this.protocolConfig); + + Assert.IsFalse(actualDiscoveryCriteria.RunSettings.Contains("LoggerRunSettings")); + } + + [TestMethod] + public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInNonDesignMode() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" False @@ -1981,39 +1981,39 @@ public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentI " - }; - - this.commandLineOptions.IsDesignMode = false; - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; - Assert.AreEqual(2, loggerSettingsList.Count); - Assert.IsNotNull(loggerSettingsList[0].Configuration); - Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); - Assert.AreEqual("console", loggerSettingsList[1].FriendlyName); - Assert.AreEqual(new Uri("logger://tempconsoleUri").ToString(), loggerSettingsList[1].Uri.ToString()); - Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); - Assert.AreNotEqual("tempCodeBase", loggerSettingsList[1].CodeBase); - Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); - Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[1].CodeBase); - } - - [TestMethod] - public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInNonDesignMode() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = false; + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; + Assert.AreEqual(2, loggerSettingsList.Count); + Assert.IsNotNull(loggerSettingsList[0].Configuration); + Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); + Assert.AreEqual("console", loggerSettingsList[1].FriendlyName); + Assert.AreEqual(new Uri("logger://tempconsoleUri").ToString(), loggerSettingsList[1].Uri.ToString()); + Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); + Assert.AreNotEqual("tempCodeBase", loggerSettingsList[1].CodeBase); + Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); + Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[1].CodeBase); + } + + [TestMethod] + public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInNonDesignMode() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" False @@ -2033,42 +2033,42 @@ public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPre " - }; - this.commandLineOptions.IsDesignMode = false; - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, - new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; - Assert.AreEqual(2, loggerSettingsList.Count); - Assert.IsNotNull(loggerSettingsList[0].Configuration); - Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); - Assert.AreEqual("consoleTemp", loggerSettingsList[1].FriendlyName); - Assert.AreEqual(new Uri("logger://Microsoft/TestPlatform/ConsoleLogger/v1").ToString(), loggerSettingsList[1].Uri.ToString()); - Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); - Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].CodeBase); - Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); - Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[1].CodeBase); - } - - [TestMethod] - public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInDesignMode() - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + this.commandLineOptions.IsDesignMode = false; + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, + new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; + Assert.AreEqual(2, loggerSettingsList.Count); + Assert.IsNotNull(loggerSettingsList[0].Configuration); + Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); + Assert.AreEqual("consoleTemp", loggerSettingsList[1].FriendlyName); + Assert.AreEqual(new Uri("logger://Microsoft/TestPlatform/ConsoleLogger/v1").ToString(), loggerSettingsList[1].Uri.ToString()); + Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); + Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].CodeBase); + Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); + Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[1].CodeBase); + } + + [TestMethod] + public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInDesignMode() + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" True @@ -2088,39 +2088,39 @@ public void RunTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentI " - }; - - this.commandLineOptions.IsDesignMode = false; - TestRunCriteria actualTestRunCriteria = null; - var mockTestRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - actualTestRunCriteria = runCriteria; - }).Returns(mockTestRunRequest.Object); - this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; - Assert.AreEqual(2, loggerSettingsList.Count); - Assert.IsNotNull(loggerSettingsList[0].Configuration); - Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); - Assert.AreEqual("console", loggerSettingsList[1].FriendlyName); - Assert.AreEqual(new Uri("logger://tempconsoleUri").ToString(), loggerSettingsList[1].Uri.ToString()); - Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); - Assert.AreNotEqual("tempCodeBase", loggerSettingsList[1].CodeBase); - Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); - Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[1].CodeBase); - } - - [TestMethod] - public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInDesignMode() - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + }; + + this.commandLineOptions.IsDesignMode = false; + TestRunCriteria actualTestRunCriteria = null; + var mockTestRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + actualTestRunCriteria = runCriteria; + }).Returns(mockTestRunRequest.Object); + this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualTestRunCriteria.TestRunSettings).LoggerSettingsList; + Assert.AreEqual(2, loggerSettingsList.Count); + Assert.IsNotNull(loggerSettingsList[0].Configuration); + Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); + Assert.AreEqual("console", loggerSettingsList[1].FriendlyName); + Assert.AreEqual(new Uri("logger://tempconsoleUri").ToString(), loggerSettingsList[1].Uri.ToString()); + Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); + Assert.AreNotEqual("tempCodeBase", loggerSettingsList[1].CodeBase); + Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); + Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[1].CodeBase); + } + + [TestMethod] + public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPresentInDesignMode() + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" True @@ -2140,209 +2140,215 @@ public void DiscoverTestsShouldOverrideOnlyAssemblyNameIfConsoleLoggerAlreadyPre " - }; - this.commandLineOptions.IsDesignMode = false; - DiscoveryCriteria actualDiscoveryCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform - .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) - .Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - actualDiscoveryCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - this.testRequestManager.DiscoverTests(payload, - new Mock().Object, this.protocolConfig); - - var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; - Assert.AreEqual(2, loggerSettingsList.Count); - Assert.IsNotNull(loggerSettingsList[0].Configuration); - Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); - Assert.AreEqual("consoleTemp", loggerSettingsList[1].FriendlyName); - Assert.AreEqual(new Uri("logger://Microsoft/TestPlatform/ConsoleLogger/v1").ToString(), loggerSettingsList[1].Uri.ToString()); - Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); - Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].CodeBase); - Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); - Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); - Assert.IsNotNull(loggerSettingsList[1].CodeBase); - } - - [TestMethod] - public void ProcessTestRunAttachmentsShouldSucceedWithTelemetryEnabled() - { - var mockEventsHandler = new Mock(); - mockAttachmentsProcessingManager - .Setup(m => m.ProcessTestRunAttachmentsAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) - .Returns((IRequestData r, ICollection a, ITestRunAttachmentsProcessingEventsHandler h, CancellationToken token) => Task.Run(() => - { - r.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsSentForProcessing, 5); - r.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsAfterProcessing, 1); - })); - - var payload = new TestRunAttachmentsProcessingPayload() - { - Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, - CollectMetrics = true - }; - - testRequestManager.ProcessTestRunAttachments(payload, mockEventsHandler.Object, this.protocolConfig); - - mockAttachmentsProcessingManager.Verify(m => m.ProcessTestRunAttachmentsAsync(It.Is(r => r.IsTelemetryOptedIn), payload.Attachments, mockEventsHandler.Object, It.IsAny())); - mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStart()); - mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStop()); - - mockMetricsPublisher.Verify(p => p.PublishMetrics(TelemetryDataConstants.TestAttachmentsProcessingCompleteEvent, - It.Is>(m => m.Count == 2 && - m.ContainsKey(TelemetryDataConstants.NumberOfAttachmentsSentForProcessing) && (int)m[TelemetryDataConstants.NumberOfAttachmentsSentForProcessing] == 5 && - m.ContainsKey(TelemetryDataConstants.NumberOfAttachmentsAfterProcessing) && (int)m[TelemetryDataConstants.NumberOfAttachmentsAfterProcessing] == 1))); - } - - [TestMethod] - public void ProcessTestRunAttachmentsShouldSucceedWithTelemetryDisabled() - { - var mockEventsHandler = new Mock(); - mockAttachmentsProcessingManager - .Setup(m => m.ProcessTestRunAttachmentsAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) - .Returns(Task.FromResult(true)); - - var payload = new TestRunAttachmentsProcessingPayload() - { - Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, - CollectMetrics = false - }; - - testRequestManager.ProcessTestRunAttachments(payload, mockEventsHandler.Object, this.protocolConfig); - - mockAttachmentsProcessingManager.Verify(m => m.ProcessTestRunAttachmentsAsync(It.Is(r => !r.IsTelemetryOptedIn), payload.Attachments, mockEventsHandler.Object, It.IsAny())); - mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStart()); - mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStop()); - } - - [TestMethod] - public async Task CancelTestRunAttachmentsProcessingShouldSucceedIfRequestInProgress() - { - var mockEventsHandler = new Mock(); - mockAttachmentsProcessingManager - .Setup(m => m.ProcessTestRunAttachmentsAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) - .Returns((IRequestData r, ICollection a, ITestRunAttachmentsProcessingEventsHandler h, CancellationToken token) => Task.Run(() => - { - int i = 0; - while (!token.IsCancellationRequested) + }; + this.commandLineOptions.IsDesignMode = false; + DiscoveryCriteria actualDiscoveryCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform + .Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + actualDiscoveryCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + this.testRequestManager.DiscoverTests(payload, + new Mock().Object, this.protocolConfig); + + var loggerSettingsList = XmlRunSettingsUtilities.GetLoggerRunSettings(actualDiscoveryCriteria.RunSettings).LoggerSettingsList; + Assert.AreEqual(2, loggerSettingsList.Count); + Assert.IsNotNull(loggerSettingsList[0].Configuration); + Assert.AreEqual("blabla", loggerSettingsList[0].FriendlyName); + Assert.AreEqual("consoleTemp", loggerSettingsList[1].FriendlyName); + Assert.AreEqual(new Uri("logger://Microsoft/TestPlatform/ConsoleLogger/v1").ToString(), loggerSettingsList[1].Uri.ToString()); + Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].AssemblyQualifiedName); + Assert.AreNotEqual("tempAssemblyName", loggerSettingsList[1].CodeBase); + Assert.IsTrue(loggerSettingsList[1].Configuration.InnerXml.Contains("Value1")); + Assert.IsNotNull(loggerSettingsList[1].AssemblyQualifiedName); + Assert.IsNotNull(loggerSettingsList[1].CodeBase); + } + + [TestMethod] + public void ProcessTestRunAttachmentsShouldSucceedWithTelemetryEnabled() + { + var mockEventsHandler = new Mock(); + mockAttachmentsProcessingManager + .Setup(m => m.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((string runSettingsXml, IRequestData r, ICollection a, ICollection b, ITestRunAttachmentsProcessingEventsHandler h, CancellationToken token) => Task.Run(() => + { + r.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsSentForProcessing, 5); + r.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsAfterProcessing, 1); + })); + + var payload = new TestRunAttachmentsProcessingPayload() + { + Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, + InvokedDataCollectors = new List(), + RunSettings = Constants.EmptyRunSettings, + CollectMetrics = true + }; + + testRequestManager.ProcessTestRunAttachments(payload, mockEventsHandler.Object, this.protocolConfig); + + mockAttachmentsProcessingManager.Verify(m => m.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, It.Is(r => r.IsTelemetryOptedIn), payload.Attachments, payload.InvokedDataCollectors, mockEventsHandler.Object, It.IsAny())); + mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStart()); + mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStop()); + + mockMetricsPublisher.Verify(p => p.PublishMetrics(TelemetryDataConstants.TestAttachmentsProcessingCompleteEvent, + It.Is>(m => m.Count == 2 && + m.ContainsKey(TelemetryDataConstants.NumberOfAttachmentsSentForProcessing) && (int)m[TelemetryDataConstants.NumberOfAttachmentsSentForProcessing] == 5 && + m.ContainsKey(TelemetryDataConstants.NumberOfAttachmentsAfterProcessing) && (int)m[TelemetryDataConstants.NumberOfAttachmentsAfterProcessing] == 1))); + } + + [TestMethod] + public void ProcessTestRunAttachmentsShouldSucceedWithTelemetryDisabled() + { + var mockEventsHandler = new Mock(); + mockAttachmentsProcessingManager + .Setup(m => m.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(true)); + + var payload = new TestRunAttachmentsProcessingPayload() + { + Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, + InvokedDataCollectors = new List(), + RunSettings = Constants.EmptyRunSettings, + CollectMetrics = false + }; + + testRequestManager.ProcessTestRunAttachments(payload, mockEventsHandler.Object, this.protocolConfig); + + mockAttachmentsProcessingManager.Verify(m => m.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, It.Is(r => !r.IsTelemetryOptedIn), payload.Attachments, payload.InvokedDataCollectors, mockEventsHandler.Object, It.IsAny())); + mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStart()); + mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStop()); + } + + [TestMethod] + public async Task CancelTestRunAttachmentsProcessingShouldSucceedIfRequestInProgress() + { + var mockEventsHandler = new Mock(); + mockAttachmentsProcessingManager + .Setup(m => m.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((string runSettingsXml, IRequestData r, ICollection a, ICollection b, ITestRunAttachmentsProcessingEventsHandler h, CancellationToken token) => Task.Run(() => + { + int i = 0; + while (!token.IsCancellationRequested) + { + i++; + Console.WriteLine($"Iteration {i}"); + Task.Delay(5).Wait(); + } + + r.MetricsCollection.Add(TelemetryDataConstants.AttachmentsProcessingState, "Canceled"); + })); + + var payload = new TestRunAttachmentsProcessingPayload() + { + Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, + InvokedDataCollectors = new List(), + RunSettings = Constants.EmptyRunSettings, + CollectMetrics = true + }; + + Task task = Task.Run(() => testRequestManager.ProcessTestRunAttachments(payload, mockEventsHandler.Object, this.protocolConfig)); + await Task.Delay(50); + testRequestManager.CancelTestRunAttachmentsProcessing(); + + await task; + + mockAttachmentsProcessingManager.Verify(m => m.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, It.IsAny(), payload.Attachments, payload.InvokedDataCollectors, mockEventsHandler.Object, It.IsAny())); + mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStart()); + mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStop()); + + mockMetricsPublisher.Verify(p => p.PublishMetrics(TelemetryDataConstants.TestAttachmentsProcessingCompleteEvent, + It.Is>(m => m.Count == 1 && m.ContainsKey(TelemetryDataConstants.AttachmentsProcessingState) && (string)m[TelemetryDataConstants.AttachmentsProcessingState] == "Canceled"))); + } + + [TestMethod] + public void CancelTestRunAttachmentsProcessingShouldSucceedIfNoRequest() + { + testRequestManager.CancelTestRunAttachmentsProcessing(); + } + + [TestMethod] + public void StartTestSessionShouldPassCorrectTelemetryOptedInOptionToTestPlatform() + { + this.mockTestPlatform.Setup( + tp => tp.StartTestSession( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(true) + .Callback( + (IRequestData rd, StartTestSessionCriteria _, ITestSessionEventsHandler __) => + { + Assert.IsTrue(rd.IsTelemetryOptedIn); + }); + + Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); + + this.testRequestManager.StartTestSession( + new StartTestSessionPayload() + { + TestPlatformOptions = new TestPlatformOptions() { - i++; - Console.WriteLine($"Iteration {i}"); - Task.Delay(5).Wait(); + CollectMetrics = true } + }, + new Mock().Object, + new Mock().Object, + this.protocolConfig); + } - r.MetricsCollection.Add(TelemetryDataConstants.AttachmentsProcessingState, "Canceled"); - })); - - var payload = new TestRunAttachmentsProcessingPayload() - { - Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, - CollectMetrics = true - }; - - Task task = Task.Run(() => testRequestManager.ProcessTestRunAttachments(payload, mockEventsHandler.Object, this.protocolConfig)); - await Task.Delay(50); - testRequestManager.CancelTestRunAttachmentsProcessing(); - - await task; - - mockAttachmentsProcessingManager.Verify(m => m.ProcessTestRunAttachmentsAsync(It.IsAny(), payload.Attachments, mockEventsHandler.Object, It.IsAny())); - mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStart()); - mockTestPlatformEventSource.Verify(es => es.TestRunAttachmentsProcessingRequestStop()); - - mockMetricsPublisher.Verify(p => p.PublishMetrics(TelemetryDataConstants.TestAttachmentsProcessingCompleteEvent, - It.Is>(m => m.Count == 1 && m.ContainsKey(TelemetryDataConstants.AttachmentsProcessingState) && (string)m[TelemetryDataConstants.AttachmentsProcessingState] == "Canceled"))); - } - - [TestMethod] - public void CancelTestRunAttachmentsProcessingShouldSucceedIfNoRequest() - { - testRequestManager.CancelTestRunAttachmentsProcessing(); - } - - [TestMethod] - public void StartTestSessionShouldPassCorrectTelemetryOptedInOptionToTestPlatform() - { - this.mockTestPlatform.Setup( - tp => tp.StartTestSession( - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Returns(true) - .Callback( - (IRequestData rd, StartTestSessionCriteria _, ITestSessionEventsHandler __) => - { - Assert.IsTrue(rd.IsTelemetryOptedIn); - }); - - Environment.SetEnvironmentVariable("VSTEST_TELEMETRY_OPTEDIN", "1"); - - this.testRequestManager.StartTestSession( - new StartTestSessionPayload() - { - TestPlatformOptions = new TestPlatformOptions() - { - CollectMetrics = true - } - }, - new Mock().Object, - new Mock().Object, - this.protocolConfig); - } - - [TestMethod] - public void StartTestSessionShouldUpdateSettings() - { - var payload = new StartTestSessionPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = - @" + [TestMethod] + public void StartTestSessionShouldUpdateSettings() + { + var payload = new StartTestSessionPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = + @" " - }; - this.commandLineOptions.IsDesignMode = true; - - this.mockAssemblyMetadataProvider.Setup( - a => a.GetArchitecture(It.IsAny())) - .Returns(Architecture.ARM); - this.mockAssemblyMetadataProvider.Setup( - a => a.GetFrameWork(It.IsAny())) - .Returns(new FrameworkName(Constants.DotNetFramework46)); - - this.mockTestPlatform.Setup( - tp => tp.StartTestSession( - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Returns(true) - .Callback( - (IRequestData _, StartTestSessionCriteria criteria, ITestSessionEventsHandler __) => - { - Assert.IsTrue(criteria.RunSettings.Contains(Constants.DotNetFramework46)); - Assert.IsTrue(criteria.RunSettings.Contains(nameof(Architecture.ARM))); - }); - - this.testRequestManager.StartTestSession( - payload, - new Mock().Object, - new Mock().Object, - this.protocolConfig); - - this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); - this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); - } - - [TestMethod] - public void StartTestSessionShouldThrowSettingsExceptionWhenFindingIncompatibleDataCollectorsInTestSettings() - { - var settingXml = @" + }; + this.commandLineOptions.IsDesignMode = true; + + this.mockAssemblyMetadataProvider.Setup( + a => a.GetArchitecture(It.IsAny())) + .Returns(Architecture.ARM); + this.mockAssemblyMetadataProvider.Setup( + a => a.GetFrameWork(It.IsAny())) + .Returns(new FrameworkName(Constants.DotNetFramework46)); + + this.mockTestPlatform.Setup( + tp => tp.StartTestSession( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(true) + .Callback( + (IRequestData _, StartTestSessionCriteria criteria, ITestSessionEventsHandler __) => + { + Assert.IsTrue(criteria.RunSettings.Contains(Constants.DotNetFramework46)); + Assert.IsTrue(criteria.RunSettings.Contains(nameof(Architecture.ARM))); + }); + + this.testRequestManager.StartTestSession( + payload, + new Mock().Object, + new Mock().Object, + this.protocolConfig); + + this.mockAssemblyMetadataProvider.Verify(a => a.GetArchitecture(It.IsAny())); + this.mockAssemblyMetadataProvider.Verify(a => a.GetFrameWork(It.IsAny())); + } + + [TestMethod] + public void StartTestSessionShouldThrowSettingsExceptionWhenFindingIncompatibleDataCollectorsInTestSettings() + { + var settingXml = @" C:\temp.testsettings true @@ -2357,120 +2363,120 @@ public void StartTestSessionShouldThrowSettingsExceptionWhenFindingIncompatibleD "; - var payload = new StartTestSessionPayload() - { - Sources = new List() { "a.dll" }, - RunSettings = settingXml - }; - - this.commandLineOptions.EnableCodeCoverage = false; - bool exceptionThrown = false; - - try - { - this.testRequestManager.StartTestSession( - payload, - new Mock().Object, - new Mock().Object, - this.protocolConfig); - } - catch (SettingsException ex) - { - exceptionThrown = true; - Assert.IsTrue(ex.Message.Contains(@"C:\temp.testsettings"), ex.Message); - } - - Assert.IsTrue(exceptionThrown, "Initialize should throw exception"); - } - - [TestMethod] - public void StartTestSessionShouldBeSuccessful() - { - this.mockTestPlatform.Setup( - tp => tp.StartTestSession( - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Returns(true); - - this.testRequestManager.StartTestSession( - new StartTestSessionPayload() - { - TestPlatformOptions = new TestPlatformOptions() - { - CollectMetrics = true - } - }, - new Mock().Object, - new Mock().Object, - this.protocolConfig); - - this.mockTestPlatformEventSource.Verify( - tpes => tpes.StartTestSessionStart(), - Times.Once()); - this.mockTestPlatformEventSource.Verify( - tpes => tpes.StartTestSessionStop(), - Times.Once()); - } - - private static DiscoveryRequestPayload CreateDiscoveryPayload(string runsettings) - { - var discoveryPayload = new DiscoveryRequestPayload - { - RunSettings = runsettings, - Sources = new[] { "c:\\testproject.dll" } - }; - return discoveryPayload; - } - - private void RunTestsIfThrowsExceptionShouldThrowOut(Exception exception) - { - var payload = new TestRunRequestPayload() - { - Sources = new List() { "a", "b" }, - RunSettings = DefaultRunsettings - }; - - var createRunRequestCalled = 0; - TestRunCriteria observedCriteria = null; - var mockRunRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => - { - createRunRequestCalled++; - observedCriteria = runCriteria; - }).Returns(mockRunRequest.Object); - - mockRunRequest.Setup(mr => mr.ExecuteAsync()).Throws(exception); - - var mockRunEventsRegistrar = new Mock(); - var mockCustomlauncher = new Mock(); - - this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); - } - - private void DiscoverTestsIfThrowsExceptionShouldThrowOut(Exception exception) - { - var payload = new DiscoveryRequestPayload() - { - Sources = new List() { "a.dll", "b.dll" }, - RunSettings = DefaultRunsettings - }; - - DiscoveryCriteria observedCriteria = null; - var mockDiscoveryRequest = new Mock(); - this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( - (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => - { - observedCriteria = discoveryCriteria; - }).Returns(mockDiscoveryRequest.Object); - - mockDiscoveryRequest.Setup(mr => mr.DiscoverAsync()).Throws(exception); - - var mockDiscoveryEventsRegistrar = new Mock(); - var mockCustomlauncher = new Mock(); - - this.testRequestManager.DiscoverTests(payload, mockDiscoveryEventsRegistrar.Object, this.protocolConfig); - } - } + var payload = new StartTestSessionPayload() + { + Sources = new List() { "a.dll" }, + RunSettings = settingXml + }; + + this.commandLineOptions.EnableCodeCoverage = false; + bool exceptionThrown = false; + + try + { + this.testRequestManager.StartTestSession( + payload, + new Mock().Object, + new Mock().Object, + this.protocolConfig); + } + catch (SettingsException ex) + { + exceptionThrown = true; + Assert.IsTrue(ex.Message.Contains(@"C:\temp.testsettings"), ex.Message); + } + + Assert.IsTrue(exceptionThrown, "Initialize should throw exception"); + } + + [TestMethod] + public void StartTestSessionShouldBeSuccessful() + { + this.mockTestPlatform.Setup( + tp => tp.StartTestSession( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(true); + + this.testRequestManager.StartTestSession( + new StartTestSessionPayload() + { + TestPlatformOptions = new TestPlatformOptions() + { + CollectMetrics = true + } + }, + new Mock().Object, + new Mock().Object, + this.protocolConfig); + + this.mockTestPlatformEventSource.Verify( + tpes => tpes.StartTestSessionStart(), + Times.Once()); + this.mockTestPlatformEventSource.Verify( + tpes => tpes.StartTestSessionStop(), + Times.Once()); + } + + private static DiscoveryRequestPayload CreateDiscoveryPayload(string runsettings) + { + var discoveryPayload = new DiscoveryRequestPayload + { + RunSettings = runsettings, + Sources = new[] { "c:\\testproject.dll" } + }; + return discoveryPayload; + } + + private void RunTestsIfThrowsExceptionShouldThrowOut(Exception exception) + { + var payload = new TestRunRequestPayload() + { + Sources = new List() { "a", "b" }, + RunSettings = DefaultRunsettings + }; + + var createRunRequestCalled = 0; + TestRunCriteria observedCriteria = null; + var mockRunRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, TestRunCriteria runCriteria, TestPlatformOptions options) => + { + createRunRequestCalled++; + observedCriteria = runCriteria; + }).Returns(mockRunRequest.Object); + + mockRunRequest.Setup(mr => mr.ExecuteAsync()).Throws(exception); + + var mockRunEventsRegistrar = new Mock(); + var mockCustomlauncher = new Mock(); + + this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig); + } + + private void DiscoverTestsIfThrowsExceptionShouldThrowOut(Exception exception) + { + var payload = new DiscoveryRequestPayload() + { + Sources = new List() { "a.dll", "b.dll" }, + RunSettings = DefaultRunsettings + }; + + DiscoveryCriteria observedCriteria = null; + var mockDiscoveryRequest = new Mock(); + this.mockTestPlatform.Setup(mt => mt.CreateDiscoveryRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( + (IRequestData requestData, DiscoveryCriteria discoveryCriteria, TestPlatformOptions options) => + { + observedCriteria = discoveryCriteria; + }).Returns(mockDiscoveryRequest.Object); + + mockDiscoveryRequest.Setup(mr => mr.DiscoverAsync()).Throws(exception); + + var mockDiscoveryEventsRegistrar = new Mock(); + var mockCustomlauncher = new Mock(); + + this.testRequestManager.DiscoverTests(payload, mockDiscoveryEventsRegistrar.Object, this.protocolConfig); + } + } } \ No newline at end of file From 2e4ce701f105572f75d537f2f7e7b8ddf3664cfb Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 23 Nov 2021 18:44:19 +0100 Subject: [PATCH 02/47] revert formatting --- scripts/build/TestPlatform.Dependencies.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 5190258c1c..815cdfe5bc 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -1,4 +1,4 @@ - + 17.0.1600 From 24edd07a53e93de6029e4e524810eb493f88aad3 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 23 Nov 2021 19:09:39 +0100 Subject: [PATCH 03/47] add comment --- .../DataCollection/DataCollectionTestRunEventsHandler.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs index 12dbcf5e12..b7f0883cd4 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionTestRunEventsHandler.cs @@ -143,6 +143,8 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, runContextAttachments = GetCombinedAttachmentSets(this.dataCollectionAttachmentSets, runContextAttachments); } + // At the moment, we don't support running data collectors inside testhost process, so it will always be empty inside "TestRunCompleteEventArgs testRunCompleteArgs". + // We load invoked data collectors from data collector process inside "DataCollectionTestRunEventsHandler.HandleRawMessage" method. if (this.invokedDataCollectors != null && this.invokedDataCollectors.Any()) { foreach (var dataCollector in this.invokedDataCollectors) From e9a1e53dd96ca575f4e1fb28352057363aee8f58 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Wed, 1 Dec 2021 14:47:49 +0100 Subject: [PATCH 04/47] add new versioning strategy --- .../TestPluginDiscoverer.cs | 29 +- .../Utilities/MetadataReaderHelper.cs | 392 ++++++++++++++++++ 2 files changed, 416 insertions(+), 5 deletions(-) create mode 100644 src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index 67bfa3c182..7d0d9c6953 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -161,21 +161,41 @@ private void GetTestExtensionsFromAssembly(Assembly ass { Debug.Assert(assembly != null, "null assembly"); Debug.Assert(pluginInfos != null, "null pluginInfos"); - IEnumerable types; + + List types = new List(); Type extension = typeof(TExtension); try { - types = TypesToLoadUtilities.GetTypesToLoad(assembly); + MetadataReaderExtensionsHelper extensionHelper = new MetadataReaderExtensionsHelper(); + var discoveredExtensions = extensionHelper.DiscoverTestPlatformExtensionVersionAttributeExtensions(assembly, filePath); + if (discoveredExtensions?.Length>0) + { + types.AddRange(discoveredExtensions); + } + } + catch (Exception e) + { + EqtTrace.Warning("TestPluginDiscoverer: Failed to get types searching for 'TestPlatformExtensionVersionAttribute' from assembly '{0}'. Error: {1}", assembly.FullName, e.ToString()); + } + + + try + { + var typesToLoad = TypesToLoadUtilities.GetTypesToLoad(assembly); + if (typesToLoad?.Any() == true) + { + types.AddRange(typesToLoad); + } if (!types.Any()) { - types = assembly.GetTypes().Where(type => type.GetTypeInfo().IsClass && !type.GetTypeInfo().IsAbstract); + types.AddRange(assembly.GetTypes().Where(type => type.GetTypeInfo().IsClass && !type.GetTypeInfo().IsAbstract)); } } catch (ReflectionTypeLoadException e) { - EqtTrace.Warning("TestPluginDiscoverer: Failed to get types from assembly '{0}'. Skipping test extension scan for this assembly. Error: {1}", assembly.FullName, e.ToString()); + EqtTrace.Warning("TestPluginDiscoverer: Failed to get types from assembly '{0}'. Error: {1}", assembly.FullName, e.ToString()); if (e.LoaderExceptions != null) { @@ -184,7 +204,6 @@ private void GetTestExtensionsFromAssembly(Assembly ass EqtTrace.Warning("LoaderExceptions: {0}", ex); } } - return; } if (types != null && types.Any()) diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs new file mode 100644 index 0000000000..e26c67e2f0 --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -0,0 +1,392 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.PortableExecutable; +using System.Text; + +namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities +{ + internal class MetadataReaderExtensionsHelper + { + private static string TestPlatformExtensionVersionAttribute = "TestPlatformExtensionVersionAttribute"; + private static string[] MethodsDefinition = new string[] { ".ctor", "get_Version" }; + private static Type[] EmptyTypeArray = new Type[0]; + + public MetadataReaderExtensionsHelper() + { + + } + + public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions(Assembly assembly, string assemblyFilePath) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); + + using (var stream = new FileStream(assemblyFilePath, FileMode.Open, FileAccess.Read)) + using (var reader = new PEReader(stream, PEStreamOptions.Default)) + { + MetadataReader metadataReader = reader.GetMetadataReader(MetadataReaderOptions.Default); + + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Search '{TestPlatformExtensionVersionAttribute}' definition inside current assembly '{assembly.FullName}'"); + Type testPlatformExtensionVersionAttributeType = SearchExtensionAttribute(assembly, metadataReader); + + if (testPlatformExtensionVersionAttributeType is null) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: '{TestPlatformExtensionVersionAttribute}' attribute not found inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); + return EmptyTypeArray; + } + + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Inspect types for extensions"); + return InspectTypes(assembly, metadataReader, testPlatformExtensionVersionAttributeType); + } + } + + private Type[] InspectTypes(Assembly assembly, MetadataReader metadataReader, Type testPlatformExtensionVersionAttributeType) + { + List> extensions = null; + + foreach (var handle in metadataReader.TypeDefinitions) + { + if (handle.IsNil) + { + continue; + } + + var typeDef = metadataReader.GetTypeDefinition(handle); + var typeName = metadataReader.GetString(typeDef.Name); + var typeNameSpace = metadataReader.GetString(typeDef.Namespace); + string fullName = $"{typeNameSpace}.{typeName}"; + + if (fullName == testPlatformExtensionVersionAttributeType.FullName) + { + continue; + } + + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Analyze TypeDefinitionHandle '{fullName}'"); + + var customAttributes = metadataReader.GetCustomAttributes(handle); + if (customAttributes.Count == 0) + { + continue; + } + + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Analyze attributes for type '{fullName}'"); + foreach (var attributeHandle in customAttributes) + { + if (!attributeHandle.IsNil) + { + if (!GetAttributeTypeAndConstructor(metadataReader, attributeHandle, out EntityHandle attributeType, out EntityHandle _)) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute found for '{fullName}' (GetAttributeTypeAndConstructor)"); + continue; + } + + if (!GetAttributeTypeNamespaceAndName(metadataReader, attributeType, out StringHandle namespaceHandle, out StringHandle nameHandle)) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute found for '{fullName}' (GetAttributeTypeNamespaceAndName)"); + continue; + } + + string attributeFullName = $"{metadataReader.GetString(namespaceHandle)}.{metadataReader.GetString(nameHandle)}"; + if (attributeFullName != testPlatformExtensionVersionAttributeType.FullName) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute found for '{fullName}' wrong type: {attributeFullName}"); + continue; + } + + try + { + var extensionType = assembly.GetType(fullName); + var version = GetVersion(metadataReader, metadataReader.GetCustomAttribute(attributeHandle)); + if(version == int.MinValue) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Unable to read the version for '{fullName}'"); + } + + if (extensions is null) extensions = new List>(); + extensions.Add(Tuple.Create(version, extensionType)); + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid extension found '{extensionType}' version '{version}'"); + } + catch (Exception ex) + { + EqtTrace.Verbose($"Failed to create extension for type '{fullName}'\n{FormatException(ex)}"); + } + } + } + } + + return extensions?.OrderByDescending(t => t.Item1).Select(t => t.Item2).ToArray() ?? EmptyTypeArray; + } + + private string FormatException(Exception ex) + { + StringBuilder log = new StringBuilder(); + Exception current = ex; + while (current != null) + { + log.AppendLine(current.ToString()); + current = current.InnerException; + } + + return log.ToString(); + } + + // https://github.com/dotnet/runtime/blob/main/src/libraries/System.Diagnostics.FileVersionInfo/src/System/Diagnostics/FileVersionInfo.Unix.cs#L288 + private int GetVersion(MetadataReader metadataReader, CustomAttribute attributeHandle) + { + EntityHandle ctorHandle = attributeHandle.Constructor; + BlobHandle signature; + switch (ctorHandle.Kind) + { + case HandleKind.MemberReference: + signature = metadataReader.GetMemberReference((MemberReferenceHandle)ctorHandle).Signature; + break; + case HandleKind.MethodDefinition: + signature = metadataReader.GetMethodDefinition((MethodDefinitionHandle)ctorHandle).Signature; + break; + default: + // Unusual case, potentially invalid IL + return int.MinValue; + } + + BlobReader signatureReader = metadataReader.GetBlobReader(signature); + BlobReader valueReader = metadataReader.GetBlobReader(attributeHandle.Value); + const ushort Prolog = 1; // two-byte "prolog" defined by ECMA-335 (II.23.3) to be at the beginning of attribute value blobs + if (valueReader.ReadUInt16() == Prolog) + { + SignatureHeader header = signatureReader.ReadSignatureHeader(); + int parameterCount; + if (header.Kind == SignatureKind.Method && // attr ctor must be a method + !header.IsGeneric && // attr ctor must be non-generic + signatureReader.TryReadCompressedInteger(out parameterCount) && // read parameter count + parameterCount == 1 && // attr ctor must have 1 parameter + signatureReader.ReadSignatureTypeCode() == SignatureTypeCode.Void && // attr ctor return type must be void + signatureReader.ReadSignatureTypeCode() == SignatureTypeCode.Int32) // attr ctor first parameter must be int32 + { + return valueReader.ReadInt32(); + } + } + + return int.MinValue; + } + + private Type SearchExtensionAttribute(Assembly assembly, MetadataReader metadataReader) + { + foreach (TypeDefinitionHandle typeDefHandle in metadataReader.TypeDefinitions) + { + try + { + if (typeDefHandle.IsNil) + { + continue; + } + + var typeDef = metadataReader.GetTypeDefinition(typeDefHandle); + var typeName = metadataReader.GetString(typeDef.Name); + var @namespace = metadataReader.GetString(typeDef.Namespace); + var fullName = $"{@namespace}.{typeName}"; + + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Analyze TypeDefinitionHandle '{fullName}'"); + + // Check the name + if (typeName == TestPlatformExtensionVersionAttribute && (typeDef.Attributes & TypeAttributes.Sealed) == TypeAttributes.Sealed) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid type name found '{fullName}'"); + + // Check it inherits from System.Attribute + if (typeDef.BaseType.Kind != HandleKind.TypeReference) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Type '{fullName}' doesn't inherit from System.Attribute (typeDef.BaseType.Kind != HandleKind.TypeReference)"); + continue; + } + + var baseTypeReferenceHandle = metadataReader.GetTypeReference((TypeReferenceHandle)typeDef.BaseType); + if ($"{metadataReader.GetString(baseTypeReferenceHandle.Namespace)}.{metadataReader.GetString(baseTypeReferenceHandle.Name)}" != "System.Attribute") + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Type '{fullName}' doesn't inherit from System.Attribute (baseTypeFullName != 'System.Attribute')"); + continue; + } + + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Verify members definition for type '{fullName}'"); + + int isGoodCandidate = 0; + foreach (var method in typeDef.GetMethods()) + { + var methodDef = metadataReader.GetMethodDefinition(method); + var methodName = metadataReader.GetString(methodDef.Name); + if (MethodsDefinition.Contains(methodName)) + { + // Verify the ctor signature, int32 for the version number. + if (methodName == ".ctor" && + (methodDef.Attributes & MethodAttributes.Public) == MethodAttributes.Public && + (methodDef.Attributes & MethodAttributes.SpecialName) == MethodAttributes.SpecialName + ) + { + var sigReader = metadataReader.GetBlobReader(methodDef.Signature); + var decoder = new SignatureDecoder(new TestPlatformExtensionVersionAttributeSignatureDecoder(), metadataReader, genericContext: null); + var methodSignature = decoder.DecodeMethodSignature(ref sigReader); + if (methodSignature.Header.IsInstance && + methodSignature.ReturnType == "void" && + methodSignature.ParameterTypes != null && + methodSignature.ParameterTypes.Length == 1 && + methodSignature.ParameterTypes[0] == "int32" && + methodSignature.GenericParameterCount == 0 && + !methodSignature.Header.IsGeneric) + { + isGoodCandidate++; + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Found '.ctor' for '{fullName}'"); + } + } + + if (methodName == "get_Version" && + (methodDef.Attributes & MethodAttributes.Public) == MethodAttributes.Public) + { + var sigReader = metadataReader.GetBlobReader(methodDef.Signature); + var decoder = new SignatureDecoder(new TestPlatformExtensionVersionAttributeSignatureDecoder(), metadataReader, genericContext: null); + var methodSignature = decoder.DecodeMethodSignature(ref sigReader); + if (methodSignature.Header.IsInstance && + methodSignature.ReturnType == "int32" && + methodSignature.ParameterTypes != null && + methodSignature.ParameterTypes.Length == 0 && + methodSignature.GenericParameterCount == 0 && + !methodSignature.Header.IsGeneric) + { + isGoodCandidate++; + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Found 'get_Version' for '{fullName}'"); + } + } + } + } + + // If all characteristics were meet we'll use this attribute to find extensions type. + if (isGoodCandidate != 2) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Members definition verification for type '{fullName}' failed"); + continue; + } + + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Members definition verification for type '{fullName}' succeded"); + return assembly.GetType(fullName); + } + } + catch (Exception ex) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Exception during TypeDefinitions analysis\n{ex}"); + } + } + + return null; + } + + // https://github.com/dotnet/runtime/blob/6cf529168a8dcdfb158738d46be40b1867fd1bfa/src/coreclr/tools/Common/TypeSystem/Ecma/MetadataExtensions.cs#L173 + private bool GetAttributeTypeNamespaceAndName(MetadataReader metadataReader, EntityHandle attributeType, + out StringHandle namespaceHandle, out StringHandle nameHandle) + { + namespaceHandle = default; + nameHandle = default; + + if (attributeType.Kind == HandleKind.TypeReference) + { + TypeReference typeRefRow = metadataReader.GetTypeReference((TypeReferenceHandle)attributeType); + HandleKind handleType = typeRefRow.ResolutionScope.Kind; + + // Nested type? + if (handleType == HandleKind.TypeReference || handleType == HandleKind.TypeDefinition) + return false; + + nameHandle = typeRefRow.Name; + namespaceHandle = typeRefRow.Namespace; + return true; + } + else if (attributeType.Kind == HandleKind.TypeDefinition) + { + var def = metadataReader.GetTypeDefinition((TypeDefinitionHandle)attributeType); + + // Nested type? + if (IsNested(def.Attributes)) + return false; + + nameHandle = def.Name; + namespaceHandle = def.Namespace; + return true; + } + else + { + // unsupported metadata + return false; + } + } + + // https://github.com/dotnet/runtime/blob/6cf529168a8dcdfb158738d46be40b1867fd1bfa/src/coreclr/tools/Common/TypeSystem/Ecma/MetadataExtensions.cs#L150 + private bool GetAttributeTypeAndConstructor(MetadataReader metadataReader, CustomAttributeHandle attributeHandle, + out EntityHandle attributeType, out EntityHandle attributeCtor) + { + attributeCtor = metadataReader.GetCustomAttribute(attributeHandle).Constructor; + + if (attributeCtor.Kind == HandleKind.MemberReference) + { + attributeType = metadataReader.GetMemberReference((MemberReferenceHandle)attributeCtor).Parent; + return true; + } + else if (attributeCtor.Kind == HandleKind.MethodDefinition) + { + attributeType = metadataReader.GetMethodDefinition((MethodDefinitionHandle)attributeCtor).GetDeclaringType(); + return true; + } + else + { + // invalid metadata + attributeType = default; + return false; + } + } + + private bool IsNested(TypeAttributes flags) + { + return (flags & (TypeAttributes)0x00000006) != 0; + } + + class TestPlatformExtensionVersionAttributeSignatureDecoder : ISignatureTypeProvider + { + public string GetArrayType(string elementType, ArrayShape shape) + => string.Empty; + public string GetByReferenceType(string elementType) + => string.Empty; + public string GetFunctionPointerType(MethodSignature signature) + => string.Empty; + public string GetGenericInstantiation(string genericType, System.Collections.Immutable.ImmutableArray typeArguments) + => string.Empty; + public string GetGenericMethodParameter(object genericContext, int index) + => string.Empty; + public string GetGenericTypeParameter(object genericContext, int index) + => string.Empty; + public string GetModifiedType(string modifier, string unmodifiedType, bool isRequired) + => string.Empty; + public string GetPinnedType(string elementType) + => string.Empty; + public string GetPointerType(string elementType) + => string.Empty; + public string GetSZArrayType(string elementType) + => string.Empty; + public string GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) + => string.Empty; + public string GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) + => string.Empty; + public string GetTypeFromSpecification(MetadataReader reader, object genericContext, TypeSpecificationHandle handle, byte rawTypeKind) + => string.Empty; + public string GetPrimitiveType(PrimitiveTypeCode typeCode) + { + switch (typeCode) + { + case PrimitiveTypeCode.Int32: return "int32"; + case PrimitiveTypeCode.Void: return "void"; + default: return ""; + } + } + } + } +} From 1a0159998b80f06a0def0fefed6a60e3b12e8ade Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Wed, 1 Dec 2021 16:16:22 +0100 Subject: [PATCH 05/47] fix public api --- .../DataCollectionExtensionManager.cs | 2 +- .../PublicAPI/PublicAPI.Shipped.txt | 10 +++++----- .../DataCollectorAttachmentsProcessorsFactory.cs | 2 +- .../PublicAPI/PublicAPI.Shipped.txt | 12 ++++++------ .../PublicAPI/PublicAPI.Shipped.txt | 2 +- .../PublicAPI/PublicAPI.Shipped.txt | 4 +++- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs index fa585077f3..e7a3cf039e 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs @@ -85,7 +85,7 @@ public static DataCollectorExtensionManager Create(string extensionAssemblyFileP /// /// Hold data about the Data Collector. /// - public class DataCollectorMetadata : IDataCollectorCapabilities + internal class DataCollectorMetadata : IDataCollectorCapabilities { /// /// Constructor for DataCollectorMetadata diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index 98840e0caf..c23e7a1cfd 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -9,9 +9,10 @@ const Microsoft.VisualStudio.TestPlatform.Common.TestPlatformDefaults.EnableBoun const Microsoft.VisualStudio.TestPlatform.Common.TestPlatformDefaults.MaxBytesLoggerEventQueueCanHold = "MaxBytesLoggerEventQueueCanHold" -> string const Microsoft.VisualStudio.TestPlatform.Common.TestPlatformDefaults.MaxNumberOfEventsLoggerEventQueueCanHold = "MaxNumberOfEventsLoggerEventQueueCanHold" -> string Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult -Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.AfterTestRunEndResult(System.Collections.ObjectModel.Collection attachmentSets, System.Collections.Generic.IDictionary metrics) -> void +Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.AfterTestRunEndResult(System.Collections.ObjectModel.Collection attachmentSets, System.Collections.ObjectModel.Collection invokedDataCollectors, System.Collections.Generic.IDictionary metrics) -> void Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.AttachmentSets.get -> System.Collections.ObjectModel.Collection Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.Metrics.get -> System.Collections.Generic.IDictionary +Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.InvokedDataCollectors.get -> System.Collections.ObjectModel.Collection Microsoft.VisualStudio.TestPlatform.Common.DataCollection.BeforeTestRunStartResult Microsoft.VisualStudio.TestPlatform.Common.DataCollection.BeforeTestRunStartResult.BeforeTestRunStartResult(System.Collections.Generic.IDictionary environmentVariables, int dataCollectionEventsPort) -> void Microsoft.VisualStudio.TestPlatform.Common.DataCollection.BeforeTestRunStartResult.DataCollectionEventsPort.get -> int @@ -19,10 +20,6 @@ Microsoft.VisualStudio.TestPlatform.Common.DataCollection.BeforeTestRunStartResu Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message) -> void Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message, System.Exception innerException) -> void -Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata -Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.DataCollectorMetadata(string extension, string friendlyName) -> void -Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.ExtensionUri.get -> string -Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.FriendlyName.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata.ExtensionUri.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata.FriendlyName.get -> string @@ -47,6 +44,7 @@ Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestExte Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.AssemblyQualifiedName.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.TestPluginInformation(System.Type testExtensionType) -> void +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.FilePath.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.VSExtensionManager Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.VSExtensionManager.GetUnitTestExtensions() -> System.Collections.Generic.IEnumerable Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.VSExtensionManager.VSExtensionManager() -> void @@ -76,6 +74,8 @@ Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IBaseTestEventsRegistrar Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IBaseTestEventsRegistrar.LogWarning(string message) -> void Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities.FriendlyName.get -> string +Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities.FilePath.get -> string +Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities.HasAttachmentProcessor.get -> bool Microsoft.VisualStudio.TestPlatform.Common.Interfaces.ISettingsProviderCapabilities Microsoft.VisualStudio.TestPlatform.Common.Interfaces.ISettingsProviderCapabilities.SettingsName.get -> string Microsoft.VisualStudio.TestPlatform.Common.Interfaces.ITestDiscovererCapabilities diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index 7852cda8d2..61e35c7555 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -12,7 +12,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing { - public class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory + internal class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory { private static Uri CoverageUri = new Uri("datacollector://microsoft/CodeCoverage/2.0"); private const string CoverageFriendlyName = "Code Coverage"; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt index a24b6bf8a3..2d1e6db286 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt @@ -61,7 +61,7 @@ Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.DataCollectio Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IDataCollectorsSettingsProvider Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IDataCollectorsSettingsProvider.Settings.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollectionRunSettings Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IProxyDataCollectionManager -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IProxyDataCollectionManager.AfterTestRunEnd(bool isCanceled, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler runEventsHandler) -> System.Collections.ObjectModel.Collection +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IProxyDataCollectionManager.AfterTestRunEnd(bool isCanceled, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler runEventsHandler) -> Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.DataCollectionResult Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IProxyDataCollectionManager.BeforeTestRunStart(bool resetDataCollectors, bool isRunStartingNow, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler runEventsHandler) -> Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.DataCollectionParameters Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IProxyDataCollectionManager.Initialize() -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces.IProxyDataCollectionManager.SettingsXml.get -> string @@ -103,10 +103,6 @@ Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.GetDiscoveryManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IDiscoveryManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.GetExecutionManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.TestHostManagerFactory(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> void -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.ProcessTestRunAttachmentsAsync(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, System.Collections.Generic.IEnumerable attachments, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.ProcessTestRunAttachmentsAsync(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, System.Collections.Generic.IEnumerable attachments, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.TestRunAttachmentsProcessingManager(Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces.ITestPlatformEventSource testPlatformEventSource, params Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor[] dataCollectorAttachmentsProcessors) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool override Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManagerWithDataCollection.Initialize(bool skipDefaultAdapters) -> void override Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManagerWithDataCollection.SetupChannel(System.Collections.Generic.IEnumerable sources, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler eventHandler) -> bool @@ -127,4 +123,8 @@ virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionMana virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool.AddSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager proxyManager) -> bool virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool.KillSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo) -> bool virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool.ReturnProxy(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, int proxyId) -> bool -virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool.TryTakeProxy(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, string source, string runSettings) -> Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManager \ No newline at end of file +virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool.TryTakeProxy(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, string source, string runSettings) -> Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.DataCollectionResult +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.DataCollectionResult.Attachments.get -> System.Collections.ObjectModel.Collection +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.DataCollectionResult.DataCollectionResult(System.Collections.ObjectModel.Collection attachments, System.Collections.ObjectModel.Collection invokedDataCollectors) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.DataCollectionResult.InvokedDataCollectors.get -> System.Collections.ObjectModel.Collection \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.Utilities/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Utilities/PublicAPI/PublicAPI.Shipped.txt index 6b354c8abc..cbb7b132a9 100644 --- a/src/Microsoft.TestPlatform.Utilities/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Utilities/PublicAPI/PublicAPI.Shipped.txt @@ -2,7 +2,7 @@ Microsoft.VisualStudio.TestPlatform.Utilities.ClientUtilities Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler.CodeCoverageDataAttachmentsHandler() -> void Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler.GetExtensionUris() -> System.Collections.Generic.IEnumerable -Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> +Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler.ProcessAttachmentSetsAsync(System.Xml.XmlElement configurationElement, System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageDataAttachmentsHandler.SupportsIncrementalProcessing.get -> bool Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageRunSettingsProcessor Microsoft.VisualStudio.TestPlatform.Utilities.CodeCoverageRunSettingsProcessor.CodeCoverageRunSettingsProcessor(System.Xml.XmlNode defaultSettingsRootNode) -> void diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt index 9e88c55c27..b18957df29 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt @@ -47,6 +47,7 @@ Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleW Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleWrapperAsync.EndSession() -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleWrapperAsync.InitializeExtensionsAsync(System.Collections.Generic.IEnumerable pathToAdditionalExtensions) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleWrapperAsync.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleWrapperAsync.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollectors, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleWrapperAsync.RunTestsAsync(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleWrapperAsync.RunTestsAsync(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces.IVsTestConsoleWrapperAsync.RunTestsAsync(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> System.Threading.Tasks.Task @@ -112,7 +113,8 @@ Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.Disco Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.EndSession() -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.InitializeExtensions(System.Collections.Generic.IEnumerable pathToAdditionalExtensions) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.InitializeExtensionsAsync(System.Collections.Generic.IEnumerable pathToAdditionalExtensions) -> System.Threading.Tasks.Task -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollectors, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void From c8d065e6d4d055cadd8a4c5c34524c99b38c0c28 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 2 Dec 2021 09:36:51 +0100 Subject: [PATCH 06/47] fix build --- .../ExtensionFramework/TestPluginDiscoverer.cs | 4 ++-- .../Microsoft.TestPlatform.Common.csproj | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index 7d0d9c6953..2282534b9c 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -256,8 +256,8 @@ private void GetTestExtensionFromType( if (extensionCollection.ContainsKey(pluginInfo.IdentifierData)) { EqtTrace.Warning( - "TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data '{0}'; keeping the first one.", - pluginInfo.IdentifierData); + "TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data '{0}' and type '{1}'; keeping the first one '{2}'.", + pluginInfo.IdentifierData, pluginInfo.AssemblyQualifiedName, extensionCollection[pluginInfo.IdentifierData].AssemblyQualifiedName); } else { diff --git a/src/Microsoft.TestPlatform.Common/Microsoft.TestPlatform.Common.csproj b/src/Microsoft.TestPlatform.Common/Microsoft.TestPlatform.Common.csproj index ad8a71623a..3a3f83a250 100644 --- a/src/Microsoft.TestPlatform.Common/Microsoft.TestPlatform.Common.csproj +++ b/src/Microsoft.TestPlatform.Common/Microsoft.TestPlatform.Common.csproj @@ -21,6 +21,7 @@ + From d72d1efea13f7f2b8985bdf49a8a0d7ee6659f0f Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 2 Dec 2021 09:56:27 +0100 Subject: [PATCH 07/47] add header --- .../Utilities/MetadataReaderHelper.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index e26c67e2f0..524f02c41e 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -1,4 +1,7 @@ -using Microsoft.VisualStudio.TestPlatform.ObjectModel; +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; using System; using System.Collections.Generic; using System.IO; From ddfea67cbc4ac2e3484fbb3a0fb49a6b295e2824 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 2 Dec 2021 09:57:42 +0100 Subject: [PATCH 08/47] reorder usings --- .../Utilities/MetadataReaderHelper.cs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index 524f02c41e..f9cde52ce4 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -1,19 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Reflection.Metadata; -using System.Reflection.Metadata.Ecma335; -using System.Reflection.PortableExecutable; -using System.Text; - namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities { + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Reflection; + using System.Reflection.Metadata; + using System.Reflection.Metadata.Ecma335; + using System.Reflection.PortableExecutable; + using System.Text; + + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + internal class MetadataReaderExtensionsHelper { private static string TestPlatformExtensionVersionAttribute = "TestPlatformExtensionVersionAttribute"; From 24e4805ec6dbbb897cf1fd09cb30a15ea02118ad Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 2 Dec 2021 18:54:19 +0100 Subject: [PATCH 09/47] add new IConfigurableAttachmentProcessor and add second TestExtensionAttribute strategy --- .../TestPluginDiscoverer.cs | 4 +- .../ITestRunAttachmentsProcessingManager.cs | 2 +- .../Utilities/MetadataReaderHelper.cs | 113 ++++++++++++++++-- ...taCollectorAttachmentsProcessorsFactory.cs | 16 +-- .../TestRunAttachmentsProcessingManager.cs | 2 +- ...gurableDataCollectorAttachmentProcessor.cs | 51 ++++++++ .../IDataCollectorAttachmentProcessor.cs | 4 +- .../PublicAPI/net/PublicAPI.Shipped.txt | 6 +- .../CodeCoverageDataAttachmentsHandler.cs | 2 +- .../DataCollectionTests.cs | 11 ++ .../TestPluginDiscovererTests.cs | 2 +- ...lectorAttachmentsProcessorsFactoryTests.cs | 4 +- ...estRunAttachmentsProcessingManagerTests.cs | 10 +- .../SampleDataCollector.cs | 9 +- .../TestExtensionTypesAttribute.cs | 45 +++++++ .../DataCollectionManagerTests.cs | 2 +- 16 files changed, 248 insertions(+), 35 deletions(-) create mode 100644 src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs create mode 100644 test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index 2282534b9c..7207b90208 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -168,8 +168,8 @@ private void GetTestExtensionsFromAssembly(Assembly ass try { MetadataReaderExtensionsHelper extensionHelper = new MetadataReaderExtensionsHelper(); - var discoveredExtensions = extensionHelper.DiscoverTestPlatformExtensionVersionAttributeExtensions(assembly, filePath); - if (discoveredExtensions?.Length>0) + var discoveredExtensions = extensionHelper.DiscoverTestPlatformExtensionVersionAttributeExtensions2(assembly, filePath); + if (discoveredExtensions?.Length > 0) { types.AddRange(discoveredExtensions); } diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs index db8ff0584d..a94c0f5227 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs @@ -48,6 +48,6 @@ internal interface IDataCollectorAttachmentsProcessorsFactory /// /// List of invoked data collectors /// List of attachments processors - IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors); + IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors); } } diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index f9cde52ce4..263fdf2f0d 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -17,7 +17,8 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities internal class MetadataReaderExtensionsHelper { - private static string TestPlatformExtensionVersionAttribute = "TestPlatformExtensionVersionAttribute"; + private static string TestPlatformExtensionVersionAttribute = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypeAttribute"; + private static string TestExtensionTypesAttributeV2 = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypesV2Attribute"; private static string[] MethodsDefinition = new string[] { ".ctor", "get_Version" }; private static Type[] EmptyTypeArray = new Type[0]; @@ -26,6 +27,105 @@ public MetadataReaderExtensionsHelper() } + public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions2(Assembly assembly, string assemblyFilePath) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); + + List> extensions = null; + using (var stream = new FileStream(assemblyFilePath, FileMode.Open, FileAccess.Read)) + using (var reader = new PEReader(stream, PEStreamOptions.Default)) + { + MetadataReader metadataReader = reader.GetMetadataReader(); + + foreach (var customAttributeHandle in metadataReader.CustomAttributes) + { + string attributeFullName = null; + try + { + if (customAttributeHandle.IsNil) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute (customAttributeHandle.IsNil)"); + continue; + } + + if (!GetAttributeTypeAndConstructor(metadataReader, customAttributeHandle, out EntityHandle attributeType)) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute (GetAttributeTypeAndConstructor)"); + continue; + } + + if (!GetAttributeTypeNamespaceAndName(metadataReader, attributeType, out StringHandle namespaceHandle, out StringHandle nameHandle)) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute (GetAttributeTypeNamespaceAndName)"); + continue; + } + + attributeFullName = $"{metadataReader.GetString(namespaceHandle)}.{metadataReader.GetString(nameHandle)}"; + if (attributeFullName != TestExtensionTypesAttributeV2) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid attribute '{attributeFullName}'"); + continue; + } + + var customAttribute = metadataReader.GetCustomAttribute(customAttributeHandle); + EntityHandle ctorHandle = customAttribute.Constructor; + BlobHandle signature; + switch (ctorHandle.Kind) + { + case HandleKind.MemberReference: + signature = metadataReader.GetMemberReference((MemberReferenceHandle)ctorHandle).Signature; + break; + case HandleKind.MethodDefinition: + signature = metadataReader.GetMethodDefinition((MethodDefinitionHandle)ctorHandle).Signature; + break; + default: + Console.WriteLine($"MetadataReaderExtensionsHelper: Potentially invalid IL for the attribute '{attributeFullName}'"); + continue; + } + + BlobReader signatureReader = metadataReader.GetBlobReader(signature); + BlobReader valueReader = metadataReader.GetBlobReader(customAttribute.Value); + const ushort Prolog = 1; // two-byte "prolog" defined by ECMA-335 (II.23.3) to be at the beginning of attribute value blobs + UInt16 prolog = valueReader.ReadUInt16(); + if (prolog != Prolog) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid blob attribute prolog '{prolog}'"); + continue; + } + + string fullName = valueReader.ReadSerializedString(); + int version = valueReader.ReadInt32(); + + try + { + var extensionType = assembly.GetType(fullName); + if (extensionType is null) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Unable to get extension for type '{fullName}'"); + continue; + } + + if (extensions is null) extensions = new List>(); + extensions.Add(Tuple.Create(version, extensionType)); + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid extension found '{extensionType}' version '{version}'"); + } + catch (Exception ex) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Failure during type creation, extension full name: '{fullName}'\n{FormatException(ex)}"); + } + } + catch (Exception ex) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Failure during custom attribute analysis, attribute full name: {attributeFullName}\n{FormatException(ex)}"); + } + } + } + + var finalExtensions = extensions?.OrderByDescending(t => t.Item1).Select(t => t.Item2).ToArray() ?? EmptyTypeArray; + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Found {extensions?.Count ?? 0} extensions"); + return finalExtensions; + } + public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions(Assembly assembly, string assemblyFilePath) { EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); @@ -83,7 +183,7 @@ private Type[] InspectTypes(Assembly assembly, MetadataReader metadataReader, Ty { if (!attributeHandle.IsNil) { - if (!GetAttributeTypeAndConstructor(metadataReader, attributeHandle, out EntityHandle attributeType, out EntityHandle _)) + if (!GetAttributeTypeAndConstructor(metadataReader, attributeHandle, out EntityHandle attributeType)) { EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute found for '{fullName}' (GetAttributeTypeAndConstructor)"); continue; @@ -106,7 +206,7 @@ private Type[] InspectTypes(Assembly assembly, MetadataReader metadataReader, Ty { var extensionType = assembly.GetType(fullName); var version = GetVersion(metadataReader, metadataReader.GetCustomAttribute(attributeHandle)); - if(version == int.MinValue) + if (version == int.MinValue) { EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Unable to read the version for '{fullName}'"); } @@ -197,7 +297,7 @@ private Type SearchExtensionAttribute(Assembly assembly, MetadataReader metadata EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Analyze TypeDefinitionHandle '{fullName}'"); // Check the name - if (typeName == TestPlatformExtensionVersionAttribute && (typeDef.Attributes & TypeAttributes.Sealed) == TypeAttributes.Sealed) + if (fullName == TestPlatformExtensionVersionAttribute && (typeDef.Attributes & TypeAttributes.Sealed) == TypeAttributes.Sealed) { EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid type name found '{fullName}'"); @@ -326,10 +426,9 @@ private bool GetAttributeTypeNamespaceAndName(MetadataReader metadataReader, Ent } // https://github.com/dotnet/runtime/blob/6cf529168a8dcdfb158738d46be40b1867fd1bfa/src/coreclr/tools/Common/TypeSystem/Ecma/MetadataExtensions.cs#L150 - private bool GetAttributeTypeAndConstructor(MetadataReader metadataReader, CustomAttributeHandle attributeHandle, - out EntityHandle attributeType, out EntityHandle attributeCtor) + private bool GetAttributeTypeAndConstructor(MetadataReader metadataReader, CustomAttributeHandle attributeHandle, out EntityHandle attributeType) { - attributeCtor = metadataReader.GetCustomAttribute(attributeHandle).Constructor; + var attributeCtor = metadataReader.GetCustomAttribute(attributeHandle).Constructor; if (attributeCtor.Kind == HandleKind.MemberReference) { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index 61e35c7555..06ed66c88d 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -17,9 +17,9 @@ internal class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachm private static Uri CoverageUri = new Uri("datacollector://microsoft/CodeCoverage/2.0"); private const string CoverageFriendlyName = "Code Coverage"; - public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) + public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) { - IDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); + IDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); bool addCodeCoverageAttachmentProcessors = true; if (invokedDataCollectors != null) @@ -44,10 +44,10 @@ public IReadOnlyDictionary Create(Inv if (dataCollectorExtension?.Metadata.HasAttachmentProcessor == true) { Type attachmentProcessorType = ((DataCollectorConfig)dataCollectorExtension.TestPluginInfo).AttachmentsProcessorType; - IDataCollectorAttachmentProcessor dataCollectorAttachmentProcessorInstance = null; + IConfigurableDataCollectorAttachmentProcessor dataCollectorAttachmentProcessorInstance = null; try { - dataCollectorAttachmentProcessorInstance = TestPluginManager.CreateTestExtension(attachmentProcessorType); + dataCollectorAttachmentProcessorInstance = TestPluginManager.CreateTestExtension(attachmentProcessorType); } catch (Exception ex) { @@ -56,7 +56,7 @@ public IReadOnlyDictionary Create(Inv if (dataCollectorAttachmentProcessorInstance != null && !datacollectorsAttachmentsProcessors.ContainsKey(attachmentProcessorType.AssemblyQualifiedName)) { - datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); + datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); if (invokedDataCollector.Uri.AbsoluteUri == CoverageUri.AbsoluteUri) { @@ -74,17 +74,17 @@ public IReadOnlyDictionary Create(Inv if (addCodeCoverageAttachmentProcessors) { - datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); + datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); } - var finalDatacollectorsAttachmentsProcessors = new Dictionary(); + var finalDatacollectorsAttachmentsProcessors = new Dictionary(); foreach (var attachementProcessor in datacollectorsAttachmentsProcessors) { EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: valid data collector attachment processor found: '{attachementProcessor.Value.Item2.GetType().AssemblyQualifiedName}'"); finalDatacollectorsAttachmentsProcessors.Add(attachementProcessor.Value.Item1, attachementProcessor.Value.Item2); } - return new ReadOnlyDictionary(finalDatacollectorsAttachmentsProcessors); + return new ReadOnlyDictionary(finalDatacollectorsAttachmentsProcessors); } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs index 3e8c23cb7e..63073568c6 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs @@ -104,7 +104,7 @@ private async Task> ProcessAttachmentsAsync(string run var dataCollectionRunSettings = XmlRunSettingsUtilities.GetDataCollectionRunSettings(runSettingsXml); var logger = CreateMessageLogger(eventsHandler); - IReadOnlyDictionary dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray()); + IReadOnlyDictionary dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray()); for (int i = 0; i < dataCollectorAttachmentsProcessors.Count; i++) { var dataCollectorAttachmentsProcessor = dataCollectorAttachmentsProcessors.ElementAt(i); diff --git a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs new file mode 100644 index 0000000000..d81738dab3 --- /dev/null +++ b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + + /// + /// Interface for data collectors add-ins that choose to reprocess generated attachments + /// + public interface IConfigurableDataCollectorAttachmentProcessor + { + /// + /// Gets the attachments Uris, which are handled by attachment processor + /// + IEnumerable GetExtensionUris(); + + /// + /// Indicates whether attachment processor is supporting incremental processing of attachments + /// + /// + /// `SupportsIncrementalProcessing` should indicate if attachment processor is supporting incremental processing of attachments. It means that `ProcessAttachmentSetsAsync` should be [associative](https://en.wikipedia.org/wiki/Associative_property). + /// By default `SupportsIncrementalProcessing` should be `False`, unless processing can take longer time and it's beneficial to start the process as soon as possible. + /// + /// If `SupportsIncrementalProcessing` is `True` Test Platform may try to speed up whole process by reprocessing data collector attachments as soon as possible when any two test executions are done.For example let's assume we have 5 test executions which are generating 5 data collector attachments: `a1`, `a2`, `a3`, `a4` and `a5`. Test platform could perform invocations: + /// * `var result1 = await ProcessAttachmentSetsAsync([a1, a2, a3], ...);` when first 3 executions are done + /// * `var result2 = await ProcessAttachmentSetsAsync(result1.Concat([a4]), ...);` when 4th execution is done + /// * `var finalResult = await ProcessAttachmentSetsAsync(result2.Concat([a5]), ...);` when last test execution is done + /// + /// If `SupportsIncrementalProcessing` is `False` then Test Platform will wait for all test executions to finish and call `ProcessAttachmentSetsAsync` only once: + /// * `var finalResult = await ProcessAttachmentSetsAsync([a1, a2, a3, a4, a5], ...);` + /// + bool SupportsIncrementalProcessing { get; } + + /// + /// Reprocess attachments generated by independent test executions + /// + /// Configuration of the attachment processor. Will be the same as the data collector that registers it. + /// Attachments to be processed + /// Progress reporter. Accepts integers from 0 to 100 + /// Message logger + /// Cancellation token + /// Attachments after reprocessing + Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs index 37565e3d89..8425477b5f 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs @@ -13,6 +13,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection /// /// Interface for data collectors add-ins that choose to reprocess generated attachments /// + [Obsolete("Interface is deprecated. Please use IConfigurableDataCollectorAttachmentProcessor instead")] public interface IDataCollectorAttachmentProcessor { /// @@ -40,12 +41,11 @@ public interface IDataCollectorAttachmentProcessor /// /// Reprocess attachments generated by independent test executions /// - /// Configuration of the attachment processor. Will be the same as the data collector that registers it. /// Attachments to be processed /// Progress reporter. Accepts integers from 0 to 100 /// Message logger /// Cancellation token /// Attachments after reprocessing - Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); + Task> ProcessAttachmentSetsAsync(ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt index 7929196a23..be5012453d 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt @@ -68,8 +68,12 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectionSi Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectionSink.SendData(Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectionContext dataCollectionContext, string key, string value) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.GetExtensionUris() -> System.Collections.Generic.IEnumerable -Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Xml.XmlElement configurationElement, System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.SupportsIncrementalProcessing.get -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor.GetExtensionUris() -> System.Collections.Generic.IEnumerable +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Xml.XmlElement configurationElement, System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor.SupportsIncrementalProcessing.get -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments.GetExtensionUri() -> System.Uri Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments.HandleDataCollectionAttachmentSets(System.Collections.Generic.ICollection dataCollectionAttachments) -> System.Collections.Generic.ICollection diff --git a/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs b/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs index e4a2fd4210..5bf8924691 100644 --- a/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs +++ b/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs @@ -17,7 +17,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Utilities using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; - public class CodeCoverageDataAttachmentsHandler : IDataCollectorAttachmentProcessor + public class CodeCoverageDataAttachmentsHandler : IConfigurableDataCollectorAttachmentProcessor { private const string CoverageUri = "datacollector://microsoft/CodeCoverage/2.0"; private const string CoverageFileExtension = ".coverage"; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 252e5a3f77..6635ca2330 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -157,6 +157,17 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) Assert.AreEqual(2, fileContent.Distinct().Count()); + var dataCollectorsLogs = Directory.GetFiles(resultsDir, "*.datacollector.*", SearchOption.TopDirectoryOnly); + Assert.AreEqual(2, dataCollectorsLogs.Distinct().Count()); + foreach (var dataCollectorLogFile in dataCollectorsLogs) + { + string dataCollectorLog = File.ReadAllText(dataCollectorLogFile); + Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Unable to get extension for type 'AttachmentProcessorDataCollector.SampleDataCollector'")); + Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found 'AttachmentProcessorDataCollector.SampleDataCollectorV1' version '1'")); + Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found 'AttachmentProcessorDataCollector.SampleDataCollectorV2' version '2'")); + Assert.IsTrue(dataCollectorLog.Contains("TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector.SampleDataCollectorV1, AttachmentProcessorDataCollector, Version=15.0.0.0, Culture=neutral, PublicKeyToken=null'; keeping the first one 'AttachmentProcessorDataCollector.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=15.0.0.0, Culture=neutral, PublicKeyToken=null'.")); + } + TryRemoveDirectory(resultsDir); } diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs index 3f623486e7..0e3a1a0bd2 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs @@ -335,7 +335,7 @@ public override void Initialize( } } - public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor + public class DataCollectorAttachmentProcessor : IConfigurableDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index a10e739de0..abdf0647b5 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -180,7 +180,7 @@ public override void Initialize( } } - public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor + public class DataCollectorAttachmentProcessor : IConfigurableDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); @@ -195,7 +195,7 @@ public Task> ProcessAttachmentSetsAsync(XmlElement co } } - public class DataCollectorAttachmentProcessor2 : IDataCollectorAttachmentProcessor + public class DataCollectorAttachmentProcessor2 : IConfigurableDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs index 4bc8a806bc..e055ba9b66 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs @@ -31,8 +31,8 @@ public class TestRunAttachmentsProcessingManagerTests private readonly Mock mockRequestData; private readonly Mock mockMetricsCollection; private readonly Mock mockEventSource; - private readonly Mock mockAttachmentHandler1; - private readonly Mock mockAttachmentHandler2; + private readonly Mock mockAttachmentHandler1; + private readonly Mock mockAttachmentHandler2; private readonly Mock mockDataCollectorAttachmentsProcessorsFactory; private readonly Mock mockEventsHandler; private readonly TestRunAttachmentsProcessingManager manager; @@ -45,15 +45,15 @@ public TestRunAttachmentsProcessingManagerTests() mockRequestData.Setup(r => r.MetricsCollection).Returns(mockMetricsCollection.Object); mockEventSource = new Mock(); - mockAttachmentHandler1 = new Mock(); - mockAttachmentHandler2 = new Mock(); + mockAttachmentHandler1 = new Mock(); + mockAttachmentHandler2 = new Mock(); mockEventsHandler = new Mock(); mockDataCollectorAttachmentsProcessorsFactory = new Mock(); mockAttachmentHandler1.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); mockAttachmentHandler2.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri2) }); mockDataCollectorAttachmentsProcessorsFactory.Setup(p => p.Create(It.IsAny())) - .Returns(new ReadOnlyDictionary(new Dictionary() + .Returns(new ReadOnlyDictionary(new Dictionary() { { "friendlyNameA", mockAttachmentHandler1.Object } , { "friendlyNameB" ,mockAttachmentHandler2.Object } })); diff --git a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs index 5a82c1999c..6273665dc9 100644 --- a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs +++ b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs @@ -8,7 +8,6 @@ namespace AttachmentProcessorDataCollector using System.Collections.ObjectModel; using System.IO; using System.Linq; - using System.Net.Mail; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -20,7 +19,11 @@ namespace AttachmentProcessorDataCollector [DataCollectorFriendlyName("SampleDataCollector")] [DataCollectorTypeUri("my://sample/datacollector")] [DataCollectorAttachmentProcessor(typeof(SampleDataCollectorAttachmentProcessor))] - public class SampleDataCollector : DataCollector + public class SampleDataCollectorV2 : SampleDataCollectorV1 { } + + [DataCollectorFriendlyName("SampleDataCollector")] + [DataCollectorTypeUri("my://sample/datacollector")] + public class SampleDataCollectorV1 : DataCollector { private DataCollectionSink dataCollectionSink; private DataCollectionEnvironmentContext context; @@ -47,7 +50,7 @@ private void SessionEnded_Handler(object sender, SessionEndEventArgs e) } } - public class SampleDataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor + public class SampleDataCollectorAttachmentProcessor : IConfigurableDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => true; diff --git a/test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs b/test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs new file mode 100644 index 0000000000..0615ef773e --- /dev/null +++ b/test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AttachmentProcessorDataCollector; +using Microsoft.VisualStudio.TestPlatform; + +[assembly: TestExtensionTypes(typeof(SampleDataCollectorV1))] +[assembly: TestExtensionTypesV2("AttachmentProcessorDataCollector.SampleDataCollector", 0)] +[assembly: TestExtensionTypesV2("AttachmentProcessorDataCollector.SampleDataCollectorV1", 1)] +[assembly: TestExtensionTypesV2("AttachmentProcessorDataCollector.SampleDataCollectorV2", 2, "unused")] + +namespace Microsoft.VisualStudio.TestPlatform +{ + using System; + + [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = false)] + internal sealed class TestExtensionTypesAttribute : Attribute + { + public TestExtensionTypesAttribute(params Type[] types) + { + this.Types = types; + } + + public Type[] Types { get; } + } + + [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = true)] + internal sealed class TestExtensionTypesV2Attribute : Attribute + { + public int Version { get; set; } + public string FullName { get; } + + public TestExtensionTypesV2Attribute(string fullName, int version) + { + this.FullName = fullName; + this.Version =version; + } + + public TestExtensionTypesV2Attribute(string fullName, int version, string unused) + { + this.FullName = fullName; + this.Version =version; + } + } +} \ No newline at end of file diff --git a/test/datacollector.UnitTests/DataCollectionManagerTests.cs b/test/datacollector.UnitTests/DataCollectionManagerTests.cs index 0b017379c1..29e1f3879e 100644 --- a/test/datacollector.UnitTests/DataCollectionManagerTests.cs +++ b/test/datacollector.UnitTests/DataCollectionManagerTests.cs @@ -598,7 +598,7 @@ public abstract class CodeCoverageDataCollector : DataCollector { } - public class AttachmentProcessorDataCollector2 : IDataCollectorAttachmentProcessor + public class AttachmentProcessorDataCollector2 : IConfigurableDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); From 4fd8447d61a148ff1160cadede5950a0d5a806d7 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 6 Dec 2021 10:21:06 +0100 Subject: [PATCH 10/47] another round on the metadata discovery --- .../TestPluginDiscoverer.cs | 2 +- .../Utilities/MetadataReaderHelper.cs | 493 +++--------------- .../DataCollectionTests.cs | 5 +- .../SampleDataCollector.cs | 10 +- .../TestExtensionTypesAttribute.cs | 25 +- 5 files changed, 99 insertions(+), 436 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index 7207b90208..542f3ab5b7 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -168,7 +168,7 @@ private void GetTestExtensionsFromAssembly(Assembly ass try { MetadataReaderExtensionsHelper extensionHelper = new MetadataReaderExtensionsHelper(); - var discoveredExtensions = extensionHelper.DiscoverTestPlatformExtensionVersionAttributeExtensions2(assembly, filePath); + var discoveredExtensions = extensionHelper.DiscoverTestExtensionTypesV2Attribute(assembly, filePath); if (discoveredExtensions?.Length > 0) { types.AddRange(discoveredExtensions); diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index 263fdf2f0d..0b78e0c851 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -9,25 +9,16 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities using System.Linq; using System.Reflection; using System.Reflection.Metadata; - using System.Reflection.Metadata.Ecma335; using System.Reflection.PortableExecutable; using System.Text; using Microsoft.VisualStudio.TestPlatform.ObjectModel; - internal class MetadataReaderExtensionsHelper { - private static string TestPlatformExtensionVersionAttribute = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypeAttribute"; private static string TestExtensionTypesAttributeV2 = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypesV2Attribute"; - private static string[] MethodsDefinition = new string[] { ".ctor", "get_Version" }; private static Type[] EmptyTypeArray = new Type[0]; - public MetadataReaderExtensionsHelper() - { - - } - - public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions2(Assembly assembly, string assemblyFilePath) + public Type[] DiscoverTestExtensionTypesV2Attribute(Assembly assembly, string assemblyFilePath) { EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); @@ -37,6 +28,7 @@ public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions2(Assembly { MetadataReader metadataReader = reader.GetMetadataReader(); + // Search for the custom attribute TestExtensionTypesAttributeV2 - ECMA-335 II.22.10 CustomAttribute : 0x0C foreach (var customAttributeHandle in metadataReader.CustomAttributes) { string attributeFullName = null; @@ -44,74 +36,98 @@ public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions2(Assembly { if (customAttributeHandle.IsNil) { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute (customAttributeHandle.IsNil)"); continue; } - if (!GetAttributeTypeAndConstructor(metadataReader, customAttributeHandle, out EntityHandle attributeType)) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute (GetAttributeTypeAndConstructor)"); - continue; - } - - if (!GetAttributeTypeNamespaceAndName(metadataReader, attributeType, out StringHandle namespaceHandle, out StringHandle nameHandle)) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute (GetAttributeTypeNamespaceAndName)"); - continue; - } + // Read custom attribute metadata row + var customAttribute = metadataReader.GetCustomAttribute(customAttributeHandle); - attributeFullName = $"{metadataReader.GetString(namespaceHandle)}.{metadataReader.GetString(nameHandle)}"; - if (attributeFullName != TestExtensionTypesAttributeV2) + // We expect that the attribute is defined inside current assembly by the extension owner + // and so ctor should point to the MethodDefinition table - ECMA-335 II.22.26 MethodDef : 0x06 + // and parent scope should be the current assembly [assembly:...] - ECMA-335 II.22.2 Assembly : 0x20 + if (customAttribute.Constructor.Kind != HandleKind.MethodDefinition || customAttribute.Parent.Kind != HandleKind.AssemblyDefinition) { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid attribute '{attributeFullName}'"); continue; } - var customAttribute = metadataReader.GetCustomAttribute(customAttributeHandle); - EntityHandle ctorHandle = customAttribute.Constructor; - BlobHandle signature; - switch (ctorHandle.Kind) - { - case HandleKind.MemberReference: - signature = metadataReader.GetMemberReference((MemberReferenceHandle)ctorHandle).Signature; - break; - case HandleKind.MethodDefinition: - signature = metadataReader.GetMethodDefinition((MethodDefinitionHandle)ctorHandle).Signature; - break; - default: - Console.WriteLine($"MetadataReaderExtensionsHelper: Potentially invalid IL for the attribute '{attributeFullName}'"); - continue; - } + // Read MethodDef metadata row + var methodDefinition = metadataReader.GetMethodDefinition((MethodDefinitionHandle)customAttribute.Constructor); - BlobReader signatureReader = metadataReader.GetBlobReader(signature); - BlobReader valueReader = metadataReader.GetBlobReader(customAttribute.Value); - const ushort Prolog = 1; // two-byte "prolog" defined by ECMA-335 (II.23.3) to be at the beginning of attribute value blobs - UInt16 prolog = valueReader.ReadUInt16(); - if (prolog != Prolog) + // Check that name is .ctor + if (metadataReader.GetString(methodDefinition.Name) != ".ctor") { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid blob attribute prolog '{prolog}'"); continue; } - string fullName = valueReader.ReadSerializedString(); - int version = valueReader.ReadInt32(); + // Get the custom attribute TypeDef handle + var typeDefinitionHandle = methodDefinition.GetDeclaringType(); + + // Read TypeDef metadata row + var typeDef = metadataReader.GetTypeDefinition(typeDefinitionHandle); - try + // Check the attribute type full name + attributeFullName = $"{metadataReader.GetString(typeDef.Namespace)}.{metadataReader.GetString(typeDef.Name)}"; + if (attributeFullName == TestExtensionTypesAttributeV2) { - var extensionType = assembly.GetType(fullName); - if (extensionType is null) + // We don't do any signature verification to allow future possibility to add new parameter in a back compat way. + // Get signature blob using methodDefinition.Signature index into Blob heap - ECMA-335 II.22.26 MethodDef : 0x06, 'Signature' column + // BlobReader signatureReader = metadataReader.GetBlobReader(methodDefinition.Signature); + // var decoder = new SignatureDecoder(new SignatureDecoder(), metadataReader, genericContext: null); + // var ctorDecodedSignature = decoder.DecodeMethodSignature(ref signatureReader); + // Log the signature for analysis purpose + // EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Find possible good attribute candidate '{attributeFullName}' with ctor signature: '{ctorDecodedSignature.ReturnType}" + + // $" ctor ({(ctorDecodedSignature.ParameterTypes.Length > 0 ? ctorDecodedSignature.ParameterTypes.Aggregate((a, b) => $"{a},{b}").Trim(',') : "(")})'"); + + // Read the ctor signature values - ECMA-335 II.23.3 Custom attributes + BlobReader valueReader = metadataReader.GetBlobReader(customAttribute.Value); + // Verify the prolog + if (valueReader.ReadUInt16() == 1) { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Unable to get extension for type '{fullName}'"); - continue; - } + // Expected ctor shape ctor(string,string,System.Type,Int32) + // [assembly: TestExtensionTypesV2(ExtensionMetadata.ExtensionType, ExtensionMetadata.Uri, typeof(ExtensionImplementation), 1)] + + // If the parameter kind is string, (middle line in above diagram) then the blob contains + // a SerString – a PackedLen count of bytes, followed by the UTF8 characters.If the + // string is null, its PackedLen has the value 0xFF(with no following characters).If + // the string is empty(“”), then PackedLen has the value 0x00(with no following + // characters). + string extension = valueReader.ReadSerializedString(); + string extensionIdentifier = valueReader.ReadSerializedString(); + + // If the parameter kind is System.Type, (also, the middle line in above diagram) its + // value is stored as a SerString(as defined in the previous paragraph), representing its + // canonical name. The canonical name is its full type name, followed optionally by + // the assembly where it is defined, its version, culture and public-key-token.If the + // assembly name is omitted, the CLI looks first in the current assembly, and then in + // the system library(mscorlib); in these two special cases, it is permitted to omit the + // assembly-name, version, culture and public-key-token. + string extensionImplementation = valueReader.ReadSerializedString(); + + // If the parameter kind is simple(first line in the above diagram) (bool, char, float32, + // float64, int8, int16, int32, int64, unsigned int8, unsigned int16, unsigned int32 or + // unsigned int64) then the 'blob' contains its binary value(Val). (A bool is a single + // byte with value 0(false) or 1(true); char is a two-byte Unicode character; and the + // others have their obvious meaning.) This pattern is also used if the parameter kind is + // an enum -- simply store the value of the enum's underlying integer type. + int version = valueReader.ReadInt32(); + try + { + var extensionType = assembly.GetType(extensionImplementation); + if (extensionType is null) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Unable to get extension type for '{extensionImplementation}'"); + continue; + } - if (extensions is null) extensions = new List>(); - extensions.Add(Tuple.Create(version, extensionType)); - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid extension found '{extensionType}' version '{version}'"); - } - catch (Exception ex) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Failure during type creation, extension full name: '{fullName}'\n{FormatException(ex)}"); + if (extensions is null) extensions = new List>(); + extensions.Add(Tuple.Create(version, extensionType)); + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid extension found: extension type '{extension}' identifier '{extensionIdentifier}' implementation '{extensionType}' version '{version}'"); + } + catch (Exception ex) + { + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Failure during type creation, extension full name: '{extensionImplementation}'\n{FormatException(ex)}"); + } + } } } catch (Exception ex) @@ -121,108 +137,6 @@ public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions2(Assembly } } - var finalExtensions = extensions?.OrderByDescending(t => t.Item1).Select(t => t.Item2).ToArray() ?? EmptyTypeArray; - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Found {extensions?.Count ?? 0} extensions"); - return finalExtensions; - } - - public Type[] DiscoverTestPlatformExtensionVersionAttributeExtensions(Assembly assembly, string assemblyFilePath) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); - - using (var stream = new FileStream(assemblyFilePath, FileMode.Open, FileAccess.Read)) - using (var reader = new PEReader(stream, PEStreamOptions.Default)) - { - MetadataReader metadataReader = reader.GetMetadataReader(MetadataReaderOptions.Default); - - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Search '{TestPlatformExtensionVersionAttribute}' definition inside current assembly '{assembly.FullName}'"); - Type testPlatformExtensionVersionAttributeType = SearchExtensionAttribute(assembly, metadataReader); - - if (testPlatformExtensionVersionAttributeType is null) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: '{TestPlatformExtensionVersionAttribute}' attribute not found inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); - return EmptyTypeArray; - } - - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Inspect types for extensions"); - return InspectTypes(assembly, metadataReader, testPlatformExtensionVersionAttributeType); - } - } - - private Type[] InspectTypes(Assembly assembly, MetadataReader metadataReader, Type testPlatformExtensionVersionAttributeType) - { - List> extensions = null; - - foreach (var handle in metadataReader.TypeDefinitions) - { - if (handle.IsNil) - { - continue; - } - - var typeDef = metadataReader.GetTypeDefinition(handle); - var typeName = metadataReader.GetString(typeDef.Name); - var typeNameSpace = metadataReader.GetString(typeDef.Namespace); - string fullName = $"{typeNameSpace}.{typeName}"; - - if (fullName == testPlatformExtensionVersionAttributeType.FullName) - { - continue; - } - - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Analyze TypeDefinitionHandle '{fullName}'"); - - var customAttributes = metadataReader.GetCustomAttributes(handle); - if (customAttributes.Count == 0) - { - continue; - } - - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Analyze attributes for type '{fullName}'"); - foreach (var attributeHandle in customAttributes) - { - if (!attributeHandle.IsNil) - { - if (!GetAttributeTypeAndConstructor(metadataReader, attributeHandle, out EntityHandle attributeType)) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute found for '{fullName}' (GetAttributeTypeAndConstructor)"); - continue; - } - - if (!GetAttributeTypeNamespaceAndName(metadataReader, attributeType, out StringHandle namespaceHandle, out StringHandle nameHandle)) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute found for '{fullName}' (GetAttributeTypeNamespaceAndName)"); - continue; - } - - string attributeFullName = $"{metadataReader.GetString(namespaceHandle)}.{metadataReader.GetString(nameHandle)}"; - if (attributeFullName != testPlatformExtensionVersionAttributeType.FullName) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Invalid custom attribute found for '{fullName}' wrong type: {attributeFullName}"); - continue; - } - - try - { - var extensionType = assembly.GetType(fullName); - var version = GetVersion(metadataReader, metadataReader.GetCustomAttribute(attributeHandle)); - if (version == int.MinValue) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Unable to read the version for '{fullName}'"); - } - - if (extensions is null) extensions = new List>(); - extensions.Add(Tuple.Create(version, extensionType)); - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid extension found '{extensionType}' version '{version}'"); - } - catch (Exception ex) - { - EqtTrace.Verbose($"Failed to create extension for type '{fullName}'\n{FormatException(ex)}"); - } - } - } - } - return extensions?.OrderByDescending(t => t.Item1).Select(t => t.Item2).ToArray() ?? EmptyTypeArray; } @@ -238,258 +152,5 @@ private string FormatException(Exception ex) return log.ToString(); } - - // https://github.com/dotnet/runtime/blob/main/src/libraries/System.Diagnostics.FileVersionInfo/src/System/Diagnostics/FileVersionInfo.Unix.cs#L288 - private int GetVersion(MetadataReader metadataReader, CustomAttribute attributeHandle) - { - EntityHandle ctorHandle = attributeHandle.Constructor; - BlobHandle signature; - switch (ctorHandle.Kind) - { - case HandleKind.MemberReference: - signature = metadataReader.GetMemberReference((MemberReferenceHandle)ctorHandle).Signature; - break; - case HandleKind.MethodDefinition: - signature = metadataReader.GetMethodDefinition((MethodDefinitionHandle)ctorHandle).Signature; - break; - default: - // Unusual case, potentially invalid IL - return int.MinValue; - } - - BlobReader signatureReader = metadataReader.GetBlobReader(signature); - BlobReader valueReader = metadataReader.GetBlobReader(attributeHandle.Value); - const ushort Prolog = 1; // two-byte "prolog" defined by ECMA-335 (II.23.3) to be at the beginning of attribute value blobs - if (valueReader.ReadUInt16() == Prolog) - { - SignatureHeader header = signatureReader.ReadSignatureHeader(); - int parameterCount; - if (header.Kind == SignatureKind.Method && // attr ctor must be a method - !header.IsGeneric && // attr ctor must be non-generic - signatureReader.TryReadCompressedInteger(out parameterCount) && // read parameter count - parameterCount == 1 && // attr ctor must have 1 parameter - signatureReader.ReadSignatureTypeCode() == SignatureTypeCode.Void && // attr ctor return type must be void - signatureReader.ReadSignatureTypeCode() == SignatureTypeCode.Int32) // attr ctor first parameter must be int32 - { - return valueReader.ReadInt32(); - } - } - - return int.MinValue; - } - - private Type SearchExtensionAttribute(Assembly assembly, MetadataReader metadataReader) - { - foreach (TypeDefinitionHandle typeDefHandle in metadataReader.TypeDefinitions) - { - try - { - if (typeDefHandle.IsNil) - { - continue; - } - - var typeDef = metadataReader.GetTypeDefinition(typeDefHandle); - var typeName = metadataReader.GetString(typeDef.Name); - var @namespace = metadataReader.GetString(typeDef.Namespace); - var fullName = $"{@namespace}.{typeName}"; - - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Analyze TypeDefinitionHandle '{fullName}'"); - - // Check the name - if (fullName == TestPlatformExtensionVersionAttribute && (typeDef.Attributes & TypeAttributes.Sealed) == TypeAttributes.Sealed) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Valid type name found '{fullName}'"); - - // Check it inherits from System.Attribute - if (typeDef.BaseType.Kind != HandleKind.TypeReference) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Type '{fullName}' doesn't inherit from System.Attribute (typeDef.BaseType.Kind != HandleKind.TypeReference)"); - continue; - } - - var baseTypeReferenceHandle = metadataReader.GetTypeReference((TypeReferenceHandle)typeDef.BaseType); - if ($"{metadataReader.GetString(baseTypeReferenceHandle.Namespace)}.{metadataReader.GetString(baseTypeReferenceHandle.Name)}" != "System.Attribute") - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Type '{fullName}' doesn't inherit from System.Attribute (baseTypeFullName != 'System.Attribute')"); - continue; - } - - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Verify members definition for type '{fullName}'"); - - int isGoodCandidate = 0; - foreach (var method in typeDef.GetMethods()) - { - var methodDef = metadataReader.GetMethodDefinition(method); - var methodName = metadataReader.GetString(methodDef.Name); - if (MethodsDefinition.Contains(methodName)) - { - // Verify the ctor signature, int32 for the version number. - if (methodName == ".ctor" && - (methodDef.Attributes & MethodAttributes.Public) == MethodAttributes.Public && - (methodDef.Attributes & MethodAttributes.SpecialName) == MethodAttributes.SpecialName - ) - { - var sigReader = metadataReader.GetBlobReader(methodDef.Signature); - var decoder = new SignatureDecoder(new TestPlatformExtensionVersionAttributeSignatureDecoder(), metadataReader, genericContext: null); - var methodSignature = decoder.DecodeMethodSignature(ref sigReader); - if (methodSignature.Header.IsInstance && - methodSignature.ReturnType == "void" && - methodSignature.ParameterTypes != null && - methodSignature.ParameterTypes.Length == 1 && - methodSignature.ParameterTypes[0] == "int32" && - methodSignature.GenericParameterCount == 0 && - !methodSignature.Header.IsGeneric) - { - isGoodCandidate++; - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Found '.ctor' for '{fullName}'"); - } - } - - if (methodName == "get_Version" && - (methodDef.Attributes & MethodAttributes.Public) == MethodAttributes.Public) - { - var sigReader = metadataReader.GetBlobReader(methodDef.Signature); - var decoder = new SignatureDecoder(new TestPlatformExtensionVersionAttributeSignatureDecoder(), metadataReader, genericContext: null); - var methodSignature = decoder.DecodeMethodSignature(ref sigReader); - if (methodSignature.Header.IsInstance && - methodSignature.ReturnType == "int32" && - methodSignature.ParameterTypes != null && - methodSignature.ParameterTypes.Length == 0 && - methodSignature.GenericParameterCount == 0 && - !methodSignature.Header.IsGeneric) - { - isGoodCandidate++; - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Found 'get_Version' for '{fullName}'"); - } - } - } - } - - // If all characteristics were meet we'll use this attribute to find extensions type. - if (isGoodCandidate != 2) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Members definition verification for type '{fullName}' failed"); - continue; - } - - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Members definition verification for type '{fullName}' succeded"); - return assembly.GetType(fullName); - } - } - catch (Exception ex) - { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Exception during TypeDefinitions analysis\n{ex}"); - } - } - - return null; - } - - // https://github.com/dotnet/runtime/blob/6cf529168a8dcdfb158738d46be40b1867fd1bfa/src/coreclr/tools/Common/TypeSystem/Ecma/MetadataExtensions.cs#L173 - private bool GetAttributeTypeNamespaceAndName(MetadataReader metadataReader, EntityHandle attributeType, - out StringHandle namespaceHandle, out StringHandle nameHandle) - { - namespaceHandle = default; - nameHandle = default; - - if (attributeType.Kind == HandleKind.TypeReference) - { - TypeReference typeRefRow = metadataReader.GetTypeReference((TypeReferenceHandle)attributeType); - HandleKind handleType = typeRefRow.ResolutionScope.Kind; - - // Nested type? - if (handleType == HandleKind.TypeReference || handleType == HandleKind.TypeDefinition) - return false; - - nameHandle = typeRefRow.Name; - namespaceHandle = typeRefRow.Namespace; - return true; - } - else if (attributeType.Kind == HandleKind.TypeDefinition) - { - var def = metadataReader.GetTypeDefinition((TypeDefinitionHandle)attributeType); - - // Nested type? - if (IsNested(def.Attributes)) - return false; - - nameHandle = def.Name; - namespaceHandle = def.Namespace; - return true; - } - else - { - // unsupported metadata - return false; - } - } - - // https://github.com/dotnet/runtime/blob/6cf529168a8dcdfb158738d46be40b1867fd1bfa/src/coreclr/tools/Common/TypeSystem/Ecma/MetadataExtensions.cs#L150 - private bool GetAttributeTypeAndConstructor(MetadataReader metadataReader, CustomAttributeHandle attributeHandle, out EntityHandle attributeType) - { - var attributeCtor = metadataReader.GetCustomAttribute(attributeHandle).Constructor; - - if (attributeCtor.Kind == HandleKind.MemberReference) - { - attributeType = metadataReader.GetMemberReference((MemberReferenceHandle)attributeCtor).Parent; - return true; - } - else if (attributeCtor.Kind == HandleKind.MethodDefinition) - { - attributeType = metadataReader.GetMethodDefinition((MethodDefinitionHandle)attributeCtor).GetDeclaringType(); - return true; - } - else - { - // invalid metadata - attributeType = default; - return false; - } - } - - private bool IsNested(TypeAttributes flags) - { - return (flags & (TypeAttributes)0x00000006) != 0; - } - - class TestPlatformExtensionVersionAttributeSignatureDecoder : ISignatureTypeProvider - { - public string GetArrayType(string elementType, ArrayShape shape) - => string.Empty; - public string GetByReferenceType(string elementType) - => string.Empty; - public string GetFunctionPointerType(MethodSignature signature) - => string.Empty; - public string GetGenericInstantiation(string genericType, System.Collections.Immutable.ImmutableArray typeArguments) - => string.Empty; - public string GetGenericMethodParameter(object genericContext, int index) - => string.Empty; - public string GetGenericTypeParameter(object genericContext, int index) - => string.Empty; - public string GetModifiedType(string modifier, string unmodifiedType, bool isRequired) - => string.Empty; - public string GetPinnedType(string elementType) - => string.Empty; - public string GetPointerType(string elementType) - => string.Empty; - public string GetSZArrayType(string elementType) - => string.Empty; - public string GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) - => string.Empty; - public string GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) - => string.Empty; - public string GetTypeFromSpecification(MetadataReader reader, object genericContext, TypeSpecificationHandle handle, byte rawTypeKind) - => string.Empty; - public string GetPrimitiveType(PrimitiveTypeCode typeCode) - { - switch (typeCode) - { - case PrimitiveTypeCode.Int32: return "int32"; - case PrimitiveTypeCode.Void: return "void"; - default: return ""; - } - } - } } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 6635ca2330..f6f3f96544 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -162,9 +162,8 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) foreach (var dataCollectorLogFile in dataCollectorsLogs) { string dataCollectorLog = File.ReadAllText(dataCollectorLogFile); - Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Unable to get extension for type 'AttachmentProcessorDataCollector.SampleDataCollector'")); - Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found 'AttachmentProcessorDataCollector.SampleDataCollectorV1' version '1'")); - Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found 'AttachmentProcessorDataCollector.SampleDataCollectorV2' version '2'")); + Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV1' version '1'")); + Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV2' version '2'")); Assert.IsTrue(dataCollectorLog.Contains("TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector.SampleDataCollectorV1, AttachmentProcessorDataCollector, Version=15.0.0.0, Culture=neutral, PublicKeyToken=null'; keeping the first one 'AttachmentProcessorDataCollector.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=15.0.0.0, Culture=neutral, PublicKeyToken=null'.")); } diff --git a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs index 6273665dc9..7ac6320b27 100644 --- a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs +++ b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs @@ -16,13 +16,19 @@ namespace AttachmentProcessorDataCollector using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + internal class ExtensionInfo + { + public const string ExtensionType = "DataCollector"; + public const string ExtensionIdentifier = "my://sample/datacollector"; + } + [DataCollectorFriendlyName("SampleDataCollector")] - [DataCollectorTypeUri("my://sample/datacollector")] + [DataCollectorTypeUri(ExtensionInfo.ExtensionIdentifier)] [DataCollectorAttachmentProcessor(typeof(SampleDataCollectorAttachmentProcessor))] public class SampleDataCollectorV2 : SampleDataCollectorV1 { } [DataCollectorFriendlyName("SampleDataCollector")] - [DataCollectorTypeUri("my://sample/datacollector")] + [DataCollectorTypeUri(ExtensionInfo.ExtensionIdentifier)] public class SampleDataCollectorV1 : DataCollector { private DataCollectionSink dataCollectionSink; diff --git a/test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs b/test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs index 0615ef773e..b38196a844 100644 --- a/test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs +++ b/test/TestAssets/AttachmentProcessorDataCollector/TestExtensionTypesAttribute.cs @@ -5,9 +5,8 @@ using Microsoft.VisualStudio.TestPlatform; [assembly: TestExtensionTypes(typeof(SampleDataCollectorV1))] -[assembly: TestExtensionTypesV2("AttachmentProcessorDataCollector.SampleDataCollector", 0)] -[assembly: TestExtensionTypesV2("AttachmentProcessorDataCollector.SampleDataCollectorV1", 1)] -[assembly: TestExtensionTypesV2("AttachmentProcessorDataCollector.SampleDataCollectorV2", 2, "unused")] +[assembly: TestExtensionTypesV2(ExtensionInfo.ExtensionType, ExtensionInfo.ExtensionIdentifier, typeof(SampleDataCollectorV1), 1, "futureUnused")] +[assembly: TestExtensionTypesV2(ExtensionInfo.ExtensionType, ExtensionInfo.ExtensionIdentifier, typeof(SampleDataCollectorV2), 2)] namespace Microsoft.VisualStudio.TestPlatform { @@ -27,19 +26,17 @@ public TestExtensionTypesAttribute(params Type[] types) [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = true)] internal sealed class TestExtensionTypesV2Attribute : Attribute { - public int Version { get; set; } - public string FullName { get; } + public string ExtensionType { get; } + public string ExtensionIdentifier { get; } + public Type ExtensionImplementation { get; } + public int Version { get; } - public TestExtensionTypesV2Attribute(string fullName, int version) + public TestExtensionTypesV2Attribute(string extensionType, string extensionIdentifier, Type extensionImplementation, int version, string unused = null) { - this.FullName = fullName; - this.Version =version; - } - - public TestExtensionTypesV2Attribute(string fullName, int version, string unused) - { - this.FullName = fullName; - this.Version =version; + ExtensionType = extensionType; + ExtensionIdentifier = extensionIdentifier; + ExtensionImplementation = extensionImplementation; + Version = version; } } } \ No newline at end of file From dfa45b47b0902b24b9d98e46d8cd1a5a3d01c700 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 6 Dec 2021 10:22:52 +0100 Subject: [PATCH 11/47] nit refactor test --- .../AttachmentProcessorDataCollector/SampleDataCollector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs index 7ac6320b27..2fb903c5db 100644 --- a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs +++ b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs @@ -61,7 +61,7 @@ public class SampleDataCollectorAttachmentProcessor : IConfigurableDataCollector public bool SupportsIncrementalProcessing => true; public IEnumerable GetExtensionUris() - => new List() { new Uri("my://sample/datacollector") }; + => new List() { new Uri(ExtensionInfo.ExtensionIdentifier) }; public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) { From 2eb7b18b762355390c65272bf5f3a04ecc6c42d0 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 6 Dec 2021 18:16:00 +0100 Subject: [PATCH 12/47] cleanup --- .../ExtensionFramework/TestPluginDiscoverer.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index 542f3ab5b7..7f0395b605 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -25,6 +25,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework internal class TestPluginDiscoverer { private static HashSet UnloadableFiles = new HashSet(); + private readonly MetadataReaderExtensionsHelper extensionHelper = new MetadataReaderExtensionsHelper(); /// /// Initializes a new instance of the class. @@ -167,8 +168,7 @@ private void GetTestExtensionsFromAssembly(Assembly ass try { - MetadataReaderExtensionsHelper extensionHelper = new MetadataReaderExtensionsHelper(); - var discoveredExtensions = extensionHelper.DiscoverTestExtensionTypesV2Attribute(assembly, filePath); + var discoveredExtensions = this.extensionHelper.DiscoverTestExtensionTypesV2Attribute(assembly, filePath); if (discoveredExtensions?.Length > 0) { types.AddRange(discoveredExtensions); @@ -179,7 +179,6 @@ private void GetTestExtensionsFromAssembly(Assembly ass EqtTrace.Warning("TestPluginDiscoverer: Failed to get types searching for 'TestPlatformExtensionVersionAttribute' from assembly '{0}'. Error: {1}", assembly.FullName, e.ToString()); } - try { var typesToLoad = TypesToLoadUtilities.GetTypesToLoad(assembly); From e4c009cdeb8b9086666bc37eba31c545a935d6f7 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 9 Dec 2021 18:43:54 +0100 Subject: [PATCH 13/47] load extension for the analysis --- .../Utilities/MetadataReaderHelper.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index 0b78e0c851..6588e0c9f2 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -13,14 +13,21 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities using System.Text; using Microsoft.VisualStudio.TestPlatform.ObjectModel; + internal class MetadataReaderExtensionsHelper { private static string TestExtensionTypesAttributeV2 = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypesV2Attribute"; private static Type[] EmptyTypeArray = new Type[0]; - public Type[] DiscoverTestExtensionTypesV2Attribute(Assembly assembly, string assemblyFilePath) + public Type[] DiscoverTestExtensionTypesV2Attribute(Assembly loadedAssembly, string assemblyFilePath) { - EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{assembly.FullName}' file path '{assemblyFilePath}'"); + EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{loadedAssembly.FullName}' file path '{assemblyFilePath}'"); + +#if !NETSTANDARD1_3 + Assembly assemblyToAnalyze = Assembly.Load(File.ReadAllBytes(assemblyFilePath)); +#else + Assembly assemblyToAnalyze = loadedAssembly; +#endif List> extensions = null; using (var stream = new FileStream(assemblyFilePath, FileMode.Open, FileAccess.Read)) @@ -112,7 +119,7 @@ public Type[] DiscoverTestExtensionTypesV2Attribute(Assembly assembly, string as int version = valueReader.ReadInt32(); try { - var extensionType = assembly.GetType(extensionImplementation); + var extensionType = assemblyToAnalyze.GetType(extensionImplementation); if (extensionType is null) { EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Unable to get extension type for '{extensionImplementation}'"); From 63db3e886cf82ee49c4e05ddae710ac8d76abcb3 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 9 Dec 2021 19:07:41 +0100 Subject: [PATCH 14/47] cache asms --- .../Utilities/MetadataReaderHelper.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index 6588e0c9f2..35af120bd3 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -4,6 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities { using System; + using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -16,15 +17,17 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities internal class MetadataReaderExtensionsHelper { - private static string TestExtensionTypesAttributeV2 = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypesV2Attribute"; - private static Type[] EmptyTypeArray = new Type[0]; + private const string TestExtensionTypesAttributeV2 = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypesV2Attribute"; + private static readonly ConcurrentDictionary assemblyCache = new ConcurrentDictionary(); + private static readonly Type[] emptyTypeArray = new Type[0]; public Type[] DiscoverTestExtensionTypesV2Attribute(Assembly loadedAssembly, string assemblyFilePath) { EqtTrace.Verbose($"MetadataReaderExtensionsHelper: Discovering extensions inside assembly '{loadedAssembly.FullName}' file path '{assemblyFilePath}'"); #if !NETSTANDARD1_3 - Assembly assemblyToAnalyze = Assembly.Load(File.ReadAllBytes(assemblyFilePath)); + // Cache assembly, in VS scenario vstest.console is not unloaded so we don't want to load same asm more times. + Assembly assemblyToAnalyze = assemblyCache.GetOrAdd(assemblyFilePath, Assembly.Load(File.ReadAllBytes(assemblyFilePath))); #else Assembly assemblyToAnalyze = loadedAssembly; #endif @@ -144,7 +147,7 @@ public Type[] DiscoverTestExtensionTypesV2Attribute(Assembly loadedAssembly, str } } - return extensions?.OrderByDescending(t => t.Item1).Select(t => t.Item2).ToArray() ?? EmptyTypeArray; + return extensions?.OrderByDescending(t => t.Item1).Select(t => t.Item2).ToArray() ?? emptyTypeArray; } private string FormatException(Exception ex) From fa281d4b3dab4797db3bad13f640372268519047 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Wed, 15 Dec 2021 19:13:56 +0100 Subject: [PATCH 15/47] skip the extension cache for the attachment data processor --- .../DataCollectionExtensionManager.cs | 8 +++- .../ExtensionFramework/TestPluginCache.cs | 46 +++++++++++-------- .../TestPluginDiscoverer.cs | 5 ++ .../ExtensionFramework/TestPluginManager.cs | 8 +++- .../Utilities/MetadataReaderHelper.cs | 25 ++++++++++ ...taCollectorAttachmentsProcessorsFactory.cs | 4 +- 6 files changed, 71 insertions(+), 25 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs index e7a3cf039e..58904cf792 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs @@ -65,18 +65,22 @@ public static DataCollectorExtensionManager Create(IMessageLogger messageLogger) /// /// File path that contains data collectors to load. /// + /// + /// Skip the extensions cache. + /// /// /// The message Logger. /// /// /// The DataCollectorExtensionManager. /// - public static DataCollectorExtensionManager Create(string extensionAssemblyFilePath, IMessageLogger messageLogger) + public static DataCollectorExtensionManager Create(string extensionAssemblyFilePath, bool skipCache, IMessageLogger messageLogger) { TestPluginManager.Instance.GetTestExtensions( extensionAssemblyFilePath, out var unfilteredTestExtensions, - out var filteredTestExtensions); + out var filteredTestExtensions, + skipCache); return new DataCollectorExtensionManager(unfilteredTestExtensions, filteredTestExtensions, messageLogger); } diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs index cc9a34b3de..eaec74411c 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs @@ -330,6 +330,9 @@ internal IEnumerable DefaultExtensionPaths /// /// The extension assembly. /// + /// + /// Skip the extensions cache. + /// /// /// Type of Test plugin info. /// @@ -339,28 +342,35 @@ internal IEnumerable DefaultExtensionPaths /// /// The . /// - internal Dictionary GetTestExtensions(string extensionAssembly) where TPluginInfo : TestPluginInformation + internal Dictionary GetTestExtensions(string extensionAssembly, bool skipCache = false) where TPluginInfo : TestPluginInformation { - // Check if extensions from this assembly have already been discovered. - var extensions = this.TestExtensions?.GetExtensionsDiscoveredFromAssembly( - this.TestExtensions.GetTestExtensionCache(), - extensionAssembly); - - if (extensions != null && extensions.Count > 0) + if (skipCache) { - return extensions; + return this.GetTestExtensions(new List() { extensionAssembly }); } + else + { + // Check if extensions from this assembly have already been discovered. + var extensions = this.TestExtensions?.GetExtensionsDiscoveredFromAssembly( + this.TestExtensions.GetTestExtensionCache(), + extensionAssembly); - var pluginInfos = this.GetTestExtensions(new List() { extensionAssembly }); + if (extensions != null && extensions.Count > 0) + { + return extensions; + } - // Add extensions discovered to the cache. - if (this.TestExtensions == null) - { - this.TestExtensions = new TestExtensions(); - } + var pluginInfos = this.GetTestExtensions(new List() { extensionAssembly }); - this.TestExtensions.AddExtension(pluginInfos); - return pluginInfos; + // Add extensions discovered to the cache. + if (this.TestExtensions == null) + { + this.TestExtensions = new TestExtensions(); + } + + this.TestExtensions.AddExtension(pluginInfos); + return pluginInfos; + } } /// @@ -491,9 +501,7 @@ private Dictionary GetTestExtensions(extensionPaths); + return new TestPluginDiscoverer().GetTestExtensionsInformation(extensionPaths); } protected void SetupAssemblyResolver(string extensionAssembly) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index 7f0395b605..b950fdefab 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -196,6 +196,11 @@ private void GetTestExtensionsFromAssembly(Assembly ass { EqtTrace.Warning("TestPluginDiscoverer: Failed to get types from assembly '{0}'. Error: {1}", assembly.FullName, e.ToString()); + if (e.Types?.Length > 0) + { + types.AddRange(e.Types.Where(type => type.GetTypeInfo().IsClass && !type.GetTypeInfo().IsAbstract)); + } + if (e.LoaderExceptions != null) { foreach (var ex in e.LoaderExceptions) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginManager.cs index 0f11cf2067..3ef28c131c 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginManager.cs @@ -167,12 +167,16 @@ public void GetSpecificTestExtensions /// Receives test extensions filtered by Identifier data /// + /// + /// Skip the extensions cache. + /// public void GetTestExtensions( string extensionAssembly, out IEnumerable>> unfiltered, - out IEnumerable> filtered) where TMetadata : IMetadata where TPluginInfo : TestPluginInformation + out IEnumerable> filtered, + bool skipCache = false) where TMetadata : IMetadata where TPluginInfo : TestPluginInformation { - var extensions = TestPluginCache.Instance.GetTestExtensions(extensionAssembly); + var extensions = TestPluginCache.Instance.GetTestExtensions(extensionAssembly, skipCache); this.GetExtensions(extensions, out unfiltered, out filtered); } diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index 35af120bd3..81da41f31c 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -15,6 +15,31 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities using Microsoft.VisualStudio.TestPlatform.ObjectModel; + /* Expected attriute shape + + namespace Microsoft.VisualStudio.TestPlatform + { + using System; + + [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = true)] + internal sealed class TestExtensionTypesV2Attribute : Attribute + { + public string ExtensionType { get; } + public string ExtensionIdentifier { get; } + public Type ExtensionImplementation { get; } + public int Version { get; } + + public TestExtensionTypesV2Attribute(string extensionType, string extensionIdentifier, Type extensionImplementation, int version) + { + ExtensionType = extensionType; + ExtensionIdentifier = extensionIdentifier; + ExtensionImplementation = extensionImplementation; + Version = version; + } + } + } + + */ internal class MetadataReaderExtensionsHelper { private const string TestExtensionTypesAttributeV2 = "Microsoft.VisualStudio.TestPlatform.TestExtensionTypesV2Attribute"; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index 06ed66c88d..3b00dd912e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -22,7 +22,7 @@ public IReadOnlyDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); bool addCodeCoverageAttachmentProcessors = true; - if (invokedDataCollectors != null) + if (invokedDataCollectors?.Length > 0) { // We order files by filename descending so in case of the same collector from the same nuget but with different versions, we'll run the newer version. // i.e. C:\Users\xxx\.nuget\packages\coverlet.collector @@ -39,7 +39,7 @@ public IReadOnlyDictionary Date: Thu, 16 Dec 2021 10:47:27 +0100 Subject: [PATCH 16/47] cache attachment processor resolution --- .../DataCollectorAttachmentsProcessorsFactory.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index 3b00dd912e..6b3eb6368b 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -6,6 +6,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.Utilities; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; @@ -16,6 +17,7 @@ internal class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachm { private static Uri CoverageUri = new Uri("datacollector://microsoft/CodeCoverage/2.0"); private const string CoverageFriendlyName = "Code Coverage"; + private static ConcurrentDictionary dataCollectorExtensionManagerCache = new ConcurrentDictionary(); public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) { @@ -39,7 +41,8 @@ public IReadOnlyDictionary(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); + // If we found inside an extension the CodeCoverage attachment processor we use it(the most up to date) and we won't add the default one inside TP. if (invokedDataCollector.Uri.AbsoluteUri == CoverageUri.AbsoluteUri) { EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Attachment data processor for data collector with friendly name '{CoverageFriendlyName}' found '{attachmentProcessorType.AssemblyQualifiedName}' inside '{invokedDataCollector.FilePath}'"); From f79e25e7fe193d1ecb76ca9fbf659f0eec6eb4e1 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 16 Dec 2021 15:14:36 +0100 Subject: [PATCH 17/47] Break the IDataCollectorAttachmentProcessor and remove IConfigurableDataCollectorAttachmentProcessor --- .../ITestRunAttachmentsProcessingManager.cs | 2 +- ...taCollectorAttachmentsProcessorsFactory.cs | 17 ++++--- .../TestRunAttachmentsProcessingManager.cs | 5 +- ...gurableDataCollectorAttachmentProcessor.cs | 51 ------------------- .../IDataCollectorAttachmentProcessor.cs | 4 +- .../PublicAPI/net/PublicAPI.Shipped.txt | 6 +-- .../CodeCoverageDataAttachmentsHandler.cs | 2 +- .../TestPluginDiscovererTests.cs | 2 +- ...lectorAttachmentsProcessorsFactoryTests.cs | 4 +- ...estRunAttachmentsProcessingManagerTests.cs | 10 ++-- .../SampleDataCollector.cs | 2 +- .../DataCollectionManagerTests.cs | 2 +- 12 files changed, 28 insertions(+), 79 deletions(-) delete mode 100644 src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs index a94c0f5227..db8ff0584d 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs @@ -48,6 +48,6 @@ internal interface IDataCollectorAttachmentsProcessorsFactory /// /// List of invoked data collectors /// List of attachments processors - IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors); + IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors); } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index 6b3eb6368b..ca50c7d787 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -19,9 +19,9 @@ internal class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachm private const string CoverageFriendlyName = "Code Coverage"; private static ConcurrentDictionary dataCollectorExtensionManagerCache = new ConcurrentDictionary(); - public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) + public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) { - IDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); + IDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); bool addCodeCoverageAttachmentProcessors = true; if (invokedDataCollectors?.Length > 0) @@ -47,10 +47,10 @@ public IReadOnlyDictionary(attachmentProcessorType); + dataCollectorAttachmentProcessorInstance = TestPluginManager.CreateTestExtension(attachmentProcessorType); } catch (Exception ex) { @@ -59,7 +59,7 @@ public IReadOnlyDictionary(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); + datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); // If we found inside an extension the CodeCoverage attachment processor we use it(the most up to date) and we won't add the default one inside TP. if (invokedDataCollector.Uri.AbsoluteUri == CoverageUri.AbsoluteUri) @@ -76,19 +76,20 @@ public IReadOnlyDictionary(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); + datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); } - var finalDatacollectorsAttachmentsProcessors = new Dictionary(); + var finalDatacollectorsAttachmentsProcessors = new Dictionary(); foreach (var attachementProcessor in datacollectorsAttachmentsProcessors) { EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: valid data collector attachment processor found: '{attachementProcessor.Value.Item2.GetType().AssemblyQualifiedName}'"); finalDatacollectorsAttachmentsProcessors.Add(attachementProcessor.Value.Item1, attachementProcessor.Value.Item2); } - return new ReadOnlyDictionary(finalDatacollectorsAttachmentsProcessors); + return new ReadOnlyDictionary(finalDatacollectorsAttachmentsProcessors); } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs index 63073568c6..8659344c4c 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs @@ -104,9 +104,12 @@ private async Task> ProcessAttachmentsAsync(string run var dataCollectionRunSettings = XmlRunSettingsUtilities.GetDataCollectionRunSettings(runSettingsXml); var logger = CreateMessageLogger(eventsHandler); - IReadOnlyDictionary dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray()); + IReadOnlyDictionary dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray()); for (int i = 0; i < dataCollectorAttachmentsProcessors.Count; i++) { + // TODO: We don't want have all or nothing...if one fails we skip it + // Add units: first failing(second merge) and all failing(no change to attachments) + var dataCollectorAttachmentsProcessor = dataCollectorAttachmentsProcessors.ElementAt(i); int attachmentsHandlerIndex = i + 1; diff --git a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs deleted file mode 100644 index d81738dab3..0000000000 --- a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IConfigurableDataCollectorAttachmentProcessor.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection -{ - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using System.Xml; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; - - /// - /// Interface for data collectors add-ins that choose to reprocess generated attachments - /// - public interface IConfigurableDataCollectorAttachmentProcessor - { - /// - /// Gets the attachments Uris, which are handled by attachment processor - /// - IEnumerable GetExtensionUris(); - - /// - /// Indicates whether attachment processor is supporting incremental processing of attachments - /// - /// - /// `SupportsIncrementalProcessing` should indicate if attachment processor is supporting incremental processing of attachments. It means that `ProcessAttachmentSetsAsync` should be [associative](https://en.wikipedia.org/wiki/Associative_property). - /// By default `SupportsIncrementalProcessing` should be `False`, unless processing can take longer time and it's beneficial to start the process as soon as possible. - /// - /// If `SupportsIncrementalProcessing` is `True` Test Platform may try to speed up whole process by reprocessing data collector attachments as soon as possible when any two test executions are done.For example let's assume we have 5 test executions which are generating 5 data collector attachments: `a1`, `a2`, `a3`, `a4` and `a5`. Test platform could perform invocations: - /// * `var result1 = await ProcessAttachmentSetsAsync([a1, a2, a3], ...);` when first 3 executions are done - /// * `var result2 = await ProcessAttachmentSetsAsync(result1.Concat([a4]), ...);` when 4th execution is done - /// * `var finalResult = await ProcessAttachmentSetsAsync(result2.Concat([a5]), ...);` when last test execution is done - /// - /// If `SupportsIncrementalProcessing` is `False` then Test Platform will wait for all test executions to finish and call `ProcessAttachmentSetsAsync` only once: - /// * `var finalResult = await ProcessAttachmentSetsAsync([a1, a2, a3, a4, a5], ...);` - /// - bool SupportsIncrementalProcessing { get; } - - /// - /// Reprocess attachments generated by independent test executions - /// - /// Configuration of the attachment processor. Will be the same as the data collector that registers it. - /// Attachments to be processed - /// Progress reporter. Accepts integers from 0 to 100 - /// Message logger - /// Cancellation token - /// Attachments after reprocessing - Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); - } -} diff --git a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs index 8425477b5f..37565e3d89 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/DataCollector/IDataCollectorAttachmentProcessor.cs @@ -13,7 +13,6 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection /// /// Interface for data collectors add-ins that choose to reprocess generated attachments /// - [Obsolete("Interface is deprecated. Please use IConfigurableDataCollectorAttachmentProcessor instead")] public interface IDataCollectorAttachmentProcessor { /// @@ -41,11 +40,12 @@ public interface IDataCollectorAttachmentProcessor /// /// Reprocess attachments generated by independent test executions /// + /// Configuration of the attachment processor. Will be the same as the data collector that registers it. /// Attachments to be processed /// Progress reporter. Accepts integers from 0 to 100 /// Message logger /// Cancellation token /// Attachments after reprocessing - Task> ProcessAttachmentSetsAsync(ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); + Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt index be5012453d..7929196a23 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Shipped.txt @@ -68,12 +68,8 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectionSi Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectionSink.SendData(Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.DataCollectionContext dataCollectionContext, string key, string value) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.GetExtensionUris() -> System.Collections.Generic.IEnumerable -Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> +Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Xml.XmlElement configurationElement, System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor.SupportsIncrementalProcessing.get -> bool -Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor -Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor.GetExtensionUris() -> System.Collections.Generic.IEnumerable -Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor.ProcessAttachmentSetsAsync(System.Xml.XmlElement configurationElement, System.Collections.Generic.ICollection attachments, System.IProgress progressReporter, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> -Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IConfigurableDataCollectorAttachmentProcessor.SupportsIncrementalProcessing.get -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments.GetExtensionUri() -> System.Uri Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachments.HandleDataCollectionAttachmentSets(System.Collections.Generic.ICollection dataCollectionAttachments) -> System.Collections.Generic.ICollection diff --git a/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs b/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs index 5bf8924691..e4a2fd4210 100644 --- a/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs +++ b/src/Microsoft.TestPlatform.Utilities/CodeCoverageDataAttachmentsHandler.cs @@ -17,7 +17,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Utilities using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; - public class CodeCoverageDataAttachmentsHandler : IConfigurableDataCollectorAttachmentProcessor + public class CodeCoverageDataAttachmentsHandler : IDataCollectorAttachmentProcessor { private const string CoverageUri = "datacollector://microsoft/CodeCoverage/2.0"; private const string CoverageFileExtension = ".coverage"; diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs index 0e3a1a0bd2..3f623486e7 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestPluginDiscovererTests.cs @@ -335,7 +335,7 @@ public override void Initialize( } } - public class DataCollectorAttachmentProcessor : IConfigurableDataCollectorAttachmentProcessor + public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index abdf0647b5..a10e739de0 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -180,7 +180,7 @@ public override void Initialize( } } - public class DataCollectorAttachmentProcessor : IConfigurableDataCollectorAttachmentProcessor + public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); @@ -195,7 +195,7 @@ public Task> ProcessAttachmentSetsAsync(XmlElement co } } - public class DataCollectorAttachmentProcessor2 : IConfigurableDataCollectorAttachmentProcessor + public class DataCollectorAttachmentProcessor2 : IDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs index e055ba9b66..4bc8a806bc 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs @@ -31,8 +31,8 @@ public class TestRunAttachmentsProcessingManagerTests private readonly Mock mockRequestData; private readonly Mock mockMetricsCollection; private readonly Mock mockEventSource; - private readonly Mock mockAttachmentHandler1; - private readonly Mock mockAttachmentHandler2; + private readonly Mock mockAttachmentHandler1; + private readonly Mock mockAttachmentHandler2; private readonly Mock mockDataCollectorAttachmentsProcessorsFactory; private readonly Mock mockEventsHandler; private readonly TestRunAttachmentsProcessingManager manager; @@ -45,15 +45,15 @@ public TestRunAttachmentsProcessingManagerTests() mockRequestData.Setup(r => r.MetricsCollection).Returns(mockMetricsCollection.Object); mockEventSource = new Mock(); - mockAttachmentHandler1 = new Mock(); - mockAttachmentHandler2 = new Mock(); + mockAttachmentHandler1 = new Mock(); + mockAttachmentHandler2 = new Mock(); mockEventsHandler = new Mock(); mockDataCollectorAttachmentsProcessorsFactory = new Mock(); mockAttachmentHandler1.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); mockAttachmentHandler2.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri2) }); mockDataCollectorAttachmentsProcessorsFactory.Setup(p => p.Create(It.IsAny())) - .Returns(new ReadOnlyDictionary(new Dictionary() + .Returns(new ReadOnlyDictionary(new Dictionary() { { "friendlyNameA", mockAttachmentHandler1.Object } , { "friendlyNameB" ,mockAttachmentHandler2.Object } })); diff --git a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs index 2fb903c5db..e3d61d6eec 100644 --- a/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs +++ b/test/TestAssets/AttachmentProcessorDataCollector/SampleDataCollector.cs @@ -56,7 +56,7 @@ private void SessionEnded_Handler(object sender, SessionEndEventArgs e) } } - public class SampleDataCollectorAttachmentProcessor : IConfigurableDataCollectorAttachmentProcessor + public class SampleDataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => true; diff --git a/test/datacollector.UnitTests/DataCollectionManagerTests.cs b/test/datacollector.UnitTests/DataCollectionManagerTests.cs index 29e1f3879e..0b017379c1 100644 --- a/test/datacollector.UnitTests/DataCollectionManagerTests.cs +++ b/test/datacollector.UnitTests/DataCollectionManagerTests.cs @@ -598,7 +598,7 @@ public abstract class CodeCoverageDataCollector : DataCollector { } - public class AttachmentProcessorDataCollector2 : IConfigurableDataCollectorAttachmentProcessor + public class AttachmentProcessorDataCollector2 : IDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); From 9b77adfa954f511ef8dc2f9c3262f92decb53b78 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 16 Dec 2021 16:14:05 +0100 Subject: [PATCH 18/47] add telemetry for invoked collectors --- .../PublicAPI/PublicAPI.Shipped.txt | 1 + .../Telemetry/TelemetryDataConstants.cs | 4 ++++ .../DataCollectionRequestHandler.cs | 8 ++++++++ .../DebuggerBreakpoint.cs | 2 +- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index c23e7a1cfd..ef71ebe7de 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -277,6 +277,7 @@ static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstan static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.TotalTestsRanByAdapter -> string static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.TotalTestsRun -> string static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.TotalTestsRunByMSTestv1 -> string +static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.InvokedDataCollector -> string static Microsoft.VisualStudio.TestPlatform.Common.Utilities.CancellationTokenExtensions.ThrowTestPlatformExceptionIfCancellationRequested(this System.Threading.CancellationToken token) -> void static Microsoft.VisualStudio.TestPlatform.Common.Utilities.ExceptionUtilities.GetExceptionMessage(System.Exception exception) -> string static Microsoft.VisualStudio.TestPlatform.Common.Utilities.FakesUtilities.GenerateFakesSettingsForRunConfiguration(string[] sources, string runSettingsXml) -> string diff --git a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs index a0ef413692..c7bf3b86e6 100644 --- a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs +++ b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs @@ -112,8 +112,12 @@ public static class TelemetryDataConstants public static string NumberOfAttachmentsAfterProcessing = "VS.AttachmentsProcessing.FinalAttachmentsCount"; public static string TimeTakenInSecForAttachmentsProcessing = "VS.AttachmentsProcessing.TotalTimeTakenInSec"; + public static string AttachmentsProcessingState = "VS.AttachmentsProcessing.State"; + // *********************Data collectors**************************** + public static string InvokedDataCollector = "VS.DataCollectors.InvokedDataCollector"; + // **************Events Name ********************************** public static string TestDiscoveryCompleteEvent = "vs/testplatform/testdiscoverysession"; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index ef4ee8fd0a..f3e98276a5 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -391,6 +391,14 @@ private void HandleAfterTestRunEnd(Message message) var attachmentsets = this.dataCollectionManager.SessionEnded(isCancelled); var invokedDataCollectors = this.dataCollectionManager.GetInvokedDataCollectors(); + if (invokedDataCollectors != null && invokedDataCollectors.Count > 0) + { + foreach (var invokedDataCollector in invokedDataCollectors) + { + this.requestData.MetricsCollection.Add(TelemetryDataConstants.InvokedDataCollector, invokedDataCollector); + } + } + var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, invokedDataCollectors, this.requestData.MetricsCollection.Metrics); // Dispose all datacollectors before sending attachments to vstest.console process. diff --git a/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs b/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs index 1f8717c4a2..f54892d4a2 100644 --- a/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs +++ b/src/Microsoft.TestPlatform.Execution.Shared/DebuggerBreakpoint.cs @@ -42,7 +42,7 @@ internal static void AttachVisualStudioDebugger(string environmentVariable) } else { - ConsoleOutput.Instance.WriteLine($"Attaching Visual Studio with PID {vsPid}... No breakpoints are automatically set.", OutputLevel.Information); + ConsoleOutput.Instance.WriteLine($"Attaching Visual Studio with PID {vsPid} to the process '{Process.GetCurrentProcess().ProcessName}({Process.GetCurrentProcess().Id})'... No breakpoints are automatically set.", OutputLevel.Information); } AttachVS(Process.GetCurrentProcess(), vsPid); From fc0577c655d009309adf867222b16085ba1eaf7f Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 16 Dec 2021 19:35:48 +0100 Subject: [PATCH 19/47] Make the processor logic fault resilient --- ...taCollectorAttachmentsProcessorsFactory.cs | 17 +-- .../TestRunAttachmentsProcessingManager.cs | 73 +++++----- ...estRunAttachmentsProcessingManagerTests.cs | 126 ++++++++++++++++++ 3 files changed, 172 insertions(+), 44 deletions(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index ca50c7d787..c7e87b3649 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -15,14 +15,12 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachments { internal class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory { - private static Uri CoverageUri = new Uri("datacollector://microsoft/CodeCoverage/2.0"); private const string CoverageFriendlyName = "Code Coverage"; private static ConcurrentDictionary dataCollectorExtensionManagerCache = new ConcurrentDictionary(); public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) { IDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); - bool addCodeCoverageAttachmentProcessors = true; if (invokedDataCollectors?.Length > 0) { @@ -60,13 +58,6 @@ public IReadOnlyDictionary Create(Inv if (dataCollectorAttachmentProcessorInstance != null && !datacollectorsAttachmentsProcessors.ContainsKey(attachmentProcessorType.AssemblyQualifiedName)) { datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); - - // If we found inside an extension the CodeCoverage attachment processor we use it(the most up to date) and we won't add the default one inside TP. - if (invokedDataCollector.Uri.AbsoluteUri == CoverageUri.AbsoluteUri) - { - EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Attachment data processor for data collector with friendly name '{CoverageFriendlyName}' found '{attachmentProcessorType.AssemblyQualifiedName}' inside '{invokedDataCollector.FilePath}'"); - addCodeCoverageAttachmentProcessors = false; - } } } else @@ -76,11 +67,9 @@ public IReadOnlyDictionary Create(Inv } } - // TODO: we can add it always as last processor...in case of error of newer one at least this one will run - if (addCodeCoverageAttachmentProcessors) - { - datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); - } + // We provide the implementation of CodeCoverageDataAttachmentsHandler through nuget package, but in case of absent registration or if for some reason + // the attachment processor from package fails we fallback to the default implementation. + datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); var finalDatacollectorsAttachmentsProcessors = new Dictionary(); foreach (var attachementProcessor in datacollectorsAttachmentsProcessors) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs index 8659344c4c..cf7b8d69a7 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs @@ -107,51 +107,64 @@ private async Task> ProcessAttachmentsAsync(string run IReadOnlyDictionary dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray()); for (int i = 0; i < dataCollectorAttachmentsProcessors.Count; i++) { - // TODO: We don't want have all or nothing...if one fails we skip it - // Add units: first failing(second merge) and all failing(no change to attachments) - var dataCollectorAttachmentsProcessor = dataCollectorAttachmentsProcessors.ElementAt(i); int attachmentsHandlerIndex = i + 1; - ICollection attachmentProcessorUris = dataCollectorAttachmentsProcessor.Value.GetExtensionUris()?.ToList(); - if (attachmentProcessorUris != null && attachmentProcessorUris.Any()) + // We run processor code inside a try/catch because we want to continue with the others in case of failure. + Collection attachmentsBackup = null; + try { - var attachmentsToBeProcessed = attachments.Where(dataCollectionAttachment => attachmentProcessorUris.Any(uri => uri.Equals(dataCollectionAttachment.Uri))).ToArray(); - if (attachmentsToBeProcessed.Any()) + // We temporarily save the attachments to process because, in case of processor exception, + // we'll restore the attachmentSets to make those available to other processors. + attachmentsBackup = new Collection(attachments.ToList()); + + ICollection attachmentProcessorUris = dataCollectorAttachmentsProcessor.Value.GetExtensionUris()?.ToList(); + if (attachmentProcessorUris != null && attachmentProcessorUris.Any()) { - foreach (var attachment in attachmentsToBeProcessed) + var attachmentsToBeProcessed = attachments.Where(dataCollectionAttachment => attachmentProcessorUris.Any(uri => uri.Equals(dataCollectionAttachment.Uri))).ToArray(); + if (attachmentsToBeProcessed.Any()) { - attachments.Remove(attachment); - } + foreach (var attachment in attachmentsToBeProcessed) + { + attachments.Remove(attachment); + } - IProgress progressReporter = new Progress((int progress) => - eventsHandler?.HandleTestRunAttachmentsProcessingProgress( - new TestRunAttachmentsProcessingProgressEventArgs(attachmentsHandlerIndex, attachmentProcessorUris, progress, dataCollectorAttachmentsProcessors.Count))); + IProgress progressReporter = new Progress((int progress) => + eventsHandler?.HandleTestRunAttachmentsProcessingProgress( + new TestRunAttachmentsProcessingProgressEventArgs(attachmentsHandlerIndex, attachmentProcessorUris, progress, dataCollectorAttachmentsProcessors.Count))); - XmlElement configuration = null; - var collectorConfiguration = dataCollectionRunSettings?.DataCollectorSettingsList.SingleOrDefault(c => c.FriendlyName == dataCollectorAttachmentsProcessor.Key); - if (collectorConfiguration != null && collectorConfiguration.IsEnabled) - { - configuration = collectorConfiguration.Configuration; - } + XmlElement configuration = null; + var collectorConfiguration = dataCollectionRunSettings?.DataCollectorSettingsList.SingleOrDefault(c => c.FriendlyName == dataCollectorAttachmentsProcessor.Key); + if (collectorConfiguration != null && collectorConfiguration.IsEnabled) + { + configuration = collectorConfiguration.Configuration; + } - EqtTrace.Info($"TestRunAttachmentsProcessingManager: invocation of data collector attachment processor '{dataCollectorAttachmentsProcessor.Value.GetType().AssemblyQualifiedName}' with configuration '{(configuration == null ? "null" : configuration.OuterXml)}'"); - ICollection processedAttachments = await dataCollectorAttachmentsProcessor.Value.ProcessAttachmentSetsAsync( - configuration, - new Collection(attachmentsToBeProcessed), - progressReporter, - logger, - cancellationToken).ConfigureAwait(false); + EqtTrace.Info($"TestRunAttachmentsProcessingManager: invocation of data collector attachment processor '{dataCollectorAttachmentsProcessor.Value.GetType().AssemblyQualifiedName}' with configuration '{(configuration == null ? "null" : configuration.OuterXml)}'"); + ICollection processedAttachments = await dataCollectorAttachmentsProcessor.Value.ProcessAttachmentSetsAsync( + configuration, + new Collection(attachmentsToBeProcessed), + progressReporter, + logger, + cancellationToken).ConfigureAwait(false); - if (processedAttachments != null && processedAttachments.Any()) - { - foreach (var attachment in processedAttachments) + if (processedAttachments != null && processedAttachments.Any()) { - attachments.Add(attachment); + foreach (var attachment in processedAttachments) + { + attachments.Add(attachment); + } } } } } + catch (Exception e) + { + EqtTrace.Error("TestRunAttachmentsProcessingManager: Exception in ProcessAttachmentsAsync: " + e); + + // Restore the attachment sets for the others attachment processors. + attachments = attachmentsBackup; + } } return attachments; diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs index 4bc8a806bc..455564df1b 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs @@ -689,6 +689,132 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldFlowCorrectDataCollectorC mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); } + [TestMethod] + public async Task ProcessTestRunAttachmentsAsync_ShouldNotConsumeAttachmentsIfProcessorFails() + { + // arrange + List inputAttachments = new List + { + new AttachmentSet(new Uri(uri1), "uri1_input_1") + }; + inputAttachments[0].Attachments.Add(UriDataAttachment.CreateFrom("file1", "Sample1")); + inputAttachments[0].Attachments.Add(UriDataAttachment.CreateFrom("file2", "Sample2")); + inputAttachments[0].Attachments.Add(UriDataAttachment.CreateFrom("file3", "Sample3")); + + + mockAttachmentHandler1.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); + mockAttachmentHandler2.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); + + bool firstProcessorFailed = false; + + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + { + try + { + throw new Exception("Processor exception"); + } + catch + { + firstProcessorFailed = true; + throw; + } + }); + + ICollection output = new List + { + new AttachmentSet(new Uri(uri1), "uri1_input_1") + }; + output.Single().Attachments.Add(UriDataAttachment.CreateFrom("file4", "Merged")); + + mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + { + // assert + Assert.IsTrue(firstProcessorFailed); + Assert.AreEqual(1, i1.Count); + Assert.AreEqual(3, i1.Single().Attachments.Count); + for (int i = 0; i < i1.Single().Attachments.Count; i++) + { + Assert.AreEqual(inputAttachments.Single().Attachments[i], i1.Single().Attachments[i]); + } + + return Task.FromResult(output); + }); + + // act + await manager.ProcessTestRunAttachmentsAsync(null, mockRequestData.Object, inputAttachments, new List(), mockEventsHandler.Object, cancellationTokenSource.Token); + + // assert + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once()); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once()); + VerifyCompleteEvent(false, false, output.ToArray()); + } + + [TestMethod] + public async Task ProcessTestRunAttachmentsAsync_ShouldNotConsumeAttachmentsIfAllProcessorsFail() + { + // arrange + List inputAttachments = new List + { + new AttachmentSet(new Uri(uri1), "uri1_input_1") + }; + inputAttachments[0].Attachments.Add(UriDataAttachment.CreateFrom("file1", "Sample1")); + inputAttachments[0].Attachments.Add(UriDataAttachment.CreateFrom("file2", "Sample2")); + inputAttachments[0].Attachments.Add(UriDataAttachment.CreateFrom("file3", "Sample3")); + + + mockAttachmentHandler1.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); + mockAttachmentHandler2.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); + + bool firstProcessorFailed = false; + mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + { + try + { + throw new Exception("Processor exception"); + } + catch + { + firstProcessorFailed = true; + throw; + } + }); + + bool secondProcessorFailed = false; + mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + { + try + { + // assert + Assert.IsTrue(firstProcessorFailed); + Assert.AreEqual(1, i1.Count); + Assert.AreEqual(3, i1.Single().Attachments.Count); + for (int i = 0; i < i1.Single().Attachments.Count; i++) + { + Assert.AreEqual(inputAttachments.Single().Attachments[i], i1.Single().Attachments[i]); + } + throw new Exception("Processor exception"); + } + catch + { + secondProcessorFailed = true; + throw; + } + }); + + // act + await manager.ProcessTestRunAttachmentsAsync(null, mockRequestData.Object, inputAttachments, new List(), mockEventsHandler.Object, cancellationTokenSource.Token); + + // assert + Assert.IsTrue(firstProcessorFailed && secondProcessorFailed); + mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once()); + mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Once()); + VerifyCompleteEvent(false, false, inputAttachments.ToArray()); + } + private void VerifyMetrics(int inputCount, int outputCount, string status = "Completed") { mockEventSource.Verify(s => s.TestRunAttachmentsProcessingStart(inputCount)); From 399dc05e3761000c19d453b29242d016f72b8ff6 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Fri, 17 Dec 2021 14:17:27 +0100 Subject: [PATCH 20/47] Add message logger for datacollector attachment factory, fix UTs, add UTs for the MetadataReaderHelper --- ...taCollectorAttachmentsProcessorsFactory.cs | 46 ++++++++++++++ .../ITestRunAttachmentsProcessingManager.cs | 18 +----- .../PublicAPI/PublicAPI.Shipped.txt | 1 - .../Telemetry/TelemetryDataConstants.cs | 3 - .../DataCollectionRequestHandler.cs | 8 --- ...taCollectorAttachmentsProcessorsFactory.cs | 19 +++--- .../TestRunAttachmentsProcessingManager.cs | 21 ++++--- .../Utilities/MetadataReaderHelperTests.cs | 61 +++++++++++++++++++ ...lectorAttachmentsProcessorsFactoryTests.cs | 57 +++++++++++------ ...estRunAttachmentsProcessingManagerTests.cs | 21 ++++--- 10 files changed, 179 insertions(+), 76 deletions(-) create mode 100644 src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs create mode 100644 test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs new file mode 100644 index 0000000000..55e1d23a8e --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine +{ + /// + /// Creates and return a list of available attachments processor + /// + internal interface IDataCollectorAttachmentsProcessorsFactory + { + /// + /// Creates and return a list of available attachments processor + /// + /// List of invoked data collectors + /// Message logger + /// List of attachments processors + DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedDataCollectors, IMessageLogger logger); + } + + /// + /// Registered data collector attachment processor + /// + internal class DataCollectorAttachmentProcessor + { + /// + /// Data collector FriendlyName + /// + public string FriendlyName { get; private set; } + + /// + /// Data collector attachment processor instance + /// + public IDataCollectorAttachmentProcessor DataCollectorAttachmentProcessorInstance { get; private set; } + + public DataCollectorAttachmentProcessor(string friendlyName, IDataCollectorAttachmentProcessor dataCollectorAttachmentProcessor) + { + this.FriendlyName = string.IsNullOrEmpty(friendlyName) ? throw new ArgumentException("Invalid FriendlyName", nameof(friendlyName)) : friendlyName; + this.DataCollectorAttachmentProcessorInstance = dataCollectorAttachmentProcessor ?? throw new ArgumentNullException(nameof(dataCollectorAttachmentProcessor)); + } + } +} diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs index db8ff0584d..02f80bf3c1 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ITestRunAttachmentsProcessingManager.cs @@ -1,14 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; -using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Threading; using System.Threading.Tasks; -using System.Xml; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine { @@ -37,17 +34,4 @@ internal interface ITestRunAttachmentsProcessingManager /// Collection of attachments. Task> ProcessTestRunAttachmentsAsync(string runSettingsXml, IRequestData requestData, IEnumerable attachments, IEnumerable invokedDataCollector, CancellationToken cancellationToken); } - - /// - /// Creates and return a list of available attachments processor - /// - internal interface IDataCollectorAttachmentsProcessorsFactory - { - /// - /// Creates and return a list of available attachments processor - /// - /// List of invoked data collectors - /// List of attachments processors - IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors); - } } diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index ef71ebe7de..c23e7a1cfd 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -277,7 +277,6 @@ static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstan static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.TotalTestsRanByAdapter -> string static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.TotalTestsRun -> string static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.TotalTestsRunByMSTestv1 -> string -static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.InvokedDataCollector -> string static Microsoft.VisualStudio.TestPlatform.Common.Utilities.CancellationTokenExtensions.ThrowTestPlatformExceptionIfCancellationRequested(this System.Threading.CancellationToken token) -> void static Microsoft.VisualStudio.TestPlatform.Common.Utilities.ExceptionUtilities.GetExceptionMessage(System.Exception exception) -> string static Microsoft.VisualStudio.TestPlatform.Common.Utilities.FakesUtilities.GenerateFakesSettingsForRunConfiguration(string[] sources, string runSettingsXml) -> string diff --git a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs index c7bf3b86e6..016ea55be3 100644 --- a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs +++ b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs @@ -115,9 +115,6 @@ public static class TelemetryDataConstants public static string AttachmentsProcessingState = "VS.AttachmentsProcessing.State"; - // *********************Data collectors**************************** - public static string InvokedDataCollector = "VS.DataCollectors.InvokedDataCollector"; - // **************Events Name ********************************** public static string TestDiscoveryCompleteEvent = "vs/testplatform/testdiscoverysession"; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index f3e98276a5..ef4ee8fd0a 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -391,14 +391,6 @@ private void HandleAfterTestRunEnd(Message message) var attachmentsets = this.dataCollectionManager.SessionEnded(isCancelled); var invokedDataCollectors = this.dataCollectionManager.GetInvokedDataCollectors(); - if (invokedDataCollectors != null && invokedDataCollectors.Count > 0) - { - foreach (var invokedDataCollector in invokedDataCollectors) - { - this.requestData.MetricsCollection.Add(TelemetryDataConstants.InvokedDataCollector, invokedDataCollector); - } - } - var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, invokedDataCollectors, this.requestData.MetricsCollection.Metrics); // Dispose all datacollectors before sending attachments to vstest.console process. diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index c7e87b3649..4124af8d1c 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -4,6 +4,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.Utilities; using System; using System.Collections.Concurrent; @@ -16,9 +17,9 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachments internal class DataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory { private const string CoverageFriendlyName = "Code Coverage"; - private static ConcurrentDictionary dataCollectorExtensionManagerCache = new ConcurrentDictionary(); + private static readonly ConcurrentDictionary dataCollectorExtensionManagerCache = new ConcurrentDictionary(); - public IReadOnlyDictionary Create(InvokedDataCollector[] invokedDataCollectors) + public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedDataCollectors, IMessageLogger logger) { IDictionary> datacollectorsAttachmentsProcessors = new Dictionary>(); @@ -53,9 +54,10 @@ public IReadOnlyDictionary Create(Inv catch (Exception ex) { EqtTrace.Error($"DataCollectorAttachmentsProcessorsFactory: Failed during the creation of data collector attachment processor '{attachmentProcessorType.AssemblyQualifiedName}'\n{ex}"); + logger?.SendMessage(TestMessageLevel.Error, $"DataCollectorAttachmentsProcessorsFactory: Failed during the creation of data collector attachment processor '{attachmentProcessorType.AssemblyQualifiedName}'\n{ex}"); } - if (dataCollectorAttachmentProcessorInstance != null && !datacollectorsAttachmentsProcessors.ContainsKey(attachmentProcessorType.AssemblyQualifiedName)) + if (dataCollectorAttachmentProcessorInstance != null) { datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); } @@ -69,16 +71,19 @@ public IReadOnlyDictionary Create(Inv // We provide the implementation of CodeCoverageDataAttachmentsHandler through nuget package, but in case of absent registration or if for some reason // the attachment processor from package fails we fallback to the default implementation. - datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); + if (!datacollectorsAttachmentsProcessors.ContainsKey(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName)) + { + datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); + } - var finalDatacollectorsAttachmentsProcessors = new Dictionary(); + var finalDatacollectorsAttachmentsProcessors = new List(); foreach (var attachementProcessor in datacollectorsAttachmentsProcessors) { EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: valid data collector attachment processor found: '{attachementProcessor.Value.Item2.GetType().AssemblyQualifiedName}'"); - finalDatacollectorsAttachmentsProcessors.Add(attachementProcessor.Value.Item1, attachementProcessor.Value.Item2); + finalDatacollectorsAttachmentsProcessors.Add(new DataCollectorAttachmentProcessor(attachementProcessor.Value.Item1, attachementProcessor.Value.Item2)); } - return new ReadOnlyDictionary(finalDatacollectorsAttachmentsProcessors); + return finalDatacollectorsAttachmentsProcessors.ToArray(); } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs index cf7b8d69a7..20b8689966 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs @@ -13,7 +13,6 @@ using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; @@ -93,7 +92,7 @@ private async Task> InternalProcessTestRunAttachmentsA { EqtTrace.Error("TestRunAttachmentsProcessingManager: Exception in ProcessTestRunAttachmentsAsync: " + e); - eventHandler?.HandleLogMessage(TestMessageLevel.Error, e.Message); + eventHandler?.HandleLogMessage(TestMessageLevel.Error, "TestRunAttachmentsProcessingManager: Exception in ProcessTestRunAttachmentsAsync: " + e); return FinalizeOperation(requestData, new TestRunAttachmentsProcessingCompleteEventArgs(false, e), attachments, stopwatch, eventHandler); } } @@ -104,10 +103,10 @@ private async Task> ProcessAttachmentsAsync(string run var dataCollectionRunSettings = XmlRunSettingsUtilities.GetDataCollectionRunSettings(runSettingsXml); var logger = CreateMessageLogger(eventsHandler); - IReadOnlyDictionary dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray()); - for (int i = 0; i < dataCollectorAttachmentsProcessors.Count; i++) + var dataCollectorAttachmentsProcessors = this.dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollector?.ToArray(), logger); + for (int i = 0; i < dataCollectorAttachmentsProcessors.Length; i++) { - var dataCollectorAttachmentsProcessor = dataCollectorAttachmentsProcessors.ElementAt(i); + var dataCollectorAttachmentsProcessor = dataCollectorAttachmentsProcessors[i]; int attachmentsHandlerIndex = i + 1; // We run processor code inside a try/catch because we want to continue with the others in case of failure. @@ -116,9 +115,10 @@ private async Task> ProcessAttachmentsAsync(string run { // We temporarily save the attachments to process because, in case of processor exception, // we'll restore the attachmentSets to make those available to other processors. + // NB. attachments.ToList() is done on purpose we need a new ref list. attachmentsBackup = new Collection(attachments.ToList()); - ICollection attachmentProcessorUris = dataCollectorAttachmentsProcessor.Value.GetExtensionUris()?.ToList(); + ICollection attachmentProcessorUris = dataCollectorAttachmentsProcessor.DataCollectorAttachmentProcessorInstance.GetExtensionUris()?.ToList(); if (attachmentProcessorUris != null && attachmentProcessorUris.Any()) { var attachmentsToBeProcessed = attachments.Where(dataCollectionAttachment => attachmentProcessorUris.Any(uri => uri.Equals(dataCollectionAttachment.Uri))).ToArray(); @@ -131,17 +131,17 @@ private async Task> ProcessAttachmentsAsync(string run IProgress progressReporter = new Progress((int progress) => eventsHandler?.HandleTestRunAttachmentsProcessingProgress( - new TestRunAttachmentsProcessingProgressEventArgs(attachmentsHandlerIndex, attachmentProcessorUris, progress, dataCollectorAttachmentsProcessors.Count))); + new TestRunAttachmentsProcessingProgressEventArgs(attachmentsHandlerIndex, attachmentProcessorUris, progress, dataCollectorAttachmentsProcessors.Length))); XmlElement configuration = null; - var collectorConfiguration = dataCollectionRunSettings?.DataCollectorSettingsList.SingleOrDefault(c => c.FriendlyName == dataCollectorAttachmentsProcessor.Key); + var collectorConfiguration = dataCollectionRunSettings?.DataCollectorSettingsList.SingleOrDefault(c => c.FriendlyName == dataCollectorAttachmentsProcessor.FriendlyName); if (collectorConfiguration != null && collectorConfiguration.IsEnabled) { configuration = collectorConfiguration.Configuration; } - EqtTrace.Info($"TestRunAttachmentsProcessingManager: invocation of data collector attachment processor '{dataCollectorAttachmentsProcessor.Value.GetType().AssemblyQualifiedName}' with configuration '{(configuration == null ? "null" : configuration.OuterXml)}'"); - ICollection processedAttachments = await dataCollectorAttachmentsProcessor.Value.ProcessAttachmentSetsAsync( + EqtTrace.Info($"TestRunAttachmentsProcessingManager: invocation of data collector attachment processor '{dataCollectorAttachmentsProcessor.DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName}' with configuration '{(configuration == null ? "null" : configuration.OuterXml)}'"); + ICollection processedAttachments = await dataCollectorAttachmentsProcessor.DataCollectorAttachmentProcessorInstance.ProcessAttachmentSetsAsync( configuration, new Collection(attachmentsToBeProcessed), progressReporter, @@ -161,6 +161,7 @@ private async Task> ProcessAttachmentsAsync(string run catch (Exception e) { EqtTrace.Error("TestRunAttachmentsProcessingManager: Exception in ProcessAttachmentsAsync: " + e); + logger.SendMessage(TestMessageLevel.Error, "TestRunAttachmentsProcessingManager: Exception in ProcessAttachmentsAsync: " + e); // Restore the attachment sets for the others attachment processors. attachments = attachmentsBackup; diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs new file mode 100644 index 0000000000..da6f352486 --- /dev/null +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform; +using Microsoft.VisualStudio.TestPlatform.Common.Utilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Reflection; +using TestPlatform.Common.UnitTests.Utilities; + +[assembly: TestExtensionTypesV2("ExcentionType", "ExtensionIdentified", typeof(ExtensionV1), 1)] +[assembly: TestExtensionTypesV2("ExcentionType", "ExtensionIdentified", typeof(ExtensionV2), 2)] + +namespace TestPlatform.Common.UnitTests.Utilities +{ + [TestClass] + public class MetadataReaderHelperTests + { + private readonly MetadataReaderExtensionsHelper metadataReaderHelper = new MetadataReaderExtensionsHelper(); + + [TestMethod] + public void MetadataReaderHelper_GetCollectorExtensionTypes() + { + var types = metadataReaderHelper.DiscoverTestExtensionTypesV2Attribute(Assembly.GetExecutingAssembly(), Assembly.GetExecutingAssembly().Location); + Assert.AreEqual(typeof(ExtensionV2).AssemblyQualifiedName, types[0].AssemblyQualifiedName); + Assert.AreEqual(typeof(ExtensionV1).AssemblyQualifiedName, types[1].AssemblyQualifiedName); + } + } + + public class ExtensionV2 : ExtensionV1 + { + + } + + public class ExtensionV1 + { + + } +} + + +namespace Microsoft.VisualStudio.TestPlatform +{ + using System; + + [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = true)] + internal sealed class TestExtensionTypesV2Attribute : Attribute + { + public string ExtensionType { get; } + public string ExtensionIdentifier { get; } + public Type ExtensionImplementation { get; } + public int Version { get; } + + public TestExtensionTypesV2Attribute(string extensionType, string extensionIdentifier, Type extensionImplementation, int version, string futureUse = null) + { + ExtensionType = extensionType; + ExtensionIdentifier = extensionIdentifier; + ExtensionImplementation = extensionImplementation; + Version = version; + } + } +} diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index a10e739de0..85c009e1c1 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -12,6 +12,7 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.DataCollectorAttachme using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; + using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -42,18 +43,18 @@ public void Create_ShouldReturnListOfAttachmentProcessors() invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData2"), typeof(SampleData2Collector).AssemblyQualifiedName, typeof(SampleData2Collector).Assembly.Location, true)); invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData3"), typeof(SampleData3Collector).AssemblyQualifiedName, typeof(SampleData3Collector).Assembly.Location, true)); // act - var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray()); + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); // assert - Assert.AreEqual(3, dataCollectorAttachmentsProcessors.Count); + Assert.AreEqual(3, dataCollectorAttachmentsProcessors.Length); - Assert.IsTrue(dataCollectorAttachmentsProcessors.ContainsKey("Sample")); - Assert.IsTrue(dataCollectorAttachmentsProcessors.ContainsKey("SampleData3")); - Assert.IsTrue(dataCollectorAttachmentsProcessors.ContainsKey("Code Coverage")); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "Sample") == 1); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "SampleData3") == 1); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "Code Coverage") == 1); - Assert.AreEqual(typeof(DataCollectorAttachmentProcessor).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Sample"].GetType().AssemblyQualifiedName); - Assert.AreEqual(typeof(DataCollectorAttachmentProcessor2).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["SampleData3"].GetType().AssemblyQualifiedName); - Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Code Coverage"].GetType().AssemblyQualifiedName); + Assert.AreEqual(typeof(DataCollectorAttachmentProcessor).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); + Assert.AreEqual(typeof(DataCollectorAttachmentProcessor2).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[1].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[2].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); } [DataTestMethod] @@ -62,11 +63,11 @@ public void Create_ShouldReturnListOfAttachmentProcessors() public void Create_EmptyOrNullInvokedDataCollector_ShouldReturnCodeCoverageDataAttachmentsHandler(bool empty) { // act - var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(empty ? new InvokedDataCollector[0] : null); + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(empty ? new InvokedDataCollector[0] : null, null); //assert - Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count); - Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Code Coverage"].GetType().AssemblyQualifiedName); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Length); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); } [TestMethod] @@ -77,26 +78,27 @@ public void Create_ShouldNotFailIfWrongDataCollectorAttachmentProcessor() invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData4"), typeof(SampleData4Collector).AssemblyQualifiedName, typeof(SampleData4Collector).Assembly.Location, true)); // act - var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray()); + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); // assert - Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count); - Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["Code Coverage"].GetType().AssemblyQualifiedName); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Length); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); } [TestMethod] - public void Create_ShouldNotAddTwoTimeCodeCoverageDataAttachmentsHandler() + public void Create_ShouldAddTwoTimeCodeCoverageDataAttachmentsHandler() { // arrange List invokedDataCollectors = new List(); invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://microsoft/CodeCoverage/2.0"), typeof(SampleData5Collector).AssemblyQualifiedName, typeof(SampleData5Collector).Assembly.Location, true)); // act - var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray()); + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); // assert - Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count); - Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors["SampleData5"].GetType().AssemblyQualifiedName); + Assert.AreEqual(2, dataCollectorAttachmentsProcessors.Length); + Assert.AreEqual(typeof(DataCollectorAttachmentProcessorCodeCoverage).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[1].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); } } @@ -164,9 +166,9 @@ public override void Initialize( } } - [DataCollectorFriendlyName("SampleData5")] + [DataCollectorFriendlyName("Code Coverage")] [DataCollectorTypeUri("datacollector://microsoft/CodeCoverage/2.0")] - [DataCollectorAttachmentProcessor(typeof(CodeCoverageDataAttachmentsHandler))] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessorCodeCoverage))] public class SampleData5Collector : DataCollector { public override void Initialize( @@ -180,6 +182,21 @@ public override void Initialize( } } + public class DataCollectorAttachmentProcessorCodeCoverage : IDataCollectorAttachmentProcessor + { + public bool SupportsIncrementalProcessing => true; + + public IEnumerable GetExtensionUris() + { + yield return new Uri("datacollector://microsoft/CodeCoverage/2.0"); + } + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + return Task.FromResult(attachments); + } + } + public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor { public bool SupportsIncrementalProcessing => throw new NotImplementedException(); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs index 455564df1b..f0666f18e0 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs @@ -52,11 +52,12 @@ public TestRunAttachmentsProcessingManagerTests() mockAttachmentHandler1.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri1) }); mockAttachmentHandler2.Setup(h => h.GetExtensionUris()).Returns(new[] { new Uri(uri2) }); - mockDataCollectorAttachmentsProcessorsFactory.Setup(p => p.Create(It.IsAny())) - .Returns(new ReadOnlyDictionary(new Dictionary() + mockDataCollectorAttachmentsProcessorsFactory.Setup(p => p.Create(It.IsAny(), It.IsAny())) + .Returns(new DataCollectorAttachmentProcessor[] { - { "friendlyNameA", mockAttachmentHandler1.Object } , { "friendlyNameB" ,mockAttachmentHandler2.Object } - })); + new DataCollectorAttachmentProcessor( "friendlyNameA", mockAttachmentHandler1.Object ), + new DataCollectorAttachmentProcessor( "friendlyNameB" ,mockAttachmentHandler2.Object ) + }); manager = new TestRunAttachmentsProcessingManager(mockEventSource.Object, mockDataCollectorAttachmentsProcessorsFactory.Object); @@ -231,15 +232,15 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT await manager.ProcessTestRunAttachmentsAsync(Constants.EmptyRunSettings, mockRequestData.Object, inputAttachments, new InvokedDataCollector[0], mockEventsHandler.Object, cancellationTokenSource.Token); // assert - VerifyCompleteEvent(false, true, inputAttachments[0]); + VerifyCompleteEvent(false, false, inputAttachments[0]); mockEventsHandler.Verify(h => h.HandleTestRunAttachmentsProcessingProgress(It.IsAny()), Times.Never); - mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "exception message"), Times.Once); + mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "exception message"), Times.Never); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); + mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Once); mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - VerifyMetrics(inputCount: 1, outputCount: 1, status: "Failed"); + VerifyMetrics(inputCount: 1, outputCount: 1); } [TestMethod] @@ -260,11 +261,11 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachments_ Assert.AreEqual(1, result.Count); Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); - mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Never); + mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Once); mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); mockAttachmentHandler2.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); - VerifyMetrics(inputCount: 1, outputCount: 1, status: "Failed"); + VerifyMetrics(inputCount: 1, outputCount: 1); } [TestMethod] From 361ab26b5228d1f0d9963b86c6ae36e37a02f94e Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Sat, 18 Dec 2021 10:34:57 +0100 Subject: [PATCH 21/47] add some UTs --- .../Utilities/MetadataReaderHelper.cs | 2 +- ...taCollectorAttachmentsProcessorsFactory.cs | 23 +- .../DataCollectionTests.cs | 4 +- .../Utilities/MetadataReaderHelperTests.cs | 65 ++--- ...lectorAttachmentsProcessorsFactoryTests.cs | 242 ++++++++++-------- .../AttachmentProcessorDataCollector.csproj | 7 +- 6 files changed, 181 insertions(+), 162 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index 81da41f31c..ac5b5d756f 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -52,7 +52,7 @@ public Type[] DiscoverTestExtensionTypesV2Attribute(Assembly loadedAssembly, str #if !NETSTANDARD1_3 // Cache assembly, in VS scenario vstest.console is not unloaded so we don't want to load same asm more times. - Assembly assemblyToAnalyze = assemblyCache.GetOrAdd(assemblyFilePath, Assembly.Load(File.ReadAllBytes(assemblyFilePath))); + Assembly assemblyToAnalyze = assemblyCache.GetOrAdd(assemblyFilePath, Assembly.LoadFile(assemblyFilePath)); #else Assembly assemblyToAnalyze = loadedAssembly; #endif diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index 4124af8d1c..e7925a5605 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -1,4 +1,11 @@ -using Microsoft.VisualStudio.TestPlatform.Common.DataCollector; +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Microsoft.VisualStudio.TestPlatform.Common.DataCollector; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; using Microsoft.VisualStudio.TestPlatform.Common.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -6,11 +13,6 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.Utilities; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing { @@ -32,14 +34,14 @@ public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedD // /3.1.0 foreach (var invokedDataCollector in invokedDataCollectors.OrderByDescending(d => d.FilePath)) { - EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Analyzing data collector attachment processor Uri: {invokedDataCollector.Uri} AssemblyQualifiedName: {invokedDataCollector.AssemblyQualifiedName} FilePath: {invokedDataCollector.FilePath} HasAttachmentProcessor: {invokedDataCollector.HasAttachmentProcessor}"); - // We'll merge using only one AQN in case of more "same processors" in different assembly. - if (!invokedDataCollector.HasAttachmentProcessor || datacollectorsAttachmentsProcessors.ContainsKey(invokedDataCollector.AssemblyQualifiedName)) + if (!invokedDataCollector.HasAttachmentProcessor) { continue; } + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Analyzing data collector attachment processor Uri: {invokedDataCollector.Uri} AssemblyQualifiedName: {invokedDataCollector.AssemblyQualifiedName} FilePath: {invokedDataCollector.FilePath} HasAttachmentProcessor: {invokedDataCollector.HasAttachmentProcessor}"); + // We cache extension locally by file path var dataCollectorExtensionManager = dataCollectorExtensionManagerCache.GetOrAdd(invokedDataCollector.FilePath, DataCollectorExtensionManager.Create(invokedDataCollector.FilePath, true, TestSessionMessageLogger.Instance)); var dataCollectorExtension = dataCollectorExtensionManager.TryGetTestExtension(invokedDataCollector.Uri); @@ -50,6 +52,7 @@ public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedD try { dataCollectorAttachmentProcessorInstance = TestPluginManager.CreateTestExtension(attachmentProcessorType); + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Creation of collector attachment processor Uri: {invokedDataCollector.Uri} AssemblyQualifiedName: {invokedDataCollector.AssemblyQualifiedName} FilePath: {invokedDataCollector.FilePath} succeded"); } catch (Exception ex) { @@ -57,7 +60,7 @@ public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedD logger?.SendMessage(TestMessageLevel.Error, $"DataCollectorAttachmentsProcessorsFactory: Failed during the creation of data collector attachment processor '{attachmentProcessorType.AssemblyQualifiedName}'\n{ex}"); } - if (dataCollectorAttachmentProcessorInstance != null) + if (dataCollectorAttachmentProcessorInstance != null && !datacollectorsAttachmentsProcessors.ContainsKey(attachmentProcessorType.AssemblyQualifiedName)) { datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index f6f3f96544..f78b5f91af 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -121,7 +121,7 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) Path.GetFileNameWithoutExtension("AttachmentProcessorDataCollector"), "bin", IntegrationTestEnvironment.BuildConfiguration, - this.testEnvironment.RunnerFramework); + "netstandard2.0"); var arguments = PrepareArguments(new string[] { assemblyPath, secondAssemblyPath }, null, runSettings, this.FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: resultsDir); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{extensionsPath}"); @@ -164,7 +164,7 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) string dataCollectorLog = File.ReadAllText(dataCollectorLogFile); Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV1' version '1'")); Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV2' version '2'")); - Assert.IsTrue(dataCollectorLog.Contains("TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector.SampleDataCollectorV1, AttachmentProcessorDataCollector, Version=15.0.0.0, Culture=neutral, PublicKeyToken=null'; keeping the first one 'AttachmentProcessorDataCollector.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=15.0.0.0, Culture=neutral, PublicKeyToken=null'.")); + Assert.IsTrue(dataCollectorLog.Contains("TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector.SampleDataCollectorV1, AttachmentProcessorDataCollector, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null'; keeping the first one 'AttachmentProcessorDataCollector.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null'.")); } TryRemoveDirectory(resultsDir); diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs index da6f352486..c325dc8c63 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs @@ -1,14 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform; +using System; +using System.IO; +using System.Linq; +using System.Reflection; using Microsoft.VisualStudio.TestPlatform.Common.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; -using System.Reflection; -using TestPlatform.Common.UnitTests.Utilities; - -[assembly: TestExtensionTypesV2("ExcentionType", "ExtensionIdentified", typeof(ExtensionV1), 1)] -[assembly: TestExtensionTypesV2("ExcentionType", "ExtensionIdentified", typeof(ExtensionV2), 2)] namespace TestPlatform.Common.UnitTests.Utilities { @@ -20,42 +18,31 @@ public class MetadataReaderHelperTests [TestMethod] public void MetadataReaderHelper_GetCollectorExtensionTypes() { - var types = metadataReaderHelper.DiscoverTestExtensionTypesV2Attribute(Assembly.GetExecutingAssembly(), Assembly.GetExecutingAssembly().Location); - Assert.AreEqual(typeof(ExtensionV2).AssemblyQualifiedName, types[0].AssemblyQualifiedName); - Assert.AreEqual(typeof(ExtensionV1).AssemblyQualifiedName, types[1].AssemblyQualifiedName); + string testAssetsPath = GetTestAssetsFolder(); + var dataCollectorFilePath = Directory.GetFiles(testAssetsPath, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Where(x => x.Contains("bin")).Single(); + var types = metadataReaderHelper.DiscoverTestExtensionTypesV2Attribute(Assembly.LoadFile(dataCollectorFilePath), dataCollectorFilePath); + Assert.IsTrue(types.Any(), $"File {dataCollectorFilePath}"); + Assert.IsTrue(types[0].AssemblyQualifiedName.StartsWith("AttachmentProcessorDataCollector.SampleDataCollectorV2"), $"File {dataCollectorFilePath}"); + Assert.AreEqual(dataCollectorFilePath.Replace("/", @"\"), types[0].Assembly.Location.Replace("/", @"\"), $"File {dataCollectorFilePath}"); + Assert.IsTrue(types[1].AssemblyQualifiedName.StartsWith("AttachmentProcessorDataCollector.SampleDataCollectorV1"), $"File {dataCollectorFilePath}"); + Assert.AreEqual(dataCollectorFilePath.Replace("/", @"\"), types[1].Assembly.Location.Replace("/", @"\"), $"File {dataCollectorFilePath}"); } - } - - public class ExtensionV2 : ExtensionV1 - { - - } - - public class ExtensionV1 - { - - } -} - - -namespace Microsoft.VisualStudio.TestPlatform -{ - using System; - - [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = true)] - internal sealed class TestExtensionTypesV2Attribute : Attribute - { - public string ExtensionType { get; } - public string ExtensionIdentifier { get; } - public Type ExtensionImplementation { get; } - public int Version { get; } - public TestExtensionTypesV2Attribute(string extensionType, string extensionIdentifier, Type extensionImplementation, int version, string futureUse = null) + private string GetTestAssetsFolder() { - ExtensionType = extensionType; - ExtensionIdentifier = extensionIdentifier; - ExtensionImplementation = extensionImplementation; - Version = version; + string current = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + while (true) + { + if (File.Exists(Path.Combine(current, "TestPlatform.sln"))) + { + return Path.Combine(current, @"test/TestAssets"); + } + current = Path.GetDirectoryName(current); + if (current == Path.GetPathRoot(current)) + { + throw new Exception("Repo root path not tound"); + } + } } } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index 85c009e1c1..de600d4223 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -3,6 +3,14 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.DataCollectorAttachmentsProcessorsFactoryTests { + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -10,12 +18,6 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.DataCollectorAttachme using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; - using System; - using System.Collections.Generic; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using System.Xml; [TestClass] public class DataCollectorAttachmentsProcessorsFactoryTests @@ -48,9 +50,9 @@ public void Create_ShouldReturnListOfAttachmentProcessors() // assert Assert.AreEqual(3, dataCollectorAttachmentsProcessors.Length); - Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "Sample") == 1); - Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "SampleData3") == 1); - Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "Code Coverage") == 1); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "Sample")); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "SampleData3")); + Assert.AreEqual(1, dataCollectorAttachmentsProcessors.Count(x => x.FriendlyName == "Code Coverage")); Assert.AreEqual(typeof(DataCollectorAttachmentProcessor).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); Assert.AreEqual(typeof(DataCollectorAttachmentProcessor2).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[1].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); @@ -97,133 +99,165 @@ public void Create_ShouldAddTwoTimeCodeCoverageDataAttachmentsHandler() // assert Assert.AreEqual(2, dataCollectorAttachmentsProcessors.Length); - Assert.AreEqual(typeof(DataCollectorAttachmentProcessorCodeCoverage).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); + Assert.AreEqual(typeof(DataCollectorAttachmentProcessor).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[1].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); } - } - [DataCollectorFriendlyName("Sample")] - [DataCollectorTypeUri("datacollector://Sample")] - [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] - public class SampleDataCollector : DataCollector - { - public override void Initialize( - XmlElement configurationElement, - DataCollectionEvents events, - DataCollectionSink dataSink, - DataCollectionLogger logger, - DataCollectionEnvironmentContext environmentContext) + [TestMethod] + public void Create_ShouldLoadOrderingByFilePath() { + // arrange + // We cannot cleanup at the end because assembly will be copied into tmp directory and loaded + string testAssetsPath = GetTestAssetsFolder(); + var dataCollectorFilePath = Directory.GetFiles(testAssetsPath, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Where(x => x.Contains("bin")).Single(); + string tmpDir = Path.Combine(Path.GetTempPath(), nameof(Create_ShouldLoadOrderingByFilePath)); + Directory.CreateDirectory(tmpDir); + string version1 = Path.Combine(tmpDir, "1.0.0"); + Directory.CreateDirectory(version1); + File.Copy(dataCollectorFilePath, Path.Combine(version1, Path.GetFileName(dataCollectorFilePath)), true); + string version2 = Path.Combine(tmpDir, "1.0.1"); + Directory.CreateDirectory(version2); + File.Copy(dataCollectorFilePath, Path.Combine(version2, Path.GetFileName(dataCollectorFilePath)), true); - } - } + List invokedDataCollectors = new List(); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("my://sample/datacollector"), "AttachmentProcessorDataCollector.SampleDataCollectorV2", Path.Combine(version1, Path.GetFileName(dataCollectorFilePath)), true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("my://sample/datacollector"), "AttachmentProcessorDataCollector.SampleDataCollectorV2", Path.Combine(version2, Path.GetFileName(dataCollectorFilePath)), true)); - [DataCollectorFriendlyName("SampleData2")] - [DataCollectorTypeUri("datacollector://SampleData2")] - [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] - public class SampleData2Collector : DataCollector - { - public override void Initialize( - XmlElement configurationElement, - DataCollectionEvents events, - DataCollectionSink dataSink, - DataCollectionLogger logger, - DataCollectionEnvironmentContext environmentContext) - { + // act + var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); + // assert + Assert.AreEqual(2, dataCollectorAttachmentsProcessors.Length); + Assert.AreEqual("AttachmentProcessorDataCollector.SampleDataCollectorAttachmentProcessor, AttachmentProcessorDataCollector, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); + Assert.AreEqual(Path.Combine(version2, Path.GetFileName(dataCollectorFilePath)), dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().Assembly.Location); + Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[1].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); } - } - [DataCollectorFriendlyName("SampleData3")] - [DataCollectorTypeUri("datacollector://SampleData3")] - [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor2))] - public class SampleData3Collector : DataCollector - { - public override void Initialize( - XmlElement configurationElement, - DataCollectionEvents events, - DataCollectionSink dataSink, - DataCollectionLogger logger, - DataCollectionEnvironmentContext environmentContext) + private string GetTestAssetsFolder() { - + string current = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + while (true) + { + if (File.Exists(Path.Combine(current, "TestPlatform.sln"))) + { + return Path.Combine(current, @"test/TestAssets"); + } + current = Path.GetDirectoryName(current); + if (current == Path.GetPathRoot(current)) + { + throw new Exception("Repo root path not tound"); + } + } } - } - [DataCollectorFriendlyName("SampleData4")] - [DataCollectorTypeUri("datacollector://SampleData4")] - [DataCollectorAttachmentProcessor(typeof(string))] - public class SampleData4Collector : DataCollector - { - public override void Initialize( - XmlElement configurationElement, - DataCollectionEvents events, - DataCollectionSink dataSink, - DataCollectionLogger logger, - DataCollectionEnvironmentContext environmentContext) + [DataCollectorFriendlyName("Sample")] + [DataCollectorTypeUri("datacollector://Sample")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] + public class SampleDataCollector : DataCollector { - + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } } - } - [DataCollectorFriendlyName("Code Coverage")] - [DataCollectorTypeUri("datacollector://microsoft/CodeCoverage/2.0")] - [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessorCodeCoverage))] - public class SampleData5Collector : DataCollector - { - public override void Initialize( - XmlElement configurationElement, - DataCollectionEvents events, - DataCollectionSink dataSink, - DataCollectionLogger logger, - DataCollectionEnvironmentContext environmentContext) + [DataCollectorFriendlyName("SampleData2")] + [DataCollectorTypeUri("datacollector://SampleData2")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] + public class SampleData2Collector : DataCollector { - + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } } - } - - public class DataCollectorAttachmentProcessorCodeCoverage : IDataCollectorAttachmentProcessor - { - public bool SupportsIncrementalProcessing => true; - public IEnumerable GetExtensionUris() + [DataCollectorFriendlyName("SampleData3")] + [DataCollectorTypeUri("datacollector://SampleData3")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor2))] + public class SampleData3Collector : DataCollector { - yield return new Uri("datacollector://microsoft/CodeCoverage/2.0"); + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } } - public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + [DataCollectorFriendlyName("SampleData4")] + [DataCollectorTypeUri("datacollector://SampleData4")] + [DataCollectorAttachmentProcessor(typeof(string))] + public class SampleData4Collector : DataCollector { - return Task.FromResult(attachments); + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } } - } - - public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor - { - public bool SupportsIncrementalProcessing => throw new NotImplementedException(); - public IEnumerable GetExtensionUris() + [DataCollectorFriendlyName("SampleData5")] + [DataCollectorTypeUri("datacollector://microsoft/CodeCoverage/2.0")] + [DataCollectorAttachmentProcessor(typeof(DataCollectorAttachmentProcessor))] + public class SampleData5Collector : DataCollector { - throw new NotImplementedException(); + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + + } } - public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + public class DataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor { - throw new NotImplementedException(); - } - } + public bool SupportsIncrementalProcessing => throw new NotImplementedException(); - public class DataCollectorAttachmentProcessor2 : IDataCollectorAttachmentProcessor - { - public bool SupportsIncrementalProcessing => throw new NotImplementedException(); + public IEnumerable GetExtensionUris() + { + throw new NotImplementedException(); + } - public IEnumerable GetExtensionUris() - { - throw new NotImplementedException(); + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } } - public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + public class DataCollectorAttachmentProcessor2 : IDataCollectorAttachmentProcessor { - throw new NotImplementedException(); + public bool SupportsIncrementalProcessing => throw new NotImplementedException(); + + public IEnumerable GetExtensionUris() + { + throw new NotImplementedException(); + } + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } } } -} +} \ No newline at end of file diff --git a/test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj b/test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj index 2410868208..91ccb29a37 100644 --- a/test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj +++ b/test/TestAssets/AttachmentProcessorDataCollector/AttachmentProcessorDataCollector.csproj @@ -1,11 +1,6 @@  - netcoreapp2.1;net451 - netcoreapp3.1 - false - 15.0.0.0 - 1.6.0 - $(MSBuildProjectDirectory)\RunSettings.runsettings + netstandard2.0 From 7cf6e314d5f3ef5d036f25b02ab8420184e53c1c Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Sat, 18 Dec 2021 10:45:01 +0100 Subject: [PATCH 22/47] improve logging --- .../DataCollectorAttachmentsProcessorsFactory.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs index e7925a5605..bd27179521 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactory.cs @@ -52,7 +52,7 @@ public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedD try { dataCollectorAttachmentProcessorInstance = TestPluginManager.CreateTestExtension(attachmentProcessorType); - EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Creation of collector attachment processor Uri: {invokedDataCollector.Uri} AssemblyQualifiedName: {invokedDataCollector.AssemblyQualifiedName} FilePath: {invokedDataCollector.FilePath} succeded"); + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Creation of collector attachment processor '{attachmentProcessorType.AssemblyQualifiedName}' from file '{invokedDataCollector.FilePath}' succeded"); } catch (Exception ex) { @@ -63,6 +63,7 @@ public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedD if (dataCollectorAttachmentProcessorInstance != null && !datacollectorsAttachmentsProcessors.ContainsKey(attachmentProcessorType.AssemblyQualifiedName)) { datacollectorsAttachmentsProcessors.Add(attachmentProcessorType.AssemblyQualifiedName, new Tuple(dataCollectorExtension.Metadata.FriendlyName, dataCollectorAttachmentProcessorInstance)); + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Collector attachment processor '{attachmentProcessorType.AssemblyQualifiedName}' from file '{invokedDataCollector.FilePath}' added to the 'run list'"); } } else @@ -77,6 +78,7 @@ public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedD if (!datacollectorsAttachmentsProcessors.ContainsKey(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName)) { datacollectorsAttachmentsProcessors.Add(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, new Tuple(CoverageFriendlyName, new CodeCoverageDataAttachmentsHandler())); + EqtTrace.Info($"DataCollectorAttachmentsProcessorsFactory: Collector attachment processor '{typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName}' for the data collector with friendly name '{CoverageFriendlyName}' added to the 'run list'"); } var finalDatacollectorsAttachmentsProcessors = new List(); From 34e49f65f815cac36e81565dd99460c5027fb5e2 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Sat, 18 Dec 2021 11:56:46 +0100 Subject: [PATCH 23/47] add some tests logs --- .../Utilities/MetadataReaderHelperTests.cs | 4 +++- .../DataCollectorAttachmentsProcessorsFactoryTests.cs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs index c325dc8c63..2edefa0507 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Utilities/MetadataReaderHelperTests.cs @@ -35,7 +35,9 @@ private string GetTestAssetsFolder() { if (File.Exists(Path.Combine(current, "TestPlatform.sln"))) { - return Path.Combine(current, @"test/TestAssets"); + string testAssetsPath = Path.Combine(current, @"test/TestAssets"); + Assert.IsTrue(Directory.Exists(testAssetsPath), $"Directory not found '{testAssetsPath}'"); + return testAssetsPath; } current = Path.GetDirectoryName(current); if (current == Path.GetPathRoot(current)) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index de600d4223..3910f8655f 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -140,7 +140,9 @@ private string GetTestAssetsFolder() { if (File.Exists(Path.Combine(current, "TestPlatform.sln"))) { - return Path.Combine(current, @"test/TestAssets"); + string testAssetsPath = Path.Combine(current, @"test/TestAssets"); + Assert.IsTrue(Directory.Exists(testAssetsPath), $"Directory not found '{testAssetsPath}'"); + return testAssetsPath; } current = Path.GetDirectoryName(current); if (current == Path.GetPathRoot(current)) From fe87ecf7147b004a03f6328b893ebd20c91af07c Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Sat, 18 Dec 2021 12:48:20 +0100 Subject: [PATCH 24/47] Build AttachmentProcessorDataCollector project for UTs --- TestPlatform.sln | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/TestPlatform.sln b/TestPlatform.sln index 4dc2355f38..f7e9b97ff0 100644 --- a/TestPlatform.sln +++ b/TestPlatform.sln @@ -179,6 +179,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestPlatform.Playground", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTest1", "playground\MSTest1\MSTest1.csproj", "{57A61A09-10AD-44BE-8DF4-A6FD108F7DF7}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataCollector", "test\TestAssets\AttachmentProcessorDataCollector\AttachmentProcessorDataCollector.csproj", "{B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 @@ -868,6 +870,18 @@ Global {57A61A09-10AD-44BE-8DF4-A6FD108F7DF7}.Release|x64.Build.0 = Release|Any CPU {57A61A09-10AD-44BE-8DF4-A6FD108F7DF7}.Release|x86.ActiveCfg = Release|Any CPU {57A61A09-10AD-44BE-8DF4-A6FD108F7DF7}.Release|x86.Build.0 = Release|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Debug|x64.ActiveCfg = Debug|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Debug|x64.Build.0 = Debug|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Debug|x86.ActiveCfg = Debug|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Debug|x86.Build.0 = Debug|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|Any CPU.Build.0 = Release|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x64.ActiveCfg = Release|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x64.Build.0 = Release|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x86.ActiveCfg = Release|Any CPU + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -942,6 +956,7 @@ Global {8238A052-D626-49EB-A011-51DC6D0DBA30} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} {545A88D3-1AE2-4D39-9B7C-C691768AD17F} = {6CE2F530-582B-4695-A209-41065E103426} {57A61A09-10AD-44BE-8DF4-A6FD108F7DF7} = {6CE2F530-582B-4695-A209-41065E103426} + {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B} = {D9A30E32-D466-4EC5-B4F2-62E17562279B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD} From 72174da3c5cf43d8bfa382617258c3dbb116cd9a Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Sun, 19 Dec 2021 17:51:05 +0100 Subject: [PATCH 25/47] fix UTs --- .../DataCollectorAttachmentsProcessorsFactoryTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index 3910f8655f..4c88d2b90c 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -8,6 +8,7 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.DataCollectorAttachme using System.IO; using System.Linq; using System.Reflection; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -128,7 +129,7 @@ public void Create_ShouldLoadOrderingByFilePath() // assert Assert.AreEqual(2, dataCollectorAttachmentsProcessors.Length); - Assert.AreEqual("AttachmentProcessorDataCollector.SampleDataCollectorAttachmentProcessor, AttachmentProcessorDataCollector, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); + Assert.IsTrue(Regex.IsMatch(dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName, @"AttachmentProcessorDataCollector\.SampleDataCollectorAttachmentProcessor, AttachmentProcessorDataCollector, Version=.*, Culture=neutral, PublicKeyToken=null")); Assert.AreEqual(Path.Combine(version2, Path.GetFileName(dataCollectorFilePath)), dataCollectorAttachmentsProcessors[0].DataCollectorAttachmentProcessorInstance.GetType().Assembly.Location); Assert.AreEqual(typeof(CodeCoverageDataAttachmentsHandler).AssemblyQualifiedName, dataCollectorAttachmentsProcessors[1].DataCollectorAttachmentProcessorInstance.GetType().AssemblyQualifiedName); } From 39fb97e89daf68c88cef188ffb229fb229ab9c38 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 14:39:58 +0100 Subject: [PATCH 26/47] Add telemetry, return friendly name for the invoked data collector --- .../DataCollection/DataCollectionManager.cs | 1 + .../Telemetry/TelemetryDataConstants.cs | 2 + .../DataCollectionRequestHandler.cs | 6 +++ .../InvokedDataCollector.cs | 19 ++++++--- .../PublicAPI/PublicAPI.Shipped.txt | 3 +- .../DataCollectionRequestSenderTests.cs | 2 +- ...lectorAttachmentsProcessorsFactoryTests.cs | 14 +++---- ...estRunAttachmentsProcessingManagerTests.cs | 42 +++++++++---------- .../ParallelRunDataAggregatorTests.cs | 4 +- ...DataCollectionTestRunEventsHandlerTests.cs | 2 +- .../VsTestConsoleRequestSenderTests.cs | 12 +++--- 11 files changed, 63 insertions(+), 44 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs index a319e78517..de09f2c22f 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs @@ -266,6 +266,7 @@ public Collection GetInvokedDataCollectors() foreach (DataCollectorInformation dataCollectorInformation in this.RunDataCollectors.Values) { invokedDataCollector.Add(new InvokedDataCollector(dataCollectorInformation.DataCollectorConfig.TypeUri, + dataCollectorInformation.DataCollectorConfig.FriendlyName, dataCollectorInformation.DataCollectorConfig.DataCollectorType.AssemblyQualifiedName, dataCollectorInformation.DataCollectorConfig.FilePath, dataCollectorInformation.DataCollectorConfig.HasAttachmentsProcessor())); diff --git a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs index 016ea55be3..985001dabc 100644 --- a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs +++ b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs @@ -26,6 +26,8 @@ public static class TelemetryDataConstants public static string DataCollectorsEnabled = "VS.TestRun.DataCollectorsEnabled"; + internal const string InvokedDataCollector = "VS.TestRun.InvokedDataCollector"; + internal const string DataCollectorsCorProfiler = "VS.TestPlatform.DataCollector.CorProfiler"; internal const string DataCollectorsCoreClrProfiler = "VS.TestPlatform.DataCollector.CoreClrProfiler"; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index ef4ee8fd0a..fc01853c53 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -8,6 +8,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollect using System.Collections.ObjectModel; using System.Globalization; using System.IO; + using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -391,6 +392,11 @@ private void HandleAfterTestRunEnd(Message message) var attachmentsets = this.dataCollectionManager.SessionEnded(isCancelled); var invokedDataCollectors = this.dataCollectionManager.GetInvokedDataCollectors(); + + // For the invoked collectors we report the same information as ProxyDataCollectionManager.cs line ~416 + var invokedDataCollectorsForMetrics = invokedDataCollectors.Select(x => new { x.Uri, x.FriendlyName, x.HasAttachmentProcessor }.ToString()); + this.requestData.MetricsCollection.Add(TelemetryDataConstants.InvokedDataCollector, string.Join(",", invokedDataCollectorsForMetrics.ToArray())); + var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, invokedDataCollectors, this.requestData.MetricsCollection.Metrics); // Dispose all datacollectors before sending attachments to vstest.console process. diff --git a/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs index 3580b5279f..f08d33c534 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs @@ -15,12 +15,13 @@ public sealed class InvokedDataCollector : IEquatable /// Data collector assembly qualified name /// Data collector file path /// True if data collector registers an attachment processor - public InvokedDataCollector(Uri uri, string assemblyQualifiedName, string filePath, bool hasAttachmentProcessor) + public InvokedDataCollector(Uri uri, string friendlyName, string assemblyQualifiedName, string filePath, bool hasAttachmentProcessor) { - this.Uri = uri; - this.AssemblyQualifiedName = assemblyQualifiedName; - this.FilePath = filePath; + this.Uri = uri ?? throw new ArgumentException(nameof(uri)); + this.AssemblyQualifiedName = assemblyQualifiedName ?? throw new ArgumentException(nameof(assemblyQualifiedName)); ; + this.FilePath = filePath ?? throw new ArgumentException(nameof(filePath)); ; this.HasAttachmentProcessor = hasAttachmentProcessor; + this.FriendlyName = friendlyName ?? throw new ArgumentException(nameof(friendlyName)); ; } /// @@ -29,6 +30,12 @@ public InvokedDataCollector(Uri uri, string assemblyQualifiedName, string filePa [DataMember] public Uri Uri { get; private set; } + /// + /// DataCollector FriednlyName. + /// + [DataMember] + public string FriendlyName { get; private set; } + /// /// AssemblyQualifiedName of data collector. /// @@ -61,6 +68,7 @@ public bool Equals(InvokedDataCollector other) return this.HasAttachmentProcessor == other.HasAttachmentProcessor && this.Uri.AbsoluteUri == other.Uri.AbsoluteUri && + this.FriendlyName == other.FriendlyName && this.AssemblyQualifiedName == other.AssemblyQualifiedName && this.FilePath == other.FilePath; } @@ -82,6 +90,7 @@ public override int GetHashCode() unchecked { var hashCode = this.Uri.GetHashCode(); + hashCode = (hashCode * 397) ^ this.FriendlyName.GetHashCode(); hashCode = (hashCode * 397) ^ this.AssemblyQualifiedName.GetHashCode(); hashCode = (hashCode * 397) ^ (this.FilePath != null ? this.FilePath.GetHashCode() : 0); hashCode = (hashCode * 397) ^ this.HasAttachmentProcessor.GetHashCode(); @@ -94,6 +103,6 @@ public override int GetHashCode() /// /// String representation public override string ToString() - => $"Uri: '{Uri}' AssemblyQualifiedName: '{AssemblyQualifiedName}' FilePath: '{FilePath}' HasAttachmentProcessor: '{HasAttachmentProcessor}'"; + => $"Uri: '{Uri}' FriendlyName: '{FriendlyName}' AssemblyQualifiedName: '{AssemblyQualifiedName}' FilePath: '{FilePath}' HasAttachmentProcessor: '{HasAttachmentProcessor}'"; } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt index 5d8ad972d4..d8b50aefa4 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt @@ -877,9 +877,10 @@ virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.Properties.ge virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.ProtectedGetPropertyValue(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestProperty property, object defaultValue) -> object virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.ProtectedSetPropertyValue(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestProperty property, object value) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector -Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.InvokedDataCollector(System.Uri uri, string assemblyQualifiedName, string filePath, bool hasAttachmentProcessor) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.InvokedDataCollector(System.Uri uri, string friendlyName, string assemblyQualifiedName, string filePath, bool hasAttachmentProcessor) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.Equals(Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector other) -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.AssemblyQualifiedName.get -> string +Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.FriendlyName.get -> string Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.FilePath.get -> string Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.HasAttachmentProcessor.get -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector.Uri.get -> System.Uri diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs index d3d578d53b..1da7998fb6 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/DataCollectionRequestSenderTests.cs @@ -38,7 +38,7 @@ public void SendAfterTestRunEndAndGetResultShouldReturnAttachments() var displayName = "CustomDataCollector"; var attachment = new AttachmentSet(datacollectorUri, displayName); attachment.Attachments.Add(new UriDataAttachment(attachmentUri, "filename.txt")); - var invokedDataCollector = new InvokedDataCollector(datacollectorUri, typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false); + var invokedDataCollector = new InvokedDataCollector(datacollectorUri, displayName, typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false); this.mockDataSerializer.Setup(x => x.DeserializePayload(It.IsAny())).Returns( new AfterTestRunEndResult(new Collection() { attachment }, new Collection() { invokedDataCollector }, new Dictionary())); this.mockCommunicationManager.Setup(x => x.ReceiveMessage()).Returns(new Message() { MessageType = MessageType.AfterTestRunEndResult, Payload = null }); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs index 4c88d2b90c..6212061853 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/DataCollectorAttachmentsProcessorsFactoryTests.cs @@ -42,9 +42,9 @@ public void Create_ShouldReturnListOfAttachmentProcessors() { // arrange List invokedDataCollectors = new List(); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://Sample"), typeof(SampleDataCollector).AssemblyQualifiedName, typeof(SampleDataCollector).Assembly.Location, true)); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData2"), typeof(SampleData2Collector).AssemblyQualifiedName, typeof(SampleData2Collector).Assembly.Location, true)); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData3"), typeof(SampleData3Collector).AssemblyQualifiedName, typeof(SampleData3Collector).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://Sample"), "Sample", typeof(SampleDataCollector).AssemblyQualifiedName, typeof(SampleDataCollector).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData2"), "SampleData2", typeof(SampleData2Collector).AssemblyQualifiedName, typeof(SampleData2Collector).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData3"), "SampleData3", typeof(SampleData3Collector).AssemblyQualifiedName, typeof(SampleData3Collector).Assembly.Location, true)); // act var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); @@ -78,7 +78,7 @@ public void Create_ShouldNotFailIfWrongDataCollectorAttachmentProcessor() { // arrange List invokedDataCollectors = new List(); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData4"), typeof(SampleData4Collector).AssemblyQualifiedName, typeof(SampleData4Collector).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://SampleData4"), "SampleData4", typeof(SampleData4Collector).AssemblyQualifiedName, typeof(SampleData4Collector).Assembly.Location, true)); // act var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); @@ -93,7 +93,7 @@ public void Create_ShouldAddTwoTimeCodeCoverageDataAttachmentsHandler() { // arrange List invokedDataCollectors = new List(); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://microsoft/CodeCoverage/2.0"), typeof(SampleData5Collector).AssemblyQualifiedName, typeof(SampleData5Collector).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://microsoft/CodeCoverage/2.0"), "SampleData5", typeof(SampleData5Collector).AssemblyQualifiedName, typeof(SampleData5Collector).Assembly.Location, true)); // act var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); @@ -121,8 +121,8 @@ public void Create_ShouldLoadOrderingByFilePath() File.Copy(dataCollectorFilePath, Path.Combine(version2, Path.GetFileName(dataCollectorFilePath)), true); List invokedDataCollectors = new List(); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("my://sample/datacollector"), "AttachmentProcessorDataCollector.SampleDataCollectorV2", Path.Combine(version1, Path.GetFileName(dataCollectorFilePath)), true)); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("my://sample/datacollector"), "AttachmentProcessorDataCollector.SampleDataCollectorV2", Path.Combine(version2, Path.GetFileName(dataCollectorFilePath)), true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("my://sample/datacollector"), "sample", "AttachmentProcessorDataCollector.SampleDataCollectorV2", Path.Combine(version1, Path.GetFileName(dataCollectorFilePath)), true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("my://sample/datacollector"), "sample", "AttachmentProcessorDataCollector.SampleDataCollectorV2", Path.Combine(version2, Path.GetFileName(dataCollectorFilePath)), true)); // act var dataCollectorAttachmentsProcessors = dataCollectorAttachmentsProcessorsFactory.Create(invokedDataCollectors.ToArray(), null); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs index f0666f18e0..3829ae5d1f 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs @@ -644,11 +644,11 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldFlowCorrectDataCollectorC List invokedDataCollectors = new List { - new InvokedDataCollector(new Uri(uri1), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) + new InvokedDataCollector(new Uri(uri1),withConfig ? "friendlyNameA" : "friendlyNameB", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }; string runSettingsXml = -$@" + $@" @@ -663,25 +663,25 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldFlowCorrectDataCollectorC "; mockAttachmentHandler1.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) - .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => - { + .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => + { // assert if (withConfig) - { - Assert.IsNotNull(configurationElement); - Assert.AreEqual("Value", configurationElement.InnerXml); - } - else - { - Assert.IsNull(configurationElement); - } + { + Assert.IsNotNull(configurationElement); + Assert.AreEqual("Value", configurationElement.InnerXml); + } + else + { + Assert.IsNull(configurationElement); + } - ICollection outputAttachments = new List - { + ICollection outputAttachments = new List + { new AttachmentSet(new Uri(uri2), "uri2_output") - }; - return Task.FromResult(outputAttachments); - }); + }; + return Task.FromResult(outputAttachments); + }); // act await manager.ProcessTestRunAttachmentsAsync(runSettingsXml, mockRequestData.Object, inputAttachments, invokedDataCollectors, mockEventsHandler.Object, cancellationTokenSource.Token); @@ -731,8 +731,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldNotConsumeAttachmentsIfPr mockAttachmentHandler2.Setup(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) .Returns((XmlElement configurationElement, ICollection i1, IProgress progress, IMessageLogger logger, CancellationToken cancellation) => { - // assert - Assert.IsTrue(firstProcessorFailed); + // assert + Assert.IsTrue(firstProcessorFailed); Assert.AreEqual(1, i1.Count); Assert.AreEqual(3, i1.Single().Attachments.Count); for (int i = 0; i < i1.Single().Attachments.Count; i++) @@ -789,8 +789,8 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldNotConsumeAttachmentsIfAl { try { - // assert - Assert.IsTrue(firstProcessorFailed); + // assert + Assert.IsTrue(firstProcessorFailed); Assert.AreEqual(1, i1.Count); Assert.AreEqual(3, i1.Single().Attachments.Count); for (int i = 0; i < i1.Single().Attachments.Count; i++) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs index ed917f2e4e..348005c78b 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs @@ -86,14 +86,14 @@ public void AggregateShouldAggregateInvokedCollectorsCorrectly() var invokedDataCollectors = new Collection() { - new InvokedDataCollector(new Uri("datacollector://sample"),typeof(string).AssemblyQualifiedName,typeof(string).Assembly.Location,false) + new InvokedDataCollector(new Uri("datacollector://sample"),"sample", typeof(string).AssemblyQualifiedName,typeof(string).Assembly.Location,false) }; aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors); Assert.AreEqual(1, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have data."); var invokedDataCollectors2 = new Collection() { - new InvokedDataCollector(new Uri("datacollector://sample2"),typeof(int).AssemblyQualifiedName,typeof(int).Assembly.Location,false) + new InvokedDataCollector(new Uri("datacollector://sample2"),"sample2", typeof(int).AssemblyQualifiedName,typeof(int).Assembly.Location,false) }; aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors2); Assert.AreEqual(2, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have aggregated data."); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs index 1744ba41cc..d801a23c3d 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs @@ -109,7 +109,7 @@ public void HandleRawMessageShouldInvokeAfterTestRunEndPassingTrueIfRequestCance public void HandleRawMessageShouldInvokeAfterTestRunEndAndReturnInvokedDataCollectors() { var invokedDataCollectors = new Collection(); - invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, true)); + invokedDataCollectors.Add(new InvokedDataCollector(new Uri("datacollector://sample"), "sample", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, true)); var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new Collection(), new TimeSpan()); this.mockDataSerializer.Setup(x => x.DeserializeMessage(It.IsAny())).Returns(new Message() { MessageType = MessageType.ExecutionComplete }); diff --git a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs index 22e38b309a..9759acdac3 100644 --- a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs +++ b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs @@ -1924,7 +1924,7 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithZeroAttachment await this.requestSender.ProcessTestRunAttachmentsAsync( new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, - new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), "sample", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, Constants.EmptyRunSettings, true, mockHandler.Object, @@ -1963,7 +1963,7 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithOneAttachment( await this.requestSender.ProcessTestRunAttachmentsAsync( new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, - new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), "sample", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, Constants.EmptyRunSettings, true, mockHandler.Object, @@ -2008,7 +2008,7 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithOneAttachmentA await this.requestSender.ProcessTestRunAttachmentsAsync( new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, - new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), "sample", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, Constants.EmptyRunSettings, false, mockHandler.Object, @@ -2061,7 +2061,7 @@ public async Task ProcessTestRunAttachmentsAsyncShouldCompleteWithOneAttachmentA await this.requestSender.ProcessTestRunAttachmentsAsync( new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, - new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), "sample", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, Constants.EmptyRunSettings, false, mockHandler.Object, @@ -2110,7 +2110,7 @@ public async Task ProcessTestRunAttachmentsAsyncShouldSendCancelMessageIfCancell await this.requestSender.ProcessTestRunAttachmentsAsync( new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, - new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), "sample", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, Constants.EmptyRunSettings, false, mockHandler.Object, @@ -2151,7 +2151,7 @@ public async Task ProcessTestRunAttachmentsAsyncShouldSendCancelMessageIfCancell await this.requestSender.ProcessTestRunAttachmentsAsync( new List { new AttachmentSet(new Uri("http://www.bing.com"), "a") }, - new List() { new InvokedDataCollector(new Uri("datacollector://sample"), typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, + new List() { new InvokedDataCollector(new Uri("datacollector://sample"), "sample", typeof(string).AssemblyQualifiedName, typeof(string).Assembly.Location, false) }, Constants.EmptyRunSettings, true, mockHandler.Object, From a25b999bdca35e062cbf83858001d16d1d395e50 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 14:41:37 +0100 Subject: [PATCH 27/47] rename telemetry key --- .../Telemetry/TelemetryDataConstants.cs | 2 +- .../DataCollectionRequestHandler.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs index 985001dabc..e9c3515892 100644 --- a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs +++ b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs @@ -26,7 +26,7 @@ public static class TelemetryDataConstants public static string DataCollectorsEnabled = "VS.TestRun.DataCollectorsEnabled"; - internal const string InvokedDataCollector = "VS.TestRun.InvokedDataCollector"; + internal const string InvokedDataCollectors = "VS.TestRun.InvokedDataCollectors"; internal const string DataCollectorsCorProfiler = "VS.TestPlatform.DataCollector.CorProfiler"; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index fc01853c53..78b32c8666 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -395,7 +395,7 @@ private void HandleAfterTestRunEnd(Message message) // For the invoked collectors we report the same information as ProxyDataCollectionManager.cs line ~416 var invokedDataCollectorsForMetrics = invokedDataCollectors.Select(x => new { x.Uri, x.FriendlyName, x.HasAttachmentProcessor }.ToString()); - this.requestData.MetricsCollection.Add(TelemetryDataConstants.InvokedDataCollector, string.Join(",", invokedDataCollectorsForMetrics.ToArray())); + this.requestData.MetricsCollection.Add(TelemetryDataConstants.InvokedDataCollectors, string.Join(",", invokedDataCollectorsForMetrics.ToArray())); var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, invokedDataCollectors, this.requestData.MetricsCollection.Metrics); From f4a83e21f405281d84c1ea80ffd5edb917639137 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 14:49:46 +0100 Subject: [PATCH 28/47] fix typo --- .../InvokedDataCollector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs index f08d33c534..fc250ca92a 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs @@ -18,10 +18,10 @@ public sealed class InvokedDataCollector : IEquatable public InvokedDataCollector(Uri uri, string friendlyName, string assemblyQualifiedName, string filePath, bool hasAttachmentProcessor) { this.Uri = uri ?? throw new ArgumentException(nameof(uri)); + this.FriendlyName = friendlyName ?? throw new ArgumentException(nameof(friendlyName)); this.AssemblyQualifiedName = assemblyQualifiedName ?? throw new ArgumentException(nameof(assemblyQualifiedName)); ; this.FilePath = filePath ?? throw new ArgumentException(nameof(filePath)); ; this.HasAttachmentProcessor = hasAttachmentProcessor; - this.FriendlyName = friendlyName ?? throw new ArgumentException(nameof(friendlyName)); ; } /// @@ -31,7 +31,7 @@ public InvokedDataCollector(Uri uri, string friendlyName, string assemblyQualifi public Uri Uri { get; private set; } /// - /// DataCollector FriednlyName. + /// DataCollector FriendlyName. /// [DataMember] public string FriendlyName { get; private set; } From 92bcc733bf23580de87ef24c37c17d1ac22c8676 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 15:27:25 +0100 Subject: [PATCH 29/47] align public api names --- .../PublicAPI/PublicAPI.Shipped.txt | 2 +- .../VsTestConsoleWrapper.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt index b18957df29..a970cc5243 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt @@ -114,7 +114,7 @@ Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.EndSe Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.InitializeExtensions(System.Collections.Generic.IEnumerable pathToAdditionalExtensions) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.InitializeExtensionsAsync(System.Collections.Generic.IEnumerable pathToAdditionalExtensions) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollectors, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollectors, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs index 56f3a0b664..2e94316a74 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs @@ -940,7 +940,7 @@ public async Task ProcessTestRunAttachmentsAsync( string processingSettings, bool isLastBatch, bool collectMetrics, - ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, + ITestRunAttachmentsProcessingEventsHandler eventsHandler, CancellationToken cancellationToken) { this.testPlatformEventSource.TranslationLayerTestRunAttachmentsProcessingStart(); @@ -951,7 +951,7 @@ await requestSender.ProcessTestRunAttachmentsAsync( invokedDataCollectors, processingSettings, collectMetrics, - testSessionEventsHandler, + eventsHandler, cancellationToken).ConfigureAwait(false); } From ae5ef169a02ac56c5501e1431e4b6cfe7a1f8e8d Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 15:30:15 +0100 Subject: [PATCH 30/47] refactor api name --- .../PublicAPI/PublicAPI.Shipped.txt | 4 ++-- .../VsTestConsoleWrapper.cs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt index a970cc5243..cb8161d464 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt @@ -113,8 +113,8 @@ Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.Disco Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.EndSession() -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.InitializeExtensions(System.Collections.Generic.IEnumerable pathToAdditionalExtensions) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.InitializeExtensionsAsync(System.Collections.Generic.IEnumerable pathToAdditionalExtensions) -> System.Threading.Tasks.Task -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollectors, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.ProcessTestRunAttachmentsAsync(System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollectors, string processingSettings, bool isLastBatch, bool collectMetrics, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void Microsoft.TestPlatform.VsTestConsole.TranslationLayer.VsTestConsoleWrapper.RunTests(System.Collections.Generic.IEnumerable testCases, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestPlatformOptions options, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunEventsHandler testRunEventsHandler) -> void diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs index 2e94316a74..7911508244 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleWrapper.cs @@ -940,7 +940,7 @@ public async Task ProcessTestRunAttachmentsAsync( string processingSettings, bool isLastBatch, bool collectMetrics, - ITestRunAttachmentsProcessingEventsHandler eventsHandler, + ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, CancellationToken cancellationToken) { this.testPlatformEventSource.TranslationLayerTestRunAttachmentsProcessingStart(); @@ -951,7 +951,7 @@ await requestSender.ProcessTestRunAttachmentsAsync( invokedDataCollectors, processingSettings, collectMetrics, - eventsHandler, + testSessionEventsHandler, cancellationToken).ConfigureAwait(false); } @@ -961,9 +961,9 @@ public Task ProcessTestRunAttachmentsAsync( string processingSettings, bool isLastBatch, bool collectMetrics, - ITestRunAttachmentsProcessingEventsHandler eventsHandler, + ITestRunAttachmentsProcessingEventsHandler testSessionEventsHandler, CancellationToken cancellationToken) - => ProcessTestRunAttachmentsAsync(attachments, Enumerable.Empty(), processingSettings, isLastBatch, collectMetrics, eventsHandler, cancellationToken); + => ProcessTestRunAttachmentsAsync(attachments, Enumerable.Empty(), processingSettings, isLastBatch, collectMetrics, testSessionEventsHandler, cancellationToken); #endregion From d5a42e7efc7c49e343e6c04b48f5aa92f9318b6d Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 16:19:16 +0100 Subject: [PATCH 31/47] fix UTs --- .../DataCollectionRequestHandler.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index 78b32c8666..dfa9dbed5d 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -393,9 +393,12 @@ private void HandleAfterTestRunEnd(Message message) var attachmentsets = this.dataCollectionManager.SessionEnded(isCancelled); var invokedDataCollectors = this.dataCollectionManager.GetInvokedDataCollectors(); - // For the invoked collectors we report the same information as ProxyDataCollectionManager.cs line ~416 - var invokedDataCollectorsForMetrics = invokedDataCollectors.Select(x => new { x.Uri, x.FriendlyName, x.HasAttachmentProcessor }.ToString()); - this.requestData.MetricsCollection.Add(TelemetryDataConstants.InvokedDataCollectors, string.Join(",", invokedDataCollectorsForMetrics.ToArray())); + if (invokedDataCollectors != null && invokedDataCollectors.Any()) + { + // For the invoked collectors we report the same information as ProxyDataCollectionManager.cs line ~416 + var invokedDataCollectorsForMetrics = invokedDataCollectors.Select(x => new { x.Uri, x.FriendlyName, x.HasAttachmentProcessor }.ToString()); + this.requestData.MetricsCollection.Add(TelemetryDataConstants.InvokedDataCollectors, string.Join(",", invokedDataCollectorsForMetrics.ToArray())); + } var afterTestRunEndResult = new AfterTestRunEndResult(attachmentsets, invokedDataCollectors, this.requestData.MetricsCollection.Metrics); From c1b4d0bb74e62b80ce9bbcad2bad229dc13fb234 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 17:03:24 +0100 Subject: [PATCH 32/47] fix UT --- .../DataCollectionManagerTests.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/datacollector.UnitTests/DataCollectionManagerTests.cs b/test/datacollector.UnitTests/DataCollectionManagerTests.cs index 0b017379c1..37ae957b66 100644 --- a/test/datacollector.UnitTests/DataCollectionManagerTests.cs +++ b/test/datacollector.UnitTests/DataCollectionManagerTests.cs @@ -5,7 +5,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.UnitTests { using System; using System.Collections.Generic; - using System.Linq; + using System.IO; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -573,12 +573,16 @@ protected override DataCollectorConfig TryGetDataCollectorConfig(string extensio { if (extensionUri.Equals("my://custom/datacollector")) { - return new DataCollectorConfig(dataCollector.GetType()); + var dc = new DataCollectorConfig(dataCollector.GetType()); + dc.FilePath = Path.GetTempFileName(); + return dc; } if (extensionUri.Equals("my://custom/ccdatacollector")) { - return new DataCollectorConfig(ccDataCollector.GetType()); + var dc = new DataCollectorConfig(ccDataCollector.GetType()); + dc.FilePath = Path.GetTempFileName(); + return dc; } return null; From 442a1b9adf95ccff2b81ab75cfefc1e33cc6faeb Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 18:09:06 +0100 Subject: [PATCH 33/47] fix build on Unix --- .../DataCollectionRequestHandler.cs | 1 - .../Microsoft.TestPlatform.CommunicationUtilities.csproj | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index dfa9dbed5d..8ed776bba3 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -5,7 +5,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.DataCollect { using System; using System.Collections.Generic; - using System.Collections.ObjectModel; using System.Globalization; using System.IO; using System.Linq; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Microsoft.TestPlatform.CommunicationUtilities.csproj b/src/Microsoft.TestPlatform.CommunicationUtilities/Microsoft.TestPlatform.CommunicationUtilities.csproj index 5358540868..959ae622c7 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Microsoft.TestPlatform.CommunicationUtilities.csproj +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Microsoft.TestPlatform.CommunicationUtilities.csproj @@ -18,13 +18,13 @@ - $(JsonNetVersion) + From 09d322ae481b908146c950f980a13112c9a1fa19 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 20 Dec 2021 20:01:16 +0100 Subject: [PATCH 34/47] Improve logging --- .../ExtensionFramework/TestPluginDiscoverer.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index b950fdefab..11cadec8cb 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -252,7 +252,7 @@ private void GetTestExtensionFromType( if (EqtTrace.IsErrorEnabled) { EqtTrace.Error( - "TryGetTestExtensionFromType: Either PluginInformation is null or PluginInformation doesn't contain IdentifierData for type {0}.", type.FullName); + "GetTestExtensionFromType: Either PluginInformation is null or PluginInformation doesn't contain IdentifierData for type {0}.", type.FullName); } return; } @@ -260,12 +260,14 @@ private void GetTestExtensionFromType( if (extensionCollection.ContainsKey(pluginInfo.IdentifierData)) { EqtTrace.Warning( - "TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data '{0}' and type '{1}'; keeping the first one '{2}'.", - pluginInfo.IdentifierData, pluginInfo.AssemblyQualifiedName, extensionCollection[pluginInfo.IdentifierData].AssemblyQualifiedName); + "GetTestExtensionFromType: Discovered multiple test extensions with identifier data '{0}' and type '{1}' inside file '{2}'; keeping the first one '{3}'.", + pluginInfo.IdentifierData, pluginInfo.AssemblyQualifiedName, filePath, extensionCollection[pluginInfo.IdentifierData].AssemblyQualifiedName); } else { extensionCollection.Add(pluginInfo.IdentifierData, pluginInfo); + EqtTrace.Info("GetTestExtensionFromType: Register extension with identifier data '{0}' and type '{1}' inside file '{2}'", + pluginInfo.IdentifierData, pluginInfo.AssemblyQualifiedName, filePath); } } } From f547abafed0f9935804332303bab22c3629f22f9 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 21 Dec 2021 09:20:30 +0100 Subject: [PATCH 35/47] fix acceptance --- .../DataCollectionTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index f78b5f91af..acd7f3e5c8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -7,6 +7,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests using System.Collections.Generic; using System.IO; using System.Linq; + using System.Text.RegularExpressions; using System.Xml; using System.Xml.Linq; using Microsoft.TestPlatform.TestUtilities; @@ -164,7 +165,7 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) string dataCollectorLog = File.ReadAllText(dataCollectorLogFile); Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV1' version '1'")); Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV2' version '2'")); - Assert.IsTrue(dataCollectorLog.Contains("TryGetTestExtensionFromType: Discovered multiple test extensions with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector.SampleDataCollectorV1, AttachmentProcessorDataCollector, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null'; keeping the first one 'AttachmentProcessorDataCollector.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null'.")); + Assert.IsTrue(Regex.IsMatch(dataCollectorLog, @"GetTestExtensionFromType: Register extension with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector\.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=.*, Culture=neutral, PublicKeyToken=null' inside file '.*AttachmentProcessorDataCollector\.dll'")); } TryRemoveDirectory(resultsDir); From aa3eb53bb15977dc83bd647a9baf8b8cf344dbaf Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 21 Dec 2021 09:23:01 +0100 Subject: [PATCH 36/47] Improve acceptance test --- .../DataCollectionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index acd7f3e5c8..197aee8f83 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -165,7 +165,7 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) string dataCollectorLog = File.ReadAllText(dataCollectorLogFile); Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV1' version '1'")); Assert.IsTrue(dataCollectorLog.Contains("MetadataReaderExtensionsHelper: Valid extension found: extension type 'DataCollector' identifier 'my://sample/datacollector' implementation 'AttachmentProcessorDataCollector.SampleDataCollectorV2' version '2'")); - Assert.IsTrue(Regex.IsMatch(dataCollectorLog, @"GetTestExtensionFromType: Register extension with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector\.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=.*, Culture=neutral, PublicKeyToken=null' inside file '.*AttachmentProcessorDataCollector\.dll'")); + Assert.IsTrue(Regex.IsMatch(dataCollectorLog, @"GetTestExtensionFromType: Discovered multiple test extensions with identifier data 'my://sample/datacollector' and type 'AttachmentProcessorDataCollector\.SampleDataCollectorV1, AttachmentProcessorDataCollector, Version=.*, Culture=neutral, PublicKeyToken=null' inside file '.*AttachmentProcessorDataCollector\.dll'; keeping the first one 'AttachmentProcessorDataCollector\.SampleDataCollectorV2, AttachmentProcessorDataCollector, Version=.*, Culture=neutral, PublicKeyToken=null'\.")); } TryRemoveDirectory(resultsDir); From ca79cfa6069372d632a7c2cb1a06bb008363cf3d Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 21 Dec 2021 09:50:22 +0100 Subject: [PATCH 37/47] fix comment typo --- .../DataCollection/DataCollectorConfig.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs index 91b541b98d..71d4c769e9 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs @@ -74,7 +74,7 @@ public override ICollection Metadata public Type AttachmentsProcessorType { get; private set; } /// - /// Check if collector registers an attachement processor. + /// Check if collector registers an attachment processor. /// /// True if collector registers an attachment processor. public bool HasAttachmentsProcessor() => AttachmentsProcessorType != null; From 869f5eafa78e16e7db42b049c0f8922038c0fb55 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 21 Dec 2021 09:56:06 +0100 Subject: [PATCH 38/47] fix comment typo --- .../Utilities/MetadataReaderHelper.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs index ac5b5d756f..ce4aa9945c 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/MetadataReaderHelper.cs @@ -15,7 +15,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Utilities using Microsoft.VisualStudio.TestPlatform.ObjectModel; - /* Expected attriute shape + /* Expected attribute shape namespace Microsoft.VisualStudio.TestPlatform { @@ -38,7 +38,6 @@ public TestExtensionTypesV2Attribute(string extensionType, string extensionIdent } } } - */ internal class MetadataReaderExtensionsHelper { From 37e3042dbaa961e23692abb923ff784f9f8708a7 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 21 Dec 2021 16:03:27 +0100 Subject: [PATCH 39/47] address PR feedback --- .../DataCollectionExtensionManager.cs | 2 +- .../IDataCollectorAttachmentsProcessorsFactory.cs | 5 ++--- .../PublicAPI/PublicAPI.Shipped.txt | 14 +++++++++++++- .../TestRunAttachmentsProcessingManager.cs | 6 +++--- .../Client/Parallel/ParallelRunDataAggregator.cs | 2 +- .../DataCollection/DataCollectionResult.cs | 6 +++--- .../PublicAPI/PublicAPI.Shipped.txt | 4 ++++ .../InvokedDataCollector.cs | 6 +++--- 8 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs index 58904cf792..a06177594e 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs @@ -89,7 +89,7 @@ public static DataCollectorExtensionManager Create(string extensionAssemblyFileP /// /// Hold data about the Data Collector. /// - internal class DataCollectorMetadata : IDataCollectorCapabilities + public class DataCollectorMetadata : IDataCollectorCapabilities { /// /// Constructor for DataCollectorMetadata diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs index 55e1d23a8e..117495550d 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -11,7 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine /// /// Creates and return a list of available attachments processor /// - internal interface IDataCollectorAttachmentsProcessorsFactory + public interface IDataCollectorAttachmentsProcessorsFactory { /// /// Creates and return a list of available attachments processor @@ -25,7 +24,7 @@ internal interface IDataCollectorAttachmentsProcessorsFactory /// /// Registered data collector attachment processor /// - internal class DataCollectorAttachmentProcessor + public class DataCollectorAttachmentProcessor { /// /// Data collector FriendlyName diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index c23e7a1cfd..5bf16e122b 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -20,6 +20,12 @@ Microsoft.VisualStudio.TestPlatform.Common.DataCollection.BeforeTestRunStartResu Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message) -> void Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message, System.Exception innerException) -> void +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.DataCollectorMetadata(string extension, string friendlyName, string filePath, bool hasAttachmentProcessor) -> void +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.ExtensionUri.get -> string +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.FriendlyName.get -> string +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.HasAttachmentProcessor.get -> bool +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.FilePath.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata.ExtensionUri.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata.FriendlyName.get -> string @@ -291,4 +297,10 @@ static Microsoft.VisualStudio.TestPlatform.Common.Utilities.RunSettingsUtilities virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestPluginCache.GetFilteredExtensions(System.Collections.Generic.List extensions, string endsWithPattern) -> System.Collections.Generic.IEnumerable virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.IdentifierData.get -> string virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.Metadata.get -> System.Collections.Generic.ICollection -virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider \ No newline at end of file +virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor.DataCollectorAttachmentProcessor(string friendlyName, Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor dataCollectorAttachmentProcessor) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor.DataCollectorAttachmentProcessorInstance.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor.FriendlyName.get -> string +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IDataCollectorAttachmentsProcessorsFactory +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IDataCollectorAttachmentsProcessorsFactory.Create(Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector[] invokedDataCollectors, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor[] \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs index 20b8689966..be36c5b599 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs @@ -22,7 +22,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachments /// /// Orchestrates test run attachments processing operations. /// - internal class TestRunAttachmentsProcessingManager : ITestRunAttachmentsProcessingManager + public class TestRunAttachmentsProcessingManager : ITestRunAttachmentsProcessingManager { private static string AttachmentsProcessingCompleted = "Completed"; private static string AttachmentsProcessingCanceled = "Canceled"; @@ -92,7 +92,7 @@ private async Task> InternalProcessTestRunAttachmentsA { EqtTrace.Error("TestRunAttachmentsProcessingManager: Exception in ProcessTestRunAttachmentsAsync: " + e); - eventHandler?.HandleLogMessage(TestMessageLevel.Error, "TestRunAttachmentsProcessingManager: Exception in ProcessTestRunAttachmentsAsync: " + e); + eventHandler?.HandleLogMessage(TestMessageLevel.Error, e.Message); return FinalizeOperation(requestData, new TestRunAttachmentsProcessingCompleteEventArgs(false, e), attachments, stopwatch, eventHandler); } } @@ -161,7 +161,7 @@ private async Task> ProcessAttachmentsAsync(string run catch (Exception e) { EqtTrace.Error("TestRunAttachmentsProcessingManager: Exception in ProcessAttachmentsAsync: " + e); - logger.SendMessage(TestMessageLevel.Error, "TestRunAttachmentsProcessingManager: Exception in ProcessAttachmentsAsync: " + e); + logger.SendMessage(TestMessageLevel.Error, e.Message); // Restore the attachment sets for the others attachment processors. attachments = attachmentsBackup; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs index 2917a5fd06..eda28500fa 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs @@ -167,7 +167,7 @@ public void Aggregate( if (executorUris != null) this.executorUris.AddRange(executorUris); if (testRunStats != null) testRunStatsList.Add(testRunStats); - if (invokedDataCollectors != null && invokedDataCollectors.Count > 0) + if (invokedDataCollectors?.Count > 0) { foreach (var invokedDataCollector in invokedDataCollectors) { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs index 8c822286b2..440dcdbf6d 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/DataCollection/DataCollectionResult.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using System.Collections.ObjectModel; - namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection { + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using System.Collections.ObjectModel; + /// /// Information returned after data collection. /// diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt index 2d1e6db286..6e0600eca2 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt @@ -103,6 +103,10 @@ Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.GetDiscoveryManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IDiscoveryManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.GetExecutionManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.TestHostManagerFactory(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.ProcessTestRunAttachmentsAsync(string runSettingsXml, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollector, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.ProcessTestRunAttachmentsAsync(string runSettingsXml, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollector, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.TestRunAttachmentsProcessingManager(Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces.ITestPlatformEventSource testPlatformEventSource, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IDataCollectorAttachmentsProcessorsFactory dataCollectorAttachmentsProcessorsFactory) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool override Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManagerWithDataCollection.Initialize(bool skipDefaultAdapters) -> void override Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManagerWithDataCollection.SetupChannel(System.Collections.Generic.IEnumerable sources, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler eventHandler) -> bool diff --git a/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs index fc250ca92a..dacdfc6f9e 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/InvokedDataCollector.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Runtime.Serialization; - namespace Microsoft.VisualStudio.TestPlatform.ObjectModel { + using System; + using System.Runtime.Serialization; + public sealed class InvokedDataCollector : IEquatable { /// From 570b983d847403268f2b362effd318cb8ca0ccc6 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 21 Dec 2021 16:52:34 +0100 Subject: [PATCH 40/47] fix test --- .../TestRunAttachmentsProcessingManagerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs index 3829ae5d1f..48464e9b54 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/AttachmentsProcessing/TestRunAttachmentsProcessingManagerTests.cs @@ -234,7 +234,7 @@ public async Task ProcessTestRunAttachmentsAsync_ShouldReturnInitialAttachmentsT // assert VerifyCompleteEvent(false, false, inputAttachments[0]); mockEventsHandler.Verify(h => h.HandleTestRunAttachmentsProcessingProgress(It.IsAny()), Times.Never); - mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "exception message"), Times.Never); + mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "exception message"), Times.Once); mockAttachmentHandler1.Verify(h => h.GetExtensionUris()); mockAttachmentHandler2.Verify(h => h.GetExtensionUris(), Times.Once); mockAttachmentHandler1.Verify(h => h.ProcessAttachmentSetsAsync(It.IsAny(), It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), It.IsAny(), cancellationTokenSource.Token)); From 67499ce52f764b71b6dc44721498b58f2c0f69e8 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Thu, 23 Dec 2021 10:38:17 +0100 Subject: [PATCH 41/47] Cleanup published api --- .../DataCollection/DataCollectorConfig.cs | 2 +- .../DataCollectionExtensionManager.cs | 12 +----------- .../IDataCollectorAttachmentsProcessorsFactory.cs | 4 ++-- .../Interfaces/IDataCollectorCapabilities.cs | 5 ----- .../PublicAPI/PublicAPI.Shipped.txt | 12 ++---------- .../TestRunAttachmentsProcessingManager.cs | 2 +- .../PublicAPI/PublicAPI.Shipped.txt | 4 ---- 7 files changed, 7 insertions(+), 34 deletions(-) diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs index 71d4c769e9..538c65692c 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorConfig.cs @@ -64,7 +64,7 @@ public override ICollection Metadata { get { - return new object[] { this.TypeUri.ToString(), this.FriendlyName, this.FilePath, this.AttachmentsProcessorType != null }; + return new object[] { this.TypeUri.ToString(), this.FriendlyName, this.AttachmentsProcessorType != null }; } } diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs index a06177594e..6ddf3ddf7f 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs @@ -100,11 +100,10 @@ public class DataCollectorMetadata : IDataCollectorCapabilities /// /// The friendly Name. /// - public DataCollectorMetadata(string extension, string friendlyName, string filePath, bool hasAttachmentProcessor) + public DataCollectorMetadata(string extension, string friendlyName, bool hasAttachmentProcessor) { this.ExtensionUri = extension; this.FriendlyName = friendlyName; - this.FilePath = filePath; this.HasAttachmentProcessor = hasAttachmentProcessor; } @@ -134,14 +133,5 @@ public bool HasAttachmentProcessor get; private set; } - - /// - /// Gets the file path of assemblies that contains the data collector. - /// - public string FilePath - { - get; - private set; - } } } diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs index 117495550d..530ae616dd 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/IDataCollectorAttachmentsProcessorsFactory.cs @@ -10,7 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine /// /// Creates and return a list of available attachments processor /// - public interface IDataCollectorAttachmentsProcessorsFactory + internal interface IDataCollectorAttachmentsProcessorsFactory { /// /// Creates and return a list of available attachments processor @@ -24,7 +24,7 @@ public interface IDataCollectorAttachmentsProcessorsFactory /// /// Registered data collector attachment processor /// - public class DataCollectorAttachmentProcessor + internal class DataCollectorAttachmentProcessor { /// /// Data collector FriendlyName diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs b/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs index b4c4a57c7f..965c3dd44f 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/IDataCollectorCapabilities.cs @@ -13,11 +13,6 @@ public interface IDataCollectorCapabilities : ITestExtensionCapabilities /// string FriendlyName { get; } - /// - /// Gets the file path of the assembly that contains the data collector. - /// - string FilePath { get; } - /// /// Check if the data collector has got attachment processor registered. /// diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index 5bf16e122b..8b699707da 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -21,11 +21,10 @@ Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message) -> void Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message, System.Exception innerException) -> void Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata -Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.DataCollectorMetadata(string extension, string friendlyName, string filePath, bool hasAttachmentProcessor) -> void +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.DataCollectorMetadata(string extension, string friendlyName, bool hasAttachmentProcessor) -> void Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.ExtensionUri.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.FriendlyName.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.HasAttachmentProcessor.get -> bool -Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.FilePath.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata.ExtensionUri.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestLoggerMetadata.FriendlyName.get -> string @@ -80,7 +79,6 @@ Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IBaseTestEventsRegistrar Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IBaseTestEventsRegistrar.LogWarning(string message) -> void Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities.FriendlyName.get -> string -Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities.FilePath.get -> string Microsoft.VisualStudio.TestPlatform.Common.Interfaces.IDataCollectorCapabilities.HasAttachmentProcessor.get -> bool Microsoft.VisualStudio.TestPlatform.Common.Interfaces.ISettingsProviderCapabilities Microsoft.VisualStudio.TestPlatform.Common.Interfaces.ISettingsProviderCapabilities.SettingsName.get -> string @@ -297,10 +295,4 @@ static Microsoft.VisualStudio.TestPlatform.Common.Utilities.RunSettingsUtilities virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestPluginCache.GetFilteredExtensions(System.Collections.Generic.List extensions, string endsWithPattern) -> System.Collections.Generic.IEnumerable virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.IdentifierData.get -> string virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.Metadata.get -> System.Collections.Generic.ICollection -virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor.DataCollectorAttachmentProcessor(string friendlyName, Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor dataCollectorAttachmentProcessor) -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor.DataCollectorAttachmentProcessorInstance.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection.IDataCollectorAttachmentProcessor -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor.FriendlyName.get -> string -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IDataCollectorAttachmentsProcessorsFactory -Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IDataCollectorAttachmentsProcessorsFactory.Create(Microsoft.VisualStudio.TestPlatform.ObjectModel.InvokedDataCollector[] invokedDataCollectors, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DataCollectorAttachmentProcessor[] \ No newline at end of file +virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string runConfiguration) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs index be36c5b599..73dfb4269e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/TestRunAttachmentsProcessingManager.cs @@ -22,7 +22,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachments /// /// Orchestrates test run attachments processing operations. /// - public class TestRunAttachmentsProcessingManager : ITestRunAttachmentsProcessingManager + internal class TestRunAttachmentsProcessingManager : ITestRunAttachmentsProcessingManager { private static string AttachmentsProcessingCompleted = "Completed"; private static string AttachmentsProcessingCanceled = "Canceled"; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt index 6e0600eca2..2d1e6db286 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Shipped.txt @@ -103,10 +103,6 @@ Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.GetDiscoveryManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IDiscoveryManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.GetExecutionManager() -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestHostManagerFactory.TestHostManagerFactory(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> void -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.ProcessTestRunAttachmentsAsync(string runSettingsXml, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollector, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunAttachmentsProcessingEventsHandler eventHandler, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.ProcessTestRunAttachmentsAsync(string runSettingsXml, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, System.Collections.Generic.IEnumerable attachments, System.Collections.Generic.IEnumerable invokedDataCollector, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> -Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing.TestRunAttachmentsProcessingManager.TestRunAttachmentsProcessingManager(Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces.ITestPlatformEventSource testPlatformEventSource, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IDataCollectorAttachmentsProcessorsFactory dataCollectorAttachmentsProcessorsFactory) -> void Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool override Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManagerWithDataCollection.Initialize(bool skipDefaultAdapters) -> void override Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyOperationManagerWithDataCollection.SetupChannel(System.Collections.Generic.IEnumerable sources, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestMessageEventHandler eventHandler) -> bool From 8e15a7d9296b7498d3d3c1721b09f868712111f2 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 10 Jan 2022 15:45:09 +0100 Subject: [PATCH 42/47] address feedback on back-compat api --- .../DataCollection/AfterTestRunEndResult.cs | 13 +++++++++++++ .../DataCollectionExtensionManager.cs | 15 +++++++++++++++ .../PublicAPI/PublicAPI.Shipped.txt | 2 ++ .../Client/Events/TestRunCompleteEventArgs.cs | 13 +++++++++++++ .../PublicAPI/PublicAPI.Shipped.txt | 1 + 5 files changed, 44 insertions(+) diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs index 4c9438fbce..2faa2bb206 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs @@ -14,6 +14,19 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollection [DataContract] public class AfterTestRunEndResult { + /// + /// Initializes a new instance of the class. + /// + /// + /// The collection of attachment sets. + /// + /// + /// The metrics. + /// + public AfterTestRunEndResult(Collection attachmentSets, IDictionary metrics) + : this(attachmentSets, new Collection(), metrics) + { } + /// /// Initializes a new instance of the class. /// diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs index 6ddf3ddf7f..b3c07f626f 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/DataCollectionExtensionManager.cs @@ -100,6 +100,21 @@ public class DataCollectorMetadata : IDataCollectorCapabilities /// /// The friendly Name. /// + public DataCollectorMetadata(string extension, string friendlyName) + : this(extension, friendlyName, false) + { } + + /// + /// Constructor for DataCollectorMetadata + /// + /// + /// Uri identifying the data collector. + /// + /// + /// The friendly Name. + /// + /// Indicates if the current data collector registers an attachment processor + /// public DataCollectorMetadata(string extension, string friendlyName, bool hasAttachmentProcessor) { this.ExtensionUri = extension; diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt index 8b699707da..123a7a6896 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt @@ -9,6 +9,7 @@ const Microsoft.VisualStudio.TestPlatform.Common.TestPlatformDefaults.EnableBoun const Microsoft.VisualStudio.TestPlatform.Common.TestPlatformDefaults.MaxBytesLoggerEventQueueCanHold = "MaxBytesLoggerEventQueueCanHold" -> string const Microsoft.VisualStudio.TestPlatform.Common.TestPlatformDefaults.MaxNumberOfEventsLoggerEventQueueCanHold = "MaxNumberOfEventsLoggerEventQueueCanHold" -> string Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult +Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.AfterTestRunEndResult(System.Collections.ObjectModel.Collection attachmentSets, System.Collections.Generic.IDictionary metrics) -> void Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.AfterTestRunEndResult(System.Collections.ObjectModel.Collection attachmentSets, System.Collections.ObjectModel.Collection invokedDataCollectors, System.Collections.Generic.IDictionary metrics) -> void Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.AttachmentSets.get -> System.Collections.ObjectModel.Collection Microsoft.VisualStudio.TestPlatform.Common.DataCollection.AfterTestRunEndResult.Metrics.get -> System.Collections.Generic.IDictionary @@ -21,6 +22,7 @@ Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message) -> void Microsoft.VisualStudio.TestPlatform.Common.Exceptions.InvalidLoggerException.InvalidLoggerException(string message, System.Exception innerException) -> void Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata +Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.DataCollectorMetadata(string extension, string friendlyName) -> void Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.DataCollectorMetadata(string extension, string friendlyName, bool hasAttachmentProcessor) -> void Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.ExtensionUri.get -> string Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.DataCollectorMetadata.FriendlyName.get -> string diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs index 6bc69a12ad..249b548c7b 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs @@ -14,6 +14,19 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client [DataContract] public class TestRunCompleteEventArgs : EventArgs { + /// + /// Default constructor. + /// + /// The final stats for the test run. This parameter is only set for communications between the test host and the clients (like VS) + /// Specifies whether the test run is canceled. + /// Specifies whether the test run is aborted. + /// Specifies the error encountered during the execution of the test run. + /// Attachment sets associated with the run. + /// Time elapsed in just running tests + public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool isAborted, Exception error, Collection attachmentSets, TimeSpan elapsedTime) + : this(stats, isCanceled, isAborted, error, attachmentSets, null, elapsedTime) + { } + /// /// Default constructor. /// diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt index d8b50aefa4..3eace443ef 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt @@ -343,6 +343,7 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunChangedEventArgs.N Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunChangedEventArgs.TestRunChangedEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics stats, System.Collections.Generic.IEnumerable newTestResults, System.Collections.Generic.IEnumerable activeTests) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunChangedEventArgs.TestRunStatistics.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.TestRunCompleteEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics stats, bool isCanceled, bool isAborted, System.Exception error, System.Collections.ObjectModel.Collection attachmentSets, System.TimeSpan elapsedTime) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.TestRunCompleteEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestRunStatistics stats, bool isCanceled, bool isAborted, System.Exception error, System.Collections.ObjectModel.Collection attachmentSets, System.Collections.ObjectModel.Collection invokedDataCollectors, System.TimeSpan elapsedTime) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.InvokedDataCollectors.get -> System.Collections.ObjectModel.Collection Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.AttachmentSets.get -> System.Collections.ObjectModel.Collection From 9d07b0521a107663744e03e3d9238c6560f58f9f Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 10 Jan 2022 20:30:38 +0100 Subject: [PATCH 43/47] add non-public ctor to avoid Newtosoft dep on the public ObjectModel --- .../Client/Events/TestRunCompleteEventArgs.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs index 249b548c7b..170a9aec6e 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs @@ -14,6 +14,13 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client [DataContract] public class TestRunCompleteEventArgs : EventArgs { + // We have more than one ctor for backward-compatibility reason but we don't want to add dependency on Newtosoft([JsonContstructor]) + // We want to fallback to the non-public default constructor https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_ConstructorHandling.htm + private TestRunCompleteEventArgs() + { + + } + /// /// Default constructor. /// From 1a30ca76dcbcd1f65d2e7b43508f0ca2bddbc892 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 11 Jan 2022 10:36:46 +0100 Subject: [PATCH 44/47] fix build --- scripts/common.lib.ps1 | 2 +- .../Client/Events/TestRunCompleteEventArgs.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index f9f95b32c2..36a479c134 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -110,7 +110,7 @@ function Install-DotNetCli Write-Log "Install-DotNetCli: Get the latest dotnet cli toolset..." $dotnetInstallPath = Join-Path $env:TP_TOOLS_DIR "dotnet" New-Item -ItemType directory -Path $dotnetInstallPath -Force | Out-Null - & $dotnetInstallScript -Channel 6.0 -Quality Preview -InstallDir $dotnetInstallPath -Version $env:DOTNET_CLI_VERSION + & $dotnetInstallScript -Channel 6.0 -InstallDir $dotnetInstallPath -Version $env:DOTNET_CLI_VERSION & $dotnetInstallScript -InstallDir "$dotnetInstallPath" -Runtime 'dotnet' -Version '2.1.30' -Channel '2.1' -Architecture x64 -NoPath $env:DOTNET_ROOT= $dotnetInstallPath diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs index 170a9aec6e..9f595fd1b7 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs @@ -15,7 +15,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client public class TestRunCompleteEventArgs : EventArgs { // We have more than one ctor for backward-compatibility reason but we don't want to add dependency on Newtosoft([JsonContstructor]) - // We want to fallback to the non-public default constructor https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_ConstructorHandling.htm + // We want to fallback to the non-public default constructor https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_ConstructorHandling.htm during deserialization private TestRunCompleteEventArgs() { From 657c1290f43f74865388c74667048c21b2e6b18d Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 11 Jan 2022 10:45:34 +0100 Subject: [PATCH 45/47] fix comment typo --- .../Client/Events/TestRunCompleteEventArgs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs index 9f595fd1b7..5f81c017d9 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs @@ -14,7 +14,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client [DataContract] public class TestRunCompleteEventArgs : EventArgs { - // We have more than one ctor for backward-compatibility reason but we don't want to add dependency on Newtosoft([JsonContstructor]) + // We have more than one ctor for backward-compatibility reason but we don't want to add dependency on Newtosoft([JsonConstructor]) // We want to fallback to the non-public default constructor https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_ConstructorHandling.htm during deserialization private TestRunCompleteEventArgs() { From 2ae07e26049ef16705073ae1e18218842de4dd7d Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 11 Jan 2022 11:16:08 +0100 Subject: [PATCH 46/47] fix test --- .../DataCollection/AfterTestRunEndResult.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs index 2faa2bb206..53cdd74e1b 100644 --- a/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs +++ b/src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs @@ -14,6 +14,13 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollection [DataContract] public class AfterTestRunEndResult { + // We have more than one ctor for backward-compatibility reason but we don't want to add dependency on Newtosoft([JsonConstructor]) + // We want to fallback to the non-public default constructor https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_ConstructorHandling.htm during deserialization + private AfterTestRunEndResult() + { + + } + /// /// Initializes a new instance of the class. /// From 936e678a34177fdd8dc6b4bbbb11d8406a090174 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Tue, 11 Jan 2022 15:50:11 +0100 Subject: [PATCH 47/47] fix TestRunCompleteEventArgs --- .../Client/Events/TestRunCompleteEventArgs.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs index 5f81c017d9..cd371d0aac 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs @@ -18,7 +18,8 @@ public class TestRunCompleteEventArgs : EventArgs // We want to fallback to the non-public default constructor https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_ConstructorHandling.htm during deserialization private TestRunCompleteEventArgs() { - + this.AttachmentSets = new Collection(); + this.InvokedDataCollectors = new Collection(); } ///