[tests][eventpipe] Port over Dotnet/Diagnostics to enable eventpipe t…

…ests on Android (#64358)

* [tests][eventpipe] Port over Dotnet/Diagnostics files for mobile eventpipe tests

Copied over files from src/Microsoft.Diagnostics.NETCore.Client based off of 99fab307

* [tests][eventpipe] Add TCP/IP logic for mobile eventpipe tests

* [tests] Remove Microsoft.Diagnostics.NETCore.Client package reference

* [tests][eventpipe] Downstream Diagnostics IpcTraceTest DiagnosticsClient bootstrap

dotnet/diagnostics#720

* [tests][eventpipe] Downstream Diagnostics roslyn analyzer IpcTraceTest change

dotnet/diagnostics#1044

* [tests][eventpipe] Enable TCPIP DiagnosticsClient in IpcTraceTest for Android

* [tests][eventpipe] Aesthetic IpcTraceTest modifications

* [tests][eventpipe] Disable subprocesses tests on Android

* [tests][eventpipe] Update processinfo

* [tests][eventpipe] Update processinfo2

* [tests][eventpipe] Update eventsourceerror

* [tests][eventpipe] Update bigevent

* [tests][eventpipe] Update buffersize

* [tests][eventpipe] Update rundownvalidation

* [tests][eventpipe] Update providervalidation

* [tests][eventpipe] Update gcdump

* [tests][JIT] Update debuginfo/tester

* [tests] Segment Microsoft.Diagnostics.NETCore.Client relevant tests for Linux arm coreclr

* Account for nonspecified RuntimeFlavor

* [tests] Moveup Default coreclr RuntimeFlavor property explicit declaration

* [tests] Duplicate Microsoft.Diagnostics.NETCore.Client dependent tests for Linux arm

* Fix debuginfo/tester test skip

* Temporarily enable bigevent on Linux arm and remove duplicate exclude

* Fix unaligned UTF16 string read in collect tracing EventPipe command.

Collect tracing 2 EventPipe command triggers an unaligned UTF16 string
read that could cause a SIGBUS on platforms not supporting unalinged
reads of UTF16 strings (so far only seen on 32-bit ARM Linux CI machine).

On CoreCLR this could even cause a unalinged int read due to
optimizations used in UTF8Encoding::GetByteCount.

* Revert "[tests] Duplicate Microsoft.Diagnostics.NETCore.Client dependent tests for Linux arm"

This reverts commit cb2cacd.

* Revert "[tests] Segment Microsoft.Diagnostics.NETCore.Client relevant tests for Linux arm coreclr"

This reverts commit dc29676.

* Revert "Fix debuginfo/tester test skip"

This reverts commit 1e90d7e.

Co-authored-by: Mitchell Hwang <[email protected]>
Co-authored-by: lateralusX <[email protected]>

eng/Versions.props

@@ -137,7 +137,6 @@
     <NETStandardLibraryRefVersion>2.1.0</NETStandardLibraryRefVersion>
     <NetStandardLibraryVersion>2.0.3</NetStandardLibraryVersion>
     <MicrosoftDiagnosticsToolsRuntimeClientVersion>1.0.4-preview6.19326.1</MicrosoftDiagnosticsToolsRuntimeClientVersion>
-    <MicrosoftDiagnosticsNETCoreClientVersion>0.2.61701</MicrosoftDiagnosticsNETCoreClientVersion>
     <DNNEVersion>1.0.27</DNNEVersion>
     <MicrosoftBuildVersion>16.10.0</MicrosoftBuildVersion>
     <MicrosoftBuildTasksCoreVersion>$(MicrosoftBuildVersion)</MicrosoftBuildTasksCoreVersion>

src/native/eventpipe/ds-eventpipe-protocol.c

@@ -140,7 +140,7 @@ eventpipe_collect_tracing_command_try_parse_rundown_requested (
 	EP_ASSERT (buffer_len != NULL);
 	EP_ASSERT (rundown_requested != NULL);
 
-	return ds_ipc_message_try_parse_value (buffer, buffer_len, (uint8_t *)rundown_requested, (uint32_t)sizeof (bool));
+	return ds_ipc_message_try_parse_value (buffer, buffer_len, (uint8_t *)rundown_requested, 1);
 }
 
 static
@@ -160,6 +160,9 @@ eventpipe_collect_tracing_command_try_parse_config (
 	const uint32_t max_count_configs = 1000;
 	uint32_t count_configs = 0;
 
+	uint8_t *provider_name_byte_array = NULL;
+	uint8_t *filter_data_byte_array = NULL;
+
 	ep_char8_t *provider_name_utf8 = NULL;
 	ep_char8_t *filter_data_utf8 = NULL;
 
@@ -176,20 +179,27 @@ eventpipe_collect_tracing_command_try_parse_config (
 		ep_raise_error_if_nok (ds_ipc_message_try_parse_uint32_t (buffer, buffer_len, &log_level));
 		ep_raise_error_if_nok (log_level <= EP_EVENT_LEVEL_VERBOSE);
 
-		const ep_char16_t *provider_name = NULL;
-		ep_raise_error_if_nok (ds_ipc_message_try_parse_string_utf16_t (buffer, buffer_len, &provider_name));
+		uint32_t provider_name_byte_array_len = 0;
+		ep_raise_error_if_nok (ds_ipc_message_try_parse_string_utf16_t_byte_array_alloc (buffer, buffer_len, &provider_name_byte_array, &provider_name_byte_array_len));
 
-		provider_name_utf8 = ep_rt_utf16_to_utf8_string (provider_name, -1);
+		provider_name_utf8 = ep_rt_utf16_to_utf8_string ((const ep_char16_t *)provider_name_byte_array, -1);
 		ep_raise_error_if_nok (provider_name_utf8 != NULL);
 
 		ep_raise_error_if_nok (!ep_rt_utf8_string_is_null_or_empty (provider_name_utf8));
 
-		const ep_char16_t *filter_data = NULL; // This parameter is optional.
-		ds_ipc_message_try_parse_string_utf16_t (buffer, buffer_len, &filter_data);
+		ep_rt_byte_array_free (provider_name_byte_array);
+		provider_name_byte_array = NULL;
+
+		uint32_t filter_data_byte_array_len = 0;
+		ep_raise_error_if_nok (ds_ipc_message_try_parse_string_utf16_t_byte_array_alloc (buffer, buffer_len, &filter_data_byte_array, &filter_data_byte_array_len));
 
-		if (filter_data) {
-			filter_data_utf8 = ep_rt_utf16_to_utf8_string (filter_data, -1);
+		// This parameter is optional.
+		if (filter_data_byte_array) {
+			filter_data_utf8 = ep_rt_utf16_to_utf8_string ((const ep_char16_t *)filter_data_byte_array, -1);
 			ep_raise_error_if_nok (filter_data_utf8 != NULL);
+
+			ep_rt_byte_array_free (filter_data_byte_array);
+			filter_data_byte_array = NULL;
 		}
 
 		EventPipeProviderConfiguration provider_config;
@@ -209,7 +219,9 @@ eventpipe_collect_tracing_command_try_parse_config (
 
 ep_on_error:
 	count_configs = 0;
+	ep_rt_byte_array_free (provider_name_byte_array);
 	ep_rt_utf8_string_free (provider_name_utf8);
+	ep_rt_byte_array_free (filter_data_byte_array);
 	ep_rt_utf8_string_free (filter_data_utf8);
 	ep_exit_error_handler ();
 }

src/native/eventpipe/ds-protocol.c

@@ -59,6 +59,14 @@ ipc_message_flatten (
 	uint16_t payload_len,
 	ds_ipc_flatten_payload_func flatten_payload);
 
+static
+bool
+ipc_message_try_parse_string_utf16_t_byte_array (
+	uint8_t **buffer,
+	uint32_t *buffer_len,
+	const uint8_t **string_byte_array,
+	uint32_t *string_byte_array_len);
+
 /*
 * DiagnosticsIpc
 */
@@ -306,6 +314,50 @@ ipc_message_flatten (
 	ep_exit_error_handler ();
 }
 
+static
+bool
+ipc_message_try_parse_string_utf16_t_byte_array (
+	uint8_t **buffer,
+	uint32_t *buffer_len,
+	const uint8_t **string_byte_array,
+	uint32_t *string_byte_array_len)
+{
+	EP_ASSERT (buffer != NULL);
+	EP_ASSERT (buffer_len != NULL);
+	EP_ASSERT (string_byte_array != NULL);
+	EP_ASSERT (string_byte_array_len != NULL);
+
+	bool result = false;
+
+	ep_raise_error_if_nok (ds_ipc_message_try_parse_uint32_t (buffer, buffer_len, string_byte_array_len));
+	*string_byte_array_len *= sizeof (ep_char16_t);
+
+	if (*string_byte_array_len != 0) {
+		if (*string_byte_array_len > *buffer_len)
+			ep_raise_error ();
+
+		if (((const ep_char16_t *)*buffer) [(*string_byte_array_len / sizeof (ep_char16_t)) - 1] != 0)
+			ep_raise_error ();
+
+		*string_byte_array = *buffer;
+
+	} else {
+		*string_byte_array = NULL;
+	}
+
+	*buffer = *buffer + *string_byte_array_len;
+	*buffer_len = *buffer_len - *string_byte_array_len;
+
+	result = true;
+
+ep_on_exit:
+	return result;
+
+ep_on_error:
+	EP_ASSERT (!result);
+	ep_exit_error_handler ();
+}
+
 DiagnosticsIpcMessage *
 ds_ipc_message_init (DiagnosticsIpcMessage *message)
 {
@@ -383,38 +435,35 @@ ds_ipc_message_try_parse_uint32_t (
 	return result;
 }
 
-// TODO: Strings are in little endian format in buffer.
 bool
-ds_ipc_message_try_parse_string_utf16_t (
+ds_ipc_message_try_parse_string_utf16_t_byte_array_alloc (
 	uint8_t **buffer,
 	uint32_t *buffer_len,
-	const ep_char16_t **value)
+	uint8_t **string_byte_array,
+	uint32_t *string_byte_array_len)
 {
 	EP_ASSERT (buffer != NULL);
 	EP_ASSERT (buffer_len != NULL);
-	EP_ASSERT (value != NULL);
+	EP_ASSERT (string_byte_array != NULL);
+	EP_ASSERT (string_byte_array_len != NULL);
 
 	bool result = false;
 
-	uint32_t string_len = 0;
-	ep_raise_error_if_nok (ds_ipc_message_try_parse_uint32_t (buffer, buffer_len, &string_len));
+	const uint8_t *temp_buffer = NULL;
+	uint32_t temp_buffer_len = 0;
 
-	if (string_len != 0) {
-		if (string_len > (*buffer_len / sizeof (ep_char16_t)))
-			ep_raise_error ();
-
-		if (((const ep_char16_t *)*buffer) [string_len - 1] != 0)
-			ep_raise_error ();
+	ep_raise_error_if_nok (ipc_message_try_parse_string_utf16_t_byte_array (buffer, buffer_len, (const uint8_t **)&temp_buffer, &temp_buffer_len));
 
-		*value = (ep_char16_t *)*buffer;
+	if (temp_buffer_len != 0) {
+		*string_byte_array = ep_rt_byte_array_alloc (temp_buffer_len);
+		ep_raise_error_if_nok (*string_byte_array != NULL);
 
+		memcpy (*string_byte_array, temp_buffer, temp_buffer_len);
 	} else {
-		*value = NULL;
+		*string_byte_array = NULL;
 	}
 
-	*buffer = *buffer + (string_len * sizeof (ep_char16_t));
-	*buffer_len = *buffer_len - (string_len * sizeof (ep_char16_t));
-
+	*string_byte_array_len = temp_buffer_len;
 	result = true;
 
 ep_on_exit:
@@ -425,6 +474,21 @@ ds_ipc_message_try_parse_string_utf16_t (
 	ep_exit_error_handler ();
 }
 
+bool
+ds_ipc_message_try_parse_string_utf16_t (
+	uint8_t **buffer,
+	uint32_t *buffer_len,
+	const ep_char16_t **value)
+{
+	EP_ASSERT (buffer != NULL);
+	EP_ASSERT (buffer_len != NULL);
+	EP_ASSERT (value != NULL);
+	EP_ASSERT (!(((size_t)*buffer) & 0x1));
+
+	uint32_t string_byte_array_len = 0;
+	return ipc_message_try_parse_string_utf16_t_byte_array (buffer, buffer_len, (const uint8_t **)value, &string_byte_array_len);
+}
+
 bool
 ds_ipc_message_initialize_header_uint32_t_payload (
 	DiagnosticsIpcMessage *message,

src/native/eventpipe/ds-protocol.h

@@ -145,6 +145,13 @@ ds_ipc_message_try_parse_int32_t (
 	return ds_ipc_message_try_parse_uint32_t (buffer, buffer_len, (uint32_t *)value);
 }
 
+bool
+ds_ipc_message_try_parse_string_utf16_t_byte_array_alloc (
+	uint8_t **buffer,
+	uint32_t *buffer_len,
+	uint8_t **string_byte_array,
+	uint32_t *string_byte_array_len);
+
 bool
 ds_ipc_message_try_parse_string_utf16_t (
 	uint8_t **buffer,

src/tests/Common/test_dependencies/test_dependencies.csproj

@@ -10,7 +10,6 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.Diagnostics.Tools.RuntimeClient" Version="$(MicrosoftDiagnosticsToolsRuntimeClientVersion)" />
-    <PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Version="$(MicrosoftDiagnosticsNETCoreClientVersion)" />
   </ItemGroup>
 
   <Target Name="Build" DependsOnTargets="$(TraversalBuildDependsOn)" />

src/tests/Directory.Build.props

@@ -71,6 +71,7 @@
 
     <TestBuildMode Condition="'$(__TestBuildMode)' != ''">$(__TestBuildMode)</TestBuildMode>
     <RuntimeFlavor Condition="'$(__RuntimeFlavor)' != ''">$(__RuntimeFlavor)</RuntimeFlavor>
+    <RuntimeFlavor Condition="'$(RuntimeFlavor)' == ''">coreclr</RuntimeFlavor>
 
     <RestoreDefaultOptimizationDataPackage Condition="'$(RestoreDefaultOptimizationDataPackage)' == ''">false</RestoreDefaultOptimizationDataPackage>
     <PortableBuild Condition="'$(PortableBuild)' == ''">true</PortableBuild>

src/tests/JIT/Directed/debugging/debuginfo/tester.cs

@@ -8,7 +8,7 @@
 using System.Linq;
 using System.Reflection;
 using System.Runtime.CompilerServices;
-using Microsoft.Diagnostics.Tools.RuntimeClient;
+using Microsoft.Diagnostics.NETCore.Client;
 using Microsoft.Diagnostics.Tracing;
 using Microsoft.Diagnostics.Tracing.Parsers;
 using Microsoft.Diagnostics.Tracing.Parsers.Clr;
@@ -23,18 +23,17 @@ public static unsafe int Main()
         var keywords =
             ClrTraceEventParser.Keywords.Jit | ClrTraceEventParser.Keywords.JittedMethodILToNativeMap;
 
-        var dotnetRuntimeProvider = new List<Provider>
+        var dotnetRuntimeProvider = new List<EventPipeProvider>
         {
-            new Provider("Microsoft-Windows-DotNETRuntime", eventLevel: EventLevel.Verbose, keywords: (ulong)keywords)
+            new EventPipeProvider("Microsoft-Windows-DotNETRuntime", eventLevel: EventLevel.Verbose, keywords: (long)keywords)
         };
 
-        var config = new SessionConfiguration(1024, EventPipeSerializationFormat.NetTrace, dotnetRuntimeProvider);
-
         return
             IpcTraceTest.RunAndValidateEventCounts(
                 new Dictionary<string, ExpectedEventCount>(),
                 JitMethods,
-                config,
+                dotnetRuntimeProvider,
+                1024,
                 ValidateMappings);
     }
 

src/tests/JIT/Directed/debugging/debuginfo/tester.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="tests_r.ilproj" Aliases="tests_r" />
     <ProjectReference Include="attribute.csproj" />
     <ProjectReference Include="../../../../tracing/eventpipe/common/common.csproj" />
+    <ProjectReference Include="../../../../tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj" />
     <Compile Include="$(MSBuildProjectName).cs" />
   </ItemGroup>
 </Project>

src/tests/issues.targets

@@ -3626,9 +3626,6 @@
         <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/complus_config/name_config_with_pid/**">
             <Issue>https://github.com/dotnet/runtime/issues/54974</Issue>
         </ExcludeList>
-        <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/**">
-            <Issue>Need to update with Microsoft.Diagnostics.NETCore.Client port https://github.com/dotnet/runtime/pull/64358</Issue>
-        </ExcludeList>
         <ExcludeList Include = "$(XunitTestBinBase)/Interop/UnmanagedCallConv/UnmanagedCallConvTest/**">
             <Issue>https://github.com/dotnet/runtime/issues/53077</Issue>
         </ExcludeList>
@@ -3641,6 +3638,21 @@
         <ExcludeList Include = "$(XunitTestBinBase)/JIT/Regression/JitBlue/DevDiv_461649/DevDiv_461649/**">
             <Issue>https://github.com/dotnet/runtime/issues/53353</Issue>
         </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/diagnosticport/**">
+            <Issue>Cannot run multiple apps on Android for subprocesses</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/pauseonstart/**">
+            <Issue>Cannot run multiple apps on Android for subprocesses</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/processenvironment/**">
+            <Issue>Cannot run multiple apps on Android for subprocesses</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/reverse/**">
+            <Issue>Cannot run multiple apps on Android for subprocesses</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/reverseouter/**">
+            <Issue>Cannot run multiple apps on Android for subprocesses</Issue>
+        </ExcludeList>
     </ItemGroup>
 
     <ItemGroup Condition=" $(TargetOS) == 'Android' And '$(TargetArchitecture)' == 'arm64' " >

src/tests/profiler/common/profiler_common.csproj

@@ -10,5 +10,6 @@
     <Compile Include="DiagnosticsIPCWorkaround.cs" />
     <ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
     <ProjectReference Include="$(TestSourceDir)tracing/eventpipe/common/common.csproj" />
+    <ProjectReference Include="$(TestSourceDir)tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj" />
   </ItemGroup>
 </Project>

src/tests/profiler/eventpipe/eventpipe.csproj

@@ -17,5 +17,6 @@
     <ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
     <ProjectReference Include="../common/profiler_common.csproj" />
     <ProjectReference Include="$(MSBuildThisFileDirectory)/../native/CMakeLists.txt" />
+    <ProjectReference Include="$(TestSourceDir)tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj" />
   </ItemGroup>
 </Project>

src/tests/profiler/eventpipe/reverse_startup.csproj

@@ -18,5 +18,6 @@
     <ProjectReference Include="../common/profiler_common.csproj" />
     <ProjectReference Include="$(TestSourceDir)tracing/eventpipe/common/common.csproj" />
     <ProjectReference Include="$(MSBuildThisFileDirectory)/../native/CMakeLists.txt" />
+    <ProjectReference Include="$(TestSourceDir)tracing/eventpipe/common/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj" />
   </ItemGroup>
 </Project>

src/tests/tracing/eventpipe/bigevent/bigevent.cs

@@ -8,10 +8,10 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Collections.Generic;
-using Microsoft.Diagnostics.Tools.RuntimeClient;
 using Microsoft.Diagnostics.Tracing;
 using Tracing.Tests.Common;
 using Microsoft.Diagnostics.Tracing.Parsers.Clr;
+using Microsoft.Diagnostics.NETCore.Client;
 
 namespace Tracing.Tests.BigEventsValidation
 {
@@ -26,10 +26,10 @@ private BigEventSource()
         }
 
         public static BigEventSource Log = new BigEventSource();
-        
+
         public void BigEvent()
         {
-            WriteEvent(1, bigString);        
+            WriteEvent(1, bigString);
         }
 
         public void SmallEvent()
@@ -38,28 +38,27 @@ public void SmallEvent()
         }
     }
 
-    
+
     public class BigEventsValidation
     {
         public static int Main(string[] args)
         {
             // This test tries to send a big event (>100KB) and checks that the app does not crash
             // See https://github.com/dotnet/runtime/issues/50515 for the regression issue
-            var providers = new List<Provider>()
+            var providers = new List<EventPipeProvider>()
             {
-                new Provider("BigEventSource")
+                new EventPipeProvider("BigEventSource", EventLevel.Verbose)
             };
 
-            var configuration = new SessionConfiguration(circularBufferSizeMB: 1024, format: EventPipeSerializationFormat.NetTrace,  providers: providers);
-            return IpcTraceTest.RunAndValidateEventCounts(_expectedEventCounts, _eventGeneratingAction, configuration, _Verify);
+            return IpcTraceTest.RunAndValidateEventCounts(_expectedEventCounts, _eventGeneratingAction, providers, 1024, _Verify);
         }
 
         private static Dictionary<string, ExpectedEventCount> _expectedEventCounts = new Dictionary<string, ExpectedEventCount>()
         {
             { "BigEventSource", -1 }
         };
 
-        private static Action _eventGeneratingAction = () => 
+        private static Action _eventGeneratingAction = () =>
         {
             // Write 10 big events
             for (int i = 0; i < 10; i++)