From ca7caba3c3ddf113a731ba70b47e5e544e2826ad Mon Sep 17 00:00:00 2001 From: Tomas Date: Sun, 3 Apr 2022 20:42:08 +0200 Subject: [PATCH 1/9] Always run CopyAllNativeProjectReferenceBinaries Previously this target was only called from the recursive traversal used when globally copying native artifacts for all projects (this is used in the lab where the native artifacts are produced on a different machine than the managed test artifacts). In case of combined (local) test build we weren't running the target, only its dependencies (ResolveCMakeNativeProjectReference and ConsolidateNativeProjectReference); this used to be sufficient as these two targets handle all logic related to copying native components of a normal project, however I have recently added special logic for copying merged test wrapper native artifacts the CopyAllNativeProjectReferenceBinaries target and so we now must make sure it also always runs. Mark JIT optimization-sensitive tests as out-of-process and fix logic around skipping of out-of-process tests on the excluded platforms. Thanks Tomas --- .../Common/CoreCLRTestLibrary/OutOfProcessTest.cs | 8 +++++++- src/tests/Directory.Build.targets | 11 ++++++++--- .../JIT/Methodical/Arrays/misc/arrres_il_r.ilproj | 1 + src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj | 1 + 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs index 976f542a6b21eb..ad68f35537b92c 100644 --- a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs +++ b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; @@ -71,6 +71,12 @@ public static void RunOutOfProcessTest(string basePath, string assemblyPath) testExecutable = Path.Combine(baseDir, Path.ChangeExtension(assemblyPath.Replace("\\", "/"), ".sh")); } + if (!File.Exists(testExecutable)) + { + // Skip platform-specific test when running on the excluded platform + return; + } + System.IO.Directory.CreateDirectory(outputDir); ret = wrapper.RunTest(testExecutable, outputFile, errorFile, Assembly.GetEntryAssembly()!.FullName!, testBinaryBase, outputDir); diff --git a/src/tests/Directory.Build.targets b/src/tests/Directory.Build.targets index 96d65ab0eabf50..582f8c7b7ae59a 100644 --- a/src/tests/Directory.Build.targets +++ b/src/tests/Directory.Build.targets @@ -217,12 +217,17 @@ + DependsOnTargets="ResolveCMakeNativeProjectReference;ConsolidateNativeProjectReference" + BeforeTargets="Build"> - + + + + + - + PdbOnly true + true diff --git a/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj b/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj index 2856022120ab82..da27e6f81deaf2 100644 --- a/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj +++ b/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj @@ -13,6 +13,7 @@ true + true From 95019a82f871fac026f30990f94622ea8610c945 Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Thu, 7 Apr 2022 23:32:30 +0200 Subject: [PATCH 2/9] Fix GetFilteredExcludeList to properly report out-of-process tests --- src/tests/issues.targets | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/issues.targets b/src/tests/issues.targets index caaae135a4f87a..e003e2d3726b1e 100644 --- a/src/tests/issues.targets +++ b/src/tests/issues.targets @@ -4007,6 +4007,8 @@ + From 1f6379dda0ed96bee7cdefde87e57c29c17ee765 Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Fri, 8 Apr 2022 15:07:27 +0200 Subject: [PATCH 3/9] Add issues.targets exclusions for compat_i4_u_il_d / compat_i4_u_il_r --- src/tests/issues.targets | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tests/issues.targets b/src/tests/issues.targets index e003e2d3726b1e..3b98b5cb59f92b 100644 --- a/src/tests/issues.targets +++ b/src/tests/issues.targets @@ -3131,6 +3131,12 @@ needs triage + + https://github.com/dotnet/runtime/issues/67756 + + + https://github.com/dotnet/runtime/issues/67756 + needs triage From 07bfeb704d75b85ee91a00af09d2b35cfab1a8c3 Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Fri, 8 Apr 2022 15:12:55 +0200 Subject: [PATCH 4/9] Log timestamps in generated wrapper for investigation of timeouts --- src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index d0db16db340b53..213fc3d36d1bbd 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -338,15 +338,15 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe builder.AppendLine($"System.TimeSpan testStart = stopwatch.Elapsed;"); builder.AppendLine("try {"); - builder.AppendLine($"System.Console.WriteLine(\"Running test: {{0}}\", {test.TestNameExpression});"); + builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();"); builder.AppendLine(testExecutionExpression); builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput());"); - builder.AppendLine($"System.Console.WriteLine(\"Passed test: {{0}}\", {test.TestNameExpression});"); + builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); builder.AppendLine("catch (System.Exception ex) {"); builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput());"); - builder.AppendLine($"System.Console.WriteLine(\"Failed test: {{0}}\", {test.TestNameExpression});"); + builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); builder.AppendLine("}"); From 7f9b8fb72b3e0dce95e4f3342f7e73bc9daeaad1 Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Fri, 8 Apr 2022 15:19:57 +0200 Subject: [PATCH 5/9] Log timestamps around Crossgen2 invocations for investigation of timeouts --- src/tests/Common/CLRTest.CrossGen.targets | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tests/Common/CLRTest.CrossGen.targets b/src/tests/Common/CLRTest.CrossGen.targets index 606c4a6cd7543a..2e275840fa1a32 100644 --- a/src/tests/Common/CLRTest.CrossGen.targets +++ b/src/tests/Common/CLRTest.CrossGen.targets @@ -77,6 +77,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then __cg2ExitCode=0 OneFileCrossgen2() { + date +%H:%M:%S __OutputFile=$1 __ResponseFile="$__OutputFile.rsp" @@ -114,6 +115,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then __cg2ExitCode=$? export COMPlus_GCName COMPlus_GCStress COMPlus_HeapVerify COMPlus_ReadyToRun COMPlus_TC_OnStackReplacement COMPlus_TC_PartialCompilation + date +%H:%M:%S } if [ ! -z ${CompositeBuildMode+x} ]%3B then @@ -208,6 +210,7 @@ if defined RunCrossGen2 ( goto :DoneCrossgen2Operations :CompileOneFileCrossgen2 + echo %time% set __ResponseFile=!__OutputFile!.rsp del /Q !__ResponseFile! @@ -253,6 +256,7 @@ if defined RunCrossGen2 ( call !__Command! endlocal set CrossGen2Status=!ERRORLEVEL! + echo %time% Exit /b 0 :DoneCrossgen2Operations From 2c0ef88a3878c76b37e1b77b2e59b9f71be48e95 Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Fri, 8 Apr 2022 16:03:53 +0200 Subject: [PATCH 6/9] Suppress noisy stderr spew for deletion of non-existent rsp files --- src/tests/Common/CLRTest.CrossGen.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/CLRTest.CrossGen.targets b/src/tests/Common/CLRTest.CrossGen.targets index 2e275840fa1a32..5bf4357e327c91 100644 --- a/src/tests/Common/CLRTest.CrossGen.targets +++ b/src/tests/Common/CLRTest.CrossGen.targets @@ -81,7 +81,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then __OutputFile=$1 __ResponseFile="$__OutputFile.rsp" - rm $__ResponseFile + rm $__ResponseFile 2>/dev/null __Command=$_DebuggerFullPath # Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path @@ -212,7 +212,7 @@ if defined RunCrossGen2 ( :CompileOneFileCrossgen2 echo %time% set __ResponseFile=!__OutputFile!.rsp - del /Q !__ResponseFile! + del /Q !__ResponseFile! 2>null set __Command=!_DebuggerFullPath! REM Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path From 689d04ac6b204bf9e09e59e2fa9aad4de462764b Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Fri, 8 Apr 2022 17:22:27 +0200 Subject: [PATCH 7/9] Suppress spurious stderr; expand test timeout to 30 minutes In the merged scenario we're intentionally reducing the number of work items at the expense of a single work item size. When Crossgen2 compilation is involved, with hundreds of tests in the Methodical_*** projects we're hitting the 15 minute test timeout so I'm bumping it up to 30. I have also suppressed uninteresting stderr spew from deletion of the pre-existing composite image. Thanks Tomas --- src/tests/Common/CLRTest.CrossGen.targets | 4 ++-- src/tests/run.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/Common/CLRTest.CrossGen.targets b/src/tests/Common/CLRTest.CrossGen.targets index 5bf4357e327c91..cdd6007bb8d6b1 100644 --- a/src/tests/Common/CLRTest.CrossGen.targets +++ b/src/tests/Common/CLRTest.CrossGen.targets @@ -65,7 +65,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then cp $CORE_ROOT/lib*.so $CORE_ROOT/lib*.dylib $(scriptPath) else cp *.dll IL-CG2/ - rm IL-CG2/composite-r2r.dll + rm IL-CG2/composite-r2r.dll 2>/dev/null fi ExtraCrossGen2Args+=" $(CrossGen2TestExtraArguments)" @@ -185,7 +185,7 @@ if defined RunCrossGen2 ( mkdir IL-CG2 copy *.dll IL-CG2\ - del IL-CG2\composite-r2r.dll + del IL-CG2\composite-r2r.dll 2>null if defined CompositeBuildMode ( set ExtraCrossGen2Args=!ExtraCrossGen2Args! --composite diff --git a/src/tests/run.py b/src/tests/run.py index aab137bafa08f7..b6a1565704eb76 100755 --- a/src/tests/run.py +++ b/src/tests/run.py @@ -844,8 +844,8 @@ def run_tests(args, if args.skip_test_run: return - # Set default per-test timeout to 15 minutes (in milliseconds). - per_test_timeout = 15*60*1000 + # Set default per-test timeout to 30 minutes (in milliseconds). + per_test_timeout = 30*60*1000 # Setup the environment if args.long_gc: From 03bf73edbce7277da346a8536f8467f0539218ac Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Sat, 9 Apr 2022 00:34:54 +0200 Subject: [PATCH 8/9] Disable the compat_i4_u tests on all Mono platforms --- src/tests/issues.targets | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/issues.targets b/src/tests/issues.targets index 3b98b5cb59f92b..31c42e2e19fd59 100644 --- a/src/tests/issues.targets +++ b/src/tests/issues.targets @@ -1868,6 +1868,12 @@ https://github.com/dotnet/runtime/issues/34196 + + https://github.com/dotnet/runtime/issues/67756 + + + https://github.com/dotnet/runtime/issues/67756 + needs triage @@ -3131,12 +3137,6 @@ needs triage - - https://github.com/dotnet/runtime/issues/67756 - - - https://github.com/dotnet/runtime/issues/67756 - needs triage From b9eac47aac3140a786a9f4e88f76bcad1f8aab6a Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Sat, 9 Apr 2022 02:03:26 +0200 Subject: [PATCH 9/9] Fix typo (null -> nul) per Bruce's PR feedback --- src/tests/Common/CLRTest.CrossGen.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/CLRTest.CrossGen.targets b/src/tests/Common/CLRTest.CrossGen.targets index cdd6007bb8d6b1..30102cca1544b6 100644 --- a/src/tests/Common/CLRTest.CrossGen.targets +++ b/src/tests/Common/CLRTest.CrossGen.targets @@ -185,7 +185,7 @@ if defined RunCrossGen2 ( mkdir IL-CG2 copy *.dll IL-CG2\ - del IL-CG2\composite-r2r.dll 2>null + del IL-CG2\composite-r2r.dll 2>nul if defined CompositeBuildMode ( set ExtraCrossGen2Args=!ExtraCrossGen2Args! --composite @@ -212,7 +212,7 @@ if defined RunCrossGen2 ( :CompileOneFileCrossgen2 echo %time% set __ResponseFile=!__OutputFile!.rsp - del /Q !__ResponseFile! 2>null + del /Q !__ResponseFile! 2>nul set __Command=!_DebuggerFullPath! REM Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path