Crossgen2 ARM64 runs & initial cross-targeting support (#37331)

This change introduces initial provisions for dynamically loading the native JIT based on the targeting OS and architecture; the change expects the potentially multiple versions of clrjit to coexist with Crossgen2 in the same folder, marked by a OS / arch suffix. Based on these prerequisites the change adds ARM64 jobs to the Crossgen2 pipeline. Thanks Tomas
dotnet · Jun 10, 2020 · 882f818 · 882f818
1 parent 6d98488
commit 882f818
Show file tree

Hide file tree

Showing 11 changed files with 131 additions and 59 deletions.
diff --git a/eng/Signing.props b/eng/Signing.props
@@ -44,9 +44,11 @@
     <ItemsToSign Include="$(CoreCLRCrossgen2Dir)ILCompiler.DependencyAnalysisFramework.dll" />
     <ItemsToSign Include="$(CoreCLRCrossgen2Dir)ILCompiler.ReadyToRun.dll" />
     <ItemsToSign Include="$(CoreCLRCrossgen2Dir)ILCompiler.TypeSystem.ReadyToRun.dll" />
-    <ItemsToSign Include="$(CoreCLRCrossgen2Dir)clrjitilc.dll" />
     <ItemsToSign Include="$(CoreCLRCrossgen2Dir)jitinterface.dll" />
 
+    <ItemsToSign Condition="'$(TargetOS)' == 'Windows_NT'" Include="$(CoreCLRCrossgen2Dir)clrjit-win-$(TargetArchitecture).dll" />
+    <ItemsToSign Condition="'$(TargetOS)' != 'Windows_NT'" Include="$(CoreCLRCrossgen2Dir)clrjit-unix-$(TargetArchitecture).dll" />
+
     <ItemsToSign Include="@(CoreCLRCrossTargetItemsToSign)" />
 
     <FileSignInfo Include="mscordaccore.dll" CertificateName="MicrosoftSHA2" />

diff --git a/eng/pipelines/coreclr/crossgen2.yml b/eng/pipelines/coreclr/crossgen2.yml
@@ -22,8 +22,10 @@ jobs:
     buildConfig: checked
     platforms:
     - Linux_x64
+    - Linux_arm64
     - OSX_x64
     - Windows_NT_x64
+    - Windows_NT_arm64
     - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     jobParameters:
       testGroup: innerloop
@@ -45,8 +47,10 @@ jobs:
     buildConfig: checked
     platforms:
     - Linux_x64
+    - Linux_arm64
     - OSX_x64
     - Windows_NT_x64
+    - Windows_NT_arm64
     jobParameters:
       testGroup: innerloop
       readyToRun: true

diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd
@@ -557,15 +557,11 @@ if defined __DoCrossgen (
 
 if defined __DoCrossgen2 (
     set __CrossgenArg="/p:Crossgen2=true"
-    if "%__BuildArch%" == "x64" (
-        echo %__MsgPrefix%Running crossgen2 on framework assemblies in CORE_ROOT: %CORE_ROOT%
-        call :PrecompileFX
-        if ERRORLEVEL 1 (
-            echo %__ErrMsgPrefix%%__MsgPrefix%Error: crossgen2 precompilation of framework assemblies failed
-            exit /b 1
-        )
-    ) else (
-        echo "%__MsgPrefix%Crossgen2 only supported on x64, for now"
+    echo %__MsgPrefix%Running crossgen2 on framework assemblies in CORE_ROOT: %CORE_ROOT%
+    call :PrecompileFX
+    if ERRORLEVEL 1 (
+        echo %__ErrMsgPrefix%%__MsgPrefix%Error: crossgen2 precompilation of framework assemblies failed
+        exit /b 1
     )
 )
 
@@ -642,10 +638,19 @@ set __FailedAssemblies=
 set __CompositeOutputDir=%CORE_ROOT%\composite.out
 set __CompositeResponseFile=%__CompositeOutputDir%\framework-r2r.dll.rsp
 
+set __CrossgenDir=%__BinDir%
+if /i "%__BuildArch%" == "arm" (set __CrossgenDir=!__CrossgenDir!\x86)
+if /i "%__BuildArch%" == "arm64" (set __CrossgenDir=!__CrossgenDir!\x64)
+
+set __CrossgenExe="%__CrossgenDir%\crossgen.exe"
+set __Crossgen2Dll="%__RepoRootDir%\dotnet.cmd" "%__CrossgenDir%\crossgen2\crossgen2.dll"
+
 if defined __CompositeBuildMode (
     mkdir !__CompositeOutputDir!
+    del /Q !__CompositeResponseFile!
     echo --composite>>!__CompositeResponseFile!
     echo -O>>!__CompositeResponseFile!
+    echo --targetarch:%__BuildArch%>>!__CompositeResponseFile!
     echo --out^:%__CompositeOutputDir%\framework-r2r.dll>>!__CompositeResponseFile!
 )
 
@@ -669,8 +674,7 @@ if defined __CompositeBuildMode (
 )
 
 if defined __CompositeBuildMode (
-    set __CompositeCommandLine="%__RepoRootDir%\dotnet.cmd"
-    set __CompositeCommandLine=!__CompositeCommandLine! "%CORE_ROOT%\crossgen2\crossgen2.dll"
+    set __CompositeCommandLine=%__Crossgen2Dll%
     set __CompositeCommandLine=!__CompositeCommandLine! "@%__CompositeResponseFile%"
     echo Building composite R2R framework^: !__CompositeCommandLine!
     call !__CompositeCommandLine!
@@ -691,15 +695,6 @@ REM Compile the managed assemblies in Core_ROOT before running the tests
 set AssemblyPath=%1
 set AssemblyName=%2
 
-set __CrossgenExe="%__BinDir%\crossgen.exe"
-if /i "%__BuildArch%" == "arm" ( set __CrossgenExe="%__BinDir%\x86\crossgen.exe" )
-if /i "%__BuildArch%" == "arm64" ( set __CrossgenExe="%__BinDir%\x64\crossgen.exe" )
-set __CrossgenExe=%__CrossgenExe%
-
-if defined __DoCrossgen2 (
-    set __CrossgenExe="%__RepoRootDir%\dotnet.cmd" "%CORE_ROOT%\crossgen2\crossgen2.dll"
-)
-
 REM Intentionally avoid using the .dll extension to prevent
 REM subsequent compilations from picking it up as a reference
 set __CrossgenOutputFile="%CORE_ROOT%\temp.ni._dll"
@@ -710,7 +705,7 @@ if defined __DoCrossgen (
     echo !__CrossgenCmd!
     !__CrossgenCmd!
 ) else (
-    set __CrossgenCmd=!__CrossgenExe! -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:"!CORE_ROOT!\netstandard.dll" -O --inputbubble --out:!__CrossgenOutputFile! !AssemblyPath!
+    set __CrossgenCmd=!__Crossgen2Dll! -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:"!CORE_ROOT!\netstandard.dll" -O --inputbubble --out:!__CrossgenOutputFile! !AssemblyPath! --targetarch %__BuildArch%
     echo !__CrossgenCmd!
     call !__CrossgenCmd!
 )

diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh
@@ -181,10 +181,11 @@ precompile_coreroot_fx()
     local totalPrecompiled=0
     local failedToPrecompile=0
     local compositeCommandLine="${__DotNetCli}"
-    compositeCommandLine+=" $__BinDir/crossgen2/crossgen2.dll"
+    compositeCommandLine+=" $__Crossgen2Dll"
     compositeCommandLine+=" --composite"
     compositeCommandLine+=" -O"
     compositeCommandLine+=" --out:$outputDir/framework-r2r.dll"
+    compositeCommandLine+=" --targetarch ${__BuildArch}"
     declare -a failedAssemblies
 
     filesToPrecompile=$(find -L "$overlayDir" -maxdepth 1 -iname Microsoft.\*.dll -o -iname System.\*.dll -o -iname netstandard.dll -o -iname mscorlib.dll -type f)
@@ -206,7 +207,7 @@ precompile_coreroot_fx()
         fi
 
         if [[ "$__DoCrossgen2" != 0 ]]; then
-            commandLine="${__DotNetCli} $overlayDir/crossgen2/crossgen2.dll $crossgen2References -O --inputbubble --out $outputDir/$(basename $filename) $filename"
+            commandLine="${__DotNetCli} $__Crossgen2Dll $crossgen2References -O --inputbubble --out $outputDir/$(basename $filename) $filename --targetarch ${__BuildArch}"
         fi
 
         echo Precompiling "$filename"
@@ -696,6 +697,7 @@ if [[ "$__CrossBuild" == 1 ]]; then
 fi
 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__TargetOS.$BuildArch.$__BuildType.log"
 __CrossgenExe="$__CrossComponentBinDir/crossgen"
+__Crossgen2Dll="$__CrossComponentBinDir/crossgen2/crossgen2.dll"
 
 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 # This is needed by CLI to function.

diff --git a/src/coreclr/crosscomponents.cmake b/src/coreclr/crosscomponents.cmake
@@ -8,6 +8,7 @@ if (CLR_CMAKE_HOST_OS STREQUAL CLR_CMAKE_TARGET_OS)
     set (CLR_CROSS_COMPONENTS_LIST
         crossgen
         clrjit
+        jitinterface
     )
 endif()
 

diff --git a/src/coreclr/src/tools/Common/JitInterface/JitConfigProvider.cs b/src/coreclr/src/tools/Common/JitInterface/JitConfigProvider.cs
@@ -8,6 +8,7 @@
 using System.Runtime.InteropServices;
 using System.Threading;
 using ILCompiler;
+using Internal.TypeSystem;
 using NumberStyles = System.Globalization.NumberStyles;
 
 namespace Internal.JitInterface
@@ -29,7 +30,11 @@ public static JitConfigProvider Instance
         private Dictionary<string, string> _config = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
         private object _keepAlive; // Keeps callback delegates alive
 
-        public static void Initialize(IEnumerable<CorJitFlag> jitFlags, IEnumerable<KeyValuePair<string, string>> parameters, string jitPath = null)
+        public static void Initialize(
+            TargetDetails target,
+            IEnumerable<CorJitFlag> jitFlags,
+            IEnumerable<KeyValuePair<string, string>> parameters,
+            string jitPath = null)
         {
             var config = new JitConfigProvider(jitFlags, parameters);
 
@@ -39,18 +44,22 @@ public static void Initialize(IEnumerable<CorJitFlag> jitFlags, IEnumerable<KeyV
                 throw new InvalidOperationException();
 
 #if READYTORUN
-            if (jitPath != null)
+            NativeLibrary.SetDllImportResolver(typeof(CorInfoImpl).Assembly, (libName, assembly, searchPath) =>
             {
-                NativeLibrary.SetDllImportResolver(typeof(CorInfoImpl).Assembly, (libName, assembly, searchPath) =>
+                IntPtr libHandle = IntPtr.Zero;
+                if (libName == CorInfoImpl.JitLibrary)
                 {
-                    IntPtr libHandle = IntPtr.Zero;
-                    if (libName == CorInfoImpl.JitLibrary)
+                    if (!string.IsNullOrEmpty(jitPath))
                     {
-                        libHandle = NativeLibrary.Load(jitPath, assembly, searchPath);
+                        libHandle = NativeLibrary.Load(jitPath);
                     }
-                    return libHandle;
-                });
-            }
+                    else
+                    {
+                        libHandle = NativeLibrary.Load("clrjit-" + GetTargetSpec(target), assembly, searchPath);
+                    }
+                }
+                return libHandle;
+            });
 #else
             Debug.Assert(jitPath == null);
 #endif
@@ -122,6 +131,20 @@ public string GetStringConfigValue(string name)
             return String.Empty;
         }
 
+        private static string GetTargetSpec(TargetDetails target)
+        {
+            string targetOSComponent = (target.OperatingSystem == TargetOS.Windows ? "win" : "unix");
+            string targetArchComponent = target.Architecture switch
+            {
+                TargetArchitecture.X86 => "x86",
+                TargetArchitecture.X64 => "x64",
+                TargetArchitecture.ARM => "arm",
+                TargetArchitecture.ARM64 => "arm64",
+                _ => throw new NotImplementedException(target.Architecture.ToString())
+            };
+            return targetOSComponent + '-' + targetArchComponent;
+        }
+
         #region Unmanaged instance
 
         private unsafe IntPtr CreateUnmanagedInstance()

diff --git a/...src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs b/...src/tools/crossgen2/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs
@@ -13,7 +13,6 @@
 using Internal.IL;
 using Internal.JitInterface;
 using Internal.ReadyToRunConstants;
-using Internal.TypeSystem;
 using Internal.TypeSystem.Ecma;
 
 namespace ILCompiler
@@ -208,7 +207,7 @@ public override ICompilation ToCompilation()
             if (_ibcTuning)
                 corJitFlags.Add(CorJitFlag.CORJIT_FLAG_BBINSTR);
 
-            JitConfigProvider.Initialize(corJitFlags, _ryujitOptions, _jitPath);
+            JitConfigProvider.Initialize(_context.Target, corJitFlags, _ryujitOptions, _jitPath);
 
             return new ReadyToRunCodegenCompilation(
                 graph,

diff --git a/src/coreclr/src/tools/crossgen2/crossgen2/crossgen2.csproj b/src/coreclr/src/tools/crossgen2/crossgen2/crossgen2.csproj
@@ -40,30 +40,64 @@
     </PackageReference>
   </ItemGroup>
 
-  <PropertyGroup Condition="'$(TargetsUnix)' == 'true'">
+  <PropertyGroup>
+    <CrossHostArch></CrossHostArch>
+    <CrossHostArch Condition="'$(TargetArchitecture)' == 'arm64'">x64</CrossHostArch>
+
+    <TargetOSComponent>unix</TargetOSComponent>
+    <TargetOSComponent Condition="'$(TargetOS)' == 'Windows_NT'">win</TargetOSComponent>
+    <TargetSpec>$(TargetOSComponent)-$(TargetArchitecture)</TargetSpec>
+
     <LibraryNamePrefix>lib</LibraryNamePrefix>
-    <LibraryNameExtension>.so</LibraryNameExtension>
-    <LibraryNameExtension Condition="'$(TargetsOSX)' == 'true'">.dylib</LibraryNameExtension>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(TargetsWindows)' == 'true'">
-    <LibraryNamePrefix></LibraryNamePrefix>
-    <LibraryNameExtension>.dll</LibraryNameExtension>
+    <LibraryNamePrefix Condition="$([MSBuild]::IsOsPlatform('WINDOWS'))"></LibraryNamePrefix>
+    <LibraryNameExtension Condition="$([MSBuild]::IsOsPlatform('WINDOWS'))">.dll</LibraryNameExtension>
+    <LibraryNameExtension Condition="$([MSBuild]::IsOsPlatform('LINUX'))">.so</LibraryNameExtension>
+    <LibraryNameExtension Condition="$([MSBuild]::IsOsPlatform('OSX'))">.dylib</LibraryNameExtension>
+
+    <JitInterfaceLibraryName>$(LibraryNamePrefix)jitinterface$(LibraryNameExtension)</JitInterfaceLibraryName>
   </PropertyGroup>
 
   <ItemGroup>
-    <Content Include="$(BinDir)\$(LibraryNamePrefix)jitinterface$(LibraryNameExtension)"
+    <Content Include="$(BinDir)\$(JitInterfaceLibraryName)"
       CopyToOutputDirectory="PreserveNewest"
       CopyToPublishDirectory="PreserveNewest"
       Link="%(FileName)%(Extension)"
      />
 
-    <Content Include="$(BinDir)\$(LibraryNamePrefix)clrjit$(LibraryNameExtension)"
+    <Content Include="$(BinDir)\$(NativeArchFolder)$(LibraryNamePrefix)clrjit$(LibraryNameExtension)"
       CopyToOutputDirectory="PreserveNewest"
       CopyToPublishDirectory="PreserveNewest"
-      Link="$(LibraryNamePrefix)clrjitilc$(LibraryNameExtension)"
+      Link="$(LibraryNamePrefix)clrjit-$(TargetSpec)$(LibraryNameExtension)"
      />
   </ItemGroup>
 
+  <Target Name="CreateCrossTargetingPackage" AfterTargets="Build" Condition="'$(CrossHostArch)' != ''">
+
+    <PropertyGroup>
+      <CrossPackageFolder>$(BinDir)\$(CrossHostArch)\crossgen2</CrossPackageFolder>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageFile Include="$(BinDir)\crossgen2\*"
+        Exclude="$(BinDir)\crossgen2\$(JitInterfaceLibraryName);$(BinDir)\crossgen2\$(LibraryNamePrefix)clrjit-*$(LibraryNameExtension)" />
+      <PackageFile Include="$(BinDir)\$(CrossHostArch)\$(JitInterfaceLibraryName)" />
+    </ItemGroup>
+
+    <MakeDir Directories="$(CrossPackageFolder)" />
+    <Copy
+      SourceFiles="@(PackageFile)"
+      DestinationFiles="@(PackageFile->'$(CrossPackageFolder)\%(FileName)%(Extension)')"
+      UseHardLinksIfPossible="true"
+      />
+
+    <Copy
+      SourceFiles="$(BinDir)\$(CrossHostArch)\$(LibraryNamePrefix)clrjit$(LibraryNameExtension)"
+      DestinationFiles="$(CrossPackageFolder)\$(LibraryNamePrefix)clrjit-$(TargetSpec)$(LibraryNameExtension)"
+      UseHardLinksIfPossible="true"
+      />
+
+  </Target>
+
   <Target Name="GenerateDepsJsonFile" Returns="$(ProjectDepsFilePath)" DependsOnTargets="GenerateBuildDependencyFile" />
 
   <Target Name="RemoveLongNameDacFromDepsJson" AfterTargets="ResolveRuntimePackAssets" BeforeTargets="GenerateBuildDependencyFile" Condition="'$(RemoveLongNameDac)' == 'true'">

diff --git a/src/coreclr/tests/issues.targets b/src/coreclr/tests/issues.targets
@@ -913,29 +913,35 @@
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Intrinsics/TypeIntrinsics_ro/*">
             <Issue>https://github.com/dotnet/runtime/issues/32725</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/tailcall_v4/hijacking/*">
+            <Issue>https://github.com/dotnet/runtime/issues/7597</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/opt/ObjectStackAllocation/ObjectStackAllocationTests/*">
+            <Issue>Not compatible with crossgen2</Issue>
+        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/CLR-x86-JIT/v2.1/DDB/B168384/LdfldaHack/*">
             <Issue>https://github.com/dotnet/runtime/issues/615</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/JIT/Stress/ABI/**/*">
-            <Issue>https://github.com/dotnet/runtime/issues/32725</Issue>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/CLR-x86-JIT/V1.2-Beta1/b219940/b219940/*">
+            <Issue>https://github.com/dotnet/runtime/issues/37579</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/tracing/tracevalidation/tracelogging/tracelogging/*">
-            <Issue>https://github.com/dotnet/runtime/issues/32728</Issue>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/JitBlue/GitHub_27924/GitHub_27924/*">
+            <Issue>https://github.com/dotnet/runtime/issues/34316</Issue>
+        </ExcludeList>        
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/SIMD/SimpleSIMDProgram/*">
+            <Issue>https://github.com/dotnet/runtime/issues/35724</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/JIT/opt/ObjectStackAllocation/ObjectStackAllocationTests/*">
-            <Issue>Not compatible with crossgen2</Issue>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Stress/ABI/**/*">
+            <Issue>https://github.com/dotnet/runtime/issues/32725</Issue>
         </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/JIT/superpmi/superpmicollect/*">
             <Issue>Not compatible with crossgen2</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/tailcall_v4/hijacking/*">
-            <Issue>https://github.com/dotnet/runtime/issues/7597</Issue>
-        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/Loader/AssemblyDependencyResolver/AssemblyDependencyResolverTests/AssemblyDependencyResolverTests/*">
             <Issue>https://github.com/dotnet/runtime/issues/34905</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/JIT/SIMD/SimpleSIMDProgram/*">
-            <Issue>https://github.com/dotnet/runtime/issues/35724</Issue>
+        <ExcludeList Include="$(XunitTestBinBase)/tracing/tracevalidation/tracelogging/tracelogging/*">
+            <Issue>https://github.com/dotnet/runtime/issues/32728</Issue>
         </ExcludeList>
     </ItemGroup>
 

diff --git a/src/coreclr/tests/src/CLRTest.CrossGen.targets b/src/coreclr/tests/src/CLRTest.CrossGen.targets
@@ -106,7 +106,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
     echo -r:$CORE_ROOT/System.*.dll>>$__ResponseFile
     echo -r:$CORE_ROOT/Microsoft.*.dll>>$__ResponseFile
     echo -r:$CORE_ROOT/mscorlib.dll>>$__ResponseFile
-    echo --targetarch=x64>>$__ResponseFile
+    echo --targetarch:$(TargetArchitecture)>>$__ResponseFile
     echo -O>>$__ResponseFile
 
     echo "Response file: $__ResponseFile"
@@ -204,7 +204,7 @@ if defined RunCrossGen2 (
     )
 
     echo -o:!__OutputFile!>>!__ResponseFile!
-    echo --targetarch:x64>>!__ResponseFile!
+    echo --targetarch:$(TargetArchitecture)>>!__ResponseFile!
     echo -O>>!__ResponseFile!
     echo -r:!CORE_ROOT!\System.*.dll>>!__ResponseFile!
     echo -r:!CORE_ROOT!\Microsoft.*.dll>>!__ResponseFile!
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,6 +8,7 @@ if (CLR_CMAKE_HOST_OS STREQUAL CLR_CMAKE_TARGET_OS) @@
         set (CLR_CROSS_COMPONENTS_LIST
             crossgen
             clrjit
+            jitinterface
         )
     endif()
@@ Expand Down @@