Merge pull request #1322 from jkotas/merge-runtime

Merge from dotnet/runtime

Directory.Build.props

@@ -73,8 +73,10 @@
     <AndroidAppBuilderDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'AndroidAppBuilder', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</AndroidAppBuilderDir>
     <WasmAppBuilderDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmAppBuilder', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</WasmAppBuilderDir>
     <WasmBuildTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</WasmBuildTasksDir>
+    <WorkloadBuildTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</WorkloadBuildTasksDir>
     <MonoAOTCompilerDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoAOTCompilerDir>
     <RuntimeConfigParserDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'RuntimeConfigParser', 'Debug', '$(NetCoreAppToolCurrent)'))</RuntimeConfigParserDir>
+    <JsonToItemsTaskFactoryDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'JsonToItemsTaskFactory', 'Debug', '$(NetCoreAppToolCurrent)'))</JsonToItemsTaskFactoryDir>
 
     <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
     <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', 'net461', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
@@ -84,8 +86,10 @@
     <AndroidAppBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(AndroidAppBuilderDir)', 'AndroidAppBuilder.dll'))</AndroidAppBuilderTasksAssemblyPath>
     <WasmAppBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WasmAppBuilderDir)', 'WasmAppBuilder.dll'))</WasmAppBuilderTasksAssemblyPath>
     <WasmBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WasmBuildTasksDir)', 'WasmBuildTasks.dll'))</WasmBuildTasksAssemblyPath>
+    <WorkloadBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll'))</WorkloadBuildTasksAssemblyPath>
     <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath>
     <RuntimeConfigParserTasksAssemblyPath>$([MSBuild]::NormalizePath('$(RuntimeConfigParserDir)', 'RuntimeConfigParser.dll'))</RuntimeConfigParserTasksAssemblyPath>
+    <JsonToItemsTaskFactoryTasksAssemblyPath>$([MSBuild]::NormalizePath('$(JsonToItemsTaskFactoryDir)', 'JsonToItemsTaskFactory.dll'))</JsonToItemsTaskFactoryTasksAssemblyPath>
     <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)'))</ILAsmToolPath>
   </PropertyGroup>
 

docs/design/mono/wasm-aot.md

@@ -0,0 +1,66 @@
+# WebAssembly AOT code generation
+
+## Basic operation
+
+The LLVM backend of the Mono JIT is used to generate an llvm .bc file for each assembly, then the .bc files are
+compiled to webassembly using emscripten, then the resulting wasm files are linked into the final app. The 'bitcode'/'llvmonly'
+variant of the LLVM backend is used since webassembly doesn't support inline assembly etc.
+
+## GC Support
+
+On wasm, the execution stack is not stored in linear memory, so its not possible to scan it for GC references. However, there
+is an additional C stack which stores variables whose addresses are taken. Variables which hold GC references are marked as
+'volatile' in the llvm backend, forcing llvm to spill those to the C stack so they can be scanned.
+
+## Interpreter support
+
+Its possible for AOTed and interpreted code to interop, this is called mixed mode.
+For the AOT -> interpreter case, every call from AOTed code which might end up in the interpreter is
+emitted as an indirect call. When the callee is not found, a wrapper function is used which
+packages up the arguments into an array and passes control to the interpreter.
+For the interpreter -> AOT case, and similar wrapper function is used which receives the
+arguments and a return value pointer from the interpreter in an array, and calls the
+AOTed code. There is usually one aot->interp and interp->aot wrapper for each signature, with
+some sharing. These wrappers are generated by the AOT compiler when the 'interp' aot option
+is used.
+
+## Null checks
+
+Since wasm has no signal support, we generate explicit null checks.
+
+## Issues
+
+The generated code is in general much bigger than the code generated on ios etc. Some of the
+current issues are described below.
+
+### Function pointers
+
+The runtime needs to be able to do a IL method -> wasm function lookup. To do this, every
+AOT image includes a table mapping from a method index to wasm functions. This means that
+every generated AOT method has its address taken, which severely limits the interprocedural
+optimizations that LLVM can do, since it cannot determine the set of callers for a function.
+This means that it cannot remove functions corresponding to unused IL methods, cannot
+specialize functions for constant/nonnull arguments, etc.
+The dotnet linker includes some support for adding a [DisablePrivateReflection] attribute to
+methods which cannot be called using reflection, and the AOT compiler could use this
+to avoid generating function pointers for methods which are not called from outside the
+AOT image. This is not enabled right now because the linker support is not complete.
+
+### Null checks
+
+The explicit null checking code adds a lot of size overhead since null checks are very common.
+
+### Virtual calls
+
+Vtable slots are lazily initialized on the first call, i.e. every virtual call looks like this:
+```C
+vt_entry = vtable [slot];
+if (vt_entry == null)
+	vt_entry = init_vt_entry ();
+```
+
+### GC overhead
+
+Since GC variables are marked as volatile and stored on the C stack, they are loaded/stored on every access,
+even if there is no GC safe point between the accesses. Instead, they should only be loaded/stored around
+GC safe points.

docs/project/list-of-diagnostics.md

@@ -85,6 +85,7 @@ The PR that reveals the implementation of the `<IncludeInternalObsoleteAttribute
 |  __`SYSLIB0028`__ | X509Certificate2.PrivateKey is obsolete. Use the appropriate method to get the private key, such as GetRSAPrivateKey, or use the CopyWithPrivateKey method to create a new instance with a private key. |
 |  __`SYSLIB0029`__ | ProduceLegacyHmacValues is obsolete. Producing legacy HMAC values is no longer supported. |
 |  __`SYSLIB0030`__ | HMACSHA1 always uses the algorithm implementation provided by the platform. Use a constructor without the useManagedSha1 parameter. |
+|  __`SYSLIB0031`__ | EncodeOID is obsolete. Use the ASN.1 functionality provided in System.Formats.Asn1. |
 
 ## Analyzer Warnings
 

docs/workflow/testing/testing-workloads.md

@@ -0,0 +1,28 @@
+# Testing Workloads
+
+Workloads based on packages in `artifacts` can be installed, and used for testing.
+
+- This is done by installing a specified SDK version (`$(SdkVersionForWorkloadTesting)`) in `artifacts/bin/dotnet-workload`.
+- Then the manifest for the workload in `@(WorkloadIdForTesting)` is installed
+    - Then workload packs are installed
+    - packs, or manifests not generated by `runtime`, are restored from nuget
+
+- The SDK is installed by `ProvisionSdkForWorkloadTesting` target
+- and the workload is installed by `InstallWorkloadUsingArtifacts` target, using the `InstallWorkloadFromArtifacts` task
+
+- `@(WorkloadIdForTesting)`:
+
+Example for wasm:
+```xml
+    <WorkloadIdForTesting Include="wasm-tools"
+                          Name="microsoft.net.workload.mono.toolchain"
+                          ManifestName="Microsoft.NET.Workload.Mono.ToolChain"
+                          Version="$(PackageVersion)"
+                          VersionBand="$(SdkBandVersion)" />
+```
+
+- Currently, this is used only by `src/tests/BuildWasmApps/Wasm.Build.Tests`
+
+## Limitations:
+
+- The cross compiler package is built manually from the `InstallWorkloadUsingArtifacts` target

eng/CodeAnalysis.ruleset

@@ -68,6 +68,7 @@
     <Rule Id="CA1416" Action="Warning" />       <!-- Validate platform compatibility -->
     <Rule Id="CA1417" Action="Warning" />       <!-- Do not use 'OutAttribute' on string parameters for P/Invokes -->
     <Rule Id="CA1418" Action="Warning" />       <!-- Use valid platform string -->
+    <Rule Id="CA1419" Action="Warning" />       <!-- Provide a public parameterless constructor for concrete types derived from 'System.Runtime.InteropServices.SafeHandle' -->
     <Rule Id="CA1501" Action="None" />          <!-- Avoid excessive inheritance -->
     <Rule Id="CA1502" Action="None" />          <!-- Avoid excessive complexity -->
     <Rule Id="CA1505" Action="None" />          <!-- Avoid unmaintainable code -->

eng/Subsets.props

@@ -363,7 +363,7 @@
         <SharedFrameworkProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono'" Include="$(InstallerProjectRoot)pkg\sfx\installers\dotnet-host.proj" />
         <SharedFrameworkProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono'" Include="$(InstallerProjectRoot)pkg\sfx\installers\dotnet-hostfxr.proj" />
         <SharedFrameworkProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono'" Include="$(InstallerProjectRoot)pkg\sfx\installers\dotnet-runtime-deps\*.proj" />
-        <SharedFrameworkProjectToBuild Condition="'$(MonoCrossAOTTargetOS)' != ''" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\monocrossaot.sfxproj" />
+        <SharedFrameworkProjectToBuild Condition="'$(MonoCrossAOTTargetOS)' != ''" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\monocrossaot.sfxproj" Pack="true" />
       </ItemGroup>
       <ItemGroup>
         <SharedFrameworkProjectToBuild Condition="'$(BuildMonoAOTCrossCompilerOnly)' != 'true'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Runtime.sfxproj" />