On demand projects loading when working with big C# codebases

src/OmniSharp.MSBuild/Options/MSBuildOptions.cs

@@ -8,6 +8,12 @@ internal class MSBuildOptions
         public string Platform { get; set; }
         public bool EnablePackageAutoRestore { get; set; }
 
+        /// <summary>
+        /// If true, MSBuild project system will only be loading projects for files that were opened in the editor
+        /// as well as referenced projects, recursively.
+        /// </summary>
+        public bool LoadProjectsOnDemand { get; set; }
+
         /// <summary>
         /// When set to true, the MSBuild project system will attempt to resolve the path to the MSBuild
         /// SDKs for a project by running 'dotnet --info' and retrieving the path. This is only needed

src/OmniSharp.MSBuild/ProjectManager.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
@@ -17,6 +18,7 @@
 using OmniSharp.MSBuild.Models.Events;
 using OmniSharp.MSBuild.Notification;
 using OmniSharp.MSBuild.ProjectFile;
+using OmniSharp.Options;
 using OmniSharp.Roslyn.Utilities;
 using OmniSharp.Services;
 using OmniSharp.Utilities;
@@ -39,12 +41,14 @@ public ProjectToUpdate(string filePath, bool allowAutoRestore)
         }
 
         private readonly ILogger _logger;
+        private readonly MSBuildOptions _options;
         private readonly IEventEmitter _eventEmitter;
         private readonly IFileSystemWatcher _fileSystemWatcher;
         private readonly MetadataFileReferenceCache _metadataFileReferenceCache;
         private readonly PackageDependencyChecker _packageDependencyChecker;
         private readonly ProjectFileInfoCollection _projectFiles;
         private readonly HashSet<string> _failedToLoadProjectFiles;
+        private readonly ConcurrentDictionary<string, int/*unused*/> _projectsRequestedOnDemand;
         private readonly ProjectLoader _projectLoader;
         private readonly OmniSharpWorkspace _workspace;
         private readonly ImmutableArray<IMSBuildEventSink> _eventSinks;
@@ -57,15 +61,25 @@ public ProjectToUpdate(string filePath, bool allowAutoRestore)
 
         private readonly FileSystemNotificationCallback _onDirectoryFileChanged;
 
-        public ProjectManager(ILoggerFactory loggerFactory, IEventEmitter eventEmitter, IFileSystemWatcher fileSystemWatcher, MetadataFileReferenceCache metadataFileReferenceCache, PackageDependencyChecker packageDependencyChecker, ProjectLoader projectLoader, OmniSharpWorkspace workspace, ImmutableArray<IMSBuildEventSink> eventSinks)
+        public ProjectManager(ILoggerFactory loggerFactory, 
+            MSBuildOptions options, 
+            IEventEmitter eventEmitter, 
+            IFileSystemWatcher fileSystemWatcher, 
+            MetadataFileReferenceCache metadataFileReferenceCache, 
+            PackageDependencyChecker packageDependencyChecker, 
+            ProjectLoader projectLoader, 
+            OmniSharpWorkspace workspace,
+            ImmutableArray<IMSBuildEventSink> eventSinks)
         {
             _logger = loggerFactory.CreateLogger<ProjectManager>();
+            _options = options ?? new MSBuildOptions();
             _eventEmitter = eventEmitter;
             _fileSystemWatcher = fileSystemWatcher;
             _metadataFileReferenceCache = metadataFileReferenceCache;
             _packageDependencyChecker = packageDependencyChecker;
             _projectFiles = new ProjectFileInfoCollection();
             _failedToLoadProjectFiles = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            _projectsRequestedOnDemand = new ConcurrentDictionary<string, int>(StringComparer.OrdinalIgnoreCase);
             _projectLoader = projectLoader;
             _workspace = workspace;
             _eventSinks = eventSinks;
@@ -75,6 +89,46 @@ public ProjectManager(ILoggerFactory loggerFactory, IEventEmitter eventEmitter,
             _processLoopTask = Task.Run(() => ProcessLoopAsync(_processLoopCancellation.Token));
 
             _onDirectoryFileChanged = OnDirectoryFileChanged;
+
+            if (_options.LoadProjectsOnDemand)
+            {
+                _workspace.AddWaitForProjectModelReadyHandler(WaitForProjectModelReadyAsync);
+            }
+        }
+
+        private async Task WaitForProjectModelReadyAsync(string documentPath)
+        {
+            // Search and queue for loading C# projects that are likely to reference the requested file.
+            // C# source files are located pretty much always in the same folder with their project file or in a subfolder below.
+            // Search up the root folder to enable on-demand project load in additional scenarios like the following:
+            // - A subfolder in a big codebase was opened in VSCode and then a document was opened that is located outside of the subfoler.
+            // - A workspace was opened in VSCode that includes multiple subfolders from a big codebase.
+            // - Documents from different codebases are opened in the same VSCode workspace.
+            string projectDir = Path.GetDirectoryName(documentPath);
+            do
+            {
+                var csProjFiles = Directory.EnumerateFiles(projectDir, "*.csproj", SearchOption.TopDirectoryOnly).ToList();
+                if (csProjFiles.Count > 0)
+                {
+                    foreach(string csProjFile in csProjFiles)
+                    {
+                        if (_projectsRequestedOnDemand.TryAdd(csProjFile, 0 /*unused*/))
+                        {
+                            QueueProjectUpdate(csProjFile, allowAutoRestore:true);
+                        }
+                    }
+
+                    break;
+                }
+
+                projectDir = Path.GetDirectoryName(projectDir);
+            } while(projectDir != null);
+
+            // Wait for all queued projects to load to ensure that workspace is fully up to date before this method completes.
+            // If the project for the document was loaded before and there are no other projects to load at the moment, the call below will be no-op.
+            _logger.LogTrace($"Started waiting for projects queue to be empty when requested '{documentPath}'");
+            await WaitForQueueEmptyAsync();
+            _logger.LogTrace($"Stopped waiting for projects queue to be empty when requested '{documentPath}'");
         }
 
         protected override void DisposeCore(bool disposing)

src/OmniSharp.MSBuild/ProjectSystem.cs

@@ -99,7 +99,14 @@ public void Initalize(IConfiguration configuration)
 
             _packageDependencyChecker = new PackageDependencyChecker(_loggerFactory, _eventEmitter, _dotNetCli, _options);
             _loader = new ProjectLoader(_options, _environment.TargetDirectory, _propertyOverrides, _loggerFactory, _sdksPathResolver);
-            _manager = new ProjectManager(_loggerFactory, _eventEmitter, _fileSystemWatcher, _metadataFileReferenceCache, _packageDependencyChecker, _loader, _workspace, _eventSinks);
+            _manager = new ProjectManager(_loggerFactory, _options, _eventEmitter, _fileSystemWatcher, _metadataFileReferenceCache, _packageDependencyChecker, 
+                _loader, _workspace, _eventSinks);
+
+            if (_options.LoadProjectsOnDemand)
+            {
+                _logger.LogInformation($"Skip loading projects listed in solution file or under target directory because {Key}:{nameof(MSBuildOptions.LoadProjectsOnDemand)} is true.");
+                return;
+            }
 
             var initialProjectPaths = GetInitialProjectPaths();
 

src/OmniSharp.Roslyn.CSharp/Services/Diagnostics/CodeCheckService.cs

@@ -24,7 +24,8 @@ public CodeCheckService(OmniSharpWorkspace workspace)
         public async Task<QuickFixResponse> Handle(CodeCheckRequest request)
         {
             var documents = request.FileName != null
-                ? _workspace.GetDocuments(request.FileName)
+                // To properly handle the request wait until all projects are loaded.
+                ? await _workspace.GetDocumentsFromFullProjectModelAsync(request.FileName)
                 : _workspace.CurrentSolution.Projects.SelectMany(project => project.Documents);
 
             var quickFixes = await documents.FindDiagnosticLocationsAsync(_workspace);

src/OmniSharp.Roslyn.CSharp/Services/Navigation/FindUsagesService.cs

@@ -26,7 +26,8 @@ public FindUsagesService(OmniSharpWorkspace workspace)
 
         public async Task<QuickFixResponse> Handle(FindUsagesRequest request)
         {
-            var document = _workspace.GetDocument(request.FileName);
+            // To produce complete list of usages for symbols in the document wait until all projects are loaded.
+            var document = await _workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
             var response = new QuickFixResponse();
             if (document != null)
             {

src/OmniSharp.Roslyn.CSharp/Services/Refactoring/V2/BaseCodeActionService.cs

@@ -62,7 +62,8 @@ protected BaseCodeActionService(OmniSharpWorkspace workspace, CodeActionHelper h
 
         protected async Task<IEnumerable<AvailableCodeAction>> GetAvailableCodeActions(ICodeActionRequest request)
         {
-            var document = this.Workspace.GetDocument(request.FileName);
+            // To produce a complete list of code actions for the document wait until all projects are loaded.
+            var document = await this.Workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
             if (document == null)
             {
                 return Array.Empty<AvailableCodeAction>();

src/OmniSharp.Roslyn.CSharp/Services/Structure/BlockStructureService.cs

@@ -50,7 +50,13 @@ public BlockStructureService(IAssemblyLoader loader, OmniSharpWorkspace workspac
 
         public async Task<BlockStructureResponse> Handle(BlockStructureRequest request)
         {
-            var document = _workspace.GetDocument(request.FileName);
+            // To provide complete code structure for the document wait until all projects are loaded.
+            var document = await _workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
+            if (document == null)
+            {
+                return null;
+            }
+
             var text = await document.GetTextAsync();
 
             var service = _blockStructureService.LazyGetMethod("GetService").InvokeStatic(new[] { document });

src/OmniSharp.Roslyn.CSharp/Services/Structure/CodeStructureService.cs

@@ -32,7 +32,8 @@ public CodeStructureService(
 
         public async Task<CodeStructureResponse> Handle(CodeStructureRequest request)
         {
-            var document = _workspace.GetDocument(request.FileName);
+            // To provide complete code structure for the document wait until all projects are loaded.
+            var document = await _workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
             if (document == null)
             {
                 return null;

src/OmniSharp.Roslyn.CSharp/Services/Structure/MembersAsTreeService.cs

@@ -25,7 +25,8 @@ public async Task<FileMemberTree> Handle(MembersTreeRequest request)
         {
             return new FileMemberTree()
             {
-                TopLevelTypeDefinitions = await StructureComputer.Compute(_workspace.GetDocuments(request.FileName), _discovers)
+                // To provide complete members tree for the document wait until all projects are loaded.
+                TopLevelTypeDefinitions = await StructureComputer.Compute(await _workspace.GetDocumentsFromFullProjectModelAsync(request.FileName), _discovers)
             };
         }
     }

src/OmniSharp.Roslyn/OmniSharpWorkspace.cs

@@ -25,6 +25,8 @@ public class OmniSharpWorkspace : Workspace
 
         private readonly ILogger<OmniSharpWorkspace> _logger;
 
+        private readonly ConcurrentBag<Func<string, Task>> _waitForProjectModelReadyHandlers = new ConcurrentBag<Func<string, Task>>();
+
         private readonly ConcurrentDictionary<string, ProjectInfo> miscDocumentsProjectInfos = new ConcurrentDictionary<string, ProjectInfo>();
 
         [ImportingConstructor]
@@ -37,6 +39,11 @@ public OmniSharpWorkspace(HostServicesAggregator aggregator, ILoggerFactory logg
 
         public override bool CanOpenDocuments => true;
 
+        public void AddWaitForProjectModelReadyHandler(Func<string, Task> handler)
+        {
+            _waitForProjectModelReadyHandlers.Add(handler);
+        }
+
         public override void OpenDocument(DocumentId documentId, bool activate = true)
         {
             var doc = this.CurrentSolution.GetDocument(documentId);
@@ -233,6 +240,18 @@ public Document GetDocument(string filePath)
             return CurrentSolution.GetDocument(documentId);
         }
 
+        public async Task<IEnumerable<Document>> GetDocumentsFromFullProjectModelAsync(string filePath)
+        {
+            await OnWaitForProjectModelReadyAsync(filePath);
+            return GetDocuments(filePath);
+        }
+
+        public async Task<Document> GetDocumentFromFullProjectModelAsync(string filePath)
+        {
+            await OnWaitForProjectModelReadyAsync(filePath);
+            return GetDocument(filePath);
+        }
+
         public override bool CanApplyChange(ApplyChangesKind feature)
         {
             return true;
@@ -363,5 +382,10 @@ public override async Task<TextAndVersion> LoadTextAndVersionAsync(
                 return textAndVersion;
             }
         }
+
+        private Task OnWaitForProjectModelReadyAsync(string filePath)
+        {
+            return Task.WhenAll(_waitForProjectModelReadyHandlers.Select(h => h(filePath)));
+        }  
     }
 }

test-assets/test-projects/DeepProjectTransitiveReference/App/App.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp1.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Lib1\Lib1.csproj" />
+  </ItemGroup>
+
+</Project>

test-assets/test-projects/DeepProjectTransitiveReference/App/Program.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace App
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            Console.WriteLine("Hello World!");
+        }
+    }
+}

test-assets/test-projects/DeepProjectTransitiveReference/DeepProjectTransitiveReference.sln

@@ -0,0 +1,64 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26124.0
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "App", "App\App.csproj", "{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lib1", "Lib1\Lib1.csproj", "{CE41561B-5D13-4688-8686-EEFF744BE8B5}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lib2", "Lib2\Lib2.csproj", "{BAC03617-3A2B-431E-A9E4-49441B745C4F}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Debug|x64.Build.0 = Debug|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Debug|x86.Build.0 = Debug|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Release|x64.ActiveCfg = Release|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Release|x64.Build.0 = Release|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Release|x86.ActiveCfg = Release|Any CPU
+		{632DFE45-B56E-4158-8F27-45E2BA0BAFCF}.Release|x86.Build.0 = Release|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Debug|x64.Build.0 = Debug|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Debug|x86.Build.0 = Debug|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Release|x64.ActiveCfg = Release|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Release|x64.Build.0 = Release|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Release|x86.ActiveCfg = Release|Any CPU
+		{CE41561B-5D13-4688-8686-EEFF744BE8B5}.Release|x86.Build.0 = Release|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Debug|x64.Build.0 = Debug|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Debug|x86.Build.0 = Debug|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Release|x64.ActiveCfg = Release|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Release|x64.Build.0 = Release|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Release|x86.ActiveCfg = Release|Any CPU
+		{BAC03617-3A2B-431E-A9E4-49441B745C4F}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {4006CDC8-9E1D-438A-9749-2B3303F9074C}
+	EndGlobalSection
+EndGlobal

test-assets/test-projects/DeepProjectTransitiveReference/Lib1/Class1.cs

@@ -0,0 +1,8 @@
+using System;
+
+namespace Lib
+{
+    public class Class1
+    {
+    }
+}

test-assets/test-projects/DeepProjectTransitiveReference/Lib1/Lib1.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard1.3</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Lib2\Lib2.csproj" />
+  </ItemGroup>
+
+</Project>

test-assets/test-projects/DeepProjectTransitiveReference/Lib2/Class2.cs

@@ -0,0 +1,8 @@
+using System;
+
+namespace Lib
+{
+    public class Class2
+    {
+    }
+}

test-assets/test-projects/DeepProjectTransitiveReference/Lib2/Lib2.csproj

@@ -0,0 +1,7 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard1.3</TargetFramework>
+  </PropertyGroup>
+
+</Project>

tests/OmniSharp.MSBuild.Tests/AbstractMSBuildTestFixture.cs

@@ -30,10 +30,12 @@ public void Dispose()
             (_msbuildLocator as IDisposable)?.Dispose();
         }
 
-        protected OmniSharpTestHost CreateMSBuildTestHost(string path, IEnumerable<ExportDescriptorProvider> additionalExports = null)
+        protected OmniSharpTestHost CreateMSBuildTestHost(string path, IEnumerable<ExportDescriptorProvider> additionalExports = null,
+            IEnumerable<KeyValuePair<string, string>> configurationData = null)
         {
             var environment = new OmniSharpEnvironment(path, logLevel: LogLevel.Trace);
-            var serviceProvider = TestServiceProvider.Create(this.TestOutput, environment, this.LoggerFactory, _assemblyLoader, _msbuildLocator);
+            var serviceProvider = TestServiceProvider.Create(this.TestOutput, environment, this.LoggerFactory, _assemblyLoader, _msbuildLocator,
+                configurationData);
 
             return OmniSharpTestHost.Create(serviceProvider, additionalExports);
         }