On demand projects loading when working with big C# codebases

src/OmniSharp.MSBuild/Options/MSBuildOptions.cs

@@ -8,6 +8,17 @@ internal class MSBuildOptions
         public string Platform { get; set; }
         public bool EnablePackageAutoRestore { get; set; }
 
+        /// <summary>
+        /// If true, MSBuild project system will only be loading projects for files that were opened in the editor
+        /// as well as referenced projects, recursively.
+        /// </summary>
+        public bool OnDemandProjectsLoad { get; set; }
+
+        /// <summary>
+        /// Search for a .csproj to load on demand is stopped after a folder containing a file or folder from the list is encountered.
+        /// </summary>
+        public string[] OnDemandProjectsLoadSearchStopsAt { get; set; } = new[] { ".git" };
+
         /// <summary>
         /// When set to true, the MSBuild project system will attempt to resolve the path to the MSBuild
         /// SDKs for a project by running 'dotnet --info' and retrieving the path. This is only needed

src/OmniSharp.MSBuild/ProjectManager.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
@@ -17,6 +18,7 @@
 using OmniSharp.MSBuild.Models.Events;
 using OmniSharp.MSBuild.Notification;
 using OmniSharp.MSBuild.ProjectFile;
+using OmniSharp.Options;
 using OmniSharp.Roslyn.Utilities;
 using OmniSharp.Services;
 using OmniSharp.Utilities;
@@ -39,12 +41,14 @@ public ProjectToUpdate(string filePath, bool allowAutoRestore)
         }
 
         private readonly ILogger _logger;
+        private readonly MSBuildOptions _options;
         private readonly IEventEmitter _eventEmitter;
         private readonly IFileSystemWatcher _fileSystemWatcher;
         private readonly MetadataFileReferenceCache _metadataFileReferenceCache;
         private readonly PackageDependencyChecker _packageDependencyChecker;
         private readonly ProjectFileInfoCollection _projectFiles;
         private readonly HashSet<string> _failedToLoadProjectFiles;
+        private readonly ConcurrentDictionary<string, int/*unused*/> _projectsRequestedOnDemand;
         private readonly ProjectLoader _projectLoader;
         private readonly OmniSharpWorkspace _workspace;
         private readonly ImmutableArray<IMSBuildEventSink> _eventSinks;
@@ -57,15 +61,25 @@ public ProjectToUpdate(string filePath, bool allowAutoRestore)
 
         private readonly FileSystemNotificationCallback _onDirectoryFileChanged;
 
-        public ProjectManager(ILoggerFactory loggerFactory, IEventEmitter eventEmitter, IFileSystemWatcher fileSystemWatcher, MetadataFileReferenceCache metadataFileReferenceCache, PackageDependencyChecker packageDependencyChecker, ProjectLoader projectLoader, OmniSharpWorkspace workspace, ImmutableArray<IMSBuildEventSink> eventSinks)
+        public ProjectManager(ILoggerFactory loggerFactory, 
+            MSBuildOptions options, 
+            IEventEmitter eventEmitter, 
+            IFileSystemWatcher fileSystemWatcher, 
+            MetadataFileReferenceCache metadataFileReferenceCache, 
+            PackageDependencyChecker packageDependencyChecker, 
+            ProjectLoader projectLoader, 
+            OmniSharpWorkspace workspace, 
+            ImmutableArray<IMSBuildEventSink> eventSinks)
         {
             _logger = loggerFactory.CreateLogger<ProjectManager>();
+            _options = options ?? new MSBuildOptions();
             _eventEmitter = eventEmitter;
             _fileSystemWatcher = fileSystemWatcher;
             _metadataFileReferenceCache = metadataFileReferenceCache;
             _packageDependencyChecker = packageDependencyChecker;
             _projectFiles = new ProjectFileInfoCollection();
             _failedToLoadProjectFiles = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            _projectsRequestedOnDemand = new ConcurrentDictionary<string, int>(StringComparer.OrdinalIgnoreCase);
             _projectLoader = projectLoader;
             _workspace = workspace;
             _eventSinks = eventSinks;
@@ -75,6 +89,61 @@ public ProjectManager(ILoggerFactory loggerFactory, IEventEmitter eventEmitter,
             _processLoopTask = Task.Run(() => ProcessLoopAsync(_processLoopCancellation.Token));
 
             _onDirectoryFileChanged = OnDirectoryFileChanged;
+
+            if (_options.OnDemandProjectsLoad)
+            {
+                _workspace.AddWaitForProjectModelReadyEventHandler(WaitForProjectModelReadyAsync);
+            }
+        }
+
+        private async Task WaitForProjectModelReadyAsync(string documentPath)
+        {
+            // Search and queue for loading C# projects that are likely to reference the requested file.
+            // C# source files are located pretty much always in the same folder with project file that is referencing them or in a subfolder.
+            // Do not limit search by OmniSharpEnvironment.TargetDirectory to allow for loading on demand projects that are outside of it,
+            // i.e. in VSCode case potentially outside of the folder opened in the IDE.
+            string projectDir = Path.GetDirectoryName(documentPath);
+            while(projectDir != null)
+            {
+                List<string> csProjFiles = Directory.EnumerateFiles(projectDir, "*.csproj", SearchOption.TopDirectoryOnly).ToList();
+                if (csProjFiles.Count > 0)
+                {
+                    foreach(string csProjFile in csProjFiles)
+                    {
+                        if (_projectsRequestedOnDemand.TryAdd(csProjFile, 0 /*unused*/))
+                        {
+                            QueueProjectUpdate(csProjFile, allowAutoRestore:true);
+                        }
+                    }
+
+                    break;
+                }
+
+                bool foundStopAtItem = false;
+                foreach (string stopAtItemName in _options.OnDemandProjectsLoadSearchStopsAt)
+                {
+                    string itemPath = Path.Combine(projectDir, stopAtItemName);
+                    if (File.Exists(itemPath) || Directory.Exists(itemPath))
+                    {
+                        foundStopAtItem = true;
+                        break;
+                    }
+                }
+
+                if (foundStopAtItem)
+                {
+                    _logger.LogTrace($"Couldn't find project to load for '{documentPath}'");
+                    break;
+                }
+
+                projectDir = Path.GetDirectoryName(projectDir);
+            }
+
+            // Wait for all queued projects to load to ensure that workspace is fully up to date before this method completes.
+            // If the project for the document was loaded before and there are no other projects to load at the moment, the call below will be no-op.
+            _logger.LogTrace($"Started waiting for projects queue to be empty when requested '{documentPath}'");
+            await WaitForQueueEmptyAsync();
+            _logger.LogTrace($"Stopped waiting for projects queue to be empty when requested '{documentPath}'");
         }
 
         protected override void DisposeCore(bool disposing)

src/OmniSharp.MSBuild/ProjectSystem.cs

@@ -99,7 +99,14 @@ public void Initalize(IConfiguration configuration)
 
             _packageDependencyChecker = new PackageDependencyChecker(_loggerFactory, _eventEmitter, _dotNetCli, _options);
             _loader = new ProjectLoader(_options, _environment.TargetDirectory, _propertyOverrides, _loggerFactory, _sdksPathResolver);
-            _manager = new ProjectManager(_loggerFactory, _eventEmitter, _fileSystemWatcher, _metadataFileReferenceCache, _packageDependencyChecker, _loader, _workspace, _eventSinks);
+            _manager = new ProjectManager(_loggerFactory, _options, _eventEmitter, _fileSystemWatcher, _metadataFileReferenceCache, _packageDependencyChecker, 
+                _loader, _workspace, _eventSinks);
+
+            if (_options.OnDemandProjectsLoad)
+            {
+                _logger.LogInformation($"Skip loading projects listed in solution file or under target directory because {Key}:{nameof(MSBuildOptions.OnDemandProjectsLoad)} is true.");
+                return;
+            }
 
             var initialProjectPaths = GetInitialProjectPaths();
 

src/OmniSharp.Roslyn.CSharp/Services/Diagnostics/CodeCheckService.cs

@@ -24,7 +24,8 @@ public CodeCheckService(OmniSharpWorkspace workspace)
         public async Task<QuickFixResponse> Handle(CodeCheckRequest request)
         {
             var documents = request.FileName != null
-                ? _workspace.GetDocuments(request.FileName)
+                // To properly handle the request wait until all projects are loaded.
+                ? await _workspace.GetDocumentsFromFullProjectModelAsync(request.FileName)
                 : _workspace.CurrentSolution.Projects.SelectMany(project => project.Documents);
 
             var quickFixes = await documents.FindDiagnosticLocationsAsync(_workspace);

src/OmniSharp.Roslyn.CSharp/Services/Navigation/FindUsagesService.cs

@@ -26,7 +26,8 @@ public FindUsagesService(OmniSharpWorkspace workspace)
 
         public async Task<QuickFixResponse> Handle(FindUsagesRequest request)
         {
-            var document = _workspace.GetDocument(request.FileName);
+            // To produce complete list of usages for symbols in the document wait until all projects are loaded.
+            var document = await _workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
             var response = new QuickFixResponse();
             if (document != null)
             {

src/OmniSharp.Roslyn.CSharp/Services/Refactoring/V2/BaseCodeActionService.cs

@@ -62,7 +62,8 @@ protected BaseCodeActionService(OmniSharpWorkspace workspace, CodeActionHelper h
 
         protected async Task<IEnumerable<AvailableCodeAction>> GetAvailableCodeActions(ICodeActionRequest request)
         {
-            var document = this.Workspace.GetDocument(request.FileName);
+            // To produce a complete list of code actions for the document wait until all projects are loaded.
+            var document = await this.Workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
             if (document == null)
             {
                 return Array.Empty<AvailableCodeAction>();

src/OmniSharp.Roslyn.CSharp/Services/Structure/BlockStructureService.cs

@@ -50,7 +50,13 @@ public BlockStructureService(IAssemblyLoader loader, OmniSharpWorkspace workspac
 
         public async Task<BlockStructureResponse> Handle(BlockStructureRequest request)
         {
-            var document = _workspace.GetDocument(request.FileName);
+            // To provide complete code structure for the document wait until all projects are loaded.
+            var document = await _workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
+            if (document == null)
+            {
+                return null;
+            }
+
             var text = await document.GetTextAsync();
 
             var service = _blockStructureService.LazyGetMethod("GetService").InvokeStatic(new[] { document });

src/OmniSharp.Roslyn.CSharp/Services/Structure/CodeStructureService.cs

@@ -32,7 +32,8 @@ public CodeStructureService(
 
         public async Task<CodeStructureResponse> Handle(CodeStructureRequest request)
         {
-            var document = _workspace.GetDocument(request.FileName);
+            // To provide complete code structure for the document wait until all projects are loaded.
+            var document = await _workspace.GetDocumentFromFullProjectModelAsync(request.FileName);
             if (document == null)
             {
                 return null;

src/OmniSharp.Roslyn.CSharp/Services/Structure/MembersAsTreeService.cs

@@ -25,7 +25,8 @@ public async Task<FileMemberTree> Handle(MembersTreeRequest request)
         {
             return new FileMemberTree()
             {
-                TopLevelTypeDefinitions = await StructureComputer.Compute(_workspace.GetDocuments(request.FileName), _discovers)
+                // To provide complete members tree for the document wait until all projects are loaded.
+                TopLevelTypeDefinitions = await StructureComputer.Compute(await _workspace.GetDocumentsFromFullProjectModelAsync(request.FileName), _discovers)
             };
         }
     }

src/OmniSharp.Roslyn/OmniSharpWorkspace.cs

@@ -25,6 +25,8 @@ public class OmniSharpWorkspace : Workspace
 
         private readonly ILogger<OmniSharpWorkspace> _logger;
 
+        private readonly ConcurrentBag<Func<string, Task>> _waitForProjectModelReadyEventHandlers = new ConcurrentBag<Func<string, Task>>();
+
         private readonly ConcurrentDictionary<string, ProjectInfo> miscDocumentsProjectInfos = new ConcurrentDictionary<string, ProjectInfo>();
 
         [ImportingConstructor]
@@ -37,6 +39,11 @@ public OmniSharpWorkspace(HostServicesAggregator aggregator, ILoggerFactory logg
 
         public override bool CanOpenDocuments => true;
 
+        public void AddWaitForProjectModelReadyEventHandler(Func<string, Task> eventHandler)
+        {
+            _waitForProjectModelReadyEventHandlers.Add(eventHandler);
+        }
+
         public override void OpenDocument(DocumentId documentId, bool activate = true)
         {
             var doc = this.CurrentSolution.GetDocument(documentId);
@@ -233,6 +240,18 @@ public Document GetDocument(string filePath)
             return CurrentSolution.GetDocument(documentId);
         }
 
+        public async Task<IEnumerable<Document>> GetDocumentsFromFullProjectModelAsync(string filePath)
+        {
+            await OnWaitForProjectModelReadyAsync(filePath);
+            return GetDocuments(filePath);
+        }
+
+        public async Task<Document> GetDocumentFromFullProjectModelAsync(string filePath)
+        {
+            await OnWaitForProjectModelReadyAsync(filePath);
+            return GetDocument(filePath);
+        }
+
         public override bool CanApplyChange(ApplyChangesKind feature)
         {
             return true;
@@ -363,5 +382,10 @@ public override async Task<TextAndVersion> LoadTextAndVersionAsync(
                 return textAndVersion;
             }
         }
+
+        private Task OnWaitForProjectModelReadyAsync(string filePath)
+        {
+            return Task.WhenAll(_waitForProjectModelReadyEventHandlers.Select(h => h(filePath)));
+        }  
     }
 }

tests/OmniSharp.MSBuild.Tests/AbstractMSBuildTestFixture.cs

@@ -30,10 +30,12 @@ public void Dispose()
             (_msbuildLocator as IDisposable)?.Dispose();
         }
 
-        protected OmniSharpTestHost CreateMSBuildTestHost(string path, IEnumerable<ExportDescriptorProvider> additionalExports = null)
+        protected OmniSharpTestHost CreateMSBuildTestHost(string path, IEnumerable<ExportDescriptorProvider> additionalExports = null,
+            IEnumerable<KeyValuePair<string, string>> configurationData = null)
         {
             var environment = new OmniSharpEnvironment(path, logLevel: LogLevel.Trace);
-            var serviceProvider = TestServiceProvider.Create(this.TestOutput, environment, this.LoggerFactory, _assemblyLoader, _msbuildLocator);
+            var serviceProvider = TestServiceProvider.Create(this.TestOutput, environment, this.LoggerFactory, _assemblyLoader, _msbuildLocator,
+                configurationData);
 
             return OmniSharpTestHost.Create(serviceProvider, additionalExports);
         }