On demand projects loading when working with big C# codebases

src/OmniSharp.Abstractions/Services/IProjectSystem.cs

@@ -29,5 +29,11 @@ public interface IProjectSystem
         /// <param name="filePath">The file path to the project to retrieve. Alternatively,
         /// a file path to a document within a proejct may be specified.</param>
         Task<object> GetProjectModelAsync(string filePath);
+
+        /// <summary>
+        /// This project system will try to locate projects that reference given file and will wait until the projects and their references are fully loaded.
+        /// </summary>
+        /// <param name="filePath">File for which to locate and load projects.</param>
+        Task WaitForProjectsToLoadForFileAsync(string filePath);
     }
 }

src/OmniSharp.Cake/CakeProjectSystem.cs

@@ -258,6 +258,12 @@ public Task<object> GetProjectModelAsync(string filePath)
             return Task.FromResult<object>(new CakeContextModel(filePath));
         }
 
+        public Task WaitForProjectsToLoadForFileAsync(string filePath)
+        {
+            // At the moment this project system doesn't support on demand projects loading
+            return Task.CompletedTask;
+        }
+
         private ProjectInfo GetProjectFileInfo(string path)
         {
             return !_projects.TryGetValue(path, out ProjectInfo projectFileInfo) ? null : projectFileInfo;

src/OmniSharp.DotNet/DotNetProjectSystem.cs

@@ -152,6 +152,12 @@ public void Update(bool allowRestore)
             }
         }
 
+        public Task WaitForProjectsToLoadForFileAsync(string filePath)
+        {
+            // At the moment this project system doesn't support on demand projects loading
+            return Task.CompletedTask;
+        }
+
         private void UpdateProject(string projectDirectory)
         {
             _logger.LogInformation($"Update project {projectDirectory}");

src/OmniSharp.MSBuild/Options/MSBuildOptions.cs

@@ -8,6 +8,12 @@ internal class MSBuildOptions
         public string Platform { get; set; }
         public bool EnablePackageAutoRestore { get; set; }
 
+        /// <summary> 
+        /// If true, MSBuild project system will only be loading projects for files that were opened in the editor
+        /// as well as referenced projects, recursively.
+        /// </summary>
+        public bool OnDemandProjectsLoad { get; set; }
+
         /// <summary>
         /// When set to true, the MSBuild project system will attempt to resolve the path to the MSBuild
         /// SDKs for a project by running 'dotnet --info' and retrieving the path. This is only needed

src/OmniSharp.MSBuild/ProjectManager.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
@@ -17,6 +18,7 @@
 using OmniSharp.MSBuild.Models.Events;
 using OmniSharp.MSBuild.Notification;
 using OmniSharp.MSBuild.ProjectFile;
+using OmniSharp.Options;
 using OmniSharp.Roslyn.Utilities;
 using OmniSharp.Services;
 using OmniSharp.Utilities;
@@ -39,12 +41,14 @@ public ProjectToUpdate(string filePath, bool allowAutoRestore)
         }
 
         private readonly ILogger _logger;
+        private readonly MSBuildOptions _options;
         private readonly IEventEmitter _eventEmitter;
         private readonly IFileSystemWatcher _fileSystemWatcher;
         private readonly MetadataFileReferenceCache _metadataFileReferenceCache;
         private readonly PackageDependencyChecker _packageDependencyChecker;
         private readonly ProjectFileInfoCollection _projectFiles;
         private readonly HashSet<string> _failedToLoadProjectFiles;
+        private readonly ConcurrentDictionary<string, int/*unused*/> _projectsRequestedOnDemand;
         private readonly ProjectLoader _projectLoader;
         private readonly OmniSharpWorkspace _workspace;
         private readonly ImmutableArray<IMSBuildEventSink> _eventSinks;
@@ -57,15 +61,25 @@ public ProjectToUpdate(string filePath, bool allowAutoRestore)
 
         private readonly FileSystemNotificationCallback _onDirectoryFileChanged;
 
-        public ProjectManager(ILoggerFactory loggerFactory, IEventEmitter eventEmitter, IFileSystemWatcher fileSystemWatcher, MetadataFileReferenceCache metadataFileReferenceCache, PackageDependencyChecker packageDependencyChecker, ProjectLoader projectLoader, OmniSharpWorkspace workspace, ImmutableArray<IMSBuildEventSink> eventSinks)
+        public ProjectManager(ILoggerFactory loggerFactory, 
+            MSBuildOptions options, 
+            IEventEmitter eventEmitter, 
+            IFileSystemWatcher fileSystemWatcher, 
+            MetadataFileReferenceCache metadataFileReferenceCache, 
+            PackageDependencyChecker packageDependencyChecker, 
+            ProjectLoader projectLoader, 
+            OmniSharpWorkspace workspace, 
+            ImmutableArray<IMSBuildEventSink> eventSinks)
         {
             _logger = loggerFactory.CreateLogger<ProjectManager>();
+            _options = options ?? new MSBuildOptions();
             _eventEmitter = eventEmitter;
             _fileSystemWatcher = fileSystemWatcher;
             _metadataFileReferenceCache = metadataFileReferenceCache;
             _packageDependencyChecker = packageDependencyChecker;
             _projectFiles = new ProjectFileInfoCollection();
             _failedToLoadProjectFiles = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            _projectsRequestedOnDemand = new ConcurrentDictionary<string, int>(StringComparer.OrdinalIgnoreCase);
             _projectLoader = projectLoader;
             _workspace = workspace;
             _eventSinks = eventSinks;
@@ -75,6 +89,36 @@ public ProjectManager(ILoggerFactory loggerFactory, IEventEmitter eventEmitter,
             _processLoopTask = Task.Run(() => ProcessLoopAsync(_processLoopCancellation.Token));
 
             _onDirectoryFileChanged = OnDirectoryFileChanged;
+
+            if (_options.OnDemandProjectsLoad)
+            {
+                _workspace.DocumentRequested += OnWorkspaceDocumentRequested;
+            }
+        }
+
+        private void OnWorkspaceDocumentRequested(object sender, DocumentRequestedEventArgs args)
+        {
+            // Search and queue for loading C# projects that are likely to reference the requested file.
+            // C# source files are located pretty much always in the same folder with project file that is referencing them or in a subfolder.
+            // Do not limit search by OmniSharpEnvironment.TargetDirectory to allow for loading on demand projects that are outside of it,
+            // i.e. in VSCode case potentially outside of the folder opened in the IDE.
+            string projectDir = Path.GetDirectoryName(args.DocumentPath);
+            while(projectDir != null)
+            {
+                List<string> csProjFiles = Directory.EnumerateFiles(projectDir, "*.csproj", SearchOption.TopDirectoryOnly).ToList();
+                if (csProjFiles.Count > 0)
+                {
+                    foreach(string csProjFile in csProjFiles)
+                    {
+                        if (_projectsRequestedOnDemand.TryAdd(csProjFile, 0 /*unused*/))
+                        {
+                            QueueProjectUpdate(csProjFile, allowAutoRestore:true);
+                        }
+                    }
+                    return;
+                }
+                projectDir = Path.GetDirectoryName(projectDir);
+            }
         }
 
         protected override void DisposeCore(bool disposing)

src/OmniSharp.MSBuild/ProjectSystem.cs

@@ -98,7 +98,14 @@ public void Initalize(IConfiguration configuration)
 
             _packageDependencyChecker = new PackageDependencyChecker(_loggerFactory, _eventEmitter, _dotNetCli, _options);
             _loader = new ProjectLoader(_options, _environment.TargetDirectory, _propertyOverrides, _loggerFactory, _sdksPathResolver);
-            _manager = new ProjectManager(_loggerFactory, _eventEmitter, _fileSystemWatcher, _metadataFileReferenceCache, _packageDependencyChecker, _loader, _workspace, _eventSinks);
+            _manager = new ProjectManager(_loggerFactory, _options, _eventEmitter, _fileSystemWatcher, _metadataFileReferenceCache, _packageDependencyChecker, 
+                _loader, _workspace, _eventSinks);
+
+            if (_options.OnDemandProjectsLoad)
+            {
+                _logger.LogInformation($"Skip loading projects listed in solution file or under target directory because {Key}:{nameof(MSBuildOptions.OnDemandProjectsLoad)} is true.");
+                return;
+            }
 
             var initialProjectPaths = GetInitialProjectPaths();
 
@@ -214,5 +221,17 @@ async Task<object> IProjectSystem.GetProjectModelAsync(string filePath)
 
             return new MSBuildProjectInfo(projectFileInfo);
         }
+
+        public async Task WaitForProjectsToLoadForFileAsync(string filePath)
+        {
+            if (_options.OnDemandProjectsLoad)
+            {
+                // Request the document only to make sure that projects referencing it are queued for loading by the project system.
+                _workspace.GetDocument(filePath);
+                // Wait for all queued projects to load to ensure that workspace is fully up to date before this method completes.
+                // If the project for the document was loaded before and there are no other projects to load at the moment, the call below will be no-op.
+                await _manager.WaitForQueueEmptyAsync();
+            }
+        }
     }
 }

src/OmniSharp.Plugins/Plugin.cs

@@ -115,5 +115,11 @@ public Task<object> GetProjectModelAsync(string filePath)
             // TODO: Call out to process
             return Task.FromResult<object>(null);
         }
+
+        public Task WaitForProjectsToLoadForFileAsync(string filePath)
+        {
+            // TODO: Call out to process
+            return Task.CompletedTask;
+        }
     }
 }

src/OmniSharp.Roslyn.CSharp/Helpers/ProjectSystemExtensions.cs

@@ -0,0 +1,30 @@
+using OmniSharp.Services;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace OmniSharp.Helpers
+{
+    public static class ProjectSystemExtensions
+    {
+        public static async Task WaitForAllProjectsToLoadForFileAsync(this IEnumerable<IProjectSystem> projectSystems, string filePath)
+        {
+            if (filePath != null)
+            {
+                await Task.WhenAll(GetProjectSystemsForFile(projectSystems, filePath).Select(ps => ps.WaitForProjectsToLoadForFileAsync(filePath)));
+            }
+        }
+
+        private static IEnumerable<IProjectSystem> GetProjectSystemsForFile(IEnumerable<IProjectSystem> projectSystems, string filePath)
+        {
+            foreach (IProjectSystem projectSystem in projectSystems)
+            {
+                if (projectSystem.Extensions.Any(extension => filePath.EndsWith(extension, StringComparison.OrdinalIgnoreCase)))
+                {
+                    yield return projectSystem;
+                }
+            }
+        }
+    }
+}

src/OmniSharp.Roslyn.CSharp/Services/Buffer/UpdateBufferService.cs

@@ -1,24 +1,32 @@
+using System.Collections.Generic;
 using System.Composition;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
+using OmniSharp.Helpers;
 using OmniSharp.Mef;
 using OmniSharp.Models.UpdateBuffer;
+using OmniSharp.Services;
 
 namespace OmniSharp.Roslyn.CSharp.Services.Buffer
 {
     [OmniSharpHandler(OmniSharpEndpoints.UpdateBuffer, LanguageNames.CSharp)]
     public class UpdateBufferService : IRequestHandler<UpdateBufferRequest, object>
     {
         private OmniSharpWorkspace _workspace;
+        private readonly IEnumerable<IProjectSystem> _projectSystems;
 
         [ImportingConstructor]
-        public UpdateBufferService(OmniSharpWorkspace workspace)
+        public UpdateBufferService(OmniSharpWorkspace workspace, [ImportMany] IEnumerable<IProjectSystem> projectSystems)
         {
             _workspace = workspace;
+            _projectSystems = projectSystems;
         }
 
         public async Task<object> Handle(UpdateBufferRequest request)
         {
+            // Waiting until the document is fully formed in memory (for project systems that have this ability) 
+            // before applying updates to it helps to reduce chances for the compiler getting confused and producing invalid compilation errors.
+            await _projectSystems.WaitForAllProjectsToLoadForFileAsync(request.FileName);
             await _workspace.BufferManager.UpdateBufferAsync(request);
             return true;
         }

src/OmniSharp.Roslyn.CSharp/Services/Diagnostics/CodeCheckService.cs

@@ -7,22 +7,29 @@
 using OmniSharp.Mef;
 using OmniSharp.Models;
 using OmniSharp.Models.CodeCheck;
+using OmniSharp.Services;
 
 namespace OmniSharp.Roslyn.CSharp.Services.Diagnostics
 {
     [OmniSharpHandler(OmniSharpEndpoints.CodeCheck, LanguageNames.CSharp)]
     public class CodeCheckService : IRequestHandler<CodeCheckRequest, QuickFixResponse>
     {
         private OmniSharpWorkspace _workspace;
+        private readonly IEnumerable<IProjectSystem> _projectSystems;
 
         [ImportingConstructor]
-        public CodeCheckService(OmniSharpWorkspace workspace)
+        public CodeCheckService(OmniSharpWorkspace workspace, [ImportMany] IEnumerable<IProjectSystem> projectSystems)
         {
             _workspace = workspace;
+            _projectSystems = projectSystems;
         }
 
         public async Task<QuickFixResponse> Handle(CodeCheckRequest request)
         {
+            // Waiting until the document is fully formed in memory (for project systems that have this ability) 
+            // helps to reduce chances of returning invalid list of errors while compilation is still in progress.
+            await _projectSystems.WaitForAllProjectsToLoadForFileAsync(request.FileName);
+
             var documents = request.FileName != null
                 ? _workspace.GetDocuments(request.FileName)
                 : _workspace.CurrentSolution.Projects.SelectMany(project => project.Documents);

src/OmniSharp.Roslyn.CSharp/Services/Navigation/FindUsagesService.cs

@@ -10,23 +10,30 @@
 using OmniSharp.Mef;
 using OmniSharp.Models;
 using OmniSharp.Models.FindUsages;
+using OmniSharp.Services;
 
 namespace OmniSharp.Roslyn.CSharp.Services.Navigation
 {
     [OmniSharpHandler(OmniSharpEndpoints.FindUsages, LanguageNames.CSharp)]
     public class FindUsagesService : IRequestHandler<FindUsagesRequest, QuickFixResponse>
     {
         private readonly OmniSharpWorkspace _workspace;
+        private readonly IEnumerable<IProjectSystem> _projectSystems;
 
         [ImportingConstructor]
-        public FindUsagesService(OmniSharpWorkspace workspace)
+        public FindUsagesService(OmniSharpWorkspace workspace, [ImportMany] IEnumerable<IProjectSystem> projectSystems)
         {
             _workspace = workspace;
+            _projectSystems = projectSystems;
         }
 
         public async Task<QuickFixResponse> Handle(FindUsagesRequest request)
         {
+            // Waiting until all projects relevant to the document are fully loaded (for project systems that have this ability) 
+            // helps to produce more complete list of usages for a symbol.
+            await _projectSystems.WaitForAllProjectsToLoadForFileAsync(request.FileName);
             var document = _workspace.GetDocument(request.FileName);
+
             var response = new QuickFixResponse();
             if (document != null)
             {

src/OmniSharp.Roslyn.CSharp/Services/Refactoring/V2/GetCodeActionsService.cs

@@ -4,6 +4,7 @@
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.Extensions.Logging;
+using OmniSharp.Helpers;
 using OmniSharp.Mef;
 using OmniSharp.Models.V2.CodeActions;
 using OmniSharp.Roslyn.CSharp.Services.CodeActions;
@@ -14,18 +15,26 @@ namespace OmniSharp.Roslyn.CSharp.Services.Refactoring.V2
     [OmniSharpHandler(OmniSharpEndpoints.V2.GetCodeActions, LanguageNames.CSharp)]
     public class GetCodeActionsService : BaseCodeActionService<GetCodeActionsRequest, GetCodeActionsResponse>
     {
+        private readonly IEnumerable<IProjectSystem> _projectSystems;
+
         [ImportingConstructor]
         public GetCodeActionsService(
             OmniSharpWorkspace workspace,
             CodeActionHelper helper,
             [ImportMany] IEnumerable<ICodeActionProvider> providers,
+            [ImportMany] IEnumerable<IProjectSystem> projectSystems,
             ILoggerFactory loggerFactory)
             : base(workspace, helper, providers, loggerFactory.CreateLogger<GetCodeActionsService>())
         {
+            _projectSystems = projectSystems;
         }
 
         public override async Task<GetCodeActionsResponse> Handle(GetCodeActionsRequest request)
         {
+            // Waiting until the document is fully formed in memory (for project systems that have this ability) 
+            // helps to reduce chances of returning invalid list of code actions while compilation is still in progress.
+            await _projectSystems.WaitForAllProjectsToLoadForFileAsync(request.FileName);
+
             var availableActions = await GetAvailableCodeActions(request);
 
             return new GetCodeActionsResponse

src/OmniSharp.Roslyn.CSharp/Services/Structure/BlockStructureService.cs

@@ -8,6 +8,7 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Text;
 using OmniSharp.Extensions;
+using OmniSharp.Helpers;
 using OmniSharp.Mef;
 using OmniSharp.Models.V2;
 using OmniSharp.Services;
@@ -29,13 +30,15 @@ public class BlockStructureService : IRequestHandler<BlockStructureRequest, Bloc
         private readonly MethodInfo _getTextSpan;
         private readonly MethodInfo _getType;
         private readonly OmniSharpWorkspace _workspace;
+        private readonly IEnumerable<IProjectSystem> _projectSystems;
 
         [ImportingConstructor]
-        public BlockStructureService(IAssemblyLoader loader, OmniSharpWorkspace workspace)
+        public BlockStructureService(IAssemblyLoader loader, OmniSharpWorkspace workspace, [ImportMany] IEnumerable<IProjectSystem> projectSystems)
         {
             _workspace = workspace;
             _loader = loader;
             _featureAssembly = _loader.LazyLoad(Configuration.RoslynFeatures);
+            _projectSystems = projectSystems;
 
             _blockStructureService = _featureAssembly.LazyGetType("Microsoft.CodeAnalysis.Structure.BlockStructureService");
             _blockStructure = _featureAssembly.LazyGetType("Microsoft.CodeAnalysis.Structure.BlockStructure");
@@ -50,7 +53,15 @@ public BlockStructureService(IAssemblyLoader loader, OmniSharpWorkspace workspac
 
         public async Task<BlockStructureResponse> Handle(BlockStructureRequest request)
         {
+            // Waiting until the document is fully formed in memory (for project systems that have this ability) 
+            // helps to reduce chances of returning invalid document block structure while compilation is still in progress.
+            await _projectSystems.WaitForAllProjectsToLoadForFileAsync(request.FileName);
             var document = _workspace.GetDocument(request.FileName);
+            if (document == null)
+            {
+                return new BlockStructureResponse();
+            }
+
             var text = await document.GetTextAsync();
 
             var service = _blockStructureService.LazyGetMethod("GetService").InvokeStatic(new[] { document });