Fix for extra paths inside workspace (microsoft#467)

Fix microsoft#281: Support "go to definition" for namespace packages
Fix microsoft#466: Fix "go to definition" and resolving imports

The fix is to put user search paths in front of workspace directory so that modules inside extra paths can be used as roots for packages

src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs

@@ -359,11 +359,11 @@ private void CreateRootsWithDefault(string rootDirectory, string[] userSearchPat
                 .ToArray();
 
             var filteredInterpreterSearchPaths = interpreterSearchPaths.Select(FixPath)
-                .Except(filteredUserSearchPaths.Prepend(rootDirectory))
+                .Except(filteredUserSearchPaths.Append(rootDirectory))
                 .ToArray();
 
             userRootsCount = filteredUserSearchPaths.Length + 1;
-            nodes = AddRootsFromSearchPaths(ImmutableArray<Node>.Empty.Add(GetOrCreateRoot(rootDirectory)), filteredUserSearchPaths, filteredInterpreterSearchPaths);
+            nodes = AddRootsFromSearchPaths(rootDirectory, filteredUserSearchPaths, filteredInterpreterSearchPaths);
 
             string FixPath(string p) => Path.IsPathRooted(p) ? PathUtils.NormalizePath(p) : PathUtils.NormalizePath(Path.Combine(rootDirectory, p));
         }
@@ -381,11 +381,18 @@ private void CreateRootsWithoutDefault(string[] userSearchPaths, string[] interp
                 .ToArray();
 
             userRootsCount = filteredUserSearchPaths.Length;
-            nodes = AddRootsFromSearchPaths(ImmutableArray<Node>.Empty, filteredUserSearchPaths, filteredInterpreterSearchPaths);
+            nodes = AddRootsFromSearchPaths(filteredUserSearchPaths, filteredInterpreterSearchPaths);
         }
 
-        private ImmutableArray<Node> AddRootsFromSearchPaths(ImmutableArray<Node> roots, string[] userSearchPaths, string[] interpreterSearchPaths) {
-            return roots
+        private ImmutableArray<Node> AddRootsFromSearchPaths(string rootDirectory, string[] userSearchPaths, string[] interpreterSearchPaths) {
+            return ImmutableArray<Node>.Empty
+                .AddRange(userSearchPaths.Select(GetOrCreateRoot).ToArray())
+                .Add(GetOrCreateRoot(rootDirectory))
+                .AddRange(interpreterSearchPaths.Select(GetOrCreateRoot).ToArray());
+        }
+
+        private ImmutableArray<Node> AddRootsFromSearchPaths(string[] userSearchPaths, string[] interpreterSearchPaths) {
+            return ImmutableArray<Node>.Empty
                 .AddRange(userSearchPaths.Select(GetOrCreateRoot).ToArray())
                 .AddRange(interpreterSearchPaths.Select(GetOrCreateRoot).ToArray());
         }

src/Analysis/Engine/Impl/PythonAnalyzer.cs

@@ -129,8 +129,8 @@ private async Task LoadKnownTypesAsync(CancellationToken token) {
         }
 
         private void ReloadModulePaths(in IEnumerable<string> rootPaths) {
-            foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => ModulePath.GetModulesInPath(p))) {
-                _pathResolver.TryAddModulePath(modulePath.SourceFile, out _);
+            foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) {
+                _pathResolver.TryAddModulePath(modulePath, out _);
             }
         }
 

src/Analysis/Engine/Test/ImportTests.cs

@@ -98,6 +98,67 @@ import package.sub_package.module2
             completionModule2.Should().HaveLabels("Y").And.NotContainLabels("X");
         }
 
+        [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)]
+        public async Task Completions_ImportResolution_UserSearchPathsInsideWorkspace(Server server) {
+            var folder1 = TestData.GetTestSpecificPath("folder1");
+            var folder2 = TestData.GetTestSpecificPath("folder2");
+            var packageInFolder1 = Path.Combine(folder1, "package");
+            var packageInFolder2 = Path.Combine(folder2, "package");
+            var module1Path = Path.Combine(packageInFolder1, "module1.py");
+            var module2Path = Path.Combine(packageInFolder2, "module2.py");
+            var module1Content = @"class A():
+    @staticmethod
+    def method1():
+        pass";
+            var module2Content = @"class B():
+    @staticmethod
+    def method2():
+        pass";
+            var mainContent = @"from package import module1 as mod1, module2 as mod2
+mod1.
+mod2.
+mod1.A.
+mod2.B.";
+
+            server.Analyzer.SetSearchPaths(new[] { folder1, folder2 });
+
+            await server.OpenDocumentAndGetUriAsync(module1Path, module1Content);
+            await server.OpenDocumentAndGetUriAsync(module2Path, module2Content);
+            var uri = await server.OpenDocumentAndGetUriAsync("main.py", mainContent);
+
+            await server.WaitForCompleteAnalysisAsync(CancellationToken.None);
+
+            var completionMod1 = await server.SendCompletion(uri, 1, 5);
+            var completionMod2 = await server.SendCompletion(uri, 2, 5);
+            var completionA = await server.SendCompletion(uri, 3, 7);
+            var completionB = await server.SendCompletion(uri, 4, 7);
+            completionMod1.Should().HaveLabels("A").And.NotContainLabels("B");
+            completionMod2.Should().HaveLabels("B").And.NotContainLabels("A");
+            completionA.Should().HaveLabels("method1");
+            completionB.Should().HaveLabels("method2");
+        }
+
+        [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)]
+        [Ignore("https://github.com/Microsoft/python-language-server/issues/468")]
+        public async Task Completions_ImportResolution_ModuleInWorkspaceAndInUserSearchPath(Server server) {
+            var extraSearchPath = TestData.GetTestSpecificPath(Path.Combine("some", "other"));
+            var module1Path = TestData.GetTestSpecificPath("module.py");
+            var module2Path = Path.Combine(extraSearchPath, "module.py");
+            var module1Content = "A = 1";
+            var module2Content = "B = 2";
+            var mainContent = @"import module as mod; mod.";
+
+            server.Analyzer.SetSearchPaths(new[] { extraSearchPath });
+
+            await server.OpenDocumentAndGetUriAsync(module1Path, module1Content);
+            await server.OpenDocumentAndGetUriAsync(module2Path, module2Content);
+            var uri = await server.OpenDocumentAndGetUriAsync("main.py", mainContent);
+
+            await server.WaitForCompleteAnalysisAsync(CancellationToken.None);
+            var completion = await server.SendCompletion(uri, 0, 26);
+            completion.Should().HaveLabels("A").And.NotContainLabels("B");
+        }
+
         [Ignore("https://github.com/Microsoft/python-language-server/issues/443")]
         [ServerTestMethod(LatestAvailable3X = true, TestSpecificRootUri = true), Priority(0)]
         public async Task Completions_ImportResolution_OneSearchPathInsideAnother(Server server) {

src/Analysis/Engine/Test/ServerExtensions.cs

@@ -164,6 +164,14 @@ await server.DidOpenTextDocument(new DidOpenTextDocumentParams {
             }, GetCancellationToken());
         }
 
+        public static async Task<IModuleAnalysis> OpenDocumentAndGetAnalysisAsync(this Server server, string relativePath, string content, int failAfter = 30000, string languageId = null) {
+            var cancellationToken = GetCancellationToken(failAfter);
+            var uri = TestData.GetTestSpecificUri(relativePath);
+            await server.SendDidOpenTextDocument(uri, content, languageId);
+            cancellationToken.ThrowIfCancellationRequested();
+            return await server.GetAnalysisAsync(uri, cancellationToken);
+        }
+
         public static async Task<IModuleAnalysis> OpenDefaultDocumentAndGetAnalysisAsync(this Server server, string content, int failAfter = 30000, string languageId = null) {
             var cancellationToken = GetCancellationToken(failAfter);
             await server.SendDidOpenTextDocument(TestData.GetDefaultModuleUri(), content, languageId);