From 2f527c1bb8df9419a49af270ac5b1d167804cab7 Mon Sep 17 00:00:00 2001 From: Zsolt Dollenstein Date: Thu, 11 Mar 2021 10:53:20 +0000 Subject: [PATCH] Add FullyQualifiedNameProvider --- docs/source/metadata.rst | 8 ++ libcst/codemod/_codemod.py | 2 +- libcst/metadata/__init__.py | 6 +- libcst/metadata/full_repo_manager.py | 5 +- libcst/metadata/name_provider.py | 105 ++++++++++++++++- libcst/metadata/scope_provider.py | 27 +++-- libcst/metadata/tests/test_name_provider.py | 121 +++++++++++++++++++- 7 files changed, 259 insertions(+), 15 deletions(-) diff --git a/docs/source/metadata.rst b/docs/source/metadata.rst index f6c9c0784..a4abcf70d 100644 --- a/docs/source/metadata.rst +++ b/docs/source/metadata.rst @@ -199,10 +199,18 @@ We don't call it `fully qualified name `_ when timeout. """ diff --git a/libcst/metadata/name_provider.py b/libcst/metadata/name_provider.py index 5072399b6..516db2b40 100644 --- a/libcst/metadata/name_provider.py +++ b/libcst/metadata/name_provider.py @@ -3,12 +3,19 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from typing import Collection, Optional, Union +import dataclasses +import re +from pathlib import Path +from typing import Collection, List, Mapping, Optional, Pattern, Union import libcst as cst from libcst._metadata_dependent import MetadataDependent from libcst.metadata.base_provider import BatchableMetadataProvider -from libcst.metadata.scope_provider import QualifiedName, ScopeProvider +from libcst.metadata.scope_provider import ( + QualifiedName, + QualifiedNameSource, + ScopeProvider, +) class QualifiedNameProvider(BatchableMetadataProvider[Collection[QualifiedName]]): @@ -76,3 +83,97 @@ def on_visit(self, node: cst.CSTNode) -> bool: self.provider.set_metadata(node, set()) super().on_visit(node) return True + + +DOT_PY: Pattern[str] = re.compile(r"(__init__)?\.py$") + + +def _module_name(path: str) -> Optional[str]: + return DOT_PY.sub("", path).replace("/", ".").rstrip(".") + + +class FullyQualifiedNameProvider(BatchableMetadataProvider[Collection[QualifiedName]]): + """ + Provide fully qualified names for CST nodes. Like :class:`QualifiedNameProvider`, + but the provided :class:`QualifiedName`s have absolute identifier names instead of + local to the current module. + + This provider is initialized with the current module's fully qualified name, and can + be used with :class:`~libcst.metadata.FullRepoManager`. The module's fully qualified + name itself is stored as a metadata of the :class:`~libcst.Module` node. Compared to + :class:`QualifiedNameProvider`, it also resolves relative imports. + + Example usage:: + + >>> mgr = FullRepoManager(".", {"dir/a.py"}, {FullyQualifiedNameProvider}) + >>> wrapper = mgr.get_metadata_wrapper_for_path("dir/a.py") + >>> fqnames = wrapper.resolve(FullyQualifiedNameProvider) + >>> {type(k): v for (k, v) in fqnames.items()} + {: {QualifiedName(name='dir.a', source=)}} + + """ + + METADATA_DEPENDENCIES = (QualifiedNameProvider,) + + @classmethod + def gen_cache( + cls, root_path: Path, paths: List[str], timeout: Optional[int] = None + ) -> Mapping[str, object]: + cache = {path: _module_name(path) for path in paths} + return cache + + def __init__(self, cache: str) -> None: + super().__init__(cache) + self.module_name: str = cache + + def visit_Module(self, node: cst.Module) -> bool: + visitor = FullyQualifiedNameVisitor(self, self.module_name) + node.visit(visitor) + self.set_metadata( + node, + {QualifiedName(name=self.module_name, source=QualifiedNameSource.LOCAL)}, + ) + return True + + +class FullyQualifiedNameVisitor(cst.CSTVisitor): + @staticmethod + def _fully_qualify_local(module_name: str, qname: QualifiedName) -> str: + name = qname.name + if not name.startswith("."): + # not a relative import + return f"{module_name}.{name}" + + # relative import + name = name.lstrip(".") + parts_to_strip = len(qname.name) - len(name) + target_module = ".".join(module_name.split(".")[: -1 * parts_to_strip]) + return f"{target_module}.{name}" + + @staticmethod + def _fully_qualify(module_name: str, qname: QualifiedName) -> QualifiedName: + if qname.source == QualifiedNameSource.BUILTIN: + # builtins are already fully qualified + return qname + name = qname.name + if qname.source == QualifiedNameSource.IMPORT and not name.startswith("."): + # non-relative imports are already fully qualified + return qname + new_name = FullyQualifiedNameVisitor._fully_qualify_local(module_name, qname) + return dataclasses.replace(qname, name=new_name) + + def __init__(self, provider: FullyQualifiedNameProvider, module_name: str) -> None: + self.module_name = module_name + self.provider = provider + + def on_visit(self, node: cst.CSTNode) -> bool: + qnames = self.provider.get_metadata(QualifiedNameProvider, node) + if qnames is not None: + self.provider.set_metadata( + node, + { + FullyQualifiedNameVisitor._fully_qualify(self.module_name, qname) + for qname in qnames + }, + ) + return True diff --git a/libcst/metadata/scope_provider.py b/libcst/metadata/scope_provider.py index 77ceafd17..ce74126f0 100644 --- a/libcst/metadata/scope_provider.py +++ b/libcst/metadata/scope_provider.py @@ -286,16 +286,25 @@ def get_name_for(node: Union[str, cst.CSTNode]) -> Optional[str]: return None @staticmethod - def find_qualified_name_for_import_alike( - assignment_node: Union[cst.Import, cst.ImportFrom], full_name: str - ) -> Set[QualifiedName]: + def get_module_name_for_import_alike( + assignment_node: Union[cst.Import, cst.ImportFrom] + ) -> str: module = "" - results = set() if isinstance(assignment_node, cst.ImportFrom): module_attr = assignment_node.module + relative = assignment_node.relative if module_attr: - # TODO: for relative import, keep the relative Dot in the qualified name - module = get_full_name_for_node(module_attr) + module = get_full_name_for_node(module_attr) or "" + if relative: + module = "." * len(relative) + module + return module + + @staticmethod + def find_qualified_name_for_import_alike( + assignment_node: Union[cst.Import, cst.ImportFrom], full_name: str + ) -> Set[QualifiedName]: + module = _NameUtil.get_module_name_for_import_alike(assignment_node) + results = set() import_names = assignment_node.names if not isinstance(import_names, cst.ImportStar): for name in import_names: @@ -308,7 +317,11 @@ def find_qualified_name_for_import_alike( real_names = [".".join(parts[:i]) for i in range(len(parts), 0, -1)] for real_name in real_names: as_name = real_name - if module: + if module and module.endswith("."): + # from . import a + # real_name should be ".a" + real_name = f"{module}{real_name}" + elif module: real_name = f"{module}.{real_name}" if name and name.asname: eval_alias = name.evaluated_alias diff --git a/libcst/metadata/tests/test_name_provider.py b/libcst/metadata/tests/test_name_provider.py index f995b65a9..107a74e70 100644 --- a/libcst/metadata/tests/test_name_provider.py +++ b/libcst/metadata/tests/test_name_provider.py @@ -3,17 +3,22 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +from pathlib import Path +from tempfile import TemporaryDirectory from textwrap import dedent from typing import Collection, Mapping, Optional, Set, Tuple import libcst as cst from libcst import ensure_type from libcst.metadata import ( + FullyQualifiedNameProvider, MetadataWrapper, QualifiedName, QualifiedNameProvider, QualifiedNameSource, ) +from libcst.metadata.full_repo_manager import FullRepoManager +from libcst.metadata.name_provider import FullyQualifiedNameVisitor from libcst.testing.utils import UnitTest @@ -25,8 +30,29 @@ def get_qualified_name_metadata_provider( def get_qualified_names(module_str: str) -> Set[QualifiedName]: - _, qnames = get_qualified_name_metadata_provider(module_str) - return set().union(*qnames.values()) + _, qnames_map = get_qualified_name_metadata_provider(module_str) + return {qname for qnames in qnames_map.values() for qname in qnames} + + +def get_fully_qualified_names(file_path: str, module_str: str) -> Set[QualifiedName]: + wrapper = cst.MetadataWrapper( + cst.parse_module(dedent(module_str)), + # pyre-fixme[6]: Incompatible parameter type [6]: Expected + # `typing.Mapping[typing.Type[cst.metadata.base_provider.BaseMetadataProvider[ + # object]], object]` for 2nd parameter `cache` to call + # `cst.metadata.wrapper.MetadataWrapper.__init__` but got + # `typing.Dict[typing.Type[FullyQualifiedNameProvider], object]` + cache={ + FullyQualifiedNameProvider: FullyQualifiedNameProvider.gen_cache( + Path(""), [file_path], None + ).get(file_path, "") + }, + ) + return { + qname + for qnames in wrapper.resolve(FullyQualifiedNameProvider).values() + for qname in qnames + } class QualifiedNameProviderTest(UnitTest): @@ -325,3 +351,94 @@ class Foo: self.assertEqual( names[attribute], {QualifiedName("a.aa.aaa", QualifiedNameSource.IMPORT)} ) + + +class FullyQualifiedNameProviderTest(UnitTest): + def test_builtins(self) -> None: + qnames = get_fully_qualified_names( + "test/module.py", + """ + int(None) + """, + ) + module_name = QualifiedName( + name="test.module", source=QualifiedNameSource.LOCAL + ) + self.assertIn(module_name, qnames) + qnames -= {module_name} + self.assertEqual( + {"builtins.int", "builtins.None"}, + {qname.name for qname in qnames}, + ) + for qname in qnames: + self.assertEqual(qname.source, QualifiedNameSource.BUILTIN, msg=f"{qname}") + + def test_imports(self) -> None: + qnames = get_fully_qualified_names( + "some/test/module.py", + """ + from a.b import c as d + from . import rel + from .lol import rel2 + from .. import thing as rel3 + d, rel, rel2, rel3 + """, + ) + module_name = QualifiedName( + name="some.test.module", source=QualifiedNameSource.LOCAL + ) + self.assertIn(module_name, qnames) + qnames -= {module_name} + self.assertEqual( + {"a.b.c", "some.test.rel", "some.test.lol.rel2", "some.thing"}, + {qname.name for qname in qnames}, + ) + for qname in qnames: + self.assertEqual(qname.source, QualifiedNameSource.IMPORT, msg=f"{qname}") + + def test_locals(self) -> None: + qnames = get_fully_qualified_names( + "some/test/module.py", + """ + class X: + a: X + """, + ) + self.assertEqual( + {"some.test.module", "some.test.module.X", "some.test.module.X.a"}, + {qname.name for qname in qnames}, + ) + for qname in qnames: + self.assertEqual(qname.source, QualifiedNameSource.LOCAL, msg=f"{qname}") + + def test_local_qualification(self) -> None: + base_module = "some.test.module" + for (name, expected) in [ + (".foo", "some.test.foo"), + ("..bar", "some.bar"), + ("foo", "some.test.module.foo"), + ]: + with self.subTest(name=name): + self.assertEqual( + FullyQualifiedNameVisitor._fully_qualify_local( + base_module, + QualifiedName(name=name, source=QualifiedNameSource.LOCAL), + ), + expected, + ) + + +class FullyQualifiedNameIntegrationTest(UnitTest): + def test_with_full_repo_manager(self) -> None: + with TemporaryDirectory() as dir: + fname = "pkg/mod.py" + (Path(dir) / "pkg").mkdir() + (Path(dir) / fname).touch() + mgr = FullRepoManager(dir, [fname], [FullyQualifiedNameProvider]) + wrapper = mgr.get_metadata_wrapper_for_path(fname) + fqnames = wrapper.resolve(FullyQualifiedNameProvider) + (mod, names) = next(iter(fqnames.items())) + self.assertIsInstance(mod, cst.Module) + self.assertEqual( + names, {QualifiedName(name="pkg.mod", source=QualifiedNameSource.LOCAL)} + )