diff --git a/docs/support.md b/docs/support.md index 76a164e2a..499697cea 100644 --- a/docs/support.md +++ b/docs/support.md @@ -17,7 +17,7 @@ of pytype. * [Third-Party Libraries](#third-party-libraries) - + @@ -79,10 +79,10 @@ Feature [PEP 613 -- Explicit Type Aliases][613] | 3.10 | ✅ | [PEP 646 -- Variadic Generics][646] | 3.11 | ❌ | [#1525][variadic-generics] [PEP 647 -- User-Defined Type Guards][647] | 3.10 | ✅ | -[PEP 655 -- Marking individual TypedDict items as required or potentially-missing][655] | 3.11 | ❌ | +[PEP 655 -- Marking individual TypedDict items as required or potentially-missing][655] | 3.11 | ❌ | [#1551][typed-dict-requirements] [PEP 673 -- Self Type][673] | 3.11 | ✅ | -[PEP 675 -- Arbitrary Literal String Type][675] | 3.11 | ❌ | -[PEP 681 -- Data Class Transforms][681] | 3.11 | 🟡 | +[PEP 675 -- Arbitrary Literal String Type][675] | 3.11 | ❌ | [#1552][literal-string] +[PEP 681 -- Data Class Transforms][681] | 3.11 | 🟡 | [#1553][dataclass-transform] [PEP 695 -- Type Parameter Syntax][695] | 3.12 | ❌ | [PEP 698 -- Override Decorator for Static Typing][698] | 3.12 | ✅ | Custom Recursive Types | 3.6 | ✅ | @@ -190,10 +190,12 @@ Tensorflow | 🟡 | Minimal, Google-internal [698]: https://peps.python.org/pep-0698/ [annotated]: https://github.com/google/pytype/issues/791 [annotation-inheritance]: https://github.com/google/pytype/issues/81 +[dataclass-transform]: https://github.com/google/pytype/issues/1553 [ellipsis-issue]: https://github.com/python/typing/issues/276 [experimental-ellipsis-commit]: https://github.com/google/pytype/commit/9f3f21e7a5bcedf6584bb41fd228878498182991 [faq-noniterable-strings]: https://google.github.io/pytype/faq.html#why-doesnt-str-match-against-string-iterables [generic-aliases]: https://github.com/google/pytype/issues/793 +[literal-string]: https://github.com/google/pytype/issues/1552 [packaging]: https://github.com/google/pytype/issues/151 [param-spec]: https://github.com/google/pytype/issues/786 [py27]: https://github.com/google/pytype/issues/545 @@ -203,4 +205,5 @@ Tensorflow | 🟡 | Minimal, Google-internal [pytype-typing-faq]: https://google.github.io/pytype/typing_faq.html [self]: https://github.com/google/pytype/issues/1283 [type-guards]: https://github.com/google/pytype/issues/916 +[typed-dict-requirements]: https://github.com/google/pytype/issues/1551 [variadic-generics]: https://github.com/google/pytype/issues/1525 diff --git a/pytype/matcher.py b/pytype/matcher.py index 70a354b5f..3fac3a862 100644 --- a/pytype/matcher.py +++ b/pytype/matcher.py @@ -1180,6 +1180,12 @@ def _match_instance_against_type(self, left, other_type, subst, view): fiddle_overlay.BuildableType)): return self._match_fiddle_instance( left.cls, left, other_type, subst, view) + elif (isinstance(other_type, abstract.PyTDClass) and + fiddle_overlay.is_fiddle_buildable_pytd(other_type.pytd_cls) and + isinstance(left, fiddle_overlay.Buildable)): + # Handle a plain `fiddle.Config` imported from a pyi file. + return self._match_fiddle_instance_against_bare_type( + left.cls, left, other_type, subst, view) elif isinstance(other_type, abstract.Class): if not self._satisfies_noniterable_str(left.cls, other_type): self._noniterable_str_error = NonIterableStrError(left.cls, other_type) @@ -1294,7 +1300,21 @@ def _match_instance_parameters(self, left, instance, other_type, subst, view): return None return subst + def _match_fiddle_instance_against_bare_type( + self, left, instance, other_type, subst, view + ): + """Match a fiddle instance against an unsubscripted buildable pytd type.""" + assert isinstance(instance, fiddle_overlay.Buildable) + assert isinstance(other_type, abstract.PyTDClass) + # This is a plain fiddle.Config we have imported from a pyi file and not + # subscripted, so it is still a PyTDClass + if instance.fiddle_type_name == other_type.name: + return subst + else: + return None + def _match_fiddle_instance(self, left, instance, other_type, subst, view): + """Match a fiddle instance against an buildable type.""" if not isinstance(instance, fiddle_overlay.Buildable): return None elif instance.fiddle_type_name != other_type.fiddle_type_name: diff --git a/pytype/overlays/enum_overlay.py b/pytype/overlays/enum_overlay.py index 5325d2e11..7990d07a3 100644 --- a/pytype/overlays/enum_overlay.py +++ b/pytype/overlays/enum_overlay.py @@ -26,7 +26,7 @@ import collections import contextlib import logging -from typing import Any, Dict, Optional, Union +from typing import Optional, Union from pytype.abstract import abstract from pytype.abstract import abstract_utils @@ -65,17 +65,7 @@ def __init__(self, ctx): super().__init__(ctx, "enum", member_map, ctx.loader.import_name("enum")) -class _DelGetAttributeMixin: - - _member_map: Dict[str, Any] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if "__getattribute__" in self._member_map: - del self._member_map["__getattribute__"] - - -class EnumBuilder(_DelGetAttributeMixin, abstract.PyTDClass): +class EnumBuilder(abstract.PyTDClass): """Overlays enum.Enum.""" def __init__(self, name, ctx, module): @@ -288,7 +278,7 @@ def call(self, node, func, args, alias_map=None): return node, self.ctx.convert.build_bool(node, this.cls == other.cls) -class EnumMeta(_DelGetAttributeMixin, abstract.PyTDClass): +class EnumMeta(abstract.PyTDClass): """Wrapper for enum.EnumMeta. EnumMeta is essentially a container for the functions that drive a lot of the diff --git a/pytype/pyi/parser.py b/pytype/pyi/parser.py index 6f4ae8792..e6bab8bc9 100644 --- a/pytype/pyi/parser.py +++ b/pytype/pyi/parser.py @@ -744,7 +744,7 @@ def post_process_ast(ast, src, name=None): if name: ast = ast.Replace(name=name) - ast = ast.Visit(visitors.AddNamePrefix()) + ast = ast.Visit(visitors.ResolveLocalNames()) else: # If there's no unique name, hash the sourcecode. ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest()) diff --git a/pytype/pytd/printer.py b/pytype/pytd/printer.py index 621adbafd..9ae082226 100644 --- a/pytype/pytd/printer.py +++ b/pytype/pytd/printer.py @@ -333,6 +333,14 @@ def _DropTypingConstant(self, node): if self.class_names or node.value: return False full_typing_name = f"typing.{node.name}" + # TODO(b/315507078): This is only necessary while these three classes + # are aliases of typing members. + if full_typing_name in ( + "typing.ChainMap", + "typing.Counter", + "typing.OrderedDict", + ): + return False if node.type == f"Type[{full_typing_name}]": self._imports.add(full_typing_name, node.name) self._imports.decrement_typing_count("Type") diff --git a/pytype/pytd/visitors.py b/pytype/pytd/visitors.py index 458b5c04e..69b38f0bd 100644 --- a/pytype/pytd/visitors.py +++ b/pytype/pytd/visitors.py @@ -1241,7 +1241,7 @@ def VisitNamedType(self, node): return node.Replace(name=new_name) -class AddNamePrefix(Visitor): +class ResolveLocalNames(Visitor): """Visitor for making names fully qualified. This will change @@ -1252,6 +1252,10 @@ def bar(x: Foo) -> Foo class baz.Foo: pass def bar(x: baz.Foo) -> baz.Foo + + References to nested classes will be full resolved, e.g. if C is nested in + B is nested in A, then `x: C` becomes `x: foo.A.B.C`. + References to attributes of Any-typed constants will be resolved to Any. """ def __init__(self): @@ -1266,6 +1270,12 @@ def _ClassStackString(self): def EnterTypeDeclUnit(self, node): self.classes = {cls.name for cls in node.classes} + # TODO(b/293451396): In certain weird cases, a local module named "typing" + # may get mixed up with the stdlib typing module. We end up doing the right + # thing in the end, but in the meantime, "typing" may get mapped to Any. + self.any_constants = {const.name for const in node.constants + if const.type == pytd.AnythingType() + and const.name != "typing"} self.name = node.name self.prefix = node.name + "." @@ -1287,10 +1297,17 @@ def VisitNamedType(self, node): # This is an external type; do not prefix it. StripExternalNamePrefix will # remove it later. return node - if node.name.split(".")[0] in self.classes: + target = node.name.split(".")[0] + if target in self.classes: # We need to check just the first part, in case we have a class constant # like Foo.BAR, or some similarly nested name. return node.Replace(name=self.prefix + node.name) + if target in self.any_constants: + # If we have a constant in module `foo` that's Any, i.e. + # mod: Any + # x: mod.Thing + # We resolve `mod.Thing` to Any. + return pytd.AnythingType() if self.cls_stack: if node.name == self.cls_stack[-1].name: # We're referencing a class from within itself. diff --git a/pytype/pytd/visitors_test.py b/pytype/pytd/visitors_test.py index c7df3fc17..28bcd4846 100644 --- a/pytype/pytd/visitors_test.py +++ b/pytype/pytd/visitors_test.py @@ -239,8 +239,12 @@ def f(x: T) -> T: ... B = A """) src2 = "from foo import *" - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertEqual("bar", ast2.name) @@ -252,7 +256,9 @@ def test_lookup_star_alias_in_unnamed_module(self): class A: ... """) src2 = "from foo import *" - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) ast2 = self.Parse(src2) name = ast2.name ast2 = ast2.Visit(visitors.LookupExternalTypes( @@ -267,9 +273,15 @@ def test_lookup_two_star_aliases(self): from foo import * from bar import * """) - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) - ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) + ast3 = ( + self.Parse(src3).Replace(name="baz").Visit(visitors.ResolveLocalNames()) + ) ast3 = ast3.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz")) self.assertSetEqual({a.name for a in ast3.aliases}, {"baz.A", "baz.B"}) @@ -281,9 +293,15 @@ def test_lookup_two_star_aliases_with_same_class(self): from foo import * from bar import * """) - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) - ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) + ast3 = ( + self.Parse(src3).Replace(name="baz").Visit(visitors.ResolveLocalNames()) + ) self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz")) @@ -294,8 +312,12 @@ def test_lookup_star_alias_with_duplicate_class(self): class A: x = ... # type: int """) - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertMultiLineEqual(pytd_utils.Print(ast2), textwrap.dedent(""" @@ -310,9 +332,15 @@ def test_lookup_two_star_aliases_with_default_pyi(self): from foo import * from bar import * """) - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) - ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) + ast3 = ( + self.Parse(src3).Replace(name="baz").Visit(visitors.ResolveLocalNames()) + ) ast3 = ast3.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz")) self.assertMultiLineEqual(pytd_utils.Print(ast3), textwrap.dedent(""" @@ -328,8 +356,12 @@ def test_lookup_star_alias_with_duplicate_getattr(self): from foo import * def __getattr__(name) -> Any: ... """) - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertMultiLineEqual(pytd_utils.Print(ast2), textwrap.dedent(""" @@ -345,9 +377,15 @@ def test_lookup_two_star_aliases_with_different_getattrs(self): from foo import * from bar import * """) - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) - ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) + ast3 = ( + self.Parse(src3).Replace(name="baz").Visit(visitors.ResolveLocalNames()) + ) self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz")) @@ -357,8 +395,12 @@ def test_lookup_star_alias_with_different_getattr(self): from foo import * def __getattr__(name) -> str: ... """) - ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) - ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) + ast1 = ( + self.Parse(src1).Replace(name="foo").Visit(visitors.ResolveLocalNames()) + ) + ast2 = ( + self.Parse(src2).Replace(name="bar").Visit(visitors.ResolveLocalNames()) + ) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertMultiLineEqual(pytd_utils.Print(ast2), textwrap.dedent(""" @@ -565,7 +607,7 @@ class X(Generic[T]): self.assertIsNone(tree.Lookup("T").scope) self.assertEqual("X", tree.Lookup("X").template[0].type_param.scope) - tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix()) + tree = tree.Replace(name="foo").Visit(visitors.ResolveLocalNames()) self.assertIsNotNone(tree.Lookup("foo.f")) self.assertIsNotNone(tree.Lookup("foo.X")) self.assertEqual("foo", tree.Lookup("foo.T").scope) @@ -580,8 +622,8 @@ def test_add_name_prefix_twice(self): class X(Generic[T]): ... """) tree = self.Parse(src) - tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix()) - tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix()) + tree = tree.Replace(name="foo").Visit(visitors.ResolveLocalNames()) + tree = tree.Replace(name="foo").Visit(visitors.ResolveLocalNames()) self.assertIsNotNone(tree.Lookup("foo.foo.x")) self.assertEqual("foo.foo", tree.Lookup("foo.foo.T").scope) self.assertEqual("foo.foo.X", @@ -596,7 +638,7 @@ class Y: ... x = tree.Lookup("x") x = x.Replace(type=pytd.ClassType("Y")) tree = tree.Replace(constants=(x,), name="foo") - tree = tree.Visit(visitors.AddNamePrefix()) + tree = tree.Visit(visitors.ResolveLocalNames()) self.assertEqual("foo.Y", tree.Lookup("foo.x").type.name) def test_add_name_prefix_on_nested_class_alias(self): @@ -614,8 +656,14 @@ class foo.A.B: class foo.A.B.C: ... D: Type[foo.A.B.C] """).strip() - self.assertMultiLineEqual(expected, pytd_utils.Print( - self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix()))) + self.assertMultiLineEqual( + expected, + pytd_utils.Print( + self.Parse(src) + .Replace(name="foo") + .Visit(visitors.ResolveLocalNames()) + ), + ) def test_add_name_prefix_on_nested_class_outside_ref(self): src = textwrap.dedent(""" @@ -643,8 +691,14 @@ def f(self, x: foo.A.B) -> foo.A.B: ... def foo.f(x: foo.A.B) -> foo.A.B: ... """).strip() - self.assertMultiLineEqual(expected, pytd_utils.Print( - self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix()))) + self.assertMultiLineEqual( + expected, + pytd_utils.Print( + self.Parse(src) + .Replace(name="foo") + .Visit(visitors.ResolveLocalNames()) + ), + ) def test_add_name_prefix_on_nested_class_method(self): src = textwrap.dedent(""" @@ -657,8 +711,14 @@ class foo.A: class foo.A.B: def copy(self) -> foo.A.B: ... """).strip() - self.assertMultiLineEqual(expected, pytd_utils.Print( - self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix()))) + self.assertMultiLineEqual( + expected, + pytd_utils.Print( + self.Parse(src) + .Replace(name="foo") + .Visit(visitors.ResolveLocalNames()) + ), + ) def test_print_merge_types(self): src = textwrap.dedent(""" diff --git a/pytype/stubs/stdlib/enum.pytd b/pytype/stubs/stdlib/enum.pytd index 1cc79ee32..2d986560e 100644 --- a/pytype/stubs/stdlib/enum.pytd +++ b/pytype/stubs/stdlib/enum.pytd @@ -1,16 +1,9 @@ -# TODO(tsudol): Improve enum support so the `__getattribute__`s can be removed: -# - Remove EnumMeta.__getattribute__ -# - Remove Enum.__getattribute__ -# - Remove auto.__getattribute__ -# The enum overlay removes __getattribute__ from EnumMeta and Enum when enabled. - from typing import Any, Dict, Iterable, Iterator, Tuple, Type, TypeVar, Union _T = TypeVar('_T') _EnumType = TypeVar('_EnumType', bound=Type[Enum]) class EnumMeta(type, Iterable): - def __getattribute__(self, name) -> Any: ... def __iter__(self: Type[_T]) -> Iterator[_T]: ... def __getitem__(cls: EnumMeta, name: str) -> Any: ... def __contains__(self, member: Enum) -> bool: ... @@ -29,7 +22,6 @@ class Enum(metaclass=EnumMeta): def __eq__(self, other: Any) -> bool: ... def __new__(cls, value: str, names: Union[str, Iterable[str], Iterable[Tuple[str, Any]], Dict[str, Any]], module = ..., type: type = ..., start: complex = ...) -> Type[Enum]: ... def __new__(cls: Type[_T], value) -> _T: ... - def __getattribute__(self, name) -> Any: ... class IntEnum(int, Enum): ... @@ -40,7 +32,6 @@ _auto_null: Any class auto: value: Any def __init__(self) -> None: ... - def __getattribute__(self, name) -> Any: ... def __or__(self: _T, other: _T) -> _T: ... def __and__(self: _T, other: _T) -> _T: ... def __xor__(self: _T, other: _T) -> _T: ... diff --git a/pytype/tests/test_fiddle_overlay.py b/pytype/tests/test_fiddle_overlay.py index 58a44630a..af20b9fd3 100644 --- a/pytype/tests/test_fiddle_overlay.py +++ b/pytype/tests/test_fiddle_overlay.py @@ -334,6 +334,30 @@ def f() -> fiddle.{self.buildable_type_name}: return a """) + def test_bare_type_in_pyi(self): + """Check that we can match fiddle.Config against fiddle.Config[A].""" + + with self.DepTree([ + ("fiddle.pyi", _FIDDLE_PYI), + ("foo.pyi", f""" + import fiddle + def f(x: fiddle.{self.buildable_type_name}): ... + """), + ]): + self.Check(f""" + import dataclasses + import fiddle + import foo + + @dataclasses.dataclass + class Simple: + x: int + y: str + + c = fiddle.{self.buildable_type_name}(Simple, 1, "2") + foo.f(c) + """) + def test_generic_dataclass(self): with self.DepTree([("fiddle.pyi", _FIDDLE_PYI)]): self.CheckWithErrors(f""" diff --git a/pytype/tests/test_import2.py b/pytype/tests/test_import2.py index 6a8f8a672..b1564bd07 100644 --- a/pytype/tests/test_import2.py +++ b/pytype/tests/test_import2.py @@ -98,6 +98,22 @@ def load() -> Component: return Component(foos=foos) """, module_name="loaders") + def test_import_any(self): + with self.DepTree([("foo.pyi", """ + from typing import Any + dep: Any + x: dep.Thing + class A(dep.Base): + def get(self) -> dep.Got: ... + """)]): + self.Check(""" + from typing import Any + import foo + assert_type(foo.dep, Any) + assert_type(foo.x, Any) + assert_type(foo.A(), foo.A) + assert_type(foo.A().get(), Any) + """) if __name__ == "__main__": test_base.main()