Merge pull request #747 from google/google_sync

Google sync

docs/developers/python_version_upgrades.md

@@ -75,15 +75,15 @@ python build_scripts/run_tests.py
 ```
 
 The new version should also be added to the
-[Travis test matrix](
-https://github.com/google/pytype/blob/ee51995a1c5937cb4ebee291acb2e049fb0f81cc/.travis.yml#L9).
-Even if the tests do not pass yet, it is helpful to see the failures, and you
-can configure them to not fail the build by adding to the matrix:
-
-```
-allow_failures:
-  - python: "3.x"  # replace x with the appropriate minor version
-```
+[CI test matrix](
+https://github.com/google/pytype/blob/a2ce16edc0ee992f97b328ce752b51318a00d513/.github/workflows/ci.yml#L15-L22).
+Even if the tests do not pass yet, it is helpful to see the failures. You can
+can configure them to not fail the workflow by [marking the check as experimental](
+https://github.com/google/pytype/blob/a2ce16edc0ee992f97b328ce752b51318a00d513/.github/workflows/ci.yml#L19-L22)
+<!-- TODO(rechen): Once https://github.com/actions/toolkit/issues/399 is
+supported, suggest that instead of the `|| exit 0` hack -->
+and using `|| exit 0` to [always report a success](
+https://github.com/google/pytype/blob/a2ce16edc0ee992f97b328ce752b51318a00d513/.github/workflows/ci.yml#L47-L49).
 
 ### GitHub release
 

pytype/CMakeLists.txt

@@ -217,10 +217,10 @@ py_library(
   SRCS
     abstract_utils.py
   DEPS
-    .cfg_utils
     .utils
     pytype.pyc.pyc
     pytype.pytd.pytd_for_parser
+    pytype.typegraph.cfg_utils
 )
 
 py_library(
@@ -247,57 +247,6 @@ py_library(
     pytype.typegraph.cfg
 )
 
-py_library(
-    NAME
-      tools
-    SRCS
-      tools/arg_parser.py
-      tools/config.py
-      tools/environment.py
-      tools/runner.py
-      tools/tool_utils.py
-    DEPS
-      .config
-      .utils
-      pytype.pytd.typeshed
-)
-
-py_library(
-    NAME
-      analyze_project
-    SRCS
-      tools/analyze_project/config.py
-      tools/analyze_project/environment.py
-      tools/analyze_project/parse_args.py
-      tools/analyze_project/pytype_runner.py
-    DEPS
-      .config
-      .io
-      .tools
-      .utils
-)
-
-# This library is disabled because cfg.py can conflict with the typegraph.cfg
-# extension module.
-# py_library(
-#   NAME
-#     cfg_py
-#   SRCS
-#     typegraph/cfg.py
-#   DEPS
-#     .debug
-#     .metrics
-# )
-
-py_library(
-  NAME
-    cfg_utils
-  SRCS
-    typegraph/cfg_utils.py
-  DEPS
-    pytype.typegraph.cfg
-)
-
 py_test(
   NAME
     config_test

pytype/analyze.py

@@ -339,10 +339,11 @@ def _call_init_on_binding(self, node, b):
       for param in b.data.instance_type_parameters.values():
         node = self.call_init(node, param)
     node = self._call_method(node, b, "__init__")
-    # Test classes initialize attributes in setUp() as well.
     cls = b.data.get_class()
-    if isinstance(cls, abstract.InterpreterClass) and cls.is_test_class:
-      node = self._call_method(node, b, "setUp")
+    if isinstance(cls, abstract.InterpreterClass):
+      # Call any additional initalizers the class has registered.
+      for method in cls.additional_init_methods:
+        node = self._call_method(node, b, method)
     return node
 
   def call_init(self, node, instance):

pytype/errors.py

@@ -534,11 +534,16 @@ def _print_as_return_types(self, node, formal, actual, bad):
     else:
       output_mode = convert.OutputMode.DETAILED
     with convert.set_output_mode(output_mode):
-      actual = self._pytd_print(pytd_utils.JoinTypes(
+      bad_actual = self._pytd_print(pytd_utils.JoinTypes(
           view[actual].data.to_type(node, view=view) for view in bad))
+      if len(actual.bindings) > len(bad):
+        full_actual = self._pytd_print(pytd_utils.JoinTypes(
+            v.to_type(node) for v in actual.data))
+      else:
+        full_actual = bad_actual
     # typing.NoReturn is a prettier alias for nothing.
     fmt = lambda ret: "NoReturn" if ret == "nothing" else ret
-    return fmt(expected), fmt(actual)
+    return fmt(expected), fmt(bad_actual), fmt(full_actual)
 
   def _join_printed_types(self, types):
     """Pretty-print the union of the printed types."""
@@ -858,14 +863,20 @@ def base_class_error(self, stack, base_var):
 
   @_error_name("bad-return-type")
   def bad_return_type(self, stack, node, formal, actual, bad):
-    expected, actual = self._print_as_return_types(node, formal, actual, bad)
+    """Logs a [bad-return-type] error."""
+    expected, bad_actual, full_actual = self._print_as_return_types(
+        node, formal, actual, bad)
+    if full_actual == bad_actual:
+      message = "bad return type"
+    else:
+      message = f"bad option {bad_actual!r} in return type"
     details = "".join(["         Expected: ", expected, "\n",
-                       "Actually returned: ", actual])
-    self.error(stack, "bad option in return type", details)
+                       "Actually returned: ", full_actual])
+    self.error(stack, message, details)
 
   @_error_name("bad-concrete-type")
   def bad_concrete_type(self, stack, node, formal, actual, bad):
-    expected, actual = self._print_as_return_types(node, formal, actual, bad)
+    expected, actual, _ = self._print_as_return_types(node, formal, actual, bad)
     details = "".join(["       Expected: ", expected, "\n",
                        "Actually passed: ", actual])
     self.error(stack, "Invalid instantiation of generic class", details)

pytype/mixin.py

@@ -158,6 +158,10 @@ def init_mixin(self, metaclass):
     self._init_overrides_bool()
     self._all_formal_type_parameters = datatypes.AliasingMonitorDict()
     self._all_formal_type_parameters_loaded = False
+    # Call these methods in addition to __init__ when constructing instances.
+    self.additional_init_methods = []
+    if self._is_test_class():
+      self.additional_init_methods.append("setUp")
 
   def bases(self):
     return []
@@ -290,8 +294,7 @@ def is_abstract(self):
     return ((self._has_explicit_abcmeta() or self._has_implicit_abcmeta()) and
             bool(self.abstract_methods))
 
-  @property
-  def is_test_class(self):
+  def _is_test_class(self):
     return any(base.full_name in ("unittest.TestCase", "unittest.case.TestCase")
                for base in self.mro)
 
@@ -386,9 +389,9 @@ def _call_method(self, node, value, method_name, args):
 
   def _call_init(self, node, value, args):
     node = self._call_method(node, value, "__init__", args)
-    # Test classes initialize attributes in setUp() as well.
-    if self.is_test_class:
-      node = self._call_method(node, value, "setUp", function.Args(()))
+    # Call any additional initalizers the class has registered.
+    for method in self.additional_init_methods:
+      node = self._call_method(node, value, method, function.Args(()))
     return node
 
   def _new_instance(self):

pytype/overlays/flax_overlay.py

@@ -101,6 +101,8 @@ def __init__(self, vm, name="Module", module="flax.linen.module"):
     super().__init__(name, pytd_cls, vm)
 
   def init_subclass(self, node, subclass):
+    # Subclasses of Module call self.setup() when creating instances.
+    subclass.additional_init_methods.append("setup")
     dc = ModuleDataclass.make(self.vm)
     subclass_var = subclass.to_variable(node)
     args = function.Args(

pytype/pyi/parser.py

@@ -874,8 +874,14 @@ def _parameterized_type(self, base_type, parameters):
             literal_parameters.append(pytd.AnythingType())
         elif isinstance(p, (int, str, bytes)):
           literal_parameters.append(pytd.Literal(p))
+        elif isinstance(p, pytd.UnionType):
+          for t in p.type_list:
+            if isinstance(t, pytd.Literal):
+              literal_parameters.append(t)
+            else:
+              raise ParseError(f"Literal[{t}] not supported")
         else:
-          raise ParseError("Literal[%s] not supported" % p)
+          raise ParseError(f"Literal[{p}] not supported")
       return pytd_utils.JoinTypes(literal_parameters)
     elif any(isinstance(p, (int, str)) for p in parameters):
       parameters = ", ".join(

pytype/pyi/parser_test.py

@@ -2474,12 +2474,14 @@ def test_unnest(self):
       from typing import Literal
       MyLiteralAlias = Literal[42]
       x: Literal[MyLiteralAlias, Literal[Literal[True]], None]
+      y: Literal[1, Literal[2, Literal[3]]]
     """, """
       from typing import Literal, Optional
 
       MyLiteralAlias = Literal[42]
 
       x: Optional[Literal[42, True]]
+      y: Literal[1, 2, 3]
     """)
 
   def test_bad_value(self):

pytype/pytd/CMakeLists.txt

@@ -145,8 +145,8 @@ py_library(
     pytd_visitors.py
   DEPS
     .pytd_defs
-    pytype.cfg_utils
     pytype.pytd.parse.parser_constants
+    pytype.typegraph.cfg_utils
 )
 
 py_library(

pytype/tests/py3/test_checker.py

@@ -161,5 +161,23 @@ def g(x: A[str]):  # not-indexable[e2]
     """)
     self.assertErrorRegexes(errorlog, {"e1": r"Generic", "e2": r"Generic"})
 
+  def test_bad_return_type(self):
+    errorlog = self.CheckWithErrors("""
+      def f() -> int:
+        return ''  # bad-return-type[e]
+    """)
+    self.assertErrorRegexes(errorlog, {
+        "e": r"bad return type.*Expected: int.*Actually returned: str"})
+
+  def test_bad_option_in_return_type(self):
+    errorlog = self.CheckWithErrors("""
+      from typing import Union
+      def f(x: Union[int, str]) -> int:
+        return x  # bad-return-type[e]
+    """)
+    self.assertErrorRegexes(errorlog, {"e": (
+        r"bad option 'str' in return type.*"
+        r"Expected: int.*Actually returned: Union\[int, str\]")})
+
 
 test_base.main(globals(), __name__ == "__main__")

pytype/tests/py3/test_flax_overlay.py

@@ -154,5 +154,18 @@ class Foo(nn.Module):  # invalid-annotation[e]
         """, pythonpath=[d.path])
       self.assertErrorRegexes(errors, {"e": r"name.*implicitly"})
 
+  def test_setup(self):
+    with file_utils.Tempdir() as d:
+      self._setup_linen_pyi(d)
+      self.Check("""
+        from flax import linen
+        class Foo(linen.Module):
+          x: int
+          def setup(self):
+            self.y = 10
+        a = Foo(10)
+        b = a.y
+      """, pythonpath=[d.path])
+
 
 test_base.main(globals(), __name__ == "__main__")

pytype/tools/CMakeLists.txt

@@ -1,13 +1,28 @@
 add_package()
 
+py_library(
+    NAME
+      tools
+    SRCS
+      arg_parser.py
+      config.py
+      environment.py
+      runner.py
+      tool_utils.py
+    DEPS
+      pytype.config
+      pytype.utils
+      pytype.pytd.typeshed
+)
+
 py_test(
   NAME
     arg_parser_test
   SRCS
     arg_parser_test.py
   DEPS
+    .tools
     pytype.config
-    pytype.tools
 )
 
 py_test(
@@ -16,7 +31,7 @@ py_test(
   SRCS
     config_test.py
   DEPS
-    pytype.tools
+    .tools
     pytype.utils
 )
 
@@ -26,7 +41,7 @@ py_test(
   SRCS
     environment_test.py
   DEPS
-    pytype.tools
+    .tools
     pytype.utils
 )
 
@@ -36,7 +51,7 @@ py_test(
   SRCS
     tool_utils_test.py
   DEPS
-    pytype.tools
+    .tools
     pytype.utils
 )
 

pytype/tools/analyze_project/CMakeLists.txt

@@ -1,12 +1,27 @@
 add_package()
 
+py_library(
+    NAME
+      analyze_project
+    SRCS
+      config.py
+      environment.py
+      parse_args.py
+      pytype_runner.py
+    DEPS
+      pytype.config
+      pytype.io
+      pytype.utils
+      pytype.tools.tools
+)
+
 py_test(
   NAME
     config_test
   SRCS
     config_test.py
   DEPS
-    pytype.analyze_project
+    .analyze_project
     pytype.utils
 )
 
@@ -16,7 +31,7 @@ py_test(
   SRCS
     parse_args_test.py
   DEPS
-    pytype.analyze_project
+    .analyze_project
 )
 
 py_test(
@@ -25,7 +40,7 @@ py_test(
   SRCS
     pytype_runner_test.py
   DEPS
-    pytype.analyze_project
+    .analyze_project
     pytype.config
     pytype.utils
 )
@@ -38,5 +53,5 @@ toplevel_py_binary(
   MAIN
     main.py
   DEPS
-    pytype.analyze_project
+    .analyze_project
 )