Merge pull request #1345 from google/google_sync

Add a new "missing_modules" parameter to load_pytd.create_loader.

CHANGELOG

@@ -1,3 +1,17 @@
+Version 2023.01.10:
+
+Updates:
+* Add a new "missing_modules" parameter to load_pytd.create_loader.
+* Support putting pytype settings in a pyproject.toml file.
+* Add a performance optimisation for outputting the type of large collections.
+
+Bug fixes:
+* Add missing int.bit_count method.
+* Improve pytype's handling of dict.update.
+* Do better matching of overloads in generic classes.
+* Show expected type in InterpreterFunction error messages with *args/**kwargs.
+* Allow 'self' as a keyword argument to str.format.
+
 Version 2022.12.15:
 
 Updates:

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2022.12.15'
+__version__ = '2023.01.10'

pytype/imports/typeshed.py

@@ -4,7 +4,7 @@
 import collections
 import os
 import re
-from typing import List, Sequence, Tuple
+from typing import Collection, List, Sequence, Tuple
 
 from pytype import module_utils
 from pytype import pytype_source_utils
@@ -174,15 +174,23 @@ class Typeshed:
   # The path is relative to typeshed's root directory, e.g. if you set this to
   # "missing.txt" you need to create $TYPESHED_HOME/missing.txt or
   # pytype/typeshed/missing.txt
-  # For testing, this file must contain the entry 'stdlib/3/pytypecanary'.
+  # For testing, this file must contain the entry 'stdlib/pytypecanary'.
   MISSING_FILE = None
 
-  def __init__(self):
+  def __init__(self, missing_modules: Collection[str] = ()):
+    """Initializer.
+
+    Args:
+      missing_modules: A collection of modules in the format
+          'stdlib/module_name', which will be combined with the contents of
+          MISSING_FILE to form a set of missing modules for which pytype will
+          not report errors.
+    """
     if os.getenv("TYPESHED_HOME"):
       self._store = ExternalTypeshedFs(missing_file=self.MISSING_FILE)
     else:
       self._store = InternalTypeshedFs(missing_file=self.MISSING_FILE)
-    self._missing = self._load_missing()
+    self._missing = self._load_missing().union(missing_modules)
     self._stdlib_versions = self._load_stdlib_versions()
     self._third_party_packages = self._load_third_party_packages()
 
@@ -406,30 +414,23 @@ def blacklisted_modules(self):
         yield mod
 
 
-_typeshed = None
-
-
-def _get_typeshed():
-  """Get the global Typeshed instance."""
-  global _typeshed
-  if _typeshed is None:
-    try:
-      _typeshed = Typeshed()
-    except OSError as e:
-      # This happens if typeshed is not available. Which is a setup error
-      # and should be propagated to the user. The IOError is caught further up
-      # in the stack.
-      raise utils.UsageError(f"Couldn't initialize typeshed:\n {str(e)}")
-  return _typeshed
+def _get_typeshed(missing_modules):
+  """Get a Typeshed instance."""
+  try:
+    return Typeshed(missing_modules)
+  except OSError as e:
+    # This happens if typeshed is not available. Which is a setup error
+    # and should be propagated to the user. The IOError is caught further up
+    # in the stack.
+    raise utils.UsageError(f"Couldn't initialize typeshed:\n {str(e)}")
 
 
 class TypeshedLoader(base.BuiltinLoader):
   """Load modules from typeshed."""
 
-  def __init__(self, options):
+  def __init__(self, options, missing_modules):
     self.options = options
-    self.typeshed = _get_typeshed()
-    assert self.typeshed is not None
+    self.typeshed = _get_typeshed(missing_modules)
     # TODO(mdemello): Inject options.open_function into self.typeshed
 
   def load_module(self, namespace, module_name):

pytype/imports/typeshed_test.py

@@ -42,7 +42,7 @@ def test_get_typeshed_dir(self):
     self.assertIn("LogRecord", data)
 
   def test_load_module(self):
-    loader = typeshed.TypeshedLoader(self.options)
+    loader = typeshed.TypeshedLoader(self.options, ())
     filename, ast = loader.load_module("stdlib", "_random")
     self.assertEqual(path_utils.basename(filename), "_random.pyi")
     self.assertIn("_random.Random", [cls.name for cls in ast.classes])

pytype/load_pytd.py

@@ -31,15 +31,15 @@
 ModuleInfo = imports_base.ModuleInfo
 
 
-def create_loader(options):
+def create_loader(options, missing_modules=()):
   """Create a pytd loader."""
   if options.precompiled_builtins:
     return PickledPyiLoader.load_from_pickle(
-        options.precompiled_builtins, options)
+        options.precompiled_builtins, options, missing_modules)
   elif options.use_pickled_files:
-    return PickledPyiLoader(options)
+    return PickledPyiLoader(options, missing_modules=missing_modules)
   else:
-    return Loader(options)
+    return Loader(options, missing_modules=missing_modules)
 
 
 def _is_package(filename):
@@ -314,15 +314,16 @@ class Loader:
     typing: The typing ast.
   """
 
-  def __init__(self, options, modules=None):
+  def __init__(self, options, modules=None, missing_modules=()):
     self.options = options
     self._modules = _ModuleMap(options, modules)
     self.builtins = self._modules["builtins"].ast
     self.typing = self._modules["typing"].ast
     self._module_loader = module_loader.ModuleLoader(options)
     pyi_options = parser.PyiOptions.from_toplevel_options(options)
     self._builtin_loader = builtin_stubs.BuiltinLoader(pyi_options)
-    self._typeshed_loader = typeshed.TypeshedLoader(pyi_options)
+    self._typeshed_loader = typeshed.TypeshedLoader(
+        pyi_options, missing_modules)
     self._resolver = _Resolver(self.builtins)
     self._import_name_cache = {}  # performance cache
     self._aliases = {}
@@ -709,7 +710,7 @@ class PickledPyiLoader(Loader):
   """A Loader which always loads pickle instead of PYI, for speed."""
 
   @classmethod
-  def load_from_pickle(cls, filename, options):
+  def load_from_pickle(cls, filename, options, missing_modules=()):
     """Load a pytd module from a pickle file."""
     items = pickle_utils.LoadPickle(filename, compress=True,
                                     open_function=options.open_function)
@@ -718,7 +719,7 @@ def load_from_pickle(cls, filename, options):
                      has_unresolved_pointers=False)
         for name, pickle in items
     }
-    return cls(options, modules=modules)
+    return cls(options, modules=modules, missing_modules=missing_modules)
 
   def load_module(self, mod_info, mod_ast=None):
     """Load (or retrieve from cache) a module and resolve its dependencies."""