From 9f0ab6f19e73d203065316cd79a64bf3405ec86c Mon Sep 17 00:00:00 2001 From: rechen Date: Tue, 10 Jan 2023 12:04:12 -0800 Subject: [PATCH 1/2] Add a new "missing_modules" parameter to load_pytd.create_loader. This will allow typeshed's pytype_test to declare missing modules. See https://github.com/python/typeshed/issues/5768 and https://github.com/python/typeshed/pull/9459 for context. PiperOrigin-RevId: 501058238 --- pytype/imports/typeshed.py | 45 +++++++++++++++++---------------- pytype/imports/typeshed_test.py | 2 +- pytype/load_pytd.py | 17 +++++++------ 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/pytype/imports/typeshed.py b/pytype/imports/typeshed.py index ea82e64d4..ff49cabed 100644 --- a/pytype/imports/typeshed.py +++ b/pytype/imports/typeshed.py @@ -4,7 +4,7 @@ import collections import os import re -from typing import List, Sequence, Tuple +from typing import Collection, List, Sequence, Tuple from pytype import module_utils from pytype import pytype_source_utils @@ -174,15 +174,23 @@ class Typeshed: # The path is relative to typeshed's root directory, e.g. if you set this to # "missing.txt" you need to create $TYPESHED_HOME/missing.txt or # pytype/typeshed/missing.txt - # For testing, this file must contain the entry 'stdlib/3/pytypecanary'. + # For testing, this file must contain the entry 'stdlib/pytypecanary'. MISSING_FILE = None - def __init__(self): + def __init__(self, missing_modules: Collection[str] = ()): + """Initializer. + + Args: + missing_modules: A collection of modules in the format + 'stdlib/module_name', which will be combined with the contents of + MISSING_FILE to form a set of missing modules for which pytype will + not report errors. + """ if os.getenv("TYPESHED_HOME"): self._store = ExternalTypeshedFs(missing_file=self.MISSING_FILE) else: self._store = InternalTypeshedFs(missing_file=self.MISSING_FILE) - self._missing = self._load_missing() + self._missing = self._load_missing().union(missing_modules) self._stdlib_versions = self._load_stdlib_versions() self._third_party_packages = self._load_third_party_packages() @@ -406,30 +414,23 @@ def blacklisted_modules(self): yield mod -_typeshed = None - - -def _get_typeshed(): - """Get the global Typeshed instance.""" - global _typeshed - if _typeshed is None: - try: - _typeshed = Typeshed() - except OSError as e: - # This happens if typeshed is not available. Which is a setup error - # and should be propagated to the user. The IOError is caught further up - # in the stack. - raise utils.UsageError(f"Couldn't initialize typeshed:\n {str(e)}") - return _typeshed +def _get_typeshed(missing_modules): + """Get a Typeshed instance.""" + try: + return Typeshed(missing_modules) + except OSError as e: + # This happens if typeshed is not available. Which is a setup error + # and should be propagated to the user. The IOError is caught further up + # in the stack. + raise utils.UsageError(f"Couldn't initialize typeshed:\n {str(e)}") class TypeshedLoader(base.BuiltinLoader): """Load modules from typeshed.""" - def __init__(self, options): + def __init__(self, options, missing_modules): self.options = options - self.typeshed = _get_typeshed() - assert self.typeshed is not None + self.typeshed = _get_typeshed(missing_modules) # TODO(mdemello): Inject options.open_function into self.typeshed def load_module(self, namespace, module_name): diff --git a/pytype/imports/typeshed_test.py b/pytype/imports/typeshed_test.py index 95752aeaa..9448d0690 100644 --- a/pytype/imports/typeshed_test.py +++ b/pytype/imports/typeshed_test.py @@ -42,7 +42,7 @@ def test_get_typeshed_dir(self): self.assertIn("LogRecord", data) def test_load_module(self): - loader = typeshed.TypeshedLoader(self.options) + loader = typeshed.TypeshedLoader(self.options, ()) filename, ast = loader.load_module("stdlib", "_random") self.assertEqual(path_utils.basename(filename), "_random.pyi") self.assertIn("_random.Random", [cls.name for cls in ast.classes]) diff --git a/pytype/load_pytd.py b/pytype/load_pytd.py index aba562937..07a649ee2 100644 --- a/pytype/load_pytd.py +++ b/pytype/load_pytd.py @@ -31,15 +31,15 @@ ModuleInfo = imports_base.ModuleInfo -def create_loader(options): +def create_loader(options, missing_modules=()): """Create a pytd loader.""" if options.precompiled_builtins: return PickledPyiLoader.load_from_pickle( - options.precompiled_builtins, options) + options.precompiled_builtins, options, missing_modules) elif options.use_pickled_files: - return PickledPyiLoader(options) + return PickledPyiLoader(options, missing_modules=missing_modules) else: - return Loader(options) + return Loader(options, missing_modules=missing_modules) def _is_package(filename): @@ -314,7 +314,7 @@ class Loader: typing: The typing ast. """ - def __init__(self, options, modules=None): + def __init__(self, options, modules=None, missing_modules=()): self.options = options self._modules = _ModuleMap(options, modules) self.builtins = self._modules["builtins"].ast @@ -322,7 +322,8 @@ def __init__(self, options, modules=None): self._module_loader = module_loader.ModuleLoader(options) pyi_options = parser.PyiOptions.from_toplevel_options(options) self._builtin_loader = builtin_stubs.BuiltinLoader(pyi_options) - self._typeshed_loader = typeshed.TypeshedLoader(pyi_options) + self._typeshed_loader = typeshed.TypeshedLoader( + pyi_options, missing_modules) self._resolver = _Resolver(self.builtins) self._import_name_cache = {} # performance cache self._aliases = {} @@ -709,7 +710,7 @@ class PickledPyiLoader(Loader): """A Loader which always loads pickle instead of PYI, for speed.""" @classmethod - def load_from_pickle(cls, filename, options): + def load_from_pickle(cls, filename, options, missing_modules=()): """Load a pytd module from a pickle file.""" items = pickle_utils.LoadPickle(filename, compress=True, open_function=options.open_function) @@ -718,7 +719,7 @@ def load_from_pickle(cls, filename, options): has_unresolved_pointers=False) for name, pickle in items } - return cls(options, modules=modules) + return cls(options, modules=modules, missing_modules=missing_modules) def load_module(self, mod_info, mod_ast=None): """Load (or retrieve from cache) a module and resolve its dependencies.""" From baff5bd48700106b0972dd107876cdb9e54e20f1 Mon Sep 17 00:00:00 2001 From: rechen Date: Tue, 10 Jan 2023 12:25:07 -0800 Subject: [PATCH 2/2] Prepare a PyPI release. PiperOrigin-RevId: 501063699 --- CHANGELOG | 14 ++++++++++++++ pytype/__version__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index c81a176c3..569cdcfe6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,17 @@ +Version 2023.01.10: + +Updates: +* Add a new "missing_modules" parameter to load_pytd.create_loader. +* Support putting pytype settings in a pyproject.toml file. +* Add a performance optimisation for outputting the type of large collections. + +Bug fixes: +* Add missing int.bit_count method. +* Improve pytype's handling of dict.update. +* Do better matching of overloads in generic classes. +* Show expected type in InterpreterFunction error messages with *args/**kwargs. +* Allow 'self' as a keyword argument to str.format. + Version 2022.12.15: Updates: diff --git a/pytype/__version__.py b/pytype/__version__.py index d88e15f70..d05d87183 100644 --- a/pytype/__version__.py +++ b/pytype/__version__.py @@ -1,2 +1,2 @@ # pylint: skip-file -__version__ = '2022.12.15' +__version__ = '2023.01.10'