Merge pull request #1340 from google/google_sync

Google sync

README.md

@@ -67,13 +67,13 @@ pip install pytype
 pytype file_or_directory
 ```
 
-To set up pytype on an entire package, add the following to a `setup.cfg` file
-in the directory immediately above the package, replacing `package_name` with
-the package name:
+To set up pytype on an entire package, add the following to a `pyproject.toml`
+file in the directory immediately above the package, replacing `package_name`
+with the package name:
 
-```ini
-[pytype]
-inputs = package_name
+```toml
+[tool.pytype]
+inputs = ['package_name']
 ```
 
 Now you can run the no-argument command `pytype` to type-check the package. It's
@@ -178,14 +178,15 @@ installation instead of its own bundled copy by setting `$TYPESHED_HOME`.
 ### Config File
 
 For convenience, you can save your pytype configuration in a file. The config
-file is an INI-style file with a `[pytype]` section; if an explicit config file
-is not supplied, pytype will look for a `[pytype]` section in the first
+file can be a TOML-style file with a `[tool.pytype]` section (preferred) or an
+INI-style file with a `[pytype]` section. If an explicit config file is not
+supplied, pytype will look for a pytype section in the first `pyproject.toml` or
 `setup.cfg` file found by walking upwards from the current working directory.
 
 Start off by generating a sample config file:
 
 ```shell
-$ pytype --generate-config pytype.cfg
+$ pytype --generate-config pytype.toml
 ```
 
 Now customize the file based on your local setup, keeping only the sections you
@@ -212,35 +213,35 @@ Here is the filled-in config file, which instructs pytype to type-check
 and ignore attribute errors. Notice that the path to a package does not include
 the package itself.
 
-```ini
-$ cat ~/repo1/pytype.cfg
+```toml
+$ cat ~/repo1/pytype.toml
 
 # NOTE: All relative paths are relative to the location of this file.
 
-[pytype]
+[tool.pytype]
 
 # Space-separated list of files or directories to process.
-inputs =
-    foo
+inputs = [
+    'foo',
+]
 
 # Python version (major.minor) of the target code.
-python_version = 3.9
+python_version = '3.9'
 
 # Paths to source code directories, separated by ':'.
-pythonpath =
-    .:
-    ~/repo2
+pythonpath = .:~/repo2
 
-# Comma or space-separated list of error names to ignore.
-disable =
-    attribute-error
+# Space-separated list of error names to ignore.
+disable = [
+    'attribute-error',
+]
 ```
 
 We could've discovered that `~/repo2` needed to be added to the pythonpath by
 running pytype's broken dependency checker:
 
 ```
-$ pytype --config=~/repo1/pytype.cfg ~/repo1/foo/*.py --unresolved
+$ pytype --config=~/repo1/pytype.toml ~/repo1/foo/*.py --unresolved
 
 Unresolved dependencies:
   bar.dependency

docs/index.md

@@ -64,13 +64,13 @@ pip install pytype
 pytype file_or_directory
 ```
 
-To set up pytype on an entire package, add the following to a `setup.cfg` file
-in the directory immediately above the package, replacing `package_name` with
-the package name:
+To set up pytype on an entire package, add the following to a `pyproject.toml`
+file in the directory immediately above the package, replacing `package_name`
+with the package name:
 
-```ini
-[pytype]
-inputs = package_name
+```toml
+[tool.pytype]
+inputs = ['package_name']
 ```
 
 Now you can run the no-argument command `pytype` to type-check the package. It's
@@ -87,9 +87,9 @@ merge-pyi -i <filepath>.py .pytype/pyi/<filename>.pyi
 
 ## Requirements
 
-You need a Python 3.7-3.9 interpreter to run pytype, as well as an
+You need a Python 3.7-3.10 interpreter to run pytype, as well as an
 interpreter in `$PATH` for the Python version of the code you're analyzing
-(supported: 3.7-3.9).
+(supported: 3.7-3.10).
 
 Platform support:
 
@@ -175,14 +175,15 @@ installation instead of its own bundled copy by setting `$TYPESHED_HOME`.
 ### Config File
 
 For convenience, you can save your pytype configuration in a file. The config
-file is an INI-style file with a `[pytype]` section; if an explicit config file
-is not supplied, pytype will look for a `[pytype]` section in the first
+file can be a TOML-style file with a `[tool.pytype]` section (preferred) or an
+INI-style file with a `[pytype]` section. If an explicit config file is not
+supplied, pytype will look for a pytype section in the first `pyproject.toml` or
 `setup.cfg` file found by walking upwards from the current working directory.
 
 Start off by generating a sample config file:
 
 ```shell
-$ pytype --generate-config pytype.cfg
+$ pytype --generate-config pytype.toml
 ```
 
 Now customize the file based on your local setup, keeping only the sections you
@@ -209,35 +210,35 @@ Here is the filled-in config file, which instructs pytype to type-check
 and ignore attribute errors. Notice that the path to a package does not include
 the package itself.
 
-```ini
-$ cat ~/repo1/pytype.cfg
+```toml
+$ cat ~/repo1/pytype.toml
 
 # NOTE: All relative paths are relative to the location of this file.
 
-[pytype]
+[tool.pytype]
 
 # Space-separated list of files or directories to process.
-inputs =
-    foo
+inputs = [
+    'foo',
+]
 
 # Python version (major.minor) of the target code.
-python_version = 3.9
+python_version = '3.9'
 
 # Paths to source code directories, separated by ':'.
-pythonpath =
-    .:
-    ~/repo2
+pythonpath = .:~/repo2
 
-# Comma or space-separated list of error names to ignore.
-disable =
-    attribute-error
+# Space-separated list of error names to ignore.
+disable = [
+    'attribute-error',
+]
 ```
 
 We could've discovered that `~/repo2` needed to be added to the pythonpath by
 running pytype's broken dependency checker:
 
 ```
-$ pytype --config=~/repo1/pytype.cfg ~/repo1/foo/*.py --unresolved
+$ pytype --config=~/repo1/pytype.toml ~/repo1/foo/*.py --unresolved
 
 Unresolved dependencies:
   bar.dependency

pyproject.toml

@@ -1,3 +1,22 @@
 [build-system]
 requires = ["setuptools>=40.8.0", "ninja", "wheel", "pybind11>=2.10.1"]
 build-backend = "setuptools.build_meta"
+
+[tool.pytype]
+inputs = [
+    "pytype/*.py",
+    "pytype/overlays/",
+    "pytype/pyc/",
+    "pytype/pyi/",
+    "pytype/pytd/",
+    "pytype/tools/",
+    "pytype/typegraph/",
+    "pytype_extensions/**/*.py",
+]
+exclude = [
+    "**/*_test.py",
+    "**/test_*.py",
+    "**/*_test_*.py",
+    "pytype/tools/merge_pyi/test_data/",
+    "pytype/tools/xref/testdata/",
+]

pytype/config.py

@@ -209,7 +209,7 @@ def add_options(o, arglist):
     _Arg(
         "-d", "--disable", action="store",
         dest="disable", default=None,
-        help="Comma separated list of error names to ignore."),
+        help="Comma-separated list of error names to ignore."),
     _Arg(
         "--no-report-errors", action="store_false",
         dest="report_errors", default=True,
@@ -295,7 +295,7 @@ def add_options(o, arglist):
     _Arg(
         "--pickle-metadata", type=str, action="store",
         dest="pickle_metadata", default=None,
-        help=("Comma separated list of metadata strings to be saved in the "
+        help=("Comma-separated list of metadata strings to be saved in the "
               "pickled file.")),
 ]
 
@@ -345,7 +345,7 @@ def add_options(o, arglist):
     _Arg(
         "-e", "--enable-only", action="store",
         dest="enable_only", default=None,
-        help="Comma separated list of error names to enable checking for."),
+        help="Comma-separated list of error names to enable checking for."),
     # TODO(rechen): --analyze-annotated and --quick would make more sense as
     # basic options but are currently used by pytype-all in a way that isn't
     # easily configurable.

pytype/metaclass.py

@@ -59,7 +59,7 @@ def call(self, node, unused_func, args):
                                       self.module_name).to_variable(node)
 
 
-class WithMetaclassInstance(abstract.BaseValue, abstract.Class):
+class WithMetaclassInstance(abstract.BaseValue, abstract.Class):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """Anonymous class created by with_metaclass."""
 
   def __init__(self, ctx, cls, bases):

pytype/output.py

@@ -25,6 +25,11 @@
 log = logging.getLogger(__name__)
 
 
+# A variable with more bindings than this is treated as a large literal constant
+# and special-cased.
+LARGE_LITERAL_SIZE = 15
+
+
 class Converter(utils.ContextWeakrefMixin):
   """Functions for converting abstract classes into PyTD."""
 
@@ -37,6 +42,7 @@ class OutputMode(enum.IntEnum):
   def __init__(self, ctx):
     super().__init__(ctx)
     self._output_mode = Converter.OutputMode.NORMAL
+    self._optimize_literals = False
     self._scopes = []
 
   @contextlib.contextmanager
@@ -61,13 +67,27 @@ def set_output_mode(self, mode):
     yield
     self._output_mode = old
 
+  @contextlib.contextmanager
+  def optimize_literals(self):
+    """Optimize output of literal data structures in pyi files."""
+    old = self._optimize_literals
+    self._optimize_literals = True
+    yield
+    self._optimize_literals = old
+
   @property
   def _detailed(self):
     return self._output_mode >= Converter.OutputMode.DETAILED
 
   def _get_values(self, node, var, view):
     if var.bindings and view is not None:
       return [view[var].data]
+    elif self._optimize_literals and len(var.bindings) > LARGE_LITERAL_SIZE:
+      # Performance optimisation: If we have so many elements in a container, it
+      # is very likely a literal, and would ideally be constructed at a single
+      # CFG node. Therefore, we do not attempt to filter its bindings (which is
+      # a very expensive operation for large collections).
+      return var.data
     elif node:
       return var.FilteredData(node, strict=False)
     else:

pytype/pytd/pytd.py

@@ -324,7 +324,7 @@ class Parameter(Node):
 
 
 @attrs.frozen(cache_hash=True)
-class TypeParameter(Node, Type):
+class TypeParameter(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """Represents a type parameter.
 
   A type parameter is a bound variable in the context of a function or class
@@ -439,7 +439,7 @@ def full_name(self):
 
 
 @attrs.frozen(cache_hash=True)
-class NamedType(Node, Type):
+class NamedType(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """A type specified by name and, optionally, the module it is in."""
   name: str
 
@@ -448,7 +448,7 @@ def __str__(self):
 
 
 @attrs.mutable(init=False, slots=False, eq=False)
-class ClassType(Node, Type):
+class ClassType(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """A type specified through an existing class node."""
   # This type is different from normal nodes:
   # (a) It's mutable, and there are functions
@@ -485,7 +485,7 @@ def __repr__(self):
 
 
 @attrs.frozen(cache_hash=True)
-class LateType(Node, Type):
+class LateType(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """A type we have yet to resolve."""
   name: str
   recursive: bool = False
@@ -495,7 +495,7 @@ def __str__(self):
 
 
 @attrs.frozen(cache_hash=True)
-class AnythingType(Node, Type):
+class AnythingType(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """A type we know nothing about yet (? in pytd)."""
 
   @property
@@ -507,7 +507,7 @@ def __bool__(self):
 
 
 @attrs.frozen(cache_hash=True)
-class NothingType(Node, Type):
+class NothingType(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """An "impossible" type, with no instances (nothing in pytd).
 
   Also known as the "uninhabited" type, or, in type systems, the "bottom" type.
@@ -534,7 +534,7 @@ def _FlattenTypes(type_list) -> Tuple[Type, ...]:
 
 
 @attrs.frozen(eq=False)
-class _SetOfTypes(Node, Type):
+class _SetOfTypes(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """Super class for shared behavior of UnionType and IntersectionType."""
   # NOTE: type_list is kept as a tuple, to preserve the original order
   #       even though in most respects it acts like a frozenset.
@@ -573,7 +573,7 @@ class IntersectionType(_SetOfTypes):
 
 
 @attrs.frozen(cache_hash=True)
-class GenericType(Node, Type):
+class GenericType(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   """Generic type. Takes a base type and type parameters.
 
   This is used for homogeneous tuples, lists, dictionaries, user classes, etc.
@@ -640,7 +640,7 @@ def paramspec(self):
 
 
 @attrs.frozen(cache_hash=True)
-class Literal(Node, Type):
+class Literal(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   value: Union[int, str, Type, Constant]
 
   @property
@@ -649,7 +649,7 @@ def name(self):
 
 
 @attrs.frozen(cache_hash=True)
-class Annotated(Node, Type):
+class Annotated(Node, Type):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
   base_type: Type
   annotations: Tuple[str, ...]
 

pytype/stubs/builtins/builtins.pytd

@@ -826,6 +826,8 @@ class int(SupportsInt, SupportsFloat, SupportsAbs[int]):
     def __pos__(self) -> int: ...
     def __trunc__(self, *args, **kwargs) -> int: ...
     def __xor__(self, y: int) -> int: ...
+    if sys.version_info >= (3, 10):
+        def bit_count(self) -> int: ...
     def bit_length(self) -> int: ...
     def conjugate(self) -> int: ...
     def to_bytes(self, length: int, byteorder: str, *, signed: bool = ...) -> bytes: ...