Improve error messages

docs/requirements-docs.txt

@@ -1,2 +1,2 @@
-sphinx>=4.2.0,<5.0.0
+sphinx>=5.1.0
 furo>=2022.3.4

docs/source/class_basics.rst

@@ -263,7 +263,7 @@ effect at runtime:
 Abstract base classes and multiple inheritance
 **********************************************
 
-Mypy supports Python :doc:`abstract base classes <library/abc>` (ABCs). Abstract classes
+Mypy supports Python :doc:`abstract base classes <python:library/abc>` (ABCs). Abstract classes
 have at least one abstract method or property that must be implemented
 by any *concrete* (non-abstract) subclass. You can define abstract base
 classes using the :py:class:`abc.ABCMeta` metaclass and the :py:func:`@abc.abstractmethod <abc.abstractmethod>`
@@ -371,8 +371,7 @@ property or an instance variable.
 Slots
 *****
 
-When a class has explicitly defined
-`__slots__ <https://docs.python.org/3/reference/datamodel.html#slots>`_,
+When a class has explicitly defined :std:term:`__slots__`,
 mypy will check that all attributes assigned to are members of ``__slots__``:
 
 .. code-block:: python

docs/source/config_file.rst

@@ -238,10 +238,8 @@ section of the command line docs.
     Crafting a single regular expression that excludes multiple files while remaining
     human-readable can be a challenge. The above example demonstrates one approach.
     ``(?x)`` enables the ``VERBOSE`` flag for the subsequent regular expression, which
-    `ignores most whitespace and supports comments`__. The above is equivalent to:
-    ``(^one\.py$|two\.pyi$|^three\.)``.
-
-    .. __: https://docs.python.org/3/library/re.html#re.X
+    :py:data:`ignores most whitespace and supports comments <re.VERBOSE>`.
+    The above is equivalent to: ``(^one\.py$|two\.pyi$|^three\.)``.
 
     For more details, see :option:`--exclude <mypy --exclude>`.
 

docs/source/error_code_list2.rst

@@ -524,8 +524,7 @@ that only existed during type-checking.
 In runtime it fails with expected ``NameError``,
 which can cause real problem in production, hidden from mypy.
 
-But, in Python3.11 ``reveal_type``
-`was added to typing.py <https://docs.python.org/3/library/typing.html#typing.reveal_type>`_.
+But, in Python3.11 :py:func:`typing.reveal_type` was added.
 ``typing_extensions`` ported this helper to all supported Python versions.
 
 Now users can actually import ``reveal_type`` to make the runtime code safe.

docs/source/getting_started.rst

@@ -256,8 +256,7 @@ Mypy can also understand how to work with types from libraries that you use.
 
 For instance, mypy comes out of the box with an intimate knowledge of the
 Python standard library. For example, here is a function which uses the
-``Path`` object from the
-`pathlib standard library module <https://docs.python.org/3/library/pathlib.html>`_:
+``Path`` object from the :doc:`pathlib standard library module <python:library/pathlib>`:
 
 .. code-block:: python
 

docs/source/html_builder.py

@@ -9,11 +9,12 @@
 from sphinx.addnodes import document
 from sphinx.application import Sphinx
 from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.environment import BuildEnvironment
 
 
 class MypyHTMLBuilder(StandaloneHTMLBuilder):
-    def __init__(self, app: Sphinx) -> None:
-        super().__init__(app)
+    def __init__(self, app: Sphinx, env: BuildEnvironment) -> None:
+        super().__init__(app, env)
         self._ref_to_doc = {}
 
     def write_doc(self, docname: str, doctree: document) -> None:

docs/source/more_types.rst

@@ -829,7 +829,7 @@ Typing async/await
 
 Mypy lets you type coroutines that use the ``async/await`` syntax.
 For more information regarding coroutines, see :pep:`492` and the
-`asyncio documentation <https://docs.python.org/3/library/asyncio.html>`_.
+`asyncio documentation <python:library/asyncio>`_.
 
 Functions defined using ``async def`` are typed similar to normal functions.
 The return type annotation should be the same as the type of the value you

misc/gen_blog_post_html.py

@@ -0,0 +1,171 @@
+"""Converter from CHANGELOG.md (Markdown) to HTML suitable for a mypy blog post.
+
+How to use:
+
+1. Write release notes in CHANGELOG.md.
+2. Make sure the heading for the next release is of form `## Mypy X.Y`.
+2. Run `misc/gen_blog_post_html.py X.Y > target.html`.
+4. Manually inspect and tweak the result.
+
+Notes:
+
+* There are some fragile assumptions. Double check the output.
+"""
+
+import argparse
+import html
+import os
+import re
+import sys
+
+
+def format_lists(h: str) -> str:
+    a = h.splitlines()
+    r = []
+    i = 0
+    bullets = ("- ", "* ", " * ")
+    while i < len(a):
+        if a[i].startswith(bullets):
+            r.append("<p><ul>")
+            while i < len(a) and a[i].startswith(bullets):
+                r.append("<li>%s" % a[i][2:].lstrip())
+                i += 1
+            r.append("</ul>")
+        else:
+            r.append(a[i])
+            i += 1
+    return "\n".join(r)
+
+
+def format_code(h: str) -> str:
+    a = h.splitlines()
+    r = []
+    i = 0
+    while i < len(a):
+        if a[i].startswith("    ") or a[i].startswith("```"):
+            indent = a[i].startswith("    ")
+            if not indent:
+                i += 1
+            r.append("<pre>")
+            while i < len(a) and (
+                (indent and a[i].startswith("    ")) or (not indent and not a[i].startswith("```"))
+            ):
+                # Undo &gt; and &lt;
+                line = a[i].replace("&gt;", ">").replace("&lt;", "<")
+                if not indent:
+                    line = "    " + line
+                r.append(html.escape(line))
+                i += 1
+            r.append("</pre>")
+            if not indent and a[i].startswith("```"):
+                i += 1
+        else:
+            r.append(a[i])
+            i += 1
+    return "\n".join(r)
+
+
+def convert(src: str) -> str:
+    h = src
+
+    # Replace < and >.
+    h = re.sub(r"<", "&lt;", h)
+    h = re.sub(r">", "&gt;", h)
+
+    # Title
+    h = re.sub(r"^## (Mypy [0-9.]+)", r"<h1>\1 Released</h1>", h, flags=re.MULTILINE)
+
+    # Subheadings
+    h = re.sub(r"\n#### ([A-Z`].*)\n", r"\n<h2>\1</h2>\n", h)
+
+    # Sub-subheadings
+    h = re.sub(r"\n\*\*([A-Z_`].*)\*\*\n", r"\n<h3>\1</h3>\n", h)
+    h = re.sub(r"\n`\*\*([A-Z_`].*)\*\*\n", r"\n<h3>`\1</h3>\n", h)
+
+    # Translate `**`
+    h = re.sub(r"`\*\*`", "<tt>**</tt>", h)
+
+    # Paragraphs
+    h = re.sub(r"\n([A-Z])", r"\n<p>\1", h)
+
+    # Bullet lists
+    h = format_lists(h)
+
+    # Code blocks
+    h = format_code(h)
+
+    # Code fragments
+    h = re.sub(r"`([^`]+)`", r"<tt>\1</tt>", h)
+
+    # Remove **** noise
+    h = re.sub(r"\*\*\*\*", "", h)
+
+    # Bold text
+    h = re.sub(r"\*\*([A-Za-z].*?)\*\*", r" <b>\1</b>", h)
+
+    # Emphasized text
+    h = re.sub(r" \*([A-Za-z].*?)\*", r" <i>\1</i>", h)
+
+    # Remove redundant PR links to avoid double links (they will be generated below)
+    h = re.sub(r"\[(#[0-9]+)\]\(https://github.com/python/mypy/pull/[0-9]+/?\)", r"\1", h)
+
+    # Issue and PR links
+    h = re.sub(r"\((#[0-9]+)\) +\(([^)]+)\)", r"(\2, \1)", h)
+    h = re.sub(
+        r"fixes #([0-9]+)",
+        r'fixes issue <a href="https://github.com/python/mypy/issues/\1">\1</a>',
+        h,
+    )
+    h = re.sub(r"#([0-9]+)", r'PR <a href="https://github.com/python/mypy/pull/\1">\1</a>', h)
+    h = re.sub(r"\) \(PR", ", PR", h)
+
+    # Markdown links
+    h = re.sub(r"\[([^]]*)\]\(([^)]*)\)", r'<a href="\2">\1</a>', h)
+
+    # Add random links in case they are missing
+    h = re.sub(
+        r"contributors to typeshed:",
+        'contributors to <a href="https://github.com/python/typeshed">typeshed</a>:',
+        h,
+    )
+
+    # Add missing top-level HTML tags
+    h = '<html>\n<meta charset="utf-8" />\n<body>\n' + h + "</body>\n</html>"
+
+    return h
+
+
+def extract_version(src: str, version: str) -> str:
+    a = src.splitlines()
+    i = 0
+    heading = f"## Mypy {version}"
+    while i < len(a):
+        if a[i].strip() == heading:
+            break
+        i += 1
+    else:
+        raise RuntimeError(f"Can't find heading {heading!r}")
+    j = i + 1
+    while not a[j].startswith("## "):
+        j += 1
+    return "\n".join(a[i:j])
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Generate HTML release blog post based on CHANGELOG.md and write to stdout."
+    )
+    parser.add_argument("version", help="mypy version, in form X.Y or X.Y.Z")
+    args = parser.parse_args()
+    version: str = args.version
+    if not re.match(r"[0-9]+(\.[0-9]+)+$", version):
+        sys.exit(f"error: Version must be of form X.Y or X.Y.Z, not {version!r}")
+    changelog_path = os.path.join(os.path.dirname(__file__), os.path.pardir, "CHANGELOG.md")
+    src = open(changelog_path).read()
+    src = extract_version(src, version)
+    dst = convert(src)
+    sys.stdout.write(dst)
+
+
+if __name__ == "__main__":
+    main()

mypy/checker.py

@@ -1879,6 +1879,7 @@ def check_explicit_override_decorator(
             found_method_base_classes
             and not defn.is_explicit_override
             and defn.name not in ("__init__", "__new__")
+            and not is_private(defn.name)
         ):
             self.msg.explicit_override_decorator_missing(
                 defn.name, found_method_base_classes[0].fullname, context or defn
@@ -1921,7 +1922,7 @@ def check_method_or_accessor_override_for_base(
             base_attr = base.names.get(name)
             if base_attr:
                 # First, check if we override a final (always an error, even with Any types).
-                if is_final_node(base_attr.node):
+                if is_final_node(base_attr.node) and not is_private(name):
                     self.msg.cant_override_final(name, base.name, defn)
                 # Second, final can't override anything writeable independently of types.
                 if defn.is_final:
@@ -2680,7 +2681,7 @@ class C(B, A[int]): ...  # this is unsafe because...
             ok = True
         # Final attributes can never be overridden, but can override
         # non-final read-only attributes.
-        if is_final_node(second.node):
+        if is_final_node(second.node) and not is_private(name):
             self.msg.cant_override_final(name, base2.name, ctx)
         if is_final_node(first.node):
             self.check_if_final_var_override_writable(name, second.node, ctx)
@@ -3308,6 +3309,8 @@ def check_compatibility_final_super(
         """
         if not isinstance(base_node, (Var, FuncBase, Decorator)):
             return True
+        if is_private(node.name):
+            return True
         if base_node.is_final and (node.is_final or not isinstance(base_node, Var)):
             # Give this error only for explicit override attempt with `Final`, or
             # if we are overriding a final method with variable.

mypy/checkexpr.py

@@ -3617,8 +3617,9 @@ def dangerous_comparison(
         self,
         left: Type,
         right: Type,
-        original_container: Type | None = None,
         *,
+        original_container: Type | None = None,
+        seen_types: set[tuple[Type, Type]] | None = None,
         prefer_literal: bool = True,
     ) -> bool:
         """Check for dangerous non-overlapping comparisons like 42 == 'no'.
@@ -3639,6 +3640,12 @@ def dangerous_comparison(
         if not self.chk.options.strict_equality:
             return False
 
+        if seen_types is None:
+            seen_types = set()
+        if (left, right) in seen_types:
+            return False
+        seen_types.add((left, right))
+
         left, right = get_proper_types((left, right))
 
         # We suppress the error if there is a custom __eq__() method on either
@@ -3694,17 +3701,21 @@ def dangerous_comparison(
                 abstract_set = self.chk.lookup_typeinfo("typing.AbstractSet")
                 left = map_instance_to_supertype(left, abstract_set)
                 right = map_instance_to_supertype(right, abstract_set)
-                return self.dangerous_comparison(left.args[0], right.args[0])
+                return self.dangerous_comparison(
+                    left.args[0], right.args[0], seen_types=seen_types
+                )
             elif left.type.has_base("typing.Mapping") and right.type.has_base("typing.Mapping"):
                 # Similar to above: Mapping ignores the classes, it just compares items.
                 abstract_map = self.chk.lookup_typeinfo("typing.Mapping")
                 left = map_instance_to_supertype(left, abstract_map)
                 right = map_instance_to_supertype(right, abstract_map)
                 return self.dangerous_comparison(
-                    left.args[0], right.args[0]
-                ) or self.dangerous_comparison(left.args[1], right.args[1])
+                    left.args[0], right.args[0], seen_types=seen_types
+                ) or self.dangerous_comparison(left.args[1], right.args[1], seen_types=seen_types)
             elif left_name in ("builtins.list", "builtins.tuple") and right_name == left_name:
-                return self.dangerous_comparison(left.args[0], right.args[0])
+                return self.dangerous_comparison(
+                    left.args[0], right.args[0], seen_types=seen_types
+                )
             elif left_name in OVERLAPPING_BYTES_ALLOWLIST and right_name in (
                 OVERLAPPING_BYTES_ALLOWLIST
             ):
@@ -4902,7 +4913,7 @@ def tuple_context_matches(self, expr: TupleExpr, ctx: TupleType) -> bool:
             return len([e for e in expr.items if not isinstance(e, StarExpr)]) <= len(ctx.items)
         # For variadic context, the only easy case is when structure matches exactly.
         # TODO: try using tuple type context in more cases.
-        if len([e for e in expr.items if not isinstance(e, StarExpr)]) != 1:
+        if len([e for e in expr.items if isinstance(e, StarExpr)]) != 1:
             return False
         expr_star_index = next(i for i, lv in enumerate(expr.items) if isinstance(lv, StarExpr))
         return len(expr.items) == len(ctx.items) and ctx_unpack_index == expr_star_index
@@ -4941,6 +4952,9 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type:
         if type_context_items is not None:
             unpack_in_context = find_unpack_in_list(type_context_items) is not None
         seen_unpack_in_items = False
+        allow_precise_tuples = (
+            unpack_in_context or PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature
+        )
 
         # Infer item types.  Give up if there's a star expression
         # that's not a Tuple.
@@ -4981,10 +4995,7 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type:
                         # result in an error later, just do something predictable here.
                         j += len(tt.items)
                 else:
-                    if (
-                        PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature
-                        and not seen_unpack_in_items
-                    ):
+                    if allow_precise_tuples and not seen_unpack_in_items:
                         # Handle (x, *y, z), where y is e.g. tuple[Y, ...].
                         if isinstance(tt, Instance) and self.chk.type_is_iterable(tt):
                             item_type = self.chk.iterable_item_type(tt, e)

mypy/constraints.py

@@ -949,7 +949,7 @@ def visit_instance(self, template: Instance) -> list[Constraint]:
             for item in actual.items:
                 if isinstance(item, UnpackType):
                     unpacked = get_proper_type(item.type)
-                    if isinstance(unpacked, TypeVarType):
+                    if isinstance(unpacked, TypeVarTupleType):
                         # Cannot infer anything for T from [T, ...] <: *Ts
                         continue
                     assert (