diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 361f4054856d89..4107907fd0ab43 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -153,6 +153,19 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | f_trace | tracing function for this | | | | frame, or ``None`` | +-----------------+-------------------+---------------------------+ +| | f_trace_lines | indicate whether a | +| | | tracing event is | +| | | triggered for each source | +| | | source line | ++-----------------+-------------------+---------------------------+ +| | f_trace_opcodes | indicate whether | +| | | per-opcode events are | +| | | requested | ++-----------------+-------------------+---------------------------+ +| | clear() | used to clear all | +| | | references to local | +| | | variables | ++-----------------+-------------------+---------------------------+ | code | co_argcount | number of arguments (not | | | | including keyword only | | | | arguments, \* or \*\* | @@ -214,6 +227,18 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | arguments and local | | | | variables | +-----------------+-------------------+---------------------------+ +| | co_lines() | returns an iterator that | +| | | yields successive | +| | | bytecode ranges | ++-----------------+-------------------+---------------------------+ +| | co_positions() | returns an iterator of | +| | | source code positions for | +| | | each bytecode instruction | ++-----------------+-------------------+---------------------------+ +| | replace() | returns a copy of the | +| | | code object with new | +| | | values | ++-----------------+-------------------+---------------------------+ | generator | __name__ | name | +-----------------+-------------------+---------------------------+ | | __qualname__ | qualified name | diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 067b34c12c4e7f..b16084aaa15515 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -21,6 +21,9 @@ extern "C" { struct _parser_runtime_state { #ifdef Py_DEBUG long memo_statistics[_PYPEGEN_NSTATISTICS]; +#ifdef Py_GIL_DISABLED + PyMutex mutex; +#endif #else int _not_used; #endif @@ -28,8 +31,10 @@ struct _parser_runtime_state { }; _Py_DECLARE_STR(empty, "") +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) #define _parser_runtime_state_INIT \ { \ + .mutex = {0}, \ .dummy_name = { \ .kind = Name_kind, \ .v.Name.id = &_Py_STR(empty), \ @@ -40,6 +45,20 @@ _Py_DECLARE_STR(empty, "") .end_col_offset = 0, \ }, \ } +#else +#define _parser_runtime_state_INIT \ + { \ + .dummy_name = { \ + .kind = Name_kind, \ + .v.Name.id = &_Py_STR(empty), \ + .v.Name.ctx = Load, \ + .lineno = 1, \ + .col_offset = 0, \ + .end_lineno = 1, \ + .end_col_offset = 0, \ + }, \ + } +#endif extern struct _mod* _PyParser_ASTFromString( const char *str, diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 590e85ea8c2db1..d5d13ec6a48e14 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -1,19 +1,20 @@ import io import sys - +from threading import RLock +from time import sleep, time # The maximum length of a log message in bytes, including the level marker and -# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in -# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages -# longer than this will be be truncated by logcat. This limit has already been -# reduced at least once in the history of Android (from 4076 to 4068 between API -# level 23 and 26), so leave some headroom. +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. MAX_BYTES_PER_WRITE = 4000 # UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this -# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE. +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. # However, if the actual number of bytes per character is smaller than that, -# then TextIOWrapper may still join multiple consecutive text writes into binary +# then we may still join multiple consecutive text writes into binary # writes containing a larger number of characters. MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 @@ -26,18 +27,22 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): if sys.executable: return # Not embedded in an app. + global logcat + logcat = Logcat(android_log_write) + sys.stdout = TextLogStream( - android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors) + stdout_prio, "python.stdout", errors=sys.stdout.errors) sys.stderr = TextLogStream( - android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors) + stderr_prio, "python.stderr", errors=sys.stderr.errors) class TextLogStream(io.TextIOWrapper): - def __init__(self, android_log_write, prio, tag, **kwargs): + def __init__(self, prio, tag, **kwargs): kwargs.setdefault("encoding", "UTF-8") - kwargs.setdefault("line_buffering", True) - super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs) - self._CHUNK_SIZE = MAX_BYTES_PER_WRITE + super().__init__(BinaryLogStream(prio, tag), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 def __repr__(self): return f"" @@ -52,19 +57,48 @@ def write(self, s): s = str.__str__(s) # We want to emit one log message per line wherever possible, so split - # the string before sending it to the superclass. Note that - # "".splitlines() == [], so nothing will be logged for an empty string. - for line in s.splitlines(keepends=True): - while line: - super().write(line[:MAX_CHARS_PER_WRITE]) - line = line[MAX_CHARS_PER_WRITE:] + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) return len(s) + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + class BinaryLogStream(io.RawIOBase): - def __init__(self, android_log_write, prio, tag): - self.android_log_write = android_log_write + def __init__(self, prio, tag): self.prio = prio self.tag = tag @@ -85,10 +119,48 @@ def write(self, b): # Writing an empty string to the stream should have no effect. if b: - # Encode null bytes using "modified UTF-8" to avoid truncating the - # message. This should not affect the return value, as the caller - # may be expecting it to match the length of the input. - self.android_log_write(self.prio, self.tag, - b.replace(b"\x00", b"\xc0\x80")) - + logcat.write(self.prio, self.tag, b) return len(b) + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + self._bucket_level = min(self._bucket_level, BUCKET_SIZE) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag, message) diff --git a/Lib/inspect.py b/Lib/inspect.py index ba3ecbb87c7026..1f287842e32498 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -264,11 +264,16 @@ def isfunction(object): Function objects provide these attributes: __doc__ documentation string __name__ name with which this function was defined + __qualname__ qualified name of this function + __module__ name of the module the function was defined in or None __code__ code object containing compiled function bytecode __defaults__ tuple of any default values for arguments __globals__ global namespace in which this function was defined __annotations__ dict of parameter annotations - __kwdefaults__ dict of keyword only parameters with defaults""" + __kwdefaults__ dict of keyword only parameters with defaults + __dict__ namespace which is supporting arbitrary function attributes + __closure__ a tuple of cells or None + __type_params__ tuple of type parameters""" return isinstance(object, types.FunctionType) def _has_code_flag(f, flag): @@ -333,17 +338,18 @@ def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support iteration over container - close raises a new GeneratorExit exception inside the - generator to terminate the iteration gi_code code object gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise - next return the next item from the container - send resumes the generator and "sends" a value that becomes + gi_yieldfrom object being iterated by yield from or None + + __iter__() defined to support iteration over container + close() raises a new GeneratorExit exception inside the + generator to terminate the iteration + send() resumes the generator and "sends" a value that becomes the result of the current yield-expression - throw used to raise an exception inside the generator""" + throw() used to raise an exception inside the generator""" return isinstance(object, types.GeneratorType) def iscoroutine(object): @@ -378,7 +384,11 @@ def isframe(object): f_lasti index of last attempted instruction in bytecode f_lineno current line number in Python source code f_locals local namespace seen by this frame - f_trace tracing function for this frame, or None""" + f_trace tracing function for this frame, or None + f_trace_lines is a tracing event triggered for each source line? + f_trace_opcodes are per-opcode events being requested? + + clear() used to clear all references to local variables""" return isinstance(object, types.FrameType) def iscode(object): @@ -403,7 +413,12 @@ def iscode(object): co_names tuple of names other than arguments and function locals co_nlocals number of local variables co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_varnames tuple of names of arguments and local variables + co_qualname fully qualified function name + + co_lines() returns an iterator that yields successive bytecode ranges + co_positions() returns an iterator of source code positions for each bytecode instruction + replace() returns a copy of the code object with a new values""" return isinstance(object, types.CodeType) def isbuiltin(object): diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py index 06507d08bee02b..fed2d84fc01473 100644 --- a/Lib/re/_casefix.py +++ b/Lib/re/_casefix.py @@ -1,4 +1,4 @@ -# Auto-generated by Tools/scripts/generate_re_casefix.py. +# Auto-generated by Tools/build/generate_re_casefix.py. # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 115882a4c281f6..82035061bb6fdd 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -1,14 +1,17 @@ +import io import platform import queue import re import subprocess import sys import unittest +from _android_support import TextLogStream from array import array -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager from threading import Thread from test.support import LOOPBACK_TIMEOUT -from time import time +from time import sleep, time +from unittest.mock import patch if sys.platform != "android": @@ -81,18 +84,39 @@ def unbuffered(self, stream): finally: stream.reconfigure(write_through=False) + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. + def stream_context(self, stream_name, level): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] + + stack = ExitStack() + stack.enter_context(self.subTest(stream_name)) + stream = getattr(sys, stream_name) + if isinstance(stream, io.StringIO): + stack.enter_context( + patch( + f"sys.{stream_name}", + TextLogStream( + prio, f"python.{stream_name}", errors="backslashreplace" + ), + ) + ) + return stack + def test_str(self): for stream_name, level in [("stdout", "I"), ("stderr", "W")]: - with self.subTest(stream=stream_name): + with self.stream_context(stream_name, level): stream = getattr(sys, stream_name) tag = f"python.{stream_name}" self.assertEqual(f"", repr(stream)) - self.assertTrue(stream.writable()) - self.assertFalse(stream.readable()) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) self.assertEqual("UTF-8", stream.encoding) - self.assertTrue(stream.line_buffering) - self.assertFalse(stream.write_through) + self.assertIs(stream.line_buffering, True) + self.assertIs(stream.write_through, False) # stderr is backslashreplace by default; stdout is configured # that way by libregrtest.main. @@ -147,6 +171,13 @@ def write(s, lines=None, *, write_len=None): write("f\n\ng", ["exxf", ""]) write("\n", ["g"]) + # Since this is a line-based logging system, line buffering + # cannot be turned off, i.e. a newline always causes a flush. + stream.reconfigure(line_buffering=False) + self.assertIs(stream.line_buffering, True) + + # However, buffering can be turned off completely if you want a + # flush after every write. with self.unbuffered(stream): write("\nx", ["", "x"]) write("\na\n", ["", "a"]) @@ -209,30 +240,30 @@ def __str__(self): # (MAX_BYTES_PER_WRITE). # # ASCII (1 byte per character) - write(("foobar" * 700) + "\n", - [("foobar" * 666) + "foob", # 4000 bytes - "ar" + ("foobar" * 33)]) # 200 bytes + write(("foobar" * 700) + "\n", # 4200 bytes in + [("foobar" * 666) + "foob", # 4000 bytes out + "ar" + ("foobar" * 33)]) # 200 bytes out # "Full-width" digits 0-9 (3 bytes per character) s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" - write((s * 150) + "\n", - [s * 100, # 3000 bytes - s * 50]) # 1500 bytes + write((s * 150) + "\n", # 4500 bytes in + [s * 100, # 3000 bytes out + s * 50]) # 1500 bytes out s = "0123456789" - write(s * 200, []) - write(s * 150, []) - write(s * 51, [s * 350]) # 3500 bytes - write("\n", [s * 51]) # 510 bytes + write(s * 200, []) # 2000 bytes in + write(s * 150, []) # 1500 bytes in + write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out + write("\n", [s * 51]) # 0 bytes in, 510 bytes out def test_bytes(self): for stream_name, level in [("stdout", "I"), ("stderr", "W")]: - with self.subTest(stream=stream_name): + with self.stream_context(stream_name, level): stream = getattr(sys, stream_name).buffer tag = f"python.{stream_name}" self.assertEqual(f"", repr(stream)) - self.assertTrue(stream.writable()) - self.assertFalse(stream.readable()) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) def write(b, lines=None, *, write_len=None): if write_len is None: @@ -330,3 +361,64 @@ def write(b, lines=None, *, write_len=None): fr"{type(obj).__name__}" ): stream.write(obj) + + def test_rate_limit(self): + # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 + PER_MESSAGE_OVERHEAD = 28 + + # https://developer.android.com/ndk/reference/group/logging + ANDROID_LOG_DEBUG = 3 + + # To avoid flooding the test script output, use a different tag rather + # than stdout or stderr. + tag = "python.rate_limit" + stream = TextLogStream(ANDROID_LOG_DEBUG, tag) + + # Make a test message which consumes 1 KB of the logcat buffer. + message = "Line {:03d} " + message += "." * ( + 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) + ) + "\n" + + # See _android_support.py. The default values of these parameters work + # well across a wide range of devices, but we'll use smaller values to + # ensure a quick and reliable test that doesn't flood the log too much. + MAX_KB_PER_SECOND = 100 + BUCKET_KB = 10 + with ( + patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), + patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + ): + # Make sure the token bucket is full. + sleep(BUCKET_KB / MAX_KB_PER_SECOND) + line_num = 0 + + # Write BUCKET_KB messages, and return the rate at which they were + # accepted in KB per second. + def write_bucketful(): + nonlocal line_num + start = time() + max_line_num = line_num + BUCKET_KB + while line_num < max_line_num: + stream.write(message.format(line_num)) + line_num += 1 + return BUCKET_KB / (time() - start) + + # The first bucketful should be written with minimal delay. The + # factor of 2 here is not arbitrary: it verifies that the system can + # write fast enough to empty the bucket within two bucketfuls, which + # the next part of the test depends on. + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + # Write another bucketful to empty the token bucket completely. + write_bucketful() + + # The next bucketful should be written at the rate limit. + self.assertAlmostEqual( + write_bucketful(), MAX_KB_PER_SECOND, + delta=MAX_KB_PER_SECOND * 0.1 + ) + + # Once the token bucket refills, we should go back to full speed. + sleep(BUCKET_KB / MAX_KB_PER_SECOND) + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-08-05-19-04-06.gh-issue-116622.3LWUzE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-08-05-19-04-06.gh-issue-116622.3LWUzE.rst new file mode 100644 index 00000000000000..9320928477af2c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-08-05-19-04-06.gh-issue-116622.3LWUzE.rst @@ -0,0 +1 @@ +Fix Android stdout and stderr messages being truncated or lost. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 5d9ee8cb6c679d..dc0ef0a184d205 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1962,9 +1962,11 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * PyErr_Format(st->PicklingError, "Can't pickle %R: import of module %R failed", global, module_name); + Py_DECREF(module_name); return NULL; } if (check_dotted_path(module, global_name, dotted_path) < 0) { + Py_DECREF(module_name); Py_DECREF(module); return NULL; } @@ -1974,6 +1976,7 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * PyErr_Format(st->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", global, global_name, module_name); + Py_DECREF(module_name); return NULL; } if (actual != global) { @@ -1981,6 +1984,7 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * PyErr_Format(st->PicklingError, "Can't pickle %R: it's not the same object as %S.%S", global, module_name, global_name); + Py_DECREF(module_name); return NULL; } Py_DECREF(actual); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a8be7d75371c16..4e77780eb39097 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1721,7 +1721,7 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(clear__doc__, -"F.clear(): clear most references held by the frame"); +"F.clear(): clear all references held by the frame"); static PyObject * frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) diff --git a/Parser/pegen.c b/Parser/pegen.c index ac428be0958bdf..0c3c4689dd7ce6 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -296,12 +296,22 @@ _PyPegen_fill_token(Parser *p) #define NSTATISTICS _PYPEGEN_NSTATISTICS #define memo_statistics _PyRuntime.parser.memo_statistics +#ifdef Py_GIL_DISABLED +#define MUTEX_LOCK() PyMutex_Lock(&_PyRuntime.parser.mutex) +#define MUTEX_UNLOCK() PyMutex_Unlock(&_PyRuntime.parser.mutex) +#else +#define MUTEX_LOCK() +#define MUTEX_UNLOCK() +#endif + void _PyPegen_clear_memo_statistics(void) { + MUTEX_LOCK(); for (int i = 0; i < NSTATISTICS; i++) { memo_statistics[i] = 0; } + MUTEX_UNLOCK(); } PyObject * @@ -311,18 +321,23 @@ _PyPegen_get_memo_statistics(void) if (ret == NULL) { return NULL; } + + MUTEX_LOCK(); for (int i = 0; i < NSTATISTICS; i++) { PyObject *value = PyLong_FromLong(memo_statistics[i]); if (value == NULL) { + MUTEX_UNLOCK(); Py_DECREF(ret); return NULL; } // PyList_SetItem borrows a reference to value. if (PyList_SetItem(ret, i, value) < 0) { + MUTEX_UNLOCK(); Py_DECREF(ret); return NULL; } } + MUTEX_UNLOCK(); return ret; } #endif @@ -348,7 +363,9 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) if (count <= 0) { count = 1; } + MUTEX_LOCK(); memo_statistics[type] += count; + MUTEX_UNLOCK(); } #endif p->mark = m->mark; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 996f997d0ca8de..a8527fe84b76f5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -287,7 +287,8 @@ dummy_func( /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { ERROR_NO_POP(); } } @@ -302,7 +303,8 @@ dummy_func( tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { ERROR_NO_POP(); } } @@ -1069,11 +1071,12 @@ dummy_func( PyStackRef_AsPyObjectBorrow(v)); } if (retval_o == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (matches) { _PyEval_MonitorRaise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + int err = _PyGen_FetchStopIterationValue(&retval_o); + if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); } @@ -1210,7 +1213,8 @@ dummy_func( assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); none = PyStackRef_None; @@ -1425,7 +1429,8 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; - if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + if (err < 0) { ERROR_NO_POP(); } if (v_o == NULL) { @@ -1596,7 +1601,8 @@ dummy_func( assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + if (err < 0) { ERROR_NO_POP(); } if (!value_o) { @@ -1676,7 +1682,8 @@ dummy_func( PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); @@ -1762,8 +1769,10 @@ dummy_func( PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (PyDict_Update(dict_o, update_o) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + int err = PyDict_Update(dict_o, update_o); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); @@ -1779,7 +1788,8 @@ dummy_func( PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + int err = _PyDict_MergeEx(dict_o, update_o, 2); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); DECREF_INPUTS(); ERROR_IF(true, error); @@ -1943,7 +1953,8 @@ dummy_func( if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; - if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -2416,8 +2427,8 @@ dummy_func( inst(CHECK_EG_MATCH, (exc_value_st, match_type_st -- rest, match)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + if (err < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } @@ -2704,7 +2715,8 @@ dummy_func( if (next_o == NULL) { next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { ERROR_NO_POP(); } _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -2729,7 +2741,8 @@ dummy_func( PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { ERROR_NO_POP(); } _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); @@ -2756,7 +2769,8 @@ dummy_func( } else { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { ERROR_NO_POP(); } _PyEval_MonitorRaise(tstate, frame, this_instr); diff --git a/Python/compile.c b/Python/compile.c index 87b2c2705474a4..9695a99d201144 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -505,21 +505,35 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) deterministic, then the generated bytecode is not deterministic. */ sorted_keys = PyDict_Keys(src); - if (sorted_keys == NULL) + if (sorted_keys == NULL) { + Py_DECREF(dest); return NULL; + } if (PyList_Sort(sorted_keys) != 0) { Py_DECREF(sorted_keys); + Py_DECREF(dest); return NULL; } num_keys = PyList_GET_SIZE(sorted_keys); for (key_i = 0; key_i < num_keys; key_i++) { - /* XXX this should probably be a macro in symtable.h */ - long vi; k = PyList_GET_ITEM(sorted_keys, key_i); v = PyDict_GetItemWithError(src, k); - assert(v && PyLong_Check(v)); - vi = PyLong_AS_LONG(v); + if (!v) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, k); + } + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + long vi = PyLong_AsLong(v); + if (vi == -1 && PyErr_Occurred()) { + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + /* XXX this should probably be a macro in symtable.h */ scope = (vi >> SCOPE_OFFSET) & SCOPE_MASK; if (scope == scope_type || vi & flag) { @@ -631,6 +645,7 @@ compiler_set_qualname(struct compiler *c) scope = _PyST_GetScope(parent->u_ste, mangled); Py_DECREF(mangled); + RETURN_IF_ERROR(scope); assert(scope != GLOBAL_IMPLICIT); if (scope == GLOBAL_EXPLICIT) force_global = 1; @@ -1648,7 +1663,7 @@ dict_lookup_arg(PyObject *dict, PyObject *name) if (v == NULL) { return ERROR; } - return PyLong_AS_LONG(v); + return PyLong_AsLong(v); } static int @@ -1671,7 +1686,7 @@ compiler_lookup_arg(struct compiler *c, PyCodeObject *co, PyObject *name) else { arg = dict_lookup_arg(c->u->u_metadata.u_freevars, name); } - if (arg == -1) { + if (arg == -1 && !PyErr_Occurred()) { PyObject *freevars = _PyCode_GetFreevars(co); if (freevars == NULL) { PyErr_Clear(); @@ -4085,6 +4100,8 @@ compiler_nameop(struct compiler *c, location loc, case GLOBAL_EXPLICIT: optype = OP_GLOBAL; break; + case -1: + goto error; default: /* scope can be 0 */ break; @@ -4638,6 +4655,7 @@ is_import_originated(struct compiler *c, expr_ty e) } long flags = _PyST_GetSymbol(SYMTABLE(c)->st_top, e->v.Name.id); + RETURN_IF_ERROR(flags); return flags & DEF_IMPORT; } @@ -4657,10 +4675,12 @@ can_optimize_super_call(struct compiler *c, expr_ty attr) PyObject *super_name = e->v.Call.func->v.Name.id; // detect statically-visible shadowing of 'super' name int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), super_name); + RETURN_IF_ERROR(scope); if (scope != GLOBAL_IMPLICIT) { return 0; } scope = _PyST_GetScope(SYMTABLE(c)->st_top, super_name); + RETURN_IF_ERROR(scope); if (scope != 0) { return 0; } @@ -4767,7 +4787,9 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) } /* Check that the base object is not something that is imported */ - if (is_import_originated(c, meth->v.Attribute.value)) { + int ret = is_import_originated(c, meth->v.Attribute.value); + RETURN_IF_ERROR(ret); + if (ret) { return 0; } @@ -4795,7 +4817,9 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) /* Alright, we can optimize the code. */ location loc = LOC(meth); - if (can_optimize_super_call(c, meth)) { + ret = can_optimize_super_call(c, meth); + RETURN_IF_ERROR(ret); + if (ret) { RETURN_IF_ERROR(load_args_for_super(c, meth->v.Attribute.value)); int opcode = asdl_seq_LEN(meth->v.Attribute.value->v.Call.args) ? LOAD_SUPER_METHOD : LOAD_ZERO_SUPER_METHOD; @@ -5367,8 +5391,10 @@ push_inlined_comprehension_state(struct compiler *c, location loc, PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { - assert(PyLong_Check(v)); - long symbol = PyLong_AS_LONG(v); + long symbol = PyLong_AsLong(v); + if (symbol == -1 && PyErr_Occurred()) { + return ERROR; + } long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; PyObject *outv = PyDict_GetItemWithError(SYMTABLE_ENTRY(c)->ste_symbols, k); if (outv == NULL) { @@ -5377,8 +5403,11 @@ push_inlined_comprehension_state(struct compiler *c, location loc, } outv = _PyLong_GetZero(); } - assert(PyLong_CheckExact(outv)); - long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; + long outsymbol = PyLong_AsLong(outv); + if (outsymbol == -1 && PyErr_Occurred()) { + return ERROR; + } + long outsc = (outsymbol >> SCOPE_OFFSET) & SCOPE_MASK; // If a name has different scope inside than outside the comprehension, // we need to temporarily handle it with the right scope while // compiling the comprehension. If it's free in the comprehension @@ -6064,14 +6093,18 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_formatted_value(c, e); /* The following exprs can be assignment targets. */ case Attribute_kind: - if (e->v.Attribute.ctx == Load && can_optimize_super_call(c, e)) { - RETURN_IF_ERROR(load_args_for_super(c, e->v.Attribute.value)); - int opcode = asdl_seq_LEN(e->v.Attribute.value->v.Call.args) ? - LOAD_SUPER_ATTR : LOAD_ZERO_SUPER_ATTR; - ADDOP_NAME(c, loc, opcode, e->v.Attribute.attr, names); - loc = update_start_location_to_match_attr(c, loc, e); - ADDOP(c, loc, NOP); - return SUCCESS; + if (e->v.Attribute.ctx == Load) { + int ret = can_optimize_super_call(c, e); + RETURN_IF_ERROR(ret); + if (ret) { + RETURN_IF_ERROR(load_args_for_super(c, e->v.Attribute.value)); + int opcode = asdl_seq_LEN(e->v.Attribute.value->v.Call.args) ? + LOAD_SUPER_ATTR : LOAD_ZERO_SUPER_ATTR; + ADDOP_NAME(c, loc, opcode, e->v.Attribute.attr, names); + loc = update_start_location_to_match_attr(c, loc, e); + ADDOP(c, loc, NOP); + return SUCCESS; + } } RETURN_IF_ERROR(compiler_maybe_add_static_attribute_to_class(c, e)); VISIT(c, expr, e->v.Attribute.value); @@ -7300,7 +7333,8 @@ consts_dict_keys_inorder(PyObject *dict) if (consts == NULL) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyLong_AS_LONG(v); + assert(PyLong_CheckExact(v)); + i = PyLong_AsLong(v); /* The keys of the dictionary can be tuples wrapping a constant. * (see dict_add_o and _PyCode_ConstantKey). In that case * the object we want is always second. */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index cbee77d5cf67fc..7f520eb7abbe80 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1777,7 +1777,8 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + if (err < 0) { JUMP_TO_ERROR(); } if (!value_o) { @@ -1907,7 +1908,8 @@ PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); @@ -2031,8 +2033,10 @@ dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (PyDict_Update(dict_o, update_o) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + int err = PyDict_Update(dict_o, update_o); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); @@ -2057,7 +2061,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + int err = _PyDict_MergeEx(dict_o, update_o, 2); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); PyStackRef_CLOSE(update); if (true) JUMP_TO_ERROR(); @@ -2182,7 +2187,8 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; - if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -2855,7 +2861,8 @@ exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (true) JUMP_TO_ERROR(); @@ -3101,7 +3108,8 @@ PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { JUMP_TO_ERROR(); } _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 879c40ab0cb6ba..31490960d3828a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2465,7 +2465,8 @@ exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (true) goto pop_2_error; @@ -2528,7 +2529,8 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); PyStackRef_CLOSE(sub_iter_st); PyStackRef_CLOSE(last_sent_val_st); @@ -2982,7 +2984,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + int err = _PyDict_MergeEx(dict_o, update_o, 2); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); PyStackRef_CLOSE(update); if (true) goto pop_1_error; @@ -3003,8 +3006,10 @@ dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - if (PyDict_Update(dict_o, update_o) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + int err = PyDict_Update(dict_o, update_o); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); @@ -3208,7 +3213,8 @@ if (next_o == NULL) { next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { goto error; } _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -3786,7 +3792,8 @@ /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { goto error; } } @@ -3807,7 +3814,8 @@ receiver = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + if (err) { goto error; } } @@ -3834,7 +3842,8 @@ } else { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { goto error; } _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4327,7 +4336,8 @@ PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); @@ -4379,7 +4389,8 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; - if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -5074,7 +5085,8 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + if (err < 0) { goto error; } if (!value_o) { @@ -5100,7 +5112,8 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; - if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + if (err < 0) { goto error; } if (v_o == NULL) { @@ -6080,11 +6093,12 @@ PyStackRef_AsPyObjectBorrow(v)); } if (retval_o == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (matches) { _PyEval_MonitorRaise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + int err = _PyGen_FetchStopIterationValue(&retval_o); + if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); } diff --git a/Python/symtable.c b/Python/symtable.c index ef81a0799de3aa..4acf762f8fca39 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -526,17 +526,31 @@ _PySymtable_LookupOptional(struct symtable *st, void *key, long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { - PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name); - if (!v) + PyObject *v; + if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { + return -1; + } + if (!v) { return 0; - assert(PyLong_Check(v)); - return PyLong_AS_LONG(v); + } + long symbol = PyLong_AsLong(v); + Py_DECREF(v); + if (symbol < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "invalid symbol"); + } + return -1; + } + return symbol; } int _PyST_GetScope(PySTEntryObject *ste, PyObject *name) { long symbol = _PyST_GetSymbol(ste, name); + if (symbol < 0) { + return -1; + } return (symbol >> SCOPE_OFFSET) & SCOPE_MASK; } @@ -715,11 +729,14 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, // global statement), we want to also treat it as a global in this scope. if (class_entry != NULL) { long class_flags = _PyST_GetSymbol(class_entry, name); + if (class_flags < 0) { + return 0; + } if (class_flags & DEF_GLOBAL) { SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); return 1; } - else if (class_flags & DEF_BOUND && !(class_flags & DEF_NONLOCAL)) { + else if ((class_flags & DEF_BOUND) && !(class_flags & DEF_NONLOCAL)) { SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); return 1; } @@ -763,6 +780,9 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) PySTEntryObject *child_ste = (PySTEntryObject *)PyList_GET_ITEM( entry->ste_children, i); long scope = _PyST_GetScope(child_ste, key); + if (scope < 0) { + return -1; + } if (scope == FREE) { return 1; } @@ -781,7 +801,10 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter - long comp_flags = PyLong_AS_LONG(v); + long comp_flags = PyLong_AsLong(v); + if (comp_flags == -1 && PyErr_Occurred()) { + return 0; + } if (comp_flags & DEF_PARAM) { assert(_PyUnicode_EqualToASCIIString(k, ".0")); continue; @@ -822,11 +845,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, SET_SCOPE(scopes, k, scope); } else { - if (PyLong_AsLong(existing) & DEF_BOUND) { + long flags = PyLong_AsLong(existing); + if (flags == -1 && PyErr_Occurred()) { + return 0; + } + if ((flags & DEF_BOUND) && ste->ste_type != ClassBlock) { // free vars in comprehension that are locals in outer scope can // now simply be locals, unless they are free in comp children, // or if the outer scope is a class block - if (!is_free_in_any_child(comp, k) && ste->ste_type != ClassBlock) { + int ok = is_free_in_any_child(comp, k); + if (ok < 0) { + return 0; + } + if (!ok) { if (PySet_Discard(comp_free, k) < 0) { return 0; } @@ -861,9 +892,10 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) if (!v_cell) return 0; while (PyDict_Next(scopes, &pos, &name, &v)) { - long scope; - assert(PyLong_Check(v)); - scope = PyLong_AS_LONG(v); + long scope = PyLong_AsLong(v); + if (scope == -1 && PyErr_Occurred()) { + goto error; + } if (scope != LOCAL) continue; int contains = PySet_Contains(free, name); @@ -926,9 +958,10 @@ update_symbols(PyObject *symbols, PyObject *scopes, /* Update scope information for all symbols in this scope */ while (PyDict_Next(symbols, &pos, &name, &v)) { - long scope, flags; - assert(PyLong_Check(v)); - flags = PyLong_AS_LONG(v); + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + return 0; + } int contains = PySet_Contains(inlined_cells, name); if (contains < 0) { return 0; @@ -936,9 +969,18 @@ update_symbols(PyObject *symbols, PyObject *scopes, if (contains) { flags |= DEF_COMP_CELL; } - v_scope = PyDict_GetItemWithError(scopes, name); - assert(v_scope && PyLong_Check(v_scope)); - scope = PyLong_AS_LONG(v_scope); + if (PyDict_GetItemRef(scopes, name, &v_scope) < 0) { + return 0; + } + if (!v_scope) { + PyErr_SetObject(PyExc_KeyError, name); + return 0; + } + long scope = PyLong_AsLong(v_scope); + Py_DECREF(v_scope); + if (scope == -1 && PyErr_Occurred()) { + return 0; + } flags |= (scope << SCOPE_OFFSET); v_new = PyLong_FromLong(flags); if (!v_new) @@ -971,7 +1013,11 @@ update_symbols(PyObject *symbols, PyObject *scopes, or global in the class scope. */ if (classflag) { - long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS; + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + goto error; + } + flags |= DEF_FREE_CLASS; v_new = PyLong_FromLong(flags); if (!v_new) { goto error; @@ -1110,7 +1156,10 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { - long flags = PyLong_AS_LONG(v); + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + goto error; + } if (!analyze_name(ste, scopes, name, flags, bound, local, free, global, type_params, class_entry)) goto error; @@ -1395,9 +1444,12 @@ symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name); if (!mangled) - return 0; + return -1; long ret = _PyST_GetSymbol(ste, mangled); Py_DECREF(mangled); + if (ret < 0) { + return -1; + } return ret; } @@ -1420,7 +1472,10 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s return 0; dict = ste->ste_symbols; if ((o = PyDict_GetItemWithError(dict, mangled))) { - val = PyLong_AS_LONG(o); + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) { + goto error; + } if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); @@ -1466,16 +1521,20 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s if (flag & DEF_PARAM) { if (PyList_Append(ste->ste_varnames, mangled) < 0) goto error; - } else if (flag & DEF_GLOBAL) { + } else if (flag & DEF_GLOBAL) { /* XXX need to update DEF_GLOBAL for other flags too; perhaps only DEF_FREE_GLOBAL */ - val = flag; + val = 0; if ((o = PyDict_GetItemWithError(st->st_global, mangled))) { - val |= PyLong_AS_LONG(o); + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) { + goto error; + } } else if (PyErr_Occurred()) { goto error; } + val |= flag; o = PyLong_FromLong(val); if (o == NULL) goto error; @@ -2176,6 +2235,9 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) */ if (ste->ste_comprehension) { long target_in_scope = symtable_lookup_entry(st, ste, target_name); + if (target_in_scope < 0) { + return 0; + } if ((target_in_scope & DEF_COMP_ITER) && (target_in_scope & DEF_LOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); @@ -2188,6 +2250,9 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */ if (ste->ste_type == FunctionBlock) { long target_in_scope = symtable_lookup_entry(st, ste, target_name); + if (target_in_scope < 0) { + return 0; + } if (target_in_scope & DEF_GLOBAL) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) return 0; @@ -2601,9 +2666,6 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) { Py_ssize_t i; - if (!args) - return -1; - for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); if (!symtable_add_def(st, arg->arg, DEF_PARAM, LOCATION(arg))) @@ -2650,9 +2712,6 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) { Py_ssize_t i; - if (!args) - return -1; - for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); if (arg->annotation) { diff --git a/Tools/build/generate_re_casefix.py b/Tools/build/generate_re_casefix.py index b57ac07426c27c..6cebfbd025c58c 100755 --- a/Tools/build/generate_re_casefix.py +++ b/Tools/build/generate_re_casefix.py @@ -23,9 +23,9 @@ def update_file(file, content): # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. -_EXTRA_CASES = { +_EXTRA_CASES = {{ %s -} +}} """ def uname(i): diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index ab8c99f1e25f97..2a339f8cd6bb66 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -57,169 +57,171 @@ def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: parens -= 1 out.emit(tkn) +ReplacementFunctionType = Callable[ + [Token, Iterator[Token], Uop, Stack, Instruction | None], None +] -def replace_deopt( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - unused: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("DEOPT_IF", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - out.emit(", ") - assert inst is not None - assert inst.family is not None - out.emit(inst.family.name) - out.emit(");\n") - +class Emitter: -def replace_error( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "COMMA") - label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit(") ") - c_offset = stack.peek_offset() - try: - offset = -int(c_offset) - except ValueError: - offset = -1 - if offset > 0: - out.emit(f"goto pop_{offset}_") - out.emit(label) - out.emit(";\n") - elif offset == 0: - out.emit("goto ") - out.emit(label) - out.emit(";\n") - else: - out.emit("{\n") - stack.flush_locally(out) - out.emit("goto ") - out.emit(label) - out.emit(";\n") - out.emit("}\n") + out: CWriter + _replacers: dict[str, ReplacementFunctionType] + def __init__(self, out: CWriter): + self._replacers = { + "EXIT_IF": self.exit_if, + "DEOPT_IF": self.deopt_if, + "ERROR_IF": self.error_if, + "ERROR_NO_POP": self.error_no_pop, + "DECREF_INPUTS": self.decref_inputs, + "CHECK_EVAL_BREAKER": self.check_eval_breaker, + "SYNC_SP": self.sync_sp, + } + self.out = out -def replace_error_no_pop( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) # LPAREN - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit_at("goto error;", tkn) + def deopt_if( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, + ) -> None: + self.out.emit_at("DEOPT_IF", tkn) + self.out.emit(next(tkn_iter)) + emit_to(self.out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + self.out.emit(", ") + assert inst is not None + assert inst.family is not None + self.out.emit(inst.family.name) + self.out.emit(");\n") + exit_if = deopt_if -def replace_decrefs( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - out.emit_at("", tkn) - for var in uop.stack.inputs: - if var.name == "unused" or var.name == "null" or var.peek: - continue - if var.size: - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") - out.emit("}\n") - elif var.condition: - if var.condition == "1": - out.emit(f"PyStackRef_CLOSE({var.name});\n") - elif var.condition != "0": - out.emit(f"PyStackRef_XCLOSE({var.name});\n") + def error_if( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + self.out.emit_at("if ", tkn) + self.out.emit(next(tkn_iter)) + emit_to(self.out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + self.out.emit(") ") + c_offset = stack.peek_offset() + try: + offset = -int(c_offset) + except ValueError: + offset = -1 + if offset > 0: + self.out.emit(f"goto pop_{offset}_") + self.out.emit(label) + self.out.emit(";\n") + elif offset == 0: + self.out.emit("goto ") + self.out.emit(label) + self.out.emit(";\n") else: - out.emit(f"PyStackRef_CLOSE({var.name});\n") + self.out.emit("{\n") + stack.flush_locally(self.out) + self.out.emit("goto ") + self.out.emit(label) + self.out.emit(";\n") + self.out.emit("}\n") + def error_no_pop( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + self.out.emit_at("goto error;", tkn) -def replace_sync_sp( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - stack.flush(out) - + def decref_inputs( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + self.out.emit_at("", tkn) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size: + self.out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + self.out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") + self.out.emit("}\n") + elif var.condition: + if var.condition == "1": + self.out.emit(f"PyStackRef_CLOSE({var.name});\n") + elif var.condition != "0": + self.out.emit(f"PyStackRef_XCLOSE({var.name});\n") + else: + self.out.emit(f"PyStackRef_CLOSE({var.name});\n") -def replace_check_eval_breaker( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - if not uop.properties.ends_with_eval_breaker: - out.emit_at("CHECK_EVAL_BREAKER();", tkn) + def sync_sp( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + stack.flush(self.out) -REPLACEMENT_FUNCTIONS = { - "EXIT_IF": replace_deopt, - "DEOPT_IF": replace_deopt, - "ERROR_IF": replace_error, - "ERROR_NO_POP": replace_error_no_pop, - "DECREF_INPUTS": replace_decrefs, - "CHECK_EVAL_BREAKER": replace_check_eval_breaker, - "SYNC_SP": replace_sync_sp, -} - -ReplacementFunctionType = Callable[ - [CWriter, Token, Iterator[Token], Uop, Stack, Instruction | None], None -] + def check_eval_breaker( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + if not uop.properties.ends_with_eval_breaker: + self.out.emit_at("CHECK_EVAL_BREAKER();", tkn) -def emit_tokens( - out: CWriter, - uop: Uop, - stack: Stack, - inst: Instruction | None, - replacement_functions: Mapping[ - str, ReplacementFunctionType - ] = REPLACEMENT_FUNCTIONS, -) -> None: - tkns = uop.body[1:-1] - if not tkns: - return - tkn_iter = iter(tkns) - out.start_line() - for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in replacement_functions: - replacement_functions[tkn.text](out, tkn, tkn_iter, uop, stack, inst) - else: - out.emit(tkn) + def emit_tokens( + self, + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + tkns = uop.body[1:-1] + if not tkns: + return + tkn_iter = iter(tkns) + self.out.start_line() + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in self._replacers: + self._replacers[tkn.text](tkn, tkn_iter, uop, stack, inst) + else: + self.out.emit(tkn) + def emit(self, txt: str | Token) -> None: + self.out.emit(txt) def cflags(p: Properties) -> str: flags: list[str] = [] diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index f6c2fea40f0dbb..e192b76b23319c 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -17,8 +17,7 @@ DEFAULT_INPUT, ROOT, write_header, - emit_tokens, - replace_sync_sp, + Emitter, ) from cwriter import CWriter from typing import TextIO, Iterator @@ -89,6 +88,10 @@ def emit_default(out: CWriter, uop: Uop) -> None: else: out.emit(f"{var.name} = sym_new_not_null(ctx);\n") +class OptimizerEmitter(Emitter): + + pass + def write_uop( override: Uop | None, @@ -126,11 +129,8 @@ def write_uop( cast = f"uint{cache.size*16}_t" out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") if override: - replacement_funcs = { - "DECREF_INPUTS": decref_inputs, - "SYNC_SP": replace_sync_sp, - } - emit_tokens(out, override, stack, None, replacement_funcs) + emitter = OptimizerEmitter(out) + emitter.emit_tokens(override, stack, None) else: emit_default(out, uop) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 1cdafbd35caea3..6c13d1f10b39f9 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -20,8 +20,8 @@ DEFAULT_INPUT, ROOT, write_header, - emit_tokens, type_and_null, + Emitter, ) from cwriter import CWriter from typing import TextIO @@ -62,26 +62,26 @@ def declare_variables(inst: Instruction, out: CWriter) -> None: declare_variable(var, out) def write_uop( - uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool + uop: Part, emitter: Emitter, offset: int, stack: Stack, inst: Instruction, braces: bool ) -> int: # out.emit(stack.as_comment() + "\n") if isinstance(uop, Skip): entries = "entries" if uop.size > 1 else "entry" - out.emit(f"/* Skip {uop.size} cache {entries} */\n") + emitter.emit(f"/* Skip {uop.size} cache {entries} */\n") return offset + uop.size if isinstance(uop, Flush): - out.emit(f"// flush\n") - stack.flush(out) + emitter.emit(f"// flush\n") + stack.flush(emitter.out) return offset try: locals: dict[str, Local] = {} - out.start_line() + emitter.out.start_line() if braces: - out.emit(f"// {uop.name}\n") + emitter.out.emit(f"// {uop.name}\n") peeks: list[Local] = [] for var in reversed(uop.stack.inputs): code, local = stack.pop(var) - out.emit(code) + emitter.emit(code) if var.peek: peeks.append(local) if local.defined: @@ -91,8 +91,8 @@ def write_uop( while peeks: stack.push(peeks.pop()) if braces: - out.emit("{\n") - out.emit(stack.define_output_arrays(uop.stack.outputs)) + emitter.emit("{\n") + emitter.out.emit(stack.define_output_arrays(uop.stack.outputs)) for cache in uop.caches: if cache.name != "unused": @@ -102,13 +102,13 @@ def write_uop( else: type = f"uint{cache.size*16}_t " reader = f"read_u{cache.size*16}" - out.emit( + emitter.emit( f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" ) if inst.family is None: - out.emit(f"(void){cache.name};\n") + emitter.emit(f"(void){cache.name};\n") offset += cache.size - emit_tokens(out, uop, stack, inst) + emitter.emit_tokens(uop, stack, inst) for i, var in enumerate(uop.stack.outputs): if not var.peek: if var.name in locals: @@ -117,11 +117,11 @@ def write_uop( local = Local.unused(var) else: local = Local.local(var) - out.emit(stack.push(local)) + emitter.emit(stack.push(local)) if braces: - out.start_line() - out.emit("}\n") - # out.emit(stack.as_comment() + "\n") + emitter.out.start_line() + emitter.emit("}\n") + # emitter.emit(stack.as_comment() + "\n") return offset except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) @@ -152,6 +152,7 @@ def generate_tier1( """ ) out = CWriter(outfile, 2, lines) + emitter = Emitter(out) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): needs_this = uses_this(inst) @@ -183,7 +184,7 @@ def generate_tier1( for part in inst.parts: # Only emit braces if more than one uop insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - offset = write_uop(part, out, offset, stack, inst, insert_braces) + offset = write_uop(part, emitter, offset, stack, inst, insert_braces) out.start_line() if not inst.parts[-1].properties.always_exits: stack.flush(out) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 18bab2c13e7eb7..8c212f75878984 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -16,11 +16,10 @@ from generators_common import ( DEFAULT_INPUT, ROOT, - write_header, - emit_tokens, emit_to, - REPLACEMENT_FUNCTIONS, + write_header, type_and_null, + Emitter ) from cwriter import CWriter from typing import TextIO, Iterator @@ -61,117 +60,112 @@ def declare_variables(uop: Uop, out: CWriter) -> None: for var in uop.stack.outputs: declare_variable(var, uop, required, out) -def tier2_replace_error( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "COMMA") - label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_ERROR();\n") +class Tier2Emitter(Emitter): -def tier2_replace_error_no_pop( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - stack: Stack, - inst: Instruction | None, -) -> None: - next(tkn_iter) # LPAREN - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit_at("JUMP_TO_ERROR();", tkn) + def __init__(self, out: CWriter): + super().__init__(out) + self._replacers["oparg"] = self.oparg -def tier2_replace_deopt( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - unused: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - out.emit(") {\n") - out.emit("UOP_STAT_INC(uopcode, miss);\n") - out.emit("JUMP_TO_JUMP_TARGET();\n"); - out.emit("}\n") + def error_if( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + self.out.emit_at("if ", tkn) + self.emit(next(tkn_iter)) + emit_to(self.out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + self.emit(") JUMP_TO_ERROR();\n") + def error_no_pop( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + self.out.emit_at("JUMP_TO_ERROR();", tkn) -def tier2_replace_exit_if( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - unused: Stack, - inst: Instruction | None, -) -> None: - out.emit_at("if ", tkn) - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - out.emit(") {\n") - out.emit("UOP_STAT_INC(uopcode, miss);\n") - out.emit("JUMP_TO_JUMP_TARGET();\n") - out.emit("}\n") + def deopt_if( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, + ) -> None: + self.out.emit_at("if ", tkn) + self.emit(next(tkn_iter)) + emit_to(self.out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + self.emit(") {\n") + self.emit("UOP_STAT_INC(uopcode, miss);\n") + self.emit("JUMP_TO_JUMP_TARGET();\n"); + self.emit("}\n") + def exit_if( # type: ignore[override] + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, + ) -> None: + self.out.emit_at("if ", tkn) + self.emit(next(tkn_iter)) + emit_to(self.out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + self.emit(") {\n") + self.emit("UOP_STAT_INC(uopcode, miss);\n") + self.emit("JUMP_TO_JUMP_TARGET();\n") + self.emit("}\n") -def tier2_replace_oparg( - out: CWriter, - tkn: Token, - tkn_iter: Iterator[Token], - uop: Uop, - unused: Stack, - inst: Instruction | None, -) -> None: - if not uop.name.endswith("_0") and not uop.name.endswith("_1"): - out.emit(tkn) - return - amp = next(tkn_iter) - if amp.text != "&": - out.emit(tkn) - out.emit(amp) - return - one = next(tkn_iter) - assert one.text == "1" - out.emit_at(uop.name[-1], tkn) - + def oparg( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, + ) -> None: + if not uop.name.endswith("_0") and not uop.name.endswith("_1"): + self.emit(tkn) + return + amp = next(tkn_iter) + if amp.text != "&": + self.emit(tkn) + self.emit(amp) + return + one = next(tkn_iter) + assert one.text == "1" + self.out.emit_at(uop.name[-1], tkn) -TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() -TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error -TIER2_REPLACEMENT_FUNCTIONS["ERROR_NO_POP"] = tier2_replace_error_no_pop -TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt -TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg -TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if - - -def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> None: locals: dict[str, Local] = {} try: - out.start_line() + emitter.out.start_line() if uop.properties.oparg: - out.emit("oparg = CURRENT_OPARG();\n") + emitter.emit("oparg = CURRENT_OPARG();\n") assert uop.properties.const_oparg < 0 elif uop.properties.const_oparg >= 0: - out.emit(f"oparg = {uop.properties.const_oparg};\n") - out.emit(f"assert(oparg == CURRENT_OPARG());\n") + emitter.emit(f"oparg = {uop.properties.const_oparg};\n") + emitter.emit(f"assert(oparg == CURRENT_OPARG());\n") for var in reversed(uop.stack.inputs): code, local = stack.pop(var) - out.emit(code) + emitter.emit(code) if local.defined: locals[local.name] = local - out.emit(stack.define_output_arrays(uop.stack.outputs)) + emitter.emit(stack.define_output_arrays(uop.stack.outputs)) for cache in uop.caches: if cache.name != "unused": if cache.size == 4: @@ -179,14 +173,14 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: else: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" - out.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") - emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS) + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") + emitter.emit_tokens(uop, stack, None) for i, var in enumerate(uop.stack.outputs): if var.name in locals: local = locals[var.name] else: local = Local.local(var) - out.emit(stack.push(local)) + emitter.emit(stack.push(local)) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) from None @@ -207,6 +201,7 @@ def generate_tier2( """ ) out = CWriter(outfile, 2, lines) + emitter = Tier2Emitter(out) out.emit("\n") for name, uop in analysis.uops.items(): if uop.properties.tier == 1: @@ -223,7 +218,7 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - write_uop(uop, out, stack) + write_uop(uop, emitter, stack) out.start_line() if not uop.properties.always_exits: stack.flush(out) diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index ab3f39ac6ee087..ca872b024a7179 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.104.2 +hypothesis==6.108.10