From 61831585b637eb621bc80ae39e2e59541c135dc5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Mar 2024 18:49:09 +0100 Subject: [PATCH 01/35] Replace stat.ST_xxx usage with os.stat().st_xxx (#116501) Modernize code to use the new API which avoids the usage of the stat module just to read os.stat() members. * Sort logging.handlers imports. * Rework reopenIfNeeded() code to make it easier to follow. * Replace "not self.stream" with "self.stream is None". --- Lib/logging/handlers.py | 55 ++++++++++++++++++++++++-------------- Lib/test/test_largefile.py | 9 +++---- Lib/test/test_mailbox.py | 3 +-- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index cdb014bca04b6e..30cfe064478281 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -23,11 +23,17 @@ To use, simply 'import logging.handlers' and log away! """ -import io, logging, socket, os, pickle, struct, time, re -from stat import ST_DEV, ST_INO, ST_MTIME +import copy +import io +import logging +import os +import pickle import queue +import re +import socket +import struct import threading -import copy +import time # # Some constants... @@ -269,7 +275,7 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, # path object (see Issue #27493), but self.baseFilename will be a string filename = self.baseFilename if os.path.exists(filename): - t = os.stat(filename)[ST_MTIME] + t = int(os.stat(filename).st_mtime) else: t = int(time.time()) self.rolloverAt = self.computeRollover(t) @@ -459,8 +465,7 @@ class WatchedFileHandler(logging.FileHandler): This handler is not appropriate for use under Windows, because under Windows open files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need - for such a handler. Furthermore, ST_INO is not supported under - Windows; stat always returns zero for this value. + for such a handler. This handler is based on a suggestion and patch by Chad J. Schroeder. @@ -476,9 +481,11 @@ def __init__(self, filename, mode='a', encoding=None, delay=False, self._statstream() def _statstream(self): - if self.stream: - sres = os.fstat(self.stream.fileno()) - self.dev, self.ino = sres[ST_DEV], sres[ST_INO] + if self.stream is None: + return + sres = os.fstat(self.stream.fileno()) + self.dev = sres.st_dev + self.ino = sres.st_ino def reopenIfNeeded(self): """ @@ -488,6 +495,9 @@ def reopenIfNeeded(self): has, close the old stream and reopen the file to get the current stream. """ + if self.stream is None: + return + # Reduce the chance of race conditions by stat'ing by path only # once and then fstat'ing our new fd if we opened a new log stream. # See issue #14632: Thanks to John Mulligan for the problem report @@ -495,18 +505,23 @@ def reopenIfNeeded(self): try: # stat the file by path, checking for existence sres = os.stat(self.baseFilename) + + # compare file system stat with that of our stream file handle + reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino) except FileNotFoundError: - sres = None - # compare file system stat with that of our stream file handle - if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None # See Issue #21742: _open () might fail. - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() + reopen = True + + if not reopen: + return + + # we have an open file handle, clean it up + self.stream.flush() + self.stream.close() + self.stream = None # See Issue #21742: _open () might fail. + + # open a new file handle and get new stat info from that fd + self.stream = self._open() + self._statstream() def emit(self, record): """ diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 3b0930fe69e30e..849b6cb3e50a19 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -2,7 +2,6 @@ """ import os -import stat import sys import unittest import socket @@ -29,7 +28,7 @@ def setUp(self): mode = 'w+b' with self.open(TESTFN, mode) as f: - current_size = os.fstat(f.fileno())[stat.ST_SIZE] + current_size = os.fstat(f.fileno()).st_size if current_size == size+1: return @@ -40,13 +39,13 @@ def setUp(self): f.seek(size) f.write(b'a') f.flush() - self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1) + self.assertEqual(os.fstat(f.fileno()).st_size, size+1) @classmethod def tearDownClass(cls): with cls.open(TESTFN, 'wb'): pass - if not os.stat(TESTFN)[stat.ST_SIZE] == 0: + if not os.stat(TESTFN).st_size == 0: raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') unlink(TESTFN2) @@ -67,7 +66,7 @@ def test_large_read(self, _size): self.assertEqual(f.tell(), size + 1) def test_osstat(self): - self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) + self.assertEqual(os.stat(TESTFN).st_size, size+1) def test_seek_read(self): with self.open(TESTFN, 'rb') as f: diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index d4628f91daf7e8..bb24d5db1f9ed4 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -702,8 +702,7 @@ def _check_basics(self, factory=None): self.assertEqual(self._box._factory, factory) for subdir in '', 'tmp', 'new', 'cur': path = os.path.join(self._path, subdir) - mode = os.stat(path)[stat.ST_MODE] - self.assertTrue(stat.S_ISDIR(mode), "Not a directory: '%s'" % path) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") def test_list_folders(self): # List folders From 7db871e4fa48eef20ea22414b226998f83facfd6 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 8 Mar 2024 09:56:36 -0800 Subject: [PATCH 02/35] gh-112075: Support freeing object memory via QSBR (#116344) Free objects with qsbr if shared --- Include/internal/pycore_gc.h | 27 +++++++++++++++++++--- Include/internal/pycore_pymem.h | 3 +++ Objects/obmalloc.c | 41 ++++++++++++++++++++++++++------- Python/gc_free_threading.c | 10 ++++++-- 4 files changed, 68 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index cf0b148c2b6463..4a7191a562cc10 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { # define _PyGC_BITS_UNREACHABLE (4) # define _PyGC_BITS_FROZEN (8) # define _PyGC_BITS_SHARED (16) +# define _PyGC_BITS_SHARED_INLINE (32) #endif /* True if the object is currently tracked by the GC. */ @@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { #ifdef Py_GIL_DISABLED -/* True if an object is shared between multiple threads and - * needs special purpose when freeing to do the possibility - * of in-flight lock-free reads occurring */ +/* True if memory the object references is shared between + * multiple threads and needs special purpose when freeing + * those references due to the possibility of in-flight + * lock-free reads occurring. The object is responsible + * for calling _PyMem_FreeDelayed on the referenced + * memory. */ static inline int _PyObject_GC_IS_SHARED(PyObject *op) { return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; } @@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { } #define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) +/* True if the memory of the object is shared between multiple + * threads and needs special purpose when freeing due to + * the possibility of in-flight lock-free reads occurring. + * Objects with this bit that are GC objects will automatically + * delay-freed by PyObject_GC_Del. */ +static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) { + return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0; +} +#define _PyObject_GC_IS_SHARED_INLINE(op) \ + _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) { + op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE; +} +#define _PyObject_GC_SET_SHARED_INLINE(op) \ + _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op)) + #endif /* Bit flags for _gc_prev */ diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1aea91abc5d69f..dd6b0762370c92 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. extern void _PyMem_FreeDelayed(void *ptr); +// Enqueue an object to be freed possibly after some delay +extern void _PyObject_FreeDelayed(void *ptr); + // Periodically process delayed free requests. extern void _PyMem_ProcessDelayed(PyThreadState *tstate); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index e7813807674abe..4fe195b63166c1 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1070,7 +1070,7 @@ _PyMem_Strdup(const char *str) // A pointer to be freed once the QSBR read sequence reaches qsbr_goal. struct _mem_work_item { - void *ptr; + uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free uint64_t qsbr_goal; }; @@ -1084,16 +1084,27 @@ struct _mem_work_chunk { struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; }; -void -_PyMem_FreeDelayed(void *ptr) +static void +free_work_item(uintptr_t ptr) +{ + if (ptr & 0x01) { + PyObject_Free((char *)(ptr - 1)); + } + else { + PyMem_Free((void *)ptr); + } +} + +static void +free_delayed(uintptr_t ptr) { #ifndef Py_GIL_DISABLED - PyMem_Free(ptr); + free_work_item(ptr); #else if (_PyRuntime.stoptheworld.world_stopped) { // Free immediately if the world is stopped, including during // interpreter shutdown. - PyMem_Free(ptr); + free_work_item(ptr); return; } @@ -1120,7 +1131,7 @@ _PyMem_FreeDelayed(void *ptr) if (buf == NULL) { // failed to allocate a buffer, free immediately _PyEval_StopTheWorld(tstate->base.interp); - PyMem_Free(ptr); + free_work_item(ptr); _PyEval_StartTheWorld(tstate->base.interp); return; } @@ -1137,6 +1148,20 @@ _PyMem_FreeDelayed(void *ptr) #endif } +void +_PyMem_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed((uintptr_t)ptr); +} + +void +_PyObject_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed(((uintptr_t)ptr)|0x01); +} + static struct _mem_work_chunk * work_queue_first(struct llist_node *head) { @@ -1156,7 +1181,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, return; } - PyMem_Free(item->ptr); + free_work_item(item->ptr); buf->rd_idx++; } @@ -1243,7 +1268,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp) // Free the remaining items immediately. There should be no other // threads accessing the memory at this point during shutdown. struct _mem_work_item *item = &buf->array[buf->rd_idx]; - PyMem_Free(item->ptr); + free_work_item(item->ptr); buf->rd_idx++; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index c7883cd38644e1..59e76012f8fc50 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1695,6 +1695,7 @@ PyObject_GC_Del(void *op) { size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -1707,8 +1708,13 @@ PyObject_GC_Del(void *op) } record_deallocation(_PyThreadState_GET()); - - PyObject_Free(((char *)op)-presize); + PyObject *self = (PyObject *)op; + if (_PyObject_GC_IS_SHARED_INLINE(self)) { + _PyObject_FreeDelayed(((char *)op)-presize); + } + else { + PyObject_Free(((char *)op)-presize); + } } int From 735fc2cbbcf875c359021b5b2af7f4c29f4cf66d Mon Sep 17 00:00:00 2001 From: infohash <46137868+infohash@users.noreply.github.com> Date: Sat, 9 Mar 2024 00:44:32 +0530 Subject: [PATCH 03/35] gh-75988: Fix issues with autospec ignoring wrapped object (#115223) * set default return value of functional types as _mock_return_value * added test of wrapping child attributes * added backward compatibility with explicit return * added docs on the order of precedence * added test to check default return_value --- Doc/library/unittest.mock.rst | 120 ++++++++++++++++++ Lib/test/test_unittest/testmock/testmock.py | 67 ++++++++++ Lib/unittest/mock.py | 13 +- ...4-02-27-13-05-51.gh-issue-75988.In6LlB.rst | 1 + 4 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 1f25a16f544da8..d1f2a96df667c6 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2831,3 +2831,123 @@ Sealing mocks >>> mock.not_submock.attribute2 # This won't raise. .. versionadded:: 3.7 + + +Order of precedence of :attr:`side_effect`, :attr:`return_value` and *wraps* +---------------------------------------------------------------------------- + +The order of their precedence is: + +1. :attr:`~Mock.side_effect` +2. :attr:`~Mock.return_value` +3. *wraps* + +If all three are set, mock will return the value from :attr:`~Mock.side_effect`, +ignoring :attr:`~Mock.return_value` and the wrapped object altogether. If any +two are set, the one with the higher precedence will return the value. +Regardless of the order of which was set first, the order of precedence +remains unchanged. + + >>> from unittest.mock import Mock + >>> class Order: + ... @staticmethod + ... def get_value(): + ... return "third" + ... + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first"] + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'first' + +As ``None`` is the default value of :attr:`~Mock.side_effect`, if you reassign +its value back to ``None``, the order of precedence will be checked between +:attr:`~Mock.return_value` and the wrapped object, ignoring +:attr:`~Mock.side_effect`. + + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + +If the value being returned by :attr:`~Mock.side_effect` is :data:`DEFAULT`, +it is ignored and the order of precedence moves to the successor to obtain the +value to return. + + >>> from unittest.mock import DEFAULT + >>> order_mock.get_value.side_effect = [DEFAULT] + >>> order_mock.get_value() + 'second' + +When :class:`Mock` wraps an object, the default value of +:attr:`~Mock.return_value` will be :data:`DEFAULT`. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.return_value + sentinel.DEFAULT + >>> order_mock.get_value.return_value + sentinel.DEFAULT + +The order of precedence will ignore this value and it will move to the last +successor which is the wrapped object. + +As the real call is being made to the wrapped object, creating an instance of +this mock will return the real instance of the class. The positional arguments, +if any, required by the wrapped object must be passed. + + >>> order_mock_instance = order_mock() + >>> isinstance(order_mock_instance, Order) + True + >>> order_mock_instance.get_value() + 'third' + + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'second' + +But if you assign ``None`` to it, this will not be ignored as it is an +explicit assignment. So, the order of precedence will not move to the wrapped +object. + + >>> order_mock.get_value.return_value = None + >>> order_mock.get_value() is None + True + +Even if you set all three at once when initializing the mock, the order of +precedence remains the same: + + >>> order_mock = Mock(spec=Order, wraps=Order, + ... **{"get_value.side_effect": ["first"], + ... "get_value.return_value": "second"} + ... ) + ... + >>> order_mock.get_value() + 'first' + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + +If :attr:`~Mock.side_effect` is exhausted, the order of precedence will not +cause a value to be obtained from the successors. Instead, ``StopIteration`` +exception is raised. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first side effect value", + ... "another side effect value"] + >>> order_mock.get_value.return_value = "second" + + >>> order_mock.get_value() + 'first side effect value' + >>> order_mock.get_value() + 'another side effect value' + + >>> order_mock.get_value() + Traceback (most recent call last): + ... + StopIteration diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index 1725406bcfb9e4..b81b3049d56bf8 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -245,6 +245,65 @@ class B(object): with mock.patch('builtins.open', mock.mock_open()): mock.mock_open() # should still be valid with open() mocked + def test_create_autospec_wraps_class(self): + """Autospec a class with wraps & test if the call is passed to the + wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + class_mock = create_autospec(spec=Result, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result (actual instance of "Result") when the mock is called. + class_mock.return_value = mock.DEFAULT + self.assertEqual(class_mock().get_result(), result) + # Autospec should also wrap child attributes of parent. + self.assertEqual(class_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_instance_wraps_class(self): + """Autospec a class instance with wraps & test if the call is passed + to the wrapped object.""" + result = "real result" + + class Result: + @staticmethod + def get_result(): + """This is a static method because when the mocked instance of + 'Result' will call this method, it won't be able to consume + 'self' argument.""" + return result + instance_mock = create_autospec(spec=Result, instance=True, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result from "Result.get_result" when the mocked instance of "Result" + # calls "get_result". + instance_mock.get_result.return_value = mock.DEFAULT + self.assertEqual(instance_mock.get_result(), result) + # Autospec should also wrap child attributes of the instance. + self.assertEqual(instance_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_wraps_function_type(self): + """Autospec a function or a method with wraps & test if the call is + passed to the wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + func_mock = create_autospec(spec=Result.get_result, wraps=Result.get_result) + self.assertEqual(func_mock(Result()), result) + + def test_explicit_return_value_even_if_mock_wraps_object(self): + """If the mock has an explicit return_value set then calls are not + passed to the wrapped object and the return_value is returned instead. + """ + def my_func(): + return None + func_mock = create_autospec(spec=my_func, wraps=my_func) + return_value = "explicit return value" + func_mock.return_value = return_value + self.assertEqual(func_mock(), return_value) + def test_explicit_parent(self): parent = Mock() mock1 = Mock(parent=parent, return_value=None) @@ -622,6 +681,14 @@ def test_wraps_calls(self): real = Mock() mock = Mock(wraps=real) + # If "Mock" wraps an object, just accessing its + # "return_value" ("NonCallableMock.__get_return_value") should not + # trigger its descriptor ("NonCallableMock.__set_return_value") so + # the default "return_value" should always be "sentinel.DEFAULT". + self.assertEqual(mock.return_value, DEFAULT) + # It will not be "sentinel.DEFAULT" if the mock is not wrapping any + # object. + self.assertNotEqual(real.return_value, DEFAULT) self.assertEqual(mock(), real()) real.reset_mock() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 93f4d9743ed2fa..1799e9bbf58592 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -573,7 +573,7 @@ def __get_return_value(self): if self._mock_delegate is not None: ret = self._mock_delegate.return_value - if ret is DEFAULT: + if ret is DEFAULT and self._mock_wraps is None: ret = self._get_child_mock( _new_parent=self, _new_name='()' ) @@ -1234,6 +1234,9 @@ def _execute_mock_call(self, /, *args, **kwargs): if self._mock_return_value is not DEFAULT: return self.return_value + if self._mock_delegate and self._mock_delegate.return_value is not DEFAULT: + return self.return_value + if self._mock_wraps is not None: return self._mock_wraps(*args, **kwargs) @@ -2785,9 +2788,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if _parent is not None and not instance: _parent._mock_children[_name] = mock + wrapped = kwargs.get('wraps') + if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, - _name='()', _parent=mock) + _name='()', _parent=mock, + wraps=wrapped) for entry in dir(spec): if _is_magic(entry): @@ -2809,6 +2815,9 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, continue kwargs = {'spec': original} + # Wrap child attributes also. + if wrapped and hasattr(wrapped, entry): + kwargs.update(wraps=original) if spec_set: kwargs = {'spec_set': original} diff --git a/Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst b/Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst new file mode 100644 index 00000000000000..682b7cfa06b868 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-27-13-05-51.gh-issue-75988.In6LlB.rst @@ -0,0 +1 @@ +Fixed :func:`unittest.mock.create_autospec` to pass the call through to the wrapped object to return the real result. From 601f3a7b3391e9d219a8ec44a6c56d00ce584d2a Mon Sep 17 00:00:00 2001 From: "Nicolas A. Oyarzabal" <79150521+nicky-eng@users.noreply.github.com> Date: Fri, 8 Mar 2024 20:22:17 +0100 Subject: [PATCH 04/35] gh-105535 Document potential performance trap during enum creation (GH-107119) Co-authored-by: Ethan Furman Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/enum.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 49bf40aa5a2869..d84d9d9b4161b1 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -279,6 +279,8 @@ Data Types >>> Color.RED.value 1 + Value of the member, can be set in :meth:`~object.__new__`. + .. note:: Enum member values Member values can be anything: :class:`int`, :class:`str`, etc. If @@ -286,6 +288,11 @@ Data Types appropriate value will be chosen for you. See :class:`auto` for the details. + While mutable/unhashable values, such as :class:`dict`, :class:`list` or + a mutable :class:`~dataclasses.dataclass`, can be used, they will have a + quadratic performance impact during creation relative to the + total number of mutable/unhashable values in the enum. + .. attribute:: Enum._name_ Name of the member. From 3cdfdc07a9dd39bcd6855b8c104584f9c34624f2 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 8 Mar 2024 15:26:36 -0500 Subject: [PATCH 05/35] gh-108724: Fix _PySemaphore_Wait call during thread deletion (#116483) In general, when `_PyThreadState_GET()` is non-NULL then the current thread is "attached", but there is a small window during `PyThreadState_DeleteCurrent()` where that's not true: tstate_delete_common() is called when the thread is detached, but before current_fast_clear(). Co-authored-by: Eric Snow --- Python/parking_lot.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 0a897f9952f648..d5877fef56e4d0 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -194,14 +194,16 @@ _PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach) PyThreadState *tstate = NULL; if (detach) { tstate = _PyThreadState_GET(); - if (tstate) { + if (tstate && tstate->state == _Py_THREAD_ATTACHED) { + // Only detach if we are attached PyEval_ReleaseThread(tstate); } + else { + tstate = NULL; + } } - int res = _PySemaphore_PlatformWait(sema, timeout); - - if (detach && tstate) { + if (tstate) { PyEval_AcquireThread(tstate); } return res; From c951e25c24910064a4c8b7959e2f0f7c0d4d0a63 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Fri, 8 Mar 2024 22:25:34 +0100 Subject: [PATCH 06/35] gh-112069: Make sets thread-safe with the GIL disabled (#113800) This makes nearly all the operations on set thread-safe in the free-threaded build, with the exception of `_PySet_NextEntry` and `setiter_iternext`. Co-authored-by: Sam Gross Co-authored-by: Erlend E. Aasland --- Objects/clinic/setobject.c.h | 171 ++++++++++++- Objects/setobject.c | 479 ++++++++++++++++++++++++----------- 2 files changed, 498 insertions(+), 152 deletions(-) diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h index f3c96995ede60d..3853ce3bce685b 100644 --- a/Objects/clinic/setobject.c.h +++ b/Objects/clinic/setobject.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(set_pop__doc__, @@ -21,7 +22,13 @@ set_pop_impl(PySetObject *so); static PyObject * set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set_pop_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_pop_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_update__doc__, @@ -74,7 +81,13 @@ set_copy_impl(PySetObject *so); static PyObject * set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set_copy_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_copy_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(frozenset_copy__doc__, @@ -92,7 +105,13 @@ frozenset_copy_impl(PySetObject *so); static PyObject * frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return frozenset_copy_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = frozenset_copy_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_clear__doc__, @@ -110,7 +129,13 @@ set_clear_impl(PySetObject *so); static PyObject * set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set_clear_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_clear_impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_union__doc__, @@ -227,6 +252,21 @@ PyDoc_STRVAR(set_isdisjoint__doc__, #define SET_ISDISJOINT_METHODDEF \ {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, set_isdisjoint__doc__}, +static PyObject * +set_isdisjoint_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_isdisjoint(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_isdisjoint_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_difference_update__doc__, "difference_update($self, /, *others)\n" "--\n" @@ -315,6 +355,21 @@ PyDoc_STRVAR(set_symmetric_difference__doc__, #define SET_SYMMETRIC_DIFFERENCE_METHODDEF \ {"symmetric_difference", (PyCFunction)set_symmetric_difference, METH_O, set_symmetric_difference__doc__}, +static PyObject * +set_symmetric_difference_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_symmetric_difference(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_symmetric_difference_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_issubset__doc__, "issubset($self, other, /)\n" "--\n" @@ -324,6 +379,21 @@ PyDoc_STRVAR(set_issubset__doc__, #define SET_ISSUBSET_METHODDEF \ {"issubset", (PyCFunction)set_issubset, METH_O, set_issubset__doc__}, +static PyObject * +set_issubset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issubset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issubset_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_issuperset__doc__, "issuperset($self, other, /)\n" "--\n" @@ -333,6 +403,21 @@ PyDoc_STRVAR(set_issuperset__doc__, #define SET_ISSUPERSET_METHODDEF \ {"issuperset", (PyCFunction)set_issuperset, METH_O, set_issuperset__doc__}, +static PyObject * +set_issuperset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issuperset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issuperset_impl(so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_add__doc__, "add($self, object, /)\n" "--\n" @@ -344,6 +429,21 @@ PyDoc_STRVAR(set_add__doc__, #define SET_ADD_METHODDEF \ {"add", (PyCFunction)set_add, METH_O, set_add__doc__}, +static PyObject * +set_add_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_add(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_add_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set___contains____doc__, "__contains__($self, object, /)\n" "--\n" @@ -353,6 +453,21 @@ PyDoc_STRVAR(set___contains____doc__, #define SET___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)set___contains__, METH_O|METH_COEXIST, set___contains____doc__}, +static PyObject * +set___contains___impl(PySetObject *so, PyObject *key); + +static PyObject * +set___contains__(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___contains___impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set_remove__doc__, "remove($self, object, /)\n" "--\n" @@ -364,6 +479,21 @@ PyDoc_STRVAR(set_remove__doc__, #define SET_REMOVE_METHODDEF \ {"remove", (PyCFunction)set_remove, METH_O, set_remove__doc__}, +static PyObject * +set_remove_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_remove(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_remove_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set_discard__doc__, "discard($self, object, /)\n" "--\n" @@ -376,6 +506,21 @@ PyDoc_STRVAR(set_discard__doc__, #define SET_DISCARD_METHODDEF \ {"discard", (PyCFunction)set_discard, METH_O, set_discard__doc__}, +static PyObject * +set_discard_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_discard(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_discard_impl(so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set___reduce____doc__, "__reduce__($self, /)\n" "--\n" @@ -391,7 +536,13 @@ set___reduce___impl(PySetObject *so); static PyObject * set___reduce__(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set___reduce___impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___reduce___impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set___sizeof____doc__, @@ -409,6 +560,12 @@ set___sizeof___impl(PySetObject *so); static PyObject * set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - return set___sizeof___impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___sizeof___impl(so); + Py_END_CRITICAL_SECTION(); + + return return_value; } -/*[clinic end generated code: output=34a30591148da884 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=de4ee725bd29f758 input=a9049054013a1b77]*/ diff --git a/Objects/setobject.c b/Objects/setobject.c index b4d803c19e1e94..592711f305cbaf 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -37,6 +37,7 @@ #include "pycore_dict.h" // _PyDict_Contains_KnownHash() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition #include // offsetof() @@ -130,6 +131,8 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) int probes; int cmp; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + /* Pre-increment is necessary to prevent arbitrary code in the rich comparison from deallocating the key just before the insertion. */ Py_INCREF(key); @@ -523,7 +526,7 @@ set_dealloc(PySetObject *so) } static PyObject * -set_repr(PySetObject *so) +set_repr_lock_held(PySetObject *so) { PyObject *result=NULL, *keys, *listrepr, *tmp; int status = Py_ReprEnter((PyObject*)so); @@ -567,14 +570,24 @@ set_repr(PySetObject *so) return result; } +static PyObject * +set_repr(PySetObject *so) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(so); + result = set_repr_lock_held(so); + Py_END_CRITICAL_SECTION(); + return result; +} + static Py_ssize_t -set_len(PyObject *so) +set_len(PySetObject *so) { - return ((PySetObject *)so)->used; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); } static int -set_merge(PySetObject *so, PyObject *otherset) +set_merge_lock_held(PySetObject *so, PyObject *otherset) { PySetObject *other; PyObject *key; @@ -584,6 +597,8 @@ set_merge(PySetObject *so, PyObject *otherset) assert (PyAnySet_Check(so)); assert (PyAnySet_Check(otherset)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(otherset); other = (PySetObject*)otherset; if (other == so || other->used == 0) @@ -645,6 +660,7 @@ set_merge(PySetObject *so, PyObject *otherset) } /*[clinic input] +@critical_section set.pop so: setobject @@ -655,7 +671,7 @@ Raises KeyError if the set is empty. static PyObject * set_pop_impl(PySetObject *so) -/*[clinic end generated code: output=4d65180f1271871b input=4a3f5552e660a260]*/ +/*[clinic end generated code: output=4d65180f1271871b input=9296c84921125060]*/ { /* Make sure the search finger is in bounds */ setentry *entry = so->table + (so->finger & so->mask); @@ -889,58 +905,60 @@ PyTypeObject PySetIter_Type = { static PyObject * set_iter(PySetObject *so) { + Py_ssize_t size = set_len(so); setiterobject *si = PyObject_GC_New(setiterobject, &PySetIter_Type); if (si == NULL) return NULL; si->si_set = (PySetObject*)Py_NewRef(so); - si->si_used = so->used; + si->si_used = size; si->si_pos = 0; - si->len = so->used; + si->len = size; _PyObject_GC_TRACK(si); return (PyObject *)si; } static int -set_update_internal(PySetObject *so, PyObject *other) +set_update_dict_lock_held(PySetObject *so, PyObject *other) { - PyObject *key, *it; + assert(PyDict_CheckExact(other)); - if (PyAnySet_Check(other)) - return set_merge(so, other); - - if (PyDict_CheckExact(other)) { - PyObject *value; - Py_ssize_t pos = 0; - Py_hash_t hash; - Py_ssize_t dictsize = PyDict_GET_SIZE(other); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new keys. Expect - * that there will be no (or few) overlapping keys. - */ - if (dictsize < 0) + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new keys. Expect + * that there will be no (or few) overlapping keys. + */ + Py_ssize_t dictsize = PyDict_GET_SIZE(other); + if ((so->fill + dictsize)*5 >= so->mask*3) { + if (set_table_resize(so, (so->used + dictsize)*2) != 0) { return -1; - if ((so->fill + dictsize)*5 >= so->mask*3) { - if (set_table_resize(so, (so->used + dictsize)*2) != 0) - return -1; } - int err = 0; - Py_BEGIN_CRITICAL_SECTION(other); - while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_entry(so, key, hash)) { - err = -1; - goto exit; - } + } + + Py_ssize_t pos = 0; + PyObject *key; + PyObject *value; + Py_hash_t hash; + while (_PyDict_Next(other, &pos, &key, &value, &hash)) { + if (set_add_entry(so, key, hash)) { + return -1; } -exit: - Py_END_CRITICAL_SECTION(); - return err; } + return 0; +} - it = PyObject_GetIter(other); - if (it == NULL) +static int +set_update_iterable_lock_held(PySetObject *so, PyObject *other) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + + PyObject *it = PyObject_GetIter(other); + if (it == NULL) { return -1; + } + PyObject *key; while ((key = PyIter_Next(it)) != NULL) { if (set_add_key(so, key)) { Py_DECREF(it); @@ -955,6 +973,69 @@ set_update_internal(PySetObject *so, PyObject *other) return 0; } +static int +set_update_lock_held(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + return set_merge_lock_held(so, other); + } + else if (PyDict_CheckExact(other)) { + return set_update_dict_lock_held(so, other); + } + return set_update_iterable_lock_held(so, other); +} + +// set_update for a `so` that is only visible to the current thread +static int +set_update_local(PySetObject *so, PyObject *other) +{ + assert(Py_REFCNT(so) == 1); + if (PyAnySet_Check(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + return set_update_iterable_lock_held(so, other); +} + +static int +set_update_internal(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + if (Py_Is((PyObject *)so, other)) { + return 0; + } + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else { + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_update_iterable_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } +} + /*[clinic input] set.update so: setobject @@ -1003,7 +1084,7 @@ make_new_set(PyTypeObject *type, PyObject *iterable) so->weakreflist = NULL; if (iterable != NULL) { - if (set_update_internal(so, iterable)) { + if (set_update_local(so, iterable)) { Py_DECREF(so); return NULL; } @@ -1126,6 +1207,7 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } /*[clinic input] +@critical_section set.copy so: setobject @@ -1134,12 +1216,22 @@ Return a shallow copy of a set. static PyObject * set_copy_impl(PySetObject *so) -/*[clinic end generated code: output=c9223a1e1cc6b041 input=2b80b288d47b8cf1]*/ +/*[clinic end generated code: output=c9223a1e1cc6b041 input=c169a4fbb8209257]*/ { - return make_new_set_basetype(Py_TYPE(so), (PyObject *)so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + PyObject *copy = make_new_set_basetype(Py_TYPE(so), NULL); + if (copy == NULL) { + return NULL; + } + if (set_merge_lock_held((PySetObject *)copy, (PyObject *)so) < 0) { + Py_DECREF(copy); + return NULL; + } + return copy; } /*[clinic input] +@critical_section frozenset.copy so: setobject @@ -1148,15 +1240,16 @@ Return a shallow copy of a set. static PyObject * frozenset_copy_impl(PySetObject *so) -/*[clinic end generated code: output=b356263526af9e70 input=3dc65577d344eff7]*/ +/*[clinic end generated code: output=b356263526af9e70 input=fbf5bef131268dd7]*/ { if (PyFrozenSet_CheckExact(so)) { return Py_NewRef(so); } - return set_copy(so, NULL); + return set_copy_impl(so); } /*[clinic input] +@critical_section set.clear so: setobject @@ -1165,7 +1258,7 @@ Remove all elements from this set. static PyObject * set_clear_impl(PySetObject *so) -/*[clinic end generated code: output=4e71d5a83904161a input=74ac19794da81a39]*/ +/*[clinic end generated code: output=4e71d5a83904161a input=c6f831b366111950]*/ { set_clear_internal(so); Py_RETURN_NONE; @@ -1196,7 +1289,7 @@ set_union_impl(PySetObject *so, PyObject *args) other = PyTuple_GET_ITEM(args, i); if ((PyObject *)so == other) continue; - if (set_update_internal(result, other)) { + if (set_update_local(result, other)) { Py_DECREF(result); return NULL; } @@ -1213,11 +1306,13 @@ set_or(PySetObject *so, PyObject *other) Py_RETURN_NOTIMPLEMENTED; result = (PySetObject *)set_copy(so, NULL); - if (result == NULL) + if (result == NULL) { return NULL; - if ((PyObject *)so == other) + } + if (Py_Is((PyObject *)so, other)) { return (PyObject *)result; - if (set_update_internal(result, other)) { + } + if (set_update_local(result, other)) { Py_DECREF(result); return NULL; } @@ -1230,8 +1325,9 @@ set_ior(PySetObject *so, PyObject *other) if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; - if (set_update_internal(so, other)) + if (set_update_internal(so, other)) { return NULL; + } return Py_NewRef(so); } @@ -1244,7 +1340,7 @@ set_intersection(PySetObject *so, PyObject *other) int rv; if ((PyObject *)so == other) - return set_copy(so, NULL); + return set_copy_impl(so); result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); if (result == NULL) @@ -1333,13 +1429,17 @@ set_intersection_multi_impl(PySetObject *so, PyObject *args) { Py_ssize_t i; - if (PyTuple_GET_SIZE(args) == 0) + if (PyTuple_GET_SIZE(args) == 0) { return set_copy(so, NULL); + } PyObject *result = Py_NewRef(so); for (i=0 ; ikey); + Py_hash_t hash = entry->hash; + int rv = set_discard_entry(so, key, hash); + if (rv < 0) { + Py_DECREF(key); + return -1; + } + if (rv == DISCARD_NOTFOUND) { + if (set_add_entry(so, key, hash)) { + Py_DECREF(key); + return -1; + } + } + Py_DECREF(key); + } + return 0; } /*[clinic input] @@ -1739,58 +1905,41 @@ static PyObject * set_symmetric_difference_update(PySetObject *so, PyObject *other) /*[clinic end generated code: output=fbb049c0806028de input=a50acf0365e1f0a5]*/ { - PySetObject *otherset; - PyObject *key; - Py_ssize_t pos = 0; - Py_hash_t hash; - setentry *entry; - int rv; - - if ((PyObject *)so == other) + if (Py_Is((PyObject *)so, other)) { return set_clear(so, NULL); + } + int rv; if (PyDict_CheckExact(other)) { - PyObject *res; - - Py_BEGIN_CRITICAL_SECTION(other); - res = set_symmetric_difference_update_dict(so, other); - Py_END_CRITICAL_SECTION(); - - return res; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION2(); } - - if (PyAnySet_Check(other)) { - otherset = (PySetObject *)Py_NewRef(other); - } else { - otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); - if (otherset == NULL) - return NULL; + else if (PyAnySet_Check(other)) { + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_set(so, (PySetObject *)other); + Py_END_CRITICAL_SECTION2(); } - - while (set_next(otherset, &pos, &entry)) { - key = entry->key; - hash = entry->hash; - Py_INCREF(key); - rv = set_discard_entry(so, key, hash); - if (rv < 0) { - Py_DECREF(otherset); - Py_DECREF(key); + else { + PySetObject *otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); + if (otherset == NULL) { return NULL; } - if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, key, hash)) { - Py_DECREF(otherset); - Py_DECREF(key); - return NULL; - } - } - Py_DECREF(key); + + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_symmetric_difference_update_set(so, otherset); + Py_END_CRITICAL_SECTION(); + + Py_DECREF(otherset); + } + if (rv < 0) { + return NULL; } - Py_DECREF(otherset); Py_RETURN_NONE; } /*[clinic input] +@critical_section so other set.symmetric_difference so: setobject other: object @@ -1800,22 +1949,22 @@ Return a new set with elements in either the set or other but not both. [clinic start generated code]*/ static PyObject * -set_symmetric_difference(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=f95364211b88775a input=f18af370ad72ebac]*/ +set_symmetric_difference_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=270ee0b5d42b0797 input=624f6e7bbdf70db1]*/ { - PyObject *rv; - PySetObject *otherset; - - otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); - if (otherset == NULL) + PySetObject *result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); + if (result == NULL) { return NULL; - rv = set_symmetric_difference_update(otherset, (PyObject *)so); - if (rv == NULL) { - Py_DECREF(otherset); + } + if (set_update_lock_held(result, other) < 0) { + Py_DECREF(result); return NULL; } - Py_DECREF(rv); - return (PyObject *)otherset; + if (set_symmetric_difference_update_set(result, so) < 0) { + Py_DECREF(result); + return NULL; + } + return (PyObject *)result; } static PyObject * @@ -1841,6 +1990,7 @@ set_ixor(PySetObject *so, PyObject *other) } /*[clinic input] +@critical_section so other set.issubset so: setobject other: object @@ -1850,8 +2000,8 @@ Report whether another set contains this set. [clinic start generated code]*/ static PyObject * -set_issubset(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=78aef1f377aedef1 input=37fbc579b609db0c]*/ +set_issubset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=b2b59d5f314555ce input=f2a4fd0f2537758b]*/ { setentry *entry; Py_ssize_t pos = 0; @@ -1885,6 +2035,7 @@ set_issubset(PySetObject *so, PyObject *other) } /*[clinic input] +@critical_section so other set.issuperset so: setobject other: object @@ -1894,8 +2045,8 @@ Report whether this set contains another set. [clinic start generated code]*/ static PyObject * -set_issuperset(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=7d2b71dd714a7ec7 input=fd5dab052f2e9bb3]*/ +set_issuperset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=ecf00ce552c09461 input=5f2e1f262e6e4ccc]*/ { if (PyAnySet_Check(other)) { return set_issubset((PySetObject *)other, (PyObject *)so); @@ -1924,6 +2075,7 @@ set_issuperset(PySetObject *so, PyObject *other) Py_RETURN_TRUE; } +// TODO: Make thread-safe in free-threaded builds static PyObject * set_richcompare(PySetObject *v, PyObject *w, int op) { @@ -1968,6 +2120,7 @@ set_richcompare(PySetObject *v, PyObject *w, int op) } /*[clinic input] +@critical_section set.add so: setobject object as key: object @@ -1979,16 +2132,16 @@ This has no effect if the element is already present. [clinic start generated code]*/ static PyObject * -set_add(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=cd9c2d5c2069c2ba input=96f1efe029e47972]*/ +set_add_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=4cc4a937f1425c96 input=03baf62cb0e66514]*/ { if (set_add_key(so, key)) return NULL; Py_RETURN_NONE; } -int -_PySet_Contains(PySetObject *so, PyObject *key) +static int +set_contains_lock_held(PySetObject *so, PyObject *key) { PyObject *tmpkey; int rv; @@ -2007,7 +2160,18 @@ _PySet_Contains(PySetObject *so, PyObject *key) return rv; } +int +_PySet_Contains(PySetObject *so, PyObject *key) +{ + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_contains_lock_held(so, key); + Py_END_CRITICAL_SECTION(); + return rv; +} + /*[clinic input] +@critical_section @coexist set.__contains__ so: setobject @@ -2018,18 +2182,19 @@ x.__contains__(y) <==> y in x. [clinic start generated code]*/ static PyObject * -set___contains__(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=b5948bc5c590d3ca input=cf4c72db704e4cf0]*/ +set___contains___impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=b44863d034b3c70e input=4a7d568459617f24]*/ { long result; - result = _PySet_Contains(so, key); + result = set_contains_lock_held(so, key); if (result < 0) return NULL; return PyBool_FromLong(result); } /*[clinic input] +@critical_section set.remove so: setobject object as key: object @@ -2041,8 +2206,8 @@ If the element is not a member, raise a KeyError. [clinic start generated code]*/ static PyObject * -set_remove(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=08ae496d0cd2b8c1 input=10132515dfe8ebd7]*/ +set_remove_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=0b9134a2a2200363 input=893e1cb1df98227a]*/ { PyObject *tmpkey; int rv; @@ -2069,6 +2234,7 @@ set_remove(PySetObject *so, PyObject *key) } /*[clinic input] +@critical_section set.discard so: setobject object as key: object @@ -2081,8 +2247,8 @@ an exception when an element is missing from the set. [clinic start generated code]*/ static PyObject * -set_discard(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=9181b60d7bb7d480 input=82a689eba94d5ad9]*/ +set_discard_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=eec3b687bf32759e input=861cb7fb69b4def0]*/ { PyObject *tmpkey; int rv; @@ -2104,6 +2270,7 @@ set_discard(PySetObject *so, PyObject *key) } /*[clinic input] +@critical_section set.__reduce__ so: setobject @@ -2112,7 +2279,7 @@ Return state information for pickling. static PyObject * set___reduce___impl(PySetObject *so) -/*[clinic end generated code: output=9af7d0e029df87ee input=531375e87a24a449]*/ +/*[clinic end generated code: output=9af7d0e029df87ee input=59405a4249e82f71]*/ { PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; @@ -2134,6 +2301,7 @@ set___reduce___impl(PySetObject *so) } /*[clinic input] +@critical_section set.__sizeof__ so: setobject @@ -2142,7 +2310,7 @@ S.__sizeof__() -> size of S in memory, in bytes. static PyObject * set___sizeof___impl(PySetObject *so) -/*[clinic end generated code: output=4bfa3df7bd38ed88 input=0f214fc2225319fc]*/ +/*[clinic end generated code: output=4bfa3df7bd38ed88 input=09e1a09f168eaa23]*/ { size_t res = _PyObject_SIZE(Py_TYPE(so)); if (so->table != so->smalltable) { @@ -2156,13 +2324,17 @@ set_init(PySetObject *self, PyObject *args, PyObject *kwds) { PyObject *iterable = NULL; - if (!_PyArg_NoKeywords("set", kwds)) + if (!_PyArg_NoKeywords("set", kwds)) return -1; if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; + + Py_BEGIN_CRITICAL_SECTION(self); if (self->fill) set_clear_internal(self); self->hash = -1; + Py_END_CRITICAL_SECTION(); + if (iterable == NULL) return 0; return set_update_internal(self, iterable); @@ -2191,7 +2363,7 @@ set_vectorcall(PyObject *type, PyObject * const*args, } static PySequenceMethods set_as_sequence = { - set_len, /* sq_length */ + (lenfunc)set_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ @@ -2424,7 +2596,7 @@ PySet_Size(PyObject *anyset) PyErr_BadInternalCall(); return -1; } - return PySet_GET_SIZE(anyset); + return set_len((PySetObject *)anyset); } int @@ -2434,7 +2606,8 @@ PySet_Clear(PyObject *set) PyErr_BadInternalCall(); return -1; } - return set_clear_internal((PySetObject *)set); + (void)set_clear((PySetObject *)set, NULL); + return 0; } int @@ -2444,7 +2617,12 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_contains_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_contains_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2454,7 +2632,12 @@ PySet_Discard(PyObject *set, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_discard_key((PySetObject *)set, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(set); + rv = set_discard_key((PySetObject *)set, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2465,9 +2648,15 @@ PySet_Add(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_add_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } +// TODO: Make thread-safe in free-threaded builds int _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash) { From 03f86b1b626ac5b0df1cc74d8f80ea11117aec8c Mon Sep 17 00:00:00 2001 From: Kerim Kabirov <39376984+Privat33r-dev@users.noreply.github.com> Date: Sat, 9 Mar 2024 10:00:37 +0100 Subject: [PATCH 07/35] GH-116218 Docs: Add availability information for the 'resource' module (#116256) Add availability data to 'resource' module Docs --- Doc/library/resource.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 4e58b043f1da31..389a63f089d850 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -177,6 +177,8 @@ platform. The largest area of mapped memory which the process may occupy. + .. availability:: FreeBSD >= 11. + .. data:: RLIMIT_AS From b4b4e764a798bab60324871074ce4cdebb9d01bb Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 9 Mar 2024 14:18:38 +0300 Subject: [PATCH 08/35] gh-116520: Fix error handling in `os_get_terminal_size_impl` in `posixmodule` (#116521) --- Modules/posixmodule.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 920a6750f5136d..940a9cc8955e11 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -14981,12 +14981,23 @@ os_get_terminal_size_impl(PyObject *module, int fd) termsize = PyStructSequence_New((PyTypeObject *)TerminalSizeType); if (termsize == NULL) return NULL; - PyStructSequence_SET_ITEM(termsize, 0, PyLong_FromLong(columns)); - PyStructSequence_SET_ITEM(termsize, 1, PyLong_FromLong(lines)); - if (PyErr_Occurred()) { - Py_DECREF(termsize); - return NULL; - } + + int pos = 0; + +#define SET_TERMSIZE(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(termsize); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(termsize, pos++, item); \ + } while(0) + + SET_TERMSIZE(PyLong_FromLong(columns)); + SET_TERMSIZE(PyLong_FromLong(lines)); +#undef SET_TERMSIZE + return termsize; } #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ From b9cb855621c0813846421e4ced97260a32e57063 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 9 Mar 2024 15:28:13 +0200 Subject: [PATCH 09/35] gh-115142: Skip ``test__xxsubinterpreters`` if ``_testinternalcapi`` is not available (#116507) --- Lib/test/test__xxsubinterpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index a76e4d0ade5b8a..35d7355680e549 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -7,7 +7,6 @@ import threading import unittest -import _testinternalcapi from test import support from test.support import import_helper from test.support import os_helper @@ -15,6 +14,7 @@ interpreters = import_helper.import_module('_xxsubinterpreters') +_testinternalcapi = import_helper.import_module('_testinternalcapi') from _xxsubinterpreters import InterpreterNotFoundError From 1e68c4b87633b17da1b602b86f5d23bbe106398f Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Sat, 9 Mar 2024 23:32:05 +0300 Subject: [PATCH 10/35] gh-111389: expose PyHASH_INF/BITS/MODULUS/IMAG macros as public (#111418) Co-authored-by: Victor Stinner --- Doc/c-api/hash.rst | 25 ++++++++++++++++++- Include/cpython/pyhash.h | 16 ++++++++---- ...-11-15-09-24-51.gh-issue-111418.FYYetY.rst | 2 ++ 3 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 91d88ae27bc9f4..1cf094cfcdca24 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -3,7 +3,7 @@ PyHash API ---------- -See also the :c:member:`PyTypeObject.tp_hash` member. +See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:type:: Py_hash_t @@ -17,6 +17,29 @@ See also the :c:member:`PyTypeObject.tp_hash` member. .. versionadded:: 3.2 +.. c:macro:: PyHASH_MODULUS + + The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_BITS + + The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_INF + + The hash value returned for a positive infinity. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_IMAG + + The multiplier used for the imaginary part of a complex number. + + .. versionadded:: 3.13 .. c:type:: PyHash_FuncDef diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index 396c208e1b106a..b476c3f357de92 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -10,14 +10,20 @@ reduction modulo the prime 2**_PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 +# define PyHASH_BITS 61 #else -# define _PyHASH_BITS 31 +# define PyHASH_BITS 31 #endif -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER +#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_INF 314159 +#define PyHASH_IMAG _PyHASH_MULTIPLIER + +/* Aliases kept for backward compatibility with Python 3.12 */ +#define _PyHASH_BITS PyHASH_BITS +#define _PyHASH_MODULUS PyHASH_MODULUS +#define _PyHASH_INF PyHASH_INF +#define _PyHASH_IMAG PyHASH_IMAG /* Helpers for hash functions */ PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); diff --git a/Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst b/Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst new file mode 100644 index 00000000000000..5f76ec1443fb44 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-15-09-24-51.gh-issue-111418.FYYetY.rst @@ -0,0 +1,2 @@ +Add :c:macro:`PyHASH_MODULUS`, :c:macro:`PyHASH_BITS`, :c:macro:`PyHASH_INF` +and :c:macro:`PyHASH_IMAG` C macros. Patch by Sergey B Kirpichev. From db8f423f58e336eb6180a70d9886b443d7203c2c Mon Sep 17 00:00:00 2001 From: Declan <5962877+dec1@users.noreply.github.com> Date: Sat, 9 Mar 2024 22:52:57 +0100 Subject: [PATCH 11/35] gh-116535: Fix distracting "TypeError" in example code (gh-116538) --- Doc/library/dataclasses.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 4ada69d63abada..c612c138fc6ea8 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -719,7 +719,7 @@ Using dataclasses, *if* this code was valid:: class D: x: list = [] # This code raises ValueError def add(self, element): - self.x += element + self.x.append(element) it would generate code similar to:: @@ -728,7 +728,7 @@ it would generate code similar to:: def __init__(self, x=x): self.x = x def add(self, element): - self.x += element + self.x.append(element) assert D().x is D().x From 17d31bf3843c384873999a15ce683cc3654f46ae Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 08:50:28 +0900 Subject: [PATCH 12/35] gh-112087: Store memory allocation information into _PyListArray (gh-116529) --- ...-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst | 1 + Objects/listobject.c | 126 ++++++++++++++++-- 2 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst new file mode 100644 index 00000000000000..07808dc325699b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-09-11-10-53.gh-issue-112087.nbI0Pw.rst @@ -0,0 +1 @@ +:class:`list` is now compatible with the implementation of :pep:`703`. diff --git a/Objects/listobject.c b/Objects/listobject.c index e013383a712bc8..c2aaab0ec3b415 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -31,6 +31,50 @@ get_list_freelist(void) } #endif +#ifdef Py_GIL_DISABLED +typedef struct { + Py_ssize_t allocated; + PyObject *ob_item[]; +} _PyListArray; + +static _PyListArray * +list_allocate_array(size_t capacity) +{ + if (capacity > PY_SSIZE_T_MAX/sizeof(PyObject*) - 1) { + return NULL; + } + _PyListArray *array = PyMem_Malloc(sizeof(_PyListArray) + capacity * sizeof(PyObject *)); + if (array == NULL) { + return NULL; + } + array->allocated = capacity; + return array; +} + +static Py_ssize_t +list_capacity(PyObject **items) +{ + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + return array->allocated; +} +#endif + +static void +free_list_items(PyObject** items, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + if (use_qsbr) { + _PyMem_FreeDelayed(array); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif +} + /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -47,8 +91,7 @@ get_list_freelist(void) static int list_resize(PyListObject *self, Py_ssize_t newsize) { - PyObject **items; - size_t new_allocated, num_allocated_bytes; + size_t new_allocated, target_bytes; Py_ssize_t allocated = self->allocated; /* Bypass realloc() when a previous overallocation is large enough @@ -80,9 +123,34 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(new_allocated); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + PyObject **old_items = self->ob_item; + if (self->ob_item) { + if (new_allocated < (size_t)allocated) { + target_bytes = new_allocated * sizeof(PyObject*); + } + else { + target_bytes = allocated * sizeof(PyObject*); + } + memcpy(array->ob_item, self->ob_item, target_bytes); + } + _Py_atomic_store_ptr_release(&self->ob_item, &array->ob_item); + self->allocated = new_allocated; + Py_SET_SIZE(self, newsize); + if (old_items != NULL) { + free_list_items(old_items, _PyObject_GC_IS_SHARED(self)); + } +#else + PyObject **items; if (new_allocated <= (size_t)PY_SSIZE_T_MAX / sizeof(PyObject *)) { - num_allocated_bytes = new_allocated * sizeof(PyObject *); - items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes); + target_bytes = new_allocated * sizeof(PyObject *); + items = (PyObject **)PyMem_Realloc(self->ob_item, target_bytes); } else { // integer overflow @@ -95,12 +163,14 @@ list_resize(PyListObject *self, Py_ssize_t newsize) self->ob_item = items; Py_SET_SIZE(self, newsize); self->allocated = new_allocated; +#endif return 0; } static int list_preallocate_exact(PyListObject *self, Py_ssize_t size) { + PyObject **items; assert(self->ob_item == NULL); assert(size > 0); @@ -110,11 +180,20 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) * allocated size up to the nearest even number. */ size = (size + 1) & ~(size_t)1; - PyObject **items = PyMem_New(PyObject*, size); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + items = array->ob_item; +#else + items = PyMem_New(PyObject*, size); if (items == NULL) { PyErr_NoMemory(); return -1; } +#endif self->ob_item = items; self->allocated = size; return 0; @@ -178,7 +257,17 @@ PyList_New(Py_ssize_t size) op->ob_item = NULL; } else { +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + memset(&array->ob_item, 0, size * sizeof(PyObject *)); + op->ob_item = array->ob_item; +#else op->ob_item = (PyObject **) PyMem_Calloc(size, sizeof(PyObject *)); +#endif if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -199,11 +288,20 @@ list_new_prealloc(Py_ssize_t size) return NULL; } assert(op->ob_item == NULL); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + op->ob_item = array->ob_item; +#else op->ob_item = PyMem_New(PyObject *, size); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); } +#endif op->allocated = size; return (PyObject *) op; } @@ -268,7 +366,7 @@ list_get_item_ref(PyListObject *op, Py_ssize_t i) if (ob_item == NULL) { return NULL; } - Py_ssize_t cap = _Py_atomic_load_ssize_relaxed(&op->allocated); + Py_ssize_t cap = list_capacity(ob_item); assert(cap != -1 && cap >= size); if (!valid_index(i, cap)) { return NULL; @@ -438,7 +536,7 @@ list_dealloc(PyObject *self) while (--i >= 0) { Py_XDECREF(op->ob_item[i]); } - PyMem_Free(op->ob_item); + free_list_items(op->ob_item, false); } #ifdef WITH_FREELISTS struct _Py_list_freelist *list_freelist = get_list_freelist(); @@ -737,12 +835,7 @@ list_clear_impl(PyListObject *a, bool is_resize) #else bool use_qsbr = false; #endif - if (use_qsbr) { - _PyMem_FreeDelayed(items); - } - else { - PyMem_Free(items); - } + free_list_items(items, use_qsbr); // Note that there is no guarantee that the list is actually empty // at this point, because XDECREF may have populated it indirectly again! } @@ -2758,7 +2851,12 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) while (--i >= 0) { Py_XDECREF(final_ob_item[i]); } - PyMem_Free(final_ob_item); +#ifdef Py_GIL_DISABLED + bool use_qsbr = _PyObject_GC_IS_SHARED(self); +#else + bool use_qsbr = false; +#endif + free_list_items(final_ob_item, use_qsbr); } return Py_XNewRef(result); } From 5b2f21faf388d8de5b388996cfd4f03430085764 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 10 Mar 2024 09:45:42 +0900 Subject: [PATCH 13/35] gh-112087: Make list.sort to be thread-safe for PEP 703. (gh-116553) --- Objects/clinic/listobject.c.h | 4 +++- Objects/listobject.c | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index a61550a49b66fc..b90dc0a0b463c6 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -268,7 +268,9 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); return_value = list_sort_impl(self, keyfunc, reverse); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -452,4 +454,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=26dfb2c9846348f9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a77eda9931ec0c20 input=a9049054013a1b77]*/ diff --git a/Objects/listobject.c b/Objects/listobject.c index c2aaab0ec3b415..164f363efe24f0 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2577,6 +2577,7 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] +@critical_section list.sort * @@ -2596,7 +2597,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ { MergeState ms; Py_ssize_t nremaining; @@ -2623,7 +2624,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) saved_ob_item = self->ob_item; saved_allocated = self->allocated; Py_SET_SIZE(self, 0); - self->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, NULL); self->allocated = -1; /* any operation will reset it to >= 0 */ if (keyfunc == NULL) { @@ -2843,8 +2844,8 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) final_ob_item = self->ob_item; i = Py_SIZE(self); Py_SET_SIZE(self, saved_ob_size); - self->ob_item = saved_ob_item; - self->allocated = saved_allocated; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, saved_ob_item); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, saved_allocated); if (final_ob_item != NULL) { /* we cannot use list_clear() for this because it does not guarantee that the list is really empty when it returns */ @@ -2870,7 +2871,9 @@ PyList_Sort(PyObject *v) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(v); v = list_sort_impl((PyListObject *)v, NULL, 0); + Py_END_CRITICAL_SECTION(); if (v == NULL) return -1; Py_DECREF(v); From 729bfb3105c7ca2ad6150a6207786096732c3b9e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Mar 2024 20:42:40 +0100 Subject: [PATCH 14/35] gh-116417: Avoid PyFloat_AS_DOUBLE() in AC limited C API (#116568) Argument Clinic no longer calls PyFloat_AS_DOUBLE() when the usage of the limited C API is requested. --- Modules/_testclinic_limited.c | 36 +++++++++++ Modules/clinic/_testclinic_limited.c.h | 84 +++++++++++++++++++++++++- Tools/clinic/clinic.py | 54 +++++++++++------ 3 files changed, 155 insertions(+), 19 deletions(-) diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index ef595be0b626db..df08ff9a369b1f 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -72,10 +72,46 @@ my_int_sum_impl(PyObject *module, int x, int y) } +/*[clinic input] +my_float_sum -> float + + x: float + y: float + / + +[clinic start generated code]*/ + +static float +my_float_sum_impl(PyObject *module, float x, float y) +/*[clinic end generated code: output=634f59a5a419cad7 input=d4b5313bdf4dc377]*/ +{ + return x + y; +} + + +/*[clinic input] +my_double_sum -> double + + x: double + y: double + / + +[clinic start generated code]*/ + +static double +my_double_sum_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=a75576d9e4d8557f input=16b11c8aba172801]*/ +{ + return x + y; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF MY_INT_FUNC_METHODDEF MY_INT_SUM_METHODDEF + MY_FLOAT_SUM_METHODDEF + MY_DOUBLE_SUM_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index eff72cee24975c..690e782b839cb5 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -91,4 +91,86 @@ my_int_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5cf64baf978d2288 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(my_float_sum__doc__, +"my_float_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_FLOAT_SUM_METHODDEF \ + {"my_float_sum", (PyCFunction)(void(*)(void))my_float_sum, METH_FASTCALL, my_float_sum__doc__}, + +static float +my_float_sum_impl(PyObject *module, float x, float y); + +static PyObject * +my_float_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float x; + float y; + float _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_float_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = (float) PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = (float) PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_float_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble((double)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(my_double_sum__doc__, +"my_double_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_DOUBLE_SUM_METHODDEF \ + {"my_double_sum", (PyCFunction)(void(*)(void))my_double_sum, METH_FASTCALL, my_double_sum__doc__}, + +static double +my_double_sum_impl(PyObject *module, double x, double y); + +static PyObject * +my_double_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_double_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_double_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=bb9f6b8c5d9e6a79 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0a8546247cc326..8353941f929eb1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3867,19 +3867,28 @@ class float_converter(CConverter): def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'f': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); - }}}} - else - {{{{ + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); + }}}} + else + {{{{ + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" {paramname} = (float) PyFloat_AsDouble({argname}); if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ goto exit; }}}} - }}}} - """, - argname=argname) + """, + argname=argname) return super().parse_arg(argname, displayname, limited_capi=limited_capi) class double_converter(CConverter): @@ -3890,19 +3899,28 @@ class double_converter(CConverter): def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'd': - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = PyFloat_AS_DOUBLE({argname}); - }}}} - else - {{{{ + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = PyFloat_AS_DOUBLE({argname}); + }}}} + else + {{{{ + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" {paramname} = PyFloat_AsDouble({argname}); if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ goto exit; }}}} - }}}} - """, - argname=argname) + """, + argname=argname) return super().parse_arg(argname, displayname, limited_capi=limited_capi) From c5fa796619a8cae5a1a8a4a043d05a99adec713d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 10 Mar 2024 21:19:47 +0100 Subject: [PATCH 15/35] gh-116417: Fix make check-c-globals for _testlimitedcapi (#116570) * Remove unused '_testcapimodule' global in Modules/_testcapi/unicode.c. * Update c-analyzer to not use the internal C API in _testlimitedcapi.c. --- Modules/_testcapi/unicode.c | 4 ---- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index a10183dddeca98..d0954bbe36ff9d 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -3,8 +3,6 @@ #include "parts.h" #include "util.h" -static struct PyModuleDef *_testcapimodule = NULL; // set at initialization - static PyObject * codec_incrementalencoder(PyObject *self, PyObject *args) { @@ -2098,8 +2096,6 @@ static PyMethodDef TestMethods[] = { int _PyTestCapi_Init_Unicode(PyObject *m) { - _testcapimodule = PyModule_GetDef(m); - if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index cc3a9be2104dbe..bb5ec5ae087a74 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -7,6 +7,7 @@ FILES_WITHOUT_INTERNAL_CAPI = frozenset(( # Modules/ '_testcapimodule.c', + '_testlimitedcapi.c', '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 0f212ecb528ac4..682dceaf8ae41e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -444,7 +444,6 @@ Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - Modules/_testcapi/mem.c - FmHook - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - -Modules/_testcapi/unicode.c - _testcapimodule - Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - From 2339e7cff745271f0e4a919573a347ab2bc1c2e9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 10 Mar 2024 22:00:33 +0000 Subject: [PATCH 16/35] gh-116057: Use relative recursion limits when testing os.walk() and Path.walk() (#116058) Replace test.support.set_recursion_limit with test.support.infinite_recursion. --- Lib/test/test_os.py | 4 ++-- Lib/test/test_pathlib/test_pathlib.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 3b2d5fccc30f3c..ae8b405dab50fc 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -34,7 +34,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot @@ -1496,7 +1496,7 @@ def test_walk_many_open_files(self): def test_walk_above_recursion_limit(self): depth = 50 os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) - with set_recursion_limit(depth - 5): + with infinite_recursion(depth - 5): all = list(self.walk(self.walk_path)) sub2_path = self.sub2_tree[0] diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 7e44ae61a5eba7..6509c08d227346 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -15,7 +15,7 @@ from test.support import import_helper from test.support import is_emscripten, is_wasi -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc @@ -1199,7 +1199,7 @@ def test_walk_above_recursion_limit(self): path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): + with infinite_recursion(recursion_limit): list(base.walk()) list(base.walk(top_down=False)) @@ -1239,7 +1239,7 @@ def test_glob_above_recursion_limit(self): path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): + with infinite_recursion(recursion_limit): list(base.glob('**/')) def test_glob_pathlike(self): From 4704e55a71c859c5d17cc2747ba62f49da58ea2d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Mar 2024 09:38:04 +0300 Subject: [PATCH 17/35] gh-116576: Fix `Tools/scripts/sortperf.py` sorting the same list (#116577) --- Tools/scripts/sortperf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/scripts/sortperf.py b/Tools/scripts/sortperf.py index b54681524ac173..1978a6c83dbe2b 100644 --- a/Tools/scripts/sortperf.py +++ b/Tools/scripts/sortperf.py @@ -130,7 +130,8 @@ def run(self, loops: int) -> float: def _prepare_data(self, loops: int) -> list[float]: bench = BENCHMARKS[self._name] - return [bench(self._size, self._random)] * loops + data = bench(self._size, self._random) + return [data.copy() for _ in range(loops)] def add_cmdline_args(cmd: list[str], args) -> None: From 8d7fde655fbb57e393831b9f30ebba80d6da366f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 11 Mar 2024 08:44:42 +0100 Subject: [PATCH 18/35] gh-116417: Argument Clinic: test generated Limited C API code for float args (#116573) --- Lib/test/test_clinic.py | 58 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 02a293e04b2182..cf3eeaadf0010c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -21,9 +21,9 @@ from clinic import DSLParser -def _make_clinic(*, filename='clinic_tests'): +def _make_clinic(*, filename='clinic_tests', limited_capi=False): clang = clinic.CLanguage(filename) - c = clinic.Clinic(clang, filename=filename, limited_capi=False) + c = clinic.Clinic(clang, filename=filename, limited_capi=limited_capi) c.block_parser = clinic.BlockParser('', clang) return c @@ -3614,6 +3614,46 @@ def test_depr_multi(self): self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") +class LimitedCAPIOutputTests(unittest.TestCase): + + def setUp(self): + self.clinic = _make_clinic(limited_capi=True) + + @staticmethod + def wrap_clinic_input(block): + return dedent(f""" + /*[clinic input] + output everything buffer + {block} + [clinic start generated code]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + + def test_limited_capi_float(self): + block = self.wrap_clinic_input(""" + func + f: float + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("float f;", generated) + self.assertIn("f = (float) PyFloat_AsDouble", generated) + + def test_limited_capi_double(self): + block = self.wrap_clinic_input(""" + func + f: double + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("double f;", generated) + self.assertIn("f = PyFloat_AsDouble", generated) + + try: import _testclinic_limited except ImportError: @@ -3644,6 +3684,20 @@ def test_my_int_sum(self): with self.assertRaises(TypeError): _testclinic_limited.my_int_sum(1, "str") + def test_my_double_sum(self): + for func in ( + _testclinic_limited.my_float_sum, + _testclinic_limited.my_double_sum, + ): + with self.subTest(func=func.__name__): + self.assertEqual(func(1.0, 2.5), 3.5) + with self.assertRaises(TypeError): + func() + with self.assertRaises(TypeError): + func(1) + with self.assertRaises(TypeError): + func(1., "2") + class PermutationTests(unittest.TestCase): From 4e5df2013fc29ed8bdb71572f1d12ff36e7028d5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 11 Mar 2024 09:30:15 +0000 Subject: [PATCH 19/35] GH-116468: Use constants instead of `oparg` in stack effects when `oparg` is known to be a constant. (GH-116469) --- Include/internal/pycore_opcode_metadata.h | 10 +-- Python/bytecodes.c | 30 ++++----- Python/executor_cases.c.h | 64 +++++++++----------- Python/generated_cases.c.h | 74 +++++++++++------------ Python/optimizer_cases.c.h | 28 +++++---- 5 files changed, 98 insertions(+), 108 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index efb731f1863283..05ff78de627caf 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -116,7 +116,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_LEN: return 2 + oparg; case CALL_LIST_APPEND: - return 2 + oparg; + return 3; case CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -130,11 +130,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_PY_WITH_DEFAULTS: return 2 + oparg; case CALL_STR_1: - return 2 + oparg; + return 3; case CALL_TUPLE_1: - return 2 + oparg; + return 3; case CALL_TYPE_1: - return 2 + oparg; + return 3; case CHECK_EG_MATCH: return 2; case CHECK_EXC_MATCH: @@ -879,7 +879,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case UNPACK_SEQUENCE_TUPLE: return oparg; case UNPACK_SEQUENCE_TWO_TUPLE: - return oparg; + return 2; case WITH_EXCEPT_START: return 5; case YIELD_VALUE: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bf9d2a316ceea0..03e5f4e330bdd8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -129,6 +129,8 @@ dummy_func( PyObject *top; PyObject *type; PyObject *typevars; + PyObject *val0; + PyObject *val1; int values_or_none; switch (opcode) { @@ -1223,13 +1225,13 @@ dummy_func( macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE; - inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) { + inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) { + assert(oparg == 2); DEOPT_IF(!PyTuple_CheckExact(seq)); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); - assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); DECREF_INPUTS(); } @@ -3236,39 +3238,33 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); - PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable + res = Py_NewRef(Py_TYPE(arg)); + Py_DECREF(arg); } - inst(CALL_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + inst(CALL_STR_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { + inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } @@ -3490,14 +3486,14 @@ dummy_func( } // This is secretly a super-instruction - tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { + tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) { assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append); assert(self != NULL); DEOPT_IF(!PyList_Check(self)); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ccd8fb32989e92..42e884c20ba04f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1061,18 +1061,20 @@ case _UNPACK_SEQUENCE_TWO_TUPLE: { PyObject *seq; - PyObject **values; + PyObject *val1; + PyObject *val0; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + assert(oparg == 2); if (!PyTuple_CheckExact(seq)) goto deoptimize; if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize; - assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); Py_DECREF(seq); - stack_pointer += -1 + oparg; + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; break; } @@ -3057,71 +3059,65 @@ /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ case _CALL_TYPE_1: { - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); if (null != NULL) goto deoptimize; - PyObject *obj = args[0]; if (callable != (PyObject *)&PyType_Type) goto deoptimize; STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + res = Py_NewRef(Py_TYPE(arg)); + Py_DECREF(arg); + stack_pointer[-3] = res; + stack_pointer += -2; break; } case _CALL_STR_1: { - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); if (null != NULL) goto deoptimize; if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize; STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error_tier_two; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); break; } case _CALL_TUPLE_1: { - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); if (null != NULL) goto deoptimize; if (callable != (PyObject *)&PyTuple_Type) goto deoptimize; STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error_tier_two; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f5d125c63b10ca..53c0211be2fe6c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1472,21 +1472,21 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *self; PyObject *callable; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); assert(self != NULL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); @@ -1810,26 +1810,24 @@ next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1839,26 +1837,24 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1868,25 +1864,23 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; + PyObject *arg; PyObject *null; PyObject *callable; PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + res = Py_NewRef(Py_TYPE(arg)); + Py_DECREF(arg); + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } @@ -5910,18 +5904,20 @@ INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; - PyObject **values; + PyObject *val1; + PyObject *val0; /* Skip 1 cache entry */ seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + assert(oparg == 2); DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); Py_DECREF(seq); - stack_pointer += -1 + oparg; + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c88b25132c0e90..fed5730d2e50c1 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -684,13 +684,15 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsSymbol **values; - values = &stack_pointer[-1]; - for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); - if (values[_i] == NULL) goto out_of_space; - } - stack_pointer += -1 + oparg; + _Py_UopsSymbol *val1; + _Py_UopsSymbol *val0; + val1 = sym_new_unknown(ctx); + if (val1 == NULL) goto out_of_space; + val0 = sym_new_unknown(ctx); + if (val0 == NULL) goto out_of_space; + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; break; } @@ -1632,8 +1634,8 @@ _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-3] = res; + stack_pointer += -2; break; } @@ -1641,8 +1643,8 @@ _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-3] = res; + stack_pointer += -2; break; } @@ -1650,8 +1652,8 @@ _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-3] = res; + stack_pointer += -2; break; } From d8712fa0c75ad5ea56543903fa45674ab47cc647 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 11 Mar 2024 11:57:07 +0200 Subject: [PATCH 20/35] gh-88352: Make TimedRotatingFileHandler tests more stable (GH-116409) The tests failed (with less than 1% probability) if for example the file was created at 11:46:03.999, but the record was emitted at 11:46:04.001, with atTime=11:46:04, which caused an unexpected rollover. Ensure that the tests are always run within the range of the same whole second. Also share code between test_rollover_at_midnight and test_rollover_at_weekday. --- Lib/test/test_logging.py | 55 +++++++++++++--------------------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d71385bf2c78d7..1dfad6e1d2609b 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -6080,46 +6080,22 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) - def test_rollover_at_midnight(self): - atTime = datetime.datetime.now().time() - fmt = logging.Formatter('%(asctime)s %(message)s') - for i in range(3): - fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) - fh.setFormatter(fmt) - r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) - fh.emit(r2) - fh.close() - self.assertLogFile(self.fn) - with open(self.fn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing1 {i}', line) - - os.utime(self.fn, (time.time() - 1,)*2) - for i in range(2): - fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) - fh.setFormatter(fmt) - r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) - fh.emit(r2) - fh.close() - rolloverDate = datetime.datetime.now() - datetime.timedelta(days=1) - otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' - self.assertLogFile(otherfn) - with open(self.fn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing2 {i}', line) - with open(otherfn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing1 {i}', line) - - def test_rollover_at_weekday(self): + def test_rollover_at_midnight(self, weekly=False): + os_helper.unlink(self.fn) now = datetime.datetime.now() atTime = now.time() + if not 0.1 < atTime.microsecond/1e6 < 0.9: + # The test requires all records to be emitted within + # the range of the same whole second. + time.sleep((0.1 - atTime.microsecond/1e6) % 1.0) + now = datetime.datetime.now() + atTime = now.time() + atTime = atTime.replace(microsecond=0) fmt = logging.Formatter('%(asctime)s %(message)s') + when = f'W{now.weekday()}' if weekly else 'MIDNIGHT' for i in range(3): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + self.fn, encoding="utf-8", when=when, atTime=atTime) fh.setFormatter(fmt) r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) fh.emit(r2) @@ -6129,15 +6105,15 @@ def test_rollover_at_weekday(self): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) - os.utime(self.fn, (time.time() - 1,)*2) + os.utime(self.fn, (now.timestamp() - 1,)*2) for i in range(2): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + self.fn, encoding="utf-8", when=when, atTime=atTime) fh.setFormatter(fmt) r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) fh.emit(r2) fh.close() - rolloverDate = datetime.datetime.now() - datetime.timedelta(days=7) + rolloverDate = now - datetime.timedelta(days=7 if weekly else 1) otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' self.assertLogFile(otherfn) with open(self.fn, encoding="utf-8") as f: @@ -6147,6 +6123,9 @@ def test_rollover_at_weekday(self): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) + def test_rollover_at_weekday(self): + self.test_rollover_at_midnight(weekly=True) + def test_invalid(self): assertRaises = self.assertRaises assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, From 1cc02ca063f50b8c527fbdde9957b03c145c1575 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Mar 2024 11:28:16 +0100 Subject: [PATCH 21/35] gh-116417: Move 4 limited C API test files to _testlimitedcapi (#116571) Move the following files from Modules/_testcapi/ to Modules/_testlimitedcapi/: * bytearray.c * bytes.c * pyos.c * sys.c Changes: * Replace PyBytes_AS_STRING() with PyBytes_AsString(). * Replace PyBytes_GET_SIZE() with PyBytes_Size(). * Update related test_capi tests. * Copy Modules/_testcapi/util.h to Modules/_testlimitedcapi/util.h. --- Lib/test/test_capi/test_bytearray.py | 18 +++++----- Lib/test/test_capi/test_bytes.py | 28 ++++++++-------- Lib/test/test_capi/test_sys.py | 16 ++++----- Modules/Setup.stdlib.in | 4 +-- Modules/_testcapi/parts.h | 4 --- Modules/_testcapimodule.c | 12 ------- Modules/_testlimitedcapi.c | 14 +++++++- .../bytearray.c | 0 .../{_testcapi => _testlimitedcapi}/bytes.c | 8 ++--- Modules/_testlimitedcapi/parts.h | 6 +++- .../{_testcapi => _testlimitedcapi}/pyos.c | 0 Modules/{_testcapi => _testlimitedcapi}/sys.c | 0 Modules/_testlimitedcapi/util.h | 33 +++++++++++++++++++ PCbuild/_testcapi.vcxproj | 4 --- PCbuild/_testcapi.vcxproj.filters | 12 ------- PCbuild/_testlimitedcapi.vcxproj | 8 +++-- PCbuild/_testlimitedcapi.vcxproj.filters | 14 ++++---- 17 files changed, 101 insertions(+), 80 deletions(-) rename Modules/{_testcapi => _testlimitedcapi}/bytearray.c (100%) rename Modules/{_testcapi => _testlimitedcapi}/bytes.c (95%) rename Modules/{_testcapi => _testlimitedcapi}/pyos.c (100%) rename Modules/{_testcapi => _testlimitedcapi}/sys.c (100%) create mode 100644 Modules/_testlimitedcapi/util.h diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 833122c4e319d8..39099f6b82240f 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,7 +1,7 @@ import unittest from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -19,7 +19,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyByteArray_Check() - check = _testcapi.bytearray_check + check = _testlimitedcapi.bytearray_check self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertTrue(check(ByteArraySubclass(b'abc'))) @@ -32,7 +32,7 @@ def test_check(self): def test_checkexact(self): # Test PyByteArray_CheckExact() - check = _testcapi.bytearray_checkexact + check = _testlimitedcapi.bytearray_checkexact self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertFalse(check(ByteArraySubclass(b'abc'))) @@ -45,7 +45,7 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyByteArray_FromStringAndSize() - fromstringandsize = _testcapi.bytearray_fromstringandsize + fromstringandsize = _testlimitedcapi.bytearray_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), bytearray(b'abc')) self.assertEqual(fromstringandsize(b'abc', 2), bytearray(b'ab')) @@ -62,7 +62,7 @@ def test_fromstringandsize(self): def test_fromobject(self): # Test PyByteArray_FromObject() - fromobject = _testcapi.bytearray_fromobject + fromobject = _testlimitedcapi.bytearray_fromobject self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) @@ -77,7 +77,7 @@ def test_fromobject(self): def test_size(self): # Test PyByteArray_Size() - size = _testcapi.bytearray_size + size = _testlimitedcapi.bytearray_size self.assertEqual(size(bytearray(b'abc')), 3) self.assertEqual(size(ByteArraySubclass(b'abc')), 3) @@ -88,7 +88,7 @@ def test_size(self): def test_asstring(self): """Test PyByteArray_AsString()""" - asstring = _testcapi.bytearray_asstring + asstring = _testlimitedcapi.bytearray_asstring self.assertEqual(asstring(bytearray(b'abc'), 4), b'abc\0') self.assertEqual(asstring(ByteArraySubclass(b'abc'), 4), b'abc\0') @@ -100,7 +100,7 @@ def test_asstring(self): def test_concat(self): """Test PyByteArray_Concat()""" - concat = _testcapi.bytearray_concat + concat = _testlimitedcapi.bytearray_concat ba = bytearray(b'abc') self.assertEqual(concat(ba, b'def'), bytearray(b'abcdef')) @@ -133,7 +133,7 @@ def test_concat(self): def test_resize(self): """Test PyByteArray_Resize()""" - resize = _testcapi.bytearray_resize + resize = _testlimitedcapi.bytearray_resize ba = bytearray(b'abcdef') self.assertEqual(resize(ba, 3), 0) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index bb5d724ff187d4..a2ba7708f8fd26 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,7 +1,7 @@ import unittest from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -19,7 +19,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyBytes_Check() - check = _testcapi.bytes_check + check = _testlimitedcapi.bytes_check self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -33,7 +33,7 @@ def test_check(self): def test_checkexact(self): # Test PyBytes_CheckExact() - check = _testcapi.bytes_checkexact + check = _testlimitedcapi.bytes_checkexact self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -47,7 +47,7 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyBytes_FromStringAndSize() - fromstringandsize = _testcapi.bytes_fromstringandsize + fromstringandsize = _testlimitedcapi.bytes_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), b'abc') self.assertEqual(fromstringandsize(b'abc', 2), b'ab') @@ -65,7 +65,7 @@ def test_fromstringandsize(self): def test_fromstring(self): # Test PyBytes_FromString() - fromstring = _testcapi.bytes_fromstring + fromstring = _testlimitedcapi.bytes_fromstring self.assertEqual(fromstring(b'abc\0def'), b'abc') self.assertEqual(fromstring(b''), b'') @@ -74,7 +74,7 @@ def test_fromstring(self): def test_fromobject(self): # Test PyBytes_FromObject() - fromobject = _testcapi.bytes_fromobject + fromobject = _testlimitedcapi.bytes_fromobject self.assertEqual(fromobject(b'abc'), b'abc') self.assertEqual(fromobject(bytearray(b'abc')), b'abc') @@ -88,7 +88,7 @@ def test_fromobject(self): def test_size(self): # Test PyBytes_Size() - size = _testcapi.bytes_size + size = _testlimitedcapi.bytes_size self.assertEqual(size(b'abc'), 3) self.assertEqual(size(BytesSubclass(b'abc')), 3) @@ -100,7 +100,7 @@ def test_size(self): def test_asstring(self): """Test PyBytes_AsString()""" - asstring = _testcapi.bytes_asstring + asstring = _testlimitedcapi.bytes_asstring self.assertEqual(asstring(b'abc', 4), b'abc\0') self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') @@ -111,8 +111,8 @@ def test_asstring(self): def test_asstringandsize(self): """Test PyBytes_AsStringAndSize()""" - asstringandsize = _testcapi.bytes_asstringandsize - asstringandsize_null = _testcapi.bytes_asstringandsize_null + asstringandsize = _testlimitedcapi.bytes_asstringandsize + asstringandsize_null = _testlimitedcapi.bytes_asstringandsize_null self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) @@ -128,7 +128,7 @@ def test_asstringandsize(self): def test_repr(self): # Test PyBytes_Repr() - bytes_repr = _testcapi.bytes_repr + bytes_repr = _testlimitedcapi.bytes_repr self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") @@ -149,7 +149,7 @@ def test_repr(self): def test_concat(self, concat=None): """Test PyBytes_Concat()""" if concat is None: - concat = _testcapi.bytes_concat + concat = _testlimitedcapi.bytes_concat self.assertEqual(concat(b'abc', b'def'), b'abcdef') self.assertEqual(concat(b'a\0b', b'c\0d'), b'a\0bc\0d') @@ -182,11 +182,11 @@ def test_concat(self, concat=None): def test_concatanddel(self): """Test PyBytes_ConcatAndDel()""" - self.test_concat(_testcapi.bytes_concatanddel) + self.test_concat(_testlimitedcapi.bytes_concatanddel) def test_decodeescape(self): """Test PyBytes_DecodeEscape()""" - decodeescape = _testcapi.bytes_decodeescape + decodeescape = _testlimitedcapi.bytes_decodeescape self.assertEqual(decodeescape(b'abc'), b'abc') self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py index 08bf0370fc59b5..54a8e026d883d4 100644 --- a/Lib/test/test_capi/test_sys.py +++ b/Lib/test/test_capi/test_sys.py @@ -5,9 +5,9 @@ from test.support import import_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None NULL = None @@ -20,10 +20,10 @@ class CAPITest(unittest.TestCase): maxDiff = None @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getobject(self): # Test PySys_GetObject() - getobject = _testcapi.sys_getobject + getobject = _testlimitedcapi.sys_getobject self.assertIs(getobject(b'stdout'), sys.stdout) with support.swap_attr(sys, '\U0001f40d', 42): @@ -38,10 +38,10 @@ def test_sys_getobject(self): # CRASHES getobject(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_setobject(self): # Test PySys_SetObject() - setobject = _testcapi.sys_setobject + setobject = _testlimitedcapi.sys_setobject value = ['value'] value2 = ['value2'] @@ -70,10 +70,10 @@ def test_sys_setobject(self): # CRASHES setobject(NULL, value) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getxoptions(self): # Test PySys_GetXOptions() - getxoptions = _testcapi.sys_getxoptions + getxoptions = _testlimitedcapi.sys_getxoptions self.assertIs(getxoptions(), sys._xoptions) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index e8eaafd8b99b98..deada66cf1a807 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -162,8 +162,8 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/sys.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/heaptype_relative.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/pyos.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 2a043cfa73fa8b..f9bdd830775a75 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -31,8 +31,6 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); int _PyTestCapi_Init_Abstract(PyObject *module); -int _PyTestCapi_Init_ByteArray(PyObject *module); -int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); int _PyTestCapi_Init_DateTime(PyObject *module); @@ -52,12 +50,10 @@ int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyAtomic(PyObject *module); -int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_File(PyObject *module); int _PyTestCapi_Init_Codec(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); -int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b6536045e645f9..b5e646f904b2d1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4017,12 +4017,6 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Abstract(m) < 0) { return NULL; } - if (_PyTestCapi_Init_ByteArray(m) < 0) { - return NULL; - } - if (_PyTestCapi_Init_Bytes(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Unicode(m) < 0) { return NULL; } @@ -4077,18 +4071,12 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Buffer(m) < 0) { return NULL; } - if (_PyTestCapi_Init_PyOS(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_File(m) < 0) { return NULL; } if (_PyTestCapi_Init_Codec(m) < 0) { return NULL; } - if (_PyTestCapi_Init_Sys(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Immortal(m) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index da09e3f5084464..49bf6a3ea39613 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -26,11 +26,23 @@ PyInit__testlimitedcapi(void) return NULL; } - if (_PyTestCapi_Init_VectorcallLimited(mod) < 0) { + if (_PyTestCapi_Init_ByteArray(mod) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Bytes(mod) < 0) { return NULL; } if (_PyTestCapi_Init_HeaptypeRelative(mod) < 0) { return NULL; } + if (_PyTestCapi_Init_PyOS(mod) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Sys(mod) < 0) { + return NULL; + } + if (_PyTestCapi_Init_VectorcallLimited(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testcapi/bytearray.c b/Modules/_testlimitedcapi/bytearray.c similarity index 100% rename from Modules/_testcapi/bytearray.c rename to Modules/_testlimitedcapi/bytearray.c diff --git a/Modules/_testcapi/bytes.c b/Modules/_testlimitedcapi/bytes.c similarity index 95% rename from Modules/_testcapi/bytes.c rename to Modules/_testlimitedcapi/bytes.c index da10503f6f6856..a14c4f9d4d30a8 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testlimitedcapi/bytes.c @@ -160,8 +160,8 @@ bytes_concat(PyObject *Py_UNUSED(module), PyObject *args) if (new) { assert(left != NULL); assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); if (left == NULL) { return NULL; } @@ -191,8 +191,8 @@ bytes_concatanddel(PyObject *Py_UNUSED(module), PyObject *args) if (new) { assert(left != NULL); assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); if (left == NULL) { return NULL; } diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 83590a7afd3228..039576d5cf4c29 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -21,7 +21,11 @@ # error "Py_BUILD_CORE macro must not be defined" #endif -int _PyTestCapi_Init_VectorcallLimited(PyObject *module); +int _PyTestCapi_Init_ByteArray(PyObject *module); +int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); +int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Sys(PyObject *module); +int _PyTestCapi_Init_VectorcallLimited(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testcapi/pyos.c b/Modules/_testlimitedcapi/pyos.c similarity index 100% rename from Modules/_testcapi/pyos.c rename to Modules/_testlimitedcapi/pyos.c diff --git a/Modules/_testcapi/sys.c b/Modules/_testlimitedcapi/sys.c similarity index 100% rename from Modules/_testcapi/sys.c rename to Modules/_testlimitedcapi/sys.c diff --git a/Modules/_testlimitedcapi/util.h b/Modules/_testlimitedcapi/util.h new file mode 100644 index 00000000000000..f26d7656a10138 --- /dev/null +++ b/Modules/_testlimitedcapi/util.h @@ -0,0 +1,33 @@ +#define NULLABLE(x) do { \ + if (x == Py_None) { \ + x = NULL; \ + } \ + } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + +/* Marker to check that pointer value was set. */ +static const char uninitialized[] = "uninitialized"; +#define UNINITIALIZED_PTR ((void *)uninitialized) +/* Marker to check that Py_ssize_t value was set. */ +#define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) +/* Marker to check that integer value was set. */ +#define UNINITIALIZED_INT (63256717) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 3ca4c5f86c783c..6522cb1fcf5c63 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -98,8 +98,6 @@ - - @@ -118,10 +116,8 @@ - - diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 651eb1d6ba0b7f..772a9a861517ec 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -30,12 +30,6 @@ Source Files - - Source Files - - - Source Files - Source Files @@ -90,18 +84,12 @@ Source Files - - Source Files - Source Files Source Files - - Source Files - Source Files diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 1b27942c1da9ae..1afeacaa93396e 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -94,8 +94,12 @@ - + + + + + @@ -113,4 +117,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index c4764a55f410a0..b3eeb86185c372 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -9,12 +9,12 @@ - - Source Files - - - Source Files - + + + + + + @@ -22,4 +22,4 @@ Resource Files - \ No newline at end of file + From ffd79bea0f032df5a2e7f75e8c823a09cdc7c7a2 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Mar 2024 13:58:24 +0300 Subject: [PATCH 22/35] gh-116545: Fix error handling in `mkpwent` in `pwdmodule` (#116548) --- Modules/pwdmodule.c | 61 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index b7034369c4731e..c59a8e41aa292a 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -64,53 +64,52 @@ static struct PyModuleDef pwdmodule; #define DEFAULT_BUFFER_SIZE 1024 -static void -sets(PyObject *v, int i, const char* val) -{ - if (val) { - PyObject *o = PyUnicode_DecodeFSDefault(val); - PyStructSequence_SET_ITEM(v, i, o); - } - else { - PyStructSequence_SET_ITEM(v, i, Py_None); - Py_INCREF(Py_None); - } -} - static PyObject * mkpwent(PyObject *module, struct passwd *p) { - int setIndex = 0; PyObject *v = PyStructSequence_New(get_pwd_state(module)->StructPwdType); - if (v == NULL) + if (v == NULL) { return NULL; + } + + int setIndex = 0; + +#define SET_STRING(VAL) \ + SET_RESULT((VAL) ? PyUnicode_DecodeFSDefault((VAL)) : Py_NewRef(Py_None)) -#define SETS(i,val) sets(v, i, val) +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SET_ITEM(v, setIndex++, item); \ + } while(0) - SETS(setIndex++, p->pw_name); + SET_STRING(p->pw_name); #if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) && !defined(__ANDROID__) - SETS(setIndex++, p->pw_passwd); + SET_STRING(p->pw_passwd); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromUid(p->pw_uid)); - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromGid(p->pw_gid)); + SET_RESULT(_PyLong_FromUid(p->pw_uid)); + SET_RESULT(_PyLong_FromGid(p->pw_gid)); #if defined(HAVE_STRUCT_PASSWD_PW_GECOS) - SETS(setIndex++, p->pw_gecos); + SET_STRING(p->pw_gecos); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - SETS(setIndex++, p->pw_dir); - SETS(setIndex++, p->pw_shell); - -#undef SETS + SET_STRING(p->pw_dir); + SET_STRING(p->pw_shell); - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_STRING +#undef SET_RESULT return v; + +error: + Py_DECREF(v); + return NULL; } /*[clinic input] From 817fe33a1da747c57b467f73a47b701c0b0eb911 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 11 Mar 2024 16:25:04 +0300 Subject: [PATCH 23/35] gh-116590: Fix unused `current_thread_holds_gil` function warning (#116591) --- Python/ceval_gil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index f5c44307a513f8..edfc466d9f20ec 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -417,6 +417,7 @@ PyEval_ThreadsInitialized(void) return _PyEval_ThreadsInitialized(); } +#ifndef NDEBUG static inline int current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { @@ -425,6 +426,7 @@ current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) } return _Py_atomic_load_int_relaxed(&gil->locked); } +#endif static void init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) From 6c4fc209e1941958164509204cdc3505130c1820 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 11 Mar 2024 22:25:55 +0900 Subject: [PATCH 24/35] gh-112536: Define MI_TSAN to 1 for --with-mimalloc and --with-thread-sanitizer (gh-116558) --- Include/internal/pycore_mimalloc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 3ef0154ba76d20..10d451398f1410 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -32,6 +32,10 @@ typedef enum { # define MI_DEBUG 0 #endif +#ifdef _Py_THREAD_SANITIZER +# define MI_TSAN 1 +#endif + #include "mimalloc.h" #include "mimalloc/types.h" #include "mimalloc/internal.h" From b6ae6da1bd987506b599a30e37fb452f909b5cbe Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 11 Mar 2024 13:37:48 +0000 Subject: [PATCH 25/35] GH-116596: Better determination of escaping uops. (GH-116597) --- Include/internal/pycore_opcode_metadata.h | 24 +++++++++---------- Include/internal/pycore_uop_metadata.h | 28 +++++++++++------------ Python/optimizer.c | 1 + Python/optimizer_analysis.c | 8 +++---- Tools/cases_generator/analyzer.py | 16 +++++++++++++ 5 files changed, 47 insertions(+), 30 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 05ff78de627caf..de93d4ef14de2a 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -959,7 +959,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, @@ -973,12 +973,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1008,8 +1008,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1035,7 +1035,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1075,9 +1075,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -1132,7 +1132,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 7f984d88299b62..62405a362fd7ab 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -65,7 +65,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -114,9 +114,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, - [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -146,12 +146,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS_1] = 0, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, + [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -175,19 +175,19 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_NEXT_TUPLE] = 0, [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, + [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, @@ -199,7 +199,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = HAS_ESCAPES_FLAG, + [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -216,12 +216,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, diff --git a/Python/optimizer.c b/Python/optimizer.c index 4dbfd918a51f62..aaf75b2339cd2e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1033,6 +1033,7 @@ uop_optimize( break; } assert(_PyOpcode_uop_name[buffer[pc].opcode]); + assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); if (executor == NULL) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 51ec14b718f291..9fd4b1967ecc3b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -430,7 +430,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) int opcode = buffer[pc].opcode; switch (opcode) { case _SET_IP: - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; last_set_ip = pc; break; case _CHECK_VALIDITY: @@ -438,7 +438,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) may_have_escaped = false; } else { - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } break; case _CHECK_VALIDITY_AND_SET_IP: @@ -447,7 +447,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _CHECK_VALIDITY; } else { - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } last_set_ip = pc; break; @@ -463,7 +463,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) last->opcode == _COPY ) { last->opcode = _NOP; - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } if (last->opcode == _REPLACE_WITH_TRUE) { last->opcode = _NOP; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b0a15e6d87c2c6..27e6ba2b3fdedf 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -335,6 +335,7 @@ def is_infallible(op: parser.InstDef) -> bool: "_PyDictOrValues_IsValues", "_PyObject_DictOrValuesPointer", "_PyDictOrValues_GetValues", + "_PyDictValues_AddToInsertionOrder", "_PyObject_MakeInstanceAttributesFromDict", "Py_DECREF", "_Py_DECREF_SPECIALIZED", @@ -355,8 +356,10 @@ def is_infallible(op: parser.InstDef) -> bool: "_PyLong_IsCompact", "_PyLong_IsNonNegativeCompact", "_PyLong_CompactValue", + "_PyLong_DigitCount", "_Py_NewRef", "_Py_IsImmortal", + "PyLong_FromLong", "_Py_STR", "_PyLong_Add", "_PyLong_Multiply", @@ -368,6 +371,17 @@ def is_infallible(op: parser.InstDef) -> bool: "_Py_atomic_load_uintptr_relaxed", "_PyFrame_GetCode", "_PyThreadState_HasStackSpace", + "_PyUnicode_Equal", + "_PyFrame_SetStackPointer", + "_PyType_HasFeature", + "PyUnicode_Concat", + "_PyList_FromArraySteal", + "_PyTuple_FromArraySteal", + "PySlice_New", + "_Py_LeaveRecursiveCallPy", + "CALL_STAT_INC", + "maybe_lltrace_resume_frame", + "_PyUnicode_JoinArray", ) ESCAPING_FUNCTIONS = ( @@ -379,6 +393,8 @@ def is_infallible(op: parser.InstDef) -> bool: def makes_escaping_api_call(instr: parser.InstDef) -> bool: if "CALL_INTRINSIC" in instr.name: return True + if instr.name == "_BINARY_OP": + return True tkns = iter(instr.tokens) for tkn in tkns: if tkn.kind != lexer.IDENTIFIER: From 546eb7a3be241c5abd8a83cebbbab8c71107edcf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Mar 2024 15:20:04 +0100 Subject: [PATCH 26/35] gh-116417: Build _testinternalcapi with limited C API version 3.5 (#116598) --- Modules/_testlimitedcapi/heaptype_relative.c | 6 ++++++ Modules/_testlimitedcapi/parts.h | 5 +++-- Modules/_testlimitedcapi/vectorcall_limited.c | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Modules/_testlimitedcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c index 9878a4daf1b2cb..d0316dd4fc63b4 100644 --- a/Modules/_testlimitedcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -1,3 +1,9 @@ +// Need limited C API version 3.12 for PyType_FromMetaclass() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +# define Py_LIMITED_API 0x030c0000 +#endif + #include "parts.h" #include // max_align_t #include // memset diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 039576d5cf4c29..9bc52413382eb5 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -7,8 +7,9 @@ #include "pyconfig.h" // Py_GIL_DISABLED // Use the limited C API -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 // 3.12 +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) + // need limited C API version 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 #endif // Make sure that the internal C API cannot be used. diff --git a/Modules/_testlimitedcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c index 24aa5e991eca3f..fc1a89c9098e1b 100644 --- a/Modules/_testlimitedcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -1,5 +1,11 @@ /* Test Vectorcall in the limited API */ +// Need limited C API version 3.12 for PyObject_Vectorcall() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +# define Py_LIMITED_API 0x030c0000 +#endif + #include "parts.h" #include "clinic/vectorcall_limited.c.h" From 2731913dd5234ff5ab630a3b7f1c98ad79d4d9df Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Mon, 11 Mar 2024 11:02:58 -0400 Subject: [PATCH 27/35] gh-116167: Allow disabling the GIL with `PYTHON_GIL=0` or `-X gil=0` (#116338) In free-threaded builds, running with `PYTHON_GIL=0` will now disable the GIL. Follow-up issues track work to re-enable the GIL when loading an incompatible extension, and to disable the GIL by default. In order to support re-enabling the GIL at runtime, all GIL-related data structures are initialized as usual, and disabling the GIL simply sets a flag that causes `take_gil()` and `drop_gil()` to return early. --- Doc/using/cmdline.rst | 18 +++++++++++ Include/cpython/initconfig.h | 3 ++ Include/internal/pycore_gil.h | 5 ++++ Include/internal/pycore_initconfig.h | 12 ++++++++ Lib/subprocess.py | 2 +- Lib/test/_test_embed_set_config.py | 14 +++++++++ Lib/test/test_cmd_line.py | 33 ++++++++++++++++++++ Lib/test/test_embed.py | 2 ++ Misc/python.man | 4 +++ Python/ceval_gil.c | 15 ++++++++++ Python/initconfig.c | 45 ++++++++++++++++++++++++++++ Python/sysmodule.c | 11 +++++++ 12 files changed, 163 insertions(+), 1 deletion(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 0a7f6363a2b628..36cddffb9eae34 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -559,6 +559,9 @@ Miscellaneous options :mod:`__main__`. This can be used to execute code early during Python initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. + * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, + respectively. Only available in builds configured with + :option:`--disable-gil`. See also :envvar:`PYTHON_GIL`. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -601,6 +604,9 @@ Miscellaneous options .. versionchanged:: 3.13 Added the ``-X cpu_count`` and ``-X presite`` options. + .. versionchanged:: 3.13 + Added the ``-X gil`` option. + .. _using-on-controlling-color: Controlling color @@ -1138,6 +1144,18 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_GIL + + If this variable is set to ``1``, the global interpreter lock (GIL) will be + forced on. Setting it to ``0`` forces the GIL off. + + See also the :option:`-X gil <-X>` command-line option, which takes + precedence over this variable. + + Needs Python configured with the :option:`--disable-gil` build option. + + .. versionadded:: 3.13 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 87c059c521cbc9..5da5ef9e5431b1 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -181,6 +181,9 @@ typedef struct PyConfig { int int_max_str_digits; int cpu_count; +#ifdef Py_GIL_DISABLED + int enable_gil; +#endif /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 19b0d23a68568a..d36b4c0db010b2 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -20,6 +20,11 @@ extern "C" { #define FORCE_SWITCHING struct _gil_runtime_state { +#ifdef Py_GIL_DISABLED + /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime + if, for example, a module that requires the GIL is loaded. */ + int enabled; +#endif /* microseconds (the Python API uses seconds, though) */ unsigned long interval; /* Last PyThreadState holding / having held the GIL. This helps us diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index c86988234f6a05..1c68161341860a 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,6 +153,18 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; +typedef enum { + /* For now, this means the GIL is enabled. + + gh-116329: This will eventually change to "the GIL is disabled but can + be reenabled by loading an incompatible extension module." */ + _PyConfig_GIL_DEFAULT = -1, + + /* The GIL has been forced off or on, and will not be affected by module loading. */ + _PyConfig_GIL_DISABLE = 0, + _PyConfig_GIL_ENABLE = 1, +} _PyConfigGILEnum; + // Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 20db7747d5db13..1437bf8148282c 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -350,7 +350,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index 75b6b7d1b39fa4..5ff521892cb6fe 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -9,6 +9,7 @@ import os import sys import unittest +from test import support from test.support import MS_WINDOWS @@ -211,6 +212,19 @@ def test_flags(self): self.set_config(use_hash_seed=1, hash_seed=123) self.assertEqual(sys.flags.hash_randomization, 1) + if support.Py_GIL_DISABLED: + self.set_config(enable_gil=-1) + self.assertEqual(sys.flags.gil, None) + self.set_config(enable_gil=0) + self.assertEqual(sys.flags.gil, 0) + self.set_config(enable_gil=1) + self.assertEqual(sys.flags.gil, 1) + else: + # Builds without Py_GIL_DISABLED don't have + # PyConfig.enable_gil. sys.flags.gil is always defined to 1, for + # consistency. + self.assertEqual(sys.flags.gil, 1) + def test_options(self): self.check(warnoptions=[]) self.check(warnoptions=["default", "ignore"]) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 6796dc6e3570e9..c633f6493cfab7 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -869,6 +869,39 @@ def test_pythondevmode_env(self): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_GIL and -X gil only supported in Py_GIL_DISABLED builds") + def test_python_gil(self): + cases = [ + # (env, opt, expected, msg) + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', None, '1', "PYTHON_GIL=1"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + (None, '1', '1', "-X gil=1"), + ] + + code = "import sys; print(sys.flags.gil)" + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') + @unittest.skipUnless(sys.platform == 'win32', 'bpo-32457 only applies on Windows') def test_argv0_normalization(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 55d3acf448540a..ab1d579ed12755 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -523,6 +523,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT['_pystats'] = 0 if support.Py_DEBUG: CONFIG_COMPAT['run_presite'] = None + if support.Py_GIL_DISABLED: + CONFIG_COMPAT['enable_gil'] = -1 if MS_WINDOWS: CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, diff --git a/Misc/python.man b/Misc/python.man index 0f5dfa2e2289f7..4c90c0e2a998ba 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -607,6 +607,10 @@ output. Setting it to 0 deactivates this behavior. .IP PYTHON_HISTORY This environment variable can be used to set the location of a history file (on Unix, it is \fI~/.python_history\fP by default). +.IP PYTHON_GIL +If this variable is set to 1, the global interpreter lock (GIL) will be forced +on. Setting it to 0 forces the GIL off. Only available in builds configured +with \fB--disable-gil\fP. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index edfc466d9f20ec..d2cd35dfa86833 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -219,6 +219,11 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) // XXX assert(tstate == NULL || !tstate->_status.cleared); struct _gil_runtime_state *gil = ceval->gil; +#ifdef Py_GIL_DISABLED + if (!gil->enabled) { + return; + } +#endif if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -294,6 +299,11 @@ take_gil(PyThreadState *tstate) assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; struct _gil_runtime_state *gil = interp->ceval.gil; +#ifdef Py_GIL_DISABLED + if (!gil->enabled) { + return; + } +#endif /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -440,6 +450,11 @@ static void init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { assert(!gil_created(gil)); +#ifdef Py_GIL_DISABLED + // gh-116329: Once it is safe to do so, change this condition to + // (enable_gil == _PyConfig_GIL_ENABLE), so the GIL is disabled by default. + gil->enabled = _PyInterpreterState_GetConfig(interp)->enable_gil != _PyConfig_GIL_DISABLE; +#endif create_gil(gil); assert(gil_created(gil)); interp->ceval.gil = gil; diff --git a/Python/initconfig.c b/Python/initconfig.c index 17c95171d9528a..e3a62e53334163 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -95,6 +95,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(safe_path, BOOL), SPEC(int_max_str_digits, INT), SPEC(cpu_count, INT), +#ifdef Py_GIL_DISABLED + SPEC(enable_gil, INT), +#endif SPEC(pathconfig_warnings, BOOL), SPEC(program_name, WSTR), SPEC(pythonpath_env, WSTR_OPT), @@ -278,6 +281,9 @@ static const char usage_envvars[] = "PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" " os.cpu_count(), and multiprocessing.cpu_count() if set to\n" " a positive integer.\n" +#ifdef Py_GIL_DISABLED +"PYTHON_GIL : When set to 0, disables the GIL.\n" +#endif "PYTHONDEVMODE : enable the development mode.\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" @@ -862,6 +868,9 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef Py_GIL_DISABLED + config->enable_gil = _PyConfig_GIL_DEFAULT; +#endif } @@ -1574,6 +1583,24 @@ config_wstr_to_int(const wchar_t *wstr, int *result) return 0; } +static PyStatus +config_read_gil(PyConfig *config, size_t len, wchar_t first_char) +{ +#ifdef Py_GIL_DISABLED + if (len == 1 && first_char == L'0') { + config->enable_gil = _PyConfig_GIL_DISABLE; + } + else if (len == 1 && first_char == L'1') { + config->enable_gil = _PyConfig_GIL_ENABLE; + } + else { + return _PyStatus_ERR("PYTHON_GIL / -X gil must be \"0\" or \"1\""); + } + return _PyStatus_OK(); +#else + return _PyStatus_ERR("PYTHON_GIL / -X gil are not supported by this build"); +#endif +} static PyStatus config_read_env_vars(PyConfig *config) @@ -1652,6 +1679,15 @@ config_read_env_vars(PyConfig *config) config->safe_path = 1; } + const char *gil = config_get_env(config, "PYTHON_GIL"); + if (gil != NULL) { + size_t len = strlen(gil); + status = config_read_gil(config, len, gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + return _PyStatus_OK(); } @@ -2207,6 +2243,15 @@ config_read(PyConfig *config, int compute_path_config) config->show_ref_count = 1; } + const wchar_t *x_gil = config_get_xoption_value(config, L"gil"); + if (x_gil != NULL) { + size_t len = wcslen(x_gil); + status = config_read_gil(config, len, x_gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + #ifdef Py_STATS if (config_get_xoption(config, L"pystats")) { config->_pystats = 1; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a4161da02980a7..cd193c1581c679 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3048,6 +3048,7 @@ static PyStructSequence_Field flags_fields[] = { {"warn_default_encoding", "-X warn_default_encoding"}, {"safe_path", "-P"}, {"int_max_str_digits", "-X int_max_str_digits"}, + {"gil", "-X gil"}, {0} }; @@ -3097,6 +3098,16 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) SetFlag(config->warn_default_encoding); SetFlagObj(PyBool_FromLong(config->safe_path)); SetFlag(config->int_max_str_digits); +#ifdef Py_GIL_DISABLED + if (config->enable_gil == _PyConfig_GIL_DEFAULT) { + SetFlagObj(Py_NewRef(Py_None)); + } + else { + SetFlag(config->enable_gil); + } +#else + SetFlagObj(PyLong_FromLong(1)); +#endif #undef SetFlagObj #undef SetFlag return 0; From 113053a070ba753101f73553ef6435c5c6c9f3f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Mar 2024 17:35:29 +0100 Subject: [PATCH 28/35] gh-110850: Fix _PyTime_FromSecondsDouble() API (#116606) Return 0 on success. Set an exception and return -1 on error. Fix os.timerfd_settime(): properly report exceptions on _PyTime_FromSecondsDouble() failure. No longer export _PyTime_FromSecondsDouble(). --- Include/internal/pycore_time.h | 6 ++++-- Modules/clinic/posixmodule.c.h | 24 ++++++++++++------------ Modules/posixmodule.c | 30 ++++++++++++++++++++---------- Python/pytime.c | 11 ++++------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 40b28e0ba221ae..138d60fdb69806 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -132,8 +132,10 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( PyAPI_FUNC(PyTime_t) _PyTime_FromSeconds(int seconds); // Create a timestamp from a number of seconds in double. -// Export for '_socket' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); +extern int _PyTime_FromSecondsDouble( + double seconds, + _PyTime_round_t round, + PyTime_t *result); // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index b49d64d4281889..b7338d138e91ce 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6279,8 +6279,8 @@ PyDoc_STRVAR(os_timerfd_settime__doc__, {"timerfd_settime", _PyCFunction_CAST(os_timerfd_settime), METH_FASTCALL|METH_KEYWORDS, os_timerfd_settime__doc__}, static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval); +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double); static PyObject * os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -6315,8 +6315,8 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; int fd; int flags = 0; - double initial = 0.0; - double interval = 0.0; + double initial_double = 0.0; + double interval_double = 0.0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -6339,12 +6339,12 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } if (args[2]) { if (PyFloat_CheckExact(args[2])) { - initial = PyFloat_AS_DOUBLE(args[2]); + initial_double = PyFloat_AS_DOUBLE(args[2]); } else { - initial = PyFloat_AsDouble(args[2]); - if (initial == -1.0 && PyErr_Occurred()) { + initial_double = PyFloat_AsDouble(args[2]); + if (initial_double == -1.0 && PyErr_Occurred()) { goto exit; } } @@ -6353,17 +6353,17 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } } if (PyFloat_CheckExact(args[3])) { - interval = PyFloat_AS_DOUBLE(args[3]); + interval_double = PyFloat_AS_DOUBLE(args[3]); } else { - interval = PyFloat_AsDouble(args[3]); - if (interval == -1.0 && PyErr_Occurred()) { + interval_double = PyFloat_AsDouble(args[3]); + if (interval_double == -1.0 && PyErr_Occurred()) { goto exit; } } skip_optional_kwonly: - return_value = os_timerfd_settime_impl(module, fd, flags, initial, interval); + return_value = os_timerfd_settime_impl(module, fd, flags, initial_double, interval_double); exit: return return_value; @@ -12588,4 +12588,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=268af5cbc8baa9d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2965306970f31c5d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 940a9cc8955e11..f3403e1675276b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10483,30 +10483,40 @@ os.timerfd_settime * flags: int = 0 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET. - initial: double = 0.0 + initial as initial_double: double = 0.0 The initial expiration time, in seconds. - interval: double = 0.0 + interval as interval_double: double = 0.0 The timer's interval, in seconds. Alter a timer file descriptor's internal timer in seconds. [clinic start generated code]*/ static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval) -/*[clinic end generated code: output=0dda31115317adb9 input=6c24e47e7a4d799e]*/ +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double) +/*[clinic end generated code: output=df4c1bce6859224e input=81d2c0d7e936e8a7]*/ { - struct itimerspec new_value; - struct itimerspec old_value; - int result; - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(initial, _PyTime_ROUND_FLOOR), &new_value.it_value) < 0) { + PyTime_t initial, interval; + if (_PyTime_FromSecondsDouble(initial_double, _PyTime_ROUND_FLOOR, + &initial) < 0) { + return NULL; + } + if (_PyTime_FromSecondsDouble(interval_double, _PyTime_ROUND_FLOOR, + &interval) < 0) { + return NULL; + } + + struct itimerspec new_value, old_value; + if (_PyTime_AsTimespec(initial, &new_value.it_value) < 0) { PyErr_SetString(PyExc_ValueError, "invalid initial value"); return NULL; } - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(interval, _PyTime_ROUND_FLOOR), &new_value.it_interval) < 0) { + if (_PyTime_AsTimespec(interval, &new_value.it_interval) < 0) { PyErr_SetString(PyExc_ValueError, "invalid interval value"); return NULL; } + + int result; Py_BEGIN_ALLOW_THREADS result = timerfd_settime(fd, flags, &new_value, &old_value); Py_END_ALLOW_THREADS diff --git a/Python/pytime.c b/Python/pytime.c index 90ef2eeb546f7f..70d92ca00ee28e 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -565,6 +565,7 @@ pytime_from_double(PyTime_t *tp, double value, _PyTime_round_t round, /* See comments in pytime_double_to_denominator */ if (!((double)PyTime_MIN <= d && d < -(double)PyTime_MIN)) { pytime_time_t_overflow(); + *tp = 0; return -1; } PyTime_t ns = (PyTime_t)d; @@ -652,14 +653,10 @@ _PyTime_AsLong(PyTime_t ns) return PyLong_FromLongLong((long long)ns); } -PyTime_t -_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) +int +_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round, PyTime_t *result) { - PyTime_t tp; - if(pytime_from_double(&tp, seconds, round, SEC_TO_NS) < 0) { - return -1; - } - return tp; + return pytime_from_double(result, seconds, round, SEC_TO_NS); } From 05070f40bbc3384c36c8b3dab76345ba92098d42 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 11 Mar 2024 11:59:09 -0700 Subject: [PATCH 29/35] GH-115976: Add WASI to CI (GH-116516) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 10 ++++ .github/workflows/reusable-wasi.yml | 71 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/reusable-wasi.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20d1fad40ecafe..ae14046935b97b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -301,6 +301,14 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build_wasi: + name: 'WASI' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-wasi.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + test_hypothesis: name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-20.04 @@ -525,6 +533,7 @@ jobs: - build_ubuntu - build_ubuntu_free_threading - build_ubuntu_ssltests + - build_wasi - build_windows - build_windows_free_threading - test_hypothesis @@ -558,6 +567,7 @@ jobs: build_ubuntu, build_ubuntu_free_threading, build_ubuntu_ssltests, + build_wasi, build_windows, build_windows_free_threading, build_asan, diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml new file mode 100644 index 00000000000000..995e669c228b5c --- /dev/null +++ b/.github/workflows/reusable-wasi.yml @@ -0,0 +1,71 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + +jobs: + build_wasi_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-20.04 + env: + WASMTIME_VERSION: 18.0.2 + WASI_SDK_VERSION: 20 + WASI_SDK_PATH: /opt/wasi-sdk + CROSS_BUILD_PYTHON: cross-build/build + CROSS_BUILD_WASI: cross-build/wasm32-wasi + steps: + - uses: actions/checkout@v4 + # No problem resolver registered as one doesn't currently exist for Clang. + - name: "Install wasmtime" + uses: jcbhmr/setup-wasmtime@v2 + with: + wasmtime-version: ${{ env.WASMTIME_VERSION }} + - name: "Restore WASI SDK" + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: ${{ env.WASI_SDK_PATH }} + key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} + - name: "Install WASI SDK" + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: | + mkdir ${{ env.WASI_SDK_PATH }} && \ + curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \ + tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip + - name: "Configure ccache action" + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: "Add ccache to PATH" + run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: "Install Python" + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: "Restore Python build config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: "Configure build Python" + run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Tools/wasm/wasi.py make-build-python + - name: "Restore host config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_WASI }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }} + - name: "Configure host" + # `--with-pydebug` inferred from configure-build-python + run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + - name: "Make host" + run: python3 Tools/wasm/wasi.py make-host + - name: "Display build info" + run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo + - name: "Test" + run: make --directory ${{ env.CROSS_BUILD_WASI }} test From 9f983e00ec55b87a098a4c8229fe5bb9acb9f3ac Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 11 Mar 2024 15:14:20 -0400 Subject: [PATCH 30/35] gh-116515: Clear thread-local state before tstate_delete_common() (#116517) This moves `current_fast_clear()` up so that the current thread state is `NULL` while running `tstate_delete_common()`. This doesn't fix any bugs, but it means that we are more consistent that `_PyThreadState_GET() != NULL` means that the thread is "attached". --- Python/pystate.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/pystate.c b/Python/pystate.c index 1418d034ca2fe9..635616c5648c18 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1609,6 +1609,7 @@ tstate_delete_common(PyThreadState *tstate) { assert(tstate->_status.cleared && !tstate->_status.finalized); assert(tstate->state != _Py_THREAD_ATTACHED); + tstate_verify_not_active(tstate); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1687,8 +1688,8 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); #endif tstate_set_detached(tstate, _Py_THREAD_DETACHED); - tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); + tstate_delete_common(tstate); _PyEval_ReleaseLock(tstate->interp, NULL); free_threadstate((_PyThreadStateImpl *)tstate); } From 872c0714fcdc168ce4a69bdd0346f2d5dd488ec2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Mar 2024 19:25:39 +0000 Subject: [PATCH 31/35] gh-71052: Change Android's `sys.platform` from "linux" to "android" Co-authored-by: Erlend E. Aasland --- Doc/library/sys.rst | 45 +++++++++---------- Lib/ctypes/__init__.py | 2 +- Lib/multiprocessing/util.py | 6 +-- Lib/shutil.py | 3 +- Lib/test/support/__init__.py | 2 +- Lib/test/support/os_helper.py | 2 +- Lib/test/test_asyncio/test_subprocess.py | 3 +- Lib/test/test_c_locale_coercion.py | 21 +++++---- Lib/test/test_fcntl.py | 4 +- Lib/test/test_logging.py | 4 +- Lib/test/test_mmap.py | 2 +- Lib/test/test_os.py | 7 ++- Lib/test/test_resource.py | 2 +- Lib/test/test_socket.py | 17 ++++--- Lib/test/test_ssl.py | 2 +- Lib/test/test_sys.py | 5 +-- Lib/test/test_sysconfig.py | 22 ++++++--- Lib/test/test_tarfile.py | 2 +- Lib/test/test_time.py | 2 +- Lib/uuid.py | 2 +- ...4-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst | 1 + Misc/platform_triplet.c | 14 +++++- configure | 1 + configure.ac | 1 + 24 files changed, 94 insertions(+), 78 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 380ba1090b39b3..087a3454c33272 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1367,47 +1367,42 @@ always available. .. data:: platform - This string contains a platform identifier that can be used to append - platform-specific components to :data:`sys.path`, for instance. - - For Unix systems, except on Linux and AIX, this is the lowercased OS name as - returned by ``uname -s`` with the first part of the version as returned by - ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time - when Python was built*. Unless you want to test for a specific system - version, it is therefore recommended to use the following idiom:: - - if sys.platform.startswith('freebsd'): - # FreeBSD-specific code here... - elif sys.platform.startswith('linux'): - # Linux-specific code here... - elif sys.platform.startswith('aix'): - # AIX-specific code here... - - For other systems, the values are: + A string containing a platform identifier. Known values are: ================ =========================== System ``platform`` value ================ =========================== AIX ``'aix'`` + Android ``'android'`` Emscripten ``'emscripten'`` + iOS ``'ios'`` Linux ``'linux'`` - WASI ``'wasi'`` + macOS ``'darwin'`` Windows ``'win32'`` Windows/Cygwin ``'cygwin'`` - macOS ``'darwin'`` + WASI ``'wasi'`` ================ =========================== + On Unix systems not listed in the table, the value is the lowercased OS name + as returned by ``uname -s``, with the first part of the version as returned by + ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time + when Python was built*. Unless you want to test for a specific system + version, it is therefore recommended to use the following idiom:: + + if sys.platform.startswith('freebsd'): + # FreeBSD-specific code here... + .. versionchanged:: 3.3 On Linux, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. .. versionchanged:: 3.8 On AIX, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. + + .. versionchanged:: 3.13 + On Android, :data:`sys.platform` now returns ``'android'`` rather than + ``'linux'``. .. seealso:: diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index d54ee05b15f5bd..f63e31a3fb0107 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -468,7 +468,7 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif hasattr(_sys, "getandroidapilevel"): +elif _sys.platform == "android": pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 32871850ddec8b..75dde02d88c533 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -102,11 +102,7 @@ def log_to_stderr(level=None): # Abstract socket support def _platform_supports_abstract_sockets(): - if sys.platform == "linux": - return True - if hasattr(sys, 'getandroidapilevel'): - return True - return False + return sys.platform in ("linux", "android") def is_abstract_socket_namespace(address): diff --git a/Lib/shutil.py b/Lib/shutil.py index c19ea0607208af..f8be82d97616d3 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -47,7 +47,8 @@ COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 505d4bea83fe8d..af43446c26120e 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -520,7 +520,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') -is_android = hasattr(sys, 'getandroidapilevel') +is_android = sys.platform == "android" if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index ffa5fc52e9b2bd..8071c248b9b67e 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -612,7 +612,7 @@ def __fspath__(self): def fd_count(): """Count the number of open file descriptors. """ - if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + if sys.platform.startswith(('linux', 'android', 'freebsd', 'emscripten')): fd_path = "/proc/self/fd" elif sys.platform == "darwin": fd_path = "/dev/fd" diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 890c0acc1720dc..cf1a1985338e40 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -480,7 +480,8 @@ async def empty_error(): self.assertEqual(output, None) self.assertEqual(exitcode, 0) - @unittest.skipIf(sys.platform != 'linux', "Don't have /dev/stdin") + @unittest.skipIf(sys.platform not in ('linux', 'android'), + "Don't have /dev/stdin") def test_devstdin_input(self): async def devstdin_input(message): diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 7334a325ba22f0..e4b0b8c451fd45 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -26,17 +26,16 @@ TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"] # Apply some platform dependent overrides -if sys.platform.startswith("linux"): - if support.is_android: - # Android defaults to using UTF-8 for all system interfaces - EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" - EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" - else: - # Linux distros typically alias the POSIX locale directly to the C - # locale. - # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be - # able to check this case unconditionally - EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +if sys.platform == "android": + # Android defaults to using UTF-8 for all system interfaces + EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +elif sys.platform.startswith("linux"): + # Linux distros typically alias the POSIX locale directly to the C + # locale. + # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be + # able to check this case unconditionally + EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") elif sys.platform.startswith("aix"): # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 8b4ed4a9e3a4fe..5fae0de7423c87 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -129,8 +129,8 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - any(platform.machine().startswith(name) for name in {"arm", "aarch"}) - and platform.system() in {"Linux", "Android"}, + platform.machine().startswith(("arm", "aarch")) + and platform.system() in ("Linux", "Android"), "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 1dfad6e1d2609b..32bb5171a3f757 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -603,7 +603,7 @@ def test_name(self): def test_builtin_handlers(self): # We can't actually *use* too many handlers in the tests, # but we can try instantiating them with various options - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): for existing in (True, False): fn = make_temp_file() if not existing: @@ -667,7 +667,7 @@ def test_path_objects(self): (logging.handlers.RotatingFileHandler, (pfn, 'a')), (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), ) - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) for cls, args in cases: h = cls(*args, encoding="utf-8") diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ac759757d24659..ee86227e026b67 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -837,7 +837,7 @@ def test_flush_return_value(self): mm.write(b'python') result = mm.flush() self.assertIsNone(result) - if sys.platform.startswith('linux'): + if sys.platform.startswith(('linux', 'android')): # 'offset' must be a multiple of mmap.PAGESIZE on Linux. # See bpo-34754 for details. self.assertRaises(OSError, mm.flush, 1, len(b'python')) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ae8b405dab50fc..fc886f967c11bf 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3533,9 +3533,8 @@ def test_set_get_priority(self): class TestSendfile(unittest.IsolatedAsyncioTestCase): DATA = b"12345abcde" * 16 * 1024 # 160 KiB - SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ - not sys.platform.startswith("solaris") and \ - not sys.platform.startswith("sunos") + SUPPORT_HEADERS_TRAILERS = ( + not sys.platform.startswith(("linux", "android", "solaris", "sunos"))) requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') requires_32b = unittest.skipUnless(sys.maxsize < 2**32, @@ -5256,7 +5255,7 @@ def test_fork(self): else: assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - @unittest.skipUnless(sys.platform in ("linux", "darwin"), + @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 317e7ca8f8c853..d23d3623235f38 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -138,7 +138,7 @@ def test_pagesize(self): self.assertIsInstance(pagesize, int) self.assertGreaterEqual(pagesize, 0) - @unittest.skipUnless(sys.platform == 'linux', 'test requires Linux') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux only') def test_linux_constants(self): for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']: with contextlib.suppress(AttributeError): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index b936e9ae91daca..a7e657f5718524 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1199,8 +1199,8 @@ def testGetServBy(self): # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. if ( - sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform == 'linux' + sys.platform.startswith( + ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) or is_apple ): # avoid the 'echo' service on this platform, as there is an @@ -1218,8 +1218,7 @@ def testGetServBy(self): raise OSError # Try same call with optional protocol omitted # Issue #26936: Android getservbyname() was broken before API 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + if (not support.is_android) or sys.getandroidapilevel() >= 23: port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1577,8 +1576,7 @@ def testGetaddrinfo(self): # port can be a string service name such as "http", a numeric # port number or None # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + if (not support.is_android) or sys.getandroidapilevel() >= 23: socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) @@ -3196,7 +3194,7 @@ def _testSendmsgTimeout(self): # Linux supports MSG_DONTWAIT when sending, but in general, it # only works when receiving. Could add other platforms if they # support it too. - @skipWithClientIf(sys.platform not in {"linux"}, + @skipWithClientIf(sys.platform not in {"linux", "android"}, "MSG_DONTWAIT not known to work on this platform when " "sending") def testSendmsgDontWait(self): @@ -5634,7 +5632,7 @@ def test_setblocking_invalidfd(self): sock.setblocking(False) -@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +@unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): UNIX_PATH_MAX = 108 @@ -5759,7 +5757,8 @@ def testUnencodableAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - @unittest.skipIf(sys.platform == 'linux', 'Linux specific test') + @unittest.skipIf(sys.platform in ('linux', 'android'), + 'Linux behavior is tested by TestLinuxAbstractNamespace') def testEmptyAddress(self): # Test that binding empty address fails. self.assertRaises(OSError, self.sock.bind, "") diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index cdf96c808b86ac..489cb5e23ba57e 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4914,7 +4914,7 @@ def run(self): pass # closed, protocol error, etc. def non_linux_skip_if_other_okay_error(self, err): - if sys.platform == "linux": + if sys.platform in ("linux", "android"): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 38dcabd84d8170..37c16cd1047885 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -668,7 +668,7 @@ def test_thread_info(self): self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "freebsd")): + if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") @@ -1101,8 +1101,7 @@ def __del__(self): self.assertEqual(stdout.rstrip(), b"") self.assertEqual(stderr.rstrip(), b"") - @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), - 'need sys.getandroidapilevel()') + @unittest.skipUnless(sys.platform == "android", "Android only") def test_getandroidapilevel(self): level = sys.getandroidapilevel() self.assertIsInstance(level, int) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index bb87bf00dc2d1a..c8315bbc8b727d 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,3 +1,5 @@ +import platform +import re import unittest import sys import os @@ -516,12 +518,9 @@ def test_EXT_SUFFIX_in_vars(self): vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -534,6 +533,19 @@ def test_triplet_in_ext_suffix(self): self.assertTrue(suffix.endswith(expected_suffixes), f'unexpected suffix {suffix!r}') + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index a047780fdd7e17..39541faa237b24 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1186,7 +1186,7 @@ def _fs_supports_holes(): # # The function returns False if page size is larger than 4 KiB. # For example, ppc64 uses pages of 64 KiB. - if sys.platform.startswith("linux"): + if sys.platform.startswith(("linux", "android")): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index a0aeea515afbd6..fb234b7bc5962a 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -511,7 +511,7 @@ def test_process_time(self): def test_thread_time(self): if not hasattr(time, 'thread_time'): - if sys.platform.startswith(('linux', 'win')): + if sys.platform.startswith(('linux', 'android', 'win')): self.fail("time.thread_time() should be available on %r" % (sys.platform,)) else: diff --git a/Lib/uuid.py b/Lib/uuid.py index d4e486da15a8d7..da2f1d50563095 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -62,7 +62,7 @@ import platform _platform_system = platform.system() _AIX = _platform_system == 'AIX' - _LINUX = _platform_system == 'Linux' + _LINUX = _platform_system in ('Linux', 'Android') _MAC_DELIM = b':' _MAC_OMITS_LEADING_ZEROES = False diff --git a/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst b/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst new file mode 100644 index 00000000000000..187475f56b4415 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-03-01-16-44-19.gh-issue-71052.Hs-9EP.rst @@ -0,0 +1 @@ +Change Android's :data:`sys.platform` from ``"linux"`` to ``"android"``. diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 0b912e332510a6..06b03bfa9a266a 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -12,8 +12,20 @@ #undef powerpc #undef sparc #undef unix + #if defined(__ANDROID__) - # Android is not a multiarch system. +# if defined(__x86_64__) +PLATFORM_TRIPLET=x86_64-linux-android +# elif defined(__i386__) +PLATFORM_TRIPLET=i686-linux-android +# elif defined(__aarch64__) +PLATFORM_TRIPLET=aarch64-linux-android +# elif defined(__arm__) +PLATFORM_TRIPLET=arm-linux-androideabi +# else +# error unknown Android platform +# endif + #elif defined(__linux__) /* * BEGIN of Linux block diff --git a/configure b/configure index 5e8c9af7befde0..267a5183379ce0 100755 --- a/configure +++ b/configure @@ -4071,6 +4071,7 @@ then case $MACHDEP in aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; diff --git a/configure.ac b/configure.ac index e615259dcb1fca..681d081f571ea5 100644 --- a/configure.ac +++ b/configure.ac @@ -362,6 +362,7 @@ then case $MACHDEP in aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; From 1d0d49a7e86257ff95b4de0685e6997d7533993c Mon Sep 17 00:00:00 2001 From: "Pierre Ossman (ThinLinc team)" Date: Mon, 11 Mar 2024 20:43:30 +0100 Subject: [PATCH 32/35] gh-113538: Add asycio.Server.{close,abort}_clients (#114432) These give applications the option of more forcefully terminating client connections for asyncio servers. Useful when terminating a service and there is limited time to wait for clients to finish up their work. --- Doc/library/asyncio-eventloop.rst | 25 +++++ Doc/whatsnew/3.13.rst | 5 + Lib/asyncio/base_events.py | 25 +++-- Lib/asyncio/events.py | 8 ++ Lib/asyncio/proactor_events.py | 4 +- Lib/asyncio/selector_events.py | 6 +- Lib/test/test_asyncio/test_server.py | 96 +++++++++++++++++-- ...-01-22-15-50-58.gh-issue-113538.v2wrwg.rst | 3 + 8 files changed, 152 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 06c5c877ccc173..d6ed817b13676f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1641,6 +1641,31 @@ Do not instantiate the :class:`Server` class directly. coroutine to wait until the server is closed (and no more connections are active). + .. method:: close_clients() + + Close all existing incoming client connections. + + Calls :meth:`~asyncio.BaseTransport.close` on all associated + transports. + + :meth:`close` should be called before :meth:`close_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + + .. method:: abort_clients() + + Close all existing incoming client connections immediately, + without waiting for pending operations to complete. + + Calls :meth:`~asyncio.WriteTransport.abort` on all associated + transports. + + :meth:`close` should be called before :meth:`abort_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + .. method:: get_loop() Return the event loop associated with the server object. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 51939909000960..95e8ff3eb2e575 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -270,6 +270,11 @@ asyncio the buffer size. (Contributed by Jamie Phan in :gh:`115199`.) +* Add :meth:`asyncio.Server.close_clients` and + :meth:`asyncio.Server.abort_clients` methods which allow to more + forcefully close an asyncio server. + (Contributed by Pierre Ossman in :gh:`113538`.) + base64 --- diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 6c5cf28e7c59d4..f0e690b61a73dd 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -279,7 +279,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -292,14 +294,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -348,9 +349,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 680749325025db..be495469a0558b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -175,6 +175,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index a512db6367b20a..397a8cda757895 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 8e888d26ea0737..f94bf10b4225e7 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -791,7 +791,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,6 +868,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -906,7 +908,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 918faac909b9bf..0c55661bfb88f4 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -125,8 +125,12 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): async def test_wait_closed_basic(self): - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -137,7 +141,8 @@ async def serve(*args): self.assertFalse(task1.done()) # active count != 0, not closed: should block - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task1.done()) @@ -152,7 +157,8 @@ async def serve(*args): self.assertFalse(task2.done()) self.assertFalse(task3.done()) - srv._detach() + wr.close() + await wr.wait_closed() # active count == 0, closed: should unblock await task1 await task2 @@ -161,8 +167,12 @@ async def serve(*args): async def test_wait_closed_race(self): # Test a regression in 3.12.0, should be fixed in 3.12.1 - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -170,13 +180,83 @@ async def serve(*args): task = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task.done()) - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) loop = asyncio.get_running_loop() loop.call_soon(srv.close) - loop.call_soon(srv._detach) + loop.call_soon(wr.close) await srv.wait_closed() + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + nonlocal s_rd, s_wr + s_rd = rd + s_wr = wr + await wr.wait_closed() + + s_rd = s_wr = None + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + # Limit the socket buffers so we can reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # socket buffers are full on both server and client sockets and + # the kernel stops accepting more data + s_wr.write(b'a' * c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) + s_wr.write(b'a' * s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)) + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) # Test the various corner cases of Unix server socket removal diff --git a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst new file mode 100644 index 00000000000000..5c59af98e136bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst @@ -0,0 +1,3 @@ +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. From 34920f36917de0d4e658cf94992d53a5a7f27f51 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 11 Mar 2024 20:39:17 +0000 Subject: [PATCH 33/35] gh-71052: Use `raise_signal` in `ThreadSignals.test_signals` (#116423) Use `raise_signal` rather than `kill` in `ThreadSignals.test_signals` --- Lib/test/test_threadsignals.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index 6a53d655015cdb..bf241ada90ea35 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -32,39 +32,28 @@ def handle_signals(sig,frame): # a function that will be spawned as a separate thread. def send_signals(): - os.kill(process_pid, signal.SIGUSR1) - os.kill(process_pid, signal.SIGUSR2) + # We use `raise_signal` rather than `kill` because: + # * It verifies that a signal delivered to a background thread still has + # its Python-level handler called on the main thread. + # * It ensures the signal is handled before the thread exits. + signal.raise_signal(signal.SIGUSR1) + signal.raise_signal(signal.SIGUSR2) signalled_all.release() @threading_helper.requires_working_threading() -@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm") class ThreadSignals(unittest.TestCase): def test_signals(self): with threading_helper.wait_threads_exit(): # Test signal handling semantics of threads. - # We spawn a thread, have the thread send two signals, and + # We spawn a thread, have the thread send itself two signals, and # wait for it to finish. Check that we got both signals # and that they were run by the main thread. signalled_all.acquire() self.spawnSignallingThread() signalled_all.acquire() - # the signals that we asked the kernel to send - # will come back, but we don't know when. - # (it might even be after the thread exits - # and might be out of order.) If we haven't seen - # the signals yet, send yet another signal and - # wait for it return. - if signal_blackboard[signal.SIGUSR1]['tripped'] == 0 \ - or signal_blackboard[signal.SIGUSR2]['tripped'] == 0: - try: - signal.alarm(1) - signal.pause() - finally: - signal.alarm(0) - self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped'], 1) self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped_by'], thread.get_ident()) From 3c0dcef9808e34744096769b15bad4f1f97569f0 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Mon, 11 Mar 2024 13:42:01 -0700 Subject: [PATCH 34/35] gh-116040: [Enum] fix test_empty_names test (GH-116508) * and fix _not_given usage --- Lib/enum.py | 4 +--- Lib/test/test_enum.py | 18 +++++++----------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 3499cb0b71547c..bcf7aae949ffdb 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -165,8 +165,6 @@ def _dedent(text): class _not_given: def __repr__(self): return('') - def __bool__(self): - return False _not_given = _not_given() class _auto_null: @@ -727,7 +725,7 @@ def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, ) return cls._create_( class_name=value, - names=names or None, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0a44b61e9049ed..a83aca406cbd83 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -3431,17 +3431,13 @@ def test_no_members(self): Flag(7) def test_empty_names(self): - for nothing, e_type in ( - ('', None), - ('', int), - ([], None), - ([], int), - ({}, None), - ({}, int), - ): - empty_enum = Enum('empty_enum', nothing, type=e_type) - self.assertEqual(len(empty_enum), 0) - self.assertRaises(TypeError, 'has no members', empty_enum, 0) + for nothing in '', [], {}: + for e_type in None, int: + empty_enum = Enum('empty_enum', nothing, type=e_type) + self.assertEqual(len(empty_enum), 0) + self.assertRaisesRegex(TypeError, 'has no members', empty_enum, 0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', names=0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', 0, type=int) class TestOrder(unittest.TestCase): From 44f9a84b67c97c94f0d581ffd63b24b73fb79610 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 11 Mar 2024 14:27:00 -0700 Subject: [PATCH 35/35] gh-90095: Make .pdbrc work properly and add some reasonable tests (#110496) --- Lib/pdb.py | 47 ++---- Lib/test/test_pdb.py | 149 ++++++++++-------- ...3-10-07-06-15-13.gh-issue-90095.gWn1ka.rst | 1 + 3 files changed, 101 insertions(+), 96 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 519c1ccd5640a1..f4d19386703de0 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -363,26 +363,9 @@ def setup(self, f, tb): self._chained_exceptions[self._chained_exception_index], ) - return self.execRcLines() - - # Can be executed earlier than 'setup' if desired - def execRcLines(self): - if not self.rcLines: - return - # local copy because of recursion - rcLines = self.rcLines - rcLines.reverse() - # execute every line only once - self.rcLines = [] - while rcLines: - line = rcLines.pop().strip() - if line and line[0] != '#': - if self.onecmd(line): - # if onecmd returns True, the command wants to exit - # from the interaction, save leftover rc lines - # to execute before next interaction - self.rcLines += reversed(rcLines) - return True + if self.rcLines: + self.cmdqueue = self.rcLines + self.rcLines = [] # Override Bdb methods @@ -571,12 +554,10 @@ def interaction(self, frame, tb_or_exc): if isinstance(tb_or_exc, BaseException): assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): - if self.setup(frame, tb): - # no interaction desired at this time (happens if .pdbrc contains - # a command like "continue") - self.forget() - return - self.print_stack_entry(self.stack[self.curindex]) + self.setup(frame, tb) + # if we have more commands to process, do not show the stack entry + if not self.cmdqueue: + self.print_stack_entry(self.stack[self.curindex]) self._cmdloop() self.forget() @@ -712,7 +693,7 @@ def precmd(self, line): if marker >= 0: # queue up everything after marker next = line[marker+2:].lstrip() - self.cmdqueue.append(next) + self.cmdqueue.insert(0, next) line = line[:marker].rstrip() # Replace all the convenience variables @@ -737,13 +718,12 @@ def handle_command_def(self, line): """Handles one command line during command list definition.""" cmd, arg, line = self.parseline(line) if not cmd: - return + return False if cmd == 'silent': self.commands_silent[self.commands_bnum] = True - return # continue to handle other cmd def in the cmd list + return False # continue to handle other cmd def in the cmd list elif cmd == 'end': - self.cmdqueue = [] - return 1 # end of cmd list + return True # end of cmd list cmdlist = self.commands[self.commands_bnum] if arg: cmdlist.append(cmd+' '+arg) @@ -757,9 +737,8 @@ def handle_command_def(self, line): # one of the resuming commands if func.__name__ in self.commands_resuming: self.commands_doprompt[self.commands_bnum] = False - self.cmdqueue = [] - return 1 - return + return True + return False # interface abstraction functions diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 3dd275caf43200..44728542787423 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2618,13 +2618,29 @@ def _run_pdb(self, pdb_args, commands, def run_pdb_script(self, script, commands, expected_returncode=0, - extra_env=None): + extra_env=None, + pdbrc=None, + remove_home=False): """Run 'script' lines with pdb and the pdb 'commands'.""" filename = 'main.py' with open(filename, 'w') as f: f.write(textwrap.dedent(script)) + + if pdbrc is not None: + with open('.pdbrc', 'w') as f: + f.write(textwrap.dedent(pdbrc)) + self.addCleanup(os_helper.unlink, '.pdbrc') self.addCleanup(os_helper.unlink, filename) - return self._run_pdb([filename], commands, expected_returncode, extra_env) + + homesave = None + if remove_home: + homesave = os.environ.pop('HOME', None) + try: + stdout, stderr = self._run_pdb([filename], commands, expected_returncode, extra_env) + finally: + if homesave is not None: + os.environ['HOME'] = homesave + return stdout, stderr def run_pdb_module(self, script, commands): """Runs the script code as part of a module""" @@ -2904,37 +2920,80 @@ def test_issue26053(self): self.assertRegex(res, "Restarting .* with arguments:\na b c") self.assertRegex(res, "Restarting .* with arguments:\nd e f") - def test_readrc_kwarg(self): + def test_pdbrc_basic(self): script = textwrap.dedent(""" - import pdb; pdb.Pdb(readrc=False).set_trace() + a = 1 + b = 2 + """) - print('hello') + pdbrc = textwrap.dedent(""" + # Comments should be fine + n + p f"{a+8=}" """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w') as f: - f.write("invalid\n") - - with open('main.py', 'w') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - with proc: - stdout, stderr = proc.communicate(b'q\n') - self.assertNotIn(b"NameError: name 'invalid' is not defined", - stdout) + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("a+8=9", stdout) - finally: - if save_home is not None: - os.environ['HOME'] = save_home + def test_pdbrc_alias(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + until 6 + pi a + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("a.attr = 1", stdout) + + def test_pdbrc_semicolon(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 5;;c;;n + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("-> b = 2", stdout) + + def test_pdbrc_commands(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 6 + commands 1 ;; p a;; end + c + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("<__main__.A object at", stdout) + + def test_readrc_kwarg(self): + script = textwrap.dedent(""" + print('hello') + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc='invalid', remove_home=True) + self.assertIn("NameError: name 'invalid' is not defined", stdout) def test_readrc_homedir(self): save_home = os.environ.pop("HOME", None) @@ -2949,40 +3008,6 @@ def test_readrc_homedir(self): if save_home is not None: os.environ["HOME"] = save_home - def test_read_pdbrc_with_ascii_encoding(self): - script = textwrap.dedent(""" - import pdb; pdb.Pdb().set_trace() - print('hello') - """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w', encoding='utf-8') as f: - f.write("Fran\u00E7ais") - - with open('main.py', 'w', encoding='utf-8') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - env = {'PYTHONIOENCODING': 'ascii'} - if sys.platform == 'win32': - env['PYTHONLEGACYWINDOWSSTDIO'] = 'non-empty-string' - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ, **env} - ) - with proc: - stdout, stderr = proc.communicate(b'c\n') - self.assertIn(b"UnicodeEncodeError: \'ascii\' codec can\'t encode character " - b"\'\\xe7\' in position 21: ordinal not in range(128)", stderr) - - finally: - if save_home is not None: - os.environ['HOME'] = save_home - def test_header(self): stdout = StringIO() header = 'Nobody expects... blah, blah, blah' diff --git a/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst b/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst new file mode 100644 index 00000000000000..d71442ef642b6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-07-06-15-13.gh-issue-90095.gWn1ka.rst @@ -0,0 +1 @@ +Make .pdbrc and -c work with any valid pdb commands.