From 90d34f0d187180398774deb4e529b348023a11e8 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sun, 16 Apr 2023 11:44:17 -0700 Subject: [PATCH] specialize: optimize for single-threaded programs --- Python/bytecodes.c | 60 +++++++++++--------------------- Python/ceval.c | 71 ++++++++++++++++++++++++++++++++++---- Python/generated_cases.c.h | 60 +++++++++++--------------------- Python/specialize.c | 51 ++++++++++++++++----------- 4 files changed, 135 insertions(+), 107 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3ad3c96172..725ae29d0d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -342,9 +342,9 @@ dummy_func( }; inst(BINARY_SUBSCR, (unused/4, container, sub -- unused)) { - _PyMutex_lock(&_PyRuntime.mutex); _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); @@ -352,8 +352,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(BINARY_SUBSCR_GENERIC); } @@ -490,9 +488,9 @@ dummy_func( }; inst(STORE_SUBSCR, (unused/1, unused, container, sub -- )) { - _PyMutex_lock(&_PyRuntime.mutex); _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); @@ -500,8 +498,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(STORE_SUBSCR_GENERIC); } @@ -928,9 +924,9 @@ dummy_func( // stack effect: (__0 -- __array[oparg]) inst(UNPACK_SEQUENCE) { - _PyMutex_lock(&_PyRuntime.mutex); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; @@ -939,8 +935,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(UNPACK_SEQUENCE_GENERIC); } @@ -1021,9 +1015,9 @@ dummy_func( }; inst(STORE_ATTR, (unused/1, unused/3, unused, owner --)) { - _PyMutex_lock(&_PyRuntime.mutex); _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1032,8 +1026,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(STORE_ATTR_GENERIC); } @@ -1136,9 +1128,9 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL) { - _PyMutex_lock(&_PyRuntime.mutex); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1147,8 +1139,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(LOAD_GLOBAL_GENERIC); } @@ -1537,9 +1527,9 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR) { - _PyMutex_lock(&_PyRuntime.mutex); _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); @@ -1549,8 +1539,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(LOAD_ATTR_GENERIC); } @@ -1885,9 +1873,9 @@ dummy_func( }; inst(COMPARE_OP, (unused/2, left, right -- unused)) { - _PyMutex_lock(&_PyRuntime.mutex); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); @@ -1895,8 +1883,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } @@ -2301,9 +2287,9 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER) { - _PyMutex_lock(&_PyRuntime.mutex); _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); @@ -2311,8 +2297,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(FOR_ITER_GENERIC); } @@ -2636,9 +2620,9 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL) { - _PyMutex_lock(&_PyRuntime.mutex); _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; @@ -2649,8 +2633,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(CALL_GENERIC); } @@ -3393,9 +3375,9 @@ dummy_func( } inst(BINARY_OP, (unused/1, lhs, rhs -- unused)) { - _PyMutex_lock(&_PyRuntime.mutex); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); @@ -3403,8 +3385,6 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } diff --git a/Python/ceval.c b/Python/ceval.c index 7b7ca3a9cc..8cbf11a47f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -887,7 +887,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - GO_TO_INSTRUCTION(INSTNAME ## _GENERIC); \ + goto INSTNAME ## _DEOPT; \ } #define DEOPT_UNLOCK_IF(COND, INSTNAME) \ @@ -896,7 +896,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ _Py_critical_section_end(&_cs); \ - GO_TO_INSTRUCTION(INSTNAME ## _GENERIC); \ + goto INSTNAME ## _DEOPT; \ } @@ -955,11 +955,17 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \ (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) -#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ - do { \ - assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ - (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \ - } while (0); +static _Py_ALWAYS_INLINE int +DECREMENT_ADAPTIVE_COUNTER(uint16_t *ptr) +{ + uint16_t counter = _Py_atomic_load_uint16_relaxed(ptr); + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + return 1; + } + counter -= (1 << ADAPTIVE_BACKOFF_BITS); + _Py_atomic_store_uint16_relaxed(ptr, counter); + return 0; +} #define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ do { \ @@ -1334,6 +1340,57 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int or goto error. */ Py_UNREACHABLE(); +BINARY_OP_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(BINARY_OP); + } + GO_TO_INSTRUCTION(BINARY_OP_GENERIC); +BINARY_SUBSCR_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(BINARY_SUBSCR); + } + GO_TO_INSTRUCTION(BINARY_SUBSCR_GENERIC); +CALL_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(CALL); + } + GO_TO_INSTRUCTION(CALL_GENERIC); +COMPARE_OP_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(COMPARE_OP); + } + GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); +FOR_ITER_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(FOR_ITER); + } + GO_TO_INSTRUCTION(FOR_ITER_GENERIC); +LOAD_ATTR_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(LOAD_ATTR); + } + GO_TO_INSTRUCTION(LOAD_ATTR_GENERIC); +LOAD_GLOBAL_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(LOAD_GLOBAL); + } + GO_TO_INSTRUCTION(LOAD_GLOBAL_GENERIC); +STORE_ATTR_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(STORE_ATTR); + } + GO_TO_INSTRUCTION(STORE_ATTR_GENERIC); +STORE_SUBSCR_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(STORE_SUBSCR); + } + GO_TO_INSTRUCTION(STORE_SUBSCR_GENERIC); +UNPACK_SEQUENCE_DEOPT: + if (!_PyRuntime.multithreaded) { + GO_TO_INSTRUCTION(UNPACK_SEQUENCE); + } + GO_TO_INSTRUCTION(UNPACK_SEQUENCE_GENERIC); + unbound_local_error: { format_exc_check_arg(tstate, PyExc_UnboundLocalError, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2ac596740d..0e000866d0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -409,9 +409,9 @@ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 4, "incorrect cache size"); PyObject *sub = PEEK(1); PyObject *container = PEEK(2); - _PyMutex_lock(&_PyRuntime.mutex); _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); @@ -419,8 +419,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(BINARY_SUBSCR_GENERIC); } @@ -605,9 +603,9 @@ PREDICTED(STORE_SUBSCR); PyObject *sub = PEEK(1); PyObject *container = PEEK(2); - _PyMutex_lock(&_PyRuntime.mutex); _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); @@ -615,8 +613,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(STORE_SUBSCR_GENERIC); } @@ -1110,9 +1106,9 @@ TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); - _PyMutex_lock(&_PyRuntime.mutex); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; @@ -1121,8 +1117,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(UNPACK_SEQUENCE_GENERIC); } @@ -1198,9 +1192,9 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); PyObject *owner = PEEK(1); - _PyMutex_lock(&_PyRuntime.mutex); _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1209,8 +1203,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(STORE_ATTR_GENERIC); } @@ -1330,9 +1322,9 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); - _PyMutex_lock(&_PyRuntime.mutex); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1341,8 +1333,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(LOAD_GLOBAL_GENERIC); } @@ -1764,9 +1754,9 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); - _PyMutex_lock(&_PyRuntime.mutex); _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); @@ -1776,8 +1766,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(LOAD_ATTR_GENERIC); } @@ -2127,9 +2115,9 @@ PREDICTED(COMPARE_OP); PyObject *right = PEEK(1); PyObject *left = PEEK(2); - _PyMutex_lock(&_PyRuntime.mutex); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); @@ -2137,8 +2125,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } @@ -2635,9 +2621,9 @@ TARGET(FOR_ITER) { PREDICTED(FOR_ITER); - _PyMutex_lock(&_PyRuntime.mutex); _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); @@ -2645,8 +2631,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(FOR_ITER_GENERIC); } @@ -2976,9 +2960,9 @@ TARGET(CALL) { PREDICTED(CALL); - _PyMutex_lock(&_PyRuntime.mutex); _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; @@ -2989,8 +2973,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(CALL_GENERIC); } @@ -3737,9 +3719,9 @@ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs = PEEK(1); PyObject *lhs = PEEK(2); - _PyMutex_lock(&_PyRuntime.mutex); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (DECREMENT_ADAPTIVE_COUNTER(&cache->counter)) { + _PyMutex_lock(&_PyRuntime.mutex); assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); @@ -3747,8 +3729,6 @@ DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - _PyMutex_unlock(&_PyRuntime.mutex); GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } diff --git a/Python/specialize.c b/Python/specialize.c index 7b38aff378..3dc26bce4e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -460,6 +460,17 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9 #define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10 +static void +_py_set_opcode_failure(_Py_CODEUNIT *instr, uint8_t opcode_generic) +{ + if (!_PyRuntime.multithreaded) { + _py_set_opcode(instr, _PyOpcode_Deopt[opcode_generic]); + } + else { + _py_set_opcode(instr, opcode_generic); + } +} + static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); @@ -712,7 +723,7 @@ static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyOb void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - if (instr->opcode != LOAD_ATTR) { + if (_PyRuntime.multithreaded && instr->opcode != LOAD_ATTR) { // another thread concurrently specialized this instruction STAT_INC(LOAD_ATTR, failure); return; @@ -884,7 +895,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, LOAD_ATTR_GENERIC); + _py_set_opcode_failure(instr, LOAD_ATTR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -896,7 +907,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - if (instr->opcode != STORE_ATTR) { + if (_PyRuntime.multithreaded && instr->opcode != STORE_ATTR) { // another thread concurrently specialized this instruction STAT_INC(STORE_ATTR, failure); return; @@ -991,7 +1002,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, STORE_ATTR_GENERIC); + _py_set_opcode_failure(instr, STORE_ATTR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1191,7 +1202,7 @@ _Py_Specialize_LoadGlobal( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) { - if (instr->opcode != LOAD_GLOBAL) { + if (_PyRuntime.multithreaded && instr->opcode != LOAD_GLOBAL) { // another thread concurrently specialized this instruction STAT_INC(LOAD_GLOBAL, failure); return; @@ -1269,7 +1280,7 @@ _Py_Specialize_LoadGlobal( fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, LOAD_GLOBAL_GENERIC); + _py_set_opcode_failure(instr, LOAD_GLOBAL_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1367,7 +1378,7 @@ void _Py_Specialize_BinarySubscr( PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { - if (instr->opcode != BINARY_SUBSCR) { + if (_PyRuntime.multithreaded && instr->opcode != BINARY_SUBSCR) { // another thread concurrently specialized this instruction STAT_INC(BINARY_SUBSCR, failure); return; @@ -1447,7 +1458,7 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, BINARY_SUBSCR_GENERIC); + _py_set_opcode_failure(instr, BINARY_SUBSCR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1459,7 +1470,7 @@ _Py_Specialize_BinarySubscr( void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { - if (instr->opcode != STORE_SUBSCR) { + if (_PyRuntime.multithreaded && instr->opcode != STORE_SUBSCR) { // another thread concurrently specialized this instruction STAT_INC(STORE_SUBSCR, failure); return; @@ -1562,7 +1573,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, STORE_SUBSCR_GENERIC); + _py_set_opcode_failure(instr, STORE_SUBSCR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1849,7 +1860,7 @@ void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { - if (instr->opcode != CALL) { + if (_PyRuntime.multithreaded && instr->opcode != CALL) { // another thread concurrently specialized this instruction STAT_INC(CALL, failure); return; @@ -1888,7 +1899,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, CALL_GENERIC); + _py_set_opcode_failure(instr, CALL_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); } else { @@ -1972,7 +1983,7 @@ void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals) { - if (instr->opcode != BINARY_OP) { + if (_PyRuntime.multithreaded && instr->opcode != BINARY_OP) { // another thread concurrently specialized this instruction STAT_INC(BINARY_OP, failure); return; @@ -2036,7 +2047,7 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - _py_set_opcode(instr, BINARY_OP_GENERIC); + _py_set_opcode_failure(instr, BINARY_OP_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2098,7 +2109,7 @@ void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg) { - if (instr->opcode != COMPARE_OP) { + if (_PyRuntime.multithreaded && instr->opcode != COMPARE_OP) { // another thread concurrently specialized this instruction STAT_INC(COMPARE_OP, failure); return; @@ -2153,7 +2164,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); - _py_set_opcode(instr, COMPARE_OP_GENERIC); + _py_set_opcode_failure(instr, COMPARE_OP_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2178,7 +2189,7 @@ unpack_sequence_fail_kind(PyObject *seq) void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) { - if (instr->opcode != UNPACK_SEQUENCE) { + if (_PyRuntime.multithreaded && instr->opcode != UNPACK_SEQUENCE) { // another thread concurrently specialized this instruction STAT_INC(UNPACK_SEQUENCE, failure); return; @@ -2209,7 +2220,7 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - _py_set_opcode(instr, UNPACK_SEQUENCE_GENERIC); + _py_set_opcode_failure(instr, UNPACK_SEQUENCE_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2292,7 +2303,7 @@ int void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) { - if (instr->opcode != FOR_ITER) { + if (_PyRuntime.multithreaded && instr->opcode != FOR_ITER) { // another thread concurrently specialized this instruction STAT_INC(FOR_ITER, failure); return; @@ -2322,7 +2333,7 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); - _py_set_opcode(instr, FOR_ITER_GENERIC); + _py_set_opcode_failure(instr, FOR_ITER_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: