From 58461fe2a5ccda09b625542ec010e3e9e0c09537 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 6 Feb 2025 23:21:57 +0800 Subject: [PATCH] gh-128563: A new tail-calling interpreter (GH-128718) Co-authored-by: Garrett Gu Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/tail-call.yml | 113 + Lib/test/test_generated_cases.py | 330 +- ...-01-10-18-56-20.gh-issue-128563.baDvls.rst | 1 + Python/bytecodes.c | 9 +- Python/ceval.c | 23 +- Python/ceval_macros.h | 47 +- Python/generated_cases.c.h | 3232 ++++++++++++++--- Python/opcode_targets.h | 505 +++ Tools/cases_generator/analyzer.py | 1 + Tools/cases_generator/generators_common.py | 24 +- Tools/cases_generator/target_generator.py | 40 + Tools/cases_generator/tier1_generator.py | 84 +- Tools/jit/template.c | 8 - configure | 48 + configure.ac | 38 + pyconfig.h.in | 3 + 16 files changed, 3883 insertions(+), 623 deletions(-) create mode 100644 .github/workflows/tail-call.yml create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml new file mode 100644 index 000000000000000..ad5d0aa55173c4d --- /dev/null +++ b/.github/workflows/tail-call.yml @@ -0,0 +1,113 @@ +name: Tail calling interpreter +on: + pull_request: + paths: + - 'Python/bytecodes.c' + - 'Python/ceval.c' + - 'Python/ceval_macros.h' + push: + paths: + - 'Python/bytecodes.c' + - 'Python/ceval.c' + - 'Python/ceval_macros.h' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + tail-call: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: +# Un-comment as we add support for more platforms for tail-calling interpreters. +# - i686-pc-windows-msvc/msvc +# - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + llvm: + - 19 + include: +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-latest +# - target: x86_64-pc-windows-msvc/msvc +# architecture: x64 +# runner: windows-latest +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-latest + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Native Windows (debug) + if: runner.os == 'Windows' && matrix.architecture != 'ARM64' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }} + ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # No tests (yet): + - name: Emulated Windows (release) + if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} + + # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. + # This is a bug in the macOS runner image where the pre-installed Python is installed in the same + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # the symlink to the pre-installed Python so that the Homebrew Python is used instead. + - name: Native macOS (debug) + if: runner.os == 'macOS' + run: | + brew update + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + export PATH="/opt/homebrew/opt/llvm/bin:$PATH" + export PATH="/usr/local/opt/llvm/bin:$PATH" + CC=clang-19 ./configure --with-tail-call-interp --with-pydebug + make all --jobs 4 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Native Linux (release) + if: runner.os == 'Linux' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-19 ./configure --with-tail-call-interp + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index d2b33706ea6b75a..35ad9ebbe5a1a90 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -304,6 +304,11 @@ def test_inst_no_args(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -322,6 +327,11 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -343,6 +353,11 @@ def test_inst_one_push(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -365,6 +380,11 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -388,6 +408,11 @@ def test_binary_op(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -414,6 +439,11 @@ def test_overlap(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -442,6 +472,11 @@ def test_predictions(self): """ output = """ TARGET(OP1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -453,12 +488,23 @@ def test_predictions(self): } TARGET(OP3) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP3; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); _PyStackRef res; - DEOPT_IF(xxx, OP1); + if (xxx) { + UPDATE_MISS_STATS(OP1); + assert(_PyOpcode_Deopt[opcode] == (OP1)); + JUMP_TO_PREDICTED(OP1); + } res = Py_None; stack_pointer[-1] = res; DISPATCH(); @@ -481,6 +527,11 @@ def test_sync_sp(self): """ output = """ TARGET(A) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = A; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(A); @@ -498,6 +549,11 @@ def test_sync_sp(self): } TARGET(B) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = B; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(B); @@ -535,11 +591,16 @@ def test_error_if_plain(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); if (cond) { - goto label; + JUMP_TO_LABEL(label); } DISPATCH(); } @@ -554,11 +615,16 @@ def test_error_if_plain_with_comment(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); if (cond) { - goto label; + JUMP_TO_LABEL(label); } // Comment is ok DISPATCH(); @@ -577,6 +643,11 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -587,7 +658,7 @@ def test_error_if_pop(self): left = stack_pointer[-2]; SPAM(left, right); if (cond) { - goto pop_2_label; + JUMP_TO_LABEL(pop_2_label); } res = 0; stack_pointer[-2] = res; @@ -608,6 +679,11 @@ def test_error_if_pop_with_result(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -618,7 +694,7 @@ def test_error_if_pop_with_result(self): left = stack_pointer[-2]; res = SPAM(left, right); if (cond) { - goto pop_2_label; + JUMP_TO_LABEL(pop_2_label); } stack_pointer[-2] = res; stack_pointer += -1; @@ -635,8 +711,14 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(OP); uint16_t counter = read_u16(&this_instr[1].cache); @@ -661,13 +743,18 @@ def test_suppress_dispatch(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - goto somewhere; + JUMP_TO_LABEL(somewhere); } - somewhere: + LABEL(somewhere) { } @@ -692,6 +779,11 @@ def test_macro_instruction(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP); @@ -729,8 +821,14 @@ def test_macro_instruction(self): } TARGET(OP1) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(OP1); _PyStackRef left; @@ -746,6 +844,11 @@ def test_macro_instruction(self): } TARGET(OP3) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP3; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP3); @@ -777,6 +880,11 @@ def test_unused_caches(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(OP); @@ -799,6 +907,11 @@ def test_pseudo_instruction_no_flags(self): """ output = """ TARGET(OP1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -818,6 +931,11 @@ def test_pseudo_instruction_with_flags(self): """ output = """ TARGET(OP1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -840,6 +958,11 @@ def test_pseudo_instruction_as_sequence(self): """ output = """ TARGET(OP1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -847,6 +970,11 @@ def test_pseudo_instruction_as_sequence(self): } TARGET(OP2) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP2; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); @@ -864,6 +992,11 @@ def test_array_input(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -887,6 +1020,11 @@ def test_array_output(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -915,6 +1053,11 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -939,13 +1082,18 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); if (oparg == 0) { stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto somewhere; + JUMP_TO_LABEL(somewhere); } stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -965,6 +1113,11 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1006,6 +1159,11 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1050,6 +1208,11 @@ def test_macro_push_push(self): """ output = """ TARGET(M) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1083,6 +1246,11 @@ def test_override_inst(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1104,6 +1272,11 @@ def test_override_op(self): """ output = """ TARGET(M) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1121,6 +1294,11 @@ def test_annotated_inst(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1139,6 +1317,11 @@ def test_annotated_op(self): """ output = """ TARGET(M) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1175,6 +1358,11 @@ def test_array_of_one(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1197,6 +1385,11 @@ def test_pointer_to_stackref(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1235,6 +1428,11 @@ def test_unused_named_values(self): """ output = """ TARGET(INST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INST); @@ -1261,6 +1459,11 @@ def test_used_unused_used(self): """ output = """ TARGET(TEST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1301,6 +1504,11 @@ def test_unused_used_used(self): """ output = """ TARGET(TEST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1340,6 +1548,11 @@ def test_flush(self): """ output = """ TARGET(TEST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1388,6 +1601,11 @@ def test_pop_on_error_peeks(self): """ output = """ TARGET(TEST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1409,7 +1627,7 @@ def test_pop_on_error_peeks(self): { // Mark j and k as used if (cond) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } } stack_pointer += -2; @@ -1436,6 +1654,11 @@ def test_push_then_error(self): output = """ TARGET(TEST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1453,7 +1676,7 @@ def test_push_then_error(self): stack_pointer[1] = b; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } } stack_pointer[0] = a; @@ -1477,17 +1700,27 @@ def test_error_if_true(self): """ output = """ TARGET(OP1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); - goto here; + JUMP_TO_LABEL(here); } TARGET(OP2) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP2; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); - goto there; + JUMP_TO_LABEL(there); } """ self.run_cases_test(input, output) @@ -1541,6 +1774,11 @@ def test_stack_save_reload(self): output = """ TARGET(BALANCED) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BALANCED; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1561,6 +1799,11 @@ def test_stack_reload_only(self): output = """ TARGET(BALANCED) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BALANCED; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1582,6 +1825,13 @@ def test_stack_save_only(self): output = """ TARGET(BALANCED) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BALANCED; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1602,6 +1852,11 @@ def test_instruction_size_macro(self): output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1637,6 +1892,11 @@ def test_escaping_call_next_to_cmacro(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1671,6 +1931,11 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1702,6 +1967,11 @@ def test_pop_input(self): """ output = """ TARGET(OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1793,26 +2063,26 @@ def test_complex_label(self): """ output = """ - other_label: + LABEL(other_label) { } - other_label2: + LABEL(other_label2) { } - my_label: + LABEL(my_label) { // Comment _PyFrame_SetStackPointer(frame, stack_pointer); do_thing(); stack_pointer = _PyFrame_GetStackPointer(frame); if (complex) { - goto other_label; + JUMP_TO_LABEL(other_label); } - goto other_label2; + JUMP_TO_LABEL(other_label2); } """ self.run_cases_test(input, output) @@ -1831,17 +2101,17 @@ def test_spilled_label(self): """ output = """ - one: + LABEL(one) { /* STACK SPILLED */ stack_pointer = _PyFrame_GetStackPointer(frame); - goto two; + JUMP_TO_LABEL(two); } - two: + LABEL(two) { _PyFrame_SetStackPointer(frame, stack_pointer); - goto one; + JUMP_TO_LABEL(one); } """ self.run_cases_test(input, output) @@ -1887,20 +2157,26 @@ def test_multiple_labels(self): """ output = """ - my_label_1: + LABEL(my_label_1) { // Comment + _PyFrame_SetStackPointer(frame, stack_pointer); do_thing1(); - goto my_label_2; + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_2); } - my_label_2: + LABEL(my_label_2) { // Comment + _PyFrame_SetStackPointer(frame, stack_pointer); do_thing2(); - goto my_label_1; + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_1); } """ + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst new file mode 100644 index 000000000000000..4d29f346cb62515 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst @@ -0,0 +1 @@ +A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-11% geometric mean faster on pyperformance (depending on platform), and up to 30% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas on how to implement this in CPython by Mark Shannon, Garret Gu, Haoran Xu, and Josh Haberman. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b650613650cf366..c3024f7f98f28c1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1387,7 +1387,9 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #ifndef Py_TAIL_CALL_INTERP assert(throwflag); + #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); @@ -5305,6 +5307,9 @@ dummy_func( } #endif RELOAD_STACK(); +#ifdef Py_TAIL_CALL_INTERP + int opcode; +#endif DISPATCH(); } @@ -5351,8 +5356,10 @@ dummy_func( caller loses its exception */ assert(!_PyErr_Occurred(tstate)); #endif - RELOAD_STACK(); +#ifdef Py_TAIL_CALL_INTERP + int opcode; +#endif DISPATCH(); } diff --git a/Python/ceval.c b/Python/ceval.c index 4f628fdbabddbb5..5e834883f355e1c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -768,13 +768,18 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #define PY_EVAL_C_STACK_UNITS 2 +#ifdef Py_TAIL_CALL_INTERP +#include "opcode_targets.h" +#include "generated_cases.c.h" +#endif + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { _Py_EnsureTstateNotNULL(tstate); CALL_STAT_INC(pyeval_calls); -#if USE_COMPUTED_GOTOS +#if USE_COMPUTED_GOTOS && !defined(Py_TAIL_CALL_INTERP) /* Import the static jump table */ #include "opcode_targets.h" #endif @@ -782,10 +787,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_STATS int lastopcode = 0; #endif +#ifndef Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ - - _PyInterpreterFrame entry_frame; +#endif + _PyInterpreterFrame entry_frame; if (_Py_EnterRecursiveCallTstate(tstate, "")) { assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -845,7 +851,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); monitor_throw(tstate, frame, next_instr); +#ifdef Py_TAIL_CALL_INTERP + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0); +#else goto error; +#endif } #if defined(_Py_TIER2) && !defined(_Py_JIT) @@ -854,9 +864,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int const _PyUOpInstruction *next_uop = NULL; #endif +#ifdef Py_TAIL_CALL_INTERP + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0); +#else goto start_frame; - -#include "generated_cases.c.h" +# include "generated_cases.c.h" +#endif #ifdef _Py_TIER2 diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b6d9df329538926..44bb52a09aab5f9 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -70,12 +70,41 @@ #define INSTRUCTION_STATS(op) ((void)0) #endif -#if USE_COMPUTED_GOTOS +#define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg +#define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg + +#ifdef Py_TAIL_CALL_INTERP + // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. +# define Py_MUSTTAIL [[clang::musttail]] +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) + Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); + +# define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) +# define DISPATCH_GOTO() \ + do { \ + Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_LABEL(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, oparg); \ + } while (0) +# define LABEL(name) TARGET(name) +#elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ @@ -92,7 +121,7 @@ do { \ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ frame->lltrace = lltrace; \ if (lltrace < 0) { \ - goto exit_unwind; \ + JUMP_TO_LABEL(exit_unwind); \ } \ } while (0) #else @@ -129,11 +158,11 @@ do { \ assert((NEW_FRAME)->previous == frame); \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ - goto start_frame; \ + JUMP_TO_LABEL(start_frame); \ } while (0) // Use this instead of 'goto error' so Tier 2 can go to a different label -#define GOTO_ERROR(LABEL) goto LABEL +#define GOTO_ERROR(LABEL) JUMP_TO_LABEL(LABEL) /* Tuple access macros */ @@ -236,14 +265,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define UPDATE_MISS_STATS(INSTNAME) ((void)0) #endif -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - /* This is only a single jump on release builds! */ \ - UPDATE_MISS_STATS((INSTNAME)); \ - assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - goto PREDICTED_##INSTNAME; \ - } - // Try to lock an object in the free threading build, if it's not already // locked. Use with a DEOPT_IF() to deopt if the object is already locked. @@ -328,7 +349,7 @@ do { \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ - goto error; \ + JUMP_TO_LABEL(error); \ } \ } \ } while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5820147ff712a96..7927fa7db95c20a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,15 +8,22 @@ #endif #define TIER_ONE 1 +#ifndef Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif { +#endif /* Py_TAIL_CALL_INTERP */ /* BEGIN INSTRUCTIONS */ TARGET(BINARY_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -58,7 +65,7 @@ PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -69,6 +76,13 @@ } TARGET(BINARY_OP_ADD_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); @@ -82,8 +96,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -98,7 +120,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -109,6 +131,13 @@ } TARGET(BINARY_OP_ADD_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); @@ -122,8 +151,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -137,7 +174,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -148,6 +185,13 @@ } TARGET(BINARY_OP_ADD_UNICODE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); @@ -161,8 +205,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -176,7 +228,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -187,8 +239,14 @@ } TARGET(BINARY_OP_EXTEND) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_EXTEND; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -209,7 +267,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -234,6 +296,13 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); @@ -246,8 +315,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -264,7 +341,11 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -284,7 +365,7 @@ *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (PyStackRef_IsNull(*target_local)) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -299,6 +380,13 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); @@ -312,8 +400,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -328,7 +424,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -339,6 +435,13 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); @@ -352,8 +455,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -367,7 +478,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -378,6 +489,13 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); @@ -391,8 +509,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -407,7 +533,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -418,6 +544,13 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); @@ -431,8 +564,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -446,7 +587,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -457,6 +598,11 @@ } TARGET(BINARY_SLICE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SLICE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -502,7 +648,7 @@ PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -513,6 +659,11 @@ } TARGET(BINARY_SUBSCR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); @@ -551,7 +702,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -562,6 +713,13 @@ } TARGET(BINARY_SUBSCR_DICT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); @@ -574,7 +732,11 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -588,7 +750,7 @@ PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); if (rc <= 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); @@ -599,6 +761,13 @@ } TARGET(BINARY_SUBSCR_GETITEM) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_GETITEM; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); @@ -610,22 +779,42 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -658,6 +847,13 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_LIST_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); @@ -670,19 +866,39 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -702,6 +918,13 @@ } TARGET(BINARY_SUBSCR_STR_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_STR_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); @@ -714,14 +937,34 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyUnicode_CheckExact(str)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -738,6 +981,13 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_TUPLE_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); @@ -750,12 +1000,28 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyTuple_CheckExact(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -774,6 +1040,11 @@ } TARGET(BUILD_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_LIST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -782,7 +1053,7 @@ values = &stack_pointer[-oparg]; PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); if (list_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; @@ -792,6 +1063,11 @@ } TARGET(BUILD_MAP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_MAP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -805,7 +1081,7 @@ } stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -820,7 +1096,7 @@ if (map_o == NULL) { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; @@ -830,6 +1106,11 @@ } TARGET(BUILD_SET) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_SET; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -845,7 +1126,7 @@ } stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } int err = 0; for (int i = 0; i < oparg; i++) { @@ -864,7 +1145,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; @@ -874,6 +1155,11 @@ } TARGET(BUILD_SLICE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_SLICE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -890,7 +1176,7 @@ if (slice_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-oparg] = slice; @@ -900,6 +1186,11 @@ } TARGET(BUILD_STRING) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_STRING; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -913,7 +1204,7 @@ } stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); @@ -923,7 +1214,7 @@ if (str_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; @@ -933,6 +1224,11 @@ } TARGET(BUILD_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_TUPLE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -941,7 +1237,7 @@ values = &stack_pointer[-oparg]; PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); if (tup_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; @@ -951,6 +1247,11 @@ } TARGET(CACHE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CACHE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -960,13 +1261,17 @@ } TARGET(CALL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - opcode = CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1041,7 +1346,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -1056,7 +1361,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -1097,7 +1402,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1113,7 +1418,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1126,8 +1431,14 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1141,7 +1452,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1152,23 +1467,39 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; @@ -1198,7 +1529,7 @@ assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); - goto error; + JUMP_TO_LABEL(error); } init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -1227,8 +1558,14 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1240,14 +1577,26 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1267,9 +1616,17 @@ { uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1277,15 +1634,27 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -1329,8 +1698,14 @@ } TARGET(CALL_BOUND_METHOD_GENERAL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1342,7 +1717,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_METHOD_VERSION { @@ -1350,11 +1729,27 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _EXPAND_METHOD { @@ -1394,7 +1789,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -1426,6 +1821,13 @@ } TARGET(CALL_BUILTIN_CLASS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); @@ -1442,7 +1844,11 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -1450,7 +1856,11 @@ arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1461,7 +1871,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -1475,7 +1885,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1491,7 +1901,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1504,6 +1914,13 @@ } TARGET(CALL_BUILTIN_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_FAST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); @@ -1527,8 +1944,16 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1541,7 +1966,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( @@ -1559,7 +1984,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1575,7 +2000,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1588,6 +2013,13 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); @@ -1611,8 +2043,16 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1629,7 +2069,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -1644,7 +2084,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1660,7 +2100,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1673,6 +2113,13 @@ } TARGET(CALL_BUILTIN_O) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_O; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); @@ -1695,11 +2142,27 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1718,7 +2181,7 @@ PyStackRef_CLOSE(callable[0]); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1734,7 +2197,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1747,11 +2210,16 @@ } TARGET(CALL_FUNCTION_EX) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_FUNCTION_EX; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef kwargs_in; @@ -1777,13 +2245,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } kwargs_out = kwargs_in; stack_pointer += -2; @@ -1824,7 +2292,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -1873,7 +2341,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 1 == 1); frame->return_offset = 1; @@ -1905,7 +2373,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -1921,7 +2389,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1934,6 +2402,11 @@ } TARGET(CALL_INTRINSIC_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_INTRINSIC_1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -1946,7 +2419,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -1954,6 +2427,11 @@ } TARGET(CALL_INTRINSIC_2) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_INTRINSIC_2; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -1971,7 +2449,7 @@ PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1981,6 +2459,13 @@ } TARGET(CALL_ISINSTANCE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_ISINSTANCE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); @@ -2002,9 +2487,17 @@ arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2012,7 +2505,7 @@ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - goto error; + JUMP_TO_LABEL(error); } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2028,13 +2521,17 @@ } TARGET(CALL_KW) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - opcode = CALL_KW; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -2123,7 +2620,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -2140,7 +2637,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2182,7 +2679,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2193,8 +2690,14 @@ } TARGET(CALL_KW_BOUND_METHOD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2207,7 +2710,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _CHECK_METHOD_VERSION_KW { @@ -2215,11 +2722,27 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _EXPAND_METHOD_KW { @@ -2269,7 +2792,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -2301,10 +2824,16 @@ } TARGET(CALL_KW_NON_PY) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_NON_PY; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef kwnames; @@ -2317,8 +2846,16 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _CALL_KW_NON_PY { @@ -2346,7 +2883,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -2371,7 +2908,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2387,7 +2924,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2400,8 +2937,14 @@ } TARGET(CALL_KW_PY) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_PY; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2413,16 +2956,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _PY_FRAME_KW { @@ -2458,7 +3013,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -2490,6 +3045,13 @@ } TARGET(CALL_LEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_LEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); @@ -2510,9 +3072,17 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2520,7 +3090,7 @@ Py_ssize_t len_i = PyObject_Length(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - goto error; + JUMP_TO_LABEL(error); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2543,6 +3113,13 @@ } TARGET(CALL_LIST_APPEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_LIST_APPEND; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); @@ -2559,10 +3136,22 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + if (!PyList_Check(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2577,7 +3166,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } #if TIER_ONE // Skip the following POP_TOP. This is done here in tier one, and @@ -2589,6 +3178,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); @@ -2613,11 +3209,23 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2629,7 +3237,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = @@ -2646,7 +3254,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2662,7 +3270,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2675,6 +3283,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); @@ -2698,12 +3313,24 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2715,7 +3342,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -2732,7 +3359,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2748,7 +3375,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2761,6 +3388,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); @@ -2783,16 +3417,36 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2810,7 +3464,7 @@ PyStackRef_CLOSE(callable[0]); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2826,7 +3480,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2839,6 +3493,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); @@ -2862,16 +3523,36 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } _PyStackRef arg_stackref = arguments[1]; _PyStackRef self_stackref = arguments[0]; - DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2890,7 +3571,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2906,7 +3587,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2919,10 +3600,16 @@ } TARGET(CALL_NON_PY_GENERAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *self_or_null; @@ -2934,8 +3621,16 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CALL_NON_PY_GENERAL { @@ -2961,7 +3656,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -2979,7 +3674,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2995,7 +3690,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3008,8 +3703,14 @@ } TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3020,16 +3721,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3038,15 +3751,27 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -3090,8 +3815,14 @@ } TARGET(CALL_PY_GENERAL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_PY_GENERAL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3102,16 +3833,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _PY_FRAME_GENERAL { @@ -3137,7 +3880,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -3169,6 +3912,13 @@ } TARGET(CALL_STR_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_STR_1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); @@ -3187,8 +3937,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3199,7 +3957,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3215,7 +3973,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3228,6 +3986,13 @@ } TARGET(CALL_TUPLE_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_TUPLE_1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); @@ -3246,8 +4011,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3258,7 +4031,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3274,7 +4047,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3287,6 +4060,13 @@ } TARGET(CALL_TYPE_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_TYPE_1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); @@ -3303,8 +4083,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); stack_pointer[-3] = res; @@ -3317,6 +4105,11 @@ } TARGET(CHECK_EG_MATCH) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CHECK_EG_MATCH; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -3334,7 +4127,7 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3345,11 +4138,11 @@ PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } if (!Py_IsNone(match_o)) { stack_pointer += -2; @@ -3368,6 +4161,11 @@ } TARGET(CHECK_EXC_MATCH) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CHECK_EXC_MATCH; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -3384,7 +4182,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3396,8 +4194,14 @@ } TARGET(CLEANUP_THROW) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CLEANUP_THROW; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter_st; @@ -3409,7 +4213,9 @@ last_sent_val_st = stack_pointer[-2]; sub_iter_st = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #ifndef Py_TAIL_CALL_INTERP assert(throwflag); + #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); @@ -3427,7 +4233,7 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } stack_pointer[-3] = none; stack_pointer[-2] = value; @@ -3437,6 +4243,11 @@ } TARGET(COMPARE_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -3475,7 +4286,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } if (oparg & 16) { stack_pointer += -2; @@ -3485,7 +4296,7 @@ Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) { - goto error; + JUMP_TO_LABEL(error); } res = res_bool ? PyStackRef_True : PyStackRef_False; } @@ -3502,6 +4313,13 @@ } TARGET(COMPARE_OP_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); @@ -3515,8 +4333,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3540,6 +4366,13 @@ } TARGET(COMPARE_OP_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); @@ -3553,16 +4386,32 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3582,6 +4431,13 @@ } TARGET(COMPARE_OP_STR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_STR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); @@ -3595,8 +4451,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3621,6 +4485,11 @@ } TARGET(CONTAINS_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -3658,7 +4527,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } @@ -3669,6 +4538,13 @@ } TARGET(CONTAINS_OP_DICT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); @@ -3681,7 +4557,11 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + if (!PyDict_CheckExact(right_o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3689,7 +4569,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -3699,6 +4579,13 @@ } TARGET(CONTAINS_OP_SET) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP_SET; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); @@ -3711,7 +4598,11 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3720,7 +4611,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -3730,6 +4621,11 @@ } TARGET(CONVERT_VALUE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONVERT_VALUE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -3748,7 +4644,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; @@ -3758,6 +4654,11 @@ } TARGET(COPY) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COPY; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -3773,6 +4674,11 @@ } TARGET(COPY_FREE_VARS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COPY_FREE_VARS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -3791,6 +4697,11 @@ } TARGET(DELETE_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -3802,7 +4713,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3810,6 +4721,11 @@ } TARGET(DELETE_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -3821,7 +4737,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); @@ -3830,6 +4746,11 @@ } TARGET(DELETE_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -3841,7 +4762,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -3852,6 +4773,11 @@ } TARGET(DELETE_GLOBAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_GLOBAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -3861,19 +4787,24 @@ stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(DELETE_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -3885,7 +4816,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); @@ -3897,12 +4828,17 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(DELETE_SUBSCR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_SUBSCR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -3918,7 +4854,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3926,6 +4862,11 @@ } TARGET(DICT_MERGE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DICT_MERGE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -3946,7 +4887,7 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3955,6 +4896,11 @@ } TARGET(DICT_UPDATE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DICT_UPDATE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -3979,7 +4925,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3988,8 +4934,14 @@ } TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_ASYNC_FOR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; @@ -4012,7 +4964,7 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -4020,6 +4972,11 @@ } TARGET(END_FOR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_FOR; + (void)(opcode); next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -4038,6 +4995,11 @@ } TARGET(END_SEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_SEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -4056,11 +5018,16 @@ } TARGET(ENTER_EXECUTOR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = ENTER_EXECUTOR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -4090,6 +5057,11 @@ } TARGET(EXIT_INIT_CHECK) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = EXIT_INIT_CHECK; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -4102,7 +5074,7 @@ "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -4110,10 +5082,14 @@ } TARGET(EXTENDED_ARG) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = EXTENDED_ARG; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -4122,6 +5098,11 @@ } TARGET(FORMAT_SIMPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FORMAT_SIMPLE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -4141,7 +5122,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4157,6 +5138,11 @@ } TARGET(FORMAT_WITH_SPEC) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FORMAT_WITH_SPEC; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -4171,7 +5157,7 @@ PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -4181,6 +5167,11 @@ } TARGET(FOR_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -4219,7 +5210,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4243,6 +5234,13 @@ } TARGET(FOR_ITER_GEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_GEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); @@ -4253,14 +5251,26 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4291,6 +5301,13 @@ } TARGET(FOR_ITER_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_LIST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); @@ -4301,7 +5318,11 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _ITER_JUMP_LIST { @@ -4342,6 +5363,13 @@ } TARGET(FOR_ITER_RANGE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_RANGE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); @@ -4353,7 +5381,11 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _ITER_JUMP_RANGE { @@ -4376,7 +5408,7 @@ r->len--; PyObject *res = PyLong_FromLong(value); if (res == NULL) { - goto error; + JUMP_TO_LABEL(error); } next = PyStackRef_FromPyObjectSteal(res); } @@ -4387,6 +5419,13 @@ } TARGET(FOR_ITER_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_TUPLE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); @@ -4397,7 +5436,11 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _ITER_JUMP_TUPLE { @@ -4435,6 +5478,11 @@ } TARGET(GET_AITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_AITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -4456,14 +5504,14 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { @@ -4476,7 +5524,7 @@ Py_TYPE(iter_o)->tp_name); Py_DECREF(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4484,6 +5532,11 @@ } TARGET(GET_ANEXT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_ANEXT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -4494,7 +5547,7 @@ PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; @@ -4504,6 +5557,11 @@ } TARGET(GET_AWAITABLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_AWAITABLE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -4515,7 +5573,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4523,6 +5581,11 @@ } TARGET(GET_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -4535,7 +5598,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4543,6 +5606,11 @@ } TARGET(GET_LEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_LEN; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -4554,11 +5622,11 @@ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - goto error; + JUMP_TO_LABEL(error); } PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; @@ -4568,6 +5636,11 @@ } TARGET(GET_YIELD_FROM_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_YIELD_FROM_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -4586,7 +5659,7 @@ "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } iter = iterable; } @@ -4600,7 +5673,7 @@ PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); PyStackRef_CLOSE(iterable); @@ -4611,6 +5684,11 @@ } TARGET(IMPORT_FROM) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IMPORT_FROM; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -4622,7 +5700,7 @@ PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -4632,6 +5710,11 @@ } TARGET(IMPORT_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IMPORT_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -4649,7 +5732,7 @@ PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -4659,11 +5742,16 @@ } TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -4716,7 +5804,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _DO_CALL @@ -4750,7 +5838,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -4765,7 +5853,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -4806,7 +5894,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4822,7 +5910,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4835,11 +5923,16 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef kwargs_in; @@ -4865,13 +5958,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } kwargs_out = kwargs_in; stack_pointer += -2; @@ -4912,7 +6005,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -4961,7 +6054,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 1 == 1); frame->return_offset = 1; @@ -4993,7 +6086,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -5009,7 +6102,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5022,11 +6115,16 @@ } TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -5063,7 +6161,7 @@ frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _MAYBE_EXPAND_METHOD_KW @@ -5125,7 +6223,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -5142,7 +6240,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5184,7 +6282,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -5195,6 +6293,11 @@ } TARGET(INSTRUMENTED_END_FOR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_END_FOR; + (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -5210,7 +6313,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } PyStackRef_CLOSE(value); @@ -5220,8 +6323,14 @@ } TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; @@ -5235,7 +6344,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } val = value; @@ -5249,8 +6358,14 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ @@ -5269,7 +6384,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -5286,17 +6401,22 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (next_opcode < 0) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -5308,8 +6428,14 @@ } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ @@ -5322,7 +6448,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5334,8 +6460,14 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); @@ -5343,12 +6475,17 @@ } TARGET(INSTRUMENTED_LINE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_LINE; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -5364,7 +6501,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = this_instr+1; - goto error; + JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -5382,11 +6519,16 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -5412,7 +6554,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } // we make no attempt to optimize here; specializations should @@ -5447,7 +6589,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (super == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; @@ -5457,7 +6599,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -5473,9 +6615,15 @@ } TARGET(INSTRUMENTED_NOT_TAKEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); (void)this_instr; // INSTRUMENTED_JUMP requires this_instr @@ -5484,9 +6632,15 @@ } TARGET(INSTRUMENTED_POP_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; @@ -5501,8 +6655,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ @@ -5517,8 +6677,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ @@ -5537,8 +6703,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ @@ -5555,8 +6727,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ @@ -5571,8 +6749,14 @@ } TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_RESUME; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE @@ -5585,7 +6769,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); @@ -5609,7 +6793,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); @@ -5626,7 +6810,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5638,7 +6822,7 @@ tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ @@ -5649,8 +6833,14 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; @@ -5665,7 +6855,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _RETURN_VALUE @@ -5694,8 +6884,14 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); _PyStackRef val; @@ -5710,7 +6906,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; @@ -5761,6 +6957,11 @@ } TARGET(INTERPRETER_EXIT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INTERPRETER_EXIT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -5780,6 +6981,11 @@ } TARGET(IS_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IS_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -5799,6 +7005,11 @@ } TARGET(JUMP_BACKWARD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -5826,7 +7037,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5844,8 +7055,14 @@ } TARGET(JUMP_BACKWARD_JIT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); @@ -5859,7 +7076,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5891,7 +7108,7 @@ if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); if (optimized < 0) { - goto error; + JUMP_TO_LABEL(error); } } else { @@ -5912,6 +7129,11 @@ } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -5926,6 +7148,11 @@ } TARGET(JUMP_BACKWARD_NO_JIT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -5940,7 +7167,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5958,6 +7185,11 @@ } TARGET(JUMP_FORWARD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_FORWARD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -5966,6 +7198,11 @@ } TARGET(LIST_APPEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LIST_APPEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -5976,7 +7213,7 @@ int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5984,6 +7221,11 @@ } TARGET(LIST_EXTEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LIST_EXTEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -6011,7 +7253,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -6021,6 +7263,11 @@ } TARGET(LOAD_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -6077,7 +7324,7 @@ */ PyStackRef_CLOSE(owner); if (attr_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } self_or_null[0] = PyStackRef_NULL; } @@ -6089,7 +7336,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -6101,8 +7348,14 @@ } TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_CLASS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6115,9 +7368,17 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -6140,8 +7401,14 @@ } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6154,16 +7421,28 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_CLASS { @@ -6185,8 +7464,14 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6198,17 +7483,33 @@ PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -6222,8 +7523,14 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6237,14 +7544,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_INSTANCE_VALUE { @@ -6252,10 +7567,18 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6278,8 +7601,14 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6293,7 +7622,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -6301,7 +7634,11 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -6322,8 +7659,14 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6337,7 +7680,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -6359,8 +7706,14 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6374,14 +7727,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_KEYS_VERSION { @@ -6389,7 +7750,11 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -6410,8 +7775,14 @@ } TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_MODULE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6425,11 +7796,19 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -6440,11 +7819,19 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } #else Py_INCREF(attr_o); @@ -6468,8 +7855,14 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6482,7 +7875,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -6500,8 +7897,14 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6514,14 +7917,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_KEYS_VERSION { @@ -6529,7 +7940,11 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -6545,8 +7960,14 @@ } TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6555,7 +7976,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_TYPE_VERSION { @@ -6563,7 +7988,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -6573,10 +8002,26 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -6611,8 +8056,14 @@ } TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_SLOT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6626,7 +8077,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_SLOT { @@ -6634,10 +8089,18 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -6657,8 +8120,14 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6673,14 +8142,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR); + if (dict_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -6689,26 +8166,46 @@ uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6728,6 +8225,11 @@ } TARGET(LOAD_BUILD_CLASS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_BUILD_CLASS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -6737,14 +8239,14 @@ int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -6754,6 +8256,11 @@ } TARGET(LOAD_COMMON_CONSTANT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -6776,6 +8283,11 @@ } TARGET(LOAD_CONST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); @@ -6809,6 +8321,11 @@ } TARGET(LOAD_CONST_IMMORTAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST_IMMORTAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); @@ -6824,6 +8341,11 @@ } TARGET(LOAD_CONST_MORTAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST_MORTAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_MORTAL); @@ -6838,6 +8360,11 @@ } TARGET(LOAD_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -6848,7 +8375,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; @@ -6858,6 +8385,11 @@ } TARGET(LOAD_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -6871,6 +8403,11 @@ } TARGET(LOAD_FAST_AND_CLEAR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -6884,6 +8421,11 @@ } TARGET(LOAD_FAST_CHECK) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_CHECK; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -6896,7 +8438,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -6906,6 +8448,11 @@ } TARGET(LOAD_FAST_LOAD_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -6923,6 +8470,11 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -6939,7 +8491,7 @@ int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (!value_o) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -6948,7 +8500,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -6964,6 +8516,11 @@ } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -6977,7 +8534,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) @@ -6999,7 +8556,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + JUMP_TO_LABEL(error); } } else { @@ -7011,7 +8568,7 @@ int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (v_o == NULL) { /* namespace 2: builtins */ @@ -7019,7 +8576,7 @@ int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7027,7 +8584,7 @@ tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } } } @@ -7040,6 +8597,11 @@ } TARGET(LOAD_GLOBAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -7076,7 +8638,7 @@ _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) { - goto error; + JUMP_TO_LABEL(error); } } // _PUSH_NULL_CONDITIONAL @@ -7090,8 +8652,14 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -7103,18 +8671,34 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -7123,10 +8707,18 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -7145,8 +8737,14 @@ } TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -7158,9 +8756,17 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -7170,10 +8776,18 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -7192,6 +8806,11 @@ } TARGET(LOAD_LOCALS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_LOCALS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -7202,7 +8821,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -7212,6 +8831,11 @@ } TARGET(LOAD_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -7221,7 +8845,7 @@ PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -7231,6 +8855,11 @@ } TARGET(LOAD_SMALL_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SMALL_INT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -7245,6 +8874,11 @@ } TARGET(LOAD_SPECIAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SPECIAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -7269,7 +8903,7 @@ Py_TYPE(owner_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); self_or_null = self_or_null_o == NULL ? @@ -7282,13 +8916,17 @@ } TARGET(LOAD_SUPER_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -7330,7 +8968,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } // we make no attempt to optimize here; specializations should @@ -7365,7 +9003,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (super == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; @@ -7375,7 +9013,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -7391,6 +9029,13 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); @@ -7407,8 +9052,16 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7418,7 +9071,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (attr == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; @@ -7428,6 +9081,13 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); @@ -7445,8 +9105,16 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7456,7 +9124,7 @@ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } if (method_found) { self_or_null = self_st; // transfer ownership @@ -7481,6 +9149,11 @@ } TARGET(MAKE_CELL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAKE_CELL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -7489,7 +9162,7 @@ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); @@ -7500,6 +9173,11 @@ } TARGET(MAKE_FUNCTION) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAKE_FUNCTION; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -7517,7 +9195,7 @@ PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (func_obj == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -7529,6 +9207,11 @@ } TARGET(MAP_ADD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAP_ADD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -7550,7 +9233,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -7558,6 +9241,11 @@ } TARGET(MATCH_CLASS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_CLASS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -7586,7 +9274,7 @@ } else { if (_PyErr_Occurred(tstate)) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } // Error! attrs = PyStackRef_None; // Failure! @@ -7598,6 +9286,11 @@ } TARGET(MATCH_KEYS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_KEYS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -7612,7 +9305,7 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; @@ -7622,6 +9315,11 @@ } TARGET(MATCH_MAPPING) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_MAPPING; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -7637,6 +9335,11 @@ } TARGET(MATCH_SEQUENCE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_SEQUENCE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -7652,6 +9355,11 @@ } TARGET(NOP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = NOP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -7659,6 +9367,11 @@ } TARGET(NOT_TAKEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = NOT_TAKEN; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -7666,6 +9379,11 @@ } TARGET(POP_EXCEPT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_EXCEPT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -7683,6 +9401,11 @@ } TARGET(POP_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -7695,8 +9418,14 @@ } TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; @@ -7712,8 +9441,14 @@ } TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; @@ -7745,8 +9480,14 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; @@ -7778,8 +9519,14 @@ } TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; @@ -7795,6 +9542,11 @@ } TARGET(POP_TOP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_TOP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -7807,6 +9559,11 @@ } TARGET(PUSH_EXC_INFO) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = PUSH_EXC_INFO; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -7832,6 +9589,11 @@ } TARGET(PUSH_NULL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = PUSH_NULL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -7844,8 +9606,14 @@ } TARGET(RAISE_VARARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RAISE_VARARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; @@ -7864,14 +9632,20 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } - goto error; + JUMP_TO_LABEL(error); } TARGET(RERAISE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RERAISE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); _PyStackRef *values; @@ -7897,7 +9671,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); Py_DECREF(exc); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -7910,10 +9684,15 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } TARGET(RESERVED) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESERVED; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -7923,6 +9702,11 @@ } TARGET(RESUME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESUME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -7939,7 +9723,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); @@ -7963,7 +9747,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); @@ -7988,7 +9772,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -7997,26 +9781,50 @@ } TARGET(RESUME_CHECK) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESUME_CHECK; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } #ifdef Py_GIL_DISABLED - DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } #endif DISPATCH(); } TARGET(RETURN_GENERATOR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RETURN_GENERATOR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -8027,7 +9835,7 @@ PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); if (gen == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8052,6 +9860,11 @@ } TARGET(RETURN_VALUE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RETURN_VALUE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -8080,6 +9893,11 @@ } TARGET(SEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -8159,7 +9977,7 @@ } else { PyStackRef_CLOSE(v); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } stack_pointer += -1; @@ -8176,6 +9994,13 @@ } TARGET(SEND_GEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SEND_GEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); @@ -8187,15 +10012,27 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -8228,6 +10065,11 @@ } TARGET(SETUP_ANNOTATIONS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SETUP_ANNOTATIONS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -8237,21 +10079,21 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), @@ -8259,7 +10101,7 @@ Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } else { @@ -8271,6 +10113,11 @@ } TARGET(SET_ADD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_ADD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -8284,7 +10131,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8292,6 +10139,11 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -8316,6 +10168,11 @@ } TARGET(SET_UPDATE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_UPDATE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -8329,7 +10186,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8337,6 +10194,11 @@ } TARGET(STORE_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -8375,7 +10237,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } } stack_pointer += -2; @@ -8384,8 +10246,14 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -8398,11 +10266,19 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } } // _GUARD_DORV_NO_DICT @@ -8413,7 +10289,11 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } } // _STORE_ATTR_INSTANCE_VALUE @@ -8443,8 +10323,14 @@ } TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_SLOT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -8457,14 +10343,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -8481,8 +10375,14 @@ } TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -8495,7 +10395,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } // _STORE_ATTR_WITH_HINT { @@ -8504,12 +10408,24 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -8517,17 +10433,29 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -8548,6 +10476,11 @@ } TARGET(STORE_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -8563,6 +10496,11 @@ } TARGET(STORE_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -8579,6 +10517,11 @@ } TARGET(STORE_FAST_LOAD_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -8598,6 +10541,11 @@ } TARGET(STORE_FAST_STORE_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST_STORE_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -8625,6 +10573,11 @@ } TARGET(STORE_GLOBAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_GLOBAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -8636,7 +10589,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8644,6 +10597,11 @@ } TARGET(STORE_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -8658,7 +10616,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8672,7 +10630,7 @@ } PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8680,6 +10638,11 @@ } TARGET(STORE_SLICE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SLICE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -8721,7 +10684,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); if (err) { - goto pop_4_error; + JUMP_TO_LABEL(pop_4_error); } } stack_pointer += -4; @@ -8730,6 +10693,11 @@ } TARGET(STORE_SUBSCR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -8768,7 +10736,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } stack_pointer += -3; @@ -8777,6 +10745,13 @@ } TARGET(STORE_SUBSCR_DICT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); @@ -8789,7 +10764,11 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -8802,12 +10781,19 @@ PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); @@ -8821,16 +10807,36 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -8848,6 +10854,11 @@ } TARGET(SWAP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SWAP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -8863,6 +10874,11 @@ } TARGET(TO_BOOL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -8896,7 +10912,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = err ? PyStackRef_True : PyStackRef_False; } @@ -8905,8 +10921,14 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -8920,7 +10942,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } } // _REPLACE_WITH_TRUE { @@ -8933,6 +10959,13 @@ } TARGET(TO_BOOL_BOOL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_BOOL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); @@ -8941,12 +10974,23 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); DISPATCH(); } TARGET(TO_BOOL_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); @@ -8957,7 +11001,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -8972,6 +11020,13 @@ } TARGET(TO_BOOL_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_LIST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); @@ -8982,7 +11037,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + if (!PyList_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -8991,6 +11050,13 @@ } TARGET(TO_BOOL_NONE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); @@ -9001,7 +11067,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -9009,6 +11079,13 @@ } TARGET(TO_BOOL_STR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_STR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); @@ -9019,7 +11096,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -9035,6 +11116,11 @@ } TARGET(UNARY_INVERT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_INVERT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -9046,7 +11132,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -9054,6 +11140,11 @@ } TARGET(UNARY_NEGATIVE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_NEGATIVE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -9065,7 +11156,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -9073,6 +11164,11 @@ } TARGET(UNARY_NOT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_NOT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -9087,6 +11183,11 @@ } TARGET(UNPACK_EX) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_EX; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -9100,7 +11201,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -9108,6 +11209,11 @@ } TARGET(UNPACK_SEQUENCE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -9144,7 +11250,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } stack_pointer += -1 + oparg; @@ -9153,6 +11259,13 @@ } TARGET(UNPACK_SEQUENCE_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); @@ -9163,11 +11276,23 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + if (!PyList_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -9182,6 +11307,13 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); @@ -9192,8 +11324,16 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -9206,6 +11346,13 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); @@ -9217,8 +11364,16 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); @@ -9231,6 +11386,11 @@ } TARGET(WITH_EXCEPT_START) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = WITH_EXCEPT_START; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -9275,7 +11435,7 @@ (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -9285,6 +11445,11 @@ } TARGET(YIELD_VALUE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = YIELD_VALUE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); @@ -9331,6 +11496,7 @@ } /* END INSTRUCTIONS */ +#ifndef Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -9340,44 +11506,46 @@ next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); + } /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); +#endif /* Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ - pop_4_error: + LABEL(pop_4_error) { STACK_SHRINK(4); - goto error; + JUMP_TO_LABEL(error); } - pop_3_error: + LABEL(pop_3_error) { STACK_SHRINK(3); - goto error; + JUMP_TO_LABEL(error); } - pop_2_error: + LABEL(pop_2_error) { STACK_SHRINK(2); - goto error; + JUMP_TO_LABEL(error); } - pop_1_error: + LABEL(pop_1_error) { STACK_SHRINK(1); - goto error; + JUMP_TO_LABEL(error); } - error: + LABEL(error) { /* Double-check exception status. */ #ifdef NDEBUG @@ -9407,10 +11575,10 @@ _PyEval_MonitorRaise(tstate, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } - exception_unwind: + LABEL(exception_unwind) { /* STACK SPILLED */ /* We can't use frame->instr_ptr here, as RERAISE may have set it */ @@ -9427,7 +11595,7 @@ PyStackRef_XCLOSE(ref); } monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } assert(STACK_LEVEL() >= level); _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; @@ -9440,7 +11608,7 @@ int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); } @@ -9453,7 +11621,7 @@ next_instr = _PyFrame_GetBytecode(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } /* Resume normal execution */ #ifdef LLTRACE @@ -9462,10 +11630,13 @@ } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_TAIL_CALL_INTERP + int opcode; + #endif DISPATCH(); } - exit_unwind: + LABEL(exit_unwind) { /* STACK SPILLED */ assert(_PyErr_Occurred(tstate)); @@ -9484,15 +11655,15 @@ } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } - start_frame: + LABEL(start_frame) { /* STACK SPILLED */ int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } next_instr = frame->instr_ptr; #ifdef LLTRACE @@ -9500,7 +11671,7 @@ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); frame->lltrace = lltrace; if (lltrace < 0) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } } #endif @@ -9512,6 +11683,9 @@ assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_TAIL_CALL_INTERP + int opcode; + #endif DISPATCH(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 2b84f0281e53563..27c4d537b8079b1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1,3 +1,4 @@ +#ifndef Py_TAIL_CALL_INTERP static void *opcode_targets[256] = { &&TARGET_CACHE, &&TARGET_BINARY_SLICE, @@ -256,3 +257,507 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +#else /* Py_TAIL_CALL_INTERP */ +static py_tail_call_funcptr INSTRUCTION_TABLE[256]; + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); +} + +static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { + [BINARY_OP] = _TAIL_CALL_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, + [BINARY_SUBSCR] = _TAIL_CALL_BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = _TAIL_CALL_BINARY_SUBSCR_DICT, + [BINARY_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_SUBSCR_GETITEM, + [BINARY_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_SUBSCR_LIST_INT, + [BINARY_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_SUBSCR_STR_INT, + [BINARY_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT, + [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, + [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_CACHE, + [CALL] = _TAIL_CALL_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, + [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, + [COPY] = _TAIL_CALL_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, + [DELETE_ATTR] = _TAIL_CALL_DELETE_ATTR, + [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, + [DELETE_GLOBAL] = _TAIL_CALL_DELETE_GLOBAL, + [DELETE_NAME] = _TAIL_CALL_DELETE_NAME, + [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_END_FOR, + [END_SEND] = _TAIL_CALL_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, + [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, + [GET_AITER] = _TAIL_CALL_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_GET_ITER, + [GET_LEN] = _TAIL_CALL_GET_LEN, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, + [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, + [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, + [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, + [LOAD_CONST_MORTAL] = _TAIL_CALL_LOAD_CONST_MORTAL, + [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_NOP, + [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_POP_ITER, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_RERAISE, + [RESERVED] = _TAIL_CALL_RESERVED, + [RESUME] = _TAIL_CALL_RESUME, + [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, + [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, + [SEND] = _TAIL_CALL_SEND, + [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_SWAP, + [TO_BOOL] = _TAIL_CALL_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [118] = _TAIL_CALL_UNKNOWN_OPCODE, + [119] = _TAIL_CALL_UNKNOWN_OPCODE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [128] = _TAIL_CALL_UNKNOWN_OPCODE, + [129] = _TAIL_CALL_UNKNOWN_OPCODE, + [130] = _TAIL_CALL_UNKNOWN_OPCODE, + [131] = _TAIL_CALL_UNKNOWN_OPCODE, + [132] = _TAIL_CALL_UNKNOWN_OPCODE, + [133] = _TAIL_CALL_UNKNOWN_OPCODE, + [134] = _TAIL_CALL_UNKNOWN_OPCODE, + [135] = _TAIL_CALL_UNKNOWN_OPCODE, + [136] = _TAIL_CALL_UNKNOWN_OPCODE, + [137] = _TAIL_CALL_UNKNOWN_OPCODE, + [138] = _TAIL_CALL_UNKNOWN_OPCODE, + [139] = _TAIL_CALL_UNKNOWN_OPCODE, + [140] = _TAIL_CALL_UNKNOWN_OPCODE, + [141] = _TAIL_CALL_UNKNOWN_OPCODE, + [142] = _TAIL_CALL_UNKNOWN_OPCODE, + [143] = _TAIL_CALL_UNKNOWN_OPCODE, + [144] = _TAIL_CALL_UNKNOWN_OPCODE, + [145] = _TAIL_CALL_UNKNOWN_OPCODE, + [146] = _TAIL_CALL_UNKNOWN_OPCODE, + [147] = _TAIL_CALL_UNKNOWN_OPCODE, + [148] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, + [234] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +#endif /* Py_TAIL_CALL_INTERP */ diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 11a559bca474b07..aa88608168c1be4 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -675,6 +675,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "assert", "backoff_counter_triggers", "initial_temperature_backoff_counter", + "JUMP_TO_LABEL", "maybe_lltrace_resume_frame", "restart_backoff_counter", ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 1c572ec0512b370..cd52f181b60e437 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import TextIO from analyzer import ( Instruction, @@ -154,28 +153,31 @@ def deopt_if( storage: Storage, inst: Instruction | None, ) -> bool: - self.out.emit_at("DEOPT_IF", tkn) + self.out.start_line() + self.out.emit("if (") lparen = next(tkn_iter) - self.emit(lparen) assert lparen.kind == "LPAREN" first_tkn = tkn_iter.peek() emit_to(self.out, tkn_iter, "RPAREN") + self.emit(") {\n") next(tkn_iter) # Semi colon - self.out.emit(", ") assert inst is not None assert inst.family is not None - self.out.emit(inst.family.name) - self.out.emit(");\n") + family_name = inst.family.name + self.emit(f"UPDATE_MISS_STATS({family_name});\n") + self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") + self.emit(f"JUMP_TO_PREDICTED({family_name});\n") + self.emit("}\n") return not always_true(first_tkn) exit_if = deopt_if def goto_error(self, offset: int, label: str, storage: Storage) -> str: if offset > 0: - return f"goto pop_{offset}_{label};" + return f"JUMP_TO_LABEL(pop_{offset}_{label});" if offset < 0: storage.copy().flush(self.out) - return f"goto {label};" + return f"JUMP_TO_LABEL({label});" def error_if( self, @@ -410,8 +412,10 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: self.emit_save(storage) elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) - self.out.emit(goto) + self.out.start_line() + self.out.emit("JUMP_TO_LABEL(") self.out.emit(label) + self.out.emit(")") def emit_save(self, storage: Storage) -> None: storage.save(self.out) @@ -603,7 +607,7 @@ def _emit_block( elif tkn.kind == "GOTO": label_tkn = next(tkn_iter) self.goto_label(tkn, label_tkn, storage) - reachable = False; + reachable = False elif tkn.kind == "IDENTIFIER": if tkn.text in self._replacers: if not self._replacers[tkn.text](tkn, tkn_iter, uop, storage, inst): diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index c5097b7584724c9..db028116db93d20 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -13,6 +13,7 @@ DEFAULT_INPUT, ROOT, ) +from tier1_generator import UNKNOWN_OPCODE_HANDLER from cwriter import CWriter @@ -25,11 +26,49 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for name, op in analysis.opmap.items(): if op < 256: targets[op] = f"&&TARGET_{name},\n" + out.emit("#ifndef Py_TAIL_CALL_INTERP\n") out.emit("static void *opcode_targets[256] = {\n") for target in targets: out.emit(target) out.emit("};\n") + out.emit("#else /* Py_TAIL_CALL_INTERP */\n") +def function_proto(name: str) -> str: + return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + + +def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n") + out.emit("\n") + + # Emit function prototypes for labels. + for name in analysis.labels: + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + + # Emit function prototypes for opcode handlers. + for name in sorted(analysis.instructions.keys()): + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + + # Emit unknown opcode handler. + out.emit(function_proto("UNKNOWN_OPCODE")) + out.emit(" {\n") + out.emit("int opcode = next_instr->op.code;\n") + out.emit(UNKNOWN_OPCODE_HANDLER) + out.emit("}\n") + out.emit("\n") + + # Emit the dispatch table. + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_{name},\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") + outfile.write("#endif /* Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( description="Generate the file with dispatch targets.", @@ -52,3 +91,4 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: with open(args.output, "w") as outfile: out = CWriter(outfile, 0, False) write_opcode_targets(data, out) + write_tailcall_dispatch_table(data, out) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index c7cf09e2ec4edef..979646a7ca65251 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -24,10 +24,9 @@ Emitter, ) from cwriter import CWriter -from typing import TextIO +from typing import TextIO, Callable from stack import Local, Stack, StackError, get_stack_effect, Storage - DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" @@ -129,31 +128,52 @@ def uses_this(inst: Instruction) -> bool: for cache in uop.caches: if cache.name != "unused": return True + # Can't be merged into the loop above, because + # this must strictly be performed at the end. + for uop in inst.parts: + if not isinstance(uop, Uop): + continue + for tkn in uop.body: + if (tkn.kind == "IDENTIFIER" + and (tkn.text in {"DEOPT_IF", "EXIT_IF"})): + return True return False +UNKNOWN_OPCODE_HANDLER ="""\ +_PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); +""" + def generate_tier1( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: write_header(__file__, filenames, outfile) - outfile.write( - f""" + outfile.write(""" #ifdef TIER_TWO #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 - +""") + outfile.write(f""" +#ifndef Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif {{ +#endif /* Py_TAIL_CALL_INTERP */ {INSTRUCTION_START_MARKER} """ ) generate_tier1_cases(analysis, outfile, lines) outfile.write(f""" {INSTRUCTION_END_MARKER} +#ifndef Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -162,41 +182,41 @@ def generate_tier1( /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; + {UNKNOWN_OPCODE_HANDLER} }} /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); +#endif /* Py_TAIL_CALL_INTERP */ {LABEL_START_MARKER} """) - generate_tier1_labels(analysis, outfile, lines) + out = CWriter(outfile, 2, lines) + emitter = Emitter(out, analysis.labels) + generate_tier1_labels(analysis, emitter) outfile.write(f"{LABEL_END_MARKER}\n") outfile.write(FOOTER) + + def generate_tier1_labels( - analysis: Analysis, outfile: TextIO, lines: bool + analysis: Analysis, emitter: Emitter ) -> None: - out = CWriter(outfile, 2, lines) - emitter = Emitter(out, analysis.labels) - out.emit("\n") + emitter.emit("\n") + # Emit tail-callable labels as function defintions for name, label in analysis.labels.items(): - out.emit(f"{name}:\n") - out.emit("{\n") + emitter.emit(f"LABEL({name})\n") + emitter.emit("{\n") storage = Storage(Stack(), [], [], []) if label.spilled: storage.spilled = 1 - out.emit("/* STACK SPILLED */\n") + emitter.emit("/* STACK SPILLED */\n") emitter.emit_tokens(label, storage, None) - out.emit("\n") - out.emit("}\n") - out.emit("\n") + emitter.emit("\n") + emitter.emit("}\n") + emitter.emit("\n") + def generate_tier1_cases( analysis: Analysis, outfile: TextIO, lines: bool @@ -205,20 +225,26 @@ def generate_tier1_cases( emitter = Emitter(out, analysis.labels) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): - needs_this = uses_this(inst) out.emit("\n") out.emit(f"TARGET({name}) {{\n") + # We need to ifdef it because this breaks platforms + # without computed gotos/tail calling. + out.emit(f"#if defined(Py_TAIL_CALL_INTERP)\n") + out.emit(f"int opcode;\n") + out.emit(f"#endif\n") + out.emit(f"opcode = {name};\n") + out.emit(f"(void)(opcode);\n") + needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") + if needs_this and not inst.is_target: - if inst.properties.no_save_ip: - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") - else: - out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") out.emit(unused_guard) - elif not inst.properties.no_save_ip: + if not inst.properties.no_save_ip: out.emit(f"frame->instr_ptr = next_instr;\n") + out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: @@ -226,8 +252,6 @@ def generate_tier1_cases( if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) - if inst.properties.uses_opcode: - out.emit(f"opcode = {name};\n") if inst.family is not None: out.emit( f"static_assert({inst.family.size} == {inst.size-1}" diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 95c90bda70f3521..929f37d2dddea1e 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -32,14 +32,6 @@ #undef CURRENT_OPERAND1 #define CURRENT_OPERAND1() (_operand1) -#undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - do { \ - if ((COND)) { \ - goto deoptimize; \ - } \ - } while (0) - #undef ENABLE_SPECIALIZATION #define ENABLE_SPECIALIZATION (0) diff --git a/configure b/configure index 3eb787f788bfb95..22456959add174f 100755 --- a/configure +++ b/configure @@ -1122,6 +1122,7 @@ with_platlibdir with_wheel_pkg_dir with_readline with_computed_gotos +with_tail_call_interp with_ensurepip with_openssl with_openssl_rpath @@ -1929,6 +1930,8 @@ Optional Packages: use libedit for backend or disable readline module --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) + --tail-call-interp enable tail-calling interpreter in evaluation loop + and rest of CPython --with-ensurepip[=install|upgrade|no] "install" or "upgrade" using bundled pip (default is upgrade) @@ -29246,6 +29249,51 @@ printf "%s\n" "#define HAVE_COMPUTED_GOTOS 1" >>confdefs.h esac +# Check for --with-tail-call-interp +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-tail-call-interp" >&5 +printf %s "checking for --with-tail-call-interp... " >&6; } + +# Check whether --with-tail-call-interp was given. +if test ${with_tail_call_interp+y} +then : + withval=$with_tail_call_interp; +if test "$withval" = yes +then + +printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +fi +if test "$withval" = no +then + +printf "%s\n" "#define Py_TAIL_CALL_INTERP 0" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 +printf "%s\n" "no value specified" >&6; } ;; +esac +fi + + +# Do not enable tail-calling interpreter if tier 2 is enabled. +if ${tier2_flags:+false} : +then : + + case "$ac_cv_tail_call" in yes*) + +printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h + + esac + +fi + + case $ac_sys_system in AIX*) diff --git a/configure.ac b/configure.ac index c0130b8082cd8ab..cf16e77f0a1503b 100644 --- a/configure.ac +++ b/configure.ac @@ -7017,6 +7017,44 @@ case "$ac_cv_computed_gotos" in yes*) [Define if the C compiler supports computed gotos.]) esac +# Check for --with-tail-call-interp +AC_MSG_CHECKING([for --with-tail-call-interp]) +AC_ARG_WITH( + [tail-call-interp], + [AS_HELP_STRING( + [--tail-call-interp], + [enable tail-calling interpreter in evaluation loop and rest of CPython] + )], +[ +if test "$withval" = yes +then + AC_DEFINE([Py_TAIL_CALL_INTERP], [1], + [Define if you want to use tail-calling interpreters in CPython.]) + AC_MSG_RESULT([yes]) +fi +if test "$withval" = no +then + AC_DEFINE([Py_TAIL_CALL_INTERP], [0], + [Define if you want to use tail-calling interpreters in CPython.]) + AC_MSG_RESULT([no]) +fi +], +[AC_MSG_RESULT([no value specified])]) + +# Do not enable tail-calling interpreter if tier 2 is enabled. +AS_VAR_IF( + [tier2_flags], + [], + [ + case "$ac_cv_tail_call" in yes*) + AC_DEFINE([Py_TAIL_CALL_INTERP], [1], + [Define if the C compiler supports efficient proper tail calls.]) + esac + ], + [] +) + + case $ac_sys_system in AIX*) AC_DEFINE([HAVE_BROKEN_PIPE_BUF], [1], diff --git a/pyconfig.h.in b/pyconfig.h.in index 30e55158bad4d64..9ea01ad3fc0a316 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1718,6 +1718,9 @@ /* The version of SunOS/Solaris as reported by `uname -r' without the dot. */ #undef Py_SUNOS_VERSION +/* Define if the C compiler supports efficient proper tail calls. */ +#undef Py_TAIL_CALL_INTERP + /* Define if you want to enable tracing references for debugging purpose */ #undef Py_TRACE_REFS