WIP bpo-44800: Rename _PyInterpreterFrame to _Py_framedata

Doc/whatsnew/3.11.rst

@@ -919,8 +919,8 @@ Porting to Python 3.11
       #if PY_VERSION_HEX < 0x030900B1
       static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame)
       {
-          Py_INCREF(frame->f_code);
-          return frame->f_code;
+          Py_INCREF(frame->code);
+          return frame->code;
       }
       #endif
 

Include/cpython/ceval.h

@@ -22,7 +22,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *);
    flag was set, else return 0. */
 PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 
-PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc);
+PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _Py_framedata *fdata, int exc);
 
 PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);

Include/cpython/genobject.h

@@ -28,7 +28,7 @@ extern "C" {
     char prefix##_running_async;                                            \
     /* The frame */                                                         \
     char prefix##_frame_valid;                                              \
-    PyObject *prefix##_iframe[1];
+    void *prefix##_fdata[1];
 
 typedef struct {
     /* The gi_ prefix is intended to remind of generator-iterator. */

Include/cpython/pystate.h

@@ -48,7 +48,7 @@ typedef struct _PyCFrame {
      */
     int use_tracing;
     /* Pointer to the currently executing frame (it can be NULL) */
-    struct _PyInterpreterFrame *current_frame;
+    struct _Py_framedata *current_frame;
     struct _PyCFrame *previous;
 } _PyCFrame;
 
@@ -260,7 +260,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
 /* Frame evaluation API */
 
-typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int);
+typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _Py_framedata *, int);
 
 PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
     PyInterpreterState *interp);

Include/internal/pycore_ceval.h

@@ -47,12 +47,12 @@ extern PyObject *_PyEval_BuiltinsFromGlobals(
 
 
 static inline PyObject*
-_PyEval_EvalFrame(PyThreadState *tstate, struct _PyInterpreterFrame *frame, int throwflag)
+_PyEval_EvalFrame(PyThreadState *tstate, struct _Py_framedata *fdata, int throwflag)
 {
     if (tstate->interp->eval_frame == NULL) {
-        return _PyEval_EvalFrameDefault(tstate, frame, throwflag);
+        return _PyEval_EvalFrameDefault(tstate, fdata, throwflag);
     }
-    return tstate->interp->eval_frame(tstate, frame, throwflag);
+    return tstate->interp->eval_frame(tstate, fdata, throwflag);
 }
 
 extern PyObject *
@@ -116,7 +116,7 @@ static inline void _Py_LeaveRecursiveCall_inline(void)  {
 
 #define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline()
 
-struct _PyInterpreterFrame *_PyEval_GetFrame(void);
+struct _Py_framedata *_PyEval_GetFrame(void);
 
 PyObject *_Py_MakeCoro(PyFunctionObject *func);
 

Include/internal/pycore_frame.h

@@ -6,17 +6,60 @@ extern "C" {
 
 #include <stdbool.h>
 
+/* Starting in CPython 3.11, CPython separates the frame state between the
+ * full frame objects exposed by the Python and C runtime state introspection
+ * APIs, and internal lighter weight frame data structs, which are simple C
+ * structures owned by either the interpreter eval loop (while executing
+ * ordinary functions), by a generator or coroutine object (for frames that
+ * are able to be suspended), or by their corresponding full frame object (if
+ * a state instrospection API has been invoked and the full frame object has
+ * taken responsibility for the lifecycle of the frame data storage).
+ *
+ * This split storage eliminates a lot of allocation and deallocation of full
+ * Python objects during code execution, providing a significant speed gain
+ * over the previous approach of using full Python objects for both
+ * introspection and code execution.
+ *
+ * Field naming conventions:
+ *
+ *   * full frame object fields have an "f_*" prefix
+ *   * frame data struct fields have no prefix
+ *
+ * Local variable and function argument naming conventions:
+ *
+ *   * "frame", "f", and "frameobj" are used for full frame objects
+ *     * Exception: "current_frame" in the thread state cframe struct is a frame data struct
+ *   * "fdata" is used for frame data structs
+ *
+ * Function/macro naming conventions:
+ *
+ *   * "PyFrame_*" functions accept a full frame object
+ *   * "_PyFrame_*" functions accept either a full frame object or a frame
+ *     data struct. Check the specific function signatures for details.
+ *   * Other public C API functions that relate to frames only accept full
+ *     frame objects
+ *   * Other private C API functions that relate to frames may accept either a
+ *     full frame object or a frame data struct. Check the specific function
+ *     signatures for details
+ *
+ * Function return types:
+ *   * Public C API functions will only ever return full frame objects
+ *   * Private C API functions with an underscore prefix may return frame
+ *     data structs instead
+ */
+
+
 struct _frame {
     PyObject_HEAD
     PyFrameObject *f_back;      /* previous frame, or NULL */
-    struct _PyInterpreterFrame *f_frame; /* points to the frame data */
+    struct _Py_framedata *f_fdata; /* points to the frame data */
     PyObject *f_trace;          /* Trace function */
     int f_lineno;               /* Current line number. Only valid if non-zero */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
     char f_owns_frame;          /* This frame owns the frame */
     /* The frame data, if this frame object owns the frame */
-    PyObject *_f_frame_data[1];
+    struct _Py_framedata *_f_owned_fdata[1];
 };
 
 extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code);
@@ -29,7 +72,8 @@ extern void _PyFrame_Fini(PyInterpreterState *interp);
 /* other API */
 
 /* These values are chosen so that the inline functions below all
- * compare f_state to zero.
+ * compare fdata->state to zero while keeping the property that most
+ * state transitions move to a higher value frame state.
  */
 enum _framestate {
     FRAME_CREATED = -2,
@@ -44,146 +88,146 @@ enum _framestate {
 typedef signed char PyFrameState;
 
 /*
-    frame->f_lasti refers to the index of the last instruction,
+    frame->lasti refers to the index of the last instruction,
     unless it's -1 in which case next_instr should be first_instr.
 */
 
-typedef struct _PyInterpreterFrame {
-    PyFunctionObject *f_func; /* Strong reference */
-    PyObject *f_globals; /* Borrowed reference */
-    PyObject *f_builtins; /* Borrowed reference */
-    PyObject *f_locals; /* Strong reference, may be NULL */
-    PyCodeObject *f_code; /* Strong reference */
+typedef struct _Py_framedata {
+    PyFunctionObject *func; /* Strong reference */
+    PyObject *globals; /* Borrowed reference */
+    PyObject *builtins; /* Borrowed reference */
+    PyObject *locals; /* Strong reference, may be NULL */
+    PyCodeObject *code; /* Strong reference */
     PyFrameObject *frame_obj; /* Strong reference, may be NULL */
-    struct _PyInterpreterFrame *previous;
-    int f_lasti;       /* Last instruction if called */
+    struct _Py_framedata *previous;
+    int lasti;       /* Last instruction if called */
     int stacktop;     /* Offset of TOS from localsplus  */
-    PyFrameState f_state;  /* What state the frame is in */
+    PyFrameState state;  /* What state the frame is in */
     bool is_entry;  // Whether this is the "root" frame for the current _PyCFrame.
     bool is_generator;
     PyObject *localsplus[1];
-} _PyInterpreterFrame;
+} _Py_framedata;
 
-static inline int _PyFrame_IsRunnable(_PyInterpreterFrame *f) {
-    return f->f_state < FRAME_EXECUTING;
+static inline int _PyFrame_IsRunnable(_Py_framedata *f) {
+    return f->state < FRAME_EXECUTING;
 }
 
-static inline int _PyFrame_IsExecuting(_PyInterpreterFrame *f) {
-    return f->f_state == FRAME_EXECUTING;
+static inline int _PyFrame_IsExecuting(_Py_framedata *f) {
+    return f->state == FRAME_EXECUTING;
 }
 
-static inline int _PyFrameHasCompleted(_PyInterpreterFrame *f) {
-    return f->f_state > FRAME_EXECUTING;
+static inline int _PyFrameHasCompleted(_Py_framedata *f) {
+    return f->state > FRAME_EXECUTING;
 }
 
-static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
-    return f->localsplus + f->f_code->co_nlocalsplus;
+static inline PyObject **_PyFrame_Stackbase(_Py_framedata *f) {
+    return f->localsplus + f->code->co_nlocalsplus;
 }
 
-static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) {
-    assert(f->stacktop > f->f_code->co_nlocalsplus);
+static inline PyObject *_PyFrame_StackPeek(_Py_framedata *f) {
+    assert(f->stacktop > f->code->co_nlocalsplus);
     assert(f->localsplus[f->stacktop-1] != NULL);
     return f->localsplus[f->stacktop-1];
 }
 
-static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) {
-    assert(f->stacktop > f->f_code->co_nlocalsplus);
+static inline PyObject *_PyFrame_StackPop(_Py_framedata *f) {
+    assert(f->stacktop > f->code->co_nlocalsplus);
     f->stacktop--;
     return f->localsplus[f->stacktop];
 }
 
-static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) {
-    f->localsplus[f->stacktop] = value;
-    f->stacktop++;
+static inline void _PyFrame_StackPush(_Py_framedata *fdata, PyObject *value) {
+    fdata->localsplus[fdata->stacktop] = value;
+    fdata->stacktop++;
 }
 
-#define FRAME_SPECIALS_SIZE ((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))
+#define FRAME_SPECIALS_SIZE ((sizeof(_Py_framedata)-1)/sizeof(PyObject *))
 
-void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest);
+void _PyFrame_Copy(_Py_framedata *src, _Py_framedata *dest);
 
 /* Consumes reference to func */
 static inline void
 _PyFrame_InitializeSpecials(
-    _PyInterpreterFrame *frame, PyFunctionObject *func,
+    _Py_framedata *fdata, PyFunctionObject *func,
     PyObject *locals, int nlocalsplus)
 {
-    frame->f_func = func;
-    frame->f_code = (PyCodeObject *)Py_NewRef(func->func_code);
-    frame->f_builtins = func->func_builtins;
-    frame->f_globals = func->func_globals;
-    frame->f_locals = Py_XNewRef(locals);
-    frame->stacktop = nlocalsplus;
-    frame->frame_obj = NULL;
-    frame->f_lasti = -1;
-    frame->f_state = FRAME_CREATED;
-    frame->is_entry = false;
-    frame->is_generator = false;
+    fdata->func = func;
+    fdata->code = (PyCodeObject *)Py_NewRef(func->func_code);
+    fdata->builtins = func->func_builtins;
+    fdata->globals = func->func_globals;
+    fdata->locals = Py_XNewRef(locals);
+    fdata->stacktop = nlocalsplus;
+    fdata->frame_obj = NULL;
+    fdata->lasti = -1;
+    fdata->state = FRAME_CREATED;
+    fdata->is_entry = false;
+    fdata->is_generator = false;
 }
 
 /* Gets the pointer to the locals array
  * that precedes this frame.
  */
 static inline PyObject**
-_PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
+_PyFrame_GetLocalsArray(_Py_framedata *fdata)
 {
-    return frame->localsplus;
+    return fdata->localsplus;
 }
 
 static inline PyObject**
-_PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
+_PyFrame_GetStackPointer(_Py_framedata *fdata)
 {
-    return frame->localsplus+frame->stacktop;
+    return fdata->localsplus+fdata->stacktop;
 }
 
 static inline void
-_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
+_PyFrame_SetStackPointer(_Py_framedata *fdata, PyObject **stack_pointer)
 {
-    frame->stacktop = (int)(stack_pointer - frame->localsplus);
+    fdata->stacktop = (int)(stack_pointer - fdata->localsplus);
 }
 
 /* For use by _PyFrame_GetFrameObject
   Do not call directly. */
 PyFrameObject *
-_PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame);
+_PyFrame_MakeAndSetFrameObject(_Py_framedata *fdata);
 
 /* Gets the PyFrameObject for this frame, lazily
  * creating it if necessary.
  * Returns a borrowed referennce */
 static inline PyFrameObject *
-_PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
+_PyFrame_GetFrameObject(_Py_framedata *fdata)
 {
-    PyFrameObject *res =  frame->frame_obj;
+    PyFrameObject *res = fdata->frame_obj;
     if (res != NULL) {
         return res;
     }
-    return _PyFrame_MakeAndSetFrameObject(frame);
+    return _PyFrame_MakeAndSetFrameObject(fdata);
 }
 
 /* Clears all references in the frame.
- * If take is non-zero, then the _PyInterpreterFrame frame
- * may be transferred to the frame object it references
+ * If take is non-zero, then the frame data
+ * may be transfered to the frame object it references
  * instead of being cleared. Either way
  * the caller no longer owns the references
  * in the frame.
- * take should  be set to 1 for heap allocated
+ * take should be set to 1 for heap allocated
  * frames like the ones in generators and coroutines.
  */
 void
-_PyFrame_Clear(_PyInterpreterFrame * frame);
+_PyFrame_Clear(_Py_framedata * fdata);
 
 int
-_PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg);
+_PyFrame_Traverse(_Py_framedata *fdata, visitproc visit, void *arg);
 
 int
-_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame);
+_PyFrame_FastToLocalsWithError(_Py_framedata *fdata);
 
 void
-_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear);
+_PyFrame_LocalsToFast(_Py_framedata *fdata, int clear);
 
-extern _PyInterpreterFrame *
+extern _Py_framedata *
 _PyThreadState_BumpFramePointerSlow(PyThreadState *tstate, size_t size);
 
-static inline _PyInterpreterFrame *
+static inline _Py_framedata *
 _PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size)
 {
     PyObject **base = tstate->datastack_top;
@@ -192,16 +236,16 @@ _PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size)
         assert(tstate->datastack_limit);
         if (top < tstate->datastack_limit) {
             tstate->datastack_top = top;
-            return (_PyInterpreterFrame *)base;
+            return (_Py_framedata *)base;
         }
     }
     return _PyThreadState_BumpFramePointerSlow(tstate, size);
 }
 
-void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame);
+void _PyThreadState_PopFrame(PyThreadState *tstate, _Py_framedata *fdata);
 
 /* Consume reference to func */
-_PyInterpreterFrame *
+_Py_framedata *
 _PyFrame_Push(PyThreadState *tstate, PyFunctionObject *func);
 
 #ifdef __cplusplus

Misc/NEWS.d/next/Core and Builtins/2021-08-01-18-18-51.bpo-44800.TCsfH3.rst

@@ -0,0 +1,8 @@
+Refactored internal APIs for the new lazy frame object creation to more
+consistently distinguish between the full ``PyFrameObject`` Python object
+implementation that is still used in the Python and C runtime state
+introspection APIs (function prefix ``PyFrame``, field prefix ``f_``, typical
+variable names ``frame`` and ``f``) and the new ``_Py_framedata`` internal
+frame data storage (C structs with no intrinsic instance lifecycle management)
+that is now used for code execution (function prefix ``_Py_framedata``, no
+field prefix, typical variable name ``fdata``).