gh-91744: Add semi-stable C API tier

Doc/c-api/code.rst

@@ -42,9 +42,14 @@ bound into a function.
    ways, meaning that subtle changes to values are likely to result in incorrect
    execution or VM crashes. Use this function only with extreme care.
 
+   This function is part of the semi-stable C API.
+   See :c:macro:`Py_USING_SEMI_STABLE_API` for usage.
+
    .. versionchanged:: 3.11
       Added ``exceptiontable`` parameter.
 
+      Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.
+
 .. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 
    Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments.
@@ -55,6 +60,8 @@ bound into a function.
    .. versionchanged:: 3.11
       Added ``exceptiontable`` parameter.
 
+      Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.
+
 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 
    Return a new empty code object with the specified filename,
@@ -90,3 +97,79 @@ bound into a function.
 
    .. versionadded:: 3.11
 
+
+Extra information
+-----------------
+
+To support low-level extensions to frame evaluation, such as external
+just-in-time compilers, it is possible to attach arbitrary extra data to
+code objects.
+
+This functionality is a CPython implementation detail, and the API
+may change without deprecation warnings.
+These functions are part of the semi-stable C API.
+See :c:macro:`Py_USING_SEMI_STABLE_API` for details.
+
+See :pep:`523` for motivation and initial specification behind this API.
+
+
+.. c:function:: Py_ssize_t PyEval_RequestCodeExtraIndex(freefunc free)
+
+   Return a new an opaque index value used to adding data to code objects.
+
+   You generally call this function once (per interpreter) and use the result
+   with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate
+   data on individual code objects.
+
+   If *free* is not ``NULL``: when a code object is deallocated,
+   *free* will be called on non-``NULL`` data stored under the new index.
+   Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`.
+
+   Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API`
+   for usage.
+
+   .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex``
+
+   .. versionchanged:: 3.11
+
+     Renamed to ``PyEval_RequestCodeExtraIndex`` (without the leading
+     undersecore). The old name is available as an alias.
+
+     Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.
+
+.. c:function:: int PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+
+   Set *extra* to the extra data stored under the given index.
+   Return 0 on success. Set an exception and return -1 on failure.
+
+   If no data was set under the index, set *extra* to ``NULL`` and return
+   0 without setting an exception.
+
+   Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API`
+   for usage.
+
+   .. versionadded:: 3.6 as ``_PyCode_GetExtra``
+
+   .. versionchanged:: 3.11
+
+     Renamed to ``PyCode_GetExtra`` (without the leading undersecore).
+     The old name is available as an alias.
+
+     Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.
+
+.. c:function:: int PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+
+   Set the extra data stored under the given index to *extra*.
+   Return 0 on success. Set an exception and return -1 on failure.
+
+   Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API`
+   for usage.
+
+   .. versionadded:: 3.6 3.6 as ``_PyCode_SetExtra``
+
+   .. versionchanged:: 3.11
+
+     Renamed to ``PyCode_SetExtra`` (without the leading undersecore).
+     The old name is available as an alias.
+
+     Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.

Doc/c-api/init.rst

@@ -1235,27 +1235,57 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
    The *throwflag* parameter is used by the ``throw()`` method of generators:
    if non-zero, handle the current exception.
 
+   Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API`
+   for usage.
+
+   .. versionadded:: 3.6 as ``_PyFrameEvalFunction``
+
    .. versionchanged:: 3.9
       The function now takes a *tstate* parameter.
 
    .. versionchanged:: 3.11
       The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``.
 
-.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp)
+      Renamed to ``PyFrameEvalFunction`` (without the leading undersecore).
+      The old name is available as an alias.
+
+      Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.
+
+.. c:function:: PyFrameEvalFunction PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp)
 
    Get the frame evaluation function.
 
    See the :pep:`523` "Adding a frame evaluation API to CPython".
 
-   .. versionadded:: 3.9
+   Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API`
+   for usage.
+
+   .. versionadded:: 3.9 as ``_PyInterpreterState_GetEvalFrameFunc``
+
+   .. versionchanged:: 3.11
 
-.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame)
+     Renamed to ``PyInterpreterState_GetEvalFrameFunc`` (without the leading undersecore).
+     The old name is available as an alias.
+
+     Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.
+
+.. c:function:: void PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, PyFrameEvalFunction eval_frame)
 
    Set the frame evaluation function.
 
    See the :pep:`523` "Adding a frame evaluation API to CPython".
 
-   .. versionadded:: 3.9
+   Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API`
+   for usage.
+
+   .. versionadded:: 3.9 as ``_PyInterpreterState_SetEvalFrameFunc``
+
+   .. versionchanged:: 3.11
+
+     Renamed to ``PyInterpreterState_SetEvalFrameFunc`` (without the leading undersecore).
+     The old name is available as an alias.
+
+     Use without ``Py_USING_SEMI_STABLE_API`` is deprecated.
 
 
 .. c:function:: PyObject* PyThreadState_GetDict()

Doc/c-api/stable.rst

@@ -6,9 +6,9 @@
 C API Stability
 ***************
 
-Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`.
-While the C API will change with every minor release (e.g. from 3.9 to 3.10),
-most changes will be source-compatible, typically by only adding new API.
+Unless documented otherwise, Python's C API is covered by the Backwards
+Compatibility Policy, :pep:`387`.
+Most changes to it are source-compatible (typically by only adding new API).
 Changing existing API or removing API is only done after a deprecation period
 or to fix serious issues.
 
@@ -18,8 +18,35 @@ way; see :ref:`stable-abi-platform` below).
 So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa,
 but will need to be compiled separately for 3.9.x and 3.10.x.
 
+There are two tiers of C API with different stability expectations,
+enabled by specific macros:
+
+- :c:macro:`Py_USING_SEMI_STABLE_API` exposes API that may change
+  without deprecation warnings.
+- :c:macro:`Py_LIMITED_API` limits exposed API to API that is compatible
+  across several minor releases.
+
+These are discussed in more detail below.
+
 Names prefixed by an underscore, such as ``_Py_InternalState``,
 are private API that can change without notice even in patch releases.
+If you need to use this API, consider reaching out to
+`CPython developers <mailto:[email protected]>`_ to discusss adding
+external API for your use case.
+
+
+Semi-stable C API
+=================
+
+.. c:macro:: Py_USING_SEMI_STABLE_API
+
+   Define this macro to access semi-stable API, intended for low-level
+   tools like debuggers.
+   This API exposes CPython implementation details, and may change
+   in every minor release (e.g. from 3.9 to 3.10) without
+   any deprecation warnings.
+   Projects that define ``Py_USING_SEMI_STABLE_API`` are expected to follow
+   CPython development and spend extra effort adjusting to changes.
 
 
 Stable Application Binary Interface

Doc/whatsnew/3.11.rst

@@ -1405,7 +1405,30 @@ C API Changes
   be used for ``size``.
   (Contributed by Kumar Aditya in :issue:`46608`.)
 
-* :c:func:`_PyFrameEvalFunction` now takes ``_PyInterpreterFrame*``
+* Introduced the *semi-stable* tier of C API, intended for low-level tools like
+  debuggers and JIT compilers.
+  This API may change in each minor release of CPython without deprecation
+  warnings, and is exposed by defining :c:macro:`Py_USING_SEMI_STABLE_API`.
+  The folllowing functions and types are part of the semi-stable API.
+  Using them without defining ``Py_USING_SEMI_STABLE_API`` is deprecated and
+  will be disallowed in Python 3.12:
+
+  - :c:func:`PyEval_RequestCodeExtraIndex`
+  - :c:func:`PyCode_GetExtra`
+  - :c:func:`PyCode_SetExtra`
+  - :c:func:`PyCode_New`
+  - :c:func:`PyCode_NewWithPosOnlyArgs`,
+  - :c:func:`PyInterpreterState_GetEvalFrameFunc`
+  - :c:func:`PyInterpreterState_SetEvalFrameFunc`
+  - :c:type:`PyFrameEvalFunction`
+
+  Some of these were named with a leading underscore in previous versions;
+  the old names are available as aliases.
+
+  (Contributed by Petr Viktorin, Victor Stinner and Nick Coghlan in
+  :gh:`91744`.)
+
+* :c:func:`PyFrameEvalFunction` now takes ``_PyInterpreterFrame*``
   as its second parameter, instead of ``PyFrameObject*``.
   See :pep:`523` for more details of how to use this function pointer type.
 

Include/README.rst

@@ -1,11 +1,12 @@
 The Python C API
 ================
 
-The C API is divided into three sections:
+The C API is divided into these sections:
 
 1. ``Include/``: Limited API
 2. ``Include/cpython/``: CPython implementation details
-3. ``Include/internal/``: The internal API
+3. ``Include/unstable/``: API that can change between minor releases
+4. ``Include/internal/``: The internal API
 
 Information on changing the C API is available `in the developer guide`_
 

Include/cpython/ceval.h

@@ -20,7 +20,5 @@ PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _P
 PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);
 
-PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc);
-
 PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *);
 PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *);

Include/cpython/code.h

@@ -140,19 +140,7 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
 #define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
 #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
 
-/* Public interface */
-PyAPI_FUNC(PyCodeObject *) PyCode_New(
-        int, int, int, int, int, PyObject *, PyObject *,
-        PyObject *, PyObject *, PyObject *, PyObject *,
-        PyObject *, PyObject *, PyObject *, int, PyObject *,
-        PyObject *);
-
-PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs(
-        int, int, int, int, int, int, PyObject *, PyObject *,
-        PyObject *, PyObject *, PyObject *, PyObject *,
-        PyObject *, PyObject *, PyObject *, int, PyObject *,
-        PyObject *);
-        /* same as struct above */
+/* See Include/semi-stable/code.h for PyCode_New* */
 
 /* Creates a new empty code object with the specified source location. */
 PyAPI_FUNC(PyCodeObject *)
@@ -196,12 +184,6 @@ PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj);
 PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts,
                                       PyObject *names, PyObject *lnotab);
 
-
-PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index,
-                                 void **extra);
-PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
-                                 void *extra);
-
 /* Equivalent to getattr(code, 'co_code') in Python.
    Returns a strong reference to a bytes object. */
 PyAPI_FUNC(PyObject *) PyCode_GetCode(PyCodeObject *code);
@@ -219,6 +201,11 @@ typedef enum _PyCodeLocationInfoKind {
     PY_CODE_LOCATION_INFO_NONE = 15
 } _PyCodeLocationInfoKind;
 
+#define Py_UNSTABLE_CODE_H
+#include "unstable/code.h"
+#undef Py_UNSTABLE_CODE_H
+
+
 #ifdef __cplusplus
 }
 #endif

Include/cpython/pystate.h

@@ -259,16 +259,6 @@ PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *);
 PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *);
 PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
-/* Frame evaluation API */
-
-typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int);
-
-PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
-    PyInterpreterState *interp);
-PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc(
-    PyInterpreterState *interp,
-    _PyFrameEvalFunction eval_frame);
-
 PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp);
 
 /* Get a copy of the current interpreter configuration.

Include/pyport.h

@@ -61,6 +61,10 @@
 #  define Py_BUILD_CORE
 #endif
 
+// Expose unstable API when building Python
+#ifdef Py_BUILD_CORE
+#define Py_USING_UNSTABLE_API
+#endif
 
 /**************************************************************************
 Symbols and macros to supply platform-independent interfaces to basic
@@ -371,6 +375,15 @@ extern "C" {
 #define _Py_COMP_DIAG_POP
 #endif
 
+/* _Py_NEWLY_UNSTABLE:  Provide deprecation warnings for users that
+ * don't opt in to unstable API.
+ */
+#ifdef Py_USING_UNSTABLE_API
+#define _Py_NEWLY_UNSTABLE(VERSION_UNUSED)
+#else
+#define _Py_NEWLY_UNSTABLE(VERSION) Py_DEPRECATED(VERSION)
+#endif
+
 /* _Py_HOT_FUNCTION
  * The hot attribute on a function is used to inform the compiler that the
  * function is a hot spot of the compiled program. The function is optimized

Include/pystate.h

@@ -121,6 +121,10 @@ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void);
 
 
 #ifndef Py_LIMITED_API
+#  define Py_UNSTABLE_PYSTATE_H
+#  include "unstable/pystate.h"
+#  undef Py_UNSTABLE_PYSTATE_H
+
 #  define Py_CPYTHON_PYSTATE_H
 #  include "cpython/pystate.h"
 #  undef Py_CPYTHON_PYSTATE_H