From 0185c4271e6b0bdf47b1fced232f37140e12ebd4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 20 Apr 2022 17:24:26 +0200 Subject: [PATCH 01/11] Add the semi-stable C-API tier, with PEP-523 functions & PyCode_New* --- Include/README.rst | 5 +++-- Include/cpython/ceval.h | 2 -- Include/cpython/code.h | 21 ++++----------------- Include/pyport.h | 13 +++++++++++++ Include/semistable/code.h | 39 +++++++++++++++++++++++++++++++++++++++ Lib/test/test_code.py | 6 +++--- Objects/codeobject.c | 4 ++-- Python/ceval.c | 2 +- 8 files changed, 65 insertions(+), 27 deletions(-) create mode 100644 Include/semistable/code.h diff --git a/Include/README.rst b/Include/README.rst index f52e690eac9a91..f6586743a5db75 100644 --- a/Include/README.rst +++ b/Include/README.rst @@ -1,11 +1,12 @@ The Python C API ================ -The C API is divided into three sections: +The C API is divided into these sections: 1. ``Include/``: Limited API 2. ``Include/cpython/``: CPython implementation details -3. ``Include/internal/``: The internal API +3. ``Include/semi-stable/``: API that can change between minor releases +4. ``Include/internal/``: The internal API Information on changing the C API is available `in the developer guide`_ diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 9d4eeafb427eb2..e4e9873555d72a 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -20,7 +20,5 @@ PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _P PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds); PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void); -PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc); - PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 6dc2290ffeb5e2..80640aad07913c 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -148,19 +148,7 @@ PyAPI_DATA(PyTypeObject) PyCode_Type; #define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive) #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) -/* Public interface */ -PyAPI_FUNC(PyCodeObject *) PyCode_New( - int, int, int, int, int, PyObject *, PyObject *, - PyObject *, PyObject *, PyObject *, PyObject *, - PyObject *, PyObject *, PyObject *, int, PyObject *, - PyObject *, PyObject *, PyObject *); - -PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs( - int, int, int, int, int, int, PyObject *, PyObject *, - PyObject *, PyObject *, PyObject *, PyObject *, - PyObject *, PyObject *, PyObject *, int, PyObject *, - PyObject *, PyObject *, PyObject *); - /* same as struct above */ +/* See Include/semi-stable/code.h for PyCode_New* */ /* Creates a new empty code object with the specified source location. */ PyAPI_FUNC(PyCodeObject *) @@ -205,10 +193,9 @@ PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lnotab); -PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index, - void **extra); -PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index, - void *extra); +#define Py_SEMISTABLE_CODE_H +#include "semistable/code.h" +#undef Py_SEMISTABLE_CODE_H #ifdef __cplusplus } diff --git a/Include/pyport.h b/Include/pyport.h index 855c382a61ee53..f81b02e2f76766 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -33,6 +33,10 @@ # define Py_BUILD_CORE #endif +// Expose semi-stable API when building Python +#ifdef Py_BUILD_CORE +#define Py_USING_SEMI_STABLE_API +#endif /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic @@ -341,6 +345,15 @@ extern "C" { #define _Py_COMP_DIAG_POP #endif +/* _Py_NEWLY_SEMISTABLE: Provide deprecation warnings for users that + * don't opt in to semi-stable API. + */ +#ifdef Py_USING_SEMI_STABLE_API +#define _Py_NEWLY_SEMISTABLE(VERSION_UNUSED) +#else +#define _Py_NEWLY_SEMISTABLE(VERSION) Py_DEPRECATED(VERSION) +#endif + /* _Py_HOT_FUNCTION * The hot attribute on a function is used to inform the compiler that the * function is a hot spot of the compiled program. The function is optimized diff --git a/Include/semistable/code.h b/Include/semistable/code.h new file mode 100644 index 00000000000000..41725d964ff110 --- /dev/null +++ b/Include/semistable/code.h @@ -0,0 +1,39 @@ +#ifndef Py_SEMISTABLE_CODE_H +# error "this header file must not be included directly" +#endif + +/* Functions added in PEP-523 */ + +_Py_NEWLY_SEMISTABLE(3.11) +PyAPI_FUNC(Py_ssize_t) PyEval_RequestCodeExtraIndex(freefunc); + +_Py_NEWLY_SEMISTABLE(3.11) +PyAPI_FUNC(int) PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra); + +_Py_NEWLY_SEMISTABLE(3.11) +PyAPI_FUNC(int) PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra); + +/* Underscore-prefixed names for the above. + * To be removed when the API changes. + */ +#define _PyEval_RequestCodeExtraIndex PyEval_RequestCodeExtraIndex +#define _PyCode_GetExtra PyCode_GetExtra +#define _PyCode_SetExtra PyCode_SetExtra + + +/* PyCodeObject constructors */ + +_Py_NEWLY_SEMISTABLE(3.11) +PyAPI_FUNC(PyCodeObject *) PyCode_New( + int, int, int, int, int, PyObject *, PyObject *, + PyObject *, PyObject *, PyObject *, PyObject *, + PyObject *, PyObject *, PyObject *, int, PyObject *, + PyObject *, PyObject *, PyObject *); + +_Py_NEWLY_SEMISTABLE(3.11) +PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs( + int, int, int, int, int, int, PyObject *, PyObject *, + PyObject *, PyObject *, PyObject *, PyObject *, + PyObject *, PyObject *, PyObject *, int, PyObject *, + PyObject *, PyObject *, PyObject *); + /* same as struct PyCodeObject */ diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 872f7283fc504e..009616f1aabe59 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -532,15 +532,15 @@ def callback(code): py = ctypes.pythonapi freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) - RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex = py.PyEval_RequestCodeExtraIndex RequestCodeExtraIndex.argtypes = (freefunc,) RequestCodeExtraIndex.restype = ctypes.c_ssize_t - SetExtra = py._PyCode_SetExtra + SetExtra = py.PyCode_SetExtra SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) SetExtra.restype = ctypes.c_int - GetExtra = py._PyCode_GetExtra + GetExtra = py.PyCode_GetExtra GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.POINTER(ctypes.c_voidp)) GetExtra.restype = ctypes.c_int diff --git a/Objects/codeobject.c b/Objects/codeobject.c index e872b398e08c8d..d935ab4c60be01 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1070,7 +1070,7 @@ typedef struct { int -_PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) +PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) { if (!PyCode_Check(code)) { PyErr_BadInternalCall(); @@ -1091,7 +1091,7 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) int -_PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) +PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) { PyInterpreterState *interp = _PyInterpreterState_GET(); diff --git a/Python/ceval.c b/Python/ceval.c index 12b0aef774e362..f3a248d442db9d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -7802,7 +7802,7 @@ _Py_GetDXProfile(PyObject *self, PyObject *args) #endif Py_ssize_t -_PyEval_RequestCodeExtraIndex(freefunc free) +PyEval_RequestCodeExtraIndex(freefunc free) { PyInterpreterState *interp = _PyInterpreterState_GET(); Py_ssize_t new_index; From 381fc111e07351e585de58e1d072baba48f20356 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 20 Apr 2022 18:49:59 +0200 Subject: [PATCH 02/11] Document Py_USING_SEMI_STABLE_API & functions that need it --- Doc/c-api/code.rst | 94 ++++++++++++++++++++++++++++++++++++++++++-- Doc/c-api/stable.rst | 17 +++++++- 2 files changed, 106 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 840b8426dbbdc3..4f35254839cb1d 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -36,9 +36,16 @@ bound into a function. .. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) Return a new code object. If you need a dummy code object to create a frame, - use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly - can bind you to a precise Python version since the definition of the bytecode - changes often. + use :c:func:`PyCode_NewEmpty` instead. + + Since the definition of the bytecode changes often, calling + :c:func:`PyCode_New` directly can bind you to a precise Python version. + This function is part of the semi-stable C API. + See :c:macro:`Py_USING_SEMI_STABLE_API` for usage. + + .. versionchanged:: 3.11 + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. .. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) @@ -46,6 +53,10 @@ bound into a function. .. versionadded:: 3.8 + .. versionchanged:: 3.11 + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. + .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) Return a new empty code object with the specified filename, @@ -67,3 +78,80 @@ bound into a function. information is not available for any particular element. Returns ``1`` if the function succeeds and 0 otherwise. + + +Extra information +----------------- + +To support low-level extensions to frame evaluation, such as external +just-in-time compilers, it is possible to attach arbitrary extra data to +code objects. + +This functionality is a CPython implementation detail, and the API +may change without deprecation warnings. +These functions are part of the semi-stable C API. +See :c:macro:`Py_USING_SEMI_STABLE_API` for details. + +See :pep:`523` for motivation and initial specification behind this API. + + +.. c:function:: Py_ssize_t PyEval_RequestCodeExtraIndex(freefunc free) + + Return a new an opaque index value used to adding data to code objects. + + You generally call this function once (per interpreter) and use the result + with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate + data on individual code objects. + + If *free* is not ``NULL``: when a code object is deallocated, + *free* will be called on non-``NULL`` data stored under the new index. + Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. + + Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API` + for usage. + + .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` + + .. versionchanged:: 3.11 + + Renamed to ``PyEval_RequestCodeExtraIndex`` (without the leading + undersecore). The old name is available as an alias. + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. + +.. c:function:: int PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) + + Set *extra* to the extra data stored under the given index. + Return 0 on success. Set an exception and return -1 on failure. + + If no data was set under the index, set *extra* to ``NULL`` and return + 0 without setting an exception. + + Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API` + for usage. + + .. versionadded:: 3.6 as ``_PyCode_GetExtra`` + + .. versionchanged:: 3.11 + + Renamed to ``PyCode_GetExtra`` (without the leading undersecore). + The old name is available as an alias. + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. + +.. c:function:: int PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) + + Set the extra data stored under the given index to *extra*. + Return 0 on success. Set an exception and return -1 on failure. + + Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API` + for usage. + + .. versionadded:: 3.6 3.6 as ``_PyCode_SetExtra`` + + .. versionchanged:: 3.11 + + Renamed to ``PyCode_SetExtra`` (without the leading undersecore). + The old name is available as an alias. + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 4ae20e93e36785..158888b3d90cb8 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -7,8 +7,7 @@ C API Stability *************** Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. -While the C API will change with every minor release (e.g. from 3.9 to 3.10), -most changes will be source-compatible, typically by only adding new API. +Most changes to it are source-compatible (typically by only adding new API). Changing existing API or removing API is only done after a deprecation period or to fix serious issues. @@ -22,6 +21,20 @@ Names prefixed by an underscore, such as ``_Py_InternalState``, are private API that can change without notice even in patch releases. +Semi-stable C API +================= + +.. c:macro:: Py_USING_SEMI_STABLE_API + + Define this macro to access semi-stable API, intended for low-level + tools like debuggers. + This API exposes CPython implementation details, and may change + in every minor release (e.g. from 3.9 to 3.10) without + any deprecation warnings. + Projects that define ``Py_USING_SEMI_STABLE_API`` are expected to follow + CPython development and spend extra effort adjusting to changes. + + Stable Application Binary Interface =================================== From 23e8a1dad2d73ac092a859401fd90c8462da36b7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 21 Apr 2022 15:29:57 +0200 Subject: [PATCH 03/11] Add PyFrameEvalFunction & related functions to semi-stable API --- Doc/c-api/init.rst | 42 +++++++++++++++++++++++++++++++----- Include/cpython/pystate.h | 10 --------- Include/pystate.h | 4 ++++ Include/semistable/pystate.h | 24 +++++++++++++++++++++ Python/pystate.c | 8 +++---- 5 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 Include/semistable/pystate.h diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 3fda9c3af4d2a5..61c32208434f74 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1228,31 +1228,63 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.8 -.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, PyFrameObject *frame, int throwflag) +.. c:type:: PyObject* (*PyFrameEvalFunction)(PyThreadState *tstate, PyFrameObject *frame, int throwflag) Type of a frame evaluation function. The *throwflag* parameter is used by the ``throw()`` method of generators: if non-zero, handle the current exception. + Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API` + for usage. + + .. versionadded:: 3.6 as ``_PyFrameEvalFunction`` + .. versionchanged:: 3.9 The function now takes a *tstate* parameter. -.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) + .. versionchanged:: 3.11 + + Renamed to ``PyFrameEvalFunction`` (without the leading undersecore). + The old name is available as an alias. + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. + +.. c:function:: PyFrameEvalFunction PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) Get the frame evaluation function. See the :pep:`523` "Adding a frame evaluation API to CPython". - .. versionadded:: 3.9 + Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API` + for usage. + + .. versionadded:: 3.9 as ``_PyInterpreterState_GetEvalFrameFunc`` + + .. versionchanged:: 3.11 -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) + Renamed to ``PyInterpreterState_GetEvalFrameFunc`` (without the leading undersecore). + The old name is available as an alias. + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. + +.. c:function:: void PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, PyFrameEvalFunction eval_frame) Set the frame evaluation function. See the :pep:`523` "Adding a frame evaluation API to CPython". - .. versionadded:: 3.9 + Part of the semi-stable API, see :c:macro:`Py_USING_SEMI_STABLE_API` + for usage. + + .. versionadded:: 3.9 as ``_PyInterpreterState_SetEvalFrameFunc`` + + .. versionchanged:: 3.11 + + Renamed to ``PyInterpreterState_SetEvalFrameFunc`` (without the leading undersecore). + The old name is available as an alias. + + Use without ``Py_USING_SEMI_STABLE_API`` is deprecated. .. c:function:: PyObject* PyThreadState_GetDict() diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 1af21a2c947d99..e346d744b6f12d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -259,16 +259,6 @@ PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *); PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *); PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void); -/* Frame evaluation API */ - -typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int); - -PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( - PyInterpreterState *interp); -PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( - PyInterpreterState *interp, - _PyFrameEvalFunction eval_frame); - PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp); /* Get a copy of the current interpreter configuration. diff --git a/Include/pystate.h b/Include/pystate.h index e6b4de979c87b8..0159a7952e31fe 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,6 +120,10 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +#define Py_SEMISTABLE_PYSTATE_H +#include "semistable/pystate.h" +#undef Py_SEMISTABLE_PYSTATE_H + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Include/semistable/pystate.h b/Include/semistable/pystate.h new file mode 100644 index 00000000000000..03bbc5d8d2ed4a --- /dev/null +++ b/Include/semistable/pystate.h @@ -0,0 +1,24 @@ +#ifndef Py_SEMISTABLE_PYSTATE_H +# error "this header file must not be included directly" +#endif + +/* Frame evaluation API (added for PEP-523) */ + +struct _PyInterpreterFrame; +typedef PyObject* (*PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int); + +_Py_NEWLY_SEMISTABLE(3.11) +PyAPI_FUNC(PyFrameEvalFunction) PyInterpreterState_GetEvalFrameFunc( + PyInterpreterState *interp); + +_Py_NEWLY_SEMISTABLE(3.11) +PyAPI_FUNC(void) PyInterpreterState_SetEvalFrameFunc( + PyInterpreterState *interp, + PyFrameEvalFunction eval_frame); + +/* Underscore-prefixed names for the above. + * To be removed when the API changes. + */ +#define _PyFrameEvalFunction PyFrameEvalFunction +#define _PyInterpreterState_GetEvalFrameFunc PyInterpreterState_GetEvalFrameFunc +#define _PyInterpreterState_SetEvalFrameFunc PyInterpreterState_SetEvalFrameFunc diff --git a/Python/pystate.c b/Python/pystate.c index 3e28a6ab69a989..73a87d33e2ec82 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2104,8 +2104,8 @@ _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) } -_PyFrameEvalFunction -_PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) +PyFrameEvalFunction +PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) { if (interp->eval_frame == NULL) { return _PyEval_EvalFrameDefault; @@ -2115,8 +2115,8 @@ _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) void -_PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, - _PyFrameEvalFunction eval_frame) +PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, + PyFrameEvalFunction eval_frame) { if (eval_frame == _PyEval_EvalFrameDefault) { interp->eval_frame = NULL; From 02c5466efd05c922e2c71395450711c65a7e374f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 21 Apr 2022 15:30:57 +0200 Subject: [PATCH 04/11] Docs: Expand the intro for C API stability --- Doc/c-api/stable.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 158888b3d90cb8..0f72aef00a9403 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -6,7 +6,8 @@ C API Stability *************** -Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. +Unless documented otherwise, Python's C API is covered by the Backwards +Compatibility Policy, :pep:`387`. Most changes to it are source-compatible (typically by only adding new API). Changing existing API or removing API is only done after a deprecation period or to fix serious issues. @@ -17,8 +18,21 @@ way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, but will need to be compiled separately for 3.9.x and 3.10.x. +There are two tiers of API with different stability exepectations, +enabled by specific macros: + +- :c:macro:`Py_USING_SEMI_STABLE_API` exposes API that may change + without deprecation warnings. +- :c:macro:`Py_LIMITED_API` exposes API that is compatible across + several minor releases. + +These are discussed in more detail below. + Names prefixed by an underscore, such as ``_Py_InternalState``, are private API that can change without notice even in patch releases. +If you need to use this API, consider reaching out to +`CPython developers `_ to discusss adding +external API for your use case. Semi-stable C API From e8e188431fed82a16b2eb9d0d407fdcd79d8d083 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 21 Apr 2022 17:26:46 +0200 Subject: [PATCH 05/11] Add blurb --- .../2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst diff --git a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst b/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst new file mode 100644 index 00000000000000..5b1f6190241f7b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst @@ -0,0 +1,10 @@ +Defining :c:macro:`Py_USING_SEMI_STABLE_API` will expose "semi-stable API", +which may change in each minor release, without deprecation warnings. The +functions :c:func:`PyEval_RequestCodeExtraIndex`, :c:func:`PyCode_GetExtra`, +:c:func:`PyCode_SetExtra`, :c:func:`PyCode_New`, +:c:func:`PyCode_NewWithPosOnlyArgs`, +:c:func:`PyInterpreterState_GetEvalFrameFunc`, +:c:func:`PyInterpreterState_SetEvalFrameFunc` and the type +:c:type:`PyFrameEvalFunction` were added to semi-stable API. Some of these +were named with a leading underscore; the old names are available as +aliases. From a44169933c1f9839980ebd075ff8472a205503e7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 21 Apr 2022 17:41:11 +0200 Subject: [PATCH 06/11] Add What's New entry --- Doc/whatsnew/3.11.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6540a255a0ed82..b69599840f1821 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1155,6 +1155,30 @@ C API Changes be used for ``size``. (Contributed by Kumar Aditya in :issue:`46608`.) +* Introduced the *semi-stable* tier of C API, intended for low-level tools like + debuggers and JIT compilers. + This API may change in each minor release of CPython without deprecation + warnings, and is exposed by defining :c:macro:`Py_USING_SEMI_STABLE_API`. + The folllowing functions and types are part of the semi-stable API. + Using them without defining ``Py_USING_SEMI_STABLE_API`` is deprecated and + will be disallowed in Python 3.12: + + - :c:func:`PyEval_RequestCodeExtraIndex` + - :c:func:`PyCode_GetExtra` + - :c:func:`PyCode_SetExtra` + - :c:func:`PyCode_New` + - :c:func:`PyCode_NewWithPosOnlyArgs`, + - :c:func:`PyInterpreterState_GetEvalFrameFunc` + - :c:func:`PyInterpreterState_SetEvalFrameFunc` + - :c:type:`PyFrameEvalFunction` + + Some of these named with a leading underscore in previous versions; the old + names are available as aliases. + + (Contributed by Petr Viktorin, Victor Stinner and Nick Coghlan in + :issue:`91744`.) + + New Features ------------ From 132fd8e42a42e556a1ac62cdc3b091383482094d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 21 Apr 2022 18:06:08 +0200 Subject: [PATCH 07/11] And one more fix for the conflict --- Include/semistable/code.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/semistable/code.h b/Include/semistable/code.h index 41725d964ff110..14ea8cbe45c911 100644 --- a/Include/semistable/code.h +++ b/Include/semistable/code.h @@ -28,12 +28,12 @@ PyAPI_FUNC(PyCodeObject *) PyCode_New( int, int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *, - PyObject *, PyObject *, PyObject *); + PyObject *); _Py_NEWLY_SEMISTABLE(3.11) PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs( int, int, int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *, - PyObject *, PyObject *, PyObject *); + PyObject *); /* same as struct PyCodeObject */ From 7dba38c16358defb7efbe8d5cbcfe6a3f636efdf Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 21 Apr 2022 18:34:47 +0200 Subject: [PATCH 08/11] Exclude semistable/ from the limited API d'oh! --- Include/pystate.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 0159a7952e31fe..79f393aa679dbe 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,11 +120,11 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); -#define Py_SEMISTABLE_PYSTATE_H -#include "semistable/pystate.h" -#undef Py_SEMISTABLE_PYSTATE_H - #ifndef Py_LIMITED_API +# define Py_SEMISTABLE_PYSTATE_H +# include "semistable/pystate.h" +# undef Py_SEMISTABLE_PYSTATE_H + # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" # undef Py_CPYTHON_PYSTATE_H From 74419bd2e1bfcc90b144bcd90dd8b78c70f69388 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 28 Apr 2022 16:04:10 +0200 Subject: [PATCH 09/11] Address some review suggestions --- Doc/c-api/code.rst | 2 +- Doc/c-api/stable.rst | 8 ++++---- Doc/whatsnew/3.11.rst | 4 ++-- Include/README.rst | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 4f35254839cb1d..818c1cd409e2c6 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -40,7 +40,7 @@ bound into a function. Since the definition of the bytecode changes often, calling :c:func:`PyCode_New` directly can bind you to a precise Python version. - This function is part of the semi-stable C API. + This function is part of the semi-stable C API. See :c:macro:`Py_USING_SEMI_STABLE_API` for usage. .. versionchanged:: 3.11 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 0f72aef00a9403..a62038cf54d15e 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -6,7 +6,7 @@ C API Stability *************** -Unless documented otherwise, Python's C API is covered by the Backwards +Unless documented otherwise, Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. Most changes to it are source-compatible (typically by only adding new API). Changing existing API or removing API is only done after a deprecation period @@ -18,13 +18,13 @@ way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, but will need to be compiled separately for 3.9.x and 3.10.x. -There are two tiers of API with different stability exepectations, +There are two tiers of C API with different stability expectations, enabled by specific macros: - :c:macro:`Py_USING_SEMI_STABLE_API` exposes API that may change without deprecation warnings. -- :c:macro:`Py_LIMITED_API` exposes API that is compatible across - several minor releases. +- :c:macro:`Py_LIMITED_API` limits exposed API to API that is compatible + across several minor releases. These are discussed in more detail below. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index b69599840f1821..b19e1b275ab7ef 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1172,8 +1172,8 @@ C API Changes - :c:func:`PyInterpreterState_SetEvalFrameFunc` - :c:type:`PyFrameEvalFunction` - Some of these named with a leading underscore in previous versions; the old - names are available as aliases. + Some of these were named with a leading underscore in previous versions; + the old names are available as aliases. (Contributed by Petr Viktorin, Victor Stinner and Nick Coghlan in :issue:`91744`.) diff --git a/Include/README.rst b/Include/README.rst index f6586743a5db75..faa6f3132a9315 100644 --- a/Include/README.rst +++ b/Include/README.rst @@ -5,7 +5,7 @@ The C API is divided into these sections: 1. ``Include/``: Limited API 2. ``Include/cpython/``: CPython implementation details -3. ``Include/semi-stable/``: API that can change between minor releases +3. ``Include/semistable/``: API that can change between minor releases 4. ``Include/internal/``: The internal API Information on changing the C API is available `in the developer guide`_ From 75abb20fede3e8772d67c5f7caaa644767b63ec3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 28 Apr 2022 16:32:21 +0200 Subject: [PATCH 10/11] Fix issue reference in docs --- Doc/whatsnew/3.11.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index e97aa77d812ecf..9a56c2b94a1f2a 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1330,7 +1330,7 @@ C API Changes the old names are available as aliases. (Contributed by Petr Viktorin, Victor Stinner and Nick Coghlan in - :issue:`91744`.) + :gh:`91744`.) * :c:func:`PyFrameEvalFunction` now takes ``_PyInterpreterFrame*`` as its second parameter, instead of ``PyFrameObject*``. From b189c81be684e317288aa22a73ce8b96fb1ad231 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 4 May 2022 01:16:15 +0200 Subject: [PATCH 11/11] Rename semi-stable to unstable --- Include/README.rst | 2 +- Include/cpython/code.h | 6 +++--- Include/pyport.h | 14 +++++++------- Include/pystate.h | 6 +++--- Include/{semistable => unstable}/code.h | 12 ++++++------ Include/{semistable => unstable}/pystate.h | 6 +++--- 6 files changed, 23 insertions(+), 23 deletions(-) rename Include/{semistable => unstable}/code.h (87%) rename Include/{semistable => unstable}/pystate.h (89%) diff --git a/Include/README.rst b/Include/README.rst index faa6f3132a9315..03c55f81eed377 100644 --- a/Include/README.rst +++ b/Include/README.rst @@ -5,7 +5,7 @@ The C API is divided into these sections: 1. ``Include/``: Limited API 2. ``Include/cpython/``: CPython implementation details -3. ``Include/semistable/``: API that can change between minor releases +3. ``Include/unstable/``: API that can change between minor releases 4. ``Include/internal/``: The internal API Information on changing the C API is available `in the developer guide`_ diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 004ebf60ecd03d..41b1a9823f2a7a 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -198,9 +198,9 @@ typedef enum _PyCodeLocationInfoKind { PY_CODE_LOCATION_INFO_NONE = 15 } _PyCodeLocationInfoKind; -#define Py_SEMISTABLE_CODE_H -#include "semistable/code.h" -#undef Py_SEMISTABLE_CODE_H +#define Py_UNSTABLE_CODE_H +#include "unstable/code.h" +#undef Py_UNSTABLE_CODE_H #ifdef __cplusplus diff --git a/Include/pyport.h b/Include/pyport.h index 71200932bcd3e0..157c127f7ecdf9 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -43,9 +43,9 @@ # define Py_BUILD_CORE #endif -// Expose semi-stable API when building Python +// Expose unstable API when building Python #ifdef Py_BUILD_CORE -#define Py_USING_SEMI_STABLE_API +#define Py_USING_UNSTABLE_API #endif /************************************************************************** @@ -356,13 +356,13 @@ extern "C" { #define _Py_COMP_DIAG_POP #endif -/* _Py_NEWLY_SEMISTABLE: Provide deprecation warnings for users that - * don't opt in to semi-stable API. +/* _Py_NEWLY_UNSTABLE: Provide deprecation warnings for users that + * don't opt in to unstable API. */ -#ifdef Py_USING_SEMI_STABLE_API -#define _Py_NEWLY_SEMISTABLE(VERSION_UNUSED) +#ifdef Py_USING_UNSTABLE_API +#define _Py_NEWLY_UNSTABLE(VERSION_UNUSED) #else -#define _Py_NEWLY_SEMISTABLE(VERSION) Py_DEPRECATED(VERSION) +#define _Py_NEWLY_UNSTABLE(VERSION) Py_DEPRECATED(VERSION) #endif /* _Py_HOT_FUNCTION diff --git a/Include/pystate.h b/Include/pystate.h index 79f393aa679dbe..4fae7b179da9d0 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -121,9 +121,9 @@ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); #ifndef Py_LIMITED_API -# define Py_SEMISTABLE_PYSTATE_H -# include "semistable/pystate.h" -# undef Py_SEMISTABLE_PYSTATE_H +# define Py_UNSTABLE_PYSTATE_H +# include "unstable/pystate.h" +# undef Py_UNSTABLE_PYSTATE_H # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Include/semistable/code.h b/Include/unstable/code.h similarity index 87% rename from Include/semistable/code.h rename to Include/unstable/code.h index 14ea8cbe45c911..5ce02f8db4c6ad 100644 --- a/Include/semistable/code.h +++ b/Include/unstable/code.h @@ -1,16 +1,16 @@ -#ifndef Py_SEMISTABLE_CODE_H +#ifndef Py_UNSTABLE_CODE_H # error "this header file must not be included directly" #endif /* Functions added in PEP-523 */ -_Py_NEWLY_SEMISTABLE(3.11) +_Py_NEWLY_UNSTABLE(3.11) PyAPI_FUNC(Py_ssize_t) PyEval_RequestCodeExtraIndex(freefunc); -_Py_NEWLY_SEMISTABLE(3.11) +_Py_NEWLY_UNSTABLE(3.11) PyAPI_FUNC(int) PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra); -_Py_NEWLY_SEMISTABLE(3.11) +_Py_NEWLY_UNSTABLE(3.11) PyAPI_FUNC(int) PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra); /* Underscore-prefixed names for the above. @@ -23,14 +23,14 @@ PyAPI_FUNC(int) PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra); /* PyCodeObject constructors */ -_Py_NEWLY_SEMISTABLE(3.11) +_Py_NEWLY_UNSTABLE(3.11) PyAPI_FUNC(PyCodeObject *) PyCode_New( int, int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *, PyObject *); -_Py_NEWLY_SEMISTABLE(3.11) +_Py_NEWLY_UNSTABLE(3.11) PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs( int, int, int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, diff --git a/Include/semistable/pystate.h b/Include/unstable/pystate.h similarity index 89% rename from Include/semistable/pystate.h rename to Include/unstable/pystate.h index 03bbc5d8d2ed4a..c88c5dc425d89c 100644 --- a/Include/semistable/pystate.h +++ b/Include/unstable/pystate.h @@ -1,4 +1,4 @@ -#ifndef Py_SEMISTABLE_PYSTATE_H +#ifndef Py_UNSTABLE_PYSTATE_H # error "this header file must not be included directly" #endif @@ -7,11 +7,11 @@ struct _PyInterpreterFrame; typedef PyObject* (*PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int); -_Py_NEWLY_SEMISTABLE(3.11) +_Py_NEWLY_UNSTABLE(3.11) PyAPI_FUNC(PyFrameEvalFunction) PyInterpreterState_GetEvalFrameFunc( PyInterpreterState *interp); -_Py_NEWLY_SEMISTABLE(3.11) +_Py_NEWLY_UNSTABLE(3.11) PyAPI_FUNC(void) PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, PyFrameEvalFunction eval_frame);