sagemathgh-36407: Support python 3.12
    
This PR adds support for running with python 3.12 from system. This has
been tested on the python 3.12 branch of void linux (x86_64, x86_64-musl
and i686).

The first two commits correspond to sagemath#36403. The rest is split is small
pieces and I tried to add reasonable explanations in the commit
messages. Reviewing by commit may be easier (ignoring the first two,
already reviewed).

See also: sagemath#36181

### ⌛ Dependencies

sagemath#36403
    
URL: sagemath#36407
Reported by: Gonzalo Tornaría
Reviewer(s): Matthias Köppe

build/sage_bootstrap/package.py

@@ -326,7 +326,7 @@ def _init_checksum(self):
         # Name of the directory containing the checksums.ini file
         self.__tarball_package_name = os.path.realpath(checksums_ini).split(os.sep)[-2]
 
-    VERSION_PATCHLEVEL = re.compile('(?P<version>.*)\.p(?P<patchlevel>[0-9]+)')
+    VERSION_PATCHLEVEL = re.compile(r'(?P<version>.*)\.p(?P<patchlevel>[0-9]+)')
 
     def _init_version(self):
         try:

src/sage/all__sagemath_repl.py

@@ -79,6 +79,23 @@
                         message=r"Use setlocale\(\), getencoding\(\) and getlocale\(\) instead",
                         module='docutils.io')
 
+# triggered by dateutil 2.8.2 and sphinx 7.0.1 on Python 3.12
+# see: https://github.com/dateutil/dateutil/pull/1285
+# see: https://github.com/sphinx-doc/sphinx/pull/11468
+warnings.filterwarnings('ignore', category=DeprecationWarning,
+                        message=r"datetime.datetime.utcfromtimestamp\(\) is deprecated",
+                        module='dateutil.tz.tz|sphinx.(builders.gettext|util.i18n)')
+
+# triggered on Python 3.12
+warnings.filterwarnings('ignore', category=DeprecationWarning,
+                        message=r"This process.* is multi-threaded, "
+                                r"use of .*\(\) may lead to deadlocks in the child.")
+
+# pickling of itertools is deprecated in Python 3.12
+warnings.filterwarnings('ignore', category=DeprecationWarning,
+                        message=r"Pickle, copy, and deepcopy support will be "
+                                r"removed from itertools in Python 3.14.")
+
 
 from .all__sagemath_objects import *
 from .all__sagemath_environment import *

src/sage/arith/long.pxd

@@ -19,6 +19,8 @@ from libc.limits cimport LONG_MIN, LONG_MAX
 from cpython.object cimport Py_SIZE
 from cpython.number cimport PyNumber_Index, PyIndex_Check
 from cpython.longintrepr cimport py_long, PyLong_SHIFT, digit
+from sage.cpython.pycore_long cimport (
+    ob_digit, _PyLong_IsNegative, _PyLong_DigitCount)
 
 from sage.libs.gmp.mpz cimport mpz_fits_slong_p, mpz_get_si
 from sage.rings.integer_fake cimport is_Integer, Integer_AS_MPZ
@@ -299,8 +301,11 @@ cdef inline bint integer_check_long_py(x, long* value, int* err):
         return 0
 
     # x is a Python "int" (aka PyLongObject or py_long in cython)
-    cdef const digit* D = (<py_long>x).ob_digit
-    cdef Py_ssize_t size = Py_SIZE(x)
+    cdef const digit* D = ob_digit(x)
+    cdef Py_ssize_t size = _PyLong_DigitCount(x)
+
+    if _PyLong_IsNegative(x):
+        size = -size
 
     # We assume PyLong_SHIFT <= BITS_IN_LONG <= 3 * PyLong_SHIFT.
     # This is true in all the default configurations:

src/sage/cpython/atexit.pyx

@@ -154,6 +154,10 @@ cdef extern from *:
     #undef _PyGC_FINALIZED
     #include "internal/pycore_interp.h"
     #include "internal/pycore_pystate.h"
+    #if PY_VERSION_HEX >= 0x030c0000
+    // struct atexit_callback was renamed in 3.12 to atexit_py_callback
+    #define atexit_callback atexit_py_callback
+    #endif
     static atexit_callback ** _atexit_callbacks(PyObject *self) {
         PyInterpreterState *interp = _PyInterpreterState_GET();
         struct atexit_state state = interp->atexit;

src/sage/cpython/debug.pyx

@@ -78,7 +78,7 @@ def getattr_debug(obj, name, default=_no_default):
 
     EXAMPLES::
 
-        sage: _ = getattr_debug(list, "reverse")
+        sage: _ = getattr_debug(list, "reverse")  # not tested - broken in python 3.12
         getattr_debug(obj=<class 'list'>, name='reverse'):
           type(obj) = <class 'type'>
           object has __dict__ slot (<class 'dict'>)

src/sage/cpython/dict_internal.h

@@ -169,6 +169,7 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix)
 #else   /* Python >= 3.11 */
 
 #define Py_BUILD_CORE
+#undef _PyGC_FINALIZED
 #include <internal/pycore_dict.h>
 
 /************************************************************/

src/sage/cpython/pycore_long.h

@@ -0,0 +1,98 @@
+#include "Python.h"
+#include <stdbool.h>
+
+#if PY_VERSION_HEX >= 0x030C00A5
+#define ob_digit(o)  (((PyLongObject*)o)->long_value.ob_digit)
+#else
+#define ob_digit(o)  (((PyLongObject*)o)->ob_digit)
+#endif
+
+#if PY_VERSION_HEX >= 0x030C00A7
+// taken from cpython:Include/internal/pycore_long.h @ 3.12
+
+/* Long value tag bits:
+ * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1.
+ * 2: Reserved for immortality bit
+ * 3+ Unsigned digit count
+ */
+#define SIGN_MASK 3
+#define SIGN_ZERO 1
+#define SIGN_NEGATIVE 2
+#define NON_SIZE_BITS 3
+
+static inline bool
+_PyLong_IsZero(const PyLongObject *op)
+{
+    return (op->long_value.lv_tag & SIGN_MASK) == SIGN_ZERO;
+}
+
+static inline bool
+_PyLong_IsNegative(const PyLongObject *op)
+{
+    return (op->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE;
+}
+
+static inline bool
+_PyLong_IsPositive(const PyLongObject *op)
+{
+    return (op->long_value.lv_tag & SIGN_MASK) == 0;
+}
+
+static inline Py_ssize_t
+_PyLong_DigitCount(const PyLongObject *op)
+{
+    assert(PyLong_Check(op));
+    return op->long_value.lv_tag >> NON_SIZE_BITS;
+}
+
+#define TAG_FROM_SIGN_AND_SIZE(sign, size) ((1 - (sign)) | ((size) << NON_SIZE_BITS))
+
+static inline void
+_PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size)
+{
+    assert(size >= 0);
+    assert(-1 <= sign && sign <= 1);
+    assert(sign != 0 || size == 0);
+    op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, (size_t)size);
+}
+
+#else
+// fallback for < 3.12
+
+static inline bool
+_PyLong_IsZero(const PyLongObject *op)
+{
+    return Py_SIZE(op) == 0;
+}
+
+static inline bool
+_PyLong_IsNegative(const PyLongObject *op)
+{
+    return Py_SIZE(op) < 0;
+}
+
+static inline bool
+_PyLong_IsPositive(const PyLongObject *op)
+{
+    return Py_SIZE(op) > 0;
+}
+
+static inline Py_ssize_t
+_PyLong_DigitCount(const PyLongObject *op)
+{
+    Py_ssize_t size = Py_SIZE(op);
+    return size < 0 ? -size : size;
+}
+
+static inline void
+_PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size)
+{
+#if (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION < 9)
+// The function Py_SET_SIZE is defined starting with python 3.9.
+    Py_SIZE(o) = size;
+#else
+    Py_SET_SIZE(op, sign < 0 ? -size : size);
+#endif
+}
+
+#endif

src/sage/cpython/pycore_long.pxd

@@ -0,0 +1,9 @@
+from cpython.longintrepr cimport py_long, digit
+
+cdef extern from "pycore_long.h":
+    digit* ob_digit(py_long o)
+    bint _PyLong_IsZero(py_long o)
+    bint _PyLong_IsNegative(py_long o)
+    bint _PyLong_IsPositive(py_long o)
+    Py_ssize_t _PyLong_DigitCount(py_long o)
+    void _PyLong_SetSignAndDigitCount(py_long o, int sign, Py_ssize_t size)

src/sage/geometry/polyhedron/combinatorial_polyhedron/face_iterator.pyx

@@ -1945,7 +1945,8 @@ cdef inline int next_face_loop(iter_t structure) nogil except -1:
         # The function is not supposed to be called,
         # just prevent it from crashing.
         # Actually raising an error here results in a bad branch prediction.
-        return -1
+        # But return -1 results in a crash with python 3.12
+        raise StopIteration
 
     # Getting ``[faces, n_faces, n_visited_all]`` according to dimension.
     cdef face_list_t* faces = &structure.new_faces[structure.current_dimension]

src/sage/lfunctions/dokchitser.py

@@ -418,8 +418,8 @@ def init_coeffs(self, v, cutoff=1,
             sage: L(14)
             0.998583063162746
             sage: a = delta_qexp(1000)
-            sage: sum(a[n]/float(n)^14 for n in range(1,1000))
-            0.9985830631627459
+            sage: sum(a[n]/float(n)^14 for n in reversed(range(1,1000)))
+            0.9985830631627461
 
         Illustrate that one can give a list of complex numbers for v
         (see :trac:`10937`)::

src/sage/lfunctions/pari.py

@@ -141,8 +141,8 @@ def init_coeffs(self, v, cutoff=None, w=1):
             sage: L(14)
             0.998583063162746
             sage: a = delta_qexp(1000)
-            sage: sum(a[n]/float(n)^14 for n in range(1,1000))
-            0.9985830631627459
+            sage: sum(a[n]/float(n)^14 for n in reversed(range(1,1000)))
+            0.9985830631627461
 
         Illustrate that one can give a list of complex numbers for v
         (see :trac:`10937`)::

src/sage/libs/gmp/pylong.pxd

@@ -2,9 +2,10 @@
 Various functions to deal with conversion mpz <-> Python int/long
 """
 
+from cpython.longintrepr cimport py_long
 from sage.libs.gmp.types cimport *
 
 cdef mpz_get_pylong(mpz_srcptr z)
 cdef mpz_get_pyintlong(mpz_srcptr z)
-cdef int mpz_set_pylong(mpz_ptr z, L) except -1
+cdef int mpz_set_pylong(mpz_ptr z, py_long L) except -1
 cdef Py_hash_t mpz_pythonhash(mpz_srcptr z)

src/sage/libs/gmp/pylong.pyx

@@ -28,6 +28,8 @@ AUTHORS:
 from cpython.object cimport Py_SIZE
 from cpython.long cimport PyLong_FromLong
 from cpython.longintrepr cimport _PyLong_New, py_long, digit, PyLong_SHIFT
+from sage.cpython.pycore_long cimport (ob_digit, _PyLong_IsNegative,
+    _PyLong_DigitCount, _PyLong_SetSignAndDigitCount)
 from .mpz cimport *
 
 cdef extern from *:
@@ -60,12 +62,9 @@ cdef mpz_get_pylong_large(mpz_srcptr z):
     """
     cdef size_t nbits = mpz_sizeinbase(z, 2)
     cdef size_t pylong_size = (nbits + PyLong_SHIFT - 1) // PyLong_SHIFT
-    L = _PyLong_New(pylong_size)
-    mpz_export(L.ob_digit, NULL,
-            -1, sizeof(digit), 0, PyLong_nails, z)
-    if mpz_sgn(z) < 0:
-        # Set correct size
-        Py_SET_SIZE(L, -pylong_size)
+    cdef py_long L = _PyLong_New(pylong_size)
+    mpz_export(ob_digit(L), NULL, -1, sizeof(digit), 0, PyLong_nails, z)
+    _PyLong_SetSignAndDigitCount(L, mpz_sgn(z), pylong_size)
     return L
 
 
@@ -88,16 +87,13 @@ cdef mpz_get_pyintlong(mpz_srcptr z):
     return mpz_get_pylong_large(z)
 
 
-cdef int mpz_set_pylong(mpz_ptr z, L) except -1:
+cdef int mpz_set_pylong(mpz_ptr z, py_long L) except -1:
     """
     Convert a Python ``long`` `L` to an ``mpz``.
     """
-    cdef Py_ssize_t pylong_size = Py_SIZE(L)
-    if pylong_size < 0:
-        pylong_size = -pylong_size
-    mpz_import(z, pylong_size, -1, sizeof(digit), 0, PyLong_nails,
-            (<py_long>L).ob_digit)
-    if Py_SIZE(L) < 0:
+    cdef Py_ssize_t pylong_size = _PyLong_DigitCount(L)
+    mpz_import(z, pylong_size, -1, sizeof(digit), 0, PyLong_nails, ob_digit(L))
+    if _PyLong_IsNegative(L):
         mpz_neg(z, z)
 
 

src/sage/matroids/lean_matrix.pyx

@@ -1,5 +1,4 @@
 # sage.doctest: optional - sage.rings.finite_rings
-# cython: profile=True
 """
 Lean matrices
 

src/sage/misc/dev_tools.py

@@ -171,6 +171,7 @@ def load_submodules(module=None, exclude_pattern=None):
         load sage.geometry.polyhedron.ppl_lattice_polygon... succeeded
     """
     from .package_dir import walk_packages
+    import importlib.util
 
     if module is None:
         import sage
@@ -194,8 +195,12 @@ def load_submodules(module=None, exclude_pattern=None):
         try:
             sys.stdout.write("load %s..." % module_name)
             sys.stdout.flush()
-            loader = importer.find_module(module_name)
-            loader.load_module(module_name)
+            # see
+            # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
+            spec = importer.find_spec(module_name)
+            module = importlib.util.module_from_spec(spec)
+            sys.modules[module_name] = module
+            spec.loader.exec_module(module)
             sys.stdout.write(" succeeded\n")
         except (ValueError, AttributeError, TypeError, ImportError):
             # we might get error because of cython code that has been

src/sage/misc/sageinspect.py

@@ -602,7 +602,7 @@ def visit_Num(self, node):
             On Python 3 negative numbers are parsed first, for some reason, as
             a UnaryOp node.
         """
-        return node.n
+        return node.value
 
     def visit_Str(self, node):
         r"""
@@ -624,7 +624,7 @@ def visit_Str(self, node):
             sage: [vis(s) for s in ['"abstract"', "'syntax'", r'''r"tr\ee"''']]
             ['abstract', 'syntax', 'tr\\ee']
         """
-        return node.s
+        return node.value
 
     def visit_List(self, node):
         """

src/sage/monoids/trace_monoid.py

@@ -40,7 +40,6 @@
 #                  https://www.gnu.org/licenses/
 # ****************************************************************************
 
-from collections import OrderedDict
 from itertools import repeat, chain, product
 
 from sage.misc.cachefunc import cached_method
@@ -633,14 +632,14 @@ def _compute_dependence_stack(self, x):
             sage: x = b*a*d*a*c*b
             sage: M._compute_dependence_stack(x)
             ({a, b, c, d},
-             OrderedDict([(a, [False, False, True, True, False]),
-                          (b, [True, False, False, False, True]),
-                          (c, [True, False, False, False]),
-                          (d, [False, False, True, False])]))
+             {a: [False, False, True, True, False],
+              b: [True, False, False, False, True],
+              c: [True, False, False, False],
+              d: [False, False, True, False]})
         """
         independence = self._independence
         generators_set = set(e for e, _ in x)
-        stacks = OrderedDict(sorted((g, []) for g in generators_set))
+        stacks = dict(sorted((g, []) for g in generators_set))
         for generator, times in reversed(list(x)):
             stacks[generator].extend(repeat(True, times))
             for other_gen in generators_set:

src/sage/sat/solvers/satsolver.pyx

@@ -375,10 +375,10 @@ def SAT(solver=None, *args, **kwds):
         DIMACS Solver: 'kissat -q {input}'
     """
     if solver is None:
-        import pkgutil
-        if pkgutil.find_loader('pycryptosat') is not None:
+        from importlib.util import find_spec
+        if find_spec('pycryptosat') is not None:
             solver = "cryptominisat"
-        elif pkgutil.find_loader('pycosat') is not None:
+        elif find_spec('pycosat') is not None:
             solver = "picosat"
         else:
             solver = "LP"