Skip to content

Commit 0d90b5c

Browse files
author
Release Manager
committed
gh-39671: fix overflow in Matrix_modn_dense Replace use of `unsigned int` in matrix indices in favor of `Py_ssize_t` to avoid overflow. Fixes #35885 URL: #39671 Reported by: Vincent Delecroix Reviewer(s): Dima Pasechnik, schmittj, Vincent Delecroix
2 parents fec0504 + 36b43a4 commit 0d90b5c

File tree

1 file changed

+12
-8
lines changed

1 file changed

+12
-8
lines changed

src/sage/matrix/matrix_modn_dense_template.pxi

+12-8
Original file line numberDiff line numberDiff line change
@@ -444,17 +444,21 @@ cdef class Matrix_modn_dense_template(Matrix_dense):
444444
raise OverflowError("p (=%s) must be < %s." % (p, MAX_MODULUS))
445445

446446
if zeroed_alloc:
447-
self._entries = <celement *>check_calloc(self._nrows * self._ncols, sizeof(celement))
447+
self._entries = <celement *> check_calloc(self._nrows * self._ncols, sizeof(celement))
448448
else:
449-
self._entries = <celement *>check_allocarray(self._nrows * self._ncols, sizeof(celement))
449+
self._entries = <celement *> check_allocarray(self._nrows * self._ncols, sizeof(celement))
450+
451+
# TODO: it is a bit of a waste to allocate _matrix when ncols=0. Though some
452+
# of the code expects self._matrix[i] to be valid, even though it points to
453+
# an empty vector.
454+
self._matrix = <celement **> check_allocarray(self._nrows, sizeof(celement*))
455+
if self._nrows == 0:
456+
return
450457

451-
self._matrix = <celement **>check_allocarray(self._nrows, sizeof(celement*))
452-
cdef unsigned int k
453458
cdef Py_ssize_t i
454-
k = 0
455-
for i in range(self._nrows):
456-
self._matrix[i] = self._entries + k
457-
k = k + self._ncols
459+
self._matrix[0] = self._entries
460+
for i in range(self._nrows - 1):
461+
self._matrix[i + 1] = self._matrix[i] + self._ncols
458462

459463
def __dealloc__(self):
460464
"""

0 commit comments

Comments
 (0)