Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gh-111928: make "memo" dict local to scan_once call #112005

Merged
merged 5 commits into from
Nov 13, 2023
Merged

gh-111928: make "memo" dict local to scan_once call #112005

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
afd9ea4
json: make JSON scanner thread safe
colesbury Apr 12, 2022
da7758a
Add new entry
aisk Nov 12, 2023
79f26fa
fix new entry
aisk Nov 12, 2023
3846ef2
fix indentation
aisk Nov 12, 2023
fe5e1d0
Delete Misc/NEWS.d/next/Library/2023-11-12-21-39-36.gh-issue-111928.S…
corona10 Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 19 additions & 23 deletions Modules/_json.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ typedef struct _PyScannerObject {
PyObject *parse_float;
PyObject *parse_int;
PyObject *parse_constant;
PyObject *memo;
} PyScannerObject;

static PyMemberDef scanner_members[] = {
Expand Down Expand Up @@ -70,7 +69,7 @@ ascii_escape_unicode(PyObject *pystr);
static PyObject *
py_encode_basestring_ascii(PyObject* Py_UNUSED(self), PyObject *pystr);
static PyObject *
scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr);
scan_once_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr);
static PyObject *
_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx);
static PyObject *
Expand Down Expand Up @@ -631,7 +630,6 @@ scanner_traverse(PyScannerObject *self, visitproc visit, void *arg)
Py_VISIT(self->parse_float);
Py_VISIT(self->parse_int);
Py_VISIT(self->parse_constant);
Py_VISIT(self->memo);
return 0;
}

Expand All @@ -643,12 +641,11 @@ scanner_clear(PyScannerObject *self)
Py_CLEAR(self->parse_float);
Py_CLEAR(self->parse_int);
Py_CLEAR(self->parse_constant);
Py_CLEAR(self->memo);
return 0;
}

static PyObject *
_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
_parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
{
/* Read a JSON object from PyUnicode pystr.
idx is the index of the first character after the opening curly brace.
Expand Down Expand Up @@ -693,7 +690,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss
key = scanstring_unicode(pystr, idx + 1, s->strict, &next_idx);
if (key == NULL)
goto bail;
memokey = PyDict_SetDefault(s->memo, key, key);
memokey = PyDict_SetDefault(memo, key, key);
if (memokey == NULL) {
goto bail;
}
Expand All @@ -710,7 +707,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss
while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++;

/* read any JSON term */
val = scan_once_unicode(s, pystr, idx, &next_idx);
val = scan_once_unicode(s, memo, pystr, idx, &next_idx);
if (val == NULL)
goto bail;

Expand Down Expand Up @@ -774,7 +771,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss
}

static PyObject *
_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
_parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
/* Read a JSON array from PyUnicode pystr.
idx is the index of the first character after the opening brace.
*next_idx_ptr is a return-by-reference index to the first character after
Expand Down Expand Up @@ -805,7 +802,7 @@ _parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssi
while (1) {

/* read any JSON term */
val = scan_once_unicode(s, pystr, idx, &next_idx);
val = scan_once_unicode(s, memo, pystr, idx, &next_idx);
if (val == NULL)
goto bail;

Expand Down Expand Up @@ -986,7 +983,7 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_
}

static PyObject *
scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
scan_once_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
{
/* Read one JSON term (of any kind) from PyUnicode pystr.
idx is the index of the first character of the term
Expand Down Expand Up @@ -1022,15 +1019,15 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_
if (_Py_EnterRecursiveCall(" while decoding a JSON object "
"from a unicode string"))
return NULL;
res = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr);
res = _parse_object_unicode(s, memo, pystr, idx + 1, next_idx_ptr);
_Py_LeaveRecursiveCall();
return res;
case '[':
/* array */
if (_Py_EnterRecursiveCall(" while decoding a JSON array "
"from a unicode string"))
return NULL;
res = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr);
res = _parse_array_unicode(s, memo, pystr, idx + 1, next_idx_ptr);
_Py_LeaveRecursiveCall();
return res;
case 'n':
Expand Down Expand Up @@ -1106,16 +1103,19 @@ scanner_call(PyScannerObject *self, PyObject *args, PyObject *kwds)
if (!PyArg_ParseTupleAndKeywords(args, kwds, "On:scan_once", kwlist, &pystr, &idx))
return NULL;

if (PyUnicode_Check(pystr)) {
rval = scan_once_unicode(self, pystr, idx, &next_idx);
}
else {
if (!PyUnicode_Check(pystr)) {
PyErr_Format(PyExc_TypeError,
"first argument must be a string, not %.80s",
Py_TYPE(pystr)->tp_name);
"first argument must be a string, not %.80s",
Py_TYPE(pystr)->tp_name);
return NULL;
}
PyDict_Clear(self->memo);

PyObject *memo = PyDict_New();
if (memo == NULL) {
return NULL;
}
rval = scan_once_unicode(self, memo, pystr, idx, &next_idx);
Py_DECREF(memo);
if (rval == NULL)
return NULL;
return _build_rval_index_tuple(rval, next_idx);
Expand All @@ -1137,10 +1137,6 @@ scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
return NULL;
}

s->memo = PyDict_New();
if (s->memo == NULL)
goto bail;

/* All of these will fail "gracefully" so we don't need to verify them */
strict = PyObject_GetAttrString(ctx, "strict");
if (strict == NULL)
Expand Down