diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 233da058f464d17..e599ba4709f13bd 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -210,6 +210,8 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) +#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1] + #ifdef Py_GIL_DISABLED #define DICT_NEXT_VERSION(INTERP) \ (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) @@ -255,7 +257,7 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) assert(ix < SHARED_KEYS_MAX_SIZE); uint8_t *size_ptr = ((uint8_t *)values)-2; int size = *size_ptr; - assert(size+2 < ((uint8_t *)values)[-1]); + assert(size+2 < DICT_VALUES_SIZE(values)); size++; size_ptr[-size] = (uint8_t)ix; *size_ptr = size; diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index daaef211b1db494..fb46c4c64334f91 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -66,21 +66,6 @@ PyDoc_STRVAR(dict___contains____doc__, #define DICT___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)dict___contains__, METH_O|METH_COEXIST, dict___contains____doc__}, -static PyObject * -dict___contains___impl(PyDictObject *self, PyObject *key); - -static PyObject * -dict___contains__(PyDictObject *self, PyObject *key) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = dict___contains___impl(self, key); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - PyDoc_STRVAR(dict_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -327,4 +312,4 @@ dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict_values_impl(self); } -/*[clinic end generated code: output=c8fda06bac5b05f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f3dd5f3fb8122aef input=a9049054013a1b77]*/ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 2df95e977a180fa..5837a2d988bd93a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -80,6 +80,8 @@ DK_ENTRIES(keys)[index] if index >= 0): Active upon key insertion. Dummy slots cannot be made Unused again else the probe sequence in case of collision would have no way to know they were once active. + In free-threaded builds dummy slots are not re-used to allow lock-free + lookups to proceed safely. 4. Pending. index >= 0, key != NULL, and value == NULL (split only) Not yet inserted in split-table. @@ -150,10 +152,48 @@ ASSERT_DICT_LOCKED(PyObject *op) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); } #define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) +#define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); +#define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value); +#define ASSERT_OWNED_OR_SHARED(mp) \ + assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)); + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_keys, keys); +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) { + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_values, values); +} + +// Defined until we get QSBR +#define _PyMem_FreeQsbr PyMem_Free #else #define ASSERT_DICT_LOCKED(op) +#define IS_DICT_SHARED(mp) (false) +#define SET_DICT_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx] +#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(keys->dk_indices))[idx] = (int##size##_t)value + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + mp->ma_keys = keys; +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + mp->ma_values = values; +} #endif @@ -257,9 +297,7 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, static PyObject* dict_iter(PyObject *dict); static int -contains_lock_held(PyDictObject *mp, PyObject *key); -static int -contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash); +contains_known_hash(PyDictObject *mp, PyObject *key, Py_ssize_t hash); static int setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value); static int @@ -330,7 +368,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyDictKeysObject *keys); +static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); /* PyDictKeysObject has refcounts like PyObject does, so we have the following two functions to mirror what Py_INCREF() and Py_DECREF() do. @@ -352,7 +390,7 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { return; @@ -378,7 +416,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) Py_XDECREF(entries[i].me_value); } } - free_keys_object(dk); + free_keys_object(dk, use_qsbr); } } @@ -390,22 +428,18 @@ dictkeys_get_index(const PyDictKeysObject *keys, Py_ssize_t i) Py_ssize_t ix; if (log2size < 8) { - const int8_t *indices = (const int8_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 8, i); } else if (log2size < 16) { - const int16_t *indices = (const int16_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 16, i); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - const int64_t *indices = (const int64_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 64, i); } #endif else { - const int32_t *indices = (const int32_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 32, i); } assert(ix >= DKIX_DUMMY); return ix; @@ -421,25 +455,21 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) assert(keys->dk_version == 0); if (log2size < 8) { - int8_t *indices = (int8_t*)(keys->dk_indices); assert(ix <= 0x7f); - indices[i] = (char)ix; + STORE_INDEX(keys, 8, i, ix); } else if (log2size < 16) { - int16_t *indices = (int16_t*)(keys->dk_indices); assert(ix <= 0x7fff); - indices[i] = (int16_t)ix; + STORE_INDEX(keys, 16, i, ix); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - int64_t *indices = (int64_t*)(keys->dk_indices); - indices[i] = ix; + STORE_INDEX(keys, 64, i, ix); } #endif else { - int32_t *indices = (int32_t*)(keys->dk_indices); assert(ix <= 0x7fffffff); - indices[i] = (int32_t)ix; + STORE_INDEX(keys, 32, i, ix); } } @@ -706,8 +736,14 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } static void -free_keys_object(PyDictKeysObject *keys) +free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeQsbr(keys); + return; + } +#endif #ifdef WITH_FREELISTS struct _Py_dict_freelist *state = get_dict_state(); if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE @@ -739,9 +775,15 @@ new_values(size_t size) } static inline void -free_values(PyDictValues *values) +free_values(PyDictValues *values, int use_qsbr) { int prefix_size = ((uint8_t *)values)[-1]; +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeQsbr(((char *)values)-prefix_size); + return; + } +#endif PyMem_Free(((char *)values)-prefix_size); } @@ -767,9 +809,9 @@ new_dict(PyInterpreterState *interp, { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); if (free_values_on_failure) { - free_values(values); + free_values(values, false); } return NULL; } @@ -795,7 +837,7 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; @@ -1118,6 +1160,262 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu return ix; } +static inline void +ensure_shared_on_read(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // We are accessing the dict from another thread then owns + // it and we haven't marked it as shared which will ensure + // that when we re-size ma_keys or ma_values that we will + // free using QSBR. We need to lock the dictionary to + // contend with writes from the owning thread, mark it as + // shared, and then we can continue with lock-free reads. + Py_BEGIN_CRITICAL_SECTION(mp); + if (!IS_DICT_SHARED(mp)) { + SET_DICT_SHARED(mp); + } + Py_END_CRITICAL_SECTION(); + } +#endif +} + +static inline void +ensure_shared_on_resize(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // We are writing to the dict from another thread then owns + // it and we haven't marked it as shared which will ensure + // that when we re-size ma_keys or ma_values that we will + // free using QSBR. We need to lock the dictionary to + // contend with writes from the owning thread, mark it as + // shared, and then we can continue with lock-free reads. + // Technically this is a little heavy handed, we could just + // free the individual old keys / old-values using qsbr + SET_DICT_SHARED(mp); + } +#endif +} + +#ifdef Py_GIL_DISABLED + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); + + if (startkey != NULL) { + if (!_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + + if (unicode_get_hash(startkey) == hash) { + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + else { + Py_DECREF(startkey); + } + } + return 0; +} + +// Search non-Unicode key from Unicode table +static Py_ssize_t +unicodekeys_lookup_generic_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_unicode_generic_threadsafe); +} + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(startkey)); + if (startkey == key) { + return 1; + } + if (startkey != NULL) { + if (_Py_IsImmortal(startkey)) { + return unicode_get_hash(startkey) == hash && unicode_eq(startkey, key); + } + else { + if (!_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + if (unicode_get_hash(startkey) == hash && unicode_eq(startkey, key)) { + Py_DECREF(startkey); + return 1; + } + Py_DECREF(startkey); + } + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode_threadsafe); +} + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + if (startkey == key) { + return 1; + } + Py_ssize_t ep_hash = _Py_atomic_load_ssize_relaxed(&ep->me_hash); + if (ep_hash == hash) { + if (startkey == NULL || !_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic_threadsafe); +} + +static Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + PyDictKeysObject *dk; + DictKeysKind kind; + Py_ssize_t ix; + PyObject *value; + + ensure_shared_on_read(mp); + + dk = _Py_atomic_load_ptr(&mp->ma_keys); + kind = dk->dk_kind; + + if (kind != DICT_KEYS_GENERAL) { + if (PyUnicode_CheckExact(key)) { + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + } + else { + ix = unicodekeys_lookup_generic_threadsafe(mp, dk, key, hash); + } + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + + if (ix >= 0) { + if (kind == DICT_KEYS_SPLIT) { + PyDictValues *values = _Py_atomic_load_ptr(&mp->ma_values); + if (values == NULL) + goto read_failed; + + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values)); + if (ix >= (Py_ssize_t)capacity) + goto read_failed; + + value = _Py_TryXGetRef(&values->values[ix]); + if (value == NULL) + goto read_failed; + + if (values != _Py_atomic_load_ptr(&mp->ma_values)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = _Py_TryXGetRef(&DK_UNICODE_ENTRIES(dk)[ix].me_value); + if (value == NULL) { + goto read_failed; + } + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + } + else { + value = NULL; + } + } + else { + ix = dictkeys_generic_lookup_threadsafe(mp, dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + if (ix >= 0) { + value = _Py_TryXGetRef(&DK_ENTRIES(dk)[ix].me_value); + if (value == NULL) + goto read_failed; + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = NULL; + } + } + + *value_addr = value; + return ix; + +read_failed: + // In addition to the normal races of the dict being modified the _Py_TryXGetRef + // can all fail if they don't yet have a shared ref count. That can happen here + // or in the *_lookup_* helper. In that case we need to take the lock to avoid + // mutation and do a normal incref which will make them shared. + Py_BEGIN_CRITICAL_SECTION(mp); + ix = _Py_dict_lookup(mp, key, hash, &value); + *value_addr = value; + if (value != NULL) { + assert(ix >= 0); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + return ix; +} + +#endif + int _PyDict_HasOnlyStringKeys(PyObject *dict) { @@ -1188,6 +1486,16 @@ _PyDict_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +static inline int +is_unusable_slot(Py_ssize_t ix) +{ +#ifdef Py_GIL_DISABLED + return ix >= 0 || ix == DKIX_DUMMY; +#else + return ix >= 0; +#endif +} + /* Internal function to find slot for an item from its hash when it is known that the key is not present in the dict. @@ -1200,7 +1508,7 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) const size_t mask = DK_MASK(keys); size_t i = hash & mask; Py_ssize_t ix = dictkeys_get_index(keys, i); - for (size_t perturb = hash; ix >= 0;) { + for (size_t perturb = hash; is_unusable_slot(ix);) { perturb >>= PERTURB_SHIFT; i = (i*5 + perturb + 1) & mask; ix = dictkeys_get_index(keys, i); @@ -1349,7 +1657,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, return -1; } -// Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. +// Same to insertdict but specialized for ma_keys == Py_EMPTY_KEYS. // Consumes key and value references. static int insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, @@ -1370,28 +1678,37 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, return -1; } /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ - mp->ma_keys = newkeys; - mp->ma_values = NULL; + assert(mp->ma_values == NULL); MAINTAIN_TRACKING(mp, key, value); size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1); - dictkeys_set_index(mp->ma_keys, hashpos, 0); + dictkeys_set_index(newkeys, hashpos, 0); if (unicode) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys); ep->me_key = key; ep->me_value = value; } else { - PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *ep = DK_ENTRIES(newkeys); ep->me_key = key; ep->me_hash = hash; ep->me_value = value; } mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; + newkeys->dk_usable--; + newkeys->dk_nentries++; + // We store the keys last so no one can see them in a partially inconsistent + // state so that we don't need to switch the keys to being shared yet for + // the case where we're inserting from the non-owner thread. We don't use + // store_keys here because the transition from empty to non-empty is safe + // as the empty keys will never be freed. +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&mp->ma_keys, newkeys); +#else + mp->ma_keys = newkeys; +#endif return 0; } @@ -1465,15 +1782,16 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, unicode = 0; } + ensure_shared_on_resize(mp); /* NOTE: Current odict checks mp->ma_keys to detect resize happen. * So we can't reuse oldkeys even if oldkeys->dk_size == newsize. * TODO: Try reusing oldkeys when reimplement odict. */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); + set_keys(mp, new_keys_object(interp, log2_newsize, unicode)); if (mp->ma_keys == NULL) { - mp->ma_keys = oldkeys; + set_keys(mp, oldkeys); return -1; } // New table must be large enough. @@ -1512,9 +1830,9 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } build_indices_unicode(mp->ma_keys, newentries, numentries); } - dictkeys_decref(interp, oldkeys); - mp->ma_values = NULL; - free_values(oldvalues); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + set_values(mp, NULL); + free_values(oldvalues, IS_DICT_SHARED(mp)); } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { @@ -1573,7 +1891,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); - free_keys_object(oldkeys); + free_keys_object(oldkeys, IS_DICT_SHARED(mp)); } } @@ -1695,9 +2013,13 @@ dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) PyObject *value; Py_ssize_t ix; (void)ix; - PyObject *exc = _PyErr_GetRaisedException(tstate); +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif /* Ignore any exception raised by the lookup */ PyObject *exc2 = _PyErr_Occurred(tstate); @@ -1753,11 +2075,47 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } +/* Gets an item and provides a new reference if the value is present. + * Returns 1 if the key is present, 0 if the key is missing, and -1 if an + * exception occurred. +*/ +int +_PyDict_GetItemRef_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +{ + PyDictObject*mp = (PyDictObject *)op; + + PyObject *value; +#ifdef Py_GIL_DISABLED + Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); +#endif + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } +#ifdef Py_GIL_DISABLED + *result = value; +#else + *result = Py_NewRef(value); +#endif + return 1; // key is present +} int PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) @@ -1767,7 +2125,6 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) *result = NULL; return -1; } - PyDictObject*mp = (PyDictObject *)op; Py_hash_t hash; if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) @@ -1779,19 +2136,7 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) } } - PyObject *value; - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - assert(ix >= 0 || value == NULL); - if (ix == DKIX_ERROR) { - *result = NULL; - return -1; - } - if (value == NULL) { - *result = NULL; - return 0; // missing key - } - *result = Py_NewRef(value); - return 1; // key is present + return _PyDict_GetItemRef_KnownHash(op, key, hash, result); } @@ -1819,7 +2164,12 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) } } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } @@ -1860,7 +2210,6 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) return rv; } - /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * @@ -1874,6 +2223,7 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) { + // TODO: Thread safety Py_ssize_t ix; Py_hash_t hash; PyObject *value; @@ -2195,9 +2545,11 @@ clear_lock_held(PyObject *op) PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); - dictkeys_incref(Py_EMPTY_KEYS); - mp->ma_keys = Py_EMPTY_KEYS; - mp->ma_values = NULL; + // We don't inc ref empty keys because they're immortal + ensure_shared_on_resize(mp); + + set_keys(mp, Py_EMPTY_KEYS); + set_values(mp, NULL); mp->ma_used = 0; mp->ma_version_tag = new_version; /* ...then clear the keys and values */ @@ -2205,12 +2557,12 @@ clear_lock_held(PyObject *op) n = oldkeys->dk_nentries; for (i = 0; i < n; i++) Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues); - dictkeys_decref(interp, oldkeys); + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); } else { assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } ASSERT_CONSISTENT(mp); } @@ -2596,12 +2948,12 @@ dict_dealloc(PyObject *self) for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { Py_XDECREF(values->values[i]); } - free_values(values); - dictkeys_decref(interp, keys); + free_values(values, false); + dictkeys_decref(interp, keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } #ifdef WITH_FREELISTS struct _Py_dict_freelist *state = get_dict_state(); @@ -2721,7 +3073,7 @@ dict_length(PyObject *self) } static PyObject * -dict_subscript_lock_held(PyObject *self, PyObject *key) +dict_subscript(PyObject *self, PyObject *key) { PyDictObject *mp = (PyDictObject *)self; Py_ssize_t ix; @@ -2733,7 +3085,11 @@ dict_subscript_lock_held(PyObject *self, PyObject *key) if (hash == -1) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -2753,17 +3109,11 @@ dict_subscript_lock_held(PyObject *self, PyObject *key) _PyErr_SetKeyError(key); return NULL; } +#ifdef Py_GIL_DISABLED + return value; +#else return Py_NewRef(value); -} - -static PyObject * -dict_subscript(PyObject *self, PyObject *key) -{ - PyObject *res; - Py_BEGIN_CRITICAL_SECTION(self); - res = dict_subscript_lock_held(self, key); - Py_END_CRITICAL_SECTION(); - return res; +#endif } static int @@ -3141,10 +3491,12 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe if (keys == NULL) return -1; - dictkeys_decref(interp, mp->ma_keys); + ensure_shared_on_resize(mp); + dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - if (mp->ma_values != NULL) { - free_values(mp->ma_values); + + if (_PyDict_HasSplitTable(mp)) { + free_values(mp->ma_values, IS_DICT_SHARED(mp)); mp->ma_values = NULL; } @@ -3187,7 +3539,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe Py_NewRef(key), hash, Py_NewRef(value)); } else { - err = contains_known_hash_lock_held(mp, key, hash); + err = contains_known_hash(mp, key, hash); if (err == 0) { err = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); @@ -3270,7 +3622,7 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) for (key = PyIter_Next(iter); key; key = PyIter_Next(iter)) { if (override != 1) { - status = contains_lock_held(mp, key); + status = PyDict_Contains(a, key); if (status != 0) { if (status > 0) { if (override == 0) { @@ -3374,7 +3726,7 @@ copy_lock_held(PyObject *o) return PyErr_NoMemory(); split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { - free_values(newvalues); + free_values(newvalues, false); return NULL; } size_t prefix_size = ((uint8_t *)newvalues)[-1]; @@ -3568,7 +3920,6 @@ dict_richcompare(PyObject *v, PyObject *w, int op) /*[clinic input] @coexist -@critical_section dict.__contains__ key: object @@ -3578,8 +3929,8 @@ True if the dictionary has the specified key, else False. [clinic start generated code]*/ static PyObject * -dict___contains___impl(PyDictObject *self, PyObject *key) -/*[clinic end generated code: output=1b314e6da7687dae input=bc76ec9c157cb81b]*/ +dict___contains__(PyDictObject *self, PyObject *key) +/*[clinic end generated code: output=a3d03db709ed6e6b input=fe1cb42ad831e820]*/ { register PyDictObject *mp = self; Py_hash_t hash; @@ -3591,11 +3942,18 @@ dict___contains___impl(PyDictObject *self, PyObject *key) if (hash == -1) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) Py_RETURN_FALSE; +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif Py_RETURN_TRUE; } @@ -3623,13 +3981,24 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) if (hash == -1) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(self, key, hash, &val); +#else ix = _Py_dict_lookup(self, key, hash, &val); +#endif if (ix == DKIX_ERROR) return NULL; +#ifdef Py_GIL_DISABLED + if (ix == DKIX_EMPTY || val == NULL) { + val = Py_NewRef(default_value); + } + return val; +#else if (ix == DKIX_EMPTY || val == NULL) { val = default_value; } return Py_NewRef(val); +#endif } static int @@ -3881,7 +4250,7 @@ dict_popitem_impl(PyDictObject *self) return NULL; } /* Convert split table to combined table */ - if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { + if (_PyDict_HasSplitTable(self)) { if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { Py_DECREF(res); return NULL; @@ -4093,48 +4462,56 @@ static PyMethodDef mapp_methods[] = { }; static int -contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash) +contains_known_hash(PyDictObject *mp, PyObject *key, Py_ssize_t hash) { Py_ssize_t ix; PyObject *value; - ASSERT_DICT_LOCKED(mp); - +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return -1; - return (ix != DKIX_EMPTY && value != NULL); + + if (ix != DKIX_EMPTY && value != NULL) { +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif + return 1; + } + return 0; } -static int -contains_lock_held(PyDictObject *mp, PyObject *key) +/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ +int +PyDict_Contains(PyObject *op, PyObject *key) { Py_hash_t hash; Py_ssize_t ix; PyObject *value; - - ASSERT_DICT_LOCKED(mp); + PyDictObject *mp = (PyDictObject *)op; if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return -1; - return (ix != DKIX_EMPTY && value != NULL); -} - -/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ -int -PyDict_Contains(PyObject *op, PyObject *key) -{ - int res; - Py_BEGIN_CRITICAL_SECTION(op); - res = contains_lock_held((PyDictObject *)op, key); - Py_END_CRITICAL_SECTION(); - return res; + if (ix != DKIX_EMPTY && value != NULL) { +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif + return 1; + } + return 0; } int @@ -4157,10 +4534,20 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyObject *value; Py_ssize_t ix; +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return -1; - return (ix != DKIX_EMPTY && value != NULL); + if (ix != DKIX_EMPTY && value != NULL) { +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif + return 1; + } + return 0; } int @@ -4876,7 +5263,7 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iter_PyDict_Next(PyDictObject *d, PyObject *self) +dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; @@ -4983,7 +5370,7 @@ dictreviter_iternext(PyObject *self) PyObject *value; Py_BEGIN_CRITICAL_SECTION(d); - value = dictreviter_iter_PyDict_Next(d, self); + value = dictreviter_iter_lock_held(d, self); Py_END_CRITICAL_SECTION(); return value; @@ -6008,6 +6395,8 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { return false; } + ensure_shared_on_resize(dict); + assert(dict->ma_values); // We have an opportunity to do something *really* cool: dematerialize it! _PyDictKeys_DecRef(dict->ma_keys); @@ -6171,7 +6560,7 @@ _PyObject_FreeInstanceAttributes(PyObject *self) for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { Py_XDECREF(values->values[i]); } - free_values(values); + free_values(values, IS_DICT_SHARED((PyDictObject*)self)); } int @@ -6212,7 +6601,7 @@ PyObject_ClearManagedDict(PyObject *obj) Py_CLEAR(values->values[i]); } dorv_ptr->dict = NULL; - free_values(values); + free_values(values, IS_DICT_SHARED((PyDictObject*)obj)); } else { PyObject *dict = dorv_ptr->dict; @@ -6322,7 +6711,7 @@ void _PyDictKeys_DecRef(PyDictKeysObject *keys) { PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp,