From 9f9b3d085fd2d02d395bcb89ccc068b873b17c44 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 6 Mar 2023 13:27:58 -0800 Subject: [PATCH] ceval: fix some thread-safety issues --- Include/internal/pycore_list.h | 9 +++ Objects/cellobject.c | 10 ++++ Objects/listobject.c | 9 +-- Objects/longobject.c | 3 +- Python/bytecodes.c | 106 ++++++++++++++++++--------------- Python/ceval.c | 12 +++- Python/generated_cases.c.h | 104 +++++++++++++++++--------------- 7 files changed, 145 insertions(+), 108 deletions(-) diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 381bbc2451f..dc88305a151 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "listobject.h" // _PyList_CAST() +#include // offsetof() /* runtime lifecycle */ @@ -59,6 +60,14 @@ _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) return _PyList_AppendTakeRefListResize(self, newitem); } +static inline Py_ssize_t +_PyList_Capacity(PyObject **ob_item) +{ + char *mem = (char *)ob_item; + _PyListArray *arr = (_PyListArray *)(mem - offsetof(_PyListArray, ob_item)); + return arr->allocated; +} + // Repeat the bytes of a buffer in place static inline void _Py_memory_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) diff --git a/Objects/cellobject.c b/Objects/cellobject.c index f516707f6f8..bf153849c8a 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -11,6 +11,10 @@ PyCell_New(PyObject *obj) op = (PyCellObject *)PyObject_GC_New(PyCellObject, &PyCell_Type); if (op == NULL) return NULL; + + if (obj) { + _PyObject_SetMaybeWeakref(obj); + } op->ob_ref = Py_XNewRef(obj); _PyObject_GC_TRACK(op); @@ -67,6 +71,9 @@ PyCell_Set(PyObject *op, PyObject *value) return -1; } PyObject *old_value = PyCell_GET(op); + if (value) { + _PyObject_SetMaybeWeakref(value); + } PyCell_SET(op, Py_XNewRef(value)); Py_XDECREF(old_value); return 0; @@ -139,6 +146,9 @@ cell_get_contents(PyCellObject *op, void *closure) static int cell_set_contents(PyCellObject *op, PyObject *obj, void *Py_UNUSED(ignored)) { + if (obj) { + _PyObject_SetMaybeWeakref(obj); + } Py_XSETREF(op->ob_ref, Py_XNewRef(obj)); return 0; } diff --git a/Objects/listobject.c b/Objects/listobject.c index 052e8eb53f5..a840c72aa5e 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -301,13 +301,6 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) return (size_t) i < (size_t) limit; } -static Py_ssize_t -list_capacity(PyObject **ob_item) -{ - _PyListArray *arr = list_array(ob_item); - return _Py_atomic_load_ssize_relaxed(&arr->allocated); -} - Py_NO_INLINE static PyObject * list_item_locked(PyListObject *self, Py_ssize_t idx, PyObject *dead) { @@ -349,7 +342,7 @@ list_fetch_item(PyListObject *self, Py_ssize_t idx) if (ob_item == NULL) { return NULL; } - if (!valid_index(idx, list_capacity(ob_item))) { + if (!valid_index(idx, _PyList_Capacity(ob_item))) { return NULL; } PyObject *item = _Py_atomic_load_ptr(&ob_item[idx]); diff --git a/Objects/longobject.c b/Objects/longobject.c index 06b32131717..ee57e6cc806 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -62,6 +62,7 @@ get_small_int(sdigit ival) { assert(IS_SMALL_INT(ival)); PyObject *v = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + ival]; + assert(_PyObject_IS_IMMORTAL(v)); return v; } @@ -277,7 +278,7 @@ _PyLong_AssignValue(PyObject **target, Py_ssize_t value) return 0; } else if (old != NULL && PyLong_CheckExact(old) && - Py_REFCNT(old) == 1 && Py_SIZE(old) == 1 && + _PyObject_HasLocalRefcnt(old, 1) && Py_SIZE(old) == 1 && (size_t)value <= PyLong_MASK) { // Mutate in place if there are no other references the old diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 66fa7ac55a1..98fbe5ea287 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -11,6 +11,7 @@ #include "pycore_call.h" // _PyObject_FastCallDictTstate() #include "pycore_ceval.h" // _PyEval_SignalAsyncExc() #include "pycore_code.h" +#include "pycore_critical_section.h" #include "pycore_function.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() @@ -65,6 +66,7 @@ do { \ /* Flow control macros */ #define DEOPT_IF(cond, instname) ((void)0) +#define DEOPT_UNLOCK_IF(cond, instname) ((void)0) #define ERROR_IF(cond, labelname) ((void)0) #define JUMPBY(offset) ((void)0) #define GO_TO_INSTRUCTION(instname) ((void)0) @@ -422,7 +424,7 @@ dummy_func( assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); + res = PyDict_FetchItemWithError(dict, sub); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); @@ -431,7 +433,6 @@ dummy_func( Py_DECREF(sub); ERROR_IF(true, error); } - Py_INCREF(res); // Do this before DECREF'ing dict, sub DECREF_INPUTS(); } @@ -1223,15 +1224,14 @@ dummy_func( } inst(DELETE_DEREF, (--)) { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyObject *oldobj = _Py_atomic_exchange_ptr(&cell->ob_ref, NULL); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. if (oldobj == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); goto error; } - PyCell_SET(cell, NULL); Py_DECREF(oldobj); } @@ -1270,19 +1270,18 @@ dummy_func( } inst(LOAD_DEREF, ( -- value)) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = _Py_XFetchRef(&cell->ob_ref); if (value == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); ERROR_IF(true, error); } - Py_INCREF(value); } inst(STORE_DEREF, (v --)) { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + _PyObject_SetMaybeWeakref(v); + PyObject *oldobj = _Py_atomic_exchange_ptr(&cell->ob_ref, v); Py_XDECREF(oldobj); } @@ -1566,12 +1565,12 @@ dummy_func( DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_dictoffset < 0); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - res = _PyDictOrValues_GetValues(dorv)->values[cache->index]; + PyDictValues *dv = _PyDictOrValues_GetValues(dorv); + res = _Py_TryXFetchRef(&dv->values[cache->index]); DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); SET_TOP(NULL); STACK_GROW((oparg & 1)); SET_TOP(res); @@ -1588,15 +1587,14 @@ dummy_func( DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->version), - LOAD_ATTR); - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(cache->index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + cache->index; - res = ep->me_value; + PyDictKeysObject *keys = _Py_atomic_load_ptr_relaxed(&dict->ma_keys); + DEOPT_IF(keys->dk_version != read_u32(cache->version), LOAD_ATTR); + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(cache->index < keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + cache->index; + res = _Py_TryXFetchRef(&ep->me_value); DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); SET_TOP(NULL); STACK_GROW((oparg & 1)); SET_TOP(res); @@ -1615,7 +1613,7 @@ dummy_func( assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, LOAD_ATTR); @@ -1758,18 +1756,17 @@ dummy_func( assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(owner); + PyDictValues *values = _PyDictValues_Lock(dorv_ptr); + DEOPT_IF(values == NULL, STORE_ATTR); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; - values->values[index] = value; + _Py_atomic_store_ptr_release(&values->values[index], value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + _PyDictValues_Unlock(dorv_ptr); + Py_XDECREF(old_value); Py_DECREF(owner); } @@ -1779,30 +1776,31 @@ dummy_func( assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + Py_BEGIN_CRITICAL_SECTION(&dict->ma_mutex); + DEOPT_UNLOCK_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_UNLOCK_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_UNLOCK_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + _Py_atomic_store_ptr_relaxed(&ep->me_value, value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_UNLOCK_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_UNLOCK_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + _Py_atomic_store_ptr_relaxed(&ep->me_value, value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); @@ -1812,6 +1810,7 @@ dummy_func( } /* PEP 509 */ dict->ma_version_tag = new_version; + Py_END_CRITICAL_SECTION; Py_DECREF(owner); } @@ -2287,15 +2286,19 @@ dummy_func( DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if (it->it_index >= 0) { - if (it->it_index < PyList_GET_SIZE(seq)) { - PyObject *next = PyList_GET_ITEM(seq, it->it_index++); - PUSH(Py_NewRef(next)); - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); - goto end_for_iter_list; // End of this instruction - } - it->it_index = -1; + Py_ssize_t index = _Py_atomic_load_ssize_relaxed(&it->it_index); + Py_ssize_t size = _Py_atomic_load_ssize_relaxed(&((PyVarObject *)seq)->ob_size); + if ((size_t)index < (size_t)size) { + PyObject **ob_item = _Py_atomic_load_ptr_relaxed(&seq->ob_item); + DEOPT_IF(index >= _PyList_Capacity(ob_item), FOR_ITER); + PyObject *next = _Py_TryXFetchRef(&ob_item[index]); + DEOPT_IF(next == NULL, FOR_ITER); + _Py_atomic_store_ssize_relaxed(&it->it_index, index + 1); + PUSH(next); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); + goto end_for_iter_list; // End of this instruction } + _Py_atomic_store_ssize_relaxed(&it->it_index, -1); STACK_SHRINK(1); Py_DECREF(it); JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); @@ -2491,7 +2494,7 @@ dummy_func( assert(type_version != 0); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + PyDictOrValues dorv = _PyObject_DictOrValues(self); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != @@ -2976,11 +2979,16 @@ dummy_func( PyObject *callable = PEEK(3); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - PyObject *list = SECOND(); - DEOPT_IF(!PyList_Check(list), CALL); + PyObject *self = SECOND(); + DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); PyObject *arg = POP(); - if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { + PyListObject *list = (PyListObject *)self; + int err; + Py_BEGIN_CRITICAL_SECTION(&list->mutex); + err = _PyList_AppendTakeRef(list, arg); + Py_END_CRITICAL_SECTION; + if (err < 0) { goto error; } STACK_SHRINK(2); diff --git a/Python/ceval.c b/Python/ceval.c index 94e757c6f52..aa6f0bbc293 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -12,6 +12,7 @@ #include "pycore_call.h" // _PyObject_FastCallDictTstate() #include "pycore_ceval.h" // _PyEval_SignalAsyncExc() #include "pycore_code.h" +#include "pycore_critical_section.h" #include "pycore_function.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() @@ -48,7 +49,7 @@ # error "ceval.c must be build with Py_BUILD_CORE define for best performance" #endif -#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) +#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(_Py_THREAD_SANITIZER) // GH-89279: The MSVC compiler does not inline these static inline functions // in PGO build in _PyEval_EvalFrameDefault(), because this function is over // the limit of PGO, and that limit cannot be configured. @@ -888,6 +889,15 @@ GETITEM(PyObject *v, Py_ssize_t i) { GO_TO_INSTRUCTION(INSTNAME); \ } +#define DEOPT_UNLOCK_IF(COND, INSTNAME) \ + if ((COND)) { \ + /* This is only a single jump on release builds! */ \ + UPDATE_MISS_STATS((INSTNAME)); \ + assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ + _Py_critical_section_end(&_cs); \ + GO_TO_INSTRUCTION(INSTNAME); \ + } + #define GLOBALS() frame->f_globals #define BUILTINS() frame->f_builtins diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b2860c2d58b..9639baebe6f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -529,7 +529,7 @@ assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyDict_GetItemWithError(dict, sub); + res = PyDict_FetchItemWithError(dict, sub); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); @@ -538,7 +538,6 @@ Py_DECREF(sub); if (true) goto pop_2_error; } - Py_INCREF(res); // Do this before DECREF'ing dict, sub Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -1420,15 +1419,14 @@ } TARGET(DELETE_DEREF) { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyObject *oldobj = _Py_atomic_exchange_ptr(&cell->ob_ref, NULL); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. if (oldobj == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); goto error; } - PyCell_SET(cell, NULL); Py_DECREF(oldobj); DISPATCH(); } @@ -1473,13 +1471,12 @@ TARGET(LOAD_DEREF) { PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = _Py_XFetchRef(&cell->ob_ref); if (value == NULL) { format_exc_unbound(tstate, frame->f_code, oparg); if (true) goto error; } - Py_INCREF(value); STACK_GROW(1); POKE(1, value); DISPATCH(); @@ -1487,9 +1484,9 @@ TARGET(STORE_DEREF) { PyObject *v = PEEK(1); - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + _PyObject_SetMaybeWeakref(v); + PyObject *oldobj = _Py_atomic_exchange_ptr(&cell->ob_ref, v); Py_XDECREF(oldobj); STACK_SHRINK(1); DISPATCH(); @@ -1793,12 +1790,12 @@ DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_dictoffset < 0); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - res = _PyDictOrValues_GetValues(dorv)->values[cache->index]; + PyDictValues *dv = _PyDictOrValues_GetValues(dorv); + res = _Py_TryXFetchRef(&dv->values[cache->index]); DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); SET_TOP(NULL); STACK_GROW((oparg & 1)); SET_TOP(res); @@ -1815,15 +1812,14 @@ DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->version), - LOAD_ATTR); - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(cache->index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + cache->index; - res = ep->me_value; + PyDictKeysObject *keys = _Py_atomic_load_ptr_relaxed(&dict->ma_keys); + DEOPT_IF(keys->dk_version != read_u32(cache->version), LOAD_ATTR); + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(cache->index < keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + cache->index; + res = _Py_TryXFetchRef(&ep->me_value); DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); SET_TOP(NULL); STACK_GROW((oparg & 1)); SET_TOP(res); @@ -1842,7 +1838,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, LOAD_ATTR); @@ -1988,18 +1984,17 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(owner); + PyDictValues *values = _PyDictValues_Lock(dorv_ptr); + DEOPT_IF(values == NULL, STORE_ATTR); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; - values->values[index] = value; + _Py_atomic_store_ptr_release(&values->values[index], value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + _PyDictValues_Unlock(dorv_ptr); + Py_XDECREF(old_value); Py_DECREF(owner); STACK_SHRINK(2); JUMPBY(4); @@ -2016,30 +2011,31 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(names, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + Py_BEGIN_CRITICAL_SECTION(&dict->ma_mutex); + DEOPT_UNLOCK_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_UNLOCK_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_UNLOCK_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + _Py_atomic_store_ptr_relaxed(&ep->me_value, value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_UNLOCK_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_UNLOCK_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + _Py_atomic_store_ptr_relaxed(&ep->me_value, value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); @@ -2049,6 +2045,7 @@ } /* PEP 509 */ dict->ma_version_tag = new_version; + Py_END_CRITICAL_SECTION; Py_DECREF(owner); STACK_SHRINK(2); JUMPBY(4); @@ -2619,15 +2616,19 @@ DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if (it->it_index >= 0) { - if (it->it_index < PyList_GET_SIZE(seq)) { - PyObject *next = PyList_GET_ITEM(seq, it->it_index++); - PUSH(Py_NewRef(next)); - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); - goto end_for_iter_list; // End of this instruction - } - it->it_index = -1; + Py_ssize_t index = _Py_atomic_load_ssize_relaxed(&it->it_index); + Py_ssize_t size = _Py_atomic_load_ssize_relaxed(&((PyVarObject *)seq)->ob_size); + if ((size_t)index < (size_t)size) { + PyObject **ob_item = _Py_atomic_load_ptr_relaxed(&seq->ob_item); + DEOPT_IF(index >= _PyList_Capacity(ob_item), FOR_ITER); + PyObject *next = _Py_TryXFetchRef(&ob_item[index]); + DEOPT_IF(next == NULL, FOR_ITER); + _Py_atomic_store_ssize_relaxed(&it->it_index, index + 1); + PUSH(next); + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER); + goto end_for_iter_list; // End of this instruction } + _Py_atomic_store_ssize_relaxed(&it->it_index, -1); STACK_SHRINK(1); Py_DECREF(it); JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); @@ -2829,7 +2830,7 @@ assert(type_version != 0); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + PyDictOrValues dorv = _PyObject_DictOrValues(self); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != @@ -3313,11 +3314,16 @@ PyObject *callable = PEEK(3); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - PyObject *list = SECOND(); - DEOPT_IF(!PyList_Check(list), CALL); + PyObject *self = SECOND(); + DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); PyObject *arg = POP(); - if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { + PyListObject *list = (PyListObject *)self; + int err; + Py_BEGIN_CRITICAL_SECTION(&list->mutex); + err = _PyList_AppendTakeRef(list, arg); + Py_END_CRITICAL_SECTION; + if (err < 0) { goto error; } STACK_SHRINK(2);