From 63b68b5531f769a69892b2d68c1fce56593fa5e1 Mon Sep 17 00:00:00 2001 From: Hai Shi Date: Mon, 11 May 2020 00:19:55 +0800 Subject: [PATCH 1/2] test --- Modules/_functoolsmodule.c | 265 +++++++++++++++++++------------------ 1 file changed, 135 insertions(+), 130 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index d158d3bae157b2..c73d7a9dcbc471 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -22,7 +22,20 @@ typedef struct { vectorcallfunc vectorcall; } partialobject; -static PyTypeObject partial_type; +typedef struct _functools_state { + /* this object is used delimit args and keywords in the cache keys */ + PyObject *kwd_mark; + PyTypeObject *partial_type; + PyTypeObject *lru_cache_type; +} _functools_state; + +static inline _functools_state* +get_functools_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (_functools_state *)state; +} static void partial_setvectorcall(partialobject *pto); @@ -40,7 +53,10 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pargs = pkw = NULL; func = PyTuple_GET_ITEM(args, 0); - if (Py_IS_TYPE(func, &partial_type) && type == &partial_type) { + PyObject *module = PyType_GetModule(type); + _functools_state *state = get_functools_state(module); + if (Py_IS_TYPE(func, state->partial_type) + && type == state->partial_type) { partialobject *part = (partialobject *)func; if (part->dict == NULL) { pargs = part->args; @@ -292,6 +308,12 @@ static PyMemberDef partial_memberlist[] = { "tuple of arguments to future partial calls"}, {"keywords", T_OBJECT, OFF(kw), READONLY, "dictionary of keyword arguments to future partial calls"}, + {"__weaklistoffset__", T_PYSSIZET, + offsetof(partialobject, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, + offsetof(partialobject, dict), READONLY}, + {"__vectorcalloffset__", T_PYSSIZET, + offsetof(partialobject, vectorcall), READONLY}, {NULL} /* Sentinel */ }; @@ -418,49 +440,28 @@ static PyMethodDef partial_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyTypeObject partial_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "functools.partial", /* tp_name */ - sizeof(partialobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)partial_dealloc, /* tp_dealloc */ - offsetof(partialobject, vectorcall),/* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)partial_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)partial_call, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */ - partial_doc, /* tp_doc */ - (traverseproc)partial_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(partialobject, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - partial_methods, /* tp_methods */ - partial_memberlist, /* tp_members */ - partial_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(partialobject, dict), /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - partial_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ +static PyType_Slot partial_type_slots[] = { + {Py_tp_dealloc, partial_dealloc}, + {Py_tp_repr, partial_repr}, + {Py_tp_call, partial_call}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_setattro, PyObject_GenericSetAttr}, + {Py_tp_doc, (void *)partial_doc}, + {Py_tp_traverse, partial_traverse}, + {Py_tp_methods, partial_methods}, + {Py_tp_members, partial_memberlist}, + {Py_tp_getset, partial_getsetlist}, + {Py_tp_new, partial_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, 0} +}; + +static PyType_Spec partial_type_spec = { + .name = "functools.partial", + .basicsize = sizeof(partialobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VECTORCALL, + .slots = partial_type_slots }; @@ -723,10 +724,6 @@ sequence is empty."); */ - -/* this object is used delimit args and keywords in the cache keys */ -static PyObject *kwd_mark = NULL; - struct lru_list_elem; struct lru_cache_object; @@ -786,10 +783,8 @@ typedef struct lru_cache_object { PyObject *weakreflist; } lru_cache_object; -static PyTypeObject lru_cache_type; - static PyObject * -lru_cache_make_key(PyObject *args, PyObject *kwds, int typed) +lru_cache_make_key(PyObject *module, PyObject *args, PyObject *kwds, int typed) { PyObject *key, *keyword, *value; Py_ssize_t key_size, pos, key_pos, kwds_size; @@ -827,9 +822,10 @@ lru_cache_make_key(PyObject *args, PyObject *kwds, int typed) Py_INCREF(item); PyTuple_SET_ITEM(key, key_pos++, item); } + _functools_state *state = get_functools_state(module); if (kwds_size) { - Py_INCREF(kwd_mark); - PyTuple_SET_ITEM(key, key_pos++, kwd_mark); + Py_INCREF(state->kwd_mark); + PyTuple_SET_ITEM(key, key_pos++, state->kwd_mark); for (pos = 0; PyDict_Next(kwds, &pos, &keyword, &value);) { Py_INCREF(keyword); PyTuple_SET_ITEM(key, key_pos++, keyword); @@ -868,12 +864,19 @@ uncached_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd return result; } +static inline PyObject* +lru_cache_object_get_module(lru_cache_object *self) +{ + return PyType_GetModule(Py_TYPE(self)); +} + static PyObject * infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds) { - PyObject *result; + PyObject *result, *module; Py_hash_t hash; - PyObject *key = lru_cache_make_key(args, kwds, self->typed); + module = lru_cache_object_get_module(self); + PyObject *key = lru_cache_make_key(module, args, kwds, self->typed); if (!key) return NULL; hash = PyObject_Hash(key); @@ -971,10 +974,11 @@ static PyObject * bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds) { lru_list_elem *link; - PyObject *key, *result, *testresult; + PyObject *key, *result, *testresult, *module; Py_hash_t hash; - key = lru_cache_make_key(args, kwds, self->typed); + module = lru_cache_object_get_module(self); + key = lru_cache_make_key(module, args, kwds, self->typed); if (!key) return NULL; hash = PyObject_Hash(key); @@ -1360,51 +1364,27 @@ static PyGetSetDef lru_cache_getsetlist[] = { {NULL} }; -static PyTypeObject lru_cache_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "functools._lru_cache_wrapper", /* tp_name */ - sizeof(lru_cache_object), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)lru_cache_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)lru_cache_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_METHOD_DESCRIPTOR, - /* tp_flags */ - lru_cache_doc, /* tp_doc */ - (traverseproc)lru_cache_tp_traverse,/* tp_traverse */ - (inquiry)lru_cache_tp_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(lru_cache_object, weakreflist), - /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - lru_cache_methods, /* tp_methods */ - 0, /* tp_members */ - lru_cache_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - lru_cache_descr_get, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(lru_cache_object, dict), /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - lru_cache_new, /* tp_new */ +static PyType_Slot lru_cache_type_slots[] = { + {Py_tp_dealloc, lru_cache_dealloc}, + {Py_tp_call, lru_cache_call}, + {Py_tp_doc, (void *)lru_cache_doc}, + {Py_tp_traverse, lru_cache_tp_traverse}, + {Py_tp_clear, lru_cache_tp_clear}, + {Py_tp_methods, lru_cache_methods}, + {Py_tp_getset, lru_cache_getsetlist}, + {Py_tp_descr_get, lru_cache_descr_get}, + {Py_tp_new, lru_cache_new} +}; + +static PyType_Spec lru_cache_type_spec = { + .name = "functools._lru_cache_wrapper", + .basicsize = sizeof(lru_cache_object), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_METHOD_DESCRIPTOR, + .slots = lru_cache_type_slots }; + /* module level code ********************************************************/ PyDoc_STRVAR(_functools_doc, @@ -1417,38 +1397,64 @@ static PyMethodDef _functools_methods[] = { {NULL, NULL} /* sentinel */ }; -static void -_functools_free(void *m) -{ - // FIXME: Do not clear kwd_mark to avoid NULL pointer dereferencing if we have - // other modules instances that could use it. Will fix when PEP-573 land - // and we could move kwd_mark to a per-module state. - // Py_CLEAR(kwd_mark); -} - static int _functools_exec(PyObject *module) { - PyTypeObject *typelist[] = { - &partial_type, - &lru_cache_type - }; - - if (!kwd_mark) { - kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); - if (!kwd_mark) { + _functools_state *state = get_functools_state(module); + if (state->kwd_mark == NULL) { + state->kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); + if (state->kwd_mark == NULL) { return -1; } } - for (size_t i = 0; i < Py_ARRAY_LENGTH(typelist); i++) { - if (PyModule_AddType(module, typelist[i]) < 0) { - return -1; - } + PyTypeObject *partial_type = + (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); + if (partial_type == NULL) { + return -1; } + if (PyModule_AddType(module, partial_type) < 0) { + return -1; + } + state->partial_type = partial_type; + + PyTypeObject *lru_cache_type = + (PyTypeObject *)PyType_FromModuleAndSpec(module, &lru_cache_type_spec, NULL); + if (lru_cache_type == NULL) { + return -1; + } + if (PyModule_AddType(module, lru_cache_type) < 0) { + return -1; + } + state->lru_cache_type = lru_cache_type; + return 0; } +static int +_functools_traverse(PyObject *module, visitproc visit, void *arg) +{ + Py_VISIT(get_functools_state(module)->kwd_mark); + Py_VISIT(get_functools_state(module)->partial_type); + Py_VISIT(get_functools_state(module)->lru_cache_type); + return 0; +} + +static int +_functools_clear(PyObject *module) +{ + Py_CLEAR(get_functools_state(module)->kwd_mark); + Py_CLEAR(get_functools_state(module)->partial_type); + Py_CLEAR(get_functools_state(module)->lru_cache_type); + return 0; +} + +static void +_functools_free(void *module) +{ + _functools_clear((PyObject *)module); +} + static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, {0, NULL} @@ -1456,14 +1462,13 @@ static struct PyModuleDef_Slot _functools_slots[] = { static struct PyModuleDef _functools_module = { PyModuleDef_HEAD_INIT, - "_functools", - _functools_doc, - 0, - _functools_methods, - _functools_slots, - NULL, - NULL, - _functools_free, + .m_name = "_functools", + .m_doc = _functools_doc, + .m_methods = _functools_methods, + .m_slots = _functools_slots, + .m_traverse = _functools_traverse, + .m_clear = _functools_clear, + .m_free = _functools_free, }; PyMODINIT_FUNC From 2ddffb00fc16261fdbb0a83f07d6912f67fae756 Mon Sep 17 00:00:00 2001 From: Hai Shi Date: Tue, 12 May 2020 23:47:10 +0800 Subject: [PATCH 2/2] update pep573 in functools extension module --- Modules/_functoolsmodule.c | 41 ++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index c73d7a9dcbc471..f03fe55b2c83d8 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -51,10 +51,14 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pargs = pkw = NULL; - func = PyTuple_GET_ITEM(args, 0); PyObject *module = PyType_GetModule(type); + if (module == NULL) { + return NULL; + } _functools_state *state = get_functools_state(module); + + pargs = pkw = NULL; + func = PyTuple_GET_ITEM(args, 0); if (Py_IS_TYPE(func, state->partial_type) && type == state->partial_type) { partialobject *part = (partialobject *)func; @@ -1364,6 +1368,14 @@ static PyGetSetDef lru_cache_getsetlist[] = { {NULL} }; +static PyMemberDef lru_cache_memberlist[] = { + {"__dictoffset__", T_PYSSIZET, + offsetof(lru_cache_object, dict), READONLY}, + {"__weaklistoffset__", T_PYSSIZET, + offsetof(lru_cache_object, weakreflist), READONLY}, + {NULL} /* Sentinel */ +}; + static PyType_Slot lru_cache_type_slots[] = { {Py_tp_dealloc, lru_cache_dealloc}, {Py_tp_call, lru_cache_call}, @@ -1371,9 +1383,11 @@ static PyType_Slot lru_cache_type_slots[] = { {Py_tp_traverse, lru_cache_tp_traverse}, {Py_tp_clear, lru_cache_tp_clear}, {Py_tp_methods, lru_cache_methods}, + {Py_tp_members, lru_cache_memberlist}, {Py_tp_getset, lru_cache_getsetlist}, {Py_tp_descr_get, lru_cache_descr_get}, - {Py_tp_new, lru_cache_new} + {Py_tp_new, lru_cache_new}, + {0, 0} }; static PyType_Spec lru_cache_type_spec = { @@ -1401,11 +1415,9 @@ static int _functools_exec(PyObject *module) { _functools_state *state = get_functools_state(module); + state->kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); if (state->kwd_mark == NULL) { - state->kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type); - if (state->kwd_mark == NULL) { - return -1; - } + return -1; } PyTypeObject *partial_type = @@ -1434,18 +1446,20 @@ _functools_exec(PyObject *module) static int _functools_traverse(PyObject *module, visitproc visit, void *arg) { - Py_VISIT(get_functools_state(module)->kwd_mark); - Py_VISIT(get_functools_state(module)->partial_type); - Py_VISIT(get_functools_state(module)->lru_cache_type); + _functools_state *state = get_functools_state(module); + Py_VISIT(state->kwd_mark); + Py_VISIT(state->partial_type); + Py_VISIT(state->lru_cache_type); return 0; } static int _functools_clear(PyObject *module) { - Py_CLEAR(get_functools_state(module)->kwd_mark); - Py_CLEAR(get_functools_state(module)->partial_type); - Py_CLEAR(get_functools_state(module)->lru_cache_type); + _functools_state *state = get_functools_state(module); + Py_CLEAR(state->kwd_mark); + Py_CLEAR(state->partial_type); + Py_CLEAR(state->lru_cache_type); return 0; } @@ -1464,6 +1478,7 @@ static struct PyModuleDef _functools_module = { PyModuleDef_HEAD_INIT, .m_name = "_functools", .m_doc = _functools_doc, + .m_size = sizeof(_functools_state), .m_methods = _functools_methods, .m_slots = _functools_slots, .m_traverse = _functools_traverse,