Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] bpo-40137: Use pep573 in functools extension module. #20055

Closed
wants to merge 2 commits into from
Closed

[WIP] bpo-40137: Use pep573 in functools extension module. #20055

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
63b68b5
test
shihai1991 May 10, 2020
2ddffb0
update pep573 in functools extension module
shihai1991 May 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
284 changes: 152 additions & 132 deletions Modules/_functoolsmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,20 @@ typedef struct {
vectorcallfunc vectorcall;
} partialobject;

static PyTypeObject partial_type;
typedef struct _functools_state {
/* this object is used delimit args and keywords in the cache keys */
PyObject *kwd_mark;
PyTypeObject *partial_type;
PyTypeObject *lru_cache_type;
} _functools_state;

static inline _functools_state*
get_functools_state(PyObject *module)
{
void *state = PyModule_GetState(module);
assert(state != NULL);
return (_functools_state *)state;
}

static void partial_setvectorcall(partialobject *pto);

Expand All @@ -38,9 +51,16 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
return NULL;
}

PyObject *module = PyType_GetModule(type);
encukou marked this conversation as resolved.
Show resolved Hide resolved
if (module == NULL) {
return NULL;
}
_functools_state *state = get_functools_state(module);

pargs = pkw = NULL;
func = PyTuple_GET_ITEM(args, 0);
if (Py_IS_TYPE(func, &partial_type) && type == &partial_type) {
if (Py_IS_TYPE(func, state->partial_type)
&& type == state->partial_type) {
partialobject *part = (partialobject *)func;
if (part->dict == NULL) {
pargs = part->args;
Expand Down Expand Up @@ -292,6 +312,12 @@ static PyMemberDef partial_memberlist[] = {
"tuple of arguments to future partial calls"},
{"keywords", T_OBJECT, OFF(kw), READONLY,
"dictionary of keyword arguments to future partial calls"},
{"__weaklistoffset__", T_PYSSIZET,
offsetof(partialobject, weakreflist), READONLY},
{"__dictoffset__", T_PYSSIZET,
offsetof(partialobject, dict), READONLY},
{"__vectorcalloffset__", T_PYSSIZET,
offsetof(partialobject, vectorcall), READONLY},
{NULL} /* Sentinel */
};

Expand Down Expand Up @@ -418,49 +444,28 @@ static PyMethodDef partial_methods[] = {
{NULL, NULL} /* sentinel */
};

static PyTypeObject partial_type = {
PyVarObject_HEAD_INIT(NULL, 0)
"functools.partial", /* tp_name */
sizeof(partialobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)partial_dealloc, /* tp_dealloc */
offsetof(partialobject, vectorcall),/* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
(reprfunc)partial_repr, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
(ternaryfunc)partial_call, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
PyObject_GenericSetAttr, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */
partial_doc, /* tp_doc */
(traverseproc)partial_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
offsetof(partialobject, weakreflist), /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
partial_methods, /* tp_methods */
partial_memberlist, /* tp_members */
partial_getsetlist, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
offsetof(partialobject, dict), /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
partial_new, /* tp_new */
PyObject_GC_Del, /* tp_free */
static PyType_Slot partial_type_slots[] = {
{Py_tp_dealloc, partial_dealloc},
{Py_tp_repr, partial_repr},
{Py_tp_call, partial_call},
{Py_tp_getattro, PyObject_GenericGetAttr},
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_doc, (void *)partial_doc},
{Py_tp_traverse, partial_traverse},
{Py_tp_methods, partial_methods},
{Py_tp_members, partial_memberlist},
{Py_tp_getset, partial_getsetlist},
{Py_tp_new, partial_new},
{Py_tp_free, PyObject_GC_Del},
{0, 0}
};

static PyType_Spec partial_type_spec = {
.name = "functools.partial",
.basicsize = sizeof(partialobject),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VECTORCALL,
.slots = partial_type_slots
};


Expand Down Expand Up @@ -723,10 +728,6 @@ sequence is empty.");

*/


/* this object is used delimit args and keywords in the cache keys */
static PyObject *kwd_mark = NULL;

struct lru_list_elem;
struct lru_cache_object;

Expand Down Expand Up @@ -786,10 +787,8 @@ typedef struct lru_cache_object {
PyObject *weakreflist;
} lru_cache_object;

static PyTypeObject lru_cache_type;

static PyObject *
lru_cache_make_key(PyObject *args, PyObject *kwds, int typed)
lru_cache_make_key(PyObject *module, PyObject *args, PyObject *kwds, int typed)
{
PyObject *key, *keyword, *value;
Py_ssize_t key_size, pos, key_pos, kwds_size;
Expand Down Expand Up @@ -827,9 +826,10 @@ lru_cache_make_key(PyObject *args, PyObject *kwds, int typed)
Py_INCREF(item);
PyTuple_SET_ITEM(key, key_pos++, item);
}
_functools_state *state = get_functools_state(module);
if (kwds_size) {
Py_INCREF(kwd_mark);
PyTuple_SET_ITEM(key, key_pos++, kwd_mark);
Py_INCREF(state->kwd_mark);
PyTuple_SET_ITEM(key, key_pos++, state->kwd_mark);
for (pos = 0; PyDict_Next(kwds, &pos, &keyword, &value);) {
Py_INCREF(keyword);
PyTuple_SET_ITEM(key, key_pos++, keyword);
Expand Down Expand Up @@ -868,12 +868,19 @@ uncached_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd
return result;
}

static inline PyObject*
lru_cache_object_get_module(lru_cache_object *self)
{
return PyType_GetModule(Py_TYPE(self));
}

static PyObject *
infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds)
{
PyObject *result;
PyObject *result, *module;
Py_hash_t hash;
PyObject *key = lru_cache_make_key(args, kwds, self->typed);
module = lru_cache_object_get_module(self);
PyObject *key = lru_cache_make_key(module, args, kwds, self->typed);
if (!key)
return NULL;
hash = PyObject_Hash(key);
Expand Down Expand Up @@ -971,10 +978,11 @@ static PyObject *
bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds)
{
lru_list_elem *link;
PyObject *key, *result, *testresult;
PyObject *key, *result, *testresult, *module;
Py_hash_t hash;

key = lru_cache_make_key(args, kwds, self->typed);
module = lru_cache_object_get_module(self);
key = lru_cache_make_key(module, args, kwds, self->typed);
if (!key)
return NULL;
hash = PyObject_Hash(key);
Expand Down Expand Up @@ -1360,51 +1368,37 @@ static PyGetSetDef lru_cache_getsetlist[] = {
{NULL}
};

static PyTypeObject lru_cache_type = {
PyVarObject_HEAD_INIT(NULL, 0)
"functools._lru_cache_wrapper", /* tp_name */
sizeof(lru_cache_object), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)lru_cache_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
(ternaryfunc)lru_cache_call, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_METHOD_DESCRIPTOR,
/* tp_flags */
lru_cache_doc, /* tp_doc */
(traverseproc)lru_cache_tp_traverse,/* tp_traverse */
(inquiry)lru_cache_tp_clear, /* tp_clear */
0, /* tp_richcompare */
offsetof(lru_cache_object, weakreflist),
/* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
lru_cache_methods, /* tp_methods */
0, /* tp_members */
lru_cache_getsetlist, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
lru_cache_descr_get, /* tp_descr_get */
0, /* tp_descr_set */
offsetof(lru_cache_object, dict), /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
lru_cache_new, /* tp_new */
static PyMemberDef lru_cache_memberlist[] = {
{"__dictoffset__", T_PYSSIZET,
offsetof(lru_cache_object, dict), READONLY},
{"__weaklistoffset__", T_PYSSIZET,
offsetof(lru_cache_object, weakreflist), READONLY},
{NULL} /* Sentinel */
};

static PyType_Slot lru_cache_type_slots[] = {
{Py_tp_dealloc, lru_cache_dealloc},
{Py_tp_call, lru_cache_call},
{Py_tp_doc, (void *)lru_cache_doc},
{Py_tp_traverse, lru_cache_tp_traverse},
{Py_tp_clear, lru_cache_tp_clear},
{Py_tp_methods, lru_cache_methods},
{Py_tp_members, lru_cache_memberlist},
{Py_tp_getset, lru_cache_getsetlist},
{Py_tp_descr_get, lru_cache_descr_get},
{Py_tp_new, lru_cache_new},
{0, 0}
};

static PyType_Spec lru_cache_type_spec = {
.name = "functools._lru_cache_wrapper",
.basicsize = sizeof(lru_cache_object),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_METHOD_DESCRIPTOR,
.slots = lru_cache_type_slots
};


/* module level code ********************************************************/

PyDoc_STRVAR(_functools_doc,
Expand All @@ -1417,53 +1411,79 @@ static PyMethodDef _functools_methods[] = {
{NULL, NULL} /* sentinel */
};

static void
_functools_free(void *m)
{
// FIXME: Do not clear kwd_mark to avoid NULL pointer dereferencing if we have
// other modules instances that could use it. Will fix when PEP-573 land
// and we could move kwd_mark to a per-module state.
// Py_CLEAR(kwd_mark);
}

static int
_functools_exec(PyObject *module)
{
PyTypeObject *typelist[] = {
&partial_type,
&lru_cache_type
};

if (!kwd_mark) {
kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type);
if (!kwd_mark) {
return -1;
}
_functools_state *state = get_functools_state(module);
state->kwd_mark = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type);
if (state->kwd_mark == NULL) {
shihai1991 marked this conversation as resolved.
Show resolved Hide resolved
return -1;
}

for (size_t i = 0; i < Py_ARRAY_LENGTH(typelist); i++) {
if (PyModule_AddType(module, typelist[i]) < 0) {
return -1;
}
PyTypeObject *partial_type =
(PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL);
if (partial_type == NULL) {
return -1;
}
if (PyModule_AddType(module, partial_type) < 0) {
return -1;
}
state->partial_type = partial_type;

PyTypeObject *lru_cache_type =
(PyTypeObject *)PyType_FromModuleAndSpec(module, &lru_cache_type_spec, NULL);
if (lru_cache_type == NULL) {
return -1;
}
if (PyModule_AddType(module, lru_cache_type) < 0) {
return -1;
}
state->lru_cache_type = lru_cache_type;

return 0;
}

static int
_functools_traverse(PyObject *module, visitproc visit, void *arg)
{
_functools_state *state = get_functools_state(module);
Py_VISIT(state->kwd_mark);
Py_VISIT(state->partial_type);
Py_VISIT(state->lru_cache_type);
return 0;
}

static int
_functools_clear(PyObject *module)
{
_functools_state *state = get_functools_state(module);
Py_CLEAR(state->kwd_mark);
Py_CLEAR(state->partial_type);
Py_CLEAR(state->lru_cache_type);
return 0;
}

static void
_functools_free(void *module)
{
_functools_clear((PyObject *)module);
}

static struct PyModuleDef_Slot _functools_slots[] = {
{Py_mod_exec, _functools_exec},
{0, NULL}
};

static struct PyModuleDef _functools_module = {
PyModuleDef_HEAD_INIT,
"_functools",
_functools_doc,
0,
_functools_methods,
_functools_slots,
NULL,
NULL,
_functools_free,
.m_name = "_functools",
.m_doc = _functools_doc,
.m_size = sizeof(_functools_state),
.m_methods = _functools_methods,
.m_slots = _functools_slots,
.m_traverse = _functools_traverse,
.m_clear = _functools_clear,
.m_free = _functools_free,
};

PyMODINIT_FUNC
Expand Down