diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index de93d4ef14de2a..f754de3706c812 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1212,21 +1212,21 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { _CALL_BUILTIN_O, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { _CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { _CALL_TUPLE_1, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 1209d736abe696..b569b80c5f110a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -37,44 +37,45 @@ extern "C" { #define _BUILD_TUPLE BUILD_TUPLE #define _CALL 311 #define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS -#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS -#define _CALL_BUILTIN_O CALL_BUILTIN_O +#define _CALL_BUILTIN_CLASS 312 +#define _CALL_BUILTIN_FAST 313 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 314 +#define _CALL_BUILTIN_O 315 #define _CALL_FUNCTION_EX CALL_FUNCTION_EX #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE #define _CALL_KW CALL_KW #define _CALL_LEN CALL_LEN -#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS -#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS -#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O +#define _CALL_METHOD_DESCRIPTOR_FAST 316 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 318 +#define _CALL_METHOD_DESCRIPTOR_O 319 #define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_STR_1 CALL_STR_1 -#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_STR_1 320 +#define _CALL_TUPLE_1 321 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 312 -#define _CHECK_ATTR_METHOD_LAZY_DICT 313 -#define _CHECK_ATTR_MODULE 314 -#define _CHECK_ATTR_WITH_HINT 315 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 316 +#define _CHECK_ATTR_CLASS 322 +#define _CHECK_ATTR_METHOD_LAZY_DICT 323 +#define _CHECK_ATTR_MODULE 324 +#define _CHECK_ATTR_WITH_HINT 325 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 326 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 317 -#define _CHECK_FUNCTION_EXACT_ARGS 318 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 -#define _CHECK_PEP_523 320 -#define _CHECK_STACK_SPACE 321 -#define _CHECK_VALIDITY 322 -#define _CHECK_VALIDITY_AND_SET_IP 323 -#define _COLD_EXIT 324 -#define _COMPARE_OP 325 -#define _COMPARE_OP_FLOAT 326 -#define _COMPARE_OP_INT 327 -#define _COMPARE_OP_STR 328 -#define _CONTAINS_OP 329 +#define _CHECK_FUNCTION 327 +#define _CHECK_FUNCTION_EXACT_ARGS 328 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 329 +#define _CHECK_PEP_523 330 +#define _CHECK_PERIODIC 331 +#define _CHECK_STACK_SPACE 332 +#define _CHECK_VALIDITY 333 +#define _CHECK_VALIDITY_AND_SET_IP 334 +#define _COLD_EXIT 335 +#define _COMPARE_OP 336 +#define _COMPARE_OP_FLOAT 337 +#define _COMPARE_OP_INT 338 +#define _COMPARE_OP_STR 339 +#define _CONTAINS_OP 340 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -90,41 +91,41 @@ extern "C" { #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 330 +#define _FATAL_ERROR 341 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 331 +#define _FOR_ITER 342 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 332 +#define _FOR_ITER_TIER_TWO 343 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 333 -#define _GUARD_BOTH_INT 334 -#define _GUARD_BOTH_UNICODE 335 -#define _GUARD_BUILTINS_VERSION 336 -#define _GUARD_DORV_VALUES 337 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 -#define _GUARD_GLOBALS_VERSION 339 -#define _GUARD_IS_FALSE_POP 340 -#define _GUARD_IS_NONE_POP 341 -#define _GUARD_IS_NOT_NONE_POP 342 -#define _GUARD_IS_TRUE_POP 343 -#define _GUARD_KEYS_VERSION 344 -#define _GUARD_NOT_EXHAUSTED_LIST 345 -#define _GUARD_NOT_EXHAUSTED_RANGE 346 -#define _GUARD_NOT_EXHAUSTED_TUPLE 347 -#define _GUARD_TYPE_VERSION 348 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 349 -#define _INIT_CALL_PY_EXACT_ARGS 350 -#define _INIT_CALL_PY_EXACT_ARGS_0 351 -#define _INIT_CALL_PY_EXACT_ARGS_1 352 -#define _INIT_CALL_PY_EXACT_ARGS_2 353 -#define _INIT_CALL_PY_EXACT_ARGS_3 354 -#define _INIT_CALL_PY_EXACT_ARGS_4 355 +#define _GUARD_BOTH_FLOAT 344 +#define _GUARD_BOTH_INT 345 +#define _GUARD_BOTH_UNICODE 346 +#define _GUARD_BUILTINS_VERSION 347 +#define _GUARD_DORV_VALUES 348 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 349 +#define _GUARD_GLOBALS_VERSION 350 +#define _GUARD_IS_FALSE_POP 351 +#define _GUARD_IS_NONE_POP 352 +#define _GUARD_IS_NOT_NONE_POP 353 +#define _GUARD_IS_TRUE_POP 354 +#define _GUARD_KEYS_VERSION 355 +#define _GUARD_NOT_EXHAUSTED_LIST 356 +#define _GUARD_NOT_EXHAUSTED_RANGE 357 +#define _GUARD_NOT_EXHAUSTED_TUPLE 358 +#define _GUARD_TYPE_VERSION 359 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 360 +#define _INIT_CALL_PY_EXACT_ARGS 361 +#define _INIT_CALL_PY_EXACT_ARGS_0 362 +#define _INIT_CALL_PY_EXACT_ARGS_1 363 +#define _INIT_CALL_PY_EXACT_ARGS_2 364 +#define _INIT_CALL_PY_EXACT_ARGS_3 365 +#define _INIT_CALL_PY_EXACT_ARGS_4 366 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -141,65 +142,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 356 -#define _IS_NONE 357 +#define _INTERNAL_INCREMENT_OPT_COUNTER 367 +#define _IS_NONE 368 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 358 -#define _ITER_CHECK_RANGE 359 -#define _ITER_CHECK_TUPLE 360 -#define _ITER_JUMP_LIST 361 -#define _ITER_JUMP_RANGE 362 -#define _ITER_JUMP_TUPLE 363 -#define _ITER_NEXT_LIST 364 -#define _ITER_NEXT_RANGE 365 -#define _ITER_NEXT_TUPLE 366 -#define _JUMP_TO_TOP 367 +#define _ITER_CHECK_LIST 369 +#define _ITER_CHECK_RANGE 370 +#define _ITER_CHECK_TUPLE 371 +#define _ITER_JUMP_LIST 372 +#define _ITER_JUMP_RANGE 373 +#define _ITER_JUMP_TUPLE 374 +#define _ITER_NEXT_LIST 375 +#define _ITER_NEXT_RANGE 376 +#define _ITER_NEXT_TUPLE 377 +#define _JUMP_TO_TOP 378 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 368 -#define _LOAD_ATTR_CLASS 369 -#define _LOAD_ATTR_CLASS_0 370 -#define _LOAD_ATTR_CLASS_1 371 +#define _LOAD_ATTR 379 +#define _LOAD_ATTR_CLASS 380 +#define _LOAD_ATTR_CLASS_0 381 +#define _LOAD_ATTR_CLASS_1 382 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 372 -#define _LOAD_ATTR_INSTANCE_VALUE_0 373 -#define _LOAD_ATTR_INSTANCE_VALUE_1 374 -#define _LOAD_ATTR_METHOD_LAZY_DICT 375 -#define _LOAD_ATTR_METHOD_NO_DICT 376 -#define _LOAD_ATTR_METHOD_WITH_VALUES 377 -#define _LOAD_ATTR_MODULE 378 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 379 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 380 +#define _LOAD_ATTR_INSTANCE_VALUE 383 +#define _LOAD_ATTR_INSTANCE_VALUE_0 384 +#define _LOAD_ATTR_INSTANCE_VALUE_1 385 +#define _LOAD_ATTR_METHOD_LAZY_DICT 386 +#define _LOAD_ATTR_METHOD_NO_DICT 387 +#define _LOAD_ATTR_METHOD_WITH_VALUES 388 +#define _LOAD_ATTR_MODULE 389 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 390 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 391 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 381 -#define _LOAD_ATTR_SLOT_0 382 -#define _LOAD_ATTR_SLOT_1 383 -#define _LOAD_ATTR_WITH_HINT 384 +#define _LOAD_ATTR_SLOT 392 +#define _LOAD_ATTR_SLOT_0 393 +#define _LOAD_ATTR_SLOT_1 394 +#define _LOAD_ATTR_WITH_HINT 395 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 385 -#define _LOAD_CONST_INLINE_BORROW 386 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 387 -#define _LOAD_CONST_INLINE_WITH_NULL 388 +#define _LOAD_CONST_INLINE 396 +#define _LOAD_CONST_INLINE_BORROW 397 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 398 +#define _LOAD_CONST_INLINE_WITH_NULL 399 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 389 -#define _LOAD_FAST_0 390 -#define _LOAD_FAST_1 391 -#define _LOAD_FAST_2 392 -#define _LOAD_FAST_3 393 -#define _LOAD_FAST_4 394 -#define _LOAD_FAST_5 395 -#define _LOAD_FAST_6 396 -#define _LOAD_FAST_7 397 +#define _LOAD_FAST 400 +#define _LOAD_FAST_0 401 +#define _LOAD_FAST_1 402 +#define _LOAD_FAST_2 403 +#define _LOAD_FAST_3 404 +#define _LOAD_FAST_4 405 +#define _LOAD_FAST_5 406 +#define _LOAD_FAST_6 407 +#define _LOAD_FAST_7 408 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 398 -#define _LOAD_GLOBAL_BUILTINS 399 -#define _LOAD_GLOBAL_MODULE 400 +#define _LOAD_GLOBAL 409 +#define _LOAD_GLOBAL_BUILTINS 410 +#define _LOAD_GLOBAL_MODULE 411 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -213,48 +214,48 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 401 -#define _POP_JUMP_IF_FALSE 402 -#define _POP_JUMP_IF_TRUE 403 +#define _POP_FRAME 412 +#define _POP_JUMP_IF_FALSE 413 +#define _POP_JUMP_IF_TRUE 414 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 404 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 415 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 405 +#define _PUSH_FRAME 416 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 406 +#define _REPLACE_WITH_TRUE 417 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 407 -#define _SEND 408 +#define _SAVE_RETURN_OFFSET 418 +#define _SEND 419 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 409 -#define _STORE_ATTR 410 -#define _STORE_ATTR_INSTANCE_VALUE 411 -#define _STORE_ATTR_SLOT 412 +#define _START_EXECUTOR 420 +#define _STORE_ATTR 421 +#define _STORE_ATTR_INSTANCE_VALUE 422 +#define _STORE_ATTR_SLOT 423 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 413 -#define _STORE_FAST_0 414 -#define _STORE_FAST_1 415 -#define _STORE_FAST_2 416 -#define _STORE_FAST_3 417 -#define _STORE_FAST_4 418 -#define _STORE_FAST_5 419 -#define _STORE_FAST_6 420 -#define _STORE_FAST_7 421 +#define _STORE_FAST 424 +#define _STORE_FAST_0 425 +#define _STORE_FAST_1 426 +#define _STORE_FAST_2 427 +#define _STORE_FAST_3 428 +#define _STORE_FAST_4 429 +#define _STORE_FAST_5 430 +#define _STORE_FAST_6 431 +#define _STORE_FAST_7 432 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 422 +#define _STORE_SUBSCR 433 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 423 +#define _TO_BOOL 434 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -264,12 +265,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 424 +#define _UNPACK_SEQUENCE 435 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 424 +#define MAX_UOP_ID 435 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 62405a362fd7ab..507bd27c01c553 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -188,6 +188,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, @@ -201,19 +202,19 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, @@ -301,6 +302,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index af2e2c8f52ee29..e8fcb6c6d0067c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3108,10 +3108,13 @@ dummy_func( Py_DECREF(args[i]); } ERROR_IF(res == NULL, error); + } + + op(_CHECK_PERIODIC, (--)) { CHECK_EVAL_BREAKER(); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL; + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { DEOPT_IF(null != NULL); @@ -3246,7 +3249,7 @@ dummy_func( Py_DECREF(arg); } - inst(CALL_STR_1, (unused/1, unused/2, callable, null, arg -- res)) { + op(_CALL_STR_1, (callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); @@ -3254,10 +3257,15 @@ dummy_func( res = PyObject_Str(arg); Py_DECREF(arg); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, arg -- res)) { + macro(CALL_STR_1) = + unused/1 + + unused/2 + + _CALL_STR_1 + + _CHECK_PERIODIC; + + op(_CALL_TUPLE_1, (callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type); @@ -3265,9 +3273,14 @@ dummy_func( res = PySequence_Tuple(arg); Py_DECREF(arg); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } + macro(CALL_TUPLE_1) = + unused/1 + + unused/2 + + _CALL_TUPLE_1 + + _CHECK_PERIODIC; + inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) @@ -3328,7 +3341,7 @@ dummy_func( } } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3345,10 +3358,15 @@ dummy_func( } Py_DECREF(tp); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_CLASS) = + unused/1 + + unused/2 + + _CALL_BUILTIN_CLASS + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ int total_args = oparg; if (self_or_null != NULL) { @@ -3373,10 +3391,15 @@ dummy_func( Py_DECREF(arg); Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_O) = + unused/1 + + unused/2 + + _CALL_BUILTIN_O + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ int total_args = oparg; if (self_or_null != NULL) { @@ -3400,15 +3423,15 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_BUILTIN_FAST) = + unused/1 + + unused/2 + + _CALL_BUILTIN_FAST + + _CHECK_PERIODIC; + + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int total_args = oparg; if (self_or_null != NULL) { @@ -3431,9 +3454,14 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } + macro(CALL_BUILTIN_FAST_WITH_KEYWORDS) = + unused/1 + + unused/2 + + _CALL_BUILTIN_FAST_WITH_KEYWORDS + + _CHECK_PERIODIC; + inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* len(o) */ int total_args = oparg; @@ -3504,7 +3532,7 @@ dummy_func( DISPATCH(); } - inst(CALL_METHOD_DESCRIPTOR_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3532,10 +3560,15 @@ dummy_func( Py_DECREF(arg); Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_O) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_O + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3561,10 +3594,15 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); int total_args = oparg; if (self_or_null != NULL) { @@ -3591,10 +3629,15 @@ dummy_func( Py_DECREF(self); Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { + macro(CALL_METHOD_DESCRIPTOR_NOARGS) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_NOARGS + + _CHECK_PERIODIC; + + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3619,9 +3662,14 @@ dummy_func( } Py_DECREF(callable); ERROR_IF(res == NULL, error); - CHECK_EVAL_BREAKER(); } + macro(CALL_METHOD_DESCRIPTOR_FAST) = + unused/1 + + unused/2 + + _CALL_METHOD_DESCRIPTOR_FAST + + _CHECK_PERIODIC; + inst(INSTRUMENTED_CALL_KW, ( -- )) { int is_meth = PEEK(oparg + 2) != NULL; int total_args = oparg + is_meth; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 42e884c20ba04f..077499ebc01687 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2819,6 +2819,11 @@ /* _CALL is not a viable micro-op for tier 2 */ + case _CHECK_PERIODIC: { + CHECK_EVAL_BREAKER(); + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *null; PyObject *callable; @@ -3096,7 +3101,6 @@ if (res == NULL) goto pop_3_error_tier_two; stack_pointer[-3] = res; stack_pointer += -2; - CHECK_EVAL_BREAKER(); break; } @@ -3118,7 +3122,6 @@ if (res == NULL) goto pop_3_error_tier_two; stack_pointer[-3] = res; stack_pointer += -2; - CHECK_EVAL_BREAKER(); break; } @@ -3165,7 +3168,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3203,7 +3205,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3238,14 +3239,8 @@ } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3281,7 +3276,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3393,7 +3387,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3432,7 +3425,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3473,7 +3465,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } @@ -3512,7 +3503,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7d02e49d040c23..645d0fb510e6fb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -842,6 +842,9 @@ } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } + // _CHECK_PERIODIC + { + } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1020,25 +1023,31 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1056,36 +1065,37 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1103,30 +1113,36 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1144,32 +1160,38 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_BUILTIN_O args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1509,33 +1531,39 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1553,33 +1581,39 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1597,35 +1631,41 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1643,36 +1683,42 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + { + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1816,16 +1862,22 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_STR_1 arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - res = PyObject_Str(arg); - Py_DECREF(arg); - if (res == NULL) goto pop_3_error; + { + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + res = PyObject_Str(arg); + Py_DECREF(arg); + if (res == NULL) goto pop_3_error; + } + // _CHECK_PERIODIC + { + } stack_pointer[-3] = res; stack_pointer += -2; CHECK_EVAL_BREAKER(); @@ -1843,16 +1895,22 @@ PyObject *res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ + // _CALL_TUPLE_1 arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - res = PySequence_Tuple(arg); - Py_DECREF(arg); - if (res == NULL) goto pop_3_error; + { + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + res = PySequence_Tuple(arg); + Py_DECREF(arg); + if (res == NULL) goto pop_3_error; + } + // _CHECK_PERIODIC + { + } stack_pointer[-3] = res; stack_pointer += -2; CHECK_EVAL_BREAKER(); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 7e4214cc9acf39..cf36f1ba792775 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1533,6 +1533,10 @@ /* _CALL is not a viable micro-op for tier 2 */ + case _CHECK_PERIODIC: { + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { _Py_UopsSymbol *null; _Py_UopsSymbol *callable;