From 686f64eff2672e9344425a5168a8dbaa6c717a06 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 21 Jan 2025 03:37:23 +0900 Subject: [PATCH 01/53] Enable inlining for late devirt --- src/coreclr/jit/compiler.h | 2 ++ src/coreclr/jit/fginline.cpp | 42 +++++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 0e3986969c161e..3257670a24bce6 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5109,6 +5109,8 @@ class Compiler SpillCliqueSucc }; + friend class SubstitutePlaceholdersAndDevirtualizeWalker; + // Abstract class for receiving a callback while walking a spill clique class SpillCliqueWalker { diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index ee75f8b13cd774..05c580b084d2ea 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -586,8 +586,36 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); + if (context != nullptr) + { + IL_OFFSET ilOffset = m_compiler->compCurStmt->GetDebugInfo().GetLocation().GetOffset(); + CORINFO_CALL_INFO callInfo = {}; + callInfo.hMethod = method; + callInfo.methodFlags = methodFlags; + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, ilOffset); + + const bool isInlineCandidate = call->IsInlineCandidate(); + + if (isInlineCandidate) + { + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), genActualType(call->TypeGet())); + *pTree = retExpr; + + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); + + JITDUMP("New inline candidate due to late devirtualization:"); + DISPTREE(call); + m_compiler->compCurStmt = stmt; + } + } m_madeChanges = true; } } @@ -730,11 +758,11 @@ PhaseStatus Compiler::fgInline() do { // Make the current basic block address available globally - compCurBB = block; - - for (Statement* const stmt : block->Statements()) + compCurBB = block; + Statement* stmt = block->firstStmt(); + while (stmt != nullptr) { - + compCurStmt = stmt; #if defined(DEBUG) // In debug builds we want the inline tree to show all failed // inlines. Some inlines may fail very early and never make it to @@ -756,6 +784,7 @@ PhaseStatus Compiler::fgInline() // replacement may have enabled optimizations by providing more // specific types for trees or variables. walker.WalkTree(stmt->GetRootNodePointer(), nullptr); + stmt = compCurStmt; GenTree* expr = stmt->GetRootNode(); @@ -805,6 +834,8 @@ PhaseStatus Compiler::fgInline() madeChanges = true; stmt->SetRootNode(expr->AsOp()->gtOp1); } + + stmt = stmt->GetNextStmt(); } block = block->Next(); @@ -1075,7 +1106,8 @@ void Compiler::fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result, Compiler::fgWalkResult Compiler::fgFindNonInlineCandidate(GenTree** pTree, fgWalkData* data) { GenTree* tree = *pTree; - if (tree->gtOper == GT_CALL) + // We may get late devirtualization opportunities for virtual calls that are not inline candidates. + if (tree->gtOper == GT_CALL && !tree->AsCall()->IsVirtual()) { Compiler* compiler = data->compiler; Statement* stmt = (Statement*)data->pCallbackData; From 27cc1ce2464fbf59391595dd58d74f4b44d0840b Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 21 Jan 2025 04:14:20 +0900 Subject: [PATCH 02/53] Pass correct IL offset --- src/coreclr/jit/fginline.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 05c580b084d2ea..675a86f5245ba9 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -591,7 +591,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorcompCurStmt->GetDebugInfo().GetLocation().GetOffset(); + IL_OFFSET ilOffset = 0; + INDEBUG(ilOffset = call->gtRawILOffset); CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; callInfo.methodFlags = methodFlags; From 21a5adb9dd2c564b69242a171fd9320de0a59b1a Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 21 Jan 2025 05:59:50 +0900 Subject: [PATCH 03/53] Only creates RET_EXPR if the node is not top-level or not returning void --- src/coreclr/jit/fginline.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 675a86f5245ba9..1f51207f5a99f8 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -602,19 +602,22 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), genActualType(call->TypeGet())); - *pTree = retExpr; - - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); - + if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID) + { + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + *pTree = retExpr; + + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); + m_compiler->compCurStmt = stmt; + } JITDUMP("New inline candidate due to late devirtualization:"); DISPTREE(call); - m_compiler->compCurStmt = stmt; } } m_madeChanges = true; From d1d51817c8805b337269e475c6e943b939e3b84f Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 21 Jan 2025 06:21:19 +0900 Subject: [PATCH 04/53] Do not try inlining if BBF_INTERNAL is set --- src/coreclr/jit/fginline.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 1f51207f5a99f8..db9d47c37a117b 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -589,7 +589,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); - if (context != nullptr) + if (context != nullptr && !m_compiler->compCurBB->HasFlag(BBF_INTERNAL)) { IL_OFFSET ilOffset = 0; INDEBUG(ilOffset = call->gtRawILOffset); @@ -597,10 +597,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpMarkInlineCandidate(call, context, false, &callInfo, ilOffset); - - const bool isInlineCandidate = call->IsInlineCandidate(); - - if (isInlineCandidate) + if (call->IsInlineCandidate()) { if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID) { From cd9f8174319bd2bc97772d44f006ecddb41ffe79 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 21 Jan 2025 07:00:47 +0900 Subject: [PATCH 05/53] Ensure the type matches --- src/coreclr/jit/fginline.cpp | 47 ++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index db9d47c37a117b..768c573492d472 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -591,30 +591,35 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorcompCurBB->HasFlag(BBF_INTERNAL)) { - IL_OFFSET ilOffset = 0; - INDEBUG(ilOffset = call->gtRawILOffset); - CORINFO_CALL_INFO callInfo = {}; - callInfo.hMethod = method; - callInfo.methodFlags = methodFlags; - m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, ilOffset); - if (call->IsInlineCandidate()) + CORINFO_METHOD_INFO methInfo; + if (m_compiler->info.compCompHnd->getMethodInfo(call->gtCallMethHnd, &methInfo, context) && + genActualType(JITtype2varType(methInfo.args.retType)) == genActualType(call->TypeGet())) { - if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID) + IL_OFFSET ilOffset = 0; + INDEBUG(ilOffset = call->gtRawILOffset); + CORINFO_CALL_INFO callInfo = {}; + callInfo.hMethod = method; + callInfo.methodFlags = methodFlags; + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, ilOffset); + if (call->IsInlineCandidate()) { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); - *pTree = retExpr; - - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); - m_compiler->compCurStmt = stmt; + if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID) + { + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + *pTree = retExpr; + + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); + m_compiler->compCurStmt = stmt; + } + JITDUMP("New inline candidate due to late devirtualization:"); + DISPTREE(call); } - JITDUMP("New inline candidate due to late devirtualization:"); - DISPTREE(call); } } m_madeChanges = true; From aad3d7e73b9a584424aca89076378130d7590c9c Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 21 Jan 2025 07:37:32 +0900 Subject: [PATCH 06/53] Set inliner context correctly --- src/coreclr/jit/fginline.cpp | 40 ++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 768c573492d472..412745e7e0f571 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -601,7 +601,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpMarkInlineCandidate(call, context, false, &callInfo, ilOffset); - if (call->IsInlineCandidate()) + + const bool isInlineCandidate = call->IsInlineCandidate(); + const bool isGuardedDevirtualizationCandidate = call->IsGuardedDevirtualizationCandidate(); + + if (isInlineCandidate || isGuardedDevirtualizationCandidate) { if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID) { @@ -610,14 +614,38 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtNewInlineCandidateReturnExpr(call->AsCall(), genActualType(call->TypeGet())); - *pTree = retExpr; - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); + if (call->IsGuardedDevirtualizationCandidate()) + { + for (uint8_t i = 0; i < call->GetInlineCandidatesCount(); i++) + { + call->GetGDVCandidateInfo(i)->retExpr = retExpr; + } + } + else + { + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + } + m_compiler->compCurStmt = stmt; + *pTree = retExpr; } - JITDUMP("New inline candidate due to late devirtualization:"); + + if (call->IsGuardedDevirtualizationCandidate()) + { + for (uint8_t i = 0; i < call->GetInlineCandidatesCount(); i++) + { + call->GetGDVCandidateInfo(i)->exactContextHandle = context; + INDEBUG(call->GetGDVCandidateInfo(i)->inlinersContext = call->gtInlineContext); + } + } + else + { + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + } + + JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); } } From 3b5b446bb8b71a4ade7ee69831706e386752bba9 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 02:18:28 +0900 Subject: [PATCH 07/53] Address review feedbacks --- src/coreclr/jit/fginline.cpp | 68 ++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 412745e7e0f571..f91f4ca424fc1e 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -204,7 +204,8 @@ bool Compiler::TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, i class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor { - bool m_madeChanges = false; + bool m_madeChanges = false; + Statement* m_curStmt = nullptr; public: enum @@ -219,11 +220,28 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetRootNodePointer(), nullptr); + return m_curStmt; + } + fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) { GenTree* tree = *use; @@ -595,56 +613,32 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorinfo.compCompHnd->getMethodInfo(call->gtCallMethHnd, &methInfo, context) && genActualType(JITtype2varType(methInfo.args.retType)) == genActualType(call->TypeGet())) { - IL_OFFSET ilOffset = 0; + IL_OFFSET ilOffset = BAD_IL_OFFSET; INDEBUG(ilOffset = call->gtRawILOffset); CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; callInfo.methodFlags = methodFlags; m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, ilOffset); - const bool isInlineCandidate = call->IsInlineCandidate(); - const bool isGuardedDevirtualizationCandidate = call->IsGuardedDevirtualizationCandidate(); - - if (isInlineCandidate || isGuardedDevirtualizationCandidate) + if (call->IsInlineCandidate()) { if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID) { Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); GenTreeRetExpr* retExpr = m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), genActualType(call->TypeGet())); - if (call->IsGuardedDevirtualizationCandidate()) - { - for (uint8_t i = 0; i < call->GetInlineCandidatesCount(); i++) - { - call->GetGDVCandidateInfo(i)->retExpr = retExpr; - } - } - else - { - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - } - - m_compiler->compCurStmt = stmt; - *pTree = retExpr; - } + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - if (call->IsGuardedDevirtualizationCandidate()) - { - for (uint8_t i = 0; i < call->GetInlineCandidatesCount(); i++) - { - call->GetGDVCandidateInfo(i)->exactContextHandle = context; - INDEBUG(call->GetGDVCandidateInfo(i)->inlinersContext = call->gtInlineContext); - } - } - else - { - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + m_curStmt = stmt; + *pTree = retExpr; } + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); } @@ -796,7 +790,6 @@ PhaseStatus Compiler::fgInline() Statement* stmt = block->firstStmt(); while (stmt != nullptr) { - compCurStmt = stmt; #if defined(DEBUG) // In debug builds we want the inline tree to show all failed // inlines. Some inlines may fail very early and never make it to @@ -817,8 +810,7 @@ PhaseStatus Compiler::fgInline() // possible further optimization, as the (now complete) GT_RET_EXPR // replacement may have enabled optimizations by providing more // specific types for trees or variables. - walker.WalkTree(stmt->GetRootNodePointer(), nullptr); - stmt = compCurStmt; + stmt = walker.WalkStatement(stmt); GenTree* expr = stmt->GetRootNode(); From 27bc0aa39c337899078c8e068c9de470bea667e2 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 02:32:33 +0900 Subject: [PATCH 08/53] Fix non inline candidate marking --- src/coreclr/jit/fginline.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index f91f4ca424fc1e..47df2efe9b8500 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -790,12 +790,6 @@ PhaseStatus Compiler::fgInline() Statement* stmt = block->firstStmt(); while (stmt != nullptr) { -#if defined(DEBUG) - // In debug builds we want the inline tree to show all failed - // inlines. Some inlines may fail very early and never make it to - // candidate stage. So scan the tree looking for those early failures. - fgWalkTreePre(stmt->GetRootNodePointer(), fgFindNonInlineCandidate, stmt); -#endif // See if we need to replace some return value place holders. // Also, see if this replacement enables further devirtualization. // @@ -861,6 +855,11 @@ PhaseStatus Compiler::fgInline() stmt->SetRootNode(expr->AsOp()->gtOp1); } +#if defined(DEBUG) + // In debug builds we want the inline tree to show all failed + // inlines. + fgWalkTreePre(stmt->GetRootNodePointer(), fgFindNonInlineCandidate, stmt); +#endif stmt = stmt->GetNextStmt(); } @@ -1132,8 +1131,7 @@ void Compiler::fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result, Compiler::fgWalkResult Compiler::fgFindNonInlineCandidate(GenTree** pTree, fgWalkData* data) { GenTree* tree = *pTree; - // We may get late devirtualization opportunities for virtual calls that are not inline candidates. - if (tree->gtOper == GT_CALL && !tree->AsCall()->IsVirtual()) + if (tree->gtOper == GT_CALL) { Compiler* compiler = data->compiler; Statement* stmt = (Statement*)data->pCallbackData; From dbf204a207db58397c8924e237ca9087cc4238a8 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 03:24:49 +0900 Subject: [PATCH 09/53] Handle calls with retbuf --- src/coreclr/jit/fginline.cpp | 57 ++++++++++++++++--------------- src/coreclr/jit/importercalls.cpp | 4 ++- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 47df2efe9b8500..b2b275c435ab1f 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -609,39 +609,42 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorcompCurBB->HasFlag(BBF_INTERNAL)) { - CORINFO_METHOD_INFO methInfo; - if (m_compiler->info.compCompHnd->getMethodInfo(call->gtCallMethHnd, &methInfo, context) && - genActualType(JITtype2varType(methInfo.args.retType)) == genActualType(call->TypeGet())) + IL_OFFSET ilOffset = BAD_IL_OFFSET; + INDEBUG(ilOffset = call->gtRawILOffset); + CORINFO_CALL_INFO callInfo = {}; + callInfo.hMethod = method; + callInfo.methodFlags = methodFlags; + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, ilOffset); + + if (call->IsInlineCandidate()) { - IL_OFFSET ilOffset = BAD_IL_OFFSET; - INDEBUG(ilOffset = call->gtRawILOffset); - CORINFO_CALL_INFO callInfo = {}; - callInfo.hMethod = method; - callInfo.methodFlags = methodFlags; - m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, ilOffset); - - if (call->IsInlineCandidate()) - { - if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID) - { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); + CallArg* retBuffer = call->gtArgs.GetRetBufferArg(); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID || retBuffer != nullptr) + { + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); - m_curStmt = stmt; - *pTree = retExpr; - } + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + m_curStmt = stmt; + *pTree = retExpr; + } - JITDUMP("New inline candidate due to late devirtualization:\n"); - DISPTREE(call); + if (retBuffer != nullptr) + { + call->GetSingleInlineCandidateInfo()->preexistingSpillTemp = + retBuffer->GetNode()->AsLclVarCommon()->GetLclNum(); } + + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + + JITDUMP("New inline candidate due to late devirtualization:\n"); + DISPTREE(call); } } m_madeChanges = true; diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 6d737b4f59d10f..7bbe5e4396f065 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -9201,9 +9201,11 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_CLASS_HANDLE clsHandle = compCompHnd->getMethodClass(ftn); unsigned const clsAttr = compCompHnd->getClassAttribs(clsHandle); + CallArg* pRetBuf = pParam->call->gtArgs.GetRetBufferArg(); + // Return type // - var_types const fncRetType = pParam->call->TypeGet(); + var_types const fncRetType = pRetBuf == nullptr ? pParam->call->TypeGet() : TYP_STRUCT; #ifdef DEBUG var_types fncRealRetType = JITtype2varType(methInfo.args.retType); From f6d38c4d86e42ff9aaad72a17e7dc3ec62fa431b Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 03:44:00 +0900 Subject: [PATCH 10/53] Handle BBF_INTERNAL --- src/coreclr/jit/fginline.cpp | 2 +- src/coreclr/jit/importercalls.cpp | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index b2b275c435ab1f..5617158ec86c19 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -607,7 +607,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); - if (context != nullptr && !m_compiler->compCurBB->HasFlag(BBF_INTERNAL)) + if (context != nullptr) { IL_OFFSET ilOffset = BAD_IL_OFFSET; INDEBUG(ilOffset = call->gtRawILOffset); diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 7bbe5e4396f065..408f47c3feae24 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -7695,6 +7695,19 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (!(methAttr & CORINFO_FLG_FORCEINLINE)) { + if (compCurBB->HasFlag(BBF_INTERNAL) && ehGetBlockHndDsc(compCurBB) != nullptr) + { +#ifdef DEBUG + if (verbose) + { + printf("\nWill not inline blocks that have EH descriptor when IL offset may not be valid\n"); + } + +#endif + inlineResult->NoteFatal(InlineObservation::CALLSITE_NOT_CANDIDATE); + return; + } + /* Don't bother inline blocks that are in the filter region */ if (bbInCatchHandlerILRange(compCurBB)) { From 10de97b672ceabda89cab357f8269dbb6ca847a5 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 16:36:56 +0900 Subject: [PATCH 11/53] Get real type from local --- src/coreclr/jit/importercalls.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 408f47c3feae24..47dafef40001b3 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -9218,7 +9218,9 @@ void Compiler::impCheckCanInline(GenTreeCall* call, // Return type // - var_types const fncRetType = pRetBuf == nullptr ? pParam->call->TypeGet() : TYP_STRUCT; + var_types const fncRetType = + pRetBuf == nullptr ? pParam->call->TypeGet() + : pParam->pThis->lvaGetRealType(pRetBuf->GetEarlyNode()->AsLclVarCommon()->GetLclNum()); #ifdef DEBUG var_types fncRealRetType = JITtype2varType(methInfo.args.retType); From 472e1405403331bf072579cbb1b4ab0bac4682b0 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 17:22:21 +0900 Subject: [PATCH 12/53] Always set IL offset --- src/coreclr/jit/fginline.cpp | 7 +++++-- src/coreclr/jit/importercalls.cpp | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 5617158ec86c19..2d0090b373a8a7 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -609,8 +609,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtRawILOffset); +#if defined(DEBUG) + IL_OFFSET ilOffset = call->gtRawILOffset; +#else + IL_OFFSET ilOffset = m_curStmt->GetDebugInfo().GetLocation().GetOffset(); +#endif // defined(DEBUG) CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; callInfo.methodFlags = methodFlags; diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 47dafef40001b3..fa3e3f55768b0b 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -329,7 +329,7 @@ var_types Compiler::impImportCall(OPCODE opcode, assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG && (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG); - call = gtNewIndCallNode(stubAddr, callRetTyp); + call = gtNewIndCallNode(stubAddr, callRetTyp, di); call->gtFlags |= GTF_EXCEPT | (stubAddr->gtFlags & GTF_GLOB_EFFECT); call->gtFlags |= GTF_CALL_VIRT_STUB; @@ -1027,7 +1027,7 @@ var_types Compiler::impImportCall(OPCODE opcode, if (!bIntrinsicImported) { // Keep track of the raw IL offset of the call - INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); + call->AsCall()->gtRawILOffset = rawILOffset; // Is it an inline candidate? impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); @@ -1237,7 +1237,7 @@ var_types Compiler::impImportCall(OPCODE opcode, } // Keep track of the raw IL offset of the call - INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); + call->AsCall()->gtRawILOffset = rawILOffset; // Is it an inline candidate? impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); From 20b1a94d28d890d055288f0bfafa1bb4cfac9830 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 17:24:54 +0900 Subject: [PATCH 13/53] Add comments --- src/coreclr/jit/fginline.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 2d0090b373a8a7..b617075f190239 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -623,6 +623,9 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtArgs.GetRetBufferArg(); + // If the call is the top-level expression in a statement, and it returns void, + // there will be no use of its return value, and we can just inline it directly. + // In this case we don't need to create a RET_EXPR node for it. if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID || retBuffer != nullptr) { Statement* stmt = m_compiler->gtNewStmt(call); From 5c9c927b8f29e83b344e328c4a823eac2b74fc01 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 17:29:15 +0900 Subject: [PATCH 14/53] Oops --- src/coreclr/jit/importercalls.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index fa3e3f55768b0b..25236eadcff9d1 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1027,7 +1027,7 @@ var_types Compiler::impImportCall(OPCODE opcode, if (!bIntrinsicImported) { // Keep track of the raw IL offset of the call - call->AsCall()->gtRawILOffset = rawILOffset; + INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); @@ -1237,7 +1237,7 @@ var_types Compiler::impImportCall(OPCODE opcode, } // Keep track of the raw IL offset of the call - call->AsCall()->gtRawILOffset = rawILOffset; + INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); From 97326e2f67ea3aebf4b2be77c137f62946da6ed8 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 17:34:41 +0900 Subject: [PATCH 15/53] Use gtReturnType instead --- src/coreclr/jit/importercalls.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 25236eadcff9d1..7ef15072a61853 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -9214,13 +9214,9 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_CLASS_HANDLE clsHandle = compCompHnd->getMethodClass(ftn); unsigned const clsAttr = compCompHnd->getClassAttribs(clsHandle); - CallArg* pRetBuf = pParam->call->gtArgs.GetRetBufferArg(); - // Return type // - var_types const fncRetType = - pRetBuf == nullptr ? pParam->call->TypeGet() - : pParam->pThis->lvaGetRealType(pRetBuf->GetEarlyNode()->AsLclVarCommon()->GetLclNum()); + var_types const fncRetType = pParam->call->gtReturnType; #ifdef DEBUG var_types fncRealRetType = JITtype2varType(methInfo.args.retType); From 096ab2a4fa7646981879665c22975ff680847a4d Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 18:51:01 +0900 Subject: [PATCH 16/53] Remove unused ilOffset --- src/coreclr/jit/compiler.h | 4 +--- src/coreclr/jit/fginline.cpp | 7 +------ src/coreclr/jit/importercalls.cpp | 16 +++++----------- src/coreclr/jit/inline.h | 2 -- 4 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3257670a24bce6..f971d0c62417b1 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5223,15 +5223,13 @@ class Compiler void impMarkInlineCandidate(GenTree* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset); + CORINFO_CALL_INFO* callInfo); void impMarkInlineCandidateHelper(GenTreeCall* call, uint8_t candidateIndex, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset, InlineResult* inlineResult); bool impTailCallRetTypeCompatible(bool allowWidening, diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index b617075f190239..1b4dbab26d3552 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -609,15 +609,10 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtRawILOffset; -#else - IL_OFFSET ilOffset = m_curStmt->GetDebugInfo().GetLocation().GetOffset(); -#endif // defined(DEBUG) CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; callInfo.methodFlags = methodFlags; - m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, ilOffset); + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); if (call->IsInlineCandidate()) { diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 7ef15072a61853..a93dd1dfea99aa 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1030,7 +1030,7 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); } // append the call node. @@ -1240,7 +1240,7 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); } // Extra checks for tail calls and tail recursion. @@ -7450,7 +7450,6 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM -// ilOffset -- the actual IL offset of the instruction that produced this inline candidate // // Notes: // Mostly a wrapper for impMarkInlineCandidateHelper that also undoes @@ -7460,8 +7459,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, void Compiler::impMarkInlineCandidate(GenTree* callNode, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset) + CORINFO_CALL_INFO* callInfo) { if (!opts.OptEnabled(CLFLG_INLINING)) { @@ -7484,7 +7482,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // Do the actual evaluation impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, - ilOffset, &inlineResult); + &inlineResult); // Ignore non-inlineable candidates // TODO: Consider keeping them to just devirtualize without inlining, at least for interface // calls on NativeAOT, but that requires more changes elsewhere too. @@ -7507,8 +7505,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, const uint8_t candidatesCount = call->GetInlineCandidatesCount(); assert(candidatesCount <= 1); InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate"); - impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, ilOffset, - &inlineResult); + impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult); } // If this call is an inline candidate or is not a guarded devirtualization @@ -7541,7 +7538,6 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM -// ilOffset -- IL offset of instruction creating the inline candidate // // Notes: // If callNode is an inline candidate, this method sets the flag @@ -7558,7 +7554,6 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset, InlineResult* inlineResult) { // Let the strategy know there's another call @@ -7780,7 +7775,6 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, // The new value should not be null. assert(inlineCandidateInfo != nullptr); inlineCandidateInfo->exactContextNeedsRuntimeLookup = exactContextNeedsRuntimeLookup; - inlineCandidateInfo->ilOffset = ilOffset; // If we're in an inlinee compiler, and have a return spill temp, and this inline candidate // is also a tail call candidate, it can use the same return spill temp. diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index ce50b2cc93dba8..3b55842aafe7d3 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -622,8 +622,6 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo unsigned clsAttr; unsigned methAttr; - // actual IL offset of instruction that resulted in this inline candidate - IL_OFFSET ilOffset; CorInfoInitClassResult initClassResult; var_types fncRetType; bool exactContextNeedsRuntimeLookup; From 095a981ea7e4f01d82775590cd4daae07c68b1ff Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 20:46:51 +0900 Subject: [PATCH 17/53] Use genActualType --- src/coreclr/jit/importercalls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index a93dd1dfea99aa..7428f2c7d3659e 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -9210,7 +9210,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, // Return type // - var_types const fncRetType = pParam->call->gtReturnType; + var_types const fncRetType = genActualType(pParam->call->gtReturnType); #ifdef DEBUG var_types fncRealRetType = JITtype2varType(methInfo.args.retType); From 59514621c561ffdc125d784c6dc714aa1ef99465 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 22 Jan 2025 22:58:15 +0900 Subject: [PATCH 18/53] Remove unnecessary spillTemp --- src/coreclr/jit/fginline.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 1b4dbab26d3552..1fdf7a7082a134 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -635,12 +635,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->preexistingSpillTemp = - retBuffer->GetNode()->AsLclVarCommon()->GetLclNum(); - } - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); From 03c0df55af4e44ae28e048434054e6df14cf83ee Mon Sep 17 00:00:00 2001 From: Steven He Date: Thu, 23 Jan 2025 03:24:07 +0900 Subject: [PATCH 19/53] Handle nested call correctly --- src/coreclr/jit/fginline.cpp | 37 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 1fdf7a7082a134..ad05834d3c9345 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -204,8 +204,8 @@ bool Compiler::TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, i class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor { - bool m_madeChanges = false; - Statement* m_curStmt = nullptr; + bool m_madeChanges = false; + Statement* m_firstNewStmt = nullptr; public: enum @@ -226,20 +226,20 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetRootNodePointer(), nullptr); - return m_curStmt; + m_firstNewStmt = nullptr; + WalkTree(stmt->GetRootNodePointer(), nullptr); + return m_firstNewStmt == nullptr ? stmt : m_firstNewStmt; } fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) @@ -616,23 +616,27 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { - CallArg* retBuffer = call->gtArgs.GetRetBufferArg(); - // If the call is the top-level expression in a statement, and it returns void, // there will be no use of its return value, and we can just inline it directly. // In this case we don't need to create a RET_EXPR node for it. - if (parent != nullptr || genActualType(call->TypeGet()) != TYP_VOID || retBuffer != nullptr) + if (parent != nullptr || call->gtReturnType != TYP_VOID) { Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); + if (m_firstNewStmt == nullptr) + { + m_firstNewStmt = stmt; + } + GenTreeRetExpr* retExpr = m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), genActualType(call->TypeGet())); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - m_curStmt = stmt; - *pTree = retExpr; + JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); + DISPTREE(retExpr); + + *pTree = retExpr; } call->GetSingleInlineCandidateInfo()->exactContextHandle = context; @@ -802,7 +806,8 @@ PhaseStatus Compiler::fgInline() // possible further optimization, as the (now complete) GT_RET_EXPR // replacement may have enabled optimizations by providing more // specific types for trees or variables. - stmt = walker.WalkStatement(stmt); + compCurStmt = stmt; + stmt = walker.WalkStatement(stmt); GenTree* expr = stmt->GetRootNode(); From 00425197b059c7bfcf208db02986e73a5d74c63c Mon Sep 17 00:00:00 2001 From: Steven He Date: Thu, 23 Jan 2025 03:40:27 +0900 Subject: [PATCH 20/53] Don't promote compCurStmt --- src/coreclr/jit/fginline.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index ad05834d3c9345..90dba19e8b1046 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -205,6 +205,7 @@ bool Compiler::TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, i class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor { bool m_madeChanges = false; + Statement* m_curStmt = nullptr; Statement* m_firstNewStmt = nullptr; public: @@ -238,8 +239,9 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetRootNodePointer(), nullptr); - return m_firstNewStmt == nullptr ? stmt : m_firstNewStmt; + m_curStmt = stmt; + WalkTree(m_curStmt->GetRootNodePointer(), nullptr); + return m_firstNewStmt == nullptr ? m_curStmt : m_firstNewStmt; } fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) @@ -622,7 +624,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtReturnType != TYP_VOID) { Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_compiler->compCurStmt, stmt); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); if (m_firstNewStmt == nullptr) { m_firstNewStmt = stmt; @@ -806,8 +808,7 @@ PhaseStatus Compiler::fgInline() // possible further optimization, as the (now complete) GT_RET_EXPR // replacement may have enabled optimizations by providing more // specific types for trees or variables. - compCurStmt = stmt; - stmt = walker.WalkStatement(stmt); + stmt = walker.WalkStatement(stmt); GenTree* expr = stmt->GetRootNode(); From 0e1355398922e32eb21d07215f07ec990b5ae5cf Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 24 Jan 2025 15:01:24 +0900 Subject: [PATCH 21/53] Remove unused ilOffset --- src/coreclr/jit/compiler.h | 4 +--- src/coreclr/jit/importercalls.cpp | 16 +++++----------- src/coreclr/jit/inline.h | 2 -- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 956e05e9b71f00..f53028db7c2797 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5223,15 +5223,13 @@ class Compiler void impMarkInlineCandidate(GenTree* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset); + CORINFO_CALL_INFO* callInfo); void impMarkInlineCandidateHelper(GenTreeCall* call, uint8_t candidateIndex, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset, InlineResult* inlineResult); bool impTailCallRetTypeCompatible(bool allowWidening, diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 6d737b4f59d10f..c3c8263a3a3da0 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1030,7 +1030,7 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); } // append the call node. @@ -1240,7 +1240,7 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); } // Extra checks for tail calls and tail recursion. @@ -7450,7 +7450,6 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM -// ilOffset -- the actual IL offset of the instruction that produced this inline candidate // // Notes: // Mostly a wrapper for impMarkInlineCandidateHelper that also undoes @@ -7460,8 +7459,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, void Compiler::impMarkInlineCandidate(GenTree* callNode, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset) + CORINFO_CALL_INFO* callInfo) { if (!opts.OptEnabled(CLFLG_INLINING)) { @@ -7484,7 +7482,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // Do the actual evaluation impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, - ilOffset, &inlineResult); + &inlineResult); // Ignore non-inlineable candidates // TODO: Consider keeping them to just devirtualize without inlining, at least for interface // calls on NativeAOT, but that requires more changes elsewhere too. @@ -7507,8 +7505,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, const uint8_t candidatesCount = call->GetInlineCandidatesCount(); assert(candidatesCount <= 1); InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate"); - impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, ilOffset, - &inlineResult); + impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult); } // If this call is an inline candidate or is not a guarded devirtualization @@ -7541,7 +7538,6 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM -// ilOffset -- IL offset of instruction creating the inline candidate // // Notes: // If callNode is an inline candidate, this method sets the flag @@ -7558,7 +7554,6 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset, InlineResult* inlineResult) { // Let the strategy know there's another call @@ -7767,7 +7762,6 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, // The new value should not be null. assert(inlineCandidateInfo != nullptr); inlineCandidateInfo->exactContextNeedsRuntimeLookup = exactContextNeedsRuntimeLookup; - inlineCandidateInfo->ilOffset = ilOffset; // If we're in an inlinee compiler, and have a return spill temp, and this inline candidate // is also a tail call candidate, it can use the same return spill temp. diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index ce50b2cc93dba8..3b55842aafe7d3 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -622,8 +622,6 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo unsigned clsAttr; unsigned methAttr; - // actual IL offset of instruction that resulted in this inline candidate - IL_OFFSET ilOffset; CorInfoInitClassResult initClassResult; var_types fncRetType; bool exactContextNeedsRuntimeLookup; From edfa2ac55f5b7cfdf16c91c96102fbfaee2cbe34 Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 24 Jan 2025 15:02:09 +0900 Subject: [PATCH 22/53] Handle BBF_INTERNAL --- src/coreclr/jit/importercalls.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index c3c8263a3a3da0..2e577cdd59d856 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -7690,6 +7690,19 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (!(methAttr & CORINFO_FLG_FORCEINLINE)) { + if (compCurBB->HasFlag(BBF_INTERNAL) && ehGetBlockHndDsc(compCurBB) != nullptr) + { +#ifdef DEBUG + if (verbose) + { + printf("\nWill not inline blocks that have EH descriptor when IL offset may not be valid\n"); + } + +#endif + inlineResult->NoteFatal(InlineObservation::CALLSITE_NOT_CANDIDATE); + return; + } + /* Don't bother inline blocks that are in the filter region */ if (bbInCatchHandlerILRange(compCurBB)) { From d40d7e1b4a620178cd7113d15b83ae0579a95382 Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 24 Jan 2025 15:03:03 +0900 Subject: [PATCH 23/53] Use correct return type --- src/coreclr/jit/importercalls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 2e577cdd59d856..9bd337bda1e3f4 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -9210,7 +9210,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, // Return type // - var_types const fncRetType = pParam->call->TypeGet(); + var_types const fncRetType = genActualType(pParam->call->gtReturnType); #ifdef DEBUG var_types fncRealRetType = JITtype2varType(methInfo.args.retType); From e540cacc7712912b09c8a3bcb9cfcd7afda80d11 Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 24 Jan 2025 20:52:47 +0900 Subject: [PATCH 24/53] Use bbInCatchHandlerBBRange and bbInFilterBBRange --- src/coreclr/jit/compiler.h | 1 + src/coreclr/jit/importercalls.cpp | 20 +++----------------- src/coreclr/jit/jiteh.cpp | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index f53028db7c2797..65ee54821f4321 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2866,6 +2866,7 @@ class Compiler bool bbInCatchHandlerILRange(BasicBlock* blk); bool bbInFilterILRange(BasicBlock* blk); + bool bbInCatchHandlerBBRange(BasicBlock* blk); bool bbInFilterBBRange(BasicBlock* blk); bool bbInTryRegions(unsigned regionIndex, BasicBlock* blk); bool bbInExnFlowRegions(unsigned regionIndex, BasicBlock* blk); diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 9bd337bda1e3f4..4d3418b6c6a6b1 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -7690,35 +7690,21 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (!(methAttr & CORINFO_FLG_FORCEINLINE)) { - if (compCurBB->HasFlag(BBF_INTERNAL) && ehGetBlockHndDsc(compCurBB) != nullptr) - { -#ifdef DEBUG - if (verbose) - { - printf("\nWill not inline blocks that have EH descriptor when IL offset may not be valid\n"); - } - -#endif - inlineResult->NoteFatal(InlineObservation::CALLSITE_NOT_CANDIDATE); - return; - } - /* Don't bother inline blocks that are in the filter region */ - if (bbInCatchHandlerILRange(compCurBB)) + if (bbInCatchHandlerBBRange(compCurBB)) { #ifdef DEBUG if (verbose) { printf("\nWill not inline blocks that are in the catch handler region\n"); } - #endif inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_CATCH); return; } - if (bbInFilterILRange(compCurBB)) + if (bbInFilterBBRange(compCurBB)) { #ifdef DEBUG if (verbose) @@ -9257,7 +9243,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, pInfo->clsAttr = clsAttr; pInfo->methAttr = pParam->methAttr; pInfo->initClassResult = initClassResult; - pInfo->fncRetType = fncRetType; + pInfo->fncRetType = genActualType(JITtype2varType(methInfo.args.retType)); pInfo->exactContextNeedsRuntimeLookup = false; pInfo->inlinersContext = pParam->pThis->compInlineContext; diff --git a/src/coreclr/jit/jiteh.cpp b/src/coreclr/jit/jiteh.cpp index 703c615fcd6339..60c830aad8d592 100644 --- a/src/coreclr/jit/jiteh.cpp +++ b/src/coreclr/jit/jiteh.cpp @@ -354,6 +354,28 @@ bool Compiler::bbInFilterILRange(BasicBlock* blk) return HBtab->InFilterRegionILRange(blk); } +//------------------------------------------------------------------------ +// bbInCatchHandlerBBRange: +// Check if this block is part of a catch handler. +// +// Arguments: +// blk - The block +// +// Return Value: +// True if the block is part of a catch handler clause. Otherwise false. +// +bool Compiler::bbInCatchHandlerBBRange(BasicBlock* blk) +{ + EHblkDsc* HBtab = ehGetBlockHndDsc(blk); + + if (HBtab == nullptr) + { + return false; + } + + return HBtab->HasCatchHandler() && HBtab->InHndRegionBBRange(blk); +} + //------------------------------------------------------------------------ // bbInFilterBBRange: // Check if this block is part of a filter. From 036010cc5ba257412918b829b8dedb88162b8c9c Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 24 Jan 2025 20:57:52 +0900 Subject: [PATCH 25/53] Cleanup fncRetType --- src/coreclr/jit/fginline.cpp | 3 ++- src/coreclr/jit/importer.cpp | 2 +- src/coreclr/jit/importercalls.cpp | 8 +++----- src/coreclr/jit/inline.h | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index ee75f8b13cd774..bbeaaab0845873 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -1323,7 +1323,8 @@ void Compiler::fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* inlineRe // (This could happen for example for a BBJ_THROW block fall through a BBJ_RETURN block which // causes the BBJ_RETURN block not to be imported at all.) // Fail the inlining attempt - if ((inlineCandidateInfo->fncRetType != TYP_VOID) && (inlineCandidateInfo->retExpr->gtSubstExpr == nullptr)) + if ((inlineCandidateInfo->methInfo.args.retType != CORINFO_TYPE_VOID) && + (inlineCandidateInfo->retExpr->gtSubstExpr == nullptr)) { #ifdef DEBUG if (verbose) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 5bd6e842daa119..2ec2a291e6d3a6 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -10981,7 +10981,7 @@ bool Compiler::impReturnInstruction(int prefixFlags, OPCODE& opcode) // Make sure the type matches the original call. var_types returnType = genActualType(op2->gtType); - var_types originalCallType = inlCandInfo->fncRetType; + var_types originalCallType = genActualType(JITtype2varType(inlCandInfo->methInfo.args.retType)); if ((returnType != originalCallType) && (originalCallType == TYP_STRUCT)) { originalCallType = impNormStructType(inlCandInfo->methInfo.args.retTypeClass); diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 4d3418b6c6a6b1..a90a23c405cc2d 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -9194,12 +9194,11 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_CLASS_HANDLE clsHandle = compCompHnd->getMethodClass(ftn); unsigned const clsAttr = compCompHnd->getClassAttribs(clsHandle); +#ifdef DEBUG // Return type // - var_types const fncRetType = genActualType(pParam->call->gtReturnType); - -#ifdef DEBUG - var_types fncRealRetType = JITtype2varType(methInfo.args.retType); + var_types const fncRetType = pParam->call->gtReturnType; + var_types const fncRealRetType = JITtype2varType(methInfo.args.retType); assert((genActualType(fncRealRetType) == genActualType(fncRetType)) || // VSW 288602 @@ -9243,7 +9242,6 @@ void Compiler::impCheckCanInline(GenTreeCall* call, pInfo->clsAttr = clsAttr; pInfo->methAttr = pParam->methAttr; pInfo->initClassResult = initClassResult; - pInfo->fncRetType = genActualType(JITtype2varType(methInfo.args.retType)); pInfo->exactContextNeedsRuntimeLookup = false; pInfo->inlinersContext = pParam->pThis->compInlineContext; diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 3b55842aafe7d3..426e6575973d4c 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -623,7 +623,6 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo unsigned methAttr; CorInfoInitClassResult initClassResult; - var_types fncRetType; bool exactContextNeedsRuntimeLookup; InlineContext* inlinersContext; }; From 121242ed7f60404555b6242a42a1ec46a74ce64f Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 24 Jan 2025 21:12:42 +0900 Subject: [PATCH 26/53] Add a runtime check to prevent accidental execution order change --- src/coreclr/jit/fginline.cpp | 63 ++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 90dba19e8b1046..dd5f326dc2e130 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -208,6 +208,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetRootNodePointer(), nullptr); return m_firstNewStmt == nullptr ? m_curStmt : m_firstNewStmt; } @@ -574,6 +577,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorAsCall(); bool tryLateDevirt = call->IsVirtual() && (call->gtCallType == CT_USER_FUNC); + bool canInline = false; #ifdef DEBUG tryLateDevirt = tryLateDevirt && (JitConfig.JitEnableLateDevirtualization() == 1); @@ -618,38 +622,57 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { + canInline = true; // If the call is the top-level expression in a statement, and it returns void, // there will be no use of its return value, and we can just inline it directly. // In this case we don't need to create a RET_EXPR node for it. if (parent != nullptr || call->gtReturnType != TYP_VOID) { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); - if (m_firstNewStmt == nullptr) + if (parent != nullptr && m_prevNonInlineCandidateCall != nullptr) { - m_firstNewStmt = stmt; + // If any call in the sibling tree is not an inline candidate, we can't inline this + // call. This shouldn't happen as we currently spill the call aggressively in the importer. + unreached(); + canInline = false; + } + else + { + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + if (m_firstNewStmt == nullptr) + { + m_firstNewStmt = stmt; + } + + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + + JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); + DISPTREE(retExpr); + + *pTree = retExpr; } - - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - - JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); - DISPTREE(retExpr); - - *pTree = retExpr; } - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + if (canInline) + { + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); - JITDUMP("New inline candidate due to late devirtualization:\n"); - DISPTREE(call); + JITDUMP("New inline candidate due to late devirtualization:\n"); + DISPTREE(call); + } } } m_madeChanges = true; } + + if (!canInline) + { + m_prevNonInlineCandidateCall = call; + } } else if (tree->OperIs(GT_STORE_LCL_VAR)) { From 8447a71f9bd8a557463d815ac4c79860ed5fd470 Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 24 Jan 2025 21:34:56 +0900 Subject: [PATCH 27/53] Format jit --- src/coreclr/jit/fginline.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index ab9851ce895781..177b6bcd8292b0 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -631,7 +631,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor Date: Sat, 25 Jan 2025 04:06:09 +0900 Subject: [PATCH 28/53] Revert some changes --- src/coreclr/jit/fginline.cpp | 65 ++++++++++++------------------------ 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 177b6bcd8292b0..758e0db7103d29 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -208,8 +208,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetRootNodePointer(), nullptr); return m_firstNewStmt == nullptr ? m_curStmt : m_firstNewStmt; } @@ -622,58 +619,40 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { - canInline = true; // If the call is the top-level expression in a statement, and it returns void, // there will be no use of its return value, and we can just inline it directly. // In this case we don't need to create a RET_EXPR node for it. + // We currently spill calls aggressively in the importer, so this won't lead to + // execution order changes. if (parent != nullptr || call->gtReturnType != TYP_VOID) { - if (parent != nullptr && m_prevNonInlineCandidateCall != nullptr) - { - // If any call in the sibling tree is not an inline candidate, we can't inline this - // call. This shouldn't happen as we currently spill the call aggressively in the - // importer. - unreached(); - canInline = false; - } - else + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + if (m_firstNewStmt == nullptr) { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); - if (m_firstNewStmt == nullptr) - { - m_firstNewStmt = stmt; - } - - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - - JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); - DISPTREE(retExpr); - - *pTree = retExpr; + m_firstNewStmt = stmt; } - } - if (canInline) - { - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + + JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); + DISPTREE(retExpr); - JITDUMP("New inline candidate due to late devirtualization:\n"); - DISPTREE(call); + *pTree = retExpr; } + + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + + JITDUMP("New inline candidate due to late devirtualization:\n"); + DISPTREE(call); } } m_madeChanges = true; } - - if (!canInline) - { - m_prevNonInlineCandidateCall = call; - } } else if (tree->OperIs(GT_STORE_LCL_VAR)) { From 4507692d3b2d205907c80d1b57d791ca6f1af047 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 04:09:13 +0900 Subject: [PATCH 29/53] Remove unused local --- src/coreclr/jit/fginline.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 758e0db7103d29..7743b46ea3d93f 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -574,7 +574,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorAsCall(); bool tryLateDevirt = call->IsVirtual() && (call->gtCallType == CT_USER_FUNC); - bool canInline = false; #ifdef DEBUG tryLateDevirt = tryLateDevirt && (JitConfig.JitEnableLateDevirtualization() == 1); From 8fe9716ea74139e36e54e0525ed5591b5ea69a6f Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 05:45:52 +0900 Subject: [PATCH 30/53] Check whether a call can be spilled without side effect --- src/coreclr/jit/fginline.cpp | 75 ++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 7743b46ea3d93f..36a51f32df042c 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -204,13 +204,18 @@ bool Compiler::TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, i class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor { + typedef JitHashTable, unsigned> GenTreeOpIndexMap; + bool m_madeChanges = false; Statement* m_curStmt = nullptr; Statement* m_firstNewStmt = nullptr; + GenTreeOpIndexMap m_genTreeOpIndexMap; + public: enum { + ComputeStack = true, DoPreOrder = true, DoPostOrder = true, UseExecutionOrder = true, @@ -218,6 +223,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgetAllocator(CMK_Inlining)) { } @@ -530,6 +536,39 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorOperands()) + { + if (opIndex-- == 0) + { + nextOp = op; + *pOpIndex = *pOpIndex + 1; + break; + } + } + + if (nextOp == nullptr) + { + *pOpIndex = BAD_VAR_NUM; + } + + return nextOp == call; + } + //------------------------------------------------------------------------ // LateDevirtualization: re-examine calls after inlining to see if we // can do more devirtualization @@ -620,27 +659,29 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtReturnType != TYP_VOID) { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); - if (m_firstNewStmt == nullptr) + if (CanSpillCallWithoutSideEffect(call, parent)) { - m_firstNewStmt = stmt; + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + if (m_firstNewStmt == nullptr) + { + m_firstNewStmt = stmt; + } + + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + + JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); + DISPTREE(retExpr); + + *pTree = retExpr; } - - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - - JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); - DISPTREE(retExpr); - - *pTree = retExpr; } call->GetSingleInlineCandidateInfo()->exactContextHandle = context; From 778edcd13a8c9fdcc815ff17413824b8a6a4d1c5 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 05:48:59 +0900 Subject: [PATCH 31/53] Get rid of BAD_VAR_NUM --- src/coreclr/jit/fginline.cpp | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 36a51f32df042c..332c5a1ae10b0e 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -544,29 +544,20 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorOperands()) { - if (opIndex-- == 0) + if (--opIndex == 0) { - nextOp = op; + nextOp = op; *pOpIndex = *pOpIndex + 1; break; } } - if (nextOp == nullptr) - { - *pOpIndex = BAD_VAR_NUM; - } - - return nextOp == call; + return opIndex == 0 && nextOp == call; } //------------------------------------------------------------------------ From 85cd617041610f0ef6e63226ccb64f050bc0fc55 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 05:54:36 +0900 Subject: [PATCH 32/53] Add comments for CanSpillCallWithoutSideEffect --- src/coreclr/jit/fginline.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 332c5a1ae10b0e..9f949d1b3394d1 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -536,6 +536,21 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor Date: Sat, 25 Jan 2025 06:10:58 +0900 Subject: [PATCH 33/53] Use ancestors to estimate whether a call can be spilled or not --- src/coreclr/jit/fginline.cpp | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 9f949d1b3394d1..f9f4e26492ba82 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -204,14 +204,10 @@ bool Compiler::TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, i class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor { - typedef JitHashTable, unsigned> GenTreeOpIndexMap; - bool m_madeChanges = false; Statement* m_curStmt = nullptr; Statement* m_firstNewStmt = nullptr; - GenTreeOpIndexMap m_genTreeOpIndexMap; - public: enum { @@ -223,7 +219,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgetAllocator(CMK_Inlining)) { } @@ -548,31 +543,31 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorOperIs(GT_STORE_LCL_VAR)) { return true; } - unsigned* pOpIndex = m_genTreeOpIndexMap.LookupPointerOrAdd(parent, 0); - unsigned opIndex = *pOpIndex + 1; + if (m_ancestors.Bottom()->OperGet() != GT_STORE_LCL_VAR) + { + return false; + } - GenTree* nextOp = nullptr; - for (GenTree* op : parent->Operands()) + for (int idx = 0; idx < m_ancestors.Height() - 1; idx++) { - if (--opIndex == 0) + if (m_ancestors.Top(idx) != *m_ancestors.Top(idx + 1)->OperandsBegin()) { - nextOp = op; - *pOpIndex = *pOpIndex + 1; - break; + return false; } } - return opIndex == 0 && nextOp == call; + return true; } //------------------------------------------------------------------------ @@ -669,6 +664,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtReturnType != TYP_VOID) { + // TODO-CQ: We should spill the call if it has side effects instead of + // conservatively estimating whether it has side effects or not. if (CanSpillCallWithoutSideEffect(call, parent)) { Statement* stmt = m_compiler->gtNewStmt(call); From 67a7b54b2e51fa2d946baaf929e178b8bd9516d7 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 06:14:11 +0900 Subject: [PATCH 34/53] Reset m_ancestors before walking --- src/coreclr/jit/fginline.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index f9f4e26492ba82..dc90a73f65d897 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -241,6 +241,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetRootNodePointer(), nullptr); return m_firstNewStmt == nullptr ? m_curStmt : m_firstNewStmt; } From ff1576c3607faa584aa95750024d360e5e025556 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 06:18:04 +0900 Subject: [PATCH 35/53] Nit --- src/coreclr/jit/fginline.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index dc90a73f65d897..a77308348c5efb 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -544,9 +544,9 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor Date: Sat, 25 Jan 2025 06:57:55 +0900 Subject: [PATCH 36/53] Fix assertion --- src/coreclr/jit/fginline.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index a77308348c5efb..99e1967701c6e8 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -546,7 +546,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { + bool tryInline = true; // If the call is the top-level expression in a statement, and it returns void, // there will be no use of its return value, and we can just inline it directly. // In this case we don't need to create a RET_EXPR node for it. Otherwise, we @@ -686,13 +688,20 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + if (tryInline) + { + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); - JITDUMP("New inline candidate due to late devirtualization:\n"); - DISPTREE(call); + JITDUMP("New inline candidate due to late devirtualization:\n"); + DISPTREE(call); + } } } m_madeChanges = true; From dbf7c211cb5e4f8ee3ad5a987f7621e41abed946 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 07:01:43 +0900 Subject: [PATCH 37/53] Limit to GT_STORE_LCL_VAR only --- src/coreclr/jit/fginline.cpp | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 99e1967701c6e8..e284b3c853c55a 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -211,7 +211,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorOperGet() != GT_STORE_LCL_VAR) - { - return false; - } - - for (int idx = 0; idx < m_ancestors.Height() - 1; idx++) - { - if (m_ancestors.Top(idx) != *m_ancestors.Top(idx + 1)->OperandsBegin()) - { - return false; - } - } - - return true; + return false; } //------------------------------------------------------------------------ From bda1438a911e2f6c63c11dd765d4559c117c6fab Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 07:04:49 +0900 Subject: [PATCH 38/53] Oops --- src/coreclr/jit/fginline.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index e284b3c853c55a..d65d10caeda6b4 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -240,7 +240,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetRootNodePointer(), nullptr); return m_firstNewStmt == nullptr ? m_curStmt : m_firstNewStmt; } From 6d2197578ae8c8b7f7683fb497f6710e384a3122 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 07:07:37 +0900 Subject: [PATCH 39/53] Inline the check --- src/coreclr/jit/fginline.cpp | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index d65d10caeda6b4..293535c728f675 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -530,31 +530,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorOperIs(GT_STORE_LCL_VAR)) - { - return true; - } - - return false; - } - //------------------------------------------------------------------------ // LateDevirtualization: re-examine calls after inlining to see if we // can do more devirtualization @@ -651,8 +626,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtReturnType != TYP_VOID) { // TODO-CQ: We should spill the call if it has side effects instead of - // conservatively estimating whether it has side effects or not. - if (CanSpillCallWithoutSideEffect(call, parent)) + // conservatively estimating it using its parent. + if (parent == nullptr || parent->OperIs(GT_STORE_LCL_VAR)) { Statement* stmt = m_compiler->gtNewStmt(call); m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); From 05492faefae738a8573feecd62ff25afc48fbbd8 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 07:08:54 +0900 Subject: [PATCH 40/53] Remove leftovers --- src/coreclr/jit/importercalls.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index f874026377e882..61021c74f4ea37 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -7690,19 +7690,6 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (!(methAttr & CORINFO_FLG_FORCEINLINE)) { - if (compCurBB->HasFlag(BBF_INTERNAL) && ehGetBlockHndDsc(compCurBB) != nullptr) - { -#ifdef DEBUG - if (verbose) - { - printf("\nWill not inline blocks that have EH descriptor when IL offset may not be valid\n"); - } - -#endif - inlineResult->NoteFatal(InlineObservation::CALLSITE_NOT_CANDIDATE); - return; - } - /* Don't bother inline blocks that are in the filter region */ if (bbInCatchHandlerBBRange(compCurBB)) { From 7164fc37dd9c80857028c283de0ed496c389b8b6 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sat, 25 Jan 2025 17:12:39 +0900 Subject: [PATCH 41/53] Hoist the check --- src/coreclr/jit/fginline.cpp | 53 ++++++++++++++---------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 293535c728f675..c9cd5dee421387 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -609,7 +609,9 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); - if (context != nullptr) + // TODO-CQ: We should spill the call if it has side effects instead of conservatively + // estimating the side effects using its parent and blocking inlining. + if (context != nullptr && (parent == nullptr || parent->OperIs(GT_STORE_LCL_VAR))) { CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; @@ -618,48 +620,35 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { - bool tryInline = true; // If the call is the top-level expression in a statement, and it returns void, // there will be no use of its return value, and we can just inline it directly. // In this case we don't need to create a RET_EXPR node for it. Otherwise, we // need to create a RET_EXPR node for it. if (parent != nullptr || call->gtReturnType != TYP_VOID) { - // TODO-CQ: We should spill the call if it has side effects instead of - // conservatively estimating it using its parent. - if (parent == nullptr || parent->OperIs(GT_STORE_LCL_VAR)) + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + if (m_firstNewStmt == nullptr) { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); - if (m_firstNewStmt == nullptr) - { - m_firstNewStmt = stmt; - } - - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - - JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); - DISPTREE(retExpr); - - *pTree = retExpr; + m_firstNewStmt = stmt; } - else - { - tryInline = false; - } - } - if (tryInline) - { - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - JITDUMP("New inline candidate due to late devirtualization:\n"); - DISPTREE(call); + JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); + DISPTREE(retExpr); + + *pTree = retExpr; } + + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + + JITDUMP("New inline candidate due to late devirtualization:\n"); + DISPTREE(call); } } m_madeChanges = true; From 0415063c1a49856376fac19300ddeccb9a9a607d Mon Sep 17 00:00:00 2001 From: Steve Date: Sun, 26 Jan 2025 01:33:05 +0900 Subject: [PATCH 42/53] Make sure the store parent is the statement root Co-authored-by: Andy Ayers --- src/coreclr/jit/fginline.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index c9cd5dee421387..523b0b2529e066 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -611,7 +611,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorOperIs(GT_STORE_LCL_VAR))) + if (context != nullptr && (parent == nullptr || (parent->OperIs(GT_STORE_LCL_VAR) && (parent == m_curStmt->GetRootNode())))) { CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; From 1eb0422d156d31e94f7e0551bb7833aeb6c12654 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sun, 26 Jan 2025 01:34:58 +0900 Subject: [PATCH 43/53] Format JIT --- src/coreclr/jit/fginline.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 523b0b2529e066..6689ad8a36c690 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -611,7 +611,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorOperIs(GT_STORE_LCL_VAR) && (parent == m_curStmt->GetRootNode())))) + if (context != nullptr && + (parent == nullptr || (parent->OperIs(GT_STORE_LCL_VAR) && (parent == m_curStmt->GetRootNode())))) { CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; From 9d72f3a447c368e4e5c46a8c0db73be11cfed324 Mon Sep 17 00:00:00 2001 From: Steven He Date: Mon, 27 Jan 2025 02:44:35 +0900 Subject: [PATCH 44/53] Check side effects before trying inlining --- src/coreclr/jit/compiler.h | 2 + src/coreclr/jit/fginline.cpp | 73 ++++++++++++++++++++---------------- src/coreclr/jit/gentree.cpp | 66 ++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 32 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 305d013badee0b..6b100afa0f8ca7 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3725,6 +3725,8 @@ class Compiler void gtUpdateNodeOperSideEffects(GenTree* tree); + bool gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree); + // Returns "true" iff the complexity (not formally defined, but first interpretation // is #of nodes in subtree) of "tree" is greater than "limit". // (This is somewhat redundant with the "GetCostEx()/GetCostSz()" fields, but can be used diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 6689ad8a36c690..5a9c13583bed61 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -609,47 +609,56 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); - // TODO-CQ: We should spill the call if it has side effects instead of conservatively - // estimating the side effects using its parent and blocking inlining. - if (context != nullptr && - (parent == nullptr || (parent->OperIs(GT_STORE_LCL_VAR) && (parent == m_curStmt->GetRootNode())))) + + if (context != nullptr) { - CORINFO_CALL_INFO callInfo = {}; - callInfo.hMethod = method; - callInfo.methodFlags = methodFlags; - m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); + bool areSideEffectsFirstExecuted = + m_compiler->gtSubTreeAndChildrenAreFirstExecutedEffects(m_curStmt->GetRootNode(), call); + + // The aggressive spilling in the importer guaranteed the subtree of a call is always + // the first executed side effect. + // See https://github.com/dotnet/runtime/issues/72323. + assert(areSideEffectsFirstExecuted); - if (call->IsInlineCandidate()) + if (areSideEffectsFirstExecuted) { - // If the call is the top-level expression in a statement, and it returns void, - // there will be no use of its return value, and we can just inline it directly. - // In this case we don't need to create a RET_EXPR node for it. Otherwise, we - // need to create a RET_EXPR node for it. - if (parent != nullptr || call->gtReturnType != TYP_VOID) + CORINFO_CALL_INFO callInfo = {}; + callInfo.hMethod = method; + callInfo.methodFlags = methodFlags; + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); + + if (call->IsInlineCandidate()) { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); - if (m_firstNewStmt == nullptr) + // If the call is the top-level expression in a statement, and it returns void, + // there will be no use of its return value, and we can just inline it directly. + // In this case we don't need to create a RET_EXPR node for it. Otherwise, we + // need to create a RET_EXPR node for it. + if (parent != nullptr || call->gtReturnType != TYP_VOID) { - m_firstNewStmt = stmt; + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + if (m_firstNewStmt == nullptr) + { + m_firstNewStmt = stmt; + } + + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + + JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); + DISPTREE(retExpr); + + *pTree = retExpr; } - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); - JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); - DISPTREE(retExpr); - - *pTree = retExpr; + JITDUMP("New inline candidate due to late devirtualization:\n"); + DISPTREE(call); } - - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); - - JITDUMP("New inline candidate due to late devirtualization:\n"); - DISPTREE(call); } } m_madeChanges = true; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 29897aae4ac7ea..e9039fd473f46f 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -10213,6 +10213,72 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) }); } +//------------------------------------------------------------------------ +// gtSubTreeAndChildrenAreFirstExecutedEffects: Check whether the subtree is the first +// executed effect in a tree +// +// Arguments: +// tree - Tree to check the side effects on +// subTree - The subtree to be the first executed effect +// +// Returns: +// A boolean that indicates whether the subtree is the first executed effect in a tree +// +bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree) +{ + struct Visitor : GenTreeVisitor + { + enum + { + DoPreOrder = true, + DoPostOrder = true, + UseExecutionOrder = true, + }; + + Visitor(Compiler* comp, GenTree* subTree) + : GenTreeVisitor(comp) + , m_subTree(subTree) + { + } + + fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) + { + if (*use == m_subTree) + { + Result = true; + return WALK_ABORT; + } + + return WALK_CONTINUE; + } + + fgWalkResult PostOrderVisit(GenTree** use, GenTree* user) + { + if (((*use)->gtFlags & GTF_ALL_EFFECT) != GTF_EMPTY) + { + Result = false; + return WALK_ABORT; + } + + if ((*use)->OperIsAnyLocal() && m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->lvHasLdAddrOp) + { + Result = false; + return WALK_ABORT; + } + + return WALK_CONTINUE; + } + + bool Result = false; + private: + GenTree* m_subTree; + }; + + Visitor visitor(this, subTree); + visitor.WalkTree(&tree, nullptr); + return visitor.Result; +} + bool GenTree::gtSetFlags() const { return (gtFlags & GTF_SET_FLAGS) != 0; From 1e0b2d1b63dac619561162511d0799583123e742 Mon Sep 17 00:00:00 2001 From: Steven He Date: Mon, 27 Jan 2025 03:16:44 +0900 Subject: [PATCH 45/53] Fix --- src/coreclr/jit/fginline.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 5a9c13583bed61..77fc24b5c4f471 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -612,15 +612,10 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtSubTreeAndChildrenAreFirstExecutedEffects(m_curStmt->GetRootNode(), call); - // The aggressive spilling in the importer guaranteed the subtree of a call is always - // the first executed side effect. - // See https://github.com/dotnet/runtime/issues/72323. - assert(areSideEffectsFirstExecuted); - - if (areSideEffectsFirstExecuted) + if (isFirstExecutedEffect) { CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; @@ -629,10 +624,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { - // If the call is the top-level expression in a statement, and it returns void, - // there will be no use of its return value, and we can just inline it directly. - // In this case we don't need to create a RET_EXPR node for it. Otherwise, we - // need to create a RET_EXPR node for it. + // If the call is the root expression in a statement, and it returns void, + // and we can just inline it directly without creating a RET_EXPR. if (parent != nullptr || call->gtReturnType != TYP_VOID) { Statement* stmt = m_compiler->gtNewStmt(call); @@ -660,6 +653,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtTreeID); + } } m_madeChanges = true; } From a90b6de105eacf574f7a3e1573bebfa1424bbf3b Mon Sep 17 00:00:00 2001 From: Steven He Date: Mon, 27 Jan 2025 03:19:59 +0900 Subject: [PATCH 46/53] Nit --- src/coreclr/jit/fginline.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 77fc24b5c4f471..578b5ce06ccbf4 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -625,7 +625,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { // If the call is the root expression in a statement, and it returns void, - // and we can just inline it directly without creating a RET_EXPR. + // we can inline it directly without creating a RET_EXPR. if (parent != nullptr || call->gtReturnType != TYP_VOID) { Statement* stmt = m_compiler->gtNewStmt(call); From 0457417de369415710de0e485e72125619956642 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 28 Jan 2025 02:21:12 +0900 Subject: [PATCH 47/53] Make lvHasLdAddrOp check optional --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/fginline.cpp | 2 +- src/coreclr/jit/gentree.cpp | 17 +++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 0fc1e6a9d00d01..8f4ba3a76f304f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3727,7 +3727,7 @@ class Compiler void gtUpdateNodeOperSideEffects(GenTree* tree); - bool gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree); + bool gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool beforeMorph = false); // Returns "true" iff the complexity (not formally defined, but first interpretation // is #of nodes in subtree) of "tree" is greater than "limit". diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 1b7a2f0ade926c..a4855df03d6dc7 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -613,7 +613,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtSubTreeAndChildrenAreFirstExecutedEffects(m_curStmt->GetRootNode(), call); + m_compiler->gtSubTreeAndChildrenAreFirstExecutedEffects(m_curStmt->GetRootNode(), call, true); if (isFirstExecutedEffect) { diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 07e72730559fc8..b170b270940f91 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -10215,16 +10215,18 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) //------------------------------------------------------------------------ // gtSubTreeAndChildrenAreFirstExecutedEffects: Check whether the subtree is the first -// executed effect in a tree +// executed effect in a tree. // // Arguments: // tree - Tree to check the side effects on // subTree - The subtree to be the first executed effect +// beforeMorph - Whether the run is before morph, in which case we need to check +// lvHasLdAddrOp as we don't have valid GTF_GLOB_REF yet // // Returns: -// A boolean that indicates whether the subtree is the first executed effect in a tree +// A boolean that indicates whether the subtree is the first executed effect in a tree. // -bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree) +bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool beforeMorph) { struct Visitor : GenTreeVisitor { @@ -10235,9 +10237,10 @@ bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTre UseExecutionOrder = true, }; - Visitor(Compiler* comp, GenTree* subTree) + Visitor(Compiler* comp, GenTree* subTree, bool checkLocal) : GenTreeVisitor(comp) , m_subTree(subTree) + , m_checkLocal(checkLocal) { } @@ -10260,7 +10263,8 @@ bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTre return WALK_ABORT; } - if ((*use)->OperIsAnyLocal() && m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->lvHasLdAddrOp) + if (m_checkLocal && (*use)->OperIsAnyLocal() && + m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->lvHasLdAddrOp) { Result = false; return WALK_ABORT; @@ -10272,9 +10276,10 @@ bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTre bool Result = false; private: GenTree* m_subTree; + bool m_checkLocal; }; - Visitor visitor(this, subTree); + Visitor visitor(this, subTree, beforeMorph); visitor.WalkTree(&tree, nullptr); return visitor.Result; } From b27a25a8c0642b3445fcc159c16a7757f7d0ecda Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 28 Jan 2025 02:47:42 +0900 Subject: [PATCH 48/53] Rename to early --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/gentree.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 8f4ba3a76f304f..0e6fde88ee4ddc 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3727,7 +3727,7 @@ class Compiler void gtUpdateNodeOperSideEffects(GenTree* tree); - bool gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool beforeMorph = false); + bool gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool early = false); // Returns "true" iff the complexity (not formally defined, but first interpretation // is #of nodes in subtree) of "tree" is greater than "limit". diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index b170b270940f91..95efdb90ceb989 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -10220,13 +10220,13 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) // Arguments: // tree - Tree to check the side effects on // subTree - The subtree to be the first executed effect -// beforeMorph - Whether the run is before morph, in which case we need to check -// lvHasLdAddrOp as we don't have valid GTF_GLOB_REF yet +// early - Whether the run is early, in which case we need to check lvHasLdAddrOp as +// we don't have valid GTF_GLOB_REF yet // // Returns: // A boolean that indicates whether the subtree is the first executed effect in a tree. // -bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool beforeMorph) +bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool early) { struct Visitor : GenTreeVisitor { @@ -10279,7 +10279,7 @@ bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTre bool m_checkLocal; }; - Visitor visitor(this, subTree, beforeMorph); + Visitor visitor(this, subTree, early); visitor.WalkTree(&tree, nullptr); return visitor.Result; } From 6c0e11b65fa7a55421b525a15070f808ecf33761 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 28 Jan 2025 20:39:50 +0900 Subject: [PATCH 49/53] Split effects if necessary --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/fginline.cpp | 69 ++++++++++++++++----------------- src/coreclr/jit/gentree.cpp | 74 +++++++++++++++++++++++++++--------- 3 files changed, 90 insertions(+), 55 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 0e6fde88ee4ddc..3104d95daab4b3 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3727,7 +3727,7 @@ class Compiler void gtUpdateNodeOperSideEffects(GenTree* tree); - bool gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool early = false); + Statement* gtSplitEffectsForSubTree(Statement* stmt, GenTree* subTree, bool early = false); // Returns "true" iff the complexity (not formally defined, but first interpretation // is #of nodes in subtree) of "tree" is greater than "limit". diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index a4855df03d6dc7..7e5f0442c93a5c 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -612,51 +612,46 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtSubTreeAndChildrenAreFirstExecutedEffects(m_curStmt->GetRootNode(), call, true); - - if (isFirstExecutedEffect) + Statement* firstSplitStmt = m_compiler->gtSplitEffectsForSubTree(m_curStmt, call, true); + if (m_firstNewStmt == nullptr && firstSplitStmt != nullptr) { - CORINFO_CALL_INFO callInfo = {}; - callInfo.hMethod = method; - callInfo.methodFlags = methodFlags; - m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); + m_firstNewStmt = firstSplitStmt; + } + + CORINFO_CALL_INFO callInfo = {}; + callInfo.hMethod = method; + callInfo.methodFlags = methodFlags; + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); - if (call->IsInlineCandidate()) + if (call->IsInlineCandidate()) + { + // If the call is the root expression in a statement, and it returns void, + // we can inline it directly without creating a RET_EXPR. + if (parent != nullptr || call->gtReturnType != TYP_VOID) { - // If the call is the root expression in a statement, and it returns void, - // we can inline it directly without creating a RET_EXPR. - if (parent != nullptr || call->gtReturnType != TYP_VOID) + Statement* stmt = m_compiler->gtNewStmt(call); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); + if (m_firstNewStmt == nullptr) { - Statement* stmt = m_compiler->gtNewStmt(call); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_curStmt, stmt); - if (m_firstNewStmt == nullptr) - { - m_firstNewStmt = stmt; - } - - GenTreeRetExpr* retExpr = - m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), - genActualType(call->TypeGet())); - call->GetSingleInlineCandidateInfo()->retExpr = retExpr; - - JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); - DISPTREE(retExpr); - - *pTree = retExpr; + m_firstNewStmt = stmt; } - call->GetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + GenTreeRetExpr* retExpr = + m_compiler->gtNewInlineCandidateReturnExpr(call->AsCall(), + genActualType(call->TypeGet())); + call->GetSingleInlineCandidateInfo()->retExpr = retExpr; + + JITDUMP("Creating new RET_EXPR for [%06u]:\n", call->gtTreeID); + DISPTREE(retExpr); - JITDUMP("New inline candidate due to late devirtualization:\n"); - DISPTREE(call); + *pTree = retExpr; } - } - else - { - // TODO-CQ: Split the effects and enable inlining. - JITDUMP("Give up inlining call [%06u] due to side effects\n", call->gtTreeID); + + call->GetSingleInlineCandidateInfo()->exactContextHandle = context; + INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + + JITDUMP("New inline candidate due to late devirtualization:\n"); + DISPTREE(call); } } m_madeChanges = true; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 95efdb90ceb989..0d79044adc19cf 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -10214,19 +10214,22 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) } //------------------------------------------------------------------------ -// gtSubTreeAndChildrenAreFirstExecutedEffects: Check whether the subtree is the first -// executed effect in a tree. +// gtSplitEffectsForSubTree: Split any effect for a subtree to make sure the subtree +// is the first executed effect in the statement. // // Arguments: -// tree - Tree to check the side effects on +// stmt - The statement to spilt the side effects for // subTree - The subtree to be the first executed effect // early - Whether the run is early, in which case we need to check lvHasLdAddrOp as // we don't have valid GTF_GLOB_REF yet // // Returns: -// A boolean that indicates whether the subtree is the first executed effect in a tree. +// The first new statement that is inserted for the split effects. // -bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTree* subTree, bool early) +// Notes: +// This method requires compCurBB to be set to the block that contains the statement. +// +Statement* Compiler::gtSplitEffectsForSubTree(Statement* stmt, GenTree* subTree, bool early) { struct Visitor : GenTreeVisitor { @@ -10237,9 +10240,10 @@ bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTre UseExecutionOrder = true, }; - Visitor(Compiler* comp, GenTree* subTree, bool checkLocal) + Visitor(Compiler* comp, GenTree* subTree, Statement* stmt, bool checkLocal) : GenTreeVisitor(comp) , m_subTree(subTree) + , m_stmt(stmt) , m_checkLocal(checkLocal) { } @@ -10248,7 +10252,6 @@ bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTre { if (*use == m_subTree) { - Result = true; return WALK_ABORT; } @@ -10259,29 +10262,66 @@ bool Compiler::gtSubTreeAndChildrenAreFirstExecutedEffects(GenTree* tree, GenTre { if (((*use)->gtFlags & GTF_ALL_EFFECT) != GTF_EMPTY) { - Result = false; - return WALK_ABORT; + SplitEffect(use); + return WALK_CONTINUE; } if (m_checkLocal && (*use)->OperIsAnyLocal() && m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->lvHasLdAddrOp) { - Result = false; - return WALK_ABORT; + SplitEffect(use); + return WALK_CONTINUE; } return WALK_CONTINUE; } - bool Result = false; + void SplitEffect(GenTree** pTree) + { + Statement* stmt; + GenTree* tree = *pTree; + if (tree->IsCall() && tree->AsCall()->gtArgs.HasRetBuffer()) + { + GenTreeCall* call = tree->AsCall(); + assert(call->gtReturnType != TYP_VOID); + GenTree* retExpr = m_compiler->gtNewInlineCandidateReturnExpr(call, call->TypeGet()); + stmt = m_compiler->gtNewStmt(retExpr); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_stmt, stmt); + *pTree = retExpr; + } + else + { + unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("split effect")); + LclVarDsc* dsc = m_compiler->lvaGetDesc(temp); + dsc->lvType = tree->TypeGet(); + dsc->lvSingleDef = 1; + if (tree->TypeIs(TYP_REF)) + { + bool isExact, isNonNull; + CORINFO_CLASS_HANDLE clsHnd = m_compiler->gtGetClassHandle(tree, &isExact, &isNonNull); + m_compiler->lvaSetClass(temp, clsHnd, isExact); + } + GenTree* store = m_compiler->gtNewStoreLclVarNode(temp, tree); + stmt = m_compiler->gtNewStmt(store); + m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_stmt, stmt); + *pTree = m_compiler->gtNewLclVarNode(temp, tree->TypeGet()); + } + if (FirstNewStmt == nullptr) + { + FirstNewStmt = stmt; + } + } + + Statement* FirstNewStmt = nullptr; private: - GenTree* m_subTree; - bool m_checkLocal; + GenTree* m_subTree; + Statement* m_stmt; + bool m_checkLocal; }; - Visitor visitor(this, subTree, early); - visitor.WalkTree(&tree, nullptr); - return visitor.Result; + Visitor visitor(this, subTree, stmt, early); + visitor.WalkTree(stmt->GetRootNodePointer(), nullptr); + return visitor.FirstNewStmt; } bool GenTree::gtSetFlags() const From 5de54f8c4c97d79a6ae6264b0dc85570780cda46 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 28 Jan 2025 23:19:58 +0900 Subject: [PATCH 50/53] Use gtSplitTree --- src/coreclr/jit/compiler.h | 2 - src/coreclr/jit/fginline.cpp | 10 +++- src/coreclr/jit/gentree.cpp | 111 ----------------------------------- 3 files changed, 7 insertions(+), 116 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3104d95daab4b3..1ac131c7a0b458 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3727,8 +3727,6 @@ class Compiler void gtUpdateNodeOperSideEffects(GenTree* tree); - Statement* gtSplitEffectsForSubTree(Statement* stmt, GenTree* subTree, bool early = false); - // Returns "true" iff the complexity (not formally defined, but first interpretation // is #of nodes in subtree) of "tree" is greater than "limit". // (This is somewhat redundant with the "GetCostEx()/GetCostSz()" fields, but can be used diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 7e5f0442c93a5c..ff9413710f2432 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -612,10 +612,14 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtSplitEffectsForSubTree(m_curStmt, call, true); - if (m_firstNewStmt == nullptr && firstSplitStmt != nullptr) + Statement* newStmt = nullptr; + GenTree** callUse = nullptr; + if (m_compiler->gtSplitTree(m_compiler->compCurBB, m_curStmt, call, &newStmt, &callUse)) { - m_firstNewStmt = firstSplitStmt; + if (m_firstNewStmt == nullptr) + { + m_firstNewStmt = newStmt; + } } CORINFO_CALL_INFO callInfo = {}; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0d79044adc19cf..06854fa563b59c 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -10213,117 +10213,6 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) }); } -//------------------------------------------------------------------------ -// gtSplitEffectsForSubTree: Split any effect for a subtree to make sure the subtree -// is the first executed effect in the statement. -// -// Arguments: -// stmt - The statement to spilt the side effects for -// subTree - The subtree to be the first executed effect -// early - Whether the run is early, in which case we need to check lvHasLdAddrOp as -// we don't have valid GTF_GLOB_REF yet -// -// Returns: -// The first new statement that is inserted for the split effects. -// -// Notes: -// This method requires compCurBB to be set to the block that contains the statement. -// -Statement* Compiler::gtSplitEffectsForSubTree(Statement* stmt, GenTree* subTree, bool early) -{ - struct Visitor : GenTreeVisitor - { - enum - { - DoPreOrder = true, - DoPostOrder = true, - UseExecutionOrder = true, - }; - - Visitor(Compiler* comp, GenTree* subTree, Statement* stmt, bool checkLocal) - : GenTreeVisitor(comp) - , m_subTree(subTree) - , m_stmt(stmt) - , m_checkLocal(checkLocal) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - if (*use == m_subTree) - { - return WALK_ABORT; - } - - return WALK_CONTINUE; - } - - fgWalkResult PostOrderVisit(GenTree** use, GenTree* user) - { - if (((*use)->gtFlags & GTF_ALL_EFFECT) != GTF_EMPTY) - { - SplitEffect(use); - return WALK_CONTINUE; - } - - if (m_checkLocal && (*use)->OperIsAnyLocal() && - m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->lvHasLdAddrOp) - { - SplitEffect(use); - return WALK_CONTINUE; - } - - return WALK_CONTINUE; - } - - void SplitEffect(GenTree** pTree) - { - Statement* stmt; - GenTree* tree = *pTree; - if (tree->IsCall() && tree->AsCall()->gtArgs.HasRetBuffer()) - { - GenTreeCall* call = tree->AsCall(); - assert(call->gtReturnType != TYP_VOID); - GenTree* retExpr = m_compiler->gtNewInlineCandidateReturnExpr(call, call->TypeGet()); - stmt = m_compiler->gtNewStmt(retExpr); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_stmt, stmt); - *pTree = retExpr; - } - else - { - unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("split effect")); - LclVarDsc* dsc = m_compiler->lvaGetDesc(temp); - dsc->lvType = tree->TypeGet(); - dsc->lvSingleDef = 1; - if (tree->TypeIs(TYP_REF)) - { - bool isExact, isNonNull; - CORINFO_CLASS_HANDLE clsHnd = m_compiler->gtGetClassHandle(tree, &isExact, &isNonNull); - m_compiler->lvaSetClass(temp, clsHnd, isExact); - } - GenTree* store = m_compiler->gtNewStoreLclVarNode(temp, tree); - stmt = m_compiler->gtNewStmt(store); - m_compiler->fgInsertStmtBefore(m_compiler->compCurBB, m_stmt, stmt); - *pTree = m_compiler->gtNewLclVarNode(temp, tree->TypeGet()); - } - if (FirstNewStmt == nullptr) - { - FirstNewStmt = stmt; - } - } - - Statement* FirstNewStmt = nullptr; - private: - GenTree* m_subTree; - Statement* m_stmt; - bool m_checkLocal; - }; - - Visitor visitor(this, subTree, stmt, early); - visitor.WalkTree(stmt->GetRootNodePointer(), nullptr); - return visitor.FirstNewStmt; -} - bool GenTree::gtSetFlags() const { return (gtFlags & GTF_SET_FLAGS) != 0; From 1e967c8c5d63876a5515e944691166d34ea0d746 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 29 Jan 2025 01:41:14 +0900 Subject: [PATCH 51/53] Teach gtSplitTree to support early use --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/fginline.cpp | 2 +- src/coreclr/jit/gentree.cpp | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 1ac131c7a0b458..cd1c78923c1483 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3790,7 +3790,7 @@ class Compiler bool ignoreRoot = false); bool gtSplitTree( - BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitPointUse); + BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitPointUse, bool early = false); bool gtStoreDefinesField( LclVarDsc* fieldVarDsc, ssize_t offset, unsigned size, ssize_t* pFieldStoreOffset, unsigned* pFieldStoreSize); diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index ff9413710f2432..3f6951d1399407 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -614,7 +614,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtSplitTree(m_compiler->compCurBB, m_curStmt, call, &newStmt, &callUse)) + if (m_compiler->gtSplitTree(m_compiler->compCurBB, m_curStmt, call, &newStmt, &callUse, true)) { if (m_firstNewStmt == nullptr) { diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 06854fa563b59c..7c86705f00dd3e 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -17005,6 +17005,8 @@ bool Compiler::gtTreeHasSideEffects(GenTree* tree, GenTreeFlags flags /* = GTF_S // firstNewStmt - [out] The first new statement that was introduced. // [firstNewStmt..stmt) are the statements added by this function. // splitNodeUse - The use of the tree to split at. +// early - The run is in the early phase where we still don't have valid +// GTF_GLOB_REF yet. // // Notes: // This method turns all non-invariant nodes that would be executed before @@ -17025,14 +17027,19 @@ bool Compiler::gtTreeHasSideEffects(GenTree* tree, GenTreeFlags flags /* = GTF_S // Returns: // True if any changes were made; false if nothing needed to be done to split the tree. // -bool Compiler::gtSplitTree( - BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitNodeUse) +bool Compiler::gtSplitTree(BasicBlock* block, + Statement* stmt, + GenTree* splitPoint, + Statement** firstNewStmt, + GenTree*** splitNodeUse, + bool early) { class Splitter final : public GenTreeVisitor { BasicBlock* m_bb; Statement* m_splitStmt; GenTree* m_splitNode; + bool m_early; struct UseInfo { @@ -17049,11 +17056,12 @@ bool Compiler::gtSplitTree( UseExecutionOrder = true }; - Splitter(Compiler* compiler, BasicBlock* bb, Statement* stmt, GenTree* splitNode) + Splitter(Compiler* compiler, BasicBlock* bb, Statement* stmt, GenTree* splitNode, bool early) : GenTreeVisitor(compiler) , m_bb(bb) , m_splitStmt(stmt) , m_splitNode(splitNode) + , m_early(early) , m_useStack(compiler->getAllocator(CMK_ArrayStack)) { } @@ -17195,7 +17203,8 @@ bool Compiler::gtSplitTree( return; } - if ((*use)->OperIs(GT_LCL_VAR) && !m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->IsAddressExposed()) + if ((*use)->OperIs(GT_LCL_VAR) && !m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->IsAddressExposed() && + !(m_early && m_compiler->lvaGetDesc((*use)->AsLclVarCommon())->lvHasLdAddrOp)) { // The splitting we do here should always guarantee that we // only introduce locals for the tree edges that overlap the @@ -17278,7 +17287,7 @@ bool Compiler::gtSplitTree( } }; - Splitter splitter(this, block, stmt, splitPoint); + Splitter splitter(this, block, stmt, splitPoint, early); splitter.WalkTree(stmt->GetRootNodePointer(), nullptr); *firstNewStmt = splitter.FirstStatement; *splitNodeUse = splitter.SplitNodeUse; From 10cd910ff663ca84a09cdd2f97dc4299d2a54f25 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 29 Jan 2025 02:08:06 +0900 Subject: [PATCH 52/53] Cleanup --- src/coreclr/jit/fginline.cpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 3f6951d1399407..ac1abe7d9bb9a1 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -610,18 +610,9 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorimpDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); - if (context != nullptr) + if (!call->IsVirtual()) { - Statement* newStmt = nullptr; - GenTree** callUse = nullptr; - if (m_compiler->gtSplitTree(m_compiler->compCurBB, m_curStmt, call, &newStmt, &callUse, true)) - { - if (m_firstNewStmt == nullptr) - { - m_firstNewStmt = newStmt; - } - } - + assert(context != nullptr); CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; callInfo.methodFlags = methodFlags; @@ -629,6 +620,16 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsInlineCandidate()) { + Statement* newStmt = nullptr; + GenTree** callUse = nullptr; + if (m_compiler->gtSplitTree(m_compiler->compCurBB, m_curStmt, call, &newStmt, &callUse, true)) + { + if (m_firstNewStmt == nullptr) + { + m_firstNewStmt = newStmt; + } + } + // If the call is the root expression in a statement, and it returns void, // we can inline it directly without creating a RET_EXPR. if (parent != nullptr || call->gtReturnType != TYP_VOID) @@ -657,6 +658,10 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorClearInlineInfo(); + } } m_madeChanges = true; } From 067064fd0b4952b5638ac4429735a9f2a379cef6 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 29 Jan 2025 04:00:18 +0900 Subject: [PATCH 53/53] ClearInlineInfo is not needed --- src/coreclr/jit/fginline.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index ac1abe7d9bb9a1..ba55f989961d8d 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -658,10 +658,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorClearInlineInfo(); - } } m_madeChanges = true; }