From 8563a18af74655e430427ed487d3a9e4d387297a Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 11 Feb 2025 15:09:46 +0900 Subject: [PATCH 01/15] Track InlinersContext in LateDevirtualizationInfo --- src/coreclr/jit/fginline.cpp | 6 ++++-- src/coreclr/jit/importercalls.cpp | 1 + src/coreclr/jit/inline.h | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index d66609dc788b4c..2a5f6d86f1f3c5 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -590,6 +590,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtCallMethHnd; unsigned methodFlags = 0; const bool isLateDevirtualization = true; @@ -597,7 +598,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0) { - context = call->gtLateDevirtualizationInfo->exactContextHnd; + context = call->gtLateDevirtualizationInfo->exactContextHnd; + inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext; // Note: we might call this multiple times for the same trees. // If the devirtualization below succeeds, the call becomes // non-virtual and we won't get here again. If it does not @@ -653,7 +655,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); + call->GetSingleInlineCandidateInfo()->inlinersContext = inlinersContext; JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 319466fd94071d..fac6a47b8fc6e2 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1444,6 +1444,7 @@ var_types Compiler::impImportCall(OPCODE opcode, origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; info->exactContextHnd = exactContextHnd; + info->inlinersContext = compInlineContext; origCall->gtLateDevirtualizationInfo = info; } diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 426e6575973d4c..c56f037f2f03f1 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -634,6 +634,7 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo struct LateDevirtualizationInfo { CORINFO_CONTEXT_HANDLE exactContextHnd; + InlineContext* inlinersContext; }; // InlArgInfo describes inline candidate argument properties. From 3678070cb5d381a7e3fda8b350a98c8a97b2ac6a Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 11 Feb 2025 16:30:51 +0900 Subject: [PATCH 02/15] Rework fix --- src/coreclr/jit/fginline.cpp | 6 ++---- src/coreclr/jit/importercalls.cpp | 1 - src/coreclr/jit/inline.h | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 2a5f6d86f1f3c5..baa5e43c0d5b22 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -590,7 +590,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtCallMethHnd; unsigned methodFlags = 0; const bool isLateDevirtualization = true; @@ -598,8 +597,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0) { - context = call->gtLateDevirtualizationInfo->exactContextHnd; - inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext; + context = call->gtLateDevirtualizationInfo->exactContextHnd; // Note: we might call this multiple times for the same trees. // If the devirtualization below succeeds, the call becomes // non-virtual and we won't get here again. If it does not @@ -655,7 +653,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; - call->GetSingleInlineCandidateInfo()->inlinersContext = inlinersContext; + INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index fac6a47b8fc6e2..319466fd94071d 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1444,7 +1444,6 @@ var_types Compiler::impImportCall(OPCODE opcode, origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; info->exactContextHnd = exactContextHnd; - info->inlinersContext = compInlineContext; origCall->gtLateDevirtualizationInfo = info; } diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index c56f037f2f03f1..426e6575973d4c 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -634,7 +634,6 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo struct LateDevirtualizationInfo { CORINFO_CONTEXT_HANDLE exactContextHnd; - InlineContext* inlinersContext; }; // InlArgInfo describes inline candidate argument properties. From 1173d971381c5d02424528e44d65f0f78e97978e Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 11 Feb 2025 18:21:24 +0900 Subject: [PATCH 03/15] Update inline context for statement as well --- src/coreclr/jit/fginline.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index baa5e43c0d5b22..2eaa7b4eec82a2 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -653,7 +653,12 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext); +#ifdef DEBUG + // Update inline context for Debug. + call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext; + DebugInfo debugInfo(call->gtInlineContext, m_curStmt->GetDebugInfo().GetLocation()); + m_curStmt->SetDebugInfo(debugInfo); +#endif JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); From dc917f276ab2f9d415005a4da27472d466e67793 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 11 Feb 2025 19:28:50 +0900 Subject: [PATCH 04/15] Use IL offset from call --- src/coreclr/jit/fginline.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 2eaa7b4eec82a2..3f3bafdd343801 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -656,7 +656,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext; - DebugInfo debugInfo(call->gtInlineContext, m_curStmt->GetDebugInfo().GetLocation()); + DebugInfo debugInfo(call->gtInlineContext, ILLocation(call->gtRawILOffset, false, true)); m_curStmt->SetDebugInfo(debugInfo); #endif From 57427a417211cc2b8f0ac0c5ac60e7edbe2fd9f6 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 11 Feb 2025 20:06:37 +0900 Subject: [PATCH 05/15] Always set DebugInfo --- src/coreclr/jit/fginline.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 3f3bafdd343801..9559509595fe48 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -653,12 +653,12 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; -#ifdef DEBUG - // Update inline context for Debug. - call->gtInlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext; - DebugInfo debugInfo(call->gtInlineContext, ILLocation(call->gtRawILOffset, false, true)); + + // Update inline context and DebugInfo for the new inline candidate. + InlineContext* inlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext; + INDEBUG(call->gtInlineContext = inlineContext); + DebugInfo debugInfo(inlineContext, inlineContext->GetLocation()); m_curStmt->SetDebugInfo(debugInfo); -#endif JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); From e0e7344b39651f97755a44b644e0cdb914859605 Mon Sep 17 00:00:00 2001 From: Steven He Date: Tue, 11 Feb 2025 23:42:31 +0900 Subject: [PATCH 06/15] Track InlineContext --- src/coreclr/jit/fginline.cpp | 18 ++++++++---------- src/coreclr/jit/importercalls.cpp | 1 + src/coreclr/jit/inline.h | 1 + 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 9559509595fe48..130ab86014c564 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -590,6 +590,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorcompInlineContext; CORINFO_METHOD_HANDLE method = call->gtCallMethHnd; unsigned methodFlags = 0; const bool isLateDevirtualization = true; @@ -597,7 +598,8 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0) { - context = call->gtLateDevirtualizationInfo->exactContextHnd; + context = call->gtLateDevirtualizationInfo->exactContextHnd; + inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext; // Note: we might call this multiple times for the same trees. // If the devirtualization below succeeds, the call becomes // non-virtual and we won't get here again. If it does not @@ -613,9 +615,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsVirtual()) { assert(context != nullptr); - CORINFO_CALL_INFO callInfo = {}; - callInfo.hMethod = method; - callInfo.methodFlags = methodFlags; + assert(inlinersContext != nullptr); + m_compiler->compInlineContext = inlinersContext; + CORINFO_CALL_INFO callInfo = {}; + callInfo.hMethod = method; + callInfo.methodFlags = methodFlags; m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); if (call->IsInlineCandidate()) @@ -654,12 +658,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; - // Update inline context and DebugInfo for the new inline candidate. - InlineContext* inlineContext = call->GetSingleInlineCandidateInfo()->inlinersContext; - INDEBUG(call->gtInlineContext = inlineContext); - DebugInfo debugInfo(inlineContext, inlineContext->GetLocation()); - m_curStmt->SetDebugInfo(debugInfo); - JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); } diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 319466fd94071d..fac6a47b8fc6e2 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1444,6 +1444,7 @@ var_types Compiler::impImportCall(OPCODE opcode, origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; info->exactContextHnd = exactContextHnd; + info->inlinersContext = compInlineContext; origCall->gtLateDevirtualizationInfo = info; } diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 426e6575973d4c..c56f037f2f03f1 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -634,6 +634,7 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo struct LateDevirtualizationInfo { CORINFO_CONTEXT_HANDLE exactContextHnd; + InlineContext* inlinersContext; }; // InlArgInfo describes inline candidate argument properties. From 4b306251507f81206511de53cbb7cbf9a0d96b15 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 12 Feb 2025 00:45:19 +0900 Subject: [PATCH 07/15] Cleanup --- src/coreclr/jit/fginline.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 130ab86014c564..5db1a7f402c44b 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -656,8 +656,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; - JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); } From d01f2555d237c9c30112cacd7c12f00305b1b78c Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 12 Feb 2025 03:49:50 +0900 Subject: [PATCH 08/15] Always record late devirt info --- src/coreclr/jit/importercalls.cpp | 41 ++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index fac6a47b8fc6e2..9490e1683279b7 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1241,6 +1241,30 @@ var_types Compiler::impImportCall(OPCODE opcode, // Is it an inline candidate? impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); + + const bool isInlineCandidate = call->AsCall()->IsInlineCandidate(); + const bool isGuardedDevirtualizationCandidate = call->AsCall()->IsGuardedDevirtualizationCandidate(); + + if (!isInlineCandidate && !isGuardedDevirtualizationCandidate) + { + // If the call is virtual, and is not going to have a class probe, record the inliner's context + // for possible use during late devirt inlining. Also record the generics context if any. + // + // If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose + // important devirtualizations, we'll want to allow both a class probe and a captured context. + // + if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT) && + (call->AsCall()->gtHandleHistogramProfileCandidateInfo == nullptr)) + { + JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd), + dspPtr(compInlineContext), dspTreeID(call->AsCall())); + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; + LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; + info->exactContextHnd = exactContextHnd; + info->inlinersContext = compInlineContext; + call->AsCall()->gtLateDevirtualizationInfo = info; + } + } } // Extra checks for tail calls and tail recursion. @@ -1431,23 +1455,6 @@ var_types Compiler::impImportCall(OPCODE opcode, } else { - // If the call is virtual, and has a generics context, and is not going to have a class probe, - // record the context for possible use during late devirt. - // - // If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose - // important devirtualizations, we'll want to allow both a class probe and a captured context. - // - if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) && - (origCall->gtHandleHistogramProfileCandidateInfo == nullptr)) - { - JITDUMP("\nSaving context %p for call [%06u]\n", dspPtr(exactContextHnd), dspTreeID(origCall)); - origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; - LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; - info->exactContextHnd = exactContextHnd; - info->inlinersContext = compInlineContext; - origCall->gtLateDevirtualizationInfo = info; - } - if (isFatPointerCandidate) { // fatPointer candidates should be in statements of the form call() or var = call(). From d746d951c96657bc88adb82162aacea3b80aa519 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 12 Feb 2025 17:18:37 +0900 Subject: [PATCH 09/15] Always record late devirt info --- src/coreclr/jit/gentree.h | 4 +++- src/coreclr/jit/importercalls.cpp | 34 +++++++++++-------------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 641f0b05e1f61a..4a60ac5861f3f6 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -5758,11 +5758,13 @@ struct GenTreeCall final : public GenTree jitstd::vector* gtInlineCandidateInfoList; HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo; - LateDevirtualizationInfo* gtLateDevirtualizationInfo; + CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen }; + LateDevirtualizationInfo* gtLateDevirtualizationInfo; + // expression evaluated after args are placed which determines the control target GenTree* gtControlExpr; diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 9490e1683279b7..c703021b7e6e40 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1242,28 +1242,18 @@ var_types Compiler::impImportCall(OPCODE opcode, // Is it an inline candidate? impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); - const bool isInlineCandidate = call->AsCall()->IsInlineCandidate(); - const bool isGuardedDevirtualizationCandidate = call->AsCall()->IsGuardedDevirtualizationCandidate(); - - if (!isInlineCandidate && !isGuardedDevirtualizationCandidate) - { - // If the call is virtual, and is not going to have a class probe, record the inliner's context - // for possible use during late devirt inlining. Also record the generics context if any. - // - // If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose - // important devirtualizations, we'll want to allow both a class probe and a captured context. - // - if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT) && - (call->AsCall()->gtHandleHistogramProfileCandidateInfo == nullptr)) - { - JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd), - dspPtr(compInlineContext), dspTreeID(call->AsCall())); - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; - LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; - info->exactContextHnd = exactContextHnd; - info->inlinersContext = compInlineContext; - call->AsCall()->gtLateDevirtualizationInfo = info; - } + // If the call is virtual, record the inliner's context for possible use during late devirt inlining. + // Also record the generics context if there is any. + // + if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT)) + { + JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd), + dspPtr(compInlineContext), dspTreeID(call->AsCall())); + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; + LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; + info->exactContextHnd = exactContextHnd; + info->inlinersContext = compInlineContext; + call->AsCall()->gtLateDevirtualizationInfo = info; } } From 6a2dcd635345b8974f0843ef0ba91a20e2058a75 Mon Sep 17 00:00:00 2001 From: Steven He Date: Wed, 12 Feb 2025 18:55:18 +0900 Subject: [PATCH 10/15] Copy gtLateDevirtualizationInfo --- src/coreclr/jit/gentree.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index ea02025450b39a..c9606ae3b29646 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -10013,6 +10013,8 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree) copy->gtInlineInfoCount = tree->gtInlineInfoCount; } + copy->gtLateDevirtualizationInfo = tree->gtLateDevirtualizationInfo; + copy->gtCallType = tree->gtCallType; copy->gtReturnType = tree->gtReturnType; From 125600ac1606c2d97426b2fb462e138a23f22679 Mon Sep 17 00:00:00 2001 From: Steven He Date: Thu, 13 Feb 2025 02:09:46 +0900 Subject: [PATCH 11/15] Refactor to take InlineContext explicitly --- src/coreclr/jit/compiler.h | 5 ++++- src/coreclr/jit/fginline.cpp | 9 ++++----- src/coreclr/jit/importercalls.cpp | 25 ++++++++++++++++++------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index dd967f33cbed9e..a15392644d9cd4 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5244,6 +5244,7 @@ class Compiler CORINFO_METHOD_HANDLE fncHandle, unsigned methAttr, CORINFO_CONTEXT_HANDLE exactContextHnd, + InlineContext* inlinersContext, InlineCandidateInfo** ppInlineCandidateInfo, InlineResult* inlineResult); @@ -5265,13 +5266,15 @@ class Compiler void impMarkInlineCandidate(GenTree* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo); + CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext); void impMarkInlineCandidateHelper(GenTreeCall* call, uint8_t candidateIndex, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext, InlineResult* inlineResult); bool impTailCallRetTypeCompatible(bool allowWidening, diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 5db1a7f402c44b..a863d023f727de 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -616,11 +616,10 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorcompInlineContext = inlinersContext; - CORINFO_CALL_INFO callInfo = {}; - callInfo.hMethod = method; - callInfo.methodFlags = methodFlags; - m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); + CORINFO_CALL_INFO callInfo = {}; + callInfo.hMethod = method; + callInfo.methodFlags = methodFlags; + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, inlinersContext); if (call->IsInlineCandidate()) { diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index c703021b7e6e40..21ee81daba7507 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1030,7 +1030,8 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, + compInlineContext); } // append the call node. @@ -1240,7 +1241,7 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, compInlineContext); // If the call is virtual, record the inliner's context for possible use during late devirt inlining. // Also record the generics context if there is any. @@ -7448,6 +7449,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM +// inlinersContext -- the inliner's context // // Notes: // Mostly a wrapper for impMarkInlineCandidateHelper that also undoes @@ -7457,7 +7459,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, void Compiler::impMarkInlineCandidate(GenTree* callNode, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo) + CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext) { if (!opts.OptEnabled(CLFLG_INLINING)) { @@ -7480,7 +7483,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // Do the actual evaluation impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, - &inlineResult); + inlinersContext, &inlineResult); // Ignore non-inlineable candidates // TODO: Consider keeping them to just devirtualize without inlining, at least for interface // calls on NativeAOT, but that requires more changes elsewhere too. @@ -7503,7 +7506,8 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, const uint8_t candidatesCount = call->GetInlineCandidatesCount(); assert(candidatesCount <= 1); InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate"); - impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult); + impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, + inlinersContext, &inlineResult); } // If this call is an inline candidate or is not a guarded devirtualization @@ -7536,6 +7540,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM +// inlinersContext -- the inliner's context // // Notes: // If callNode is an inline candidate, this method sets the flag @@ -7552,6 +7557,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext, InlineResult* inlineResult) { // Let the strategy know there's another call @@ -7746,7 +7752,8 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, } InlineCandidateInfo* inlineCandidateInfo = nullptr; - impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, inlineResult); + impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, inlinersContext, &inlineCandidateInfo, + inlineResult); if (inlineResult->IsFailure()) { @@ -9054,6 +9061,7 @@ bool Compiler::impTailCallRetTypeCompatible(bool allowWideni // fncHandle - method that will be called // methAttr - attributes for the method // exactContextHnd - exact context for the method +// inlinersContext - the inliner's context // ppInlineCandidateInfo [out] - information needed later for inlining // inlineResult - result of ongoing inline evaluation // @@ -9066,6 +9074,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_METHOD_HANDLE fncHandle, unsigned methAttr, CORINFO_CONTEXT_HANDLE exactContextHnd, + InlineContext* inlinersContext, InlineCandidateInfo** ppInlineCandidateInfo, InlineResult* inlineResult) { @@ -9080,6 +9089,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_METHOD_HANDLE fncHandle; unsigned methAttr; CORINFO_CONTEXT_HANDLE exactContextHnd; + InlineContext* inlinersContext; InlineResult* result; InlineCandidateInfo** ppInlineCandidateInfo; } param; @@ -9091,6 +9101,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, param.fncHandle = fncHandle; param.methAttr = methAttr; param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle); + param.inlinersContext = inlinersContext; param.result = inlineResult; param.ppInlineCandidateInfo = ppInlineCandidateInfo; @@ -9241,7 +9252,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, pInfo->methAttr = pParam->methAttr; pInfo->initClassResult = initClassResult; pInfo->exactContextNeedsRuntimeLookup = false; - pInfo->inlinersContext = pParam->pThis->compInlineContext; + pInfo->inlinersContext = pParam->inlinersContext; // Note exactContextNeedsRuntimeLookup is reset later on, // over in impMarkInlineCandidate. From 1c711a353e2a2166731fba21943fbf5f5e2b0a5e Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 14 Feb 2025 00:42:08 +0900 Subject: [PATCH 12/15] Set inlinersContext to nullptr --- src/coreclr/jit/fginline.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index a863d023f727de..bf7de5c46d3a52 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -590,7 +590,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorcompInlineContext; + InlineContext* inlinersContext = nullptr; CORINFO_METHOD_HANDLE method = call->gtCallMethHnd; unsigned methodFlags = 0; const bool isLateDevirtualization = true; From 7f7c0f4b2f0bc37a7c921b7dc30b620884ce201d Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 14 Feb 2025 01:47:52 +0900 Subject: [PATCH 13/15] Late devirt info always available --- src/coreclr/jit/fginline.cpp | 17 +++-------------- src/coreclr/jit/gentree.h | 2 +- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index bf7de5c46d3a52..aed831829a6c15 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -589,26 +589,15 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0); + CORINFO_CONTEXT_HANDLE context = call->gtLateDevirtualizationInfo->exactContextHnd; + InlineContext* inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext; CORINFO_METHOD_HANDLE method = call->gtCallMethHnd; unsigned methodFlags = 0; const bool isLateDevirtualization = true; const bool explicitTailCall = call->IsTailPrefixedCall(); - if ((call->gtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0) - { - context = call->gtLateDevirtualizationInfo->exactContextHnd; - inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext; - // Note: we might call this multiple times for the same trees. - // If the devirtualization below succeeds, the call becomes - // non-virtual and we won't get here again. If it does not - // succeed we might get here again so we keep the late devirt - // info. - } - CORINFO_CONTEXT_HANDLE contextInput = context; - context = nullptr; m_compiler->impDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 4a60ac5861f3f6..e689490f19fa39 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -5763,7 +5763,7 @@ struct GenTreeCall final : public GenTree void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen }; - LateDevirtualizationInfo* gtLateDevirtualizationInfo; + LateDevirtualizationInfo* gtLateDevirtualizationInfo; // Always available for user virtual calls // expression evaluated after args are placed which determines the control target GenTree* gtControlExpr; From 55585bd1461d7220bd2cb7178533699483ed1af0 Mon Sep 17 00:00:00 2001 From: Steven He Date: Fri, 14 Feb 2025 01:49:32 +0900 Subject: [PATCH 14/15] Oops --- src/coreclr/jit/fginline.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index aed831829a6c15..e416af43c19462 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -598,6 +598,7 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsTailPrefixedCall(); CORINFO_CONTEXT_HANDLE contextInput = context; + context = nullptr; m_compiler->impDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, isLateDevirtualization, explicitTailCall); From 4fa6b92d4998c02da503139b6f5b275871bf7299 Mon Sep 17 00:00:00 2001 From: Steven He Date: Sun, 16 Feb 2025 01:25:49 +0900 Subject: [PATCH 15/15] Remove the reduandant flag --- src/coreclr/jit/fginline.cpp | 1 - src/coreclr/jit/gentree.h | 9 ++++----- src/coreclr/jit/importercalls.cpp | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index e416af43c19462..5b9052b28c6699 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -589,7 +589,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0); CORINFO_CONTEXT_HANDLE context = call->gtLateDevirtualizationInfo->exactContextHnd; InlineContext* inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext; CORINFO_METHOD_HANDLE method = call->gtCallMethHnd; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index e689490f19fa39..29c563f5fc5817 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4222,12 +4222,11 @@ enum GenTreeCallFlags : unsigned int GTF_CALL_M_ALLOC_SIDE_EFFECTS = 0x00100000, // this is a call to an allocator with side effects GTF_CALL_M_SUPPRESS_GC_TRANSITION = 0x00200000, // suppress the GC transition (i.e. during a pinvoke) but a separate GC safe point is required. GTF_CALL_M_EXPANDED_EARLY = 0x00800000, // the Virtual Call target address is expanded and placed in gtControlExpr in Morph rather than in Lower - GTF_CALL_M_HAS_LATE_DEVIRT_INFO = 0x01000000, // this call has late devirtualzation info - GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x02000000, // ldvirtftn on an interface type - GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed. - GTF_CALL_M_CAST_OBJ_NONNULL = 0x08000000, // if we expand this specific cast we don't need to check the input object for null + GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x01000000, // ldvirtftn on an interface type + GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x02000000, // this cast (helper call) can be expanded if it's profitable. To be removed. + GTF_CALL_M_CAST_OBJ_NONNULL = 0x04000000, // if we expand this specific cast we don't need to check the input object for null // NOTE: if needed, this flag can be removed, and we can introduce new _NONNUL cast helpers - GTF_CALL_M_STACK_ARRAY = 0x10000000, // this call is a new array helper for a stack allocated array. + GTF_CALL_M_STACK_ARRAY = 0x08000000, // this call is a new array helper for a stack allocated array. }; inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 21ee81daba7507..cb8a8b8078e8aa 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1250,7 +1250,6 @@ var_types Compiler::impImportCall(OPCODE opcode, { JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd), dspPtr(compInlineContext), dspTreeID(call->AsCall())); - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; info->exactContextHnd = exactContextHnd; info->inlinersContext = compInlineContext; @@ -8382,7 +8381,6 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, // it's a union field used for other things by virtual // stubs) call->ClearInlineInfo(); - call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO; #if defined(DEBUG) if (verbose)