diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index dd967f33cbed9e..a15392644d9cd4 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5244,6 +5244,7 @@ class Compiler CORINFO_METHOD_HANDLE fncHandle, unsigned methAttr, CORINFO_CONTEXT_HANDLE exactContextHnd, + InlineContext* inlinersContext, InlineCandidateInfo** ppInlineCandidateInfo, InlineResult* inlineResult); @@ -5265,13 +5266,15 @@ class Compiler void impMarkInlineCandidate(GenTree* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo); + CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext); void impMarkInlineCandidateHelper(GenTreeCall* call, uint8_t candidateIndex, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext, InlineResult* inlineResult); bool impTailCallRetTypeCompatible(bool allowWidening, diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index d66609dc788b4c..5b9052b28c6699 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -589,22 +589,13 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorgtLateDevirtualizationInfo->exactContextHnd; + InlineContext* inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext; CORINFO_METHOD_HANDLE method = call->gtCallMethHnd; unsigned methodFlags = 0; const bool isLateDevirtualization = true; const bool explicitTailCall = call->IsTailPrefixedCall(); - if ((call->gtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0) - { - context = call->gtLateDevirtualizationInfo->exactContextHnd; - // Note: we might call this multiple times for the same trees. - // If the devirtualization below succeeds, the call becomes - // non-virtual and we won't get here again. If it does not - // succeed we might get here again so we keep the late devirt - // info. - } - CORINFO_CONTEXT_HANDLE contextInput = context; context = nullptr; m_compiler->impDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context, @@ -613,10 +604,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsVirtual()) { assert(context != nullptr); + assert(inlinersContext != nullptr); CORINFO_CALL_INFO callInfo = {}; callInfo.hMethod = method; callInfo.methodFlags = methodFlags; - m_compiler->impMarkInlineCandidate(call, context, false, &callInfo); + m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, inlinersContext); if (call->IsInlineCandidate()) { @@ -652,9 +644,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorGetSingleInlineCandidateInfo()->exactContextHandle = context; - INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext); - JITDUMP("New inline candidate due to late devirtualization:\n"); DISPTREE(call); } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index ea02025450b39a..c9606ae3b29646 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -10013,6 +10013,8 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree) copy->gtInlineInfoCount = tree->gtInlineInfoCount; } + copy->gtLateDevirtualizationInfo = tree->gtLateDevirtualizationInfo; + copy->gtCallType = tree->gtCallType; copy->gtReturnType = tree->gtReturnType; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 641f0b05e1f61a..29c563f5fc5817 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4222,12 +4222,11 @@ enum GenTreeCallFlags : unsigned int GTF_CALL_M_ALLOC_SIDE_EFFECTS = 0x00100000, // this is a call to an allocator with side effects GTF_CALL_M_SUPPRESS_GC_TRANSITION = 0x00200000, // suppress the GC transition (i.e. during a pinvoke) but a separate GC safe point is required. GTF_CALL_M_EXPANDED_EARLY = 0x00800000, // the Virtual Call target address is expanded and placed in gtControlExpr in Morph rather than in Lower - GTF_CALL_M_HAS_LATE_DEVIRT_INFO = 0x01000000, // this call has late devirtualzation info - GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x02000000, // ldvirtftn on an interface type - GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed. - GTF_CALL_M_CAST_OBJ_NONNULL = 0x08000000, // if we expand this specific cast we don't need to check the input object for null + GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x01000000, // ldvirtftn on an interface type + GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x02000000, // this cast (helper call) can be expanded if it's profitable. To be removed. + GTF_CALL_M_CAST_OBJ_NONNULL = 0x04000000, // if we expand this specific cast we don't need to check the input object for null // NOTE: if needed, this flag can be removed, and we can introduce new _NONNUL cast helpers - GTF_CALL_M_STACK_ARRAY = 0x10000000, // this call is a new array helper for a stack allocated array. + GTF_CALL_M_STACK_ARRAY = 0x08000000, // this call is a new array helper for a stack allocated array. }; inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a) @@ -5758,11 +5757,13 @@ struct GenTreeCall final : public GenTree jitstd::vector* gtInlineCandidateInfoList; HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo; - LateDevirtualizationInfo* gtLateDevirtualizationInfo; + CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen }; + LateDevirtualizationInfo* gtLateDevirtualizationInfo; // Always available for user virtual calls + // expression evaluated after args are placed which determines the control target GenTree* gtControlExpr; diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 319466fd94071d..cb8a8b8078e8aa 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1030,7 +1030,8 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, + compInlineContext); } // append the call node. @@ -1240,7 +1241,20 @@ var_types Compiler::impImportCall(OPCODE opcode, INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset); // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, compInlineContext); + + // If the call is virtual, record the inliner's context for possible use during late devirt inlining. + // Also record the generics context if there is any. + // + if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT)) + { + JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd), + dspPtr(compInlineContext), dspTreeID(call->AsCall())); + LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; + info->exactContextHnd = exactContextHnd; + info->inlinersContext = compInlineContext; + call->AsCall()->gtLateDevirtualizationInfo = info; + } } // Extra checks for tail calls and tail recursion. @@ -1431,22 +1445,6 @@ var_types Compiler::impImportCall(OPCODE opcode, } else { - // If the call is virtual, and has a generics context, and is not going to have a class probe, - // record the context for possible use during late devirt. - // - // If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose - // important devirtualizations, we'll want to allow both a class probe and a captured context. - // - if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) && - (origCall->gtHandleHistogramProfileCandidateInfo == nullptr)) - { - JITDUMP("\nSaving context %p for call [%06u]\n", dspPtr(exactContextHnd), dspTreeID(origCall)); - origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; - LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; - info->exactContextHnd = exactContextHnd; - origCall->gtLateDevirtualizationInfo = info; - } - if (isFatPointerCandidate) { // fatPointer candidates should be in statements of the form call() or var = call(). @@ -7450,6 +7448,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM +// inlinersContext -- the inliner's context // // Notes: // Mostly a wrapper for impMarkInlineCandidateHelper that also undoes @@ -7459,7 +7458,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, void Compiler::impMarkInlineCandidate(GenTree* callNode, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo) + CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext) { if (!opts.OptEnabled(CLFLG_INLINING)) { @@ -7482,7 +7482,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // Do the actual evaluation impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, - &inlineResult); + inlinersContext, &inlineResult); // Ignore non-inlineable candidates // TODO: Consider keeping them to just devirtualize without inlining, at least for interface // calls on NativeAOT, but that requires more changes elsewhere too. @@ -7505,7 +7505,8 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, const uint8_t candidatesCount = call->GetInlineCandidatesCount(); assert(candidatesCount <= 1); InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate"); - impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult); + impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, + inlinersContext, &inlineResult); } // If this call is an inline candidate or is not a guarded devirtualization @@ -7538,6 +7539,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // exactContextHnd -- context handle for inlining // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM +// inlinersContext -- the inliner's context // // Notes: // If callNode is an inline candidate, this method sets the flag @@ -7554,6 +7556,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo, + InlineContext* inlinersContext, InlineResult* inlineResult) { // Let the strategy know there's another call @@ -7748,7 +7751,8 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, } InlineCandidateInfo* inlineCandidateInfo = nullptr; - impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, inlineResult); + impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, inlinersContext, &inlineCandidateInfo, + inlineResult); if (inlineResult->IsFailure()) { @@ -8377,7 +8381,6 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, // it's a union field used for other things by virtual // stubs) call->ClearInlineInfo(); - call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO; #if defined(DEBUG) if (verbose) @@ -9056,6 +9059,7 @@ bool Compiler::impTailCallRetTypeCompatible(bool allowWideni // fncHandle - method that will be called // methAttr - attributes for the method // exactContextHnd - exact context for the method +// inlinersContext - the inliner's context // ppInlineCandidateInfo [out] - information needed later for inlining // inlineResult - result of ongoing inline evaluation // @@ -9068,6 +9072,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_METHOD_HANDLE fncHandle, unsigned methAttr, CORINFO_CONTEXT_HANDLE exactContextHnd, + InlineContext* inlinersContext, InlineCandidateInfo** ppInlineCandidateInfo, InlineResult* inlineResult) { @@ -9082,6 +9087,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_METHOD_HANDLE fncHandle; unsigned methAttr; CORINFO_CONTEXT_HANDLE exactContextHnd; + InlineContext* inlinersContext; InlineResult* result; InlineCandidateInfo** ppInlineCandidateInfo; } param; @@ -9093,6 +9099,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, param.fncHandle = fncHandle; param.methAttr = methAttr; param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle); + param.inlinersContext = inlinersContext; param.result = inlineResult; param.ppInlineCandidateInfo = ppInlineCandidateInfo; @@ -9243,7 +9250,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, pInfo->methAttr = pParam->methAttr; pInfo->initClassResult = initClassResult; pInfo->exactContextNeedsRuntimeLookup = false; - pInfo->inlinersContext = pParam->pThis->compInlineContext; + pInfo->inlinersContext = pParam->inlinersContext; // Note exactContextNeedsRuntimeLookup is reset later on, // over in impMarkInlineCandidate. diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 426e6575973d4c..c56f037f2f03f1 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -634,6 +634,7 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo struct LateDevirtualizationInfo { CORINFO_CONTEXT_HANDLE exactContextHnd; + InlineContext* inlinersContext; }; // InlArgInfo describes inline candidate argument properties.