Fix bug introduced by preventing unwind through stack bottom (#81956)

The irecent fix to prevent unwinding through stack bottom was
incorrect for secondary threads, as it just compared the SP
being above the frame of the hosting API. However, other threads
can have their stacks anywhere in the memory, thus this
sometimes broke exception handling on secondary threads.

I have also found that there was one more case where the
unwind through the hosting API need to be checked - the
Thread::VirtualUnwindToFirstManagedCallFrame.

I have decided to use the return address of the hosting
API for the checks instead of the frame address. That
makes the check work properly.

src/coreclr/dlls/mscoree/exports.cpp

@@ -24,20 +24,20 @@
 #define ASSERTE_ALL_BUILDS(expr) _ASSERTE_ALL_BUILDS((expr))
 
 #ifdef TARGET_UNIX
-#define NO_HOSTING_API_FRAME_ADDRESS ((void*)ULONG_PTR_MAX)
-void* g_hostingApiFrameAddress = NO_HOSTING_API_FRAME_ADDRESS;
+#define NO_HOSTING_API_RETURN_ADDRESS ((void*)ULONG_PTR_MAX)
+void* g_hostingApiReturnAddress = NO_HOSTING_API_RETURN_ADDRESS;
 
 class HostingApiFrameHolder
 {
 public:
-    HostingApiFrameHolder(void* frameAddress)
+    HostingApiFrameHolder(void* returnAddress)
     {
-        g_hostingApiFrameAddress = frameAddress;
+        g_hostingApiReturnAddress = returnAddress;
     }
 
     ~HostingApiFrameHolder()
     {
-        g_hostingApiFrameAddress = NO_HOSTING_API_FRAME_ADDRESS;
+        g_hostingApiReturnAddress = NO_HOSTING_API_RETURN_ADDRESS;
     }
 };
 #endif // TARGET_UNIX
@@ -213,6 +213,7 @@ extern "C" int coreclr_create_delegate(void*, unsigned int, const char*, const c
 //  HRESULT indicating status of the operation. S_OK if the assembly was successfully executed
 //
 extern "C"
+NOINLINE
 DLLEXPORT
 int coreclr_initialize(
             const char* exePath,
@@ -232,7 +233,7 @@ int coreclr_initialize(
     PInvokeOverrideFn* pinvokeOverride = nullptr;
 
 #ifdef TARGET_UNIX
-    HostingApiFrameHolder apiFrameHolder(__builtin_frame_address(0));
+    HostingApiFrameHolder apiFrameHolder(_ReturnAddress());
 #endif
 
     ConvertConfigPropertiesToUnicode(
@@ -443,6 +444,7 @@ int coreclr_create_delegate(
 //  HRESULT indicating status of the operation. S_OK if the assembly was successfully executed
 //
 extern "C"
+NOINLINE
 DLLEXPORT
 int coreclr_execute_assembly(
             void* hostHandle,
@@ -459,7 +461,7 @@ int coreclr_execute_assembly(
     *exitCode = -1;
 
 #ifdef TARGET_UNIX
-    HostingApiFrameHolder apiFrameHolder(__builtin_frame_address(0));
+    HostingApiFrameHolder apiFrameHolder(_ReturnAddress());
 #endif
 
     ICLRRuntimeHost4* host = reinterpret_cast<ICLRRuntimeHost4*>(hostHandle);

src/coreclr/vm/exceptionhandling.cpp

@@ -4523,7 +4523,7 @@ VOID UnwindManagedExceptionPass2(PAL_SEHException& ex, CONTEXT* unwindStartConte
     EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);
 }
 
-extern void* g_hostingApiFrameAddress;
+extern void* g_hostingApiReturnAddress;
 
 //---------------------------------------------------------------------------------------
 //
@@ -4727,7 +4727,7 @@ VOID DECLSPEC_NORETURN UnwindManagedExceptionPass1(PAL_SEHException& ex, CONTEXT
             STRESS_LOG2(LF_EH, LL_INFO100, "Processing exception at native frame: IP = %p, SP = %p \n", controlPc, sp);
 
             // Consider the exception unhandled if the unwinding cannot proceed further or if it went past the coreclr_initialize or coreclr_execute_assembly
-            if ((controlPc == 0) || (sp > (UINT_PTR)g_hostingApiFrameAddress))
+            if ((controlPc == 0) || (controlPc == (UINT_PTR)g_hostingApiReturnAddress))
             {
                 if (!GetThread()->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException))
                 {

src/coreclr/vm/stackwalk.cpp

@@ -709,6 +709,8 @@ PCODE Thread::VirtualUnwindNonLeafCallFrame(T_CONTEXT* pContext, KNONVOLATILE_CO
     return uControlPc;
 }
 
+extern void* g_hostingApiReturnAddress;
+
 // static
 UINT_PTR Thread::VirtualUnwindToFirstManagedCallFrame(T_CONTEXT* pContext)
 {
@@ -751,8 +753,9 @@ UINT_PTR Thread::VirtualUnwindToFirstManagedCallFrame(T_CONTEXT* pContext)
 
         uControlPc = GetIP(pContext);
 
-        if (uControlPc == 0)
+        if ((uControlPc == 0) || (uControlPc == (PCODE)g_hostingApiReturnAddress))
         {
+            uControlPc = 0;
             break;
         }
 #endif // !TARGET_UNIX