Handle > 64 registers + predicate registers for Arm64

src/coreclr/inc/patchpointinfo.h

@@ -37,7 +37,7 @@ struct PatchpointInfo
     // Initialize
     void Initialize(unsigned localCount, int totalFrameSize)
     {
-        m_calleeSaveRegisters     = 0;
+        m_calleeSaveGprRegisters     = 0;
         m_totalFrameSize          = totalFrameSize;
         m_numberOfLocals          = localCount;
         m_genericContextArgOffset = -1;
@@ -49,7 +49,7 @@ struct PatchpointInfo
     // Copy
     void Copy(const PatchpointInfo* original)
     {
-        m_calleeSaveRegisters = original->m_calleeSaveRegisters;
+        m_calleeSaveGprRegisters = original->m_calleeSaveGprRegisters;
         m_genericContextArgOffset = original->m_genericContextArgOffset;
         m_keptAliveThisOffset = original->m_keptAliveThisOffset;
         m_securityCookieOffset = original->m_securityCookieOffset;
@@ -163,14 +163,14 @@ struct PatchpointInfo
     // Callee save registers saved by the original method.
     // Includes all saves that must be restored (eg includes pushed RBP on x64).
     //
-    uint64_t CalleeSaveRegisters() const
+    uint64_t CalleeSaveGprRegisters() const
     {
-        return m_calleeSaveRegisters;
+        return m_calleeSaveGprRegisters;
     }
 
-    void SetCalleeSaveRegisters(uint64_t registerMask)
+    void SetCalleeSaveGprRegisters(uint64_t gprRegs)
     {
-        m_calleeSaveRegisters = registerMask;
+        m_calleeSaveGprRegisters = gprRegs;
     }
 
 private:
@@ -180,7 +180,7 @@ struct PatchpointInfo
         EXPOSURE_MASK = 0x1
     };
 
-    uint64_t m_calleeSaveRegisters;
+    uint64_t m_calleeSaveGprRegisters;
     unsigned m_numberOfLocals;
     int      m_totalFrameSize;
     int      m_genericContextArgOffset;

src/coreclr/jit/abi.cpp

@@ -48,10 +48,10 @@ regNumber ABIPassingSegment::GetRegister() const
 // Return Value:
 //   The register mask.
 //
-regMaskTP ABIPassingSegment::GetRegisterMask() const
+regMaskOnlyOne ABIPassingSegment::GetRegisterMask() const
 {
     assert(IsPassedInRegister());
-    regMaskTP reg = genRegMask(m_register);
+    regMaskOnlyOne reg = genRegMask(m_register);
 
 #ifdef TARGET_ARM
     if (genIsValidFloatReg(m_register) && (Size == 8))

src/coreclr/jit/abi.h

@@ -25,7 +25,7 @@ class ABIPassingSegment
     // If this segment is passed in a register, return the particular register.
     regNumber GetRegister() const;
 
-    regMaskTP GetRegisterMask() const;
+    regMaskOnlyOne GetRegisterMask() const;
 
     // If this segment is passed on the stack then return the particular stack
     // offset, relative to the base of stack arguments.

src/coreclr/jit/block.h

@@ -1601,8 +1601,8 @@ struct BasicBlock : private LIR::Range
                              // is bbCodeOffsEnd - bbCodeOffs, assuming neither are BAD_IL_OFFSET.
 
 #ifdef DEBUG
-    void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???" for
-                                  // BAD_IL_OFFSET.
+    void dspBlockILRange() const; // Display the block's IL range as [XXX...YYY), where XXX and YYY might be "???"
+                                  // for BAD_IL_OFFSET.
 #endif                            // DEBUG
 
     VARSET_TP bbVarUse; // variables used     by block (before a definition)
@@ -1642,8 +1642,8 @@ struct BasicBlock : private LIR::Range
     };
     static MemoryPhiArg* EmptyMemoryPhiDef; // Special value (0x1, FWIW) to represent a to-be-filled in Phi arg list
                                             // for Heap.
-    MemoryPhiArg* bbMemorySsaPhiFunc[MemoryKindCount]; // If the "in" Heap SSA var is not a phi definition, this value
-                                                       // is NULL.
+    MemoryPhiArg* bbMemorySsaPhiFunc[MemoryKindCount]; // If the "in" Heap SSA var is not a phi definition, this
+                                                       // value is NULL.
     // Otherwise, it is either the special value EmptyMemoryPhiDefn, to indicate
     // that Heap needs a phi definition on entry, or else it is the linked list
     // of the phi arguments.
@@ -1677,7 +1677,8 @@ struct BasicBlock : private LIR::Range
     {
         EXPSET_TP bbCseOut;              // CSEs available on exit
         ASSERT_TP bbAssertionOut;        // assertions available on exit (global prop, local prop & !BBJ_COND)
-        ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local prop)
+        ASSERT_TP bbAssertionOutIfFalse; // assertions available on exit along false/next edge (BBJ_COND, local
+                                         // prop)
     };
 
     void* bbEmitCookie;
@@ -1981,8 +1982,8 @@ struct BasicBlock : private LIR::Range
         }
     };
 
-    // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`,
-    // normally used via BasicBlock::SuccEdges(), e.g.:
+    // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based
+    // `for`, normally used via BasicBlock::SuccEdges(), e.g.:
     //    for (FlowEdge* const succEdge : block->SuccEdges(compiler)) ...
     //
     // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation
@@ -2448,7 +2449,8 @@ inline BasicBlock* BBArrayIterator::operator*() const
     return edgeTarget->getDestinationBlock();
 }
 
-// Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and FlowEdge)
+// Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and
+// FlowEdge)
 
 inline PredEdgeList::iterator::iterator(FlowEdge* pred)
     : m_pred(pred)

src/coreclr/jit/clrjit.natvis

@@ -139,26 +139,26 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en-
         </CustomListItems>
         <Item Name="AvailableRegs mask">this-&gt;m_AvailableRegs</Item>
         <CustomListItems>
-          <Variable Name="reg" InitialValue="this->m_AvailableRegs" />
-          <Variable Name="regIndex" InitialValue="0" />
-          <Loop Condition="reg != 0">
-            <Item Condition="(reg &amp; 1) != 0">((regNumber)regIndex),en</Item>
-            <Exec>regIndex++</Exec>
-            <Exec>reg = reg >> 1</Exec>
-          </Loop>
+            <Variable Name="reg" InitialValue="this->m_AvailableRegs" />
+            <Variable Name="regIndex" InitialValue="0" />
+            <Loop Condition="reg != 0">
+                <Item Condition="(reg &amp; 1) != 0">((regNumber)regIndex),en</Item>
+                <Exec>regIndex++</Exec>
+                <Exec>reg = reg >> 1</Exec>
+            </Loop>
         </CustomListItems>
         <Item Name="RegistersWithConstants mask">this-&gt;m_RegistersWithConstants</Item>
         <CustomListItems>
-          <Variable Name="reg" InitialValue="this->m_RegistersWithConstants" />
-          <Variable Name="regIndex" InitialValue="0" />
-          <Loop Condition="reg != 0">
-            <Item Condition="(reg &amp; 1) != 0">((regNumber)regIndex),en</Item>
-            <Exec>regIndex++</Exec>
-            <Exec>reg = reg >> 1</Exec>
-          </Loop>
+            <Variable Name="reg" InitialValue="this->m_RegistersWithConstants" />
+            <Variable Name="regIndex" InitialValue="0" />
+            <Loop Condition="reg != 0">
+                <Item Condition="(reg &amp; 1) != 0">((regNumber)regIndex),en</Item>
+                <Exec>regIndex++</Exec>
+                <Exec>reg = reg >> 1</Exec>
+            </Loop>
         </CustomListItems>
     </Expand>
-  </Type>
+</Type>
 
   <Type Name="RefPosition">
     <DisplayString>[#{rpNum,d} - {refType,en}]</DisplayString>
@@ -177,6 +177,21 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en-
     </Expand>
   </Type>
 
+  <Type Name="_regMaskAll">
+      <!--<DisplayString></DisplayString>-->
+      <Expand>
+          <CustomListItems>
+              <Variable Name="reg" InitialValue="this->_combinedRegisters" />
+              <Variable Name="regIndex" InitialValue="0" />
+              <Loop Condition="reg != 0">
+                  <Item Condition="(reg &amp; 1) != 0">((regNumber)regIndex),en</Item>
+                  <Exec>regIndex++</Exec>
+                  <Exec>reg = reg >> 1</Exec>
+              </Loop>
+          </CustomListItems>
+      </Expand>
+  </Type>
+
   <Type Name="Interval">
     <DisplayString Condition="this->isUpperVector">[U{this->relatedInterval->varNum,d}, #{this->intervalIndex, d}, reg={(regNumber)physReg, en}]</DisplayString>
     <DisplayString Condition="this->isLocalVar">[V{this->varNum,d}, #{this->intervalIndex, d}, reg={(regNumber)physReg, en}]</DisplayString>

src/coreclr/jit/codegen.h

@@ -261,9 +261,9 @@ class CodeGen final : public CodeGenInterface
     // Prolog functions and data (there are a few exceptions for more generally used things)
     //
 
-    void      genEstablishFramePointer(int delta, bool reportUnwindData);
-    void      genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed);
-    regMaskTP genGetParameterHomingTempRegisterCandidates();
+    void        genEstablishFramePointer(int delta, bool reportUnwindData);
+    void        genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed);
+    RegBitSet64 genGetParameterHomingTempRegisterCandidates();
 
     var_types genParamStackStoreType(LclVarDsc* dsc, const ABIPassingSegment& seg);
     void      genSpillOrAddRegisterParam(unsigned lclNum, class RegGraph* graph);
@@ -335,16 +335,23 @@ class CodeGen final : public CodeGenInterface
         }
     };
 
-    static void genBuildRegPairsStack(regMaskTP regsMask, ArrayStack<RegPair>* regStack);
+    static void genBuildRegPairsStack(regMaskOnlyOne                regsMask,
+                                      ArrayStack<RegPair>* regStack MORE_THAN_64_REG_ARG(var_types type));
     static void genSetUseSaveNextPairs(ArrayStack<RegPair>* regStack);
 
-    static int genGetSlotSizeForRegsInMask(regMaskTP regsMask);
+    static int genGetSlotSizeForRegsInMask(regMaskOnlyOne regsMask);
 
-    void genSaveCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset);
-    void genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset);
+    void genSaveCalleeSavedRegisterGroup(regMaskOnlyOne regsMask,
+                                         int            spDelta,
+                                         int spOffset   MORE_THAN_64_REG_ARG(var_types type));
+    void genRestoreCalleeSavedRegisterGroup(regMaskOnlyOne regsMask,
+                                            int            spDelta,
+                                            int spOffset   MORE_THAN_64_REG_ARG(var_types type));
 
-    void genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowestCalleeSavedOffset, int spDelta);
-    void genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta);
+    void genSaveCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToSaveMask, int lowestCalleeSavedOffset, int spDelta);
+    void genRestoreCalleeSavedRegistersHelp(CONSTREF_AllRegsMask regsToRestoreMask,
+                                            int                  lowestCalleeSavedOffset,
+                                            int                  spDelta);
 
     void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed);
 
@@ -361,9 +368,9 @@ class CodeGen final : public CodeGenInterface
     void genStackProbe(ssize_t frameSize, regNumber rOffset, regNumber rLimit, regNumber rPageSize);
 #endif
 
-    void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn);
+    void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskGpr maskArgRegsLiveIn);
 
-    void genPoisonFrame(regMaskTP bbRegLiveIn);
+    void genPoisonFrame(regMaskGpr bbRegLiveIn);
 
 #if defined(TARGET_ARM)
 
@@ -372,11 +379,9 @@ class CodeGen final : public CodeGenInterface
 
     bool genStackPointerAdjustment(ssize_t spAdjustment, regNumber tmpReg);
 
-    void      genPushFltRegs(regMaskTP regMask);
-    void      genPopFltRegs(regMaskTP regMask);
-    regMaskTP genStackAllocRegisterMask(unsigned frameSize, regMaskTP maskCalleeSavedFloat);
-
-    regMaskTP genJmpCallArgMask();
+    void       genPushFltRegs(regMaskFloat regMask);
+    void       genPopFltRegs(regMaskFloat regMask);
+    regMaskGpr genStackAllocRegisterMask(unsigned frameSize, regMaskFloat maskCalleeSavedFloat);
 
     void genFreeLclFrame(unsigned           frameSize,
                          /* IN OUT */ bool* pUnwindStarted);
@@ -393,11 +398,12 @@ class CodeGen final : public CodeGenInterface
     // same.
     struct FuncletFrameInfoDsc
     {
-        regMaskTP fiSaveRegs;                  // Set of registers saved in the funclet prolog (includes LR)
-        unsigned  fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer
-        unsigned  fiSpDelta;                   // Stack pointer delta
-        unsigned  fiPSP_slot_SP_offset;        // PSP slot offset from SP
-        int       fiPSP_slot_CallerSP_offset;  // PSP slot offset from Caller SP
+        regMaskGpr   fiSaveGprRegs;               // Set of GPR registers saved in the funclet prolog (includes LR)
+        regMaskFloat fiSaveFloatRegs;             // Set of Float registers saved in the funclet prolog (includes LR)
+        unsigned     fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer
+        unsigned     fiSpDelta;                   // Stack pointer delta
+        unsigned     fiPSP_slot_SP_offset;        // PSP slot offset from SP
+        int          fiPSP_slot_CallerSP_offset;  // PSP slot offset from Caller SP
     };
 
     FuncletFrameInfoDsc genFuncletInfo;
@@ -409,7 +415,12 @@ class CodeGen final : public CodeGenInterface
     // same.
     struct FuncletFrameInfoDsc
     {
-        regMaskTP fiSaveRegs;                // Set of callee-saved registers saved in the funclet prolog (includes LR)
+        regMaskGpr   fiSaveGprRegs;   // Set of callee-saved GPR registers saved in the funclet prolog (includes LR)
+        regMaskFloat fiSaveFloatRegs; // Set of callee-saved float registers saved in the funclet prolog (includes LR)
+#ifdef FEATURE_MASKED_HW_INTRINSICS
+        regMaskPredicate fiSavePredicateRegs; // Set of callee-saved predicate registers saved in the funclet prolog
+                                              // (includes LR)
+#endif
         int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function
                                              // (negative)
         int fiSP_to_FPLR_save_delta;         // FP/LR register save offset from SP (positive)
@@ -444,7 +455,7 @@ class CodeGen final : public CodeGenInterface
     // and used by all funclet prologs and epilogs, which must all be the same.
     struct FuncletFrameInfoDsc
     {
-        regMaskTP fiSaveRegs;                // Set of callee-saved registers saved in the funclet prolog (includes RA)
+        regMaskMixed fiSaveRegs;             // Set of callee-saved registers saved in the funclet prolog (includes RA)
         int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function
                                              // (negative)
         int fiSP_to_CalleeSaved_delta;       // CalleeSaved register save offset from SP (positive)
@@ -463,7 +474,7 @@ class CodeGen final : public CodeGenInterface
     // and used by all funclet prologs and epilogs, which must all be the same.
     struct FuncletFrameInfoDsc
     {
-        regMaskTP fiSaveRegs;                // Set of callee-saved registers saved in the funclet prolog (includes RA)
+        regMaskMixed fiSaveRegs;             // Set of callee-saved registers saved in the funclet prolog (includes RA)
         int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function
                                              // (negative)
         int fiSP_to_CalleeSaved_delta;       // CalleeSaved register save offset from SP (positive)
@@ -485,7 +496,7 @@ class CodeGen final : public CodeGenInterface
 
 #endif // TARGET_XARCH
 
-    void genZeroInitFltRegs(const regMaskTP& initFltRegs, const regMaskTP& initDblRegs, const regNumber& initReg);
+    void genZeroInitFltRegs(const regMaskFloat& initFltRegs, const regMaskFloat& initDblRegs, const regNumber& initReg);
 
     regNumber genGetZeroReg(regNumber initReg, bool* pInitRegZeroed);
 
@@ -533,7 +544,7 @@ class CodeGen final : public CodeGenInterface
     //
 
 #if defined(TARGET_ARM)
-    bool genCanUsePopToReturn(regMaskTP maskPopRegsInt, bool jmpEpilog);
+    bool genCanUsePopToReturn(bool jmpEpilog);
 #endif
 
 #if defined(TARGET_ARM64)
@@ -545,7 +556,7 @@ class CodeGen final : public CodeGenInterface
     void genPopCalleeSavedRegisters(bool jmpEpilog = false);
 
 #if defined(TARGET_XARCH)
-    unsigned genPopCalleeSavedRegistersFromMask(regMaskTP rsPopRegs);
+    unsigned genPopCalleeSavedRegistersFromMask(regMaskGpr rsPopRegs);
 #endif // !defined(TARGET_XARCH)
 
 #endif // !defined(TARGET_ARM64)
@@ -671,10 +682,10 @@ class CodeGen final : public CodeGenInterface
     //
     //-------------------------------------------------------------------------
 
-    void      genSinglePush();
-    void      genSinglePop();
-    regMaskTP genPushRegs(regMaskTP regs, regMaskTP* byrefRegs, regMaskTP* noRefRegs);
-    void      genPopRegs(regMaskTP regs, regMaskTP byrefRegs, regMaskTP noRefRegs);
+    void       genSinglePush();
+    void       genSinglePop();
+    regMaskGpr genPushRegs(regMaskGpr regs, regMaskGpr* byrefRegs, regMaskGpr* noRefRegs);
+    void       genPopRegs(regMaskGpr regs, regMaskGpr byrefRegs, regMaskGpr noRefRegs);
 
     /*
     XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX