Usage of unused render states for special game scenarios

- use unused  RenderState 42 to assign a remix texture category for the next surface
- use unused RenderState 150 to set a custom material hash for the next surface
- add hidden remix option 'useUnusedRenderstates' (default: false) to enable usage of unused render states

review changes

src/d3d9/d3d9_rtx_utils.cpp

@@ -176,6 +176,11 @@ namespace dxvk {
 
     materialData.alphaBlendEnabled = d3d9State.renderStates[D3DRS_ALPHABLENDENABLE] != FALSE;
 
+    if (RtxOptions::Get()->useUnusedRenderstates()) {
+      materialData.remixTextureCategoryFlagsFromD3D = d3d9State.renderStates[42] != 0xfefefefe ? d3d9State.renderStates[42] : 0u; // D3DRENDERSTATETYPE index 42 is not in use - unused values are set to 0xfefefefe
+      materialData.remixHashFromD3D = d3d9State.renderStates[150] != 0xfefefefe ? d3d9State.renderStates[150] : 0u; // D3DRENDERSTATETYPE index 150 is not in use - unused values are set to 0xfefefefe
+    }
+
     if (materialData.alphaBlendEnabled) {
       D3D9BlendState color;
       color.Src = D3DBLEND(d3d9State.renderStates[D3DRS_SRCBLEND]);

src/dxvk/rtx_render/rtx_materials.h

@@ -1451,6 +1451,8 @@ struct LegacyMaterialData {
   uint32_t tFactor = 0xffffffff;  // Value for D3DRS_TEXTUREFACTOR, default value of is opaque white
   D3DMATERIAL9 d3dMaterial = {};
   bool isTextureFactorBlend = false;
+  uint32_t  remixTextureCategoryFlagsFromD3D = 0u;
+  XXH64_hash_t remixHashFromD3D = 0;
 
   void setHashOverride(XXH64_hash_t hash) {
     m_cachedHash = hash;
@@ -1469,6 +1471,11 @@ struct LegacyMaterialData {
     // plain data hash used by the RtSurfaceMaterial for storage in map-like data structures, but rather
     // one used to identify a material and compare to user-provided hashes.
     m_cachedHash = colorTextures[0].getImageHash();
+
+    // Custom hash set via unused D3D RenderState
+    if (remixHashFromD3D) {
+      m_cachedHash = remixHashFromD3D;
+    }
   }
 
   const static uint32_t kMaxSupportedTextures = 2;

src/dxvk/rtx_render/rtx_options.h

@@ -989,6 +989,9 @@ namespace dxvk {
     RTX_OPTION("rtx", float, effectLightRadius, 5.f, "");
     RTX_OPTION("rtx", bool, effectLightPlasmaBall, false, "");
 
+    RTX_OPTION("rtx", bool, useUnusedRenderstates, false, "Enable usage of unused D3D renderstates to aid with special game scenarios whilst having game code access.\n"
+               "(RS 42) Setting the remix texture category for the next surface. (RS 150) Setting a user defined material hash for the next surface.");
+
     RTX_OPTION("rtx", bool, useObsoleteHashOnTextureUpload, false,
                "Whether or not to use slower XXH64 hash on texture upload.\n"
                "New projects should not enable this option as this solely exists for compatibility with older hashing schemes.");

src/dxvk/rtx_render/rtx_types.cpp

@@ -133,6 +133,10 @@ namespace dxvk {
     //                   than doing N lookups per texture hash for each category.
     const XXH64_hash_t& textureHash = materialData.getColorTexture().getImageHash();
 
+    if (RtxOptions::Get()->useUnusedRenderstates()) {
+      categories = CategoryFlags(materialData.remixTextureCategoryFlagsFromD3D);
+    }
+
     setCategory(InstanceCategories::WorldUI, lookupHash(RtxOptions::worldSpaceUiTextures(), textureHash));
     setCategory(InstanceCategories::WorldMatte, lookupHash(RtxOptions::worldSpaceUiBackgroundTextures(), textureHash));
 
@@ -297,9 +301,12 @@ namespace dxvk {
       return true;
     }
 
-    // NOTE: we use color texture hash for sky detection, however the replacement is hashed with
-    // the whole legacy material hash (which, as of 12/9/2022, equals to color texture hash). Adding a check just in case.
-    assert(drawCallState.getMaterialData().getColorTexture().getImageHash() == drawCallState.getMaterialData().getHash() && "Texture or material hash method changed!");
+    // NOTE: disable assert if material is using a custom hash that was set via an unused D3D RenderState
+    if (!drawCallState.getMaterialData().remixHashFromD3D) {
+      // NOTE: we use color texture hash for sky detection, however the replacement is hashed with
+      // the whole legacy material hash (which, as of 12/9/2022, equals to color texture hash). Adding a check just in case.
+      assert(drawCallState.getMaterialData().getColorTexture().getImageHash() == drawCallState.getMaterialData().getHash() && "Texture or material hash method changed!");
+    }
 
     if (drawCallState.getMaterialData().usesTexture()) {
       if (lookupHash(RtxOptions::skyBoxTextures(), drawCallState.getMaterialData().getHash())) {