[Bugfix] Fix _get_lora_device for HQQ marlin (vllm-project#12090)

Signed-off-by: Varun Sundar Rabindranath <[email protected]> Co-authored-by: Varun Sundar Rabindranath <[email protected]> Signed-off-by: Isotr0py <[email protected]>
Isotr0py · Feb 2, 2025 · 2c73b3b · 2c73b3b
1 parent 090d990
commit 2c73b3b
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/vllm/lora/layers.py b/vllm/lora/layers.py
@@ -51,6 +51,9 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
     # marlin
     elif hasattr(base_layer, "B"):
         return base_layer.B.device
+    # HQQ marlin
+    elif hasattr(base_layer, "W_q"):
+        return base_layer.W_q.device
     else:
         raise ValueError(f"Unsupported base layer: {base_layer}")