set torch_dtype to float16 for XPU (#875)

ModelCloud · Dec 16, 2024 · 9426835 · 9426835
1 parent f24603c
commit 9426835
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/gptqmodel/models/loader.py b/gptqmodel/models/loader.py
@@ -220,7 +220,7 @@ def from_quantized(
             **cached_file_kwargs,
         )
 
-        if torch_dtype is None or torch_dtype == "auto":
+        if torch_dtype is None or torch_dtype == "auto" or device == DEVICE.XPU:
             # TODO FIX ME for `dynamic`, non-quantized modules should be in native type
             torch_dtype = torch.float16
             # auto_dtype_from_config(config=config, device=device, device_map=device_map)