Warning for missing FP8 checkpoint support for vLLM deployment (NVIDI…

…A#10906) Signed-off-by: Jan Lasek <[email protected]> Signed-off-by: Hainan Xu <[email protected]>
hainan-xv · Nov 5, 2024 · b09e27b · b09e27b
1 parent 72d53c0
commit b09e27b
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/nemo/export/vllm_exporter.py b/nemo/export/vllm_exporter.py
@@ -144,6 +144,11 @@ def export(
             max_seq_len_to_capture=None,
         )
 
+        if model_config.nemo_model_config.get("fp8", False):
+            LOGGER.warning(
+                "NeMo FP8 checkpoint detected, but exporting FP8 quantized engines is not supported for vLLM."
+            )
+
         parallel_config = ParallelConfig(
             pipeline_parallel_size=pipeline_parallel_size, tensor_parallel_size=tensor_parallel_size
         )