Warning for missing FP8 checkpoint support for vLLM deployment (NVIDI…

…A#10906) Signed-off-by: Jan Lasek <[email protected]>
artbataev · Oct 22, 2024 · ca66a78 · ca66a78
1 parent 9a900ec
commit ca66a78
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/nemo/export/vllm_exporter.py b/nemo/export/vllm_exporter.py
@@ -144,6 +144,11 @@ def export(
             max_seq_len_to_capture=None,
         )
 
+        if model_config.nemo_model_config.get("fp8", False):
+            LOGGER.warning(
+                "NeMo FP8 checkpoint detected, but exporting FP8 quantized engines is not supported for vLLM."
+            )
+
         parallel_config = ParallelConfig(
             pipeline_parallel_size=pipeline_parallel_size, tensor_parallel_size=tensor_parallel_size
         )