fix non

Signed-off-by: Yi Dong <[email protected]>
NVIDIA · Oct 7, 2023 · 3ddd9cd · 3ddd9cd
1 parent 019afa4
commit 3ddd9cd
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/nemo/collections/nlp/data/language_modeling/megatron/gpt_sft_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/gpt_sft_dataset.py
@@ -107,7 +107,8 @@ def __init__(
                 "end_of_turn": "\n",
                 "end_of_name": "\n",
             }
-        self.special_tokens = special_tokens
+        else:
+            self.special_tokens = special_tokens
 
         if hf_dataset:
             self.indexed_dataset = load_dataset(