Add WPM models for testing

ggml-org · May 23, 2024 · e92c3f8 · e92c3f8
1 parent 76b4dca
commit e92c3f8
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py
@@ -328,8 +328,10 @@ def func_tokenize2(text: str):
     # import os
     # tokenizers = os.listdir(path_tokenizers)
     tokenizers = [
-        "llama-spm",   # SPM
-        "phi-3",       # SPM
+        # "llama-spm",   # SPM
+        # "phi-3",       # SPM
+        "jina-v2-en",  # WPM
+        "bert-bge",    # WPM
     ]
 
     for tokenizer in tokenizers: