Bugfix: fix broken of download models from modelscope

tests/entrypoints/test_model_from_modelscope.py

@@ -0,0 +1,20 @@
+from vllm import LLM, SamplingParams
+
+# model: https://modelscope.cn/models/qwen/Qwen1.5-0.5B-Chat/summary
+MODEL_NAME = "qwen/Qwen1.5-0.5B-Chat"
+
+
+def test_offline_inference(monkeypatch):
+    monkeypatch.setenv("VLLM_USE_MODELSCOPE", "True")
+    llm = LLM(model=MODEL_NAME)
+
+    prompts = [
+        "Hello, my name is",
+        "The president of the United States is",
+        "The capital of France is",
+        "The future of AI is",
+    ]
+    sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+
+    outputs = llm.generate(prompts, sampling_params)
+    assert len(outputs) == 4

vllm/config.py

@@ -112,7 +112,11 @@ def __init__(
         self.revision = revision
         self.code_revision = code_revision
         self.rope_scaling = rope_scaling
-        self.tokenizer_revision = tokenizer_revision
+        # The tokenizer version is consistent with the model version by default.
+        if tokenizer_revision is None:
+            self.tokenizer_revision = revision
+        else:
+            self.tokenizer_revision = tokenizer_revision
         self.quantization = quantization
         self.quantization_param_path = quantization_param_path
         self.enforce_eager = enforce_eager

vllm/transformers_utils/config.py

@@ -1,7 +1,8 @@
 from typing import Dict, Optional
 
-from transformers import AutoConfig, PretrainedConfig
+from transformers import PretrainedConfig
 
+from vllm.envs import VLLM_USE_MODELSCOPE
 from vllm.logger import init_logger
 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
                                              JAISConfig, MPTConfig, RWConfig)
@@ -24,11 +25,20 @@ def get_config(model: str,
                code_revision: Optional[str] = None,
                rope_scaling: Optional[dict] = None) -> PretrainedConfig:
     try:
-        config = AutoConfig.from_pretrained(
-            model,
-            trust_remote_code=trust_remote_code,
-            revision=revision,
-            code_revision=code_revision)
+        if VLLM_USE_MODELSCOPE:
+            from modelscope import AutoConfig
+            config = AutoConfig.from_pretrained(
+                model,
+                trust_remote_code=trust_remote_code,
+                revision=revision,
+                code_revision=code_revision)
+        else:
+            from transformers import AutoConfig
+            config = AutoConfig.from_pretrained(
+                model,
+                trust_remote_code=trust_remote_code,
+                revision=revision,
+                code_revision=code_revision)
     except ValueError as e:
         if (not trust_remote_code and
                 "requires you to execute the configuration file" in str(e)):