55

benchmark/blog_v0_2/405b_sglang.sh

@@ -6,7 +6,7 @@
 #   wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer_config.json
 
 # Launch sglang
-# python -m sglang.launch_server --model ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --tp 8 --quant fp8 --disable-radix --mem-frac 0.87
+# python -m sglang.launch_server --model-path ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --tp 8 --quant fp8 --disable-radix --mem-frac 0.87
 
 # offline
 python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 > sglang_log11

benchmark/gsm8k/README.md

@@ -1,8 +1,3 @@
-## Download data
-```
-bash download_data.sh
-```
-
 ## Run benchmark
 
 ### Benchmark sglang

benchmark/gsm8k/bench_other.py

@@ -10,7 +10,7 @@
 from tqdm import tqdm
 
 from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate
-from sglang.utils import dump_state_text, read_jsonl
+from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl
 
 INVALID = -9999999
 
@@ -41,24 +41,28 @@ def get_answer_value(answer_str):
 
 
 def main(args):
-    lines = read_jsonl(args.data_path)
+    # Select backend
+    call_generate = get_call_generate(args)
+
+    # Read data
+    url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
+    filename = download_and_cache_file(url)
+    lines = list(read_jsonl(filename))
 
     # Construct prompts
-    k = args.num_shot
-    few_shot_examples = get_few_shot_examples(lines, k)
+    num_questions = args.num_questions
+    num_shots = args.num_shots
+    few_shot_examples = get_few_shot_examples(lines, num_shots)
 
     questions = []
     labels = []
-    for i in range(len(lines[: args.num_questions])):
+    for i in range(len(lines[:num_questions])):
         questions.append(get_one_example(lines, i, False))
         labels.append(get_answer_value(lines[i]["answer"]))
     assert all(l != INVALID for l in labels)
 
     states = [None] * len(labels)
 
-    # Select backend
-    call_generate = get_call_generate(args)
-
     # Run requests
     if args.backend != "lmql":
         # Use thread pool
@@ -113,11 +117,13 @@ async def batched_call(batch_size):
     # Compute accuracy
     acc = np.mean(np.array(preds) == np.array(labels))
     invalid = np.mean(np.array(preds) == INVALID)
-    print(f"Latency: {latency:.3f}")
-    print(f"Invalid: {invalid:.3f}")
+
+    # Print results
     print(f"Accuracy: {acc:.3f}")
+    print(f"Invalid: {invalid:.3f}")
+    print(f"Latency: {latency:.3f} s")
 
-    # Write results
+    # Dump results
     dump_state_text(f"tmp_output_{args.backend}.txt", states)
 
     with open(args.result_file, "a") as fout:
@@ -138,7 +144,7 @@ async def batched_call(batch_size):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--num-shot", type=int, default=5)
+    parser.add_argument("--num-shots", type=int, default=5)
     parser.add_argument("--data-path", type=str, default="test.jsonl")
     parser.add_argument("--num-questions", type=int, default=200)
     args = add_common_other_args_and_parse(parser)

benchmark/gsm8k/bench_sglang.py

@@ -6,11 +6,12 @@
 
 import numpy as np
 
+from sglang.api import set_default_backend
 from sglang.test.test_utils import (
     add_common_sglang_args_and_parse,
     select_sglang_backend,
 )
-from sglang.utils import dump_state_text, read_jsonl
+from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl
 
 INVALID = -9999999
 
@@ -41,15 +42,22 @@ def get_answer_value(answer_str):
 
 
 def main(args):
-    lines = read_jsonl(args.data_path)
+    # Select backend
+    set_default_backend(select_sglang_backend(args))
+
+    # Read data
+    url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
+    filename = download_and_cache_file(url)
+    lines = list(read_jsonl(filename))
 
     # Construct prompts
-    k = args.num_shot
-    few_shot_examples = get_few_shot_examples(lines, k)
+    num_questions = args.num_questions
+    num_shots = args.num_shots
+    few_shot_examples = get_few_shot_examples(lines, num_shots)
 
     questions = []
     labels = []
-    for i in range(len(lines[: args.num_questions])):
+    for i in range(len(lines[:num_questions])):
         questions.append(get_one_example(lines, i, False))
         labels.append(get_answer_value(lines[i]["answer"]))
     assert all(l != INVALID for l in labels)
@@ -72,15 +80,11 @@ def few_shot_gsm8k(s, question):
     ########## SGL Program End ##########
     #####################################
 
-    # Select backend
-    backend = select_sglang_backend(args)
-
     # Run requests
     tic = time.time()
     states = few_shot_gsm8k.run_batch(
         arguments,
         temperature=0,
-        backend=backend,
         num_threads=args.parallel,
         progress_bar=True,
     )
@@ -96,11 +100,20 @@ def few_shot_gsm8k(s, question):
     # Compute accuracy
     acc = np.mean(np.array(preds) == np.array(labels))
     invalid = np.mean(np.array(preds) == INVALID)
-    print(f"Latency: {latency:.3f}")
-    print(f"Invalid: {invalid:.3f}")
+
+    # Compute speed
+    num_output_tokens = sum(
+        s.get_meta_info("answer")["completion_tokens"] for s in states
+    )
+    output_throughput = num_output_tokens / latency
+
+    # Print results
     print(f"Accuracy: {acc:.3f}")
+    print(f"Invalid: {invalid:.3f}")
+    print(f"Latency: {latency:.3f} s")
+    print(f"Output throughput: {output_throughput:.3f} token/s")
 
-    # Write results
+    # Dump results
     dump_state_text(f"tmp_output_{args.backend}.txt", states)
 
     with open(args.result_file, "a") as fout:
@@ -121,7 +134,7 @@ def few_shot_gsm8k(s, question):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--num-shot", type=int, default=5)
+    parser.add_argument("--num-shots", type=int, default=5)
     parser.add_argument("--data-path", type=str, default="test.jsonl")
     parser.add_argument("--num-questions", type=int, default=200)
     args = add_common_sglang_args_and_parse(parser)

benchmark/hellaswag/README.md

@@ -1,8 +1,3 @@
-## Download data
-```
-wget https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl
-```
-
 ## Run benchmark
 
 ### Benchmark sglang

benchmark/hellaswag/bench_other.py

@@ -8,7 +8,7 @@
 from tqdm import tqdm
 
 from sglang.test.test_utils import add_common_other_args_and_parse, get_call_select
-from sglang.utils import read_jsonl
+from sglang.utils import download_and_cache_file, read_jsonl
 
 
 def get_one_example(lines, i, include_answer):
@@ -26,25 +26,29 @@ def get_few_shot_examples(lines, k):
 
 
 def main(args):
-    lines = read_jsonl(args.data_path)
+    # Select backend
+    call_select = get_call_select(args)
+
+    # Read data
+    url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl"
+    filename = download_and_cache_file(url)
+    lines = list(read_jsonl(filename))
 
     # Construct prompts
-    k = args.num_shot
-    few_shot_examples = get_few_shot_examples(lines, k)
+    num_questions = args.num_questions
+    num_shots = args.num_shots
+    few_shot_examples = get_few_shot_examples(lines, num_shots)
 
     questions = []
     choices = []
     labels = []
-    for i in range(len(lines[: args.num_questions])):
+    for i in range(len(lines[:num_questions])):
         questions.append(get_one_example(lines, i, False))
         choices.append(lines[i]["endings"])
         labels.append(lines[i]["label"])
 
     preds = [None] * len(labels)
 
-    # Select backend
-    call_select = get_call_select(args)
-
     # Run requests
     if args.backend != "lmql":
         # Use thread pool
@@ -65,7 +69,6 @@ def get_one_answer(i):
                         total=len(questions),
                     )
                 )
-
     else:
         # Use asyncio
         async def batched_call(batch_size):
@@ -108,7 +111,7 @@ async def batched_call(batch_size):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--num-shot", type=int, default=20)
+    parser.add_argument("--num-shots", type=int, default=20)
     parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl")
     parser.add_argument("--num-questions", type=int, default=200)
     args = add_common_other_args_and_parse(parser)

benchmark/hellaswag/bench_sglang.py

@@ -4,11 +4,12 @@
 
 import numpy as np
 
+from sglang.api import set_default_backend
 from sglang.test.test_utils import (
     add_common_sglang_args_and_parse,
     select_sglang_backend,
 )
-from sglang.utils import read_jsonl
+from sglang.utils import download_and_cache_file, read_jsonl
 
 
 def get_one_example(lines, i, include_answer):
@@ -26,16 +27,23 @@ def get_few_shot_examples(lines, k):
 
 
 def main(args):
-    lines = read_jsonl(args.data_path)
+    # Select backend
+    set_default_backend(select_sglang_backend(args))
+
+    # Read data
+    url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl"
+    filename = download_and_cache_file(url)
+    lines = list(read_jsonl(filename))
 
     # Construct prompts
-    k = args.num_shot
-    few_shot_examples = get_few_shot_examples(lines, k)
+    num_questions = args.num_questions
+    num_shots = args.num_shots
+    few_shot_examples = get_few_shot_examples(lines, num_shots)
 
     questions = []
     choices = []
     labels = []
-    for i in range(len(lines[: args.num_questions])):
+    for i in range(len(lines[:num_questions])):
         questions.append(get_one_example(lines, i, False))
         choices.append(lines[i]["endings"])
         labels.append(lines[i]["label"])
@@ -56,15 +64,11 @@ def few_shot_hellaswag(s, question, choices):
     ########## SGL Program End ##########
     #####################################
 
-    # Select backend
-    backend = select_sglang_backend(args)
-
     # Run requests
     tic = time.time()
     rets = few_shot_hellaswag.run_batch(
         arguments,
         temperature=0,
-        backend=backend,
         num_threads=args.parallel,
         progress_bar=True,
     )
@@ -95,7 +99,7 @@ def few_shot_hellaswag(s, question, choices):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--num-shot", type=int, default=20)
+    parser.add_argument("--num-shots", type=int, default=20)
     parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl")
     parser.add_argument("--num-questions", type=int, default=200)
     args = add_common_sglang_args_and_parse(parser)

examples/frontend_language/usage/llava_video/srt_example_llava_v.py

@@ -7,6 +7,7 @@
 
 import argparse
 import csv
+import json
 import os
 import time
 
@@ -223,7 +224,7 @@ def batch(video_dir, save_dir, cur_chunk, num_chunks, num_frames=16, batch_size=
         tokenizer_path=tokenizer_path,
         port=cur_port,
         additional_ports=[cur_port + 1, cur_port + 2, cur_port + 3, cur_port + 4],
-        model_override_args=model_override_args,
+        json_model_override_args=json.dumps(model_override_args),
         tp_size=1,
     )
     sgl.set_default_backend(runtime)

examples/runtime/llava_onevision/http_llava_onevision_test.py

@@ -93,12 +93,14 @@ def multi_image_stream_request_test(client):
                         "image_url": {
                             "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png"
                         },
+                        "modalities": "multi-images",
                     },
                     {
                         "type": "image_url",
                         "image_url": {
                             "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png"
                         },
+                        "modalities": "multi-images",
                     },
                     {
                         "type": "text",
@@ -218,6 +220,7 @@ def prepare_video_messages(video_path):
     frame_format = {
         "type": "image_url",
         "image_url": {"url": "data:image/jpeg;base64,{}"},
+        "modalities": "video",
     }
 
     for base64_frame in base64_frames:

python/pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
 [project.optional-dependencies]
 srt = ["aiohttp", "decord", "fastapi", "hf_transfer", "huggingface_hub", "interegular",
        "packaging", "pillow", "psutil", "pydantic", "python-multipart",
-       "torch", "uvicorn", "uvloop", "zmq",
+       "torch", "torchao", "uvicorn", "uvloop", "zmq",
        "vllm==0.5.5", "outlines>=0.0.44"]
 openai = ["openai>=1.0", "tiktoken"]
 anthropic = ["anthropic>=0.20.0"]

python/sglang/bench_latency.py

@@ -480,6 +480,7 @@ def main(server_args, bench_args):
 
 
 if __name__ == "__main__":
+    # TODO(kevin85421): Make the parser setup unit testable.
     parser = argparse.ArgumentParser()
     ServerArgs.add_cli_args(parser)
     BenchArgs.add_cli_args(parser)