optimize

sgl-project · Oct 4, 2024 · 7ec9132 · 7ec9132
1 parent 3834869
commit 7ec9132
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py
@@ -444,6 +444,9 @@ def handle_embedding_request(
         self.waiting_queue.append(req)
 
     def get_new_prefill_batch(self) -> Optional[ScheduleBatch]:
+        if len(self.waiting_queue) == 0 and self.current_inflight_req is None:
+            return None
+
         running_bs = (
             len(self.running_batch.reqs) if self.running_batch is not None else 0
         )

diff --git a/test/srt/test_bench_serving.py b/test/srt/test_bench_serving.py
@@ -25,7 +25,7 @@ def test_offline_throughput_default(self):
     def test_offline_throughput_non_stream_small_batch_size(self):
         res = run_bench_serving(
             model=DEFAULT_MODEL_NAME_FOR_TEST,
-            num_prompts=50,
+            num_prompts=200,
             request_rate=float("inf"),
             dataset_name="sharegpt",
             random_input_len=None,