docker-compose.yml with new speculative decoding doesn't seem to work

[daviddl9]

I've tried to use the new speculative decoding introduced in this MR: #2150. From the PR, this seems to be usage for speculative decoding:

version: '3.8'

services:
  sglang:
    image: lmsysorg/sglang:latest
    container_name: sglang
    volumes:
      - ${HOME}/.cache/huggingface:/root/.cache/huggingface
    restart: always
    network_mode: host
    environment:
      - HF_TOKEN=<token>
    entrypoint: python3 -m sglang.launch_server
    command:
      - --model-path
      - meta-llama/Llama-3.1-8B-Instruct
      - --speculative-algo
      - EAGLE
      - --speculative-draft
      - lmzheng/sglang-EAGLE-llama3-instruct-8B
      - --speculative-num-steps
      - "5"
      - --speculative-eagle-topk
      - "8"
      - --speculative-num-draft-tokens
      - "64"
      - --mem-fraction
      - "0.7"
      - --enable-torch-compile
      - --cuda-graph-max-bs
      - "2"
      - --host
      - 0.0.0.0
      - --port
      - "30000"
    ulimits:
      memlock: -1
      stack: 67108864
    ipc: host
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ['0','1']
              capabilities: [gpu]

am running on a machine with 2 A100 GPUs.

I run into this error:

sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 508, in kill_process_tree
sglang    |     itself.send_signal(signal.SIGQUIT)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1285, in send_signal
sglang    |     self._send_signal(sig)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1266, in _send_signal
sglang    |     os.kill(self.pid, sig)
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py", line 332, in sigquit_handler
sglang    |     kill_process_tree(os.getpid())
sglang    |   File "/sgl-workspace/sglang/python/sglang/srt/utils.py", line 498, in kill_process_tree
sglang    |     child.kill()
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1331, in kill
sglang    |     self._send_signal(signal.SIGKILL)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 1256, in _send_signal
sglang    |     self._raise_if_pid_reused()
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 461, in _raise_if_pid_reused
sglang    |     if self._pid_reused or (not self.is_running() and self._pid_reused):
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 636, in is_running
sglang    |     self._pid_reused = self != Process(self.pid)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 319, in __init__
sglang    |     self._init(pid)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 355, in _init
sglang    |     self._ident = self._get_ident()
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 396, in _get_ident
sglang    |     return (self.pid, self.create_time())
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/__init__.py", line 778, in create_time
sglang    |     self._create_time = self._proc.create_time()
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_pslinux.py", line 1716, in wrapper
sglang    |     return fun(self, *args, **kwargs)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_pslinux.py", line 1957, in create_time
sglang    |     ctime = float(self._parse_stat_file()['create_time'])
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_pslinux.py", line 1716, in wrapper
sglang    |     return fun(self, *args, **kwargs)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_common.py", line 508, in wrapper
sglang    |     raise raise_from(err, None)
sglang    |   File "<string>", line 3, in raise_from
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_common.py", line 506, in wrapper
sglang    |     return fun(self)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_pslinux.py", line 1784, in _parse_stat_file
sglang    |     data = bcat("%s/%s/stat" % (self._procfs_path, self.pid))
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_common.py", line 851, in bcat
sglang    |     return cat(fname, fallback=fallback, _open=open_binary)
sglang    |   File "/usr/local/lib/python3.10/dist-packages/psutil/_common.py", line 839, in cat
sglang    |     with _open(fname) as f:
sglang    | RecursionError: maximum recursion depth exceeded while calling a Python object

Previously, i had run with this docker-compose.yml config and it worked fine:

version: '3.8'

services:
  sglang:
    image: lmsysorg/sglang:latest
    container_name: sglang
    volumes:
      - ${HOME}/.cache/huggingface:/root/.cache/huggingface
    restart: always
    network_mode: host
    environment:
      - HF_TOKEN=<token>
    entrypoint: python3 -m sglang.launch_server
    command:
      - --model-path
      - meta-llama/Llama-3.1-8B-Instruct
      - --host
      - 0.0.0.0
      - --port
      - "30000"
    ulimits:
      memlock: -1
      stack: 67108864
    ipc: host
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ['0','1']
              capabilities: [gpu]