Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🐛 fix issue with starting both servers #11

Closed
wants to merge 1 commit into from
Closed

🐛 fix issue with starting both servers #11

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 5 additions & 15 deletions src/vllm_tgis_adapter/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,11 @@
import re
from contextlib import asynccontextmanager
from http import HTTPStatus
from typing import TYPE_CHECKING

import fastapi
import uvicorn
import vllm
from fastapi import APIRouter

if TYPE_CHECKING:
from fastapi import Request
from fastapi import APIRouter, Request # noqa: TCH002
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, Response, StreamingResponse
Expand All @@ -24,15 +20,11 @@
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.async_llm_engine import AsyncLLMEngine
from vllm.entrypoints.openai.cli_args import make_arg_parser

if TYPE_CHECKING:
from vllm.entrypoints.openai.protocol import (
ChatCompletionRequest,
CompletionRequest,
EmbeddingRequest,
)
from vllm.entrypoints.openai.protocol import (
ChatCompletionRequest, # noqa: TCH002
ChatCompletionResponse,
CompletionRequest, # noqa: TCH002
EmbeddingRequest, # noqa: TCH002
ErrorResponse,
)
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
Expand Down Expand Up @@ -205,7 +197,7 @@ def run_server(args, llm_engine: AsyncLLMEngine = None) -> None: # noqa: ANN001

engine = (
llm_engine
if not llm_engine
if llm_engine is not None
else AsyncLLMEngine.from_engine_args(
engine_args, usage_context=UsageContext.OPENAI_API_SERVER
)
Expand Down Expand Up @@ -274,8 +266,6 @@ async def _force_log() -> None:

server = await start_grpc_server(async_llm_engine, args)

yield

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm assuming the yield was a no-op in this case?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this won't work. see my comments on #9.

Perhaps something like this would work (although untested) https://gist.github.com/dtrifiro/251649959af43e3b5853bb8ab862d5a6

logger.info("Gracefully stopping gRPC server")
await server.stop(30) # TODO configurable grace
await server.wait_for_termination()
Expand Down
Loading