Skip to content
This repository was archived by the owner on Jun 13, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions vllm/entrypoints/openai/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@ def get_logits_processors(processors: Optional[LogitsProcessors],


class ChatCompletionRequest(OpenAIBaseModel):
## Start custom Lamini code ##
lora_request: Optional['LoadLoraAdapterRequest'] = None # This is a forward reference to LoadLoraAdapterRequest since it's defined later in this file
## End custom Lamini code ##

# Ordered by official OpenAI API documentation
# https://platform.openai.com/docs/api-reference/chat/create
messages: List[ChatCompletionMessageParam]
Expand Down
22 changes: 21 additions & 1 deletion vllm/entrypoints/openai/serving_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,16 @@ async def create_chat_completion(
if self.engine_client.errored:
raise self.engine_client.dead_error

## Start custom Lamini code ##
if request.lora_request is not None and not request.stream: # Support for non-streaming requests only for now, streaming requests would need more complicated unloading logic
logger.info("[Lamini] Completion request has LoRA request: %s", request.lora_request)
logger.info("[Lamini] Loading LoRA adapter...")
from vllm.entrypoints.openai.api_server import load_lora_adapter
await load_lora_adapter(request.lora_request, raw_request)
logger.info("[Lamini] LoRA adapter loaded")
logger.info("[Lamini] Set model to LoRA name: %s", request.lora_request.lora_name)
request.model = request.lora_request.lora_name
## End custom Lamini code ##
try:
(
lora_request,
Expand Down Expand Up @@ -260,9 +270,19 @@ async def create_chat_completion(
conversation, tokenizer, request_metadata)

try:
return await self.chat_completion_full_generator(
## Start modified Lamini code ##
response = await self.chat_completion_full_generator(
request, result_generator, request_id, model_name,
conversation, tokenizer, request_metadata)
## End modified Lamini code ##
## Start custom Lamini code ##
if request.lora_request is not None:
logger.info("[Lamini] Completion request completed, unloading LoRA adapter...: %s", request.lora_request)
from vllm.entrypoints.openai.api_server import unload_lora_adapter
await unload_lora_adapter(request.lora_request, raw_request)
logger.info("[Lamini] LoRA adapter unloaded")
## End custom Lamini code ##
return response
except ValueError as e:
# TODO: Use a vllm-specific Validation Error
return self.create_error_response(str(e))
Expand Down
2 changes: 1 addition & 1 deletion vllm/entrypoints/openai/serving_completion.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ async def create_completion(
raw_request.state.request_metadata = request_metadata

## Start custom Lamini code ##
if request.lora_request is not None:
if request.lora_request is not None and not request.stream: # Support for non-streaming requests only for now, streaming requests would need more complicated unloading logic
logger.info("[Lamini] Completion request has LoRA request: %s", request.lora_request)
logger.info("[Lamini] Loading LoRA adapter...")
from vllm.entrypoints.openai.api_server import load_lora_adapter
Expand Down