valknar/runpod
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: reduce max_model_len to 20000 to fit in 24GB VRAM

Browse Source
This commit is contained in:
Sebastian Krüger
2025-11-23 15:43:37 +01:00
parent 5cfd03f1ef
commit cc0f55df38
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/server.py
View File

@@ -72,8 +72,8 @@ async def startup_event():
engine_args = AsyncEngineArgs( engine_args = AsyncEngineArgs(
model=model_name, model=model_name,
tensor_parallel_size=1, # Single GPU tensor_parallel_size=1, # Single GPU
gpu_memory_utilization=0.85, # Use 85% of GPU memory gpu_memory_utilization=0.90, # Use 90% of GPU memory
max_model_len=32768, # Context length (increased for LLMX) max_model_len=20000, # Context length (balanced for 24GB VRAM)
dtype="auto", # Auto-detect dtype dtype="auto", # Auto-detect dtype
download_dir="/workspace/huggingface_cache", # Large disk download_dir="/workspace/huggingface_cache", # Large disk
trust_remote_code=True, # Some models require this trust_remote_code=True, # Some models require this