valknar/docker-compose
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use hosted_vllm/openai/ prefix for vLLM model via orchestrator

Browse Source
This commit is contained in:
Sebastian Krüger
2025-11-21 19:18:33 +01:00
parent 155016da97
commit ec903c16c2
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ai/litellm-config.yaml
View File

@@ -32,7 +32,7 @@ model_list:
# Text Generation # Text Generation
- model_name: qwen-2.5-7b - model_name: qwen-2.5-7b
litellm_params: litellm_params:
model: hosted_vllm/qwen-2.5-7b # vLLM model ID from /v1/models endpoint model: hosted_vllm/openai/qwen-2.5-7b # hosted_vllm/openai/ for vLLM via orchestrator
api_base: http://100.121.199.88:9000/v1 # RunPod GPU via Tailscale api_base: http://100.121.199.88:9000/v1 # RunPod GPU via Tailscale
api_key: dummy api_key: dummy
rpm: 1000 rpm: 1000