valknar/docker-compose
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: disable drop_params to preserve streaming metadata in LiteLLM

Browse Source 
- Set drop_params: false in litellm_settings
- Set modify_params: false in litellm_settings
- Set drop_params: false in default_litellm_params
- Commented out LITELLM_DROP_PARAMS env var
- Removed --drop_params command flag

These settings were stripping critical streaming parameters causing
vLLM streaming responses to collapse into empty deltas
This commit is contained in:
Sebastian Krüger
2025-11-21 18:46:33 +01:00
parent c58b5d36ba
commit 01a345979b
2 changed files with 5 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ai/compose.yaml
View File

@@ -100,7 +100,7 @@ services:
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY}
LITELLM_MASTER_KEY: ${AI_LITELLM_API_KEY} LITELLM_MASTER_KEY: ${AI_LITELLM_API_KEY}
DATABASE_URL: postgresql://${AI_DB_USER}:${AI_DB_PASSWORD}@ai_postgres:5432/litellm DATABASE_URL: postgresql://${AI_DB_USER}:${AI_DB_PASSWORD}@ai_postgres:5432/litellm
LITELLM_DROP_PARAMS: 'true' # LITELLM_DROP_PARAMS: 'true' # DISABLED: Was breaking streaming
NO_DOCS: 'true' NO_DOCS: 'true'
NO_REDOC: 'true' NO_REDOC: 'true'
# Performance optimizations # Performance optimizations
@@ -115,8 +115,7 @@ services:
'--host', '--host',
'0.0.0.0', '0.0.0.0',
'--port', '--port',
'4000', '4000'
'--drop_params'
] ]
depends_on: depends_on:
- ai_postgres - ai_postgres

6
ai/litellm-config.yaml
View File

@@ -61,7 +61,7 @@ model_list:
max_parallel_requests: 1 max_parallel_requests: 1
litellm_settings: litellm_settings:
drop_params: true drop_params: false # DISABLED: Was breaking streaming
set_verbose: true # Enable verbose logging for debugging streaming issues set_verbose: true # Enable verbose logging for debugging streaming issues
# Disable caching - it breaks streaming responses # Disable caching - it breaks streaming responses
cache: false cache: false
@@ -73,7 +73,7 @@ litellm_settings:
# Force strip specific parameters globally # Force strip specific parameters globally
allowed_fails: 0 allowed_fails: 0
# Modify params before sending to provider # Modify params before sending to provider
modify_params: true modify_params: false # DISABLED: Was breaking streaming
# Enable success and failure logging but minimize overhead # Enable success and failure logging but minimize overhead
success_callback: [] # Disable all success callbacks to reduce DB writes success_callback: [] # Disable all success callbacks to reduce DB writes
failure_callback: [] # Disable all failure callbacks failure_callback: [] # Disable all failure callbacks
@@ -83,7 +83,7 @@ router_settings:
# Drop unsupported parameters # Drop unsupported parameters
default_litellm_params: default_litellm_params:
drop_params: true drop_params: false # DISABLED: Was breaking streaming
general_settings: general_settings:
disable_responses_id_security: true disable_responses_id_security: true