feat: Make max_tokens configurable via provider config
Some checks failed
ci / build-test (push) Failing after 4m51s
Codespell / Check for spelling errors (push) Successful in 10s
sdk / sdks (push) Successful in 11m9s
rust-ci / Detect changed areas (push) Has been cancelled
rust-ci / Format / etc (push) Has been cancelled
rust-ci / cargo shear (push) Has been cancelled
rust-ci / Lint/Build — macos-14 - aarch64-apple-darwin (push) Has been cancelled
rust-ci / Lint/Build — macos-14 - x86_64-apple-darwin (push) Has been cancelled
rust-ci / Lint/Build — ubuntu-24.04 - x86_64-unknown-linux-gnu (push) Has been cancelled
rust-ci / Lint/Build — ubuntu-24.04 - x86_64-unknown-linux-musl (push) Has been cancelled
rust-ci / Lint/Build — ubuntu-24.04-arm - aarch64-unknown-linux-gnu (push) Has been cancelled
rust-ci / Lint/Build — ubuntu-24.04-arm - aarch64-unknown-linux-musl (push) Has been cancelled
rust-ci / Lint/Build — windows-11-arm - aarch64-pc-windows-msvc (push) Has been cancelled
rust-ci / Lint/Build — windows-latest - x86_64-pc-windows-msvc (push) Has been cancelled
rust-ci / Lint/Build — macos-14 - aarch64-apple-darwin (release) (push) Has been cancelled
rust-ci / Lint/Build — ubuntu-24.04 - x86_64-unknown-linux-musl (release) (push) Has been cancelled
rust-ci / Lint/Build — windows-11-arm - aarch64-pc-windows-msvc (release) (push) Has been cancelled
rust-ci / Lint/Build — windows-latest - x86_64-pc-windows-msvc (release) (push) Has been cancelled
rust-ci / Tests — macos-14 - aarch64-apple-darwin (push) Has been cancelled
rust-ci / Tests — ubuntu-24.04 - x86_64-unknown-linux-gnu (push) Has been cancelled
rust-ci / Tests — ubuntu-24.04-arm - aarch64-unknown-linux-gnu (push) Has been cancelled
rust-ci / Tests — windows-11-arm - aarch64-pc-windows-msvc (push) Has been cancelled
rust-ci / Tests — windows-latest - x86_64-pc-windows-msvc (push) Has been cancelled
rust-ci / CI results (required) (push) Has been cancelled

- Add max_tokens field to ModelProviderInfo struct
- Read from provider config, default to 8192 if not set
- Add debug logging for max_tokens value
- Allows per-provider customization in config.toml

Example config:
```toml
[model_providers.litellm]
max_tokens = 16384
```

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-11-17 10:40:15 +01:00
parent ee75cfaa7f
commit c0775ad8a3
2 changed files with 11 additions and 2 deletions

View File

@@ -443,10 +443,12 @@ pub(crate) async fn stream_chat_completions(
});
// Add max_tokens - required by Anthropic Messages API
// Use a sensible default of 8192 if not configured
// Use provider config value or default to 8192
let max_tokens = provider.max_tokens.unwrap_or(8192);
if let Some(obj) = payload.as_object_mut() {
obj.insert("max_tokens".to_string(), json!(8192));
obj.insert("max_tokens".to_string(), json!(max_tokens));
}
debug!("Using max_tokens: {}", max_tokens);
debug!(
"POST to {}: {}",

View File

@@ -87,6 +87,10 @@ pub struct ModelProviderInfo {
/// the connection as lost.
pub stream_idle_timeout_ms: Option<u64>,
/// Maximum number of tokens to generate in the response. If not specified, defaults to 8192.
/// This is required by some providers (e.g., Anthropic via LiteLLM).
pub max_tokens: Option<i64>,
/// Does this provider require an OpenAI API Key or ChatGPT login token? If true,
/// user is presented with login screen on first run, and login preference and token/key
/// are stored in auth.json. If false (which is the default), login screen is skipped,
@@ -290,6 +294,7 @@ pub fn built_in_model_providers() -> HashMap<String, ModelProviderInfo> {
request_max_retries: None,
stream_max_retries: None,
stream_idle_timeout_ms: None,
max_tokens: None,
requires_openai_auth: false,
},
),
@@ -330,6 +335,7 @@ pub fn built_in_model_providers() -> HashMap<String, ModelProviderInfo> {
request_max_retries: None,
stream_max_retries: None,
stream_idle_timeout_ms: None,
max_tokens: None,
requires_openai_auth: true,
},
),
@@ -375,6 +381,7 @@ pub fn create_oss_provider_with_base_url(base_url: &str) -> ModelProviderInfo {
request_max_retries: None,
stream_max_retries: None,
stream_idle_timeout_ms: None,
max_tokens: None,
requires_openai_auth: false,
}
}