From e5e13479d04bba8da2cb56db6cc829e66f98da5d Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Mon, 3 Nov 2025 10:02:23 -0800 Subject: [PATCH] Include reasoning tokens in the context window calculation (#6161) This value is used to determine whether mid-turn compaction is required. Reasoning items are only excluded between turns (and soon will start to be preserved even across turns) so it's incorrect to subtract reasoning_output_tokens mid term. This will result in higher values reported between turns but we are also looking into preserving reasoning items for the entire conversation to improve performance and caching. --- codex-rs/protocol/src/protocol.rs | 6 +----- ...__tests__status_snapshot_includes_reasoning_details.snap | 2 +- ...__tests__status_snapshot_shows_stale_limits_message.snap | 2 +- ...tests__status_snapshot_truncates_in_narrow_terminal.snap | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index c30ba42b..2d0b0f01 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -812,12 +812,8 @@ impl TokenUsage { (self.non_cached_input() + self.output_tokens.max(0)).max(0) } - /// For estimating what % of the model's context window is used, we need to account - /// for reasoning output tokens from prior turns being dropped from the context window. - /// We approximate this here by subtracting reasoning output tokens from the total. - /// This will be off for the current turn and pending function calls. pub fn tokens_in_context_window(&self) -> i64 { - (self.total_tokens - self.reasoning_output_tokens).max(0) + self.total_tokens } /// Estimate the remaining user-controllable percentage of the model's context window. diff --git a/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_includes_reasoning_details.snap b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_includes_reasoning_details.snap index 0b7b74d7..123547cb 100644 --- a/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_includes_reasoning_details.snap +++ b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_includes_reasoning_details.snap @@ -17,7 +17,7 @@ expression: sanitized │ Agents.md: │ │ │ │ Token usage: 1.9K total (1K input + 900 output) │ -│ Context window: 100% left (2.1K used / 272K) │ +│ Context window: 100% left (2.25K used / 272K) │ │ 5h limit: [███████████████░░░░░] 72% used (resets 03:14) │ │ Weekly limit: [█████████░░░░░░░░░░░] 45% used (resets 03:24) │ ╰─────────────────────────────────────────────────────────────────────╯ diff --git a/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_shows_stale_limits_message.snap b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_shows_stale_limits_message.snap index 2fc0d887..3a98ce95 100644 --- a/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_shows_stale_limits_message.snap +++ b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_shows_stale_limits_message.snap @@ -17,7 +17,7 @@ expression: sanitized │ Agents.md: │ │ │ │ Token usage: 1.9K total (1K input + 900 output) │ -│ Context window: 100% left (2.1K used / 272K) │ +│ Context window: 100% left (2.25K used / 272K) │ │ 5h limit: [███████████████░░░░░] 72% used (resets 03:14) │ │ Weekly limit: [████████░░░░░░░░░░░░] 40% used (resets 03:34) │ │ Warning: limits may be stale - start new turn to refresh. │ diff --git a/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_truncates_in_narrow_terminal.snap b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_truncates_in_narrow_terminal.snap index d86e43a4..5bbaaef7 100644 --- a/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_truncates_in_narrow_terminal.snap +++ b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_truncates_in_narrow_terminal.snap @@ -19,7 +19,7 @@ expression: sanitized │ Agents.md: │ │ │ │ Token usage: 1.9K total (1K input + │ -│ Context window: 100% left (2.1K used / │ +│ Context window: 100% left (2.25K used / │ │ 5h limit: [███████████████░░░░░] │ │ (resets 03:14) │ ╰────────────────────────────────────────────╯