valknar/llmx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tui: drop citation rendering (#4855)

Browse Source 
We don't instruct the model to use citations, so it never emits them.
Further, ratatui [doesn't currently support rendering links into the
terminal with OSC 8](https://github.com/ratatui/ratatui/issues/1028), so
even if we did parse citations, we can't correctly render them.

So, remove all the code related to rendering them.
This commit is contained in:
Jeremy Rose
2025-10-20 14:08:19 -07:00
committed by GitHub
parent 9c903c4716
commit 39a2446716
11 changed files with 79 additions and 418 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
codex-rs/Cargo.lock generated
View File

@@ -1446,7 +1446,6 @@ dependencies = [
"libc",
"mcp-types",
"opentelemetry-appender-tracing",
"path-clean",
"pathdiff",
"pretty_assertions",
"pulldown-cmark",
@@ -4277,12 +4276,6 @@ dependencies = [
"path-dedot",
]
[[package]]
name = "path-clean"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
[[package]]
name = "path-dedot"
version = "3.1.1"

1
codex-rs/Cargo.toml
View File

@@ -142,7 +142,6 @@ os_info = "3.12.0"
owo-colors = "4.2.0"
paste = "1.0.15"
path-absolutize = "3.1.1"
path-clean = "1.0.1"
pathdiff = "0.2"
portable-pty = "0.9.0"
predicates = "3"

1
codex-rs/tui/Cargo.toml
View File

@@ -50,7 +50,6 @@ image = { workspace = true, features = ["jpeg", "png"] }
itertools = { workspace = true }
lazy_static = { workspace = true }
mcp-types = { workspace = true }
path-clean = { workspace = true }
pathdiff = { workspace = true }
pulldown-cmark = { workspace = true }
rand = { workspace = true }

5
codex-rs/tui/src/chatwidget.rs
View File

@@ -715,7 +715,6 @@ impl ChatWidget {
self.needs_final_message_separator = false;
}
self.stream_controller = Some(StreamController::new(
self.config.clone(),
self.last_rendered_width.get().map(|w| w.saturating_sub(2)),
));
}
@@ -1539,7 +1538,7 @@ impl ChatWidget {
} else {
// Show explanation when there are no structured findings.
let mut rendered: Vec<ratatui::text::Line<'static>> = vec!["".into()];
append_markdown(&explanation, None, &mut rendered, &self.config);
append_markdown(&explanation, None, &mut rendered);
let body_cell = AgentMessageCell::new(rendered, false);
self.app_event_tx
.send(AppEvent::InsertHistoryCell(Box::new(body_cell)));
@@ -1548,7 +1547,7 @@ impl ChatWidget {
let message_text =
codex_core::review_format::format_review_findings_block(&output.findings, None);
let mut message_lines: Vec<ratatui::text::Line<'static>> = Vec::new();
append_markdown(&message_text, None, &mut message_lines, &self.config);
append_markdown(&message_text, None, &mut message_lines);
let body_cell = AgentMessageCell::new(message_lines, true);
self.app_event_tx
.send(AppEvent::InsertHistoryCell(Box::new(body_cell)));

22
codex-rs/tui/src/citation_regex.rs
View File

@@ -1,22 +0,0 @@
#![expect(clippy::expect_used)]
use regex_lite::Regex;
// This is defined in its own file so we can limit the scope of
// `allow(clippy::expect_used)` because we cannot scope it to the `lazy_static!`
// macro.
lazy_static::lazy_static! {
/// Regular expression that matches Codex-style source file citations such as:
///
/// ```text
/// 【F:src/main.rs†L10-L20】
/// ```
///
/// Capture groups:
/// 1. file path (anything except the dagger `†` symbol)
/// 2. start line number (digits)
/// 3. optional end line (digits or `?`)
pub(crate) static ref CITATION_REGEX: Regex = Regex::new(
r"【F:([^†]+)†L(\d+)(?:-L(\d+|\?))?】"
).expect("failed to compile citation regex");
}

13
codex-rs/tui/src/history_cell.rs
View File

@@ -7,7 +7,6 @@ use crate::exec_cell::output_lines;
use crate::exec_cell::spinner;
use crate::exec_command::relativize_to_home;
use crate::exec_command::strip_bash_lc_and_escape;
use crate::markdown::MarkdownCitationContext;
use crate::markdown::append_markdown;
use crate::render::line_utils::line_to_static;
use crate::render::line_utils::prefix_lines;
@@ -147,21 +146,14 @@ impl HistoryCell for UserHistoryCell {
pub(crate) struct ReasoningSummaryCell {
_header: String,
content: String,
citation_context: MarkdownCitationContext,
transcript_only: bool,
}
impl ReasoningSummaryCell {
pub(crate) fn new(
header: String,
content: String,
citation_context: MarkdownCitationContext,
transcript_only: bool,
) -> Self {
pub(crate) fn new(header: String, content: String, transcript_only: bool) -> Self {
Self {
_header: header,
content,
citation_context,
transcript_only,
}
}
@@ -172,7 +164,6 @@ impl ReasoningSummaryCell {
&self.content,
Some((width as usize).saturating_sub(2)),
&mut lines,
self.citation_context.clone(),
);
let summary_style = Style::default().dim().italic();
let summary_lines = lines
@@ -1303,7 +1294,6 @@ pub(crate) fn new_reasoning_summary_block(
return Box::new(ReasoningSummaryCell::new(
header_buffer,
summary_buffer,
config.into(),
false,
));
}
@@ -1313,7 +1303,6 @@ pub(crate) fn new_reasoning_summary_block(
Box::new(ReasoningSummaryCell::new(
"".to_string(),
full_reasoning_buffer,
config.into(),
true,
))
}

1
codex-rs/tui/src/lib.rs
View File

@@ -36,7 +36,6 @@ mod app_event_sender;
mod ascii_animation;
mod bottom_pane;
mod chatwidget;
mod citation_regex;
mod cli;
mod clipboard_paste;
mod color;

175
codex-rs/tui/src/markdown.rs
View File

@@ -1,59 +1,10 @@
use codex_core::config::Config;
use codex_core::config_types::UriBasedFileOpener;
use ratatui::text::Line;
use std::path::Path;
use std::path::PathBuf;
#[derive(Clone, Debug)]
pub struct MarkdownCitationContext {
file_opener: UriBasedFileOpener,
cwd: PathBuf,
}
impl MarkdownCitationContext {
pub(crate) fn new(file_opener: UriBasedFileOpener, cwd: PathBuf) -> Self {
Self { file_opener, cwd }
}
}
impl From<&Config> for MarkdownCitationContext {
fn from(config: &Config) -> Self {
MarkdownCitationContext::new(config.file_opener, config.cwd.clone())
}
}
pub(crate) fn append_markdown<C>(
pub(crate) fn append_markdown(
markdown_source: &str,
width: Option<usize>,
lines: &mut Vec<Line<'static>>,
citation_context: C,
) where
C: Into<MarkdownCitationContext>,
{
let citation_context: MarkdownCitationContext = citation_context.into();
append_markdown_with_opener_and_cwd(
markdown_source,
width,
lines,
citation_context.file_opener,
&citation_context.cwd,
);
}
pub(crate) fn append_markdown_with_opener_and_cwd(
markdown_source: &str,
width: Option<usize>,
lines: &mut Vec<Line<'static>>,
file_opener: UriBasedFileOpener,
cwd: &Path,
) {
// Render via pulldown-cmark and rewrite citations during traversal (outside code blocks).
let rendered = crate::markdown_render::render_markdown_text_with_citations(
markdown_source,
width,
file_opener.get_scheme(),
cwd,
);
let rendered = crate::markdown_render::render_markdown_text_with_width(markdown_source, width);
crate::render::line_utils::push_owned_lines(&rendered.lines, lines);
}
@@ -61,14 +12,10 @@ pub(crate) fn append_markdown_with_opener_and_cwd(
mod tests {
use super::*;
use pretty_assertions::assert_eq;
use ratatui::text::Line;
#[test]
fn citations_not_rewritten_inside_code_blocks() {
let src = "Before 【F:/x.rs†L1】\n```\nInside 【F:/x.rs†L2】\n```\nAfter 【F:/x.rs†L3】\n";
let cwd = Path::new("/");
let mut out = Vec::new();
append_markdown_with_opener_and_cwd(src, None, &mut out, UriBasedFileOpener::VsCode, cwd);
let rendered: Vec<String> = out
fn lines_to_strings(lines: &[Line<'static>]) -> Vec<String> {
lines
.iter()
.map(|l| {
l.spans
@@ -76,21 +23,21 @@ mod tests {
.map(|s| s.content.clone())
.collect::<String>()
})
.collect();
// Expect a line containing the inside text unchanged.
assert!(rendered.iter().any(|s| s.contains("Inside 【F:/x.rs†L2】")));
// And first/last sections rewritten.
assert!(
rendered
.first()
.map(|s| s.contains("vscode://file"))
.unwrap_or(false)
);
assert!(
rendered
.last()
.map(|s| s.contains("vscode://file"))
.unwrap_or(false)
.collect()
}
#[test]
fn citations_render_as_plain_text() {
let src = "Before 【F:/x.rs†L1】\nAfter 【F:/x.rs†L3】\n";
let mut out = Vec::new();
append_markdown(src, None, &mut out);
let rendered = lines_to_strings(&out);
assert_eq!(
rendered,
vec![
"Before 【F:/x.rs†L1】".to_string(),
"After 【F:/x.rs†L3】".to_string()
]
);
}
@@ -98,57 +45,17 @@ mod tests {
fn indented_code_blocks_preserve_leading_whitespace() {
// Basic sanity: indented code with surrounding blank lines should produce the indented line.
let src = "Before\n\n code 1\n\nAfter\n";
let cwd = Path::new("/");
let mut out = Vec::new();
append_markdown_with_opener_and_cwd(src, None, &mut out, UriBasedFileOpener::None, cwd);
let lines: Vec<String> = out
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<String>()
})
.collect();
append_markdown(src, None, &mut out);
let lines = lines_to_strings(&out);
assert_eq!(lines, vec!["Before", "", " code 1", "", "After"]);
}
#[test]
fn citations_not_rewritten_inside_indented_code_blocks() {
let src = "Start 【F:/x.rs†L1】\n\n Inside 【F:/x.rs†L2】\n\nEnd 【F:/x.rs†L3】\n";
let cwd = Path::new("/");
let mut out = Vec::new();
append_markdown_with_opener_and_cwd(src, None, &mut out, UriBasedFileOpener::VsCode, cwd);
let rendered: Vec<String> = out
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<String>()
})
.collect();
assert!(
rendered
.iter()
.any(|s| s.contains("Start") && s.contains("vscode://file"))
);
assert!(
rendered
.iter()
.any(|s| s.contains("End") && s.contains("vscode://file"))
);
assert!(rendered.iter().any(|s| s.contains("Inside 【F:/x.rs†L2】")));
}
#[test]
fn append_markdown_preserves_full_text_line() {
use codex_core::config_types::UriBasedFileOpener;
use std::path::Path;
let src = "Hi! How can I help with codex-rs today? Want me to explore the repo, run tests, or work on a specific change?\n";
let cwd = Path::new("/");
let mut out = Vec::new();
append_markdown_with_opener_and_cwd(src, None, &mut out, UriBasedFileOpener::None, cwd);
append_markdown(src, None, &mut out);
assert_eq!(
out.len(),
1,
@@ -168,47 +75,19 @@ mod tests {
#[test]
fn append_markdown_matches_tui_markdown_for_ordered_item() {
use codex_core::config_types::UriBasedFileOpener;
use std::path::Path;
let cwd = Path::new("/");
let mut out = Vec::new();
append_markdown_with_opener_and_cwd(
"1. Tight item\n",
None,
&mut out,
UriBasedFileOpener::None,
cwd,
);
let lines: Vec<String> = out
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<String>()
})
.collect();
append_markdown("1. Tight item\n", None, &mut out);
let lines = lines_to_strings(&out);
assert_eq!(lines, vec!["1. Tight item".to_string()]);
}
#[test]
fn append_markdown_keeps_ordered_list_line_unsplit_in_context() {
use codex_core::config_types::UriBasedFileOpener;
use std::path::Path;
let src = "Loose vs. tight list items:\n1. Tight item\n";
let cwd = Path::new("/");
let mut out = Vec::new();
append_markdown_with_opener_and_cwd(src, None, &mut out, UriBasedFileOpener::None, cwd);
append_markdown(src, None, &mut out);
let lines: Vec<String> = out
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<String>()
})
.collect();
let lines = lines_to_strings(&out);
// Expect to find the ordered list line rendered as a single line,
// not split into a marker-only line followed by the text.

146
codex-rs/tui/src/markdown_render.rs
View File

@@ -1,4 +1,3 @@
use crate::citation_regex::CITATION_REGEX;
use crate::render::line_utils::line_to_static;
use crate::wrapping::RtOptions;
use crate::wrapping::word_wrap_line;
@@ -15,8 +14,6 @@ use ratatui::style::Stylize;
use ratatui::text::Line;
use ratatui::text::Span;
use ratatui::text::Text;
use std::borrow::Cow;
use std::path::Path;
#[derive(Clone, Debug)]
struct IndentContext {
@@ -36,29 +33,14 @@ impl IndentContext {
}
pub fn render_markdown_text(input: &str) -> Text<'static> {
let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
let parser = Parser::new_ext(input, options);
let mut w = Writer::new(parser, None, None, None);
w.run();
w.text
render_markdown_text_with_width(input, None)
}
pub(crate) fn render_markdown_text_with_citations(
input: &str,
width: Option<usize>,
scheme: Option<&str>,
cwd: &Path,
) -> Text<'static> {
pub(crate) fn render_markdown_text_with_width(input: &str, width: Option<usize>) -> Text<'static> {
let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
let parser = Parser::new_ext(input, options);
let mut w = Writer::new(
parser,
scheme.map(str::to_string),
Some(cwd.to_path_buf()),
width,
);
let mut w = Writer::new(parser, width);
w.run();
w.text
}
@@ -76,8 +58,6 @@ where
needs_newline: bool,
pending_marker_line: bool,
in_paragraph: bool,
scheme: Option<String>,
cwd: Option<std::path::PathBuf>,
in_code_block: bool,
wrap_width: Option<usize>,
current_line_content: Option<Line<'static>>,
@@ -91,12 +71,7 @@ impl<'a, I> Writer<'a, I>
where
I: Iterator<Item = Event<'a>>,
{
fn new(
iter: I,
scheme: Option<String>,
cwd: Option<std::path::PathBuf>,
wrap_width: Option<usize>,
) -> Self {
fn new(iter: I, wrap_width: Option<usize>) -> Self {
Self {
iter,
text: Text::default(),
@@ -107,8 +82,6 @@ where
needs_newline: false,
pending_marker_line: false,
in_paragraph: false,
scheme,
cwd,
in_code_block: false,
wrap_width,
current_line_content: None,
@@ -288,15 +261,7 @@ where
if i > 0 {
self.push_line(Line::default());
}
let mut content = line.to_string();
if !self.in_code_block
&& let (Some(scheme), Some(cwd)) = (&self.scheme, &self.cwd)
{
let cow = rewrite_file_citations_with_scheme(&content, Some(scheme.as_str()), cwd);
if let std::borrow::Cow::Owned(s) = cow {
content = s;
}
}
let content = line.to_string();
let span = Span::styled(
content,
self.inline_styles.last().copied().unwrap_or_default(),
@@ -524,44 +489,6 @@ where
}
}
pub(crate) fn rewrite_file_citations_with_scheme<'a>(
src: &'a str,
scheme_opt: Option<&str>,
cwd: &Path,
) -> Cow<'a, str> {
let scheme: &str = match scheme_opt {
Some(s) => s,
None => return Cow::Borrowed(src),
};
CITATION_REGEX.replace_all(src, |caps: &regex_lite::Captures<'_>| {
let file = &caps[1];
let start_line = &caps[2];
// Resolve the path against `cwd` when it is relative.
let absolute_path = {
let p = Path::new(file);
let absolute_path = if p.is_absolute() {
path_clean::clean(p)
} else {
path_clean::clean(cwd.join(p))
};
// VS Code expects forward slashes even on Windows because URIs use
// `/` as the path separator.
absolute_path.to_string_lossy().replace('\\', "/")
};
// Render as a normal markdown link so the downstream renderer emits
// the hyperlink escape sequence (when supported by the terminal).
//
// In practice, sometimes multiple citations for the same file, but with a
// different line number, are shown sequentially, so we:
// - include the line number in the label to disambiguate them
// - add a space after the link to make it easier to read
format!("[{file}:{start_line}]({scheme}://file{absolute_path}:{start_line}) ")
})
}
#[cfg(test)]
mod markdown_render_tests {
include!("markdown_render_tests.rs");
@@ -585,50 +512,10 @@ mod tests {
.collect()
}
#[test]
fn citation_is_rewritten_with_absolute_path() {
let markdown = "See 【F:/src/main.rs†L42-L50】 for details.";
let cwd = Path::new("/workspace");
let result = rewrite_file_citations_with_scheme(markdown, Some("vscode"), cwd);
assert_eq!(
"See [/src/main.rs:42](vscode://file/src/main.rs:42) for details.",
result
);
}
#[test]
fn citation_followed_by_space_so_they_do_not_run_together() {
let markdown = "References on lines 【F:src/foo.rs†L24】【F:src/foo.rs†L42】";
let cwd = Path::new("/home/user/project");
let result = rewrite_file_citations_with_scheme(markdown, Some("vscode"), cwd);
assert_eq!(
"References on lines [src/foo.rs:24](vscode://file/home/user/project/src/foo.rs:24) [src/foo.rs:42](vscode://file/home/user/project/src/foo.rs:42) ",
result
);
}
#[test]
fn citation_unchanged_without_file_opener() {
let markdown = "Look at 【F:file.rs†L1】.";
let cwd = Path::new("/");
let unchanged = rewrite_file_citations_with_scheme(markdown, Some("vscode"), cwd);
// The helper itself always rewrites – this test validates behaviour of
// append_markdown when `file_opener` is None.
let rendered = render_markdown_text_with_citations(markdown, None, None, cwd);
// Convert lines back to string for comparison.
let rendered: String = lines_to_strings(&rendered).join("");
assert_eq!(markdown, rendered);
// Ensure helper rewrites.
assert_ne!(markdown, unchanged);
}
#[test]
fn wraps_plain_text_when_width_provided() {
let markdown = "This is a simple sentence that should wrap.";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(16), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(16));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,
@@ -643,8 +530,7 @@ mod tests {
#[test]
fn wraps_list_items_preserving_indent() {
let markdown = "- first second third fourth";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(14), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(14));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,
@@ -656,8 +542,7 @@ mod tests {
fn wraps_nested_lists() {
let markdown =
"- outer item with several words to wrap\n - inner item that also needs wrapping";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(20), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(20));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,
@@ -675,8 +560,7 @@ mod tests {
#[test]
fn wraps_ordered_lists() {
let markdown = "1. ordered item contains many words for wrapping";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(18), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(18));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,
@@ -692,8 +576,7 @@ mod tests {
#[test]
fn wraps_blockquotes() {
let markdown = "> block quote with content that should wrap nicely";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(22), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(22));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,
@@ -708,8 +591,7 @@ mod tests {
#[test]
fn wraps_blockquotes_inside_lists() {
let markdown = "- list item\n > block quote inside list that wraps";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(24), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(24));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,
@@ -724,8 +606,7 @@ mod tests {
#[test]
fn wraps_list_items_containing_blockquotes() {
let markdown = "1. item with quote\n > quoted text that should wrap";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(24), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(24));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,
@@ -740,8 +621,7 @@ mod tests {
#[test]
fn does_not_wrap_code_blocks() {
let markdown = "````\nfn main() { println!(\"hi from a long line\"); }\n````";
let cwd = Path::new("/");
let rendered = render_markdown_text_with_citations(markdown, Some(10), None, cwd);
let rendered = render_markdown_text_with_width(markdown, Some(10));
let lines = lines_to_strings(&rendered);
assert_eq!(
lines,

97
codex-rs/tui/src/markdown_stream.rs
View File

@@ -1,4 +1,3 @@
use codex_core::config::Config;
use ratatui::text::Line;
use crate::markdown;
@@ -33,7 +32,7 @@ impl MarkdownStreamCollector {
/// Render the full buffer and return only the newly completed logical lines
/// since the last commit. When the buffer does not end with a newline, the
/// final rendered line is considered incomplete and is not emitted.
pub fn commit_complete_lines(&mut self, config: &Config) -> Vec<Line<'static>> {
pub fn commit_complete_lines(&mut self) -> Vec<Line<'static>> {
let source = self.buffer.clone();
let last_newline_idx = source.rfind('\n');
let source = if let Some(last_newline_idx) = last_newline_idx {
@@ -42,7 +41,7 @@ impl MarkdownStreamCollector {
return Vec::new();
};
let mut rendered: Vec<Line<'static>> = Vec::new();
markdown::append_markdown(&source, self.width, &mut rendered, config);
markdown::append_markdown(&source, self.width, &mut rendered);
let mut complete_line_count = rendered.len();
if complete_line_count > 0
&& crate::render::line_utils::is_blank_line_spaces_only(
@@ -67,7 +66,7 @@ impl MarkdownStreamCollector {
/// If the buffer does not end with a newline, a temporary one is appended
/// for rendering. Optionally unwraps ```markdown language fences in
/// non-test builds.
pub fn finalize_and_drain(&mut self, config: &Config) -> Vec<Line<'static>> {
pub fn finalize_and_drain(&mut self) -> Vec<Line<'static>> {
let raw_buffer = self.buffer.clone();
let mut source: String = raw_buffer.clone();
if !source.ends_with('\n') {
@@ -83,7 +82,7 @@ impl MarkdownStreamCollector {
tracing::trace!("markdown finalize (raw source):\n---\n{source}\n---");
let mut rendered: Vec<Line<'static>> = Vec::new();
markdown::append_markdown(&source, self.width, &mut rendered, config);
markdown::append_markdown(&source, self.width, &mut rendered);
let out = if self.committed_line_count >= rendered.len() {
Vec::new()
@@ -101,18 +100,17 @@ impl MarkdownStreamCollector {
pub(crate) fn simulate_stream_markdown_for_tests(
deltas: &[&str],
finalize: bool,
config: &Config,
) -> Vec<Line<'static>> {
let mut collector = MarkdownStreamCollector::new(None);
let mut out = Vec::new();
for d in deltas {
collector.push_delta(d);
if d.contains('\n') {
out.extend(collector.commit_complete_lines(config));
out.extend(collector.commit_complete_lines());
}
}
if finalize {
out.extend(collector.finalize_and_drain(config));
out.extend(collector.finalize_and_drain());
}
out
}
@@ -120,45 +118,30 @@ pub(crate) fn simulate_stream_markdown_for_tests(
#[cfg(test)]
mod tests {
use super::*;
use codex_core::config::Config;
use codex_core::config::ConfigOverrides;
use ratatui::style::Color;
async fn test_config() -> Config {
let overrides = ConfigOverrides {
cwd: std::env::current_dir().ok(),
..Default::default()
};
Config::load_with_cli_overrides(vec![], overrides)
.await
.expect("load test config")
}
#[tokio::test]
async fn no_commit_until_newline() {
let cfg = test_config().await;
let mut c = super::MarkdownStreamCollector::new(None);
c.push_delta("Hello, world");
let out = c.commit_complete_lines(&cfg);
let out = c.commit_complete_lines();
assert!(out.is_empty(), "should not commit without newline");
c.push_delta("!\n");
let out2 = c.commit_complete_lines(&cfg);
let out2 = c.commit_complete_lines();
assert_eq!(out2.len(), 1, "one completed line after newline");
}
#[tokio::test]
async fn finalize_commits_partial_line() {
let cfg = test_config().await;
let mut c = super::MarkdownStreamCollector::new(None);
c.push_delta("Line without newline");
let out = c.finalize_and_drain(&cfg);
let out = c.finalize_and_drain();
assert_eq!(out.len(), 1);
}
#[tokio::test]
async fn e2e_stream_blockquote_simple_is_green() {
let cfg = test_config().await;
let out = super::simulate_stream_markdown_for_tests(&["> Hello\n"], true, &cfg);
let out = super::simulate_stream_markdown_for_tests(&["> Hello\n"], true);
assert_eq!(out.len(), 1);
let l = &out[0];
assert_eq!(
@@ -171,9 +154,7 @@ mod tests {
#[tokio::test]
async fn e2e_stream_blockquote_nested_is_green() {
let cfg = test_config().await;
let out =
super::simulate_stream_markdown_for_tests(&["> Level 1\n>> Level 2\n"], true, &cfg);
let out = super::simulate_stream_markdown_for_tests(&["> Level 1\n>> Level 2\n"], true);
// Filter out any blank lines that may be inserted at paragraph starts.
let non_blank: Vec<_> = out
.into_iter()
@@ -196,9 +177,7 @@ mod tests {
#[tokio::test]
async fn e2e_stream_blockquote_with_list_items_is_green() {
let cfg = test_config().await;
let out =
super::simulate_stream_markdown_for_tests(&["> - item 1\n> - item 2\n"], true, &cfg);
let out = super::simulate_stream_markdown_for_tests(&["> - item 1\n> - item 2\n"], true);
assert_eq!(out.len(), 2);
assert_eq!(out[0].style.fg, Some(Color::Green));
assert_eq!(out[1].style.fg, Some(Color::Green));
@@ -206,7 +185,6 @@ mod tests {
#[tokio::test]
async fn e2e_stream_nested_mixed_lists_ordered_marker_is_light_blue() {
let cfg = test_config().await;
let md = [
"1. First\n",
" - Second level\n",
@@ -214,7 +192,7 @@ mod tests {
" - Fourth level (bullet)\n",
" - Fifth level to test indent consistency\n",
];
let out = super::simulate_stream_markdown_for_tests(&md, true, &cfg);
let out = super::simulate_stream_markdown_for_tests(&md, true);
// Find the line that contains the third-level ordered text
let find_idx = out.iter().position(|l| {
l.spans
@@ -238,9 +216,8 @@ mod tests {
#[tokio::test]
async fn e2e_stream_blockquote_wrap_preserves_green_style() {
let cfg = test_config().await;
let long = "> This is a very long quoted line that should wrap across multiple columns to verify style preservation.";
let out = super::simulate_stream_markdown_for_tests(&[long, "\n"], true, &cfg);
let out = super::simulate_stream_markdown_for_tests(&[long, "\n"], true);
// Wrap to a narrow width to force multiple output lines.
let wrapped =
crate::wrapping::word_wrap_lines(out.iter(), crate::wrapping::RtOptions::new(24));
@@ -274,13 +251,11 @@ mod tests {
#[tokio::test]
async fn heading_starts_on_new_line_when_following_paragraph() {
let cfg = test_config().await;
// Stream a paragraph line, then a heading on the next line.
// Expect two distinct rendered lines: "Hello." and "Heading".
let mut c = super::MarkdownStreamCollector::new(None);
c.push_delta("Hello.\n");
let out1 = c.commit_complete_lines(&cfg);
let out1 = c.commit_complete_lines();
let s1: Vec<String> = out1
.iter()
.map(|l| {
@@ -300,7 +275,7 @@ mod tests {
);
c.push_delta("## Heading\n");
let out2 = c.commit_complete_lines(&cfg);
let out2 = c.commit_complete_lines();
let s2: Vec<String> = out2
.iter()
.map(|l| {
@@ -331,19 +306,17 @@ mod tests {
#[tokio::test]
async fn heading_not_inlined_when_split_across_chunks() {
let cfg = test_config().await;
// Paragraph without trailing newline, then a chunk that starts with the newline
// and the heading text, then a final newline. The collector should first commit
// only the paragraph line, and later commit the heading as its own line.
let mut c = super::MarkdownStreamCollector::new(None);
c.push_delta("Sounds good!");
// No commit yet
assert!(c.commit_complete_lines(&cfg).is_empty());
assert!(c.commit_complete_lines().is_empty());
// Introduce the newline that completes the paragraph and the start of the heading.
c.push_delta("\n## Adding Bird subcommand");
let out1 = c.commit_complete_lines(&cfg);
let out1 = c.commit_complete_lines();
let s1: Vec<String> = out1
.iter()
.map(|l| {
@@ -362,7 +335,7 @@ mod tests {
// Now finish the heading line with the trailing newline.
c.push_delta("\n");
let out2 = c.commit_complete_lines(&cfg);
let out2 = c.commit_complete_lines();
let s2: Vec<String> = out2
.iter()
.map(|l| {
@@ -381,7 +354,7 @@ mod tests {
// Sanity check raw markdown rendering for a simple line does not produce spurious extras.
let mut rendered: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown("Hello.\n", None, &mut rendered, &cfg);
crate::markdown::append_markdown("Hello.\n", None, &mut rendered);
let rendered_strings: Vec<String> = rendered
.iter()
.map(|l| {
@@ -423,8 +396,6 @@ mod tests {
#[tokio::test]
async fn utf8_boundary_safety_and_wide_chars() {
let cfg = test_config().await;
// Emoji (wide), CJK, control char, digit + combining macron sequences
let input = "🙂🙂🙂\n汉字漢字\nA\u{0003}0\u{0304}\n";
let deltas = vec![
@@ -439,11 +410,11 @@ mod tests {
"\n",
];
let streamed = simulate_stream_markdown_for_tests(&deltas, true, &cfg);
let streamed = simulate_stream_markdown_for_tests(&deltas, true);
let streamed_str = lines_to_plain_strings(&streamed);
let mut rendered_all: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(input, None, &mut rendered_all, &cfg);
crate::markdown::append_markdown(input, None, &mut rendered_all);
let rendered_all_str = lines_to_plain_strings(&rendered_all);
assert_eq!(
@@ -454,9 +425,8 @@ mod tests {
#[tokio::test]
async fn e2e_stream_deep_nested_third_level_marker_is_light_blue() {
let cfg = test_config().await;
let md = "1. First\n - Second level\n 1. Third level (ordered)\n - Fourth level (bullet)\n - Fifth level to test indent consistency\n";
let streamed = super::simulate_stream_markdown_for_tests(&[md], true, &cfg);
let streamed = super::simulate_stream_markdown_for_tests(&[md], true);
let streamed_strs = lines_to_plain_strings(&streamed);
// Locate the third-level line in the streamed output; avoid relying on exact indent.
@@ -504,11 +474,10 @@ mod tests {
#[tokio::test]
async fn empty_fenced_block_is_dropped_and_separator_preserved_before_heading() {
let cfg = test_config().await;
// An empty fenced code block followed by a heading should not render the fence,
// but should preserve a blank separator line so the heading starts on a new line.
let deltas = vec!["```bash\n```\n", "## Heading\n"]; // empty block and close in same commit
let streamed = simulate_stream_markdown_for_tests(&deltas, true, &cfg);
let streamed = simulate_stream_markdown_for_tests(&deltas, true);
let texts = lines_to_plain_strings(&streamed);
assert!(
texts.iter().all(|s| !s.contains("```")),
@@ -523,9 +492,8 @@ mod tests {
#[tokio::test]
async fn paragraph_then_empty_fence_then_heading_keeps_heading_on_new_line() {
let cfg = test_config().await;
let deltas = vec!["Para.\n", "```\n```\n", "## Title\n"]; // empty fence block in one commit
let streamed = simulate_stream_markdown_for_tests(&deltas, true, &cfg);
let streamed = simulate_stream_markdown_for_tests(&deltas, true);
let texts = lines_to_plain_strings(&streamed);
let para_idx = match texts.iter().position(|s| s == "Para.") {
Some(i) => i,
@@ -543,17 +511,16 @@ mod tests {
#[tokio::test]
async fn loose_list_with_split_dashes_matches_full_render() {
let cfg = test_config().await;
// Minimized failing sequence discovered by the helper: two chunks
// that still reproduce the mismatch.
let deltas = vec!["- item.\n\n", "-"];
let streamed = simulate_stream_markdown_for_tests(&deltas, true, &cfg);
let streamed = simulate_stream_markdown_for_tests(&deltas, true);
let streamed_strs = lines_to_plain_strings(&streamed);
let full: String = deltas.iter().copied().collect();
let mut rendered_all: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(&full, None, &mut rendered_all, &cfg);
crate::markdown::append_markdown(&full, None, &mut rendered_all);
let rendered_all_strs = lines_to_plain_strings(&rendered_all);
assert_eq!(
@@ -564,7 +531,6 @@ mod tests {
#[tokio::test]
async fn loose_vs_tight_list_items_streaming_matches_full() {
let cfg = test_config().await;
// Deltas extracted from the session log around 2025-08-27T00:33:18.216Z
let deltas = vec![
"\n\n",
@@ -636,13 +602,13 @@ mod tests {
"\n\n",
];
let streamed = simulate_stream_markdown_for_tests(&deltas, true, &cfg);
let streamed = simulate_stream_markdown_for_tests(&deltas, true);
let streamed_strs = lines_to_plain_strings(&streamed);
// Compute a full render for diagnostics only.
let full: String = deltas.iter().copied().collect();
let mut rendered_all: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(&full, None, &mut rendered_all, &cfg);
crate::markdown::append_markdown(&full, None, &mut rendered_all);
// Also assert exact expected plain strings for clarity.
let expected = vec![
@@ -665,12 +631,11 @@ mod tests {
// Targeted tests derived from fuzz findings. Each asserts streamed == full render.
async fn assert_streamed_equals_full(deltas: &[&str]) {
let cfg = test_config().await;
let streamed = simulate_stream_markdown_for_tests(deltas, true, &cfg);
let streamed = simulate_stream_markdown_for_tests(deltas, true);
let streamed_strs = lines_to_plain_strings(&streamed);
let full: String = deltas.iter().copied().collect();
let mut rendered: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(&full, None, &mut rendered, &cfg);
crate::markdown::append_markdown(&full, None, &mut rendered);
let rendered_strs = lines_to_plain_strings(&rendered);
assert_eq!(streamed_strs, rendered_strs, "full:\n---\n{full}\n---");
}

29
codex-rs/tui/src/streaming/controller.rs
View File

@@ -1,6 +1,5 @@
use crate::history_cell::HistoryCell;
use crate::history_cell::{self};
use codex_core::config::Config;
use ratatui::text::Line;
use super::StreamState;
@@ -8,16 +7,14 @@ use super::StreamState;
/// Controller that manages newline-gated streaming, header emission, and
/// commit animation across streams.
pub(crate) struct StreamController {
config: Config,
state: StreamState,
finishing_after_drain: bool,
header_emitted: bool,
}
impl StreamController {
pub(crate) fn new(config: Config, width: Option<usize>) -> Self {
pub(crate) fn new(width: Option<usize>) -> Self {
Self {
config,
state: StreamState::new(width),
finishing_after_drain: false,
header_emitted: false,
@@ -26,14 +23,13 @@ impl StreamController {
/// Push a delta; if it contains a newline, commit completed lines and start animation.
pub(crate) fn push(&mut self, delta: &str) -> bool {
let cfg = self.config.clone();
let state = &mut self.state;
if !delta.is_empty() {
state.has_seen_delta = true;
}
state.collector.push_delta(delta);
if delta.contains('\n') {
let newly_completed = state.collector.commit_complete_lines(&cfg);
let newly_completed = state.collector.commit_complete_lines();
if !newly_completed.is_empty() {
state.enqueue(newly_completed);
return true;
@@ -44,11 +40,10 @@ impl StreamController {
/// Finalize the active stream. Drain and emit now.
pub(crate) fn finalize(&mut self) -> Option<Box<dyn HistoryCell>> {
let cfg = self.config.clone();
// Finalize collector first.
let remaining = {
let state = &mut self.state;
state.collector.finalize_and_drain(&cfg)
state.collector.finalize_and_drain()
};
// Collect all output first to avoid emitting headers when there is no content.
let mut out_lines = Vec::new();
@@ -88,19 +83,6 @@ impl StreamController {
#[cfg(test)]
mod tests {
use super::*;
use codex_core::config::Config;
use codex_core::config::ConfigOverrides;
use pretty_assertions::assert_eq;
async fn test_config() -> Config {
let overrides = ConfigOverrides {
cwd: std::env::current_dir().ok(),
..Default::default()
};
Config::load_with_cli_overrides(vec![], overrides)
.await
.expect("load test config")
}
fn lines_to_plain_strings(lines: &[ratatui::text::Line<'_>]) -> Vec<String> {
lines
@@ -117,8 +99,7 @@ mod tests {
#[tokio::test]
async fn controller_loose_vs_tight_with_commit_ticks_matches_full() {
let cfg = test_config().await;
let mut ctrl = StreamController::new(cfg.clone(), None);
let mut ctrl = StreamController::new(None);
let mut lines = Vec::new();
// Exact deltas from the session log (section: Loose vs. tight list items)
@@ -216,7 +197,7 @@ mod tests {
// Full render of the same source
let source: String = deltas.iter().copied().collect();
let mut rendered: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(&source, None, &mut rendered, &cfg);
crate::markdown::append_markdown(&source, None, &mut rendered);
let rendered_strs = lines_to_plain_strings(&rendered);
assert_eq!(streamed, rendered_strs);