diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index d1b9cb39..7bf6c2a7 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -311,15 +311,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bit-set" version = "0.5.3" @@ -655,7 +646,7 @@ dependencies = [ "tokio-test", "tokio-util", "toml", - "toml_edit 0.23.4", + "toml_edit", "tracing", "tree-sitter", "tree-sitter-bash", @@ -879,6 +870,7 @@ dependencies = [ "path-clean", "pathdiff", "pretty_assertions", + "pulldown-cmark", "rand 0.9.2", "ratatui", "regex-lite", @@ -895,7 +887,6 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", - "tui-markdown", "unicode-segmentation", "unicode-width 0.1.14", "url", @@ -1763,12 +1754,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" - [[package]] name = "futures-util" version = "0.3.31" @@ -1854,12 +1839,6 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" -[[package]] -name = "glob" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" - [[package]] name = "globset" version = "0.4.16" @@ -2567,12 +2546,6 @@ dependencies = [ "libc", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -3014,28 +2987,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" -[[package]] -name = "onig" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" -dependencies = [ - "bitflags 2.9.1", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "openssl" version = "0.10.73" @@ -3361,15 +3312,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "proc-macro-crate" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" -dependencies = [ - "toml_edit 0.22.27", -] - [[package]] name = "proc-macro2" version = "1.0.95" @@ -3381,9 +3323,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.13.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" +checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" dependencies = [ "bitflags 2.9.1", "getopts", @@ -3394,9 +3336,9 @@ dependencies = [ [[package]] name = "pulldown-cmark-escape" -version = "0.11.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" [[package]] name = "pxfm" @@ -3627,12 +3569,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "relative-path" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" - [[package]] name = "reqwest" version = "0.12.23" @@ -3691,51 +3627,12 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rstest" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" -dependencies = [ - "futures-timer", - "futures-util", - "rstest_macros", - "rustc_version", -] - -[[package]] -name = "rstest_macros" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" -dependencies = [ - "cfg-if", - "glob", - "proc-macro-crate", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version", - "syn 2.0.104", - "unicode-ident", -] - [[package]] name = "rustc-demangle" version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.38.44" @@ -3975,12 +3872,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - [[package]] name = "serde" version = "1.0.219" @@ -4464,28 +4355,6 @@ dependencies = [ "syn 2.0.104", ] -[[package]] -name = "syntect" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1" -dependencies = [ - "bincode", - "bitflags 1.3.2", - "flate2", - "fnv", - "once_cell", - "onig", - "plist", - "regex-syntax 0.8.5", - "serde", - "serde_derive", - "serde_json", - "thiserror 1.0.69", - "walkdir", - "yaml-rust", -] - [[package]] name = "sys-locale" version = "0.3.2" @@ -4809,18 +4678,12 @@ dependencies = [ "indexmap 2.10.0", "serde", "serde_spanned", - "toml_datetime 0.7.0", + "toml_datetime", "toml_parser", "toml_writer", "winnow", ] -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" - [[package]] name = "toml_datetime" version = "0.7.0" @@ -4830,17 +4693,6 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.10.0", - "toml_datetime 0.6.11", - "winnow", -] - [[package]] name = "toml_edit" version = "0.23.4" @@ -4848,7 +4700,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93" dependencies = [ "indexmap 2.10.0", - "toml_datetime 0.7.0", + "toml_datetime", "toml_parser", "toml_writer", "winnow", @@ -5058,22 +4910,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "tui-markdown" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10648c25931bfaaf5334ff4e7dc5f3d830e0c50d7b0119b1d5cfe771f540536" -dependencies = [ - "ansi-to-tui", - "itertools 0.14.0", - "pretty_assertions", - "pulldown-cmark", - "ratatui", - "rstest", - "syntect", - "tracing", -] - [[package]] name = "typenum" version = "1.18.0" @@ -5855,15 +5691,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d" -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index bedcf827..88e68875 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -79,7 +79,7 @@ tokio-stream = "0.1.17" tracing = { version = "0.1.41", features = ["log"] } tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -tui-markdown = "0.3.3" +pulldown-cmark = "0.10" unicode-segmentation = "1.12.0" unicode-width = "0.1" url = "2" diff --git a/codex-rs/tui/src/bin/md-events.rs b/codex-rs/tui/src/bin/md-events.rs new file mode 100644 index 00000000..f1117fad --- /dev/null +++ b/codex-rs/tui/src/bin/md-events.rs @@ -0,0 +1,15 @@ +use std::io::Read; +use std::io::{self}; + +fn main() { + let mut input = String::new(); + if let Err(err) = io::stdin().read_to_string(&mut input) { + eprintln!("failed to read stdin: {err}"); + std::process::exit(1); + } + + let parser = pulldown_cmark::Parser::new(&input); + for event in parser { + println!("{event:?}"); + } +} diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__chatwidget_markdown_code_blocks_vt100_snapshot.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__chatwidget_markdown_code_blocks_vt100_snapshot.snap new file mode 100644 index 00000000..ca4e72a9 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__chatwidget_markdown_code_blocks_vt100_snapshot.snap @@ -0,0 +1,18 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: visual +--- +> -- Indented code block (4 spaces) + SELECT * + FROM "users" + WHERE "email" LIKE '%@example.com'; + + ```sh + printf 'fenced within fenced\n' + ``` + + { + // comment allowed in jsonc + "path": "C:\\Program Files\\App", + "regex": "^foo.*(bar)?$" + } diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index e097c201..aad6e839 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -1756,3 +1756,123 @@ fn chatwidget_exec_and_status_layout_vt100_snapshot() { let visual = vt_lines.join("\n"); assert_snapshot!(visual); } + +// E2E vt100 snapshot for complex markdown with indented and nested fenced code blocks +#[test] +fn chatwidget_markdown_code_blocks_vt100_snapshot() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(); + + // Simulate a final agent message via streaming deltas instead of a single message + + chat.handle_codex_event(Event { + id: "t1".into(), + msg: EventMsg::TaskStarted(TaskStartedEvent { + model_context_window: None, + }), + }); + // Build a vt100 visual from the history insertions only (no UI overlay) + let width: u16 = 80; + let height: u16 = 50; + let backend = ratatui::backend::TestBackend::new(width, height); + let mut term = crate::custom_terminal::Terminal::with_options(backend).expect("terminal"); + // Place viewport at the last line so that history lines insert above it + term.set_viewport_area(Rect::new(0, height - 1, width, 1)); + + let mut ansi: Vec = Vec::new(); + + // Simulate streaming via AgentMessageDelta in 2-character chunks (no final AgentMessage). + let source: &str = r#" + + -- Indented code block (4 spaces) + SELECT * + FROM "users" + WHERE "email" LIKE '%@example.com'; + +````markdown +```sh +printf 'fenced within fenced\n' +``` +```` + +```jsonc +{ + // comment allowed in jsonc + "path": "C:\\Program Files\\App", + "regex": "^foo.*(bar)?$" +} +``` +"#; + + let mut it = source.chars(); + loop { + let mut delta = String::new(); + match it.next() { + Some(c) => delta.push(c), + None => break, + } + if let Some(c2) = it.next() { + delta.push(c2); + } + + chat.handle_codex_event(Event { + id: "t1".into(), + msg: EventMsg::AgentMessageDelta(AgentMessageDeltaEvent { delta }), + }); + // Drive commit ticks and drain emitted history lines into the vt100 buffer. + loop { + chat.on_commit_tick(); + let mut inserted_any = false; + while let Ok(app_ev) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = app_ev { + let lines = cell.display_lines(width); + crate::insert_history::insert_history_lines_to_writer( + &mut term, &mut ansi, lines, + ); + inserted_any = true; + } + } + if !inserted_any { + break; + } + } + } + + // Finalize the stream without sending a final AgentMessage, to flush any tail. + chat.handle_codex_event(Event { + id: "t1".into(), + msg: EventMsg::TaskComplete(TaskCompleteEvent { + last_agent_message: None, + }), + }); + for lines in drain_insert_history(&mut rx) { + crate::insert_history::insert_history_lines_to_writer(&mut term, &mut ansi, lines); + } + + let mut parser = vt100::Parser::new(height, width, 0); + parser.process(&ansi); + + let mut vt_lines: Vec = (0..height) + .map(|row| { + let mut s = String::with_capacity(width as usize); + for col in 0..width { + if let Some(cell) = parser.screen().cell(row, col) { + if let Some(ch) = cell.contents().chars().next() { + s.push(ch); + } else { + s.push(' '); + } + } else { + s.push(' '); + } + } + s.trim_end().to_string() + }) + .collect(); + + // Compact trailing blank rows for a stable snapshot + while matches!(vt_lines.last(), Some(l) if l.trim().is_empty()) { + vt_lines.pop(); + } + let visual = vt_lines.join("\n"); + assert_snapshot!(visual); +} diff --git a/codex-rs/tui/src/insert_history.rs b/codex-rs/tui/src/insert_history.rs index bffd3f2b..75731c7f 100644 --- a/codex-rs/tui/src/insert_history.rs +++ b/codex-rs/tui/src/insert_history.rs @@ -97,7 +97,17 @@ pub fn insert_history_lines_to_writer( for line in wrapped { queue!(writer, Print("\r\n")).ok(); - write_spans(writer, &line).ok(); + // Merge line-level style into each span so that ANSI colors reflect + // line styles (e.g., blockquotes with green fg). + let merged_spans: Vec = line + .spans + .iter() + .map(|s| Span { + style: s.style.patch(line.style), + content: s.content.clone(), + }) + .collect(); + write_spans(writer, merged_spans.iter()).ok(); } queue!(writer, ResetScrollRegion).ok(); @@ -264,6 +274,10 @@ where #[cfg(test)] mod tests { use super::*; + use crate::markdown_render::render_markdown_text; + use ratatui::layout::Rect; + use ratatui::style::Color; + use vt100::Parser; #[test] fn writes_bold_then_regular_spans() { @@ -292,4 +306,240 @@ mod tests { String::from_utf8(expected).unwrap() ); } + + #[test] + fn vt100_blockquote_line_emits_green_fg() { + // Set up a small off-screen terminal + let width: u16 = 40; + let height: u16 = 10; + let backend = ratatui::backend::TestBackend::new(width, height); + let mut term = crate::custom_terminal::Terminal::with_options(backend).expect("terminal"); + // Place viewport on the last line so history inserts scroll upward + let viewport = Rect::new(0, height - 1, width, 1); + term.set_viewport_area(viewport); + + // Build a blockquote-like line: apply line-level green style and prefix "> " + let mut line: Line<'static> = Line::from(vec!["> ".into(), "Hello world".into()]); + line = line.style(Color::Green); + let mut ansi: Vec = Vec::new(); + insert_history_lines_to_writer(&mut term, &mut ansi, vec![line]); + + // Parse ANSI using vt100 and assert at least one non-default fg color appears + let mut parser = Parser::new(height, width, 0); + parser.process(&ansi); + + let mut saw_colored = false; + 'outer: for row in 0..height { + for col in 0..width { + if let Some(cell) = parser.screen().cell(row, col) + && cell.has_contents() + && cell.fgcolor() != vt100::Color::Default + { + saw_colored = true; + break 'outer; + } + } + } + assert!( + saw_colored, + "expected at least one colored cell in vt100 output" + ); + } + + #[test] + fn vt100_blockquote_wrap_preserves_color_on_all_wrapped_lines() { + // Force wrapping by using a narrow viewport width and a long blockquote line. + let width: u16 = 20; + let height: u16 = 8; + let backend = ratatui::backend::TestBackend::new(width, height); + let mut term = crate::custom_terminal::Terminal::with_options(backend).expect("terminal"); + // Viewport is the last line so history goes directly above it. + let viewport = Rect::new(0, height - 1, width, 1); + term.set_viewport_area(viewport); + + // Create a long blockquote with a distinct prefix and enough text to wrap. + let mut line: Line<'static> = Line::from(vec![ + "> ".into(), + "This is a long quoted line that should wrap".into(), + ]); + line = line.style(Color::Green); + + let mut ansi: Vec = Vec::new(); + insert_history_lines_to_writer(&mut term, &mut ansi, vec![line]); + + // Parse and inspect the final screen buffer. + let mut parser = Parser::new(height, width, 0); + parser.process(&ansi); + let screen = parser.screen(); + + // Collect rows that are non-empty; these should correspond to our wrapped lines. + let mut non_empty_rows: Vec = Vec::new(); + for row in 0..height { + let mut any = false; + for col in 0..width { + if let Some(cell) = screen.cell(row, col) + && cell.has_contents() + && cell.contents() != "\0" + && cell.contents() != " " + { + any = true; + break; + } + } + if any { + non_empty_rows.push(row); + } + } + + // Expect at least two rows due to wrapping. + assert!( + non_empty_rows.len() >= 2, + "expected wrapped output to span >=2 rows, got {non_empty_rows:?}", + ); + + // For each non-empty row, ensure all non-space cells are using a non-default fg color. + for row in non_empty_rows { + for col in 0..width { + if let Some(cell) = screen.cell(row, col) { + let contents = cell.contents(); + if !contents.is_empty() && contents != " " { + assert!( + cell.fgcolor() != vt100::Color::Default, + "expected non-default fg on row {row} col {col}, got {:?}", + cell.fgcolor() + ); + } + } + } + } + } + + #[test] + fn vt100_colored_prefix_then_plain_text_resets_color() { + let width: u16 = 40; + let height: u16 = 6; + let backend = ratatui::backend::TestBackend::new(width, height); + let mut term = crate::custom_terminal::Terminal::with_options(backend).expect("terminal"); + let viewport = Rect::new(0, height - 1, width, 1); + term.set_viewport_area(viewport); + + // First span colored, rest plain. + let line: Line<'static> = Line::from(vec![ + Span::styled("1. ", ratatui::style::Style::default().fg(Color::LightBlue)), + Span::raw("Hello world"), + ]); + + let mut ansi: Vec = Vec::new(); + insert_history_lines_to_writer(&mut term, &mut ansi, vec![line]); + + let mut parser = Parser::new(height, width, 0); + parser.process(&ansi); + let screen = parser.screen(); + + // Find the first non-empty row; verify first three cells are colored, following cells default. + 'rows: for row in 0..height { + let mut has_text = false; + for col in 0..width { + if let Some(cell) = screen.cell(row, col) + && cell.has_contents() + && cell.contents() != " " + { + has_text = true; + break; + } + } + if !has_text { + continue; + } + + // Expect "1. Hello world" starting at col 0. + for col in 0..3 { + let cell = screen.cell(row, col).unwrap(); + assert!( + cell.fgcolor() != vt100::Color::Default, + "expected colored prefix at col {col}, got {:?}", + cell.fgcolor() + ); + } + for col in 3..(3 + "Hello world".len() as u16) { + let cell = screen.cell(row, col).unwrap(); + assert_eq!( + cell.fgcolor(), + vt100::Color::Default, + "expected default color for plain text at col {col}, got {:?}", + cell.fgcolor() + ); + } + break 'rows; + } + } + + #[test] + fn vt100_deep_nested_mixed_list_third_level_marker_is_colored() { + // Markdown with five levels (ordered → unordered → ordered → unordered → unordered). + let md = "1. First\n - Second level\n 1. Third level (ordered)\n - Fourth level (bullet)\n - Fifth level to test indent consistency\n"; + let text = render_markdown_text(md); + let lines: Vec> = text.lines.clone(); + + let width: u16 = 60; + let height: u16 = 12; + let backend = ratatui::backend::TestBackend::new(width, height); + let mut term = crate::custom_terminal::Terminal::with_options(backend).expect("terminal"); + let viewport = ratatui::layout::Rect::new(0, height - 1, width, 1); + term.set_viewport_area(viewport); + + let mut ansi: Vec = Vec::new(); + insert_history_lines_to_writer(&mut term, &mut ansi, lines); + + let mut parser = Parser::new(height, width, 0); + parser.process(&ansi); + let screen = parser.screen(); + + // Reconstruct screen rows as strings to locate the 3rd level line. + let mut rows: Vec = Vec::with_capacity(height as usize); + for row in 0..height { + let mut s = String::with_capacity(width as usize); + for col in 0..width { + if let Some(cell) = screen.cell(row, col) { + if let Some(ch) = cell.contents().chars().next() { + s.push(ch); + } else { + s.push(' '); + } + } else { + s.push(' '); + } + } + rows.push(s.trim_end().to_string()); + } + + let needle = "1. Third level (ordered)"; + let row_idx = rows + .iter() + .position(|r| r.contains(needle)) + .unwrap_or_else(|| { + panic!("expected to find row containing {needle:?}, have rows: {rows:?}") + }); + let col_start = rows[row_idx].find(needle).unwrap() as u16; // column where '1' starts + + // Verify that the numeric marker ("1.") at the third level is colored + // (non-default fg) and the content after the following space resets to default. + for c in [col_start, col_start + 1] { + let cell = screen.cell(row_idx as u16, c).unwrap(); + assert!( + cell.fgcolor() != vt100::Color::Default, + "expected colored 3rd-level marker at row {row_idx} col {c}, got {:?}", + cell.fgcolor() + ); + } + let content_col = col_start + 3; // skip '1', '.', and the space + if let Some(cell) = screen.cell(row_idx as u16, content_col) { + assert_eq!( + cell.fgcolor(), + vt100::Color::Default, + "expected default color for 3rd-level content at row {row_idx} col {content_col}, got {:?}", + cell.fgcolor() + ); + } + } } diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 5f002e53..710312ce 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -45,6 +45,7 @@ pub mod insert_history; mod key_hint; pub mod live_wrap; mod markdown; +mod markdown_render; mod markdown_stream; pub mod onboarding; mod pager_overlay; diff --git a/codex-rs/tui/src/markdown.rs b/codex-rs/tui/src/markdown.rs index 6aff205a..010ff78a 100644 --- a/codex-rs/tui/src/markdown.rs +++ b/codex-rs/tui/src/markdown.rs @@ -1,8 +1,6 @@ -use crate::citation_regex::CITATION_REGEX; use codex_core::config::Config; use codex_core::config_types::UriBasedFileOpener; use ratatui::text::Line; -use std::borrow::Cow; use std::path::Path; pub(crate) fn append_markdown( @@ -19,238 +17,13 @@ fn append_markdown_with_opener_and_cwd( file_opener: UriBasedFileOpener, cwd: &Path, ) { - // Historically, we fed the entire `markdown_source` into the renderer in - // one pass. However, fenced code blocks sometimes lost leading whitespace - // when formatted by the markdown renderer/highlighter. To preserve code - // block content exactly, split the source into "text" and "code" segments: - // - Render non-code text through `tui_markdown` (with citation rewrite). - // - Render code block content verbatim as plain lines without additional - // formatting, preserving leading spaces. - for seg in split_text_and_fences(markdown_source) { - match seg { - Segment::Text(s) => { - let processed = rewrite_file_citations(&s, file_opener, cwd); - let rendered = tui_markdown::from_str(&processed); - crate::render::line_utils::push_owned_lines(&rendered.lines, lines); - } - Segment::Code { content, .. } => { - // Emit the code content exactly as-is, line by line. - // We don't attempt syntax highlighting to avoid whitespace bugs. - for line in content.split_inclusive('\n') { - // split_inclusive keeps the trailing \n; we want lines without it. - let line = if let Some(stripped) = line.strip_suffix('\n') { - stripped - } else { - line - }; - let owned_line: Line<'static> = line.to_string().into(); - lines.push(owned_line); - } - } - } - } -} - -/// Rewrites file citations in `src` into markdown hyperlinks using the -/// provided `scheme` (`vscode`, `cursor`, etc.). The resulting URI follows the -/// format expected by VS Code-compatible file openers: -/// -/// ```text -/// ://file: -/// ``` -fn rewrite_file_citations<'a>( - src: &'a str, - file_opener: UriBasedFileOpener, - cwd: &Path, -) -> Cow<'a, str> { - // Map enum values to the corresponding URI scheme strings. - let scheme: &str = match file_opener.get_scheme() { - Some(scheme) => scheme, - None => return Cow::Borrowed(src), - }; - - CITATION_REGEX.replace_all(src, |caps: ®ex_lite::Captures<'_>| { - let file = &caps[1]; - let start_line = &caps[2]; - - // Resolve the path against `cwd` when it is relative. - let absolute_path = { - let p = Path::new(file); - let absolute_path = if p.is_absolute() { - path_clean::clean(p) - } else { - path_clean::clean(cwd.join(p)) - }; - // VS Code expects forward slashes even on Windows because URIs use - // `/` as the path separator. - absolute_path.to_string_lossy().replace('\\', "/") - }; - - // Render as a normal markdown link so the downstream renderer emits - // the hyperlink escape sequence (when supported by the terminal). - // - // In practice, sometimes multiple citations for the same file, but with a - // different line number, are shown sequentially, so we: - // - include the line number in the label to disambiguate them - // - add a space after the link to make it easier to read - format!("[{file}:{start_line}]({scheme}://file{absolute_path}:{start_line}) ") - }) -} - -// use shared helper from `line_utils` - -// Minimal code block splitting. -// - Recognizes fenced blocks opened by ``` or ~~~ (allowing leading whitespace). -// The opening fence may include a language string which we ignore. -// The closing fence must be on its own line (ignoring surrounding whitespace). -// - Additionally recognizes indented code blocks that begin after a blank line -// with a line starting with at least 4 spaces or a tab, and continue for -// consecutive lines that are blank or also indented by >= 4 spaces or a tab. -enum Segment { - Text(String), - Code { - _lang: Option, - content: String, - }, -} - -fn split_text_and_fences(src: &str) -> Vec { - let mut segments = Vec::new(); - let mut curr_text = String::new(); - #[derive(Copy, Clone, PartialEq)] - enum CodeMode { - None, - Fenced, - Indented, - } - let mut code_mode = CodeMode::None; - let mut fence_token = ""; - let mut code_lang: Option = None; - let mut code_content = String::new(); - // We intentionally do not require a preceding blank line for indented code blocks, - // since streamed model output often omits it. This favors preserving indentation. - - for line in src.split_inclusive('\n') { - let line_no_nl = line.strip_suffix('\n'); - let trimmed_start = match line_no_nl { - Some(l) => l.trim_start(), - None => line.trim_start(), - }; - if code_mode == CodeMode::None { - let open = if trimmed_start.starts_with("```") { - Some("```") - } else if trimmed_start.starts_with("~~~") { - Some("~~~") - } else { - None - }; - if let Some(tok) = open { - // Flush pending text segment. - if !curr_text.is_empty() { - segments.push(Segment::Text(curr_text.clone())); - curr_text.clear(); - } - fence_token = tok; - // Capture language after the token on this line (before newline). - let after = &trimmed_start[tok.len()..]; - let lang = after.trim(); - code_lang = if lang.is_empty() { - None - } else { - Some(lang.to_string()) - }; - code_mode = CodeMode::Fenced; - code_content.clear(); - // Do not include the opening fence line in output. - continue; - } - // Check for start of an indented code block: only after a blank line - // (or at the beginning), and the line must start with >=4 spaces or a tab. - let raw_line = match line_no_nl { - Some(l) => l, - None => line, - }; - let leading_spaces = raw_line.chars().take_while(|c| *c == ' ').count(); - let starts_with_tab = raw_line.starts_with('\t'); - // Consider any line that begins with >=4 spaces or a tab to start an - // indented code block. This favors preserving indentation even when a - // preceding blank line is omitted (common in streamed model output). - let starts_indented_code = (leading_spaces >= 4) || starts_with_tab; - if starts_indented_code { - // Flush pending text and begin an indented code block. - if !curr_text.is_empty() { - segments.push(Segment::Text(curr_text.clone())); - curr_text.clear(); - } - code_mode = CodeMode::Indented; - code_content.clear(); - code_content.push_str(line); - // Inside code now; do not treat this line as normal text. - continue; - } - // Normal text line. - curr_text.push_str(line); - } else { - match code_mode { - CodeMode::Fenced => { - // inside fenced code: check for closing fence on its own line - let trimmed = match line_no_nl { - Some(l) => l.trim(), - None => line.trim(), - }; - if trimmed == fence_token { - // End code block: emit segment without fences - segments.push(Segment::Code { - _lang: code_lang.take(), - content: code_content.clone(), - }); - code_content.clear(); - code_mode = CodeMode::None; - fence_token = ""; - continue; - } - // Accumulate code content exactly as-is. - code_content.push_str(line); - } - CodeMode::Indented => { - // Continue while the line is blank, or starts with >=4 spaces, or a tab. - let raw_line = match line_no_nl { - Some(l) => l, - None => line, - }; - let is_blank = raw_line.trim().is_empty(); - let leading_spaces = raw_line.chars().take_while(|c| *c == ' ').count(); - let starts_with_tab = raw_line.starts_with('\t'); - if is_blank || leading_spaces >= 4 || starts_with_tab { - code_content.push_str(line); - } else { - // Close the indented code block and reprocess this line as normal text. - segments.push(Segment::Code { - _lang: None, - content: code_content.clone(), - }); - code_content.clear(); - code_mode = CodeMode::None; - // Now handle current line as text. - curr_text.push_str(line); - } - } - CodeMode::None => unreachable!(), - } - } - } - - if code_mode != CodeMode::None { - // Unterminated code fence: treat accumulated content as a code segment. - segments.push(Segment::Code { - _lang: code_lang.take(), - content: code_content.clone(), - }); - } else if !curr_text.is_empty() { - segments.push(Segment::Text(curr_text.clone())); - } - - segments + // Render via pulldown-cmark and rewrite citations during traversal (outside code blocks). + let rendered = crate::markdown_render::render_markdown_text_with_citations( + markdown_source, + file_opener.get_scheme(), + cwd, + ); + crate::render::line_utils::push_owned_lines(&rendered.lines, lines); } #[cfg(test)] @@ -258,88 +31,6 @@ mod tests { use super::*; use pretty_assertions::assert_eq; - #[test] - fn citation_is_rewritten_with_absolute_path() { - let markdown = "See 【F:/src/main.rs†L42-L50】 for details."; - let cwd = Path::new("/workspace"); - let result = rewrite_file_citations(markdown, UriBasedFileOpener::VsCode, cwd); - - assert_eq!( - "See [/src/main.rs:42](vscode://file/src/main.rs:42) for details.", - result - ); - } - - #[test] - fn citation_is_rewritten_with_relative_path() { - let markdown = "Refer to 【F:lib/mod.rs†L5】 here."; - let cwd = Path::new("/home/user/project"); - let result = rewrite_file_citations(markdown, UriBasedFileOpener::Windsurf, cwd); - - assert_eq!( - "Refer to [lib/mod.rs:5](windsurf://file/home/user/project/lib/mod.rs:5) here.", - result - ); - } - - #[test] - fn citation_followed_by_space_so_they_do_not_run_together() { - let markdown = "References on lines 【F:src/foo.rs†L24】【F:src/foo.rs†L42】"; - let cwd = Path::new("/home/user/project"); - let result = rewrite_file_citations(markdown, UriBasedFileOpener::VsCode, cwd); - - assert_eq!( - "References on lines [src/foo.rs:24](vscode://file/home/user/project/src/foo.rs:24) [src/foo.rs:42](vscode://file/home/user/project/src/foo.rs:42) ", - result - ); - } - - #[test] - fn citation_unchanged_without_file_opener() { - let markdown = "Look at 【F:file.rs†L1】."; - let cwd = Path::new("/"); - let unchanged = rewrite_file_citations(markdown, UriBasedFileOpener::VsCode, cwd); - // The helper itself always rewrites – this test validates behaviour of - // append_markdown when `file_opener` is None. - let mut out = Vec::new(); - append_markdown_with_opener_and_cwd(markdown, &mut out, UriBasedFileOpener::None, cwd); - // Convert lines back to string for comparison. - let rendered: String = out - .iter() - .flat_map(|l| l.spans.iter()) - .map(|s| s.content.clone()) - .collect::>() - .join(""); - assert_eq!(markdown, rendered); - // Ensure helper rewrites. - assert_ne!(markdown, unchanged); - } - - #[test] - fn fenced_code_blocks_preserve_leading_whitespace() { - let src = "```\n indented\n\t\twith tabs\n four spaces\n```\n"; - let cwd = Path::new("/"); - let mut out = Vec::new(); - append_markdown_with_opener_and_cwd(src, &mut out, UriBasedFileOpener::None, cwd); - let rendered: Vec = out - .iter() - .map(|l| { - l.spans - .iter() - .map(|s| s.content.clone()) - .collect::() - }) - .collect(); - assert_eq!( - rendered, - vec![ - " indented".to_string(), - "\t\twith tabs".to_string(), - " four spaces".to_string() - ] - ); - } - #[test] fn citations_not_rewritten_inside_code_blocks() { let src = "Before 【F:/x.rs†L1】\n```\nInside 【F:/x.rs†L2】\n```\nAfter 【F:/x.rs†L3】\n"; @@ -355,19 +46,31 @@ mod tests { .collect::() }) .collect(); - // Expect first and last lines rewritten, middle line unchanged. - assert!(rendered[0].contains("vscode://file")); - assert_eq!(rendered[1], "Inside 【F:/x.rs†L2】"); - assert!(matches!(rendered.last(), Some(s) if s.contains("vscode://file"))); + // Expect a line containing the inside text unchanged. + assert!(rendered.iter().any(|s| s.contains("Inside 【F:/x.rs†L2】"))); + // And first/last sections rewritten. + assert!( + rendered + .first() + .map(|s| s.contains("vscode://file")) + .unwrap_or(false) + ); + assert!( + rendered + .last() + .map(|s| s.contains("vscode://file")) + .unwrap_or(false) + ); } #[test] fn indented_code_blocks_preserve_leading_whitespace() { - let src = "Before\n code 1\n\tcode with tab\n code 2\nAfter\n"; + // Basic sanity: indented code with surrounding blank lines should produce the indented line. + let src = "Before\n\n code 1\n\nAfter\n"; let cwd = Path::new("/"); let mut out = Vec::new(); append_markdown_with_opener_and_cwd(src, &mut out, UriBasedFileOpener::None, cwd); - let rendered: Vec = out + let lines: Vec = out .iter() .map(|l| { l.spans @@ -376,16 +79,7 @@ mod tests { .collect::() }) .collect(); - assert_eq!( - rendered, - vec![ - "Before".to_string(), - " code 1".to_string(), - "\tcode with tab".to_string(), - " code 2".to_string(), - "After".to_string() - ] - ); + assert_eq!(lines, vec!["Before", "", " code 1", "", "After"]); } #[test] @@ -403,11 +97,17 @@ mod tests { .collect::() }) .collect(); - // Expect first and last lines rewritten, and the indented code line present - // unchanged (citations inside not rewritten). We do not assert on blank - // separator lines since the markdown renderer may normalize them. - assert!(rendered.iter().any(|s| s.contains("vscode://file"))); - assert!(rendered.iter().any(|s| s == " Inside 【F:/x.rs†L2】")); + assert!( + rendered + .iter() + .any(|s| s.contains("Start") && s.contains("vscode://file")) + ); + assert!( + rendered + .iter() + .any(|s| s.contains("End") && s.contains("vscode://file")) + ); + assert!(rendered.iter().any(|s| s.contains("Inside 【F:/x.rs†L2】"))); } #[test] @@ -435,27 +135,6 @@ mod tests { ); } - #[test] - fn tui_markdown_splits_ordered_marker_and_text() { - // With marker and content on the same line, tui_markdown keeps it as one line - // even in the surrounding section context. - let rendered = tui_markdown::from_str("Loose vs. tight list items:\n1. Tight item\n"); - let lines: Vec = rendered - .lines - .iter() - .map(|l| { - l.spans - .iter() - .map(|s| s.content.clone()) - .collect::() - }) - .collect(); - assert!( - lines.iter().any(|w| w == "1. Tight item"), - "expected single line '1. Tight item' in context: {lines:?}" - ); - } - #[test] fn append_markdown_matches_tui_markdown_for_ordered_item() { use codex_core::config_types::UriBasedFileOpener; @@ -480,72 +159,6 @@ mod tests { assert_eq!(lines, vec!["1. Tight item".to_string()]); } - #[test] - fn tui_markdown_shape_for_loose_tight_section() { - // Use the exact source from the session deltas used in tests. - let source = r#" -Loose vs. tight list items: -1. Tight item -2. Another tight item - -3. - Loose item -"#; - - let rendered = tui_markdown::from_str(source); - let lines: Vec = rendered - .lines - .iter() - .map(|l| { - l.spans - .iter() - .map(|s| s.content.clone()) - .collect::() - }) - .collect(); - // Join into a single string and assert the exact shape we observe - // from tui_markdown in this larger context (marker and content split). - let joined = { - let mut s = String::new(); - for (i, l) in lines.iter().enumerate() { - s.push_str(l); - if i + 1 < lines.len() { - s.push('\n'); - } - } - s - }; - let expected = r#"Loose vs. tight list items: - -1. -Tight item -2. -Another tight item -3. -Loose item"#; - assert_eq!( - joined, expected, - "unexpected tui_markdown shape: {joined:?}" - ); - } - - #[test] - fn split_text_and_fences_keeps_ordered_list_line_as_text() { - // No fences here; expect a single Text segment containing the full input. - let src = "Loose vs. tight list items:\n1. Tight item\n"; - let segs = super::split_text_and_fences(src); - assert_eq!( - segs.len(), - 1, - "expected single text segment, got {}", - segs.len() - ); - match &segs[0] { - super::Segment::Text(s) => assert_eq!(s, src), - _ => panic!("expected Text segment for non-fence input"), - } - } - #[test] fn append_markdown_keeps_ordered_list_line_unsplit_in_context() { use codex_core::config_types::UriBasedFileOpener; diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs new file mode 100644 index 00000000..f20bfd75 --- /dev/null +++ b/codex-rs/tui/src/markdown_render.rs @@ -0,0 +1,566 @@ +use crate::citation_regex::CITATION_REGEX; +use pulldown_cmark::CodeBlockKind; +use pulldown_cmark::CowStr; +use pulldown_cmark::Event; +use pulldown_cmark::HeadingLevel; +use pulldown_cmark::Options; +use pulldown_cmark::Parser; +use pulldown_cmark::Tag; +use pulldown_cmark::TagEnd; +use ratatui::style::Style; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::text::Span; +use ratatui::text::Text; +use std::borrow::Cow; +use std::path::Path; + +#[derive(Clone, Debug)] +struct IndentContext { + prefix: Vec>, + marker: Option>>, + is_list: bool, +} + +impl IndentContext { + fn new(prefix: Vec>, marker: Option>>, is_list: bool) -> Self { + Self { + prefix, + marker, + is_list, + } + } +} + +#[allow(dead_code)] +pub(crate) fn render_markdown_text(input: &str) -> Text<'static> { + let mut options = Options::empty(); + options.insert(Options::ENABLE_STRIKETHROUGH); + let parser = Parser::new_ext(input, options); + let mut w = Writer::new(parser, None, None); + w.run(); + w.text +} + +pub(crate) fn render_markdown_text_with_citations( + input: &str, + scheme: Option<&str>, + cwd: &Path, +) -> Text<'static> { + let mut options = Options::empty(); + options.insert(Options::ENABLE_STRIKETHROUGH); + let parser = Parser::new_ext(input, options); + let mut w = Writer::new( + parser, + scheme.map(|s| s.to_string()), + Some(cwd.to_path_buf()), + ); + w.run(); + w.text +} + +struct Writer<'a, I> +where + I: Iterator>, +{ + iter: I, + text: Text<'static>, + inline_styles: Vec